From 4822f5d7166996c1a619f7c51d156a029e85dd53 Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Tue, 5 Mar 2024 13:57:22 +0800 Subject: [PATCH 4/7] libhns: Introduce hns direct verbs driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I95UWO ------------------------------------------------------------------ Introduce the frame of hns direct verbs, including hnsdv_is_supported(), hnsdv_create_qp() and hnsdv_query_device(). Signed-off-by: Junxian Huang Signed-off-by: Ran Zhou --- debian/ibverbs-providers.install | 1 + debian/ibverbs-providers.lintian-overrides | 4 +- debian/ibverbs-providers.symbols | 6 ++ debian/libibverbs-dev.install | 4 ++ providers/hns/CMakeLists.txt | 9 ++- providers/hns/hns_roce_u.c | 13 ++++ providers/hns/hns_roce_u.h | 2 + providers/hns/hns_roce_u_abi.h | 1 + providers/hns/hns_roce_u_verbs.c | 69 ++++++++++++++++++++-- providers/hns/hnsdv.h | 37 ++++++++++++ providers/hns/libhns.map | 9 +++ redhat/rdma-core.spec | 5 +- suse/rdma-core.spec | 21 ++++++- 13 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 providers/hns/hnsdv.h create mode 100644 providers/hns/libhns.map diff --git a/debian/ibverbs-providers.install b/debian/ibverbs-providers.install index a003a30..fea15e0 100644 --- a/debian/ibverbs-providers.install +++ b/debian/ibverbs-providers.install @@ -1,6 +1,7 @@ etc/libibverbs.d/ usr/lib/*/libefa.so.* usr/lib/*/libibverbs/lib*-rdmav*.so +usr/lib/*/libhns.so.* usr/lib/*/libmana.so.* usr/lib/*/libmlx4.so.* usr/lib/*/libmlx5.so.* diff --git a/debian/ibverbs-providers.lintian-overrides b/debian/ibverbs-providers.lintian-overrides index 5815058..fd73a76 100644 --- a/debian/ibverbs-providers.lintian-overrides +++ b/debian/ibverbs-providers.lintian-overrides @@ -1,2 +1,2 @@ -# libefa, libmana, libmlx4 and libmlx5 are ibverbs provider that provides more functions. -ibverbs-providers: package-name-doesnt-match-sonames libefa1 libmana1 libmlx4-1 libmlx5-1 +# libefa, libhns, libmana, libmlx4 and libmlx5 are ibverbs provider that provides more functions. +ibverbs-providers: package-name-doesnt-match-sonames libefa1 libhns-1 libmana1 libmlx4-1 libmlx5-1 diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols index 72361bd..d2c0989 100644 --- a/debian/ibverbs-providers.symbols +++ b/debian/ibverbs-providers.symbols @@ -174,6 +174,12 @@ libefa.so.1 ibverbs-providers #MINVER# efadv_cq_from_ibv_cq_ex@EFA_1.2 43 efadv_create_cq@EFA_1.2 43 efadv_query_mr@EFA_1.3 50 +libhns.so.1 ibverbs-providers #MINVER# +* Build-Depends-Package: libibverbs-dev + HNS_1.0@HNS_1.0 51 + hnsdv_is_supported@HNS_1.0 51 + hnsdv_create_qp@HNS_1.0 51 + hnsdv_query_device@HNS_1.0 51 libmana.so.1 ibverbs-providers #MINVER# * Build-Depends-Package: libibverbs-dev MANA_1.0@MANA_1.0 41 diff --git a/debian/libibverbs-dev.install b/debian/libibverbs-dev.install index 5f2ffd5..ef5b9a4 100644 --- a/debian/libibverbs-dev.install +++ b/debian/libibverbs-dev.install @@ -1,5 +1,6 @@ usr/include/infiniband/arch.h usr/include/infiniband/efadv.h +usr/include/infiniband/hnsdv.h usr/include/infiniband/ib_user_ioctl_verbs.h usr/include/infiniband/manadv.h usr/include/infiniband/mlx4dv.h @@ -15,6 +16,8 @@ usr/include/infiniband/verbs_api.h usr/lib/*/lib*-rdmav*.a usr/lib/*/libefa.a usr/lib/*/libefa.so +usr/lib/*/libhns.a +usr/lib/*/libhns.so usr/lib/*/libibverbs*.so usr/lib/*/libibverbs.a usr/lib/*/libmana.a @@ -24,6 +27,7 @@ usr/lib/*/libmlx4.so usr/lib/*/libmlx5.a usr/lib/*/libmlx5.so usr/lib/*/pkgconfig/libefa.pc +usr/lib/*/pkgconfig/libhns.pc usr/lib/*/pkgconfig/libibverbs.pc usr/lib/*/pkgconfig/libmana.pc usr/lib/*/pkgconfig/libmlx4.pc diff --git a/providers/hns/CMakeLists.txt b/providers/hns/CMakeLists.txt index 7aaca75..58139ae 100644 --- a/providers/hns/CMakeLists.txt +++ b/providers/hns/CMakeLists.txt @@ -1,7 +1,14 @@ -rdma_provider(hns +rdma_shared_provider(hns libhns.map + 1 1.0.${PACKAGE_VERSION} hns_roce_u.c hns_roce_u_buf.c hns_roce_u_db.c hns_roce_u_hw_v2.c hns_roce_u_verbs.c ) + +publish_headers(infiniband + hnsdv.h +) + +rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c index 266e73e..0b254fb 100644 --- a/providers/hns/hns_roce_u.c +++ b/providers/hns/hns_roce_u.c @@ -216,4 +216,17 @@ static const struct verbs_device_ops hns_roce_dev_ops = { .uninit_device = hns_uninit_device, .alloc_context = hns_roce_alloc_context, }; + +bool is_hns_dev(struct ibv_device *device) +{ + struct verbs_device *verbs_device = verbs_get_device(device); + + return verbs_device->ops == &hns_roce_dev_ops; +} + +bool hnsdv_is_supported(struct ibv_device *device) +{ + return is_hns_dev(device); +} + PROVIDER_DRIVER(hns, hns_roce_dev_ops); diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 5ec2734..99fa23f 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -501,6 +501,8 @@ void hns_roce_free_qp_buf(struct hns_roce_qp *qp, struct hns_roce_context *ctx); void hns_roce_init_qp_indices(struct hns_roce_qp *qp); +bool is_hns_dev(struct ibv_device *device); + extern const struct hns_roce_u_hw hns_roce_u_hw_v2; #endif /* _HNS_ROCE_U_H */ diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h index ec47c4b..7e9bbc1 100644 --- a/providers/hns/hns_roce_u_abi.h +++ b/providers/hns/hns_roce_u_abi.h @@ -36,6 +36,7 @@ #include #include #include +#include "hnsdv.h" DECLARE_DRV_CMD(hns_roce_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty, hns_roce_ib_alloc_pd_resp); diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index d081bb3..997b7e0 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -785,6 +785,25 @@ int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq) return 0; } +enum { + HNSDV_QP_SUP_COMP_MASK = 0, +}; + +static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, + struct hnsdv_qp_init_attr *hns_attr) +{ + if (!hns_attr) + return 0; + + if (!check_comp_mask(hns_attr->comp_mask, HNSDV_QP_SUP_COMP_MASK)) { + verbs_err(&ctx->ibv_ctx, "invalid hnsdv comp_mask 0x%x.\n", + hns_attr->comp_mask); + return EINVAL; + } + + return 0; +} + enum { CREATE_QP_SUP_COMP_MASK = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_XRCD | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS, @@ -866,7 +885,8 @@ static int verify_qp_create_cap(struct hns_roce_context *ctx, } static int verify_qp_create_attr(struct hns_roce_context *ctx, - struct ibv_qp_init_attr_ex *attr) + struct ibv_qp_init_attr_ex *attr, + struct hnsdv_qp_init_attr *hns_attr) { int ret; @@ -874,6 +894,10 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx, if (ret) return ret; + ret = check_hnsdv_qp_attr(ctx, hns_attr); + if (ret) + return ret; + return verify_qp_create_cap(ctx, attr); } @@ -1274,14 +1298,15 @@ static int mmap_dwqe(struct ibv_context *ibv_ctx, struct hns_roce_qp *qp, } static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, - struct ibv_qp_init_attr_ex *attr) + struct ibv_qp_init_attr_ex *attr, + struct hnsdv_qp_init_attr *hns_attr) { struct hns_roce_context *context = to_hr_ctx(ibv_ctx); struct hns_roce_qp *qp; uint64_t dwqe_mmap_key; int ret; - ret = verify_qp_create_attr(context, attr); + ret = verify_qp_create_attr(context, attr, hns_attr); if (ret) goto err; @@ -1345,7 +1370,7 @@ struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd, attrx.comp_mask = IBV_QP_INIT_ATTR_PD; attrx.pd = pd; - qp = create_qp(pd->context, &attrx); + qp = create_qp(pd->context, &attrx, NULL); if (qp) memcpy(attr, &attrx, sizeof(*attr)); @@ -1355,7 +1380,41 @@ struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd, struct ibv_qp *hns_roce_u_create_qp_ex(struct ibv_context *context, struct ibv_qp_init_attr_ex *attr) { - return create_qp(context, attr); + return create_qp(context, attr, NULL); +} + +struct ibv_qp *hnsdv_create_qp(struct ibv_context *context, + struct ibv_qp_init_attr_ex *qp_attr, + struct hnsdv_qp_init_attr *hns_attr) +{ + if (!context || !qp_attr) { + errno = EINVAL; + return NULL; + } + + if (!is_hns_dev(context->device)) { + errno = EOPNOTSUPP; + return NULL; + } + + return create_qp(context, qp_attr, hns_attr); +} + +int hnsdv_query_device(struct ibv_context *context, + struct hnsdv_context *attrs_out) +{ + struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL; + + if (!ctx || !attrs_out) + return EINVAL; + + if (!is_hns_dev(context->device)) { + verbs_err(verbs_get_ctx(context), "not a HNS RoCE device!\n"); + return EOPNOTSUPP; + } + memset(attrs_out, 0, sizeof(*attrs_out)); + + return 0; } struct ibv_qp *hns_roce_u_open_qp(struct ibv_context *context, diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h new file mode 100644 index 0000000..49ba08a --- /dev/null +++ b/providers/hns/hnsdv.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (c) 2024 Hisilicon Limited. + */ + +#ifndef __HNSDV_H__ +#define __HNSDV_H__ + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct hnsdv_qp_init_attr { + uint64_t comp_mask; +}; + +struct hnsdv_context { + uint64_t comp_mask; +}; + +bool hnsdv_is_supported(struct ibv_device *device); +int hnsdv_query_device(struct ibv_context *ctx_in, + struct hnsdv_context *attrs_out); +struct ibv_qp *hnsdv_create_qp(struct ibv_context *context, + struct ibv_qp_init_attr_ex *qp_attr, + struct hnsdv_qp_init_attr *hns_qp_attr); + +#ifdef __cplusplus +} +#endif + +#endif /* __HNSDV_H__ */ diff --git a/providers/hns/libhns.map b/providers/hns/libhns.map new file mode 100644 index 0000000..e9bf417 --- /dev/null +++ b/providers/hns/libhns.map @@ -0,0 +1,9 @@ +/* Export symbols should be added below according to + Documentation/versioning.md document. */ +HNS_1.0 { + global: + hnsdv_is_supported; + hnsdv_create_qp; + hnsdv_query_device; + local: *; +}; diff --git a/redhat/rdma-core.spec b/redhat/rdma-core.spec index c6ddcfd..c347195 100644 --- a/redhat/rdma-core.spec +++ b/redhat/rdma-core.spec @@ -158,6 +158,8 @@ Provides: liberdma = %{version}-%{release} Obsoletes: liberdma < %{version}-%{release} Provides: libhfi1 = %{version}-%{release} Obsoletes: libhfi1 < %{version}-%{release} +Provides: libhns = %{version}-%{release} +Obsoletes: libhns < %{version}-%{release} Provides: libipathverbs = %{version}-%{release} Obsoletes: libipathverbs < %{version}-%{release} Provides: libirdma = %{version}-%{release} @@ -188,7 +190,7 @@ Device-specific plug-in ibverbs userspace drivers are included: - libefa: Amazon Elastic Fabric Adapter - liberdma: Alibaba Elastic RDMA (iWarp) Adapter - libhfi1: Intel Omni-Path HFI -- libhns: HiSilicon Hip06 SoC +- libhns: HiSilicon Hip08+ SoC - libipathverbs: QLogic InfiniPath HCA - libirdma: Intel Ethernet Connection RDMA - libmana: Microsoft Azure Network Adapter @@ -575,6 +577,7 @@ fi %dir %{_sysconfdir}/libibverbs.d %dir %{_libdir}/libibverbs %{_libdir}/libefa.so.* +%{_libdir}/libhns.so.* %{_libdir}/libibverbs*.so.* %{_libdir}/libibverbs/*.so %{_libdir}/libmana.so.* diff --git a/suse/rdma-core.spec b/suse/rdma-core.spec index d534dbc..f2bd0c0 100644 --- a/suse/rdma-core.spec +++ b/suse/rdma-core.spec @@ -35,6 +35,7 @@ License: BSD-2-Clause OR GPL-2.0-only Group: Productivity/Networking/Other %define efa_so_major 1 +%define hns_so_major 1 %define verbs_so_major 1 %define rdmacm_so_major 1 %define umad_so_major 3 @@ -45,6 +46,7 @@ Group: Productivity/Networking/Other %define mad_major 5 %define efa_lname libefa%{efa_so_major} +%define hns_lname libhns%{hns_so_major} %define verbs_lname libibverbs%{verbs_so_major} %define rdmacm_lname librdmacm%{rdmacm_so_major} %define umad_lname libibumad%{umad_so_major} @@ -159,6 +161,7 @@ Requires: %{umad_lname} = %{version}-%{release} Requires: %{verbs_lname} = %{version}-%{release} %if 0%{?dma_coherent} Requires: %{efa_lname} = %{version}-%{release} +Requires: %{hns_lname} = %{version}-%{release} Requires: %{mana_lname} = %{version}-%{release} Requires: %{mlx4_lname} = %{version}-%{release} Requires: %{mlx5_lname} = %{version}-%{release} @@ -200,6 +203,7 @@ Requires: %{name}%{?_isa} = %{version}-%{release} Obsoletes: libcxgb4-rdmav2 < %{version}-%{release} Obsoletes: libefa-rdmav2 < %{version}-%{release} Obsoletes: libhfi1verbs-rdmav2 < %{version}-%{release} +Obsoletes: libhns-rdmav2 < %{version}-%{release} Obsoletes: libipathverbs-rdmav2 < %{version}-%{release} Obsoletes: libmana-rdmav2 < %{version}-%{release} Obsoletes: libmlx4-rdmav2 < %{version}-%{release} @@ -209,6 +213,7 @@ Obsoletes: libocrdma-rdmav2 < %{version}-%{release} Obsoletes: librxe-rdmav2 < %{version}-%{release} %if 0%{?dma_coherent} Requires: %{efa_lname} = %{version}-%{release} +Requires: %{hns_lname} = %{version}-%{release} Requires: %{mana_lname} = %{version}-%{release} Requires: %{mlx4_lname} = %{version}-%{release} Requires: %{mlx5_lname} = %{version}-%{release} @@ -228,7 +233,7 @@ Device-specific plug-in ibverbs userspace drivers are included: - libcxgb4: Chelsio T4 iWARP HCA - libefa: Amazon Elastic Fabric Adapter - libhfi1: Intel Omni-Path HFI -- libhns: HiSilicon Hip06 SoC +- libhns: HiSilicon Hip08+ SoC - libipathverbs: QLogic InfiniPath HCA - libirdma: Intel Ethernet Connection RDMA - libmana: Microsoft Azure Network Adapter @@ -256,6 +261,13 @@ Group: System/Libraries %description -n %efa_lname This package contains the efa runtime library. +%package -n %hns_lname +Summary: HNS runtime library +Group: System/Libraries + +%description -n %hns_lname +This package contains the hns runtime library. + %package -n %mana_lname Summary: MANA runtime library Group: System/Libraries @@ -508,6 +520,9 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh %post -n %efa_lname -p /sbin/ldconfig %postun -n %efa_lname -p /sbin/ldconfig +%post -n %hns_lname -p /sbin/ldconfig +%postun -n %hns_lname -p /sbin/ldconfig + %post -n %mana_lname -p /sbin/ldconfig %postun -n %mana_lname -p /sbin/ldconfig @@ -700,6 +715,10 @@ done %files -n %efa_lname %{_libdir}/libefa*.so.* +%files -n %hns_lname +%defattr(-,root,root) +%{_libdir}/libhns*.so.* + %files -n %mana_lname %{_libdir}/libmana*.so.* -- 2.33.0