packages/rdma-core
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!89 [sync] PR-71: Support user to choose using UD sl or pktype to adapt MPI APP

Browse Source 
From: @openeuler-sync-bot 
Reviewed-by: @hellotcc 
Signed-off-by: @hellotcc
This commit is contained in:
openeuler-ci-bot 2023-11-02 13:03:15 +00:00 committed by Gitee
commit 7bdd37a1ff
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
2 changed files with 186 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
0050-libhns-Support-user-to-choose-using-UD-sl-or-pktype-.patch Normal file
View File

@ -0,0 +1,178 @@
From fb07638be493fe0667e3fa0554ee434b7d6b3502 Mon Sep 17 00:00:00 2001
From: Junxian Huang <huangjunxian6@hisilicon.com>
Date: Fri, 9 Jun 2023 11:04:41 +0800
Subject: [PATCH] libhns: Support user to choose using UD sl or pktype to adapt
MPI APP
driver inclusion
category: bugfix
bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I7A5YM
---------------------------------------------------------------
According to Annex17_RoCEv2 (A17.4.5.2), for RoCEv2 UD, a CQE should
carry a flag that indicates if the received frame is an IPv4, IPv6 or
RoCE packet. But currently, the values of the flag corresponding to
these packet types haven't been defined yet in WC.
In UCX, 'sl' in ibv_wc for UD is used as the packet type flag, and the
packet type values have already been defined in the UCX patch of
https://github.com/openucx/ucx/commit/ed28845b88633e65d64fce8ec880060aa61bd59c
Therefore, to adapt UCX, add a create flag to hnsdv_create_qp() to allow
users to choose whether they use 'sl' in ibv_wc as service level or
packet type for UD. For the latter, obtain and translate the packet type
from CQE and fill it to 'sl' in ibv_wc.
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
Signed-off-by: Haoyue Xu <xuhaoyue1@hisilicon.com>
---
providers/hns/hns_roce_u.h | 11 +++++++++++
providers/hns/hns_roce_u_hw_v2.c | 32 +++++++++++++++++++++++++++++---
providers/hns/hns_roce_u_verbs.c | 16 ++++++++++++++++
providers/hns/hnsdv.h | 1 +
4 files changed, 57 insertions(+), 3 deletions(-)
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 41e9599..e3012e1 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -195,6 +195,12 @@ enum hns_roce_tc_map_mode {
HNS_ROCE_TC_MAP_MODE_DSCP,
};
+enum hns_roce_pktype {
+ HNS_ROCE_PKTYPE_ROCE_V1,
+ HNS_ROCE_PKTYPE_ROCE_V2_IPV6,
+ HNS_ROCE_PKTYPE_ROCE_V2_IPV4,
+};
+
struct hns_roce_db_page {
struct hns_roce_db_page *prev, *next;
struct hns_roce_buf buf;
@@ -406,6 +412,11 @@ struct hns_roce_qp {
void *cur_wqe;
unsigned int rb_sq_head; /* roll back sq head */
struct hns_roce_sge_info sge_info;
+
+ /* Just for UD. If not enabled, 'sl' in ibv_wc
+ * will be filled with 'port_type' in cqe.
+ */
+ bool enable_ud_sl;
};
struct hns_roce_av {
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 4b0ec5c..ee2fffe 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -522,9 +522,24 @@ static void handle_recv_rq_inl(struct hns_roce_v2_cqe *cqe,
handle_recv_inl_data(cqe, &(cur_qp->rq_rinl_buf), wr_num, wqe_buf);
}
-static void parse_for_ud_qp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc)
+static const uint8_t pktype_for_ud[] = {
+ HNS_ROCE_PKTYPE_ROCE_V1,
+ HNS_ROCE_PKTYPE_ROCE_V2_IPV4,
+ HNS_ROCE_PKTYPE_ROCE_V2_IPV6
+};
+
+static void parse_for_ud_qp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
+ bool enable_ud_sl)
{
- wc->sl = hr_reg_read(cqe, CQE_SL);
+ uint8_t port_type;
+
+ if (enable_ud_sl) {
+ wc->sl = hr_reg_read(cqe, CQE_SL);
+ } else {
+ port_type = hr_reg_read(cqe, CQE_PORT_TYPE);
+ wc->sl = pktype_for_ud[port_type];
+ }
+
wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN);
wc->slid = 0;
wc->wc_flags |= hr_reg_read(cqe, CQE_GRH) ? IBV_WC_GRH : 0;
@@ -554,7 +569,7 @@ static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
++wq->tail;
if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD)
- parse_for_ud_qp(cqe, wc);
+ parse_for_ud_qp(cqe, wc, hr_qp->enable_ud_sl);
if (hr_reg_read(cqe, CQE_CQE_INLINE))
handle_recv_cqe_inl_from_rq(cqe, hr_qp);
@@ -2126,6 +2141,17 @@ static uint32_t wc_read_slid(struct ibv_cq_ex *current)
static uint8_t wc_read_sl(struct ibv_cq_ex *current)
{
struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+ struct hns_roce_context *ctx = to_hr_ctx(current->context);
+ struct hns_roce_qp *hr_qp;
+ uint8_t port_type;
+ uint32_t qpn;
+
+ qpn = hr_reg_read(cq->cqe, CQE_LCL_QPN);
+ hr_qp = hns_roce_v2_find_qp(ctx, qpn);
+ if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD && !hr_qp->enable_ud_sl) {
+ port_type = hr_reg_read(cq->cqe, CQE_PORT_TYPE);
+ return pktype_for_ud[port_type];
+ }
return (uint8_t)hr_reg_read(cq->cqe, CQE_SL);
}
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index fa27fc1..4b641ea 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -1019,6 +1019,11 @@ static int check_qp_congest_type(struct hns_roce_context *ctx,
return 0;
}
+enum {
+ HNSDV_QP_SUP_CREATE_FLAGS = HNSDV_QP_CREATE_ENABLE_DCA_MODE |
+ HNSDV_QP_CREATE_ENABLE_UD_SL,
+};
+
static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
struct hnsdv_qp_init_attr *hns_attr,
struct hns_roce_cmd_flag *cmd_flag)
@@ -1034,6 +1039,14 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
return -EINVAL;
}
+ if (hns_attr->comp_mask & HNSDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS &&
+ !check_comp_mask(hns_attr->create_flags,
+ HNSDV_QP_SUP_CREATE_FLAGS)) {
+ verbs_err(&ctx->ibv_ctx, "invalid create_flags 0x%x.\n",
+ hns_attr->create_flags);
+ return -EOPNOTSUPP;
+ }
+
ret = check_qp_congest_type(ctx, hns_attr, cmd_flag);
if (ret)
return ret;
@@ -1685,6 +1698,9 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx,
qp_setup_config(attr, qp, context);
+ if (hns_attr && hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_UD_SL)
+ qp->enable_ud_sl = true;
+
return &qp->verbs_qp.qp;
err_dwqe:
diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h
index e15b428..365c314 100644
--- a/providers/hns/hnsdv.h
+++ b/providers/hns/hnsdv.h
@@ -43,6 +43,7 @@ struct ibv_context *hnsdv_open_device(struct ibv_device *device,
enum hnsdv_qp_create_flags {
HNSDV_QP_CREATE_ENABLE_DCA_MODE = 1 << 0,
+ HNSDV_QP_CREATE_ENABLE_UD_SL = 1 << 1,
};
enum hnsdv_qp_congest_ctrl_type {
--
2.25.1

9
rdma-core.spec
View File

@ -1,6 +1,6 @@
Name: rdma-core Name: rdma-core
Version: 41.0 Version: 41.0
Release: 12 Release: 13
Summary: RDMA core userspace libraries and daemons Summary: RDMA core userspace libraries and daemons
License: GPLv2 or BSD License: GPLv2 or BSD
Url: https://github.com/linux-rdma/rdma-core Url: https://github.com/linux-rdma/rdma-core
@ -55,6 +55,7 @@ Patch45: 0046-libhns-Disable-local-invalidate-operation.patch
Patch46: 0047-libhns-Fix-the-owner-bit-error-of-sq-in-new-io.patch Patch46: 0047-libhns-Fix-the-owner-bit-error-of-sq-in-new-io.patch
Patch47: 0048-libhns-Fix-incorrect-post-send-with-direct-wqe-of-wr.patch Patch47: 0048-libhns-Fix-incorrect-post-send-with-direct-wqe-of-wr.patch
Patch48: 0049-libhns-Add-a-judgment-to-the-congestion-control-algo.patch Patch48: 0049-libhns-Add-a-judgment-to-the-congestion-control-algo.patch
Patch49: 0050-libhns-Support-user-to-choose-using-UD-sl-or-pktype-.patch
BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0)
BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel
@ -302,6 +303,12 @@ fi
%{_mandir}/* %{_mandir}/*
%changelog %changelog
* Fri Jun 9 2023 Juan Zhou <zhoujuan51@h-partners.com> - 41.0-13
- Type: bugfix
- ID: NA
- SUG: NA
- DESC: Support user to choose using UD sl or pktype to adapt MPI APP
* Fri Jun 2 2023 Juan Zhou <zhoujuan51@h-partners.com> - 41.0-12 * Fri Jun 2 2023 Juan Zhou <zhoujuan51@h-partners.com> - 41.0-12
- Type: bugfix - Type: bugfix
- ID: NA - ID: NA