packages/rdma-core
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
rdma-core/0067-libhns-Support-STARS-mode-QP.patch
Ran Zhou c350a79ab7 STARS is a HW scheduler. These patches support hns RoCE working in STARS mode which means RoCE will be scheduled by STARS. 
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com
Signed-off-by: Ran Zhou <zhouran10@h-partners.com>
(cherry picked from commit 6407ae1c796015fecebc9c82cfbc8f2988e23d43)
2023-11-03 11:54:55 +08:00

204 lines
6.5 KiB
Diff
Raw Blame History

From 904688d4b1cd02c27994c192d58b26903ac4e951 Mon Sep 17 00:00:00 2001
From: Chengchang Tang <tangchengchang@huawei.com>
Date: Mon, 30 Oct 2023 16:59:20 +0800
Subject: [PATCH 6/8] libhns: Support STARS mode QP
driver inclusion
category: feature
bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I8C6X4
---------------------------------------------------------------
STARS is a HW scheduler. QP in STARS mode will be taken over by STARS's
HW.
In this case, there is no need to drive the doorbell, otherwise it may
cause a CQE error.
Currently STARS only supports taking over SQ, so it only supports RDMA
operations.
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
---
providers/hns/hns_roce_u_hw_v2.c | 23 ++++++++-----
providers/hns/hns_roce_u_verbs.c | 59 ++++++++++++++++++++++++++++++--
providers/hns/hnsdv.h | 2 ++
3 files changed, 72 insertions(+), 12 deletions(-)
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 688b760..ab6d652 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -1675,11 +1675,13 @@ out:
udma_to_device_barrier();
- if (nreq == 1 && !ret &&
- (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
- hns_roce_write_dwqe(qp, wqe);
- else
- hns_roce_update_sq_db(ctx, qp);
+ if (!(qp->flags & HNS_ROCE_QP_CAP_STARS_SQ_MODE)) {
+ if (nreq == 1 && !ret &&
+ (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
+ hns_roce_write_dwqe(qp, wqe);
+ else
+ hns_roce_update_sq_db(ctx, qp);
+ }
if (qp->flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
*(qp->sdb) = qp->sq.head & 0xffff;
@@ -3008,10 +3010,13 @@ static int wr_complete(struct ibv_qp_ex *ibv_qp)
qp->next_sge = qp->sge_info.start_idx;
udma_to_device_barrier();
- if (nreq == 1 && (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
- hns_roce_write_dwqe(qp, qp->cur_wqe);
- else
- hns_roce_update_sq_db(ctx, qp);
+ if (!(qp->flags & HNS_ROCE_QP_CAP_STARS_SQ_MODE)) {
+ if (nreq == 1 &&
+ (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
+ hns_roce_write_dwqe(qp, qp->cur_wqe);
+ else
+ hns_roce_update_sq_db(ctx, qp);
+ }
if (qp->flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
*(qp->sdb) = qp->sq.head & 0xffff;
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index c7863d7..2ad9ea0 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -1211,7 +1211,8 @@ static int check_qp_congest_type(struct hns_roce_context *ctx,
enum {
HNSDV_QP_SUP_CREATE_FLAGS = HNSDV_QP_CREATE_ENABLE_DCA_MODE |
- HNSDV_QP_CREATE_ENABLE_UD_SL,
+ HNSDV_QP_CREATE_ENABLE_UD_SL |
+ HNSDV_QP_CREATE_ENABLE_STARS_MODE,
};
static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
@@ -1224,7 +1225,7 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
return 0;
if (!check_comp_mask(hns_attr->comp_mask, HNSDV_QP_SUP_COMP_MASK)) {
- verbs_err(&ctx->ibv_ctx, "invalid hnsdv comp_mask 0x%x.\n",
+ verbs_err(&ctx->ibv_ctx, "invalid comp_mask 0x%"PRIu64".\n",
hns_attr->comp_mask);
return -EINVAL;
}
@@ -1232,7 +1233,7 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
if (hns_attr->comp_mask & HNSDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS &&
!check_comp_mask(hns_attr->create_flags,
HNSDV_QP_SUP_CREATE_FLAGS)) {
- verbs_err(&ctx->ibv_ctx, "invalid create_flags 0x%x.\n",
+ verbs_err(&ctx->ibv_ctx, "invalid create_flags 0x%"PRIu32".\n",
hns_attr->create_flags);
return -EOPNOTSUPP;
}
@@ -1244,6 +1245,41 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
return 0;
}
+static int check_hnsdv_qp_create_flag(struct hns_roce_context *ctx,
+ struct ibv_qp_init_attr_ex *attr,
+ struct hnsdv_qp_init_attr *hns_attr,
+ uint32_t *hns_qp_create_flags)
+{
+ struct hns_roce_cq *send_cq = attr->send_cq ?
+ to_hr_cq(attr->send_cq) : NULL;
+
+ if (!hns_attr)
+ return 0;
+
+ if (hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_STARS_MODE) {
+ if (attr->qp_type != IBV_QPT_RC) {
+ verbs_err(&ctx->ibv_ctx,
+ "STARS mode only support RC\n");
+ return EINVAL;
+ }
+
+ if (hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_DCA_MODE) {
+ verbs_err(&ctx->ibv_ctx,
+ "STARS mode don't support DCA\n");
+ return EINVAL;
+ }
+
+ if (!send_cq || !(send_cq->flags & HNS_ROCE_CQ_FLAG_POE_EN)) {
+ verbs_err(&ctx->ibv_ctx,
+ "STARS QP should bind POE CQ with its SQ.\n");
+ return EINVAL;
+ }
+ *hns_qp_create_flags |= HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE;
+ }
+
+ return 0;
+}
+
enum {
CREATE_QP_SUP_COMP_MASK = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_XRCD |
IBV_QP_INIT_ATTR_SEND_OPS_FLAGS,
@@ -1352,6 +1388,11 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx,
if (ret)
return ret;
+ ret = check_hnsdv_qp_create_flag(ctx, attr, hns_attr,
+ &cmd_flag->create_flags);
+ if (ret)
+ return ret;
+
return verify_qp_create_cap(ctx, attr);
}
@@ -1801,6 +1842,11 @@ static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr,
cmd_ex.congest_type_flags = cmd_flag->congest_type_flags;
}
+ if (cmd_flag->create_flags) {
+ cmd_ex.comp_mask |= HNS_ROCE_CREATE_QP_MASK_CREATE_FLAGS;
+ cmd_ex.create_flags = cmd_flag->create_flags;
+ }
+
ret = ibv_cmd_create_qp_ex2(&ctx->ibv_ctx.context, &qp->verbs_qp, attr,
&cmd_ex.ibv_cmd, sizeof(cmd_ex),
&resp_ex.ibv_resp, sizeof(resp_ex));
@@ -2000,6 +2046,13 @@ struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
struct ibv_qp_init_attr_ex *qp_attr,
struct hnsdv_qp_init_attr *hns_attr)
{
+ struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL;
+
+ if (!ctx || !qp_attr) {
+ errno = EINVAL;
+ return NULL;
+ }
+
if (!is_hns_dev(context->device)) {
errno = EOPNOTSUPP;
return NULL;
diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h
index c5c7c11..98c68fd 100644
--- a/providers/hns/hnsdv.h
+++ b/providers/hns/hnsdv.h
@@ -44,6 +44,7 @@ struct ibv_context *hnsdv_open_device(struct ibv_device *device,
enum hnsdv_qp_create_flags {
HNSDV_QP_CREATE_ENABLE_DCA_MODE = 1 << 0,
HNSDV_QP_CREATE_ENABLE_UD_SL = 1 << 1,
+ HNSDV_QP_CREATE_ENABLE_STARS_MODE = 1 << 2,
};
enum hnsdv_qp_congest_ctrl_type {
@@ -62,6 +63,7 @@ struct hnsdv_qp_init_attr {
uint64_t comp_mask; /* Use enum hnsdv_qp_init_attr_mask */
uint32_t create_flags; /* Use enum hnsdv_qp_create_flags */
uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */
+ uint8_t reserved[7];
};
struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
--
2.25.1
Reference in New Issue View Git Blame Copy Permalink