STARS is a HW scheduler. These patches support hns RoCE working in STARS mode which means RoCE will be scheduled by STARS.

Signed-off-by: Chengchang Tang <tangchengchang@huawei.com Signed-off-by: Ran Zhou <zhouran10@h-partners.com> (cherry picked from commit 6407ae1c796015fecebc9c82cfbc8f2988e23d43)
2023-10-31 10:04:56 +08:00 · 2023-10-31 10:04:56 +08:00 · c350a79ab7
commit c350a79ab7
parent bc114599bc
9 changed files with 1213 additions and 1 deletions
--- a/0062-Update-kernel-headers-for-libhns-query_device.patch
+++ b/0062-Update-kernel-headers-for-libhns-query_device.patch
@ -0,0 +1,44 @@
+From 3bf0c428672478f2460bd16483d33fd0ccdcb718 Mon Sep 17 00:00:00 2001
+From: Chengchang Tang <tangchengchang@huawei.com>
+Date: Mon, 30 Oct 2023 16:59:15 +0800
+Subject: [PATCH 1/8] Update kernel headers for libhns query_device()
+
+driver inclusion
+category: feature
+bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I8C6X4
+
+---------------------------------------------------------------
+
+To commit 513ecf0e21d3 ("RDMA/hns: Support query HW ID from user space.")
+
+Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
+---
+ kernel-headers/rdma/hns-abi.h | 14 ++++++++++++++
+ 1 file changed, 14 insertions(+)
+
+diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h
+index 6b94a89..d411c33 100644
+--- a/kernel-headers/rdma/hns-abi.h
+++ b/kernel-headers/rdma/hns-abi.h
+@@ -232,4 +232,18 @@ enum hns_ib_dca_mem_query_attrs {
+ 	HNS_IB_ATTR_DCA_MEM_QUERY_OUT_PAGE_COUNT,
+ };
+ 
+#define HNS_IB_INVALID_ID 0XFFFF
+struct hns_roce_ib_hw_id {
+	__u16 chip_id;
+	__u16 die_id;
+	__u16 func_id;
+	__u16 reserved;
+};
+
+struct hns_roce_ib_query_device_resp {
+	__u32   comp_mask;
+	__u32   len;
+	struct hns_roce_ib_hw_id hw_id;
+};
+
+ #endif /* HNS_ABI_USER_H */
+-- 
+2.25.1
+
--- a/0063-libhns-Support-query-HW-ID-by-hnsdv_query_device.patch
+++ b/0063-libhns-Support-query-HW-ID-by-hnsdv_query_device.patch
@ -0,0 +1,230 @@
+From 26d1c2339d0cc633b3817a78303eff6635a5fe88 Mon Sep 17 00:00:00 2001
+From: Chengchang Tang <tangchengchang@huawei.com>
+Date: Mon, 30 Oct 2023 16:59:16 +0800
+Subject: [PATCH 2/8] libhns: Support query HW ID by hnsdv_query_device()
+
+driver inclusion
+category: feature
+bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I8C6X4
+
+---------------------------------------------------------------
+
+Supports obtaining hardware-related ID through hnsdv_query_device()
+
+Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
+---
+ providers/hns/hns_roce_u.c       | 43 ++++++++++++++++++++++----------
+ providers/hns/hns_roce_u.h       |  1 +
+ providers/hns/hns_roce_u_abi.h   |  3 +++
+ providers/hns/hns_roce_u_verbs.c | 32 ++++++++++++++++++------
+ providers/hns/hnsdv.h            | 11 ++++++++
+ 5 files changed, 70 insertions(+), 20 deletions(-)
+
+diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
+index cef64ec..1085b85 100644
+--- a/providers/hns/hns_roce_u.c
+++ b/providers/hns/hns_roce_u.c
+@@ -313,6 +313,35 @@ static void ucontext_set_cmd(struct hns_roce_alloc_ucontext *cmd,
+ 	}
+ }
+ 
+static void set_default_hw_id(struct hns_roce_context *context)
+{
+	context->hw_id.chip_id = HNSDV_INVALID_HW_ID;
+	context->hw_id.die_id = HNSDV_INVALID_HW_ID;
+	context->hw_id.func_id = HNSDV_INVALID_HW_ID;
+}
+
+static int query_dev_attr(struct hns_roce_context *context,
+			  struct hns_roce_device *hr_dev)
+{
+	struct ibv_device_attr_ex attrx = {};
+	struct ibv_device_attr *dev_attrs = &attrx.orig_attr;
+
+	set_default_hw_id(context);
+
+	if (hns_roce_u_query_device(&context->ibv_ctx.context, NULL, &attrx,
+				    sizeof(attrx)))
+		return EINVAL;
+
+	hr_dev->hw_version = dev_attrs->hw_ver;
+	context->max_qp_wr = dev_attrs->max_qp_wr;
+	context->max_sge = dev_attrs->max_sge;
+	context->max_cqe = dev_attrs->max_cqe;
+	context->max_srq_wr = dev_attrs->max_srq_wr;
+	context->max_srq_sge = dev_attrs->max_srq_sge;
+
+	return 0;
+}
+
+ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
+ 						    int cmd_fd,
+ 						    void *private_data)
+@@ -321,7 +350,6 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
+ 	struct hns_roce_device *hr_dev = to_hr_dev(ibdev);
+ 	struct hns_roce_alloc_ucontext_resp resp = {};
+ 	struct hns_roce_alloc_ucontext cmd = {};
+-	struct ibv_device_attr dev_attrs;
+ 	struct hns_roce_context *context;
+ 	int i;
+ 
+@@ -362,23 +390,12 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
+ 	for (i = 0; i < HNS_ROCE_SRQ_TABLE_SIZE; ++i)
+ 		context->srq_table[i].refcnt = 0;
+ 
+-	if (hns_roce_u_query_device(&context->ibv_ctx.context, NULL,
+-				    container_of(&dev_attrs,
+-						 struct ibv_device_attr_ex,
+-						 orig_attr),
+-				    sizeof(dev_attrs)))
+	if (query_dev_attr(context, hr_dev))
+ 		goto err_free;
+ 
+-	hr_dev->hw_version = dev_attrs.hw_ver;
+ 	if (get_link_type(dev_attrs.vendor_part_id, &hr_dev->link_type))
+ 		hr_dev->link_type = resp.mac_type;
+ 
+-	context->max_qp_wr = dev_attrs.max_qp_wr;
+-	context->max_sge = dev_attrs.max_sge;
+-	context->max_cqe = dev_attrs.max_cqe;
+-	context->max_srq_wr = dev_attrs.max_srq_wr;
+-	context->max_srq_sge = dev_attrs.max_srq_sge;
+-
+ 	if (init_dca_context(context, cmd_fd,
+ 			     &resp, ctx_attr, hr_dev->page_size))
+ 		goto err_free;
+diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
+index 662eb8a..323d2f9 100644
+--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
+@@ -280,6 +280,7 @@ struct hns_roce_context {
+ 	unsigned int			max_inline_data;
+ 
+ 	struct hns_roce_dca_ctx		dca_ctx;
+	struct hnsdv_hw_id hw_id;
+ 
+ 	bool use_new_reset_flag;
+ 	bool reseted;
+diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h
+index 1eaf62d..c73a30d 100644
+--- a/providers/hns/hns_roce_u_abi.h
+++ b/providers/hns/hns_roce_u_abi.h
+@@ -68,4 +68,7 @@ DECLARE_DRV_CMD(hns_roce_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ,
+ DECLARE_DRV_CMD(hns_roce_modify_qp_ex, IB_USER_VERBS_EX_CMD_MODIFY_QP,
+ 		empty, hns_roce_ib_modify_qp_resp);
+ 
+DECLARE_DRV_CMD(hns_roce_query_device_ex, IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
+		empty, hns_roce_ib_query_device_resp);
+
+ #endif /* _HNS_ROCE_U_ABI_H */
+diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
+index c3b01a0..5b86077 100644
+--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
+@@ -84,22 +84,27 @@ void hns_roce_init_qp_indices(struct hns_roce_qp *qp)
+ 	qp->next_sge = 0;
+ }
+ 
+#define HNSDV_CONVERT_HW_ID(val) \
+	((val) == HNS_IB_INVALID_ID ? HNSDV_INVALID_HW_ID : (val))
+
+ int hns_roce_u_query_device(struct ibv_context *context,
+ 			    const struct ibv_query_device_ex_input *input,
+ 			    struct ibv_device_attr_ex *attr, size_t attr_size)
+ {
+-	struct ib_uverbs_ex_query_device_resp resp;
+	struct hns_roce_context *ctx = to_hr_ctx(context);
+	struct hns_roce_query_device_ex_resp resp = {};
+ 	unsigned int major, minor, sub_minor;
+ 	size_t resp_size = sizeof(resp);
+ 	uint64_t raw_fw_ver;
+ 	int ret;
+ 
+-	ret = ibv_cmd_query_device_any(context, input, attr, attr_size, &resp,
+-				       &resp_size);
+	ctx = container_of(context, struct hns_roce_context, ibv_ctx.context);
+	ret = ibv_cmd_query_device_any(context, input, attr, attr_size,
+				       &resp.ibv_resp, &resp_size);
+ 	if (ret)
+ 		return ret;
+ 
+-	raw_fw_ver = resp.base.fw_ver;
+	raw_fw_ver = resp.ibv_resp.base.fw_ver;
+ 	major = (raw_fw_ver >> 32) & 0xffff;
+ 	minor = (raw_fw_ver >> 16) & 0xffff;
+ 	sub_minor = raw_fw_ver & 0xffff;
+@@ -107,27 +112,40 @@ int hns_roce_u_query_device(struct ibv_context *context,
+ 	snprintf(attr->orig_attr.fw_ver, sizeof(attr->orig_attr.fw_ver),
+ 		 "%u.%u.%03u", major, minor, sub_minor);
+ 
+	if (resp.len >= offsetofend(typeof(resp.drv_payload), hw_id)) {
+		ctx->hw_id.chip_id = HNSDV_CONVERT_HW_ID(resp.hw_id.chip_id);
+		ctx->hw_id.die_id = HNSDV_CONVERT_HW_ID(resp.hw_id.die_id);
+		ctx->hw_id.func_id = HNSDV_CONVERT_HW_ID(resp.hw_id.func_id);
+	}
+
+ 	return 0;
+ }
+ 
+ int hnsdv_query_device(struct ibv_context *context,
+ 		       struct hnsdv_context *attrs_out)
+ {
+	struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL;
+ 	struct hns_roce_device *hr_dev = to_hr_dev(context->device);
+ 
+	if (!ctx || !attrs_out)
+		return EINVAL;
+
+ 	if (!hr_dev) {
+ 		verbs_err(verbs_get_ctx(context), "not a HNS RoCE device!\n");
+ 		return EOPNOTSUPP;
+ 	}
+ 
+-	if (!attrs_out)
+-		return EINVAL;
+-
+ 	memset(attrs_out, 0, sizeof(*attrs_out));
+ 
+ 	attrs_out->comp_mask |= HNSDV_CONTEXT_MASK_CONGEST_TYPE;
+ 	attrs_out->congest_type = hr_dev->congest_type;
+ 
+	if (ctx->hw_id.chip_id != HNSDV_INVALID_HW_ID) {
+		attrs_out->comp_mask |= HNSDV_CONTEXT_MASK_HW_ID;
+		memcpy(&attrs_out->hw_id, &ctx->hw_id,
+		       sizeof(struct hnsdv_hw_id));
+	}
+
+ 	return 0;
+ }
+ 
+diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h
+index 365c314..159edb8 100644
+--- a/providers/hns/hnsdv.h
+++ b/providers/hns/hnsdv.h
+@@ -70,12 +70,23 @@ struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
+ 
+ enum hnsdv_query_context_comp_mask {
+ 	HNSDV_CONTEXT_MASK_CONGEST_TYPE = 1 << 0,
+	HNSDV_CONTEXT_MASK_HW_ID = 1 << 1,
+};
+
+#define HNSDV_INVALID_HW_ID -1
+struct hnsdv_hw_id {
+	int32_t chip_id;
+	int32_t die_id;
+	int32_t func_id;
+	int32_t reserved;
+ };
+ 
+ struct hnsdv_context {
+ 	uint64_t comp_mask; /* use enum hnsdv_query_context_comp_mask */
+ 	uint64_t flags;
+ 	uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */
+	uint8_t rsv[7];
+	struct hnsdv_hw_id hw_id;
+ };
+ 
+ int hnsdv_query_device(struct ibv_context *ctx_in,
+-- 
+2.25.1
+
--- a/0064-Update-kernel-headers-for-supporting-POE-CQs.patch
+++ b/0064-Update-kernel-headers-for-supporting-POE-CQs.patch
@ -0,0 +1,49 @@
+From 4fe79a9dc4b3b735ededf17dada62d022d170394 Mon Sep 17 00:00:00 2001
+From: Chengchang Tang <tangchengchang@huawei.com>
+Date: Mon, 30 Oct 2023 16:59:17 +0800
+Subject: [PATCH 3/8] Update kernel headers for supporting POE CQs
+
+driver inclusion
+category: feature
+bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I8C6X4
+
+---------------------------------------------------------------
+
+To commit eb68cb8c47ed ("RDMA/hns: Support configuring POE channels and creating POE CQs")
+
+Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
+---
+ kernel-headers/rdma/hns-abi.h | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+
+diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h
+index d411c33..817fc06 100644
+--- a/kernel-headers/rdma/hns-abi.h
+++ b/kernel-headers/rdma/hns-abi.h
+@@ -36,15 +36,23 @@
+ 
+ #include <linux/types.h>
+ 
+enum hns_roce_create_cq_create_flags {
+	HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE = 1 << 0,
+};
+
+ struct hns_roce_ib_create_cq {
+ 	__aligned_u64 buf_addr;
+ 	__aligned_u64 db_addr;
+ 	__u32 cqe_size;
+ 	__u32 reserved;
+	__aligned_u64 create_flags; /* Use enum hns_roce_create_cq_create_flags */
+	__u8 poe_channel;
+	__u8 rsv[7];
+ };
+ 
+ enum hns_roce_cq_cap_flags {
+ 	HNS_ROCE_CQ_FLAG_RECORD_DB = 1 << 0,
+	HNS_ROCE_CQ_FLAG_POE_EN = 1 << 2,
+ };
+ 
+ struct hns_roce_ib_create_cq_resp {
+-- 
+2.25.1
+
--- a/0065-libhns-Add-support-for-POE-CQs.patch
+++ b/0065-libhns-Add-support-for-POE-CQs.patch
@ -0,0 +1,354 @@
+From 80a8e63d5e764868cbaf2af3e522a320b5781508 Mon Sep 17 00:00:00 2001
+From: Chengchang Tang <tangchengchang@huawei.com>
+Date: Mon, 30 Oct 2023 16:59:18 +0800
+Subject: [PATCH 4/8] libhns: Add support for POE CQs
+
+driver inclusion
+category: feature
+bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I8C6X4
+
+---------------------------------------------------------------
+
+Added support for POE CQs. POE CQs will forward CQE directly to
+the corresponding POE channel.
+
+In this case, the driver cannot update the QP CI, so
+hnsdv_update_sq_ci() is added to allow users to change the QP CI.
+
+Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
+---
+ providers/hns/hns_roce_u.c       |  11 ++--
+ providers/hns/hns_roce_u_hw_v2.c |  19 ++++++
+ providers/hns/hns_roce_u_verbs.c | 110 ++++++++++++++++++++++++++++---
+ providers/hns/hnsdv.h            |  22 +++++++
+ providers/hns/libhns.map         |   2 +
+ 5 files changed, 150 insertions(+), 14 deletions(-)
+
+diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
+index 1085b85..084385b 100644
+--- a/providers/hns/hns_roce_u.c
+++ b/providers/hns/hns_roce_u.c
+@@ -321,7 +321,8 @@ static void set_default_hw_id(struct hns_roce_context *context)
+ }
+ 
+ static int query_dev_attr(struct hns_roce_context *context,
+-			  struct hns_roce_device *hr_dev)
+			  struct hns_roce_device *hr_dev,
+			  struct hns_roce_alloc_ucontext_resp *resp)
+ {
+ 	struct ibv_device_attr_ex attrx = {};
+ 	struct ibv_device_attr *dev_attrs = &attrx.orig_attr;
+@@ -339,6 +340,9 @@ static int query_dev_attr(struct hns_roce_context *context,
+ 	context->max_srq_wr = dev_attrs->max_srq_wr;
+ 	context->max_srq_sge = dev_attrs->max_srq_sge;
+ 
+	if (get_link_type(dev_attrs->vendor_part_id, &hr_dev->link_type))
+		hr_dev->link_type = resp->mac_type;
+
+ 	return 0;
+ }
+ 
+@@ -390,12 +394,9 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
+ 	for (i = 0; i < HNS_ROCE_SRQ_TABLE_SIZE; ++i)
+ 		context->srq_table[i].refcnt = 0;
+ 
+-	if (query_dev_attr(context, hr_dev))
+	if (query_dev_attr(context, hr_dev, &resp))
+ 		goto err_free;
+ 
+-	if (get_link_type(dev_attrs.vendor_part_id, &hr_dev->link_type))
+-		hr_dev->link_type = resp.mac_type;
+-
+ 	if (init_dca_context(context, cmd_fd,
+ 			     &resp, ctx_attr, hr_dev->page_size))
+ 		goto err_free;
+diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
+index fbd02dc..688b760 100644
+--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
+@@ -1029,6 +1029,9 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne,
+ 	int err = V2_CQ_OK;
+ 	int npolled;
+ 
+	if (cq->flags & HNS_ROCE_CQ_FLAG_POE_EN)
+		return V2_CQ_POLL_ERR;
+
+ 	hns_roce_spin_lock(&cq->hr_lock);
+ 
+ 	if (unlikely(hns_roce_reseted(ctx))) {
+@@ -2202,6 +2205,9 @@ static int wc_start_poll_cq(struct ibv_cq_ex *current,
+ 	if (attr->comp_mask)
+ 		return EINVAL;
+ 
+	if (cq->flags & HNS_ROCE_CQ_FLAG_POE_EN)
+		return EOPNOTSUPP;
+
+ 	hns_roce_spin_lock(&cq->hr_lock);
+ 
+ 	if (unlikely(hns_roce_reseted(ctx))) {
+@@ -3119,6 +3125,19 @@ int hns_roce_attach_qp_ex_ops(struct ibv_qp_init_attr_ex *attr,
+ 	return 0;
+ }
+ 
+void hnsdv_update_sq_ci(struct ibv_qp *ibv_qp, uint32_t inc_cnt)
+{
+	struct hns_roce_qp *qp;
+	struct hns_roce_wq *wq;
+
+	if (!ibv_qp)
+		return;
+
+	qp = to_hr_qp(ibv_qp);
+	wq = &qp->sq;
+	wq->tail += inc_cnt & (wq->wqe_cnt - 1);
+}
+
+ const struct hns_roce_u_hw hns_roce_u_hw_v2 = {
+ 	.hw_version = HNS_ROCE_HW_VER2,
+ 	.hw_ops = {
+diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
+index 5b86077..c7863d7 100644
+--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
+@@ -36,6 +36,7 @@
+ #include <math.h>
+ #include <errno.h>
+ #include <pthread.h>
+#include <inttypes.h>
+ #include <sys/mman.h>
+ #include <ccan/ilog.h>
+ #include <ccan/minmax.h>
+@@ -470,8 +471,40 @@ enum {
+ 				       IBV_WC_EX_WITH_CVLAN,
+ };
+ 
+enum {
+	HNSDV_CQ_SUP_COMP_MASK = HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS,
+};
+
+enum {
+	HNSDV_CQ_SUP_CREATE_FLAGS = HNSDV_CQ_CREATE_ENABLE_POE_MODE,
+};
+
+static int verify_hnsdv_cq_create_attr(struct hns_roce_context *ctx,
+				       struct hnsdv_cq_init_attr *hns_cq_attr)
+{
+	if (!hns_cq_attr)
+		return 0;
+
+	if (!check_comp_mask(hns_cq_attr->comp_mask, HNSDV_CQ_SUP_COMP_MASK)) {
+		verbs_err(&ctx->ibv_ctx, "Unsupported cq comps 0x%"PRIu64"\n",
+				hns_cq_attr->comp_mask);
+		return EOPNOTSUPP;
+	}
+
+	if ((hns_cq_attr->comp_mask & HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS) &&
+	    !check_comp_mask(hns_cq_attr->create_flags,
+			     HNSDV_CQ_SUP_CREATE_FLAGS)) {
+		verbs_err(&ctx->ibv_ctx, "Unsupported cq flags 0x%"PRIu64"\n",
+				hns_cq_attr->create_flags);
+		return EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr,
+-				 struct hns_roce_context *context)
+				 struct hns_roce_context *context,
+				 struct hnsdv_cq_init_attr *hns_cq_attr)
+ {
+ 	if (!attr->cqe || attr->cqe > context->max_cqe)
+ 		return -EINVAL;
+@@ -495,7 +528,7 @@ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr,
+ 	attr->cqe = max_t(uint32_t, HNS_ROCE_MIN_CQE_NUM,
+ 			  roundup_pow_of_two(attr->cqe));
+ 
+-	return 0;
+	return verify_hnsdv_cq_create_attr(context, hns_cq_attr);
+ }
+ 
+ static int hns_roce_alloc_cq_buf(struct hns_roce_cq *cq)
+@@ -508,9 +541,34 @@ static int hns_roce_alloc_cq_buf(struct hns_roce_cq *cq)
+ 	return 0;
+ }
+ 
+static void set_hnsdv_cq_attr(struct hns_roce_ib_create_cq *cmd_drv,
+			      struct hnsdv_cq_init_attr *hns_cq_attr)
+{
+	if (!hns_cq_attr)
+		return;
+
+	if (hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_POE_MODE) {
+		cmd_drv->create_flags |= HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE;
+		cmd_drv->poe_channel = hns_cq_attr->poe_channel;
+	}
+}
+
+static int check_hnsdv_cq_flags(struct hnsdv_cq_init_attr *hns_cq_attr,
+			  struct hns_roce_cq *cq)
+{
+	if (!hns_cq_attr)
+		return 0;
+
+	if ((hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_POE_MODE) &&
+			!(cq->flags & HNS_ROCE_CQ_FLAG_POE_EN))
+		return EOPNOTSUPP;
+	return 0;
+}
+
+ static int exec_cq_create_cmd(struct ibv_context *context,
+ 			      struct hns_roce_cq *cq,
+-			      struct ibv_cq_init_attr_ex *attr)
+			      struct ibv_cq_init_attr_ex *attr,
+			      struct hnsdv_cq_init_attr *hns_cq_attr)
+ {
+ 	struct hns_roce_create_cq_ex_resp resp_ex = {};
+ 	struct hns_roce_ib_create_cq_resp *resp_drv;
+@@ -525,6 +583,8 @@ static int exec_cq_create_cmd(struct ibv_context *context,
+ 	cmd_drv->db_addr = (uintptr_t)cq->db;
+ 	cmd_drv->cqe_size = (uintptr_t)cq->cqe_size;
+ 
+	set_hnsdv_cq_attr(cmd_drv, hns_cq_attr);
+
+ 	ret = ibv_cmd_create_cq_ex(context, attr, &cq->verbs_cq,
+ 				   &cmd_ex.ibv_cmd, sizeof(cmd_ex),
+ 				   &resp_ex.ibv_resp, sizeof(resp_ex), 0);
+@@ -534,7 +594,14 @@ static int exec_cq_create_cmd(struct ibv_context *context,
+ 	cq->cqn = resp_drv->cqn;
+ 	cq->flags = resp_drv->cap_flags;
+ 
+-	return 0;
+	ret = check_hnsdv_cq_flags(hns_cq_attr, cq);
+	if (ret)
+		goto flags_err;
+	return ret;
+
+flags_err:
+	ibv_cmd_destroy_cq(&cq->verbs_cq.cq);
+	return ret;
+ }
+ 
+ static int hns_roce_init_cq_swc(struct hns_roce_cq *cq,
+@@ -581,13 +648,14 @@ static int hns_roce_cq_spinlock_init(struct ibv_context *context,
+ }
+ 
+ static struct ibv_cq_ex *create_cq(struct ibv_context *context,
+-			 struct ibv_cq_init_attr_ex *attr)
+				   struct ibv_cq_init_attr_ex *attr,
+				   struct hnsdv_cq_init_attr *hns_cq_attr)
+ {
+ 	struct hns_roce_context *hr_ctx = to_hr_ctx(context);
+ 	struct hns_roce_cq *cq;
+ 	int ret;
+ 
+-	ret = verify_cq_create_attr(attr, hr_ctx);
+	ret = verify_cq_create_attr(attr, hr_ctx, hns_cq_attr);
+ 	if (ret)
+ 		goto err;
+ 
+@@ -618,7 +686,7 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *context,
+ 	if (ret)
+ 		goto err_swc;
+ 
+-	ret = exec_cq_create_cmd(context, cq, attr);
+	ret = exec_cq_create_cmd(context, cq, attr, hns_cq_attr);
+ 	if (ret)
+ 		goto err_cmd;
+ 
+@@ -652,7 +720,7 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe,
+ 		.comp_vector = comp_vector,
+ 	};
+ 
+-	cq = create_cq(context, &attr);
+	cq = create_cq(context, &attr, NULL);
+ 	return cq ? ibv_cq_ex_to_cq(cq) : NULL;
+ }
+ 
+@@ -661,13 +729,37 @@ struct ibv_cq_ex *hns_roce_u_create_cq_ex(struct ibv_context *context,
+ {
+ 	struct ibv_cq_ex *cq;
+ 
+-	cq = create_cq(context, attr);
+	cq = create_cq(context, attr, NULL);
+ 	if (cq)
+ 		hns_roce_attach_cq_ex_ops(cq, attr->wc_flags);
+ 
+ 	return cq;
+ }
+ 
+struct ibv_cq_ex *hnsdv_create_cq_ex(struct ibv_context *context,
+				     struct ibv_cq_init_attr_ex *cq_attr,
+				     struct hnsdv_cq_init_attr *hns_cq_attr)
+{
+	struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL;
+	struct ibv_cq_ex *cq;
+
+	if (!ctx || !cq_attr) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	if (!is_hns_dev(context->device)) {
+		errno = EOPNOTSUPP;
+		return NULL;
+	}
+
+	cq = create_cq(context, cq_attr, hns_cq_attr);
+	if (cq)
+		hns_roce_attach_cq_ex_ops(cq, cq_attr->wc_flags);
+
+	return cq;
+}
+
+ void hns_roce_u_cq_event(struct ibv_cq *cq)
+ {
+ 	to_hr_cq(cq)->arm_sn++;
+diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h
+index 159edb8..c5c7c11 100644
+--- a/providers/hns/hnsdv.h
+++ b/providers/hns/hnsdv.h
+@@ -92,6 +92,28 @@ struct hnsdv_context {
+ int hnsdv_query_device(struct ibv_context *ctx_in,
+ 		       struct hnsdv_context *attrs_out);
+ 
+enum hnsdv_cq_init_attr_mask {
+	HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS = 1 << 0,
+};
+
+enum hnsdv_cq_create_flags {
+	HNSDV_CQ_CREATE_ENABLE_POE_MODE = 1 << 0,
+};
+
+struct hnsdv_cq_init_attr {
+	uint64_t comp_mask; /* Use enum hnsdv_cq_init_attr_mask */
+	uint64_t create_flags; /* Use enum hnsdv_cq_create_flags */
+	uint8_t poe_channel; /* poe channel to use */
+	uint8_t reserved[7];
+};
+
+struct ibv_cq_ex *hnsdv_create_cq_ex(struct ibv_context *context,
+				     struct ibv_cq_init_attr_ex *cq_attr,
+				     struct hnsdv_cq_init_attr *hns_cq_attr);
+
+/* used in stars mode */
+void hnsdv_update_sq_ci(struct ibv_qp *qp, uint32_t inc_idx);
+
+ #ifdef __cplusplus
+ }
+ #endif
+diff --git a/providers/hns/libhns.map b/providers/hns/libhns.map
+index ebf28eb..27efc83 100644
+--- a/providers/hns/libhns.map
+++ b/providers/hns/libhns.map
+@@ -6,5 +6,7 @@ HNS_1.0 {
+ 		hnsdv_open_device;
+ 		hnsdv_create_qp;
+ 		hnsdv_query_device;
+		hnsdv_create_cq_ex;
+		hnsdv_update_sq_ci;
+ 	local: *;
+ };
+-- 
+2.25.1
+
--- a/0066-Update-kernel-headers-for-supporting-STARS-QP-in-HNS.patch
+++ b/0066-Update-kernel-headers-for-supporting-STARS-QP-in-HNS.patch
@ -0,0 +1,60 @@
+From 56e347494c9d534e521ce836ad8419d0e3857e95 Mon Sep 17 00:00:00 2001
+From: Chengchang Tang <tangchengchang@huawei.com>
+Date: Mon, 30 Oct 2023 16:59:19 +0800
+Subject: [PATCH 5/8] Update kernel headers for supporting STARS QP in HNS
+
+driver inclusion
+category: feature
+bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I8C6X4
+
+---------------------------------------------------------------
+
+To commit 9044608a998e ("RDMA/hns: Support STARS mode QP")
+
+Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
+---
+ kernel-headers/rdma/hns-abi.h | 11 ++++++++---
+ 1 file changed, 8 insertions(+), 3 deletions(-)
+
+diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h
+index 817fc06..5969bde 100644
+--- a/kernel-headers/rdma/hns-abi.h
+++ b/kernel-headers/rdma/hns-abi.h
+@@ -86,6 +86,10 @@ enum hns_roce_create_qp_comp_mask {
+ 	HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE = 1 << 1,
+ };
+ 
+enum hns_roce_create_qp_flags {
+	HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE = 1 << 0,
+};
+
+ enum hns_roce_congest_type_flags {
+ 	HNS_ROCE_CREATE_QP_FLAGS_DCQCN = 1 << 0,
+ 	HNS_ROCE_CREATE_QP_FLAGS_LDCP = 1 << 1,
+@@ -102,8 +106,8 @@ struct hns_roce_ib_create_qp {
+ 	__u8    reserved[4];
+ 	__u8    pageshift;
+ 	__aligned_u64 sdb_addr;
+-	__aligned_u64 comp_mask;
+-	__aligned_u64 create_flags;
+	__aligned_u64 comp_mask; /* Use enum hns_roce_create_qp_comp_mask */
+	__aligned_u64 create_flags;  /* Use enum hns_roce_create_qp_flags */
+ 	__aligned_u64 congest_type_flags;
+ };
+ 
+@@ -115,10 +119,11 @@ enum hns_roce_qp_cap_flags {
+ 	HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH = 1 << 4,
+ 	HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5,
+ 	HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH = 1 << 6,
+	HNS_ROCE_QP_CAP_STARS_SQ_MODE = 1 << 7,
+ };
+ 
+ struct hns_roce_ib_create_qp_resp {
+-	__aligned_u64 cap_flags;
+	__aligned_u64 cap_flags; /* Use enum hns_roce_qp_cap_flags */
+ 	__aligned_u64 dwqe_mmap_key;
+ };
+ 
+-- 
+2.25.1
+
--- a/0067-libhns-Support-STARS-mode-QP.patch
+++ b/0067-libhns-Support-STARS-mode-QP.patch
@ -0,0 +1,203 @@
+From 904688d4b1cd02c27994c192d58b26903ac4e951 Mon Sep 17 00:00:00 2001
+From: Chengchang Tang <tangchengchang@huawei.com>
+Date: Mon, 30 Oct 2023 16:59:20 +0800
+Subject: [PATCH 6/8] libhns: Support STARS mode QP
+
+driver inclusion
+category: feature
+bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I8C6X4
+
+---------------------------------------------------------------
+
+STARS is a HW scheduler. QP in STARS mode will be taken over by STARS's
+HW.
+
+In this case, there is no need to drive the doorbell, otherwise it may
+cause a CQE error.
+
+Currently STARS only supports taking over SQ, so it only supports RDMA
+operations.
+
+Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
+---
+ providers/hns/hns_roce_u_hw_v2.c | 23 ++++++++-----
+ providers/hns/hns_roce_u_verbs.c | 59 ++++++++++++++++++++++++++++++--
+ providers/hns/hnsdv.h            |  2 ++
+ 3 files changed, 72 insertions(+), 12 deletions(-)
+
+diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
+index 688b760..ab6d652 100644
+--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
+@@ -1675,11 +1675,13 @@ out:
+ 
+ 		udma_to_device_barrier();
+ 
+-		if (nreq == 1 && !ret &&
+-		    (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
+-			hns_roce_write_dwqe(qp, wqe);
+-		else
+-			hns_roce_update_sq_db(ctx, qp);
+		if (!(qp->flags & HNS_ROCE_QP_CAP_STARS_SQ_MODE)) {
+			if (nreq == 1 && !ret &&
+			    (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
+				hns_roce_write_dwqe(qp, wqe);
+			else
+				hns_roce_update_sq_db(ctx, qp);
+		}
+ 
+ 		if (qp->flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
+ 			*(qp->sdb) = qp->sq.head & 0xffff;
+@@ -3008,10 +3010,13 @@ static int wr_complete(struct ibv_qp_ex *ibv_qp)
+ 		qp->next_sge = qp->sge_info.start_idx;
+ 		udma_to_device_barrier();
+ 
+-		if (nreq == 1 && (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
+-			hns_roce_write_dwqe(qp, qp->cur_wqe);
+-		else
+-			hns_roce_update_sq_db(ctx, qp);
+		if (!(qp->flags & HNS_ROCE_QP_CAP_STARS_SQ_MODE)) {
+			if (nreq == 1 &&
+			    (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
+				hns_roce_write_dwqe(qp, qp->cur_wqe);
+			else
+				hns_roce_update_sq_db(ctx, qp);
+		}
+ 
+ 		if (qp->flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
+ 			*(qp->sdb) = qp->sq.head & 0xffff;
+diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
+index c7863d7..2ad9ea0 100644
+--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
+@@ -1211,7 +1211,8 @@ static int check_qp_congest_type(struct hns_roce_context *ctx,
+ 
+ enum {
+ 	HNSDV_QP_SUP_CREATE_FLAGS = HNSDV_QP_CREATE_ENABLE_DCA_MODE |
+-				    HNSDV_QP_CREATE_ENABLE_UD_SL,
+				    HNSDV_QP_CREATE_ENABLE_UD_SL |
+				    HNSDV_QP_CREATE_ENABLE_STARS_MODE,
+ };
+ 
+ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
+@@ -1224,7 +1225,7 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
+ 		return 0;
+ 
+ 	if (!check_comp_mask(hns_attr->comp_mask, HNSDV_QP_SUP_COMP_MASK)) {
+-		verbs_err(&ctx->ibv_ctx, "invalid hnsdv comp_mask 0x%x.\n",
+		verbs_err(&ctx->ibv_ctx, "invalid comp_mask 0x%"PRIu64".\n",
+ 			  hns_attr->comp_mask);
+ 		return -EINVAL;
+ 	}
+@@ -1232,7 +1233,7 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
+ 	if (hns_attr->comp_mask & HNSDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS &&
+ 	    !check_comp_mask(hns_attr->create_flags,
+ 			     HNSDV_QP_SUP_CREATE_FLAGS)) {
+-		verbs_err(&ctx->ibv_ctx, "invalid create_flags 0x%x.\n",
+		verbs_err(&ctx->ibv_ctx, "invalid create_flags 0x%"PRIu32".\n",
+ 			  hns_attr->create_flags);
+ 		return -EOPNOTSUPP;
+ 	}
+@@ -1244,6 +1245,41 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
+ 	return 0;
+ }
+ 
+static int check_hnsdv_qp_create_flag(struct hns_roce_context *ctx,
+				      struct ibv_qp_init_attr_ex *attr,
+				      struct hnsdv_qp_init_attr *hns_attr,
+				      uint32_t *hns_qp_create_flags)
+{
+	struct hns_roce_cq *send_cq = attr->send_cq ?
+				      to_hr_cq(attr->send_cq) : NULL;
+
+	if (!hns_attr)
+		return 0;
+
+	if (hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_STARS_MODE) {
+		if (attr->qp_type != IBV_QPT_RC) {
+			verbs_err(&ctx->ibv_ctx,
+				  "STARS mode only support RC\n");
+			return EINVAL;
+		}
+
+		if (hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_DCA_MODE) {
+			verbs_err(&ctx->ibv_ctx,
+				  "STARS mode don't support DCA\n");
+			return EINVAL;
+		}
+
+		if (!send_cq || !(send_cq->flags & HNS_ROCE_CQ_FLAG_POE_EN)) {
+			verbs_err(&ctx->ibv_ctx,
+				  "STARS QP should bind POE CQ with its SQ.\n");
+			return EINVAL;
+		}
+		*hns_qp_create_flags |= HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE;
+	}
+
+	return 0;
+}
+
+ enum {
+ 	CREATE_QP_SUP_COMP_MASK = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_XRCD |
+ 				  IBV_QP_INIT_ATTR_SEND_OPS_FLAGS,
+@@ -1352,6 +1388,11 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx,
+ 	if (ret)
+ 		return ret;
+ 
+	ret = check_hnsdv_qp_create_flag(ctx, attr, hns_attr,
+					 &cmd_flag->create_flags);
+	if (ret)
+		return ret;
+
+ 	return verify_qp_create_cap(ctx, attr);
+ }
+ 
+@@ -1801,6 +1842,11 @@ static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr,
+ 		cmd_ex.congest_type_flags = cmd_flag->congest_type_flags;
+ 	}
+ 
+	if (cmd_flag->create_flags) {
+		cmd_ex.comp_mask |= HNS_ROCE_CREATE_QP_MASK_CREATE_FLAGS;
+		cmd_ex.create_flags = cmd_flag->create_flags;
+	}
+
+ 	ret = ibv_cmd_create_qp_ex2(&ctx->ibv_ctx.context, &qp->verbs_qp, attr,
+ 				    &cmd_ex.ibv_cmd, sizeof(cmd_ex),
+ 				    &resp_ex.ibv_resp, sizeof(resp_ex));
+@@ -2000,6 +2046,13 @@ struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
+ 			       struct ibv_qp_init_attr_ex *qp_attr,
+ 			       struct hnsdv_qp_init_attr *hns_attr)
+ {
+	struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL;
+
+	if (!ctx || !qp_attr) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+ 	if (!is_hns_dev(context->device)) {
+ 		errno = EOPNOTSUPP;
+ 		return NULL;
+diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h
+index c5c7c11..98c68fd 100644
+--- a/providers/hns/hnsdv.h
+++ b/providers/hns/hnsdv.h
+@@ -44,6 +44,7 @@ struct ibv_context *hnsdv_open_device(struct ibv_device *device,
+ enum hnsdv_qp_create_flags {
+ 	HNSDV_QP_CREATE_ENABLE_DCA_MODE = 1 << 0,
+ 	HNSDV_QP_CREATE_ENABLE_UD_SL = 1 << 1,
+	HNSDV_QP_CREATE_ENABLE_STARS_MODE = 1 << 2,
+ };
+ 
+ enum hnsdv_qp_congest_ctrl_type {
+@@ -62,6 +63,7 @@ struct hnsdv_qp_init_attr {
+ 	uint64_t comp_mask;	/* Use enum hnsdv_qp_init_attr_mask */
+ 	uint32_t create_flags;	/* Use enum hnsdv_qp_create_flags */
+ 	uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */
+	uint8_t reserved[7];
+ };
+ 
+ struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
+-- 
+2.25.1
+
--- a/0068-Update-kernel-headers-for-supporting-write-with-noti.patch
+++ b/0068-Update-kernel-headers-for-supporting-write-with-noti.patch
@ -0,0 +1,58 @@
+From d13868688d40281f1da602befb19da850b59b725 Mon Sep 17 00:00:00 2001
+From: Chengchang Tang <tangchengchang@huawei.com>
+Date: Mon, 30 Oct 2023 16:59:21 +0800
+Subject: [PATCH 7/8] Update kernel headers for supporting write with notify
+
+driver inclusion
+category: feature
+bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I8C6X4
+
+---------------------------------------------------------------
+
+To commit 7261847876e4 ("RDMA/hns: Support write with notify")
+
+Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
+---
+ kernel-headers/rdma/hns-abi.h | 7 ++++++-
+ 1 file changed, 6 insertions(+), 1 deletion(-)
+
+diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h
+index 5969bde..785c4e1 100644
+--- a/kernel-headers/rdma/hns-abi.h
+++ b/kernel-headers/rdma/hns-abi.h
+@@ -38,6 +38,7 @@
+ 
+ enum hns_roce_create_cq_create_flags {
+ 	HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE = 1 << 0,
+	HNS_ROCE_CREATE_CQ_FLAGS_WRITE_WITH_NOTIFY = 1 << 1,
+ };
+ 
+ struct hns_roce_ib_create_cq {
+@@ -47,12 +48,15 @@ struct hns_roce_ib_create_cq {
+ 	__u32 reserved;
+ 	__aligned_u64 create_flags; /* Use enum hns_roce_create_cq_create_flags */
+ 	__u8 poe_channel;
+-	__u8 rsv[7];
+	__u8 notify_mode;
+	__u16 notify_idx;
+	__u16 rsv[2];
+ };
+ 
+ enum hns_roce_cq_cap_flags {
+ 	HNS_ROCE_CQ_FLAG_RECORD_DB = 1 << 0,
+ 	HNS_ROCE_CQ_FLAG_POE_EN = 1 << 2,
+	HNS_ROCE_CQ_FLAG_NOTIFY_EN = 1 << 3,
+ };
+ 
+ struct hns_roce_ib_create_cq_resp {
+@@ -120,6 +124,7 @@ enum hns_roce_qp_cap_flags {
+ 	HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5,
+ 	HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH = 1 << 6,
+ 	HNS_ROCE_QP_CAP_STARS_SQ_MODE = 1 << 7,
+	HNS_ROCE_QP_CAP_WRITE_WITH_NOTIFY = 1 << 8,
+ };
+ 
+ struct hns_roce_ib_create_qp_resp {
+-- 
+2.25.1
+
--- a/0069-libhns-Support-write-with-notify.patch
+++ b/0069-libhns-Support-write-with-notify.patch
@ -0,0 +1,200 @@
+From 6852a8b49a1af09fc41b7b5432d9487323d0f394 Mon Sep 17 00:00:00 2001
+From: Chengchang Tang <tangchengchang@huawei.com>
+Date: Mon, 30 Oct 2023 16:59:22 +0800
+Subject: [PATCH 8/8] libhns: Support write with notify
+
+driver inclusion
+category: feature
+bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I8C6X4
+
+---------------------------------------------------------------
+
+This patch supports write with notify operation.
+
+To use this operation, you need to configure the CQ to enable write
+with notify and bind the CQ to the corresponding RQ.
+
+The address of notify depends on the notify_idx configured by the
+user.  This index will point to the notify mem array configured by
+the kernel ULP.
+
+Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
+---
+ providers/hns/hns_roce_u_hw_v2.c | 23 +++++++++++++++++++++++
+ providers/hns/hns_roce_u_hw_v2.h |  1 +
+ providers/hns/hns_roce_u_verbs.c | 32 ++++++++++++++++++++++++++++++--
+ providers/hns/hnsdv.h            | 17 ++++++++++++++++-
+ providers/hns/libhns.map         |  1 +
+ 5 files changed, 71 insertions(+), 3 deletions(-)
+
+diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
+index ab6d652..495fbcb 100644
+--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
+@@ -2993,6 +2993,29 @@ static void wr_start(struct ibv_qp_ex *ibv_qp)
+ 	qp->err = 0;
+ }
+ 
+void hnsdv_wr_write_notify(struct ibv_qp_ex *ibv_qp, uint32_t rkey,
+			   uint64_t remote_addr, uint32_t offset, uint8_t value)
+{
+#define NOTIFY_OFFSET_MASK GENMASK(23, 2)
+#define NOTIFY_VAL_SHIFT 24
+	struct hns_roce_rc_sq_wqe *wqe;
+	struct hns_roce_qp *qp;
+
+	if (!ibv_qp)
+		return;
+
+	qp = to_hr_qp(&ibv_qp->qp_base);
+	wqe = init_rc_wqe(qp, ibv_qp->wr_id,
+			  HNS_ROCE_WQE_OP_RDMA_WRITE_WITH_NOTIFY);
+	if (unlikely(!wqe))
+		return;
+
+	wqe->va = htole64(remote_addr);
+	wqe->rkey = htole32(rkey);
+	wqe->immtdata = htole32(((uint32_t)value << NOTIFY_VAL_SHIFT) |
+				(offset & NOTIFY_OFFSET_MASK));
+}
+
+ static int wr_complete(struct ibv_qp_ex *ibv_qp)
+ {
+ 	struct hns_roce_context *ctx = to_hr_ctx(ibv_qp->qp_base.context);
+diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h
+index 50a920f..fa83bbe 100644
+--- a/providers/hns/hns_roce_u_hw_v2.h
+++ b/providers/hns/hns_roce_u_hw_v2.h
+@@ -61,6 +61,7 @@ enum {
+ 	HNS_ROCE_WQE_OP_ATOMIC_MASK_FETCH_AND_ADD = 0x9,
+ 	HNS_ROCE_WQE_OP_FAST_REG_PMR = 0xa,
+ 	HNS_ROCE_WQE_OP_BIND_MW_TYPE = 0xc,
+	HNS_ROCE_WQE_OP_RDMA_WRITE_WITH_NOTIFY = 0x16,
+ 	HNS_ROCE_WQE_OP_MASK = 0x1f
+ };
+ 
+diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
+index 2ad9ea0..5e46f89 100644
+--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
+@@ -476,9 +476,26 @@ enum {
+ };
+ 
+ enum {
+-	HNSDV_CQ_SUP_CREATE_FLAGS = HNSDV_CQ_CREATE_ENABLE_POE_MODE,
+	HNSDV_CQ_SUP_CREATE_FLAGS = HNSDV_CQ_CREATE_ENABLE_POE_MODE |
+				    HNSDV_CQ_CREATE_ENABLE_NOTIFY,
+ };
+ 
+static int verify_notify_attr(struct hns_roce_context *ctx,
+			      struct hnsdv_cq_init_attr *hns_cq_attr)
+{
+	if (!(hns_cq_attr->comp_mask & HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS) ||
+	    !(hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_NOTIFY))
+		return 0;
+
+	if (hns_cq_attr->notify_mode >= HNSDV_CQ_NOTIFY_MODE_GUARD) {
+		verbs_err(&ctx->ibv_ctx, "Invalid notify mode %u\n",
+			  hns_cq_attr->notify_mode);
+		return EINVAL;
+	}
+
+	return 0;
+}
+
+ static int verify_hnsdv_cq_create_attr(struct hns_roce_context *ctx,
+ 				       struct hnsdv_cq_init_attr *hns_cq_attr)
+ {
+@@ -499,7 +516,7 @@ static int verify_hnsdv_cq_create_attr(struct hns_roce_context *ctx,
+ 		return EOPNOTSUPP;
+ 	}
+ 
+-	return 0;
+	return verify_notify_attr(ctx, hns_cq_attr);
+ }
+ 
+ static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr,
+@@ -551,6 +568,12 @@ static void set_hnsdv_cq_attr(struct hns_roce_ib_create_cq *cmd_drv,
+ 		cmd_drv->create_flags |= HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE;
+ 		cmd_drv->poe_channel = hns_cq_attr->poe_channel;
+ 	}
+
+	if (hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_NOTIFY) {
+		cmd_drv->create_flags |= HNS_ROCE_CREATE_CQ_FLAGS_WRITE_WITH_NOTIFY;
+		cmd_drv->notify_mode = hns_cq_attr->notify_mode;
+		cmd_drv->notify_idx = hns_cq_attr->notify_idx;
+	}
+ }
+ 
+ static int check_hnsdv_cq_flags(struct hnsdv_cq_init_attr *hns_cq_attr,
+@@ -562,6 +585,11 @@ static int check_hnsdv_cq_flags(struct hnsdv_cq_init_attr *hns_cq_attr,
+ 	if ((hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_POE_MODE) &&
+ 			!(cq->flags & HNS_ROCE_CQ_FLAG_POE_EN))
+ 		return EOPNOTSUPP;
+
+	if ((hns_cq_attr->create_flags & HNSDV_CQ_CREATE_ENABLE_NOTIFY) &&
+			!(cq->flags & HNS_ROCE_CQ_FLAG_NOTIFY_EN))
+		return EOPNOTSUPP;
+
+ 	return 0;
+ }
+ 
+diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h
+index 98c68fd..792a23e 100644
+--- a/providers/hns/hnsdv.h
+++ b/providers/hns/hnsdv.h
+@@ -96,17 +96,28 @@ int hnsdv_query_device(struct ibv_context *ctx_in,
+ 
+ enum hnsdv_cq_init_attr_mask {
+ 	HNSDV_CQ_INIT_ATTR_MASK_CREATE_FLAGS = 1 << 0,
+	HNSDV_CQ_CREATE_ENABLE_NOTIFY = 1 << 1,
+ };
+ 
+ enum hnsdv_cq_create_flags {
+ 	HNSDV_CQ_CREATE_ENABLE_POE_MODE = 1 << 0,
+ };
+ 
+enum hnsdv_cq_notify_mode {
+	HNSDV_CQ_NOTIFY_MODE_64B_ALIGN = 0,
+	HNSDV_CQ_NOTIFY_MODE_4B_ALIGN = 1,
+	HNSDV_CQ_NOTIFY_MODE_DDR_64B_ALIGN = 2,
+	HNSDV_CQ_NOTIFY_MODE_DDR_4B_ALIGN = 3,
+	HNSDV_CQ_NOTIFY_MODE_GUARD = 4, /* Invalid for user */
+};
+
+ struct hnsdv_cq_init_attr {
+ 	uint64_t comp_mask; /* Use enum hnsdv_cq_init_attr_mask */
+ 	uint64_t create_flags; /* Use enum hnsdv_cq_create_flags */
+ 	uint8_t poe_channel; /* poe channel to use */
+-	uint8_t reserved[7];
+	uint8_t notify_mode;
+	uint16_t notify_idx;
+	uint32_t reserved;
+ };
+ 
+ struct ibv_cq_ex *hnsdv_create_cq_ex(struct ibv_context *context,
+@@ -116,6 +127,10 @@ struct ibv_cq_ex *hnsdv_create_cq_ex(struct ibv_context *context,
+ /* used in stars mode */
+ void hnsdv_update_sq_ci(struct ibv_qp *qp, uint32_t inc_idx);
+ 
+/* write with notify */
+void hnsdv_wr_write_notify(struct ibv_qp_ex *qpex, uint32_t rkey,
+			uint64_t remote_addr, uint32_t offset, uint8_t value);
+
+ #ifdef __cplusplus
+ }
+ #endif
+diff --git a/providers/hns/libhns.map b/providers/hns/libhns.map
+index 27efc83..74d85e1 100644
+--- a/providers/hns/libhns.map
+++ b/providers/hns/libhns.map
+@@ -8,5 +8,6 @@ HNS_1.0 {
+ 		hnsdv_query_device;
+ 		hnsdv_create_cq_ex;
+ 		hnsdv_update_sq_ci;
+		hnsdv_wr_write_notify;
+ 	local: *;
+ };
+-- 
+2.25.1
+
--- a/rdma-core.spec
+++ b/rdma-core.spec
@ -1,6 +1,6 @@
 Name:           rdma-core
 Version:        41.0
-Release:        18
+Release:        19
 Summary:        RDMA core userspace libraries and daemons
 License:        GPLv2 or BSD
 Url:            https://github.com/linux-rdma/rdma-core
@ -67,6 +67,14 @@ patch57: 0058-libhns-Support-flexible-WQE-buffer-page-size.patch
 patch58: 0059-Update-kernel-headers.patch
 patch59: 0060-libhns-Support-SRQ-record-doorbell.patch
 patch60: 0061-libhns-Skip-resolving-MAC-for-RDMA-over-UBLink.patch
+patch61: 0062-Update-kernel-headers-for-libhns-query_device.patch
+patch62: 0063-libhns-Support-query-HW-ID-by-hnsdv_query_device.patch
+patch63: 0064-Update-kernel-headers-for-supporting-POE-CQs.patch
+patch64: 0065-libhns-Add-support-for-POE-CQs.patch
+patch65: 0066-Update-kernel-headers-for-supporting-STARS-QP-in-HNS.patch
+patch66: 0067-libhns-Support-STARS-mode-QP.patch
+patch67: 0068-Update-kernel-headers-for-supporting-write-with-noti.patch
+patch68: 0069-libhns-Support-write-with-notify.patch

 BuildRequires:  binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0)
 BuildRequires:  pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel
@ -314,6 +322,12 @@ fi
 %{_mandir}/*

 %changelog
+* Tue Oct 31 2023 Ran Zhou <zhouran10@h-partners.com> - 41.0-19
+- Type: requirement
+- ID: NA
+- SUG: NA
+- DESC: Support STARS over RDMA
+
 * Thu Oct 26 2023 Juan Zhou <zhoujuan51@h-partners.com> - 41.0-18
 - Type: requirement
 - ID: NA