Support SRQ record doorbell

driver inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I8A08Z

Compared with normal doorbell, using record doorbell can shorten the
process of ringing the doorbell and reduce the latency.

Signed-off-by: Yangyang Li <liyangyang20@huawei.com>
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
Signed-off-by: Ran Zhou <zhouran10@h-partners.com>
(cherry picked from commit 23f6e3ca5e9fa66b41e3d33a9c4a88429cfc61ab)

0059-Update-kernel-headers.patch

@@ -0,0 +1,46 @@
+From 73a5a85a9fd75e2dd461bfd129d263fde44aa5ea Mon Sep 17 00:00:00 2001
+From: Junxian Huang <huangjunxian6@hisilicon.com>
+Date: Mon, 16 Oct 2023 16:10:05 +0800
+Subject: [PATCH 1/2] Update kernel headers
+
+To commit: c9813b0b9992 ("RDMA/hns: Support SRQ record doorbell").
+
+Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
+---
+ kernel-headers/rdma/hns-abi.h | 12 +++++++++++-
+ 1 file changed, 11 insertions(+), 1 deletion(-)
+
+diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h
+index 157dc9d..6b94a89 100644
+--- a/kernel-headers/rdma/hns-abi.h
++++ b/kernel-headers/rdma/hns-abi.h
+@@ -52,15 +52,25 @@ struct hns_roce_ib_create_cq_resp {
+ 	__aligned_u64 cap_flags;
+ };
+ 
++enum hns_roce_srq_cap_flags {
++       HNS_ROCE_SRQ_CAP_RECORD_DB = 1 << 0,
++};
++
++enum hns_roce_srq_cap_flags_resp {
++       HNS_ROCE_RSP_SRQ_CAP_RECORD_DB = 1 << 0,
++};
++
+ struct hns_roce_ib_create_srq {
+ 	__aligned_u64 buf_addr;
+ 	__aligned_u64 db_addr;
+ 	__aligned_u64 que_addr;
++	__u32 req_cap_flags; /* Use enum hns_roce_srq_cap_flags */
++	__u32 reserved;
+ };
+ 
+ struct hns_roce_ib_create_srq_resp {
+ 	__u32	srqn;
+-	__u32	reserved;
++	__u32   cap_flags; /* Use enum hns_roce_srq_cap_flags */
+ };
+ 
+ enum hns_roce_create_qp_comp_mask {
+-- 
+2.25.1
+

0060-libhns-Support-SRQ-record-doorbell.patch

@@ -0,0 +1,150 @@
+From 2880d64c8d73375978d2767c5dd7803b444f9016 Mon Sep 17 00:00:00 2001
+From: Yangyang Li <liyangyang20@huawei.com>
+Date: Mon, 16 Oct 2023 16:10:06 +0800
+Subject: [PATCH] libhns: Support SRQ record doorbell
+
+Compared with normal doorbell, using record doorbell can shorten the
+process of ringing the doorbell and reduce the latency.
+
+During SRQ creation, the kernel driver will allocate doorbell buffer
+and notify userspace whether the SRQ record doorbell is enabled with
+the flag HNS_ROCE_RSP_SRQ_CAP_RECORD_DB. The userspace driver will decide
+whether to use record doorbell or normal doorbell based on this flag
+in post SRQ recv process.
+
+This patch relies on the corresponding kernel patch:
+RDMA/hns: Support SRQ record doorbell
+
+Signed-off-by: Yangyang Li <liyangyang20@huawei.com>
+Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
+---
+ providers/hns/hns_roce_u.h       |  4 +++-
+ providers/hns/hns_roce_u_db.c    |  1 +
+ providers/hns/hns_roce_u_hw_v2.c | 14 +++++++++-----
+ providers/hns/hns_roce_u_verbs.c | 12 +++++++-----
+ 4 files changed, 20 insertions(+), 11 deletions(-)
+
+diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
+index ae9ae51..197bde9 100644
+--- a/providers/hns/hns_roce_u.h
++++ b/providers/hns/hns_roce_u.h
+@@ -187,6 +187,7 @@ struct hns_roce_buf {
+ enum hns_roce_db_type {
+ 	HNS_ROCE_QP_TYPE_DB,
+ 	HNS_ROCE_CQ_TYPE_DB,
++	HNS_ROCE_SRQ_TYPE_DB,
+ 	HNS_ROCE_DB_TYPE_NUM
+ };
+ 
+@@ -351,7 +352,8 @@ struct hns_roce_srq {
+ 	unsigned int			max_gs;
+ 	unsigned int			rsv_sge;
+ 	unsigned int			wqe_shift;
+-	unsigned int			*db;
++	unsigned int			*rdb;
++	unsigned int                    cap_flags;
+ 	unsigned short			counter;
+ 	struct list_node xrc_srcq_node;
+ };
+diff --git a/providers/hns/hns_roce_u_db.c b/providers/hns/hns_roce_u_db.c
+index 73a71de..bbef988 100644
+--- a/providers/hns/hns_roce_u_db.c
++++ b/providers/hns/hns_roce_u_db.c
+@@ -41,6 +41,7 @@
+ static const unsigned int db_size[] = {
+ 	[HNS_ROCE_QP_TYPE_DB] = 4,
+ 	[HNS_ROCE_CQ_TYPE_DB] = 4,
++	[HNS_ROCE_SRQ_TYPE_DB] = 4,
+ };
+ 
+ static struct hns_roce_db_page *hns_roce_add_db_page(
+diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
+index ac40d5d..714a34e 100644
+--- a/providers/hns/hns_roce_u_hw_v2.c
++++ b/providers/hns/hns_roce_u_hw_v2.c
+@@ -2109,11 +2109,15 @@ static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx)
+ 	idx_que->head++;
+ }
+ 
+-static void update_srq_db(struct hns_roce_db *db, struct hns_roce_srq *srq)
++static void update_srq_db(struct hns_roce_context *ctx, struct hns_roce_db *db,
++                         struct hns_roce_srq *srq)
+ {
+ 	hr_reg_write(db, DB_TAG, srq->srqn);
+ 	hr_reg_write(db, DB_CMD, HNS_ROCE_V2_SRQ_DB);
+ 	hr_reg_write(db, DB_PI, srq->idx_que.head);
++
++	hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
++                        (__le32 *)db);
+ }
+ 
+ static int check_srq_recv(struct hns_roce_context *ctx)
+@@ -2176,10 +2180,10 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq,
+ 		 */
+ 		udma_to_device_barrier();
+ 
+-		update_srq_db(&srq_db, srq);
+-
+-		hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
+-				 (__le32 *)&srq_db);
++               if (srq->cap_flags & HNS_ROCE_RSP_SRQ_CAP_RECORD_DB)
++                       *srq->rdb = srq->idx_que.head & 0xffff;
++               else
++                       update_srq_db(ctx, &srq_db, srq);
+ 	}
+ 
+ 	hns_roce_spin_unlock(&srq->hr_lock);
+diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
+index f76341c..1c2d94d 100644
+--- a/providers/hns/hns_roce_u_verbs.c
++++ b/providers/hns/hns_roce_u_verbs.c
+@@ -866,7 +866,8 @@ static int exec_srq_create_cmd(struct ibv_context *context,
+ 
+ 	cmd_ex.buf_addr = (uintptr_t)srq->wqe_buf.buf;
+ 	cmd_ex.que_addr = (uintptr_t)srq->idx_que.buf.buf;
+-	cmd_ex.db_addr = (uintptr_t)srq->db;
++	cmd_ex.db_addr = (uintptr_t)srq->rdb;
++	cmd_ex.req_cap_flags |= HNS_ROCE_SRQ_CAP_RECORD_DB;
+ 
+ 	ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq, init_attr,
+ 				    &cmd_ex.ibv_cmd, sizeof(cmd_ex),
+@@ -875,6 +876,7 @@ static int exec_srq_create_cmd(struct ibv_context *context,
+ 		return ret;
+ 
+ 	srq->srqn = resp_ex.srqn;
++	srq->cap_flags = resp_ex.cap_flags;
+ 
+ 	return 0;
+ }
+@@ -932,8 +934,8 @@ static struct ibv_srq *create_srq(struct ibv_context *context,
+ 	if (alloc_srq_buf(srq))
+ 		goto err_free_srq;
+ 
+-	srq->db = hns_roce_alloc_db(hr_ctx, HNS_ROCE_QP_TYPE_DB);
+-	if (!srq->db)
++	srq->rdb = hns_roce_alloc_db(hr_ctx, HNS_ROCE_SRQ_TYPE_DB);
++	if (!srq->rdb)
+ 		goto err_srq_buf;
+ 
+ 	ret = exec_srq_create_cmd(context, srq, init_attr);
+@@ -956,7 +958,7 @@ err_destroy_srq:
+ 	ibv_cmd_destroy_srq(&srq->verbs_srq.srq);
+ 
+ err_srq_db:
+-	hns_roce_free_db(hr_ctx, srq->db, HNS_ROCE_QP_TYPE_DB);
++	hns_roce_free_db(hr_ctx, srq->rdb, HNS_ROCE_SRQ_TYPE_DB);
+ 
+ err_srq_buf:
+ 	free_srq_buf(srq);
+@@ -1048,7 +1050,7 @@ int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq)
+ 
+ 	hns_roce_clear_srq(ctx, srq->srqn);
+ 
+-	hns_roce_free_db(ctx, srq->db, HNS_ROCE_QP_TYPE_DB);
++	hns_roce_free_db(ctx, srq->rdb, HNS_ROCE_SRQ_TYPE_DB);
+ 	free_srq_buf(srq);
+ 	free(srq);
+ 
+-- 
+2.25.1
+

rdma-core.spec

@@ -1,6 +1,6 @@
 Name:           rdma-core
 Version:        41.0
-Release:        16
+Release:        17
 Summary:        RDMA core userspace libraries and daemons
 License:        GPLv2 or BSD
 Url:            https://github.com/linux-rdma/rdma-core
@@ -64,6 +64,8 @@ Patch54: 0055-libhns-separate-the-initialization-steps-of-lock.patch
 Patch55: 0056-libhns-assign-doorbell-to-zero-when-allocate-it.patch
 patch56: 0057-libhns-Fix-missing-reset-notification.patch
 patch57: 0058-libhns-Support-flexible-WQE-buffer-page-size.patch
+patch58: 0059-Update-kernel-headers.patch
+patch59: 0060-libhns-Support-SRQ-record-doorbell.patch
 
 BuildRequires:  binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0)
 BuildRequires:  pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel
@@ -311,6 +313,12 @@ fi
 %{_mandir}/*
 
 %changelog
+* Wed Oct 25 2023 Ran Zhou <zhouran10@h-partners.com> - 41.0-17
+- Type: requirement
+- ID: NA
+- SUG: NA
+- DESC: Support SRQ record doorbell
+
 * Tue Oct 24 2023 Ran Zhou <zhouran10@h-partners.com> - 41.0-16
 - Type: requirement
 - ID: NA