packages/rdma-core
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!93 [sync] PR-80: Support SRQ record doorbell

Browse Source 
From: @openeuler-sync-bot 
Reviewed-by: @hellotcc 
Signed-off-by: @hellotcc
This commit is contained in:
openeuler-ci-bot 2023-11-03 03:31:54 +00:00 committed by Gitee
commit a0dcb93164
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
3 changed files with 205 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
0059-Update-kernel-headers.patch Normal file
View File

@ -0,0 +1,46 @@
From 73a5a85a9fd75e2dd461bfd129d263fde44aa5ea Mon Sep 17 00:00:00 2001
From: Junxian Huang <huangjunxian6@hisilicon.com>
Date: Mon, 16 Oct 2023 16:10:05 +0800
Subject: [PATCH 1/2] Update kernel headers
To commit: c9813b0b9992 ("RDMA/hns: Support SRQ record doorbell").
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
---
kernel-headers/rdma/hns-abi.h | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h
index 157dc9d..6b94a89 100644
--- a/kernel-headers/rdma/hns-abi.h
+++ b/kernel-headers/rdma/hns-abi.h
@@ -52,15 +52,25 @@ struct hns_roce_ib_create_cq_resp {
__aligned_u64 cap_flags;
};
+enum hns_roce_srq_cap_flags {
+ HNS_ROCE_SRQ_CAP_RECORD_DB = 1 << 0,
+};
+
+enum hns_roce_srq_cap_flags_resp {
+ HNS_ROCE_RSP_SRQ_CAP_RECORD_DB = 1 << 0,
+};
+
struct hns_roce_ib_create_srq {
__aligned_u64 buf_addr;
__aligned_u64 db_addr;
__aligned_u64 que_addr;
+ __u32 req_cap_flags; /* Use enum hns_roce_srq_cap_flags */
+ __u32 reserved;
};
struct hns_roce_ib_create_srq_resp {
__u32 srqn;
- __u32 reserved;
+ __u32 cap_flags; /* Use enum hns_roce_srq_cap_flags */
};
enum hns_roce_create_qp_comp_mask {
--
2.25.1

150
0060-libhns-Support-SRQ-record-doorbell.patch Normal file
View File

@ -0,0 +1,150 @@
From 2880d64c8d73375978d2767c5dd7803b444f9016 Mon Sep 17 00:00:00 2001
From: Yangyang Li <liyangyang20@huawei.com>
Date: Mon, 16 Oct 2023 16:10:06 +0800
Subject: [PATCH] libhns: Support SRQ record doorbell
Compared with normal doorbell, using record doorbell can shorten the
process of ringing the doorbell and reduce the latency.
During SRQ creation, the kernel driver will allocate doorbell buffer
and notify userspace whether the SRQ record doorbell is enabled with
the flag HNS_ROCE_RSP_SRQ_CAP_RECORD_DB. The userspace driver will decide
whether to use record doorbell or normal doorbell based on this flag
in post SRQ recv process.
This patch relies on the corresponding kernel patch:
RDMA/hns: Support SRQ record doorbell
Signed-off-by: Yangyang Li <liyangyang20@huawei.com>
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
---
providers/hns/hns_roce_u.h | 4 +++-
providers/hns/hns_roce_u_db.c | 1 +
providers/hns/hns_roce_u_hw_v2.c | 14 +++++++++-----
providers/hns/hns_roce_u_verbs.c | 12 +++++++-----
4 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index ae9ae51..197bde9 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -187,6 +187,7 @@ struct hns_roce_buf {
enum hns_roce_db_type {
HNS_ROCE_QP_TYPE_DB,
HNS_ROCE_CQ_TYPE_DB,
+ HNS_ROCE_SRQ_TYPE_DB,
HNS_ROCE_DB_TYPE_NUM
};
@@ -351,7 +352,8 @@ struct hns_roce_srq {
unsigned int max_gs;
unsigned int rsv_sge;
unsigned int wqe_shift;
- unsigned int *db;
+ unsigned int *rdb;
+ unsigned int cap_flags;
unsigned short counter;
struct list_node xrc_srcq_node;
};
diff --git a/providers/hns/hns_roce_u_db.c b/providers/hns/hns_roce_u_db.c
index 73a71de..bbef988 100644
--- a/providers/hns/hns_roce_u_db.c
+++ b/providers/hns/hns_roce_u_db.c
@@ -41,6 +41,7 @@
static const unsigned int db_size[] = {
[HNS_ROCE_QP_TYPE_DB] = 4,
[HNS_ROCE_CQ_TYPE_DB] = 4,
+ [HNS_ROCE_SRQ_TYPE_DB] = 4,
};
static struct hns_roce_db_page *hns_roce_add_db_page(
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index ac40d5d..714a34e 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -2109,11 +2109,15 @@ static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx)
idx_que->head++;
}
-static void update_srq_db(struct hns_roce_db *db, struct hns_roce_srq *srq)
+static void update_srq_db(struct hns_roce_context *ctx, struct hns_roce_db *db,
+ struct hns_roce_srq *srq)
{
hr_reg_write(db, DB_TAG, srq->srqn);
hr_reg_write(db, DB_CMD, HNS_ROCE_V2_SRQ_DB);
hr_reg_write(db, DB_PI, srq->idx_que.head);
+
+ hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
+ (__le32 *)db);
}
static int check_srq_recv(struct hns_roce_context *ctx)
@@ -2176,10 +2180,10 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq,
*/
udma_to_device_barrier();
- update_srq_db(&srq_db, srq);
-
- hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
- (__le32 *)&srq_db);
+ if (srq->cap_flags & HNS_ROCE_RSP_SRQ_CAP_RECORD_DB)
+ *srq->rdb = srq->idx_que.head & 0xffff;
+ else
+ update_srq_db(ctx, &srq_db, srq);
}
hns_roce_spin_unlock(&srq->hr_lock);
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index f76341c..1c2d94d 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -866,7 +866,8 @@ static int exec_srq_create_cmd(struct ibv_context *context,
cmd_ex.buf_addr = (uintptr_t)srq->wqe_buf.buf;
cmd_ex.que_addr = (uintptr_t)srq->idx_que.buf.buf;
- cmd_ex.db_addr = (uintptr_t)srq->db;
+ cmd_ex.db_addr = (uintptr_t)srq->rdb;
+ cmd_ex.req_cap_flags |= HNS_ROCE_SRQ_CAP_RECORD_DB;
ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq, init_attr,
&cmd_ex.ibv_cmd, sizeof(cmd_ex),
@@ -875,6 +876,7 @@ static int exec_srq_create_cmd(struct ibv_context *context,
return ret;
srq->srqn = resp_ex.srqn;
+ srq->cap_flags = resp_ex.cap_flags;
return 0;
}
@@ -932,8 +934,8 @@ static struct ibv_srq *create_srq(struct ibv_context *context,
if (alloc_srq_buf(srq))
goto err_free_srq;
- srq->db = hns_roce_alloc_db(hr_ctx, HNS_ROCE_QP_TYPE_DB);
- if (!srq->db)
+ srq->rdb = hns_roce_alloc_db(hr_ctx, HNS_ROCE_SRQ_TYPE_DB);
+ if (!srq->rdb)
goto err_srq_buf;
ret = exec_srq_create_cmd(context, srq, init_attr);
@@ -956,7 +958,7 @@ err_destroy_srq:
ibv_cmd_destroy_srq(&srq->verbs_srq.srq);
err_srq_db:
- hns_roce_free_db(hr_ctx, srq->db, HNS_ROCE_QP_TYPE_DB);
+ hns_roce_free_db(hr_ctx, srq->rdb, HNS_ROCE_SRQ_TYPE_DB);
err_srq_buf:
free_srq_buf(srq);
@@ -1048,7 +1050,7 @@ int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq)
hns_roce_clear_srq(ctx, srq->srqn);
- hns_roce_free_db(ctx, srq->db, HNS_ROCE_QP_TYPE_DB);
+ hns_roce_free_db(ctx, srq->rdb, HNS_ROCE_SRQ_TYPE_DB);
free_srq_buf(srq);
free(srq);
--
2.25.1

10
rdma-core.spec
View File

@ -1,6 +1,6 @@
Name: rdma-core
Version: 41.0
Release: 16
Release: 17
Summary: RDMA core userspace libraries and daemons
License: GPLv2 or BSD
Url: https://github.com/linux-rdma/rdma-core
@ -64,6 +64,8 @@ Patch54: 0055-libhns-separate-the-initialization-steps-of-lock.patch
Patch55: 0056-libhns-assign-doorbell-to-zero-when-allocate-it.patch
patch56: 0057-libhns-Fix-missing-reset-notification.patch
patch57: 0058-libhns-Support-flexible-WQE-buffer-page-size.patch
patch58: 0059-Update-kernel-headers.patch
patch59: 0060-libhns-Support-SRQ-record-doorbell.patch
BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0)
BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel
@ -311,6 +313,12 @@ fi
%{_mandir}/*
%changelog
* Wed Oct 25 2023 Ran Zhou <zhouran10@h-partners.com> - 41.0-17
- Type: requirement
- ID: NA
- SUG: NA
- DESC: Support SRQ record doorbell
* Tue Oct 24 2023 Ran Zhou <zhouran10@h-partners.com> - 41.0-16
- Type: requirement
- ID: NA