Support libhns stop sending db mechanism after reset
Add an interface to the user space, which is used to receive the kernel reset state. After receiving the reset flag, the user space stops sending db. Signed-off-by: Yixing Liu <liuyixing1@huawei.com>
This commit is contained in:
Yixing Liu
Chengchang Tang
parent
675fd56ef0
commit
8b56ab8b70
28
0038-Update-kernel-headers.patch
Normal file
28
0038-Update-kernel-headers.patch
Normal file
@ -0,0 +1,28 @@
|
||||
From 8a5429161e6932d4031ec705b695973d67729c71 Mon Sep 17 00:00:00 2001
|
||||
From: Yixing Liu <liuyixing1@huawei.com>
|
||||
Date: Wed, 14 Dec 2022 16:37:26 +0800
|
||||
Subject: [PATCH rdma-core 1/2] Update kernel headers
|
||||
|
||||
To commit ?? ("RDMA/hns: Kernel notify usr space to stop ring db").
|
||||
|
||||
Signed-off-by: Yixing Liu <liuyixing1@huawei.com>
|
||||
Reviewed-by: Yangyang Li <liyangyang20@huawei.com>
|
||||
---
|
||||
kernel-headers/rdma/hns-abi.h | 1 +
|
||||
1 file changed, 1 insertion(+)
|
||||
|
||||
diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h
|
||||
index 6950841..5988a62 100644
|
||||
--- a/kernel-headers/rdma/hns-abi.h
|
||||
+++ b/kernel-headers/rdma/hns-abi.h
|
||||
@@ -127,6 +127,7 @@ struct hns_roce_ib_alloc_ucontext_resp {
|
||||
__u32 dca_qps;
|
||||
__u32 dca_mmap_size;
|
||||
__aligned_u64 dca_mmap_key;
|
||||
+ __aligned_u64 reset_mmap_key;
|
||||
};
|
||||
|
||||
enum hns_roce_uctx_comp_mask {
|
||||
--
|
||||
2.30.0
|
||||
|
||||
195
0039-libhns-Add-reset-stop-flow-mechanism.patch
Normal file
195
0039-libhns-Add-reset-stop-flow-mechanism.patch
Normal file
@ -0,0 +1,195 @@
|
||||
From c3ee7375c80c7a8f0a943679566c87f17f87aa17 Mon Sep 17 00:00:00 2001
|
||||
From: Guofeng Yue <yueguofeng@hisilicon.com>
|
||||
Date: Mon, 9 May 2022 16:03:38 +0800
|
||||
Subject: [PATCH rdma-core 2/2] libhns: Add reset stop flow mechanism
|
||||
|
||||
driver inclusion
|
||||
category: bugfix
|
||||
bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I65WI7
|
||||
|
||||
------------------------------------------------------------------
|
||||
|
||||
Add an interface to the user space, which is used to receive
|
||||
the kernel reset state. After receiving the reset flag, the
|
||||
user space stops sending db.
|
||||
|
||||
Signed-off-by: Yixing Liu <liuyixing1@huawei.com>
|
||||
Signed-off-by: Guofeng Yue <yueguofeng@hisilicon.com>
|
||||
Reviewed-by: Yangyang Li <liyangyang20@huawei.com>
|
||||
---
|
||||
providers/hns/hns_roce_u.c | 30 ++++++++++++++++++++++++++++--
|
||||
providers/hns/hns_roce_u.h | 5 +++++
|
||||
providers/hns/hns_roce_u_db.h | 8 +++++++-
|
||||
providers/hns/hns_roce_u_hw_v2.c | 19 ++++++++++++++-----
|
||||
4 files changed, 54 insertions(+), 8 deletions(-)
|
||||
|
||||
diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
|
||||
index 0cf6d4b..3d29838 100644
|
||||
--- a/providers/hns/hns_roce_u.c
|
||||
+++ b/providers/hns/hns_roce_u.c
|
||||
@@ -221,6 +221,24 @@ static void uninit_dca_context(struct hns_roce_context *ctx)
|
||||
pthread_spin_destroy(&dca_ctx->lock);
|
||||
}
|
||||
|
||||
+static int init_reset_context(struct hns_roce_context *ctx, int cmd_fd,
|
||||
+ struct hns_roce_alloc_ucontext_resp *resp,
|
||||
+ int page_size)
|
||||
+{
|
||||
+ uint64_t reset_mmap_key = resp->reset_mmap_key;
|
||||
+
|
||||
+ /* The reset mmap key is 0, which means it is not supported. */
|
||||
+ if (reset_mmap_key == 0)
|
||||
+ return 0;
|
||||
+
|
||||
+ ctx->reset_state = mmap(NULL, page_size, PROT_READ, MAP_SHARED,
|
||||
+ cmd_fd, reset_mmap_key);
|
||||
+ if (ctx->reset_state == MAP_FAILED)
|
||||
+ return -ENOMEM;
|
||||
+
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
static int hns_roce_mmap(struct hns_roce_device *hr_dev,
|
||||
struct hns_roce_context *context, int cmd_fd)
|
||||
{
|
||||
@@ -325,8 +343,11 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
|
||||
&resp, ctx_attr, hr_dev->page_size))
|
||||
goto err_free;
|
||||
|
||||
+ if (init_reset_context(context, cmd_fd, &resp, hr_dev->page_size))
|
||||
+ goto reset_free;
|
||||
+
|
||||
if (hns_roce_mmap(hr_dev, context, cmd_fd))
|
||||
- goto dca_free;
|
||||
+ goto uar_free;
|
||||
|
||||
pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE);
|
||||
|
||||
@@ -335,7 +356,10 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
|
||||
|
||||
return &context->ibv_ctx;
|
||||
|
||||
-dca_free:
|
||||
+uar_free:
|
||||
+ if (context->reset_state)
|
||||
+ munmap(context->reset_state, hr_dev->page_size);
|
||||
+reset_free:
|
||||
uninit_dca_context(context);
|
||||
err_free:
|
||||
verbs_uninit_context(&context->ibv_ctx);
|
||||
@@ -349,6 +373,8 @@ static void hns_roce_free_context(struct ibv_context *ibctx)
|
||||
struct hns_roce_context *context = to_hr_ctx(ibctx);
|
||||
|
||||
munmap(context->uar, hr_dev->page_size);
|
||||
+ if (context->reset_state)
|
||||
+ munmap(context->reset_state, hr_dev->page_size);
|
||||
uninit_dca_context(context);
|
||||
verbs_uninit_context(&context->ibv_ctx);
|
||||
free(context);
|
||||
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
|
||||
index 71c35c5..76c7adb 100644
|
||||
--- a/providers/hns/hns_roce_u.h
|
||||
+++ b/providers/hns/hns_roce_u.h
|
||||
@@ -226,9 +226,14 @@ struct hns_roce_dca_ctx {
|
||||
atomic_bitmap_t *sync_status;
|
||||
};
|
||||
|
||||
+struct hns_roce_v2_reset_state {
|
||||
+ uint32_t is_reset;
|
||||
+};
|
||||
+
|
||||
struct hns_roce_context {
|
||||
struct verbs_context ibv_ctx;
|
||||
void *uar;
|
||||
+ void *reset_state;
|
||||
pthread_spinlock_t uar_lock;
|
||||
|
||||
struct {
|
||||
diff --git a/providers/hns/hns_roce_u_db.h b/providers/hns/hns_roce_u_db.h
|
||||
index 8c47a53..de288de 100644
|
||||
--- a/providers/hns/hns_roce_u_db.h
|
||||
+++ b/providers/hns/hns_roce_u_db.h
|
||||
@@ -40,8 +40,14 @@
|
||||
|
||||
#define HNS_ROCE_WORD_NUM 2
|
||||
|
||||
-static inline void hns_roce_write64(void *dest, __le32 val[HNS_ROCE_WORD_NUM])
|
||||
+static inline void hns_roce_write64(struct hns_roce_context *ctx, void *dest,
|
||||
+ __le32 val[HNS_ROCE_WORD_NUM])
|
||||
{
|
||||
+ struct hns_roce_v2_reset_state *state = ctx->reset_state;
|
||||
+
|
||||
+ if (state && state->is_reset)
|
||||
+ return;
|
||||
+
|
||||
mmio_write64_le(dest, *(__le64 *)val);
|
||||
}
|
||||
|
||||
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
|
||||
index 7661863..d0067d3 100644
|
||||
--- a/providers/hns/hns_roce_u_hw_v2.c
|
||||
+++ b/providers/hns/hns_roce_u_hw_v2.c
|
||||
@@ -298,7 +298,8 @@ static void hns_roce_update_rq_db(struct hns_roce_context *ctx,
|
||||
hr_reg_write(&rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB);
|
||||
hr_reg_write(&rq_db, DB_PI, rq_head);
|
||||
|
||||
- hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&rq_db);
|
||||
+ hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
|
||||
+ (__le32 *)&rq_db);
|
||||
}
|
||||
|
||||
static void hns_roce_update_sq_db(struct hns_roce_context *ctx,
|
||||
@@ -312,7 +313,7 @@ static void hns_roce_update_sq_db(struct hns_roce_context *ctx,
|
||||
hr_reg_write(&sq_db, DB_PI, qp->sq.head);
|
||||
hr_reg_write(&sq_db, DB_SL, qp->sl);
|
||||
|
||||
- hns_roce_write64(qp->sq.db_reg, (__le32 *)&sq_db);
|
||||
+ hns_roce_write64(ctx, qp->sq.db_reg, (__le32 *)&sq_db);
|
||||
}
|
||||
|
||||
static void hns_roce_write512(uint64_t *dest, uint64_t *val)
|
||||
@@ -323,6 +324,12 @@ static void hns_roce_write512(uint64_t *dest, uint64_t *val)
|
||||
static void hns_roce_write_dwqe(struct hns_roce_qp *qp, void *wqe)
|
||||
{
|
||||
struct hns_roce_rc_sq_wqe *rc_sq_wqe = wqe;
|
||||
+ struct ibv_qp *ibvqp = &qp->verbs_qp.qp;
|
||||
+ struct hns_roce_context *ctx = to_hr_ctx(ibvqp->context);
|
||||
+ struct hns_roce_v2_reset_state *state = ctx->reset_state;
|
||||
+
|
||||
+ if (state && state->is_reset)
|
||||
+ return;
|
||||
|
||||
/* All kinds of DirectWQE have the same header field layout */
|
||||
hr_reg_enable(rc_sq_wqe, RCWQE_FLAG);
|
||||
@@ -342,7 +349,8 @@ static void update_cq_db(struct hns_roce_context *ctx, struct hns_roce_cq *cq)
|
||||
hr_reg_write(&cq_db, DB_CQ_CI, cq->cons_index);
|
||||
hr_reg_write(&cq_db, DB_CQ_CMD_SN, 1);
|
||||
|
||||
- hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&cq_db);
|
||||
+ hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
|
||||
+ (__le32 *)&cq_db);
|
||||
}
|
||||
|
||||
static struct hns_roce_qp *hns_roce_v2_find_qp(struct hns_roce_context *ctx,
|
||||
@@ -857,7 +865,8 @@ static int hns_roce_u_v2_arm_cq(struct ibv_cq *ibvcq, int solicited)
|
||||
hr_reg_write(&cq_db, DB_CQ_CMD_SN, cq->arm_sn);
|
||||
hr_reg_write(&cq_db, DB_CQ_NOTIFY, solicited_flag);
|
||||
|
||||
- hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&cq_db);
|
||||
+ hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
|
||||
+ (__le32 *)&cq_db);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1934,7 +1943,7 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq,
|
||||
|
||||
update_srq_db(&srq_db, srq);
|
||||
|
||||
- hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
|
||||
+ hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET,
|
||||
(__le32 *)&srq_db);
|
||||
}
|
||||
|
||||
--
|
||||
2.30.0
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Name: rdma-core
|
||||
Version: 41.0
|
||||
Release: 7
|
||||
Release: 8
|
||||
Summary: RDMA core userspace libraries and daemons
|
||||
License: GPLv2 or BSD
|
||||
Url: https://github.com/linux-rdma/rdma-core
|
||||
@ -43,6 +43,8 @@ Patch33: 0034-libhns-Add-support-for-attaching-QP-s-WQE-buffer.patch
|
||||
Patch34: 0035-libhns-Use-shared-memory-to-sync-DCA-status.patch
|
||||
Patch35: 0036-libhns-Sync-DCA-status-by-shared-memory.patch
|
||||
Patch36: 0037-libhns-Add-direct-verbs-support-to-config-DCA.patch
|
||||
Patch37: 0038-Update-kernel-headers.patch
|
||||
Patch38: 0039-libhns-Add-reset-stop-flow-mechanism.patch
|
||||
|
||||
BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0)
|
||||
BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel
|
||||
@ -290,6 +292,12 @@ fi
|
||||
%{_mandir}/*
|
||||
|
||||
%changelog
|
||||
* Wed Dec 14 2022 Yixing Liu <liuyixing1@huawei.com> - 41.0-6
|
||||
- Type: requirement
|
||||
- ID: NA
|
||||
- SUG: NA
|
||||
- DESC: Support libhns reset stop ring db mechanism
|
||||
|
||||
* Wed Nov 30 2022 tangchengchang <tangchengchang@huawei.com> - 41.0-7
|
||||
- Type: requirement
|
||||
- ID: NA
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user