packages/perftest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Perftest: Add support for TD lock-free mode

Browse Source 
Add support for TD lock-free mode
New option:  --no_lock

        Usage example:
        ib_send_bw -d hns_0 --no_lock
        ib_send_bw -d hns_0 --no_lock 192.168.100.100

Signed-off-by: Xinghai Cen <cenxinghai@h-partners.com>
(cherry picked from commit 621caab096092f45ac8e5b5d74a16467e4f8df4c)
This commit is contained in:
Xinghai Cen 2024-08-21 19:25:44 +08:00 committed by openeuler-sync-bot
parent 3201f7db0e
commit e6d00e04be
3 changed files with 307 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

266
0012-Perftest-Add-support-for-TD-lock-free-mode.patch Normal file
View File

@ -0,0 +1,266 @@
From 641ab294a6bc344dffd042019c609d4e9839a2f5 Mon Sep 17 00:00:00 2001
From: Guofeng Yue <yueguofeng@h-partners.com>
Date: Tue, 13 Aug 2024 21:48:44 +0800
Subject: [PATCH] Perftest: Add support for TD lock-free mode
Add support for TD lock-free mode
New option: --no_lock
Usage example:
ib_send_bw -d hns_0 --no_lock
ib_send_bw -d hns_0 --no_lock 192.168.100.100
Signed-off-by: Guofeng Yue <yueguofeng@h-partners.com>
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
---
configure.ac | 14 +++++++
src/perftest_parameters.c | 30 ++++++++++++++-
src/perftest_parameters.h | 1 +
src/perftest_resources.c | 81 +++++++++++++++++++++++++++++++++++++++
src/perftest_resources.h | 4 ++
5 files changed, 128 insertions(+), 2 deletions(-)
diff --git a/configure.ac b/configure.ac
index 54fc2cc..d976663 100755
--- a/configure.ac
+++ b/configure.ac
@@ -298,6 +298,20 @@ if [test $HAVE_HNSDV = yes]; then
AC_SUBST([LIBHNS])
fi
+AC_TRY_LINK([#include <infiniband/verbs.h>],
+ [ibv_cq_ex_to_cq], [HAVE_CQ_EX=yes], [HAVE_CQ_EX=no])
+AM_CONDITIONAL([HAVE_CQ_EX], [test "x$HAVE_CQ_EX" = "xyes"])
+if [test $HAVE_CQ_EX = yes]; then
+ AC_DEFINE([HAVE_CQ_EX], [1], [Have CQ EX API support])
+fi
+
+AC_TRY_LINK([#include <infiniband/verbs.h>],
+ [ibv_alloc_td], [HAVE_TD_API=yes], [HAVE_TD_API=no])
+AM_CONDITIONAL([HAVE_TD_API], [test "x$HAVE_TD_API" = "xyes"])
+if [test $HAVE_TD_API = yes]; then
+ AC_DEFINE([HAVE_TD_API], [1], [Have TD API support])
+fi
+
CFLAGS="-g -Wall -D_GNU_SOURCE -O3 $CFLAGS"
LIBS=$LIBS" -lpthread"
AC_SUBST([LIBUMAD])
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 16af503..4c6ba4f 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -464,6 +464,11 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
printf(" Use a Shared Receive Queue. --rx-depth controls max-wr size of the SRQ \n");
}
+ #ifdef HAVE_TD_API
+ printf(" --no_lock ");
+ printf(" No lock in IO, including post send, post recv, post srq recv and poll cq \n");
+ #endif
+
if (connection_type != RawEth) {
printf(" --ipv6 ");
printf(" Use IPv6 GID. Default is IPv4\n");
@@ -2225,6 +2230,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
#ifdef HAVE_HNSDV
static int congest_type_flag = 0;
#endif
+ #ifdef HAVE_TD_API
+ static int no_lock_flag = 0;
+ #endif
char *server_ip = NULL;
char *client_ip = NULL;
@@ -2300,6 +2308,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
{ .name = "run_infinitely", .has_arg = 0, .flag = &run_inf_flag, .val = 1 },
{ .name = "report_gbits", .has_arg = 0, .flag = &report_fmt_flag, .val = 1},
{ .name = "use-srq", .has_arg = 0, .flag = &srq_flag, .val = 1},
+ #ifdef HAVE_TD_API
+ { .name = "no_lock", .has_arg = 0, .flag = &no_lock_flag, .val = 1},
+ #endif
{ .name = "report-both", .has_arg = 0, .flag = &report_both_flag, .val = 1},
{ .name = "reversed", .has_arg = 0, .flag = &is_reversed_flag, .val = 1},
{ .name = "pkey_index", .has_arg = 1, .flag = &pkey_flag, .val = 1},
@@ -2944,6 +2955,12 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
user_param->use_srq = 1;
}
+ #ifdef HAVE_TD_API
+ if (no_lock_flag) {
+ user_param->no_lock = 1;
+ }
+ #endif
+
if (report_fmt_flag) {
user_param->report_fmt = GBS;
}
@@ -3284,15 +3301,24 @@ void ctx_print_test_info(struct perftest_parameters *user_param)
printf(" Number of qps : %d\t\tTransport type : %s\n", user_param->num_of_qps, transport_str(user_param->transport_type));
printf(" Connection type : %s\t\tUsing SRQ : %s\n", connStr[user_param->connection_type], user_param->use_srq ? "ON" : "OFF");
#ifdef HAVE_RO
- printf(" PCIe relax order: %s\n", user_param->disable_pcir ? "OFF" : "ON");
+ #ifdef HAVE_TD_API
+ printf(" PCIe relax order: %s\t\tLock-free : %s\n", user_param->disable_pcir ? "OFF" : "ON", user_param->no_lock ? "ON" : "OFF");
+ #else
+ printf(" PCIe relax order: %s\t\tLock-free : %s\n", user_param->disable_pcir ? "OFF" : "ON", "Unsupported");
+ #endif //HAVE_TD_API
if ((check_pcie_relaxed_ordering_compliant() == false) &&
(user_param->disable_pcir == 0)) {
printf(" WARNING: CPU is not PCIe relaxed ordering compliant.\n");
printf(" WARNING: You should disable PCIe RO with `--disable_pcie_relaxed` for both server and client.\n");
}
#else
- printf(" PCIe relax order: %s\n", "Unsupported");
+ #ifdef HAVE_TD_API
+ printf(" PCIe relax order: %s\t\tLock-free : %s\n", "Unsupported", user_param->no_lock ? "ON" : "OFF");
+ #else
+ printf(" PCIe relax order: %s\t\tLock-free : %s\n", "Unsupported", "Unsupported");
+ #endif //HAVE_TD_API
#endif
+
printf(" ibv_wr* API : %s\n", user_param->use_old_post_send ? "OFF" : "ON");
if (user_param->machine == CLIENT || user_param->duplex) {
printf(" TX depth : %d\n",user_param->tx_depth);
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
index ffc19ab..f49fb0f 100755
--- a/src/perftest_parameters.h
+++ b/src/perftest_parameters.h
@@ -462,6 +462,7 @@ struct perftest_parameters {
int recv_post_list;
int duration;
int use_srq;
+ int no_lock;
int congest_type;
int use_xrc;
int use_rss;
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 451f11d..0e2b9a3 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1360,6 +1360,20 @@ int destroy_ctx(struct pingpong_context *ctx,
}
#endif
+ #ifdef HAVE_TD_API
+ if (user_param->no_lock) {
+ if (ibv_dealloc_pd(ctx->pad)) {
+ fprintf(stderr, "Failed to deallocate PAD - %s\n", strerror(errno));
+ test_result = 1;
+ }
+
+ if (ibv_dealloc_td(ctx->td)) {
+ fprintf(stderr, "Failed to deallocate TD - %s\n", strerror(errno));
+ test_result = 1;
+ }
+ }
+ #endif
+
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "Failed to deallocate PD - %s\n", strerror(errno));
test_result = 1;
@@ -1502,6 +1516,47 @@ int create_reg_cqs(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
int tx_buffer_depth, int need_recv_cq)
{
+ #ifdef HAVE_CQ_EX
+ struct ibv_cq_init_attr_ex send_cq_attr = {
+ .cqe = tx_buffer_depth * user_param->num_of_qps,
+ .cq_context = NULL,
+ .channel = ctx->channel,
+ .comp_vector = user_param->eq_num,
+ };
+
+ #ifdef HAVE_TD_API
+ if (user_param->no_lock) {
+ send_cq_attr.parent_domain = ctx->pad;
+ send_cq_attr.comp_mask = IBV_CQ_INIT_ATTR_MASK_PD;
+ }
+ #endif
+ ctx->send_cq = ibv_cq_ex_to_cq(ibv_create_cq_ex(ctx->context, &send_cq_attr));
+ if (!ctx->send_cq) {
+ fprintf(stderr, "Couldn't create CQ\n");
+ return FAILURE;
+ }
+
+ if (need_recv_cq) {
+ struct ibv_cq_init_attr_ex recv_cq_attr = {
+ .cqe = user_param->rx_depth * user_param->num_of_qps,
+ .cq_context = NULL,
+ .channel = ctx->channel,
+ .comp_vector = user_param->eq_num,
+ };
+ #ifdef HAVE_TD_API
+ if (user_param->no_lock) {
+ recv_cq_attr.parent_domain = ctx->pad;
+ recv_cq_attr.comp_mask = IBV_CQ_INIT_ATTR_MASK_PD;
+ }
+ #endif
+ ctx->recv_cq = ibv_cq_ex_to_cq(ibv_create_cq_ex(ctx->context, &recv_cq_attr));
+ if (!ctx->recv_cq) {
+ fprintf(stderr, "Couldn't create a receiver CQ\n");
+ return FAILURE;
+ }
+ }
+ return SUCCESS;
+#else
ctx->send_cq = ibv_create_cq(ctx->context,tx_buffer_depth *
user_param->num_of_qps, NULL, ctx->channel, user_param->eq_num);
if (!ctx->send_cq) {
@@ -1519,6 +1574,7 @@ int create_reg_cqs(struct pingpong_context *ctx,
}
return SUCCESS;
+#endif
}
/******************************************************************************
@@ -1906,6 +1962,31 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
fprintf(stderr, "Couldn't allocate PD\n");
return FAILURE;
}
+
+ #ifdef HAVE_TD_API
+ /* Allocating the Thread domain, Parent domain. */
+ if (user_param->no_lock) {
+ struct ibv_td_init_attr td_attr = {0};
+ ctx->td = ibv_alloc_td(ctx->context, &td_attr);
+ if (!ctx->td) {
+ fprintf(stderr, "Couldn't allocate TD\n");
+ return FAILURE;
+ }
+
+ struct ibv_parent_domain_init_attr pad_attr = {
+ .pd = ctx->pd,
+ .td = ctx->td,
+ .comp_mask = 0,
+ };
+
+ ctx->pad = ibv_alloc_parent_domain(ctx->context, &pad_attr);
+ if (!ctx->pad) {
+ fprintf(stderr, "Couldn't allocate PAD\n");
+ return FAILURE;
+ }
+ }
+ #endif
+
#ifdef HAVE_AES_XTS
if(user_param->aes_xts){
struct mlx5dv_dek_init_attr dek_attr = {};
diff --git a/src/perftest_resources.h b/src/perftest_resources.h
index cf0502c..ba8630b 100755
--- a/src/perftest_resources.h
+++ b/src/perftest_resources.h
@@ -170,6 +170,10 @@ struct pingpong_context {
#endif
struct ibv_comp_channel *channel;
struct ibv_pd *pd;
+ #ifdef HAVE_TD_API
+ struct ibv_td *td;
+ struct ibv_pd *pad;
+ #endif
struct ibv_mr **mr;
struct ibv_cq *send_cq;
struct ibv_cq *recv_cq;
--
2.25.1

32
0013-Perftest-Fix-TD-lock-free-mode-not-working-for-QP.patch Normal file
View File

@ -0,0 +1,32 @@
From f8f97b189e9af6cff8559b10c749ff289219b754 Mon Sep 17 00:00:00 2001
From: Xinghai Cen <cenxinghai@h-partners.com>
Date: Wed, 21 Aug 2024 20:06:50 +0800
Subject: [PATCH] Perftest: Fix TD lock-free mode not working for QP When
creating QP in TD lock-free mode, set attr_ex.pd with ctx->pad instead of
ctx->pd, otherwise the lock-free won't work.
Fixes: "Perftest: Add support for TD lock-free mode"
Signed-off-by: Guofeng Yue <yueguofeng@h-partners.com>
---
src/perftest_resources.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 0e2b9a3..1bdf34e 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -2309,7 +2309,11 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
else if (opcode == IBV_WR_RDMA_READ)
attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
}
+ #ifdef HAVE_TD_API
+ attr_ex.pd = user_param->no_lock ? ctx->pad : ctx->pd;
+ #else
attr_ex.pd = ctx->pd;
+ #endif
attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD;
attr_ex.send_cq = attr.send_cq;
attr_ex.recv_cq = attr.recv_cq;
--
2.25.1

10
perftest.spec
View File

@ -1,6 +1,6 @@
Name: perftest
Version: 4.5
Release: 9
Release: 10
License: GPLv2 or BSD
Summary: RDMA Performance Testing Tools
Url: https://github.com/linux-rdma/perftest
@ -17,6 +17,8 @@ Patch8: 0008-Get-CPU-MHz-on-RISC-V.patch
Patch9: 0009-Get-CPU-cycles-on-RISC-V.patch
Patch10: 0010-Perftest-Support-selecting-congestion-control-algori.patch
Patch11: 0011-Perftest-Fix-rx_depth-check-for-XRC.patch
Patch12: 0012-Perftest-Add-support-for-TD-lock-free-mode.patch
Patch13: 0013-Perftest-Fix-TD-lock-free-mode-not-working-for-QP.patch
BuildRequires: automake gcc libibverbs-devel >= 1.2.0 librdmacm-devel >= 1.0.21 libibumad-devel >= 1.3.10.2
BuildRequires: pciutils-devel libibverbs librdmacm libibumad
@ -44,6 +46,12 @@ done
%_bindir/*
%changelog
* Wed Aug 21 2024 Xinghai Cen <cenxinghai@h-partners.com> - 4.5-10
- Type: feature
- ID: NA
- SUG: NA
- DESC: Add support for TD lock-free mode
* Mon Jul 22 2024 liweigang <liweiganga@uniontech.com> - 4.5-9
- Type: bugfix
- ID: NA