packages/perftest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add hns roce to device list to support new io

Browse Source 
Devices which are not in the device list does not support new io
(ibv_wr_xx).

Add patch to support the ibv_wr_xx() API test for hns roce.

These patches also set a correct max inline size for hns.

Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
This commit is contained in:
Chengchang Tang 2022-11-08 20:20:39 +08:00 committed by Chengchang Tang
parent a302b9786f
commit df00d8ddda
6 changed files with 367 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
0001-Perftest-Fix-verification-of-max_inline_data-for-_cr.patch Normal file
View File

@ -0,0 +1,75 @@
From 950e7e2da1b50e5aab20b2efb450890df1ef0ed1 Mon Sep 17 00:00:00 2001
From: Chengchang Tang <tangchengchang@huawei.com>
Date: Mon, 7 Nov 2022 21:09:08 +0800
Subject: Perftest: Fix verification of max_inline_data for *_create_qp_ex()
Currently, attr.cap.max_inline_data is used for validation in
*_create_qp() and *_create_qp_ex(). But actually, when entering
the create_qp_ex path, the variable attr is not used. So the current
check of the *_create_qp_ex() branch is meaningless.
The attr_ex.cap.max_inline_data is used to check the max_inline_data
in *_create_qp_ex() path. And related printing error has also been
fixed.
Fixes: 13f71777e6f0 ("Added new post_send API usage for RC,UC,UD,XRC")
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
---
src/perftest_resources.c | 24 ++++++++++++++----------
1 file changed, 14 insertions(+), 10 deletions(-)
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index b6f1cb8..5065181 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -2364,22 +2364,21 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
int dc_num_of_qps = user_param->num_of_qps / 2;
int is_dc_server_side = 0;
+ struct ibv_qp_init_attr attr;
+ memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
+ struct ibv_qp_cap *qp_cap = &attr.cap;
+
#ifdef HAVE_IBV_WR_API
enum ibv_wr_opcode opcode;
- struct ibv_qp_init_attr attr;
struct ibv_qp_init_attr_ex attr_ex;
+ memset(&attr_ex, 0, sizeof(struct ibv_qp_init_attr_ex));
#ifdef HAVE_MLX5DV
struct mlx5dv_qp_init_attr attr_dv;
memset(&attr_dv, 0, sizeof(attr_dv));
#endif
- memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
- memset(&attr_ex, 0, sizeof(struct ibv_qp_init_attr_ex));
#ifdef HAVE_SRD
struct efadv_qp_init_attr efa_attr = {};
#endif
- #else
- struct ibv_qp_init_attr attr;
- memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
#endif
attr.send_cq = ctx->send_cq;
@@ -2554,10 +2553,15 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
fprintf(stderr, "Current TX depth is %d and inline size is %d .\n", user_param->tx_depth, user_param->inline_size);
}
- if (user_param->inline_size > attr.cap.max_inline_data) {
- user_param->inline_size = attr.cap.max_inline_data;
- printf(" Actual inline-size(%d) > requested inline-size(%d)\n",
- attr.cap.max_inline_data, user_param->inline_size);
+ #ifdef HAVE_IBV_WR_API
+ if (!user_param->use_old_post_send)
+ qp_cap = &attr_ex.cap;
+ #endif
+
+ if (user_param->inline_size > qp_cap->max_inline_data) {
+ printf(" Actual inline-size(%d) < requested inline-size(%d)\n",
+ qp_cap->max_inline_data, user_param->inline_size);
+ user_param->inline_size = qp_cap->max_inline_data;
}
return qp;
--
2.30.0

72
0002-Perftest-Add-support-for-HNS.patch Normal file
View File

@ -0,0 +1,72 @@
From da0dc664ee473f6e35abfadcf121fd4e4081ee9b Mon Sep 17 00:00:00 2001
From: Chengchang Tang <tangchengchang@huawei.com>
Date: Fri, 16 Sep 2022 11:28:29 +0800
Subject: Perftest: Add support for HNS
Add support for HNS device by making it recognized by perftest.
Make the perftest allow testing new post send method for hns roce.
And a suitable default inline data size is applied.
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
---
src/perftest_parameters.c | 9 +++++++++
src/perftest_parameters.h | 1 +
src/perftest_resources.c | 3 ++-
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 5fd9f7d..73a8df5 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -1851,6 +1851,13 @@ enum ctx_device ib_dev_name(struct ibv_context *context)
case 55300 : dev_fname = NETXTREME; break;
case 61344 : dev_fname = EFA; break;
case 61345 : dev_fname = EFA; break;
+ case 41506 : dev_fname = HNS; break;
+ case 41507 : dev_fname = HNS; break;
+ case 41508 : dev_fname = HNS; break;
+ case 41509 : dev_fname = HNS; break;
+ case 41510 : dev_fname = HNS; break;
+ case 41512 : dev_fname = HNS; break;
+ case 41519 : dev_fname = HNS; break;
default : dev_fname = UNKNOWN;
}
}
@@ -2054,6 +2061,8 @@ static void ctx_set_max_inline(struct ibv_context *context,struct perftest_param
user_param->inline_size = 32;
else if (current_dev == QLOGIC_E4)
user_param->inline_size = 128;
+ else if (current_dev == HNS)
+ user_param->inline_size = 32;
} else {
user_param->inline_size = 0;
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
index bad587c..717a154 100755
--- a/src/perftest_parameters.h
+++ b/src/perftest_parameters.h
@@ -380,6 +380,7 @@ enum ctx_device {
CONNECTX7 = 26,
QLOGIC_AHP = 27,
BLUEFIELD3 = 28,
+ HNS = 29,
};
/* Units for rate limiter */
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 5065181..dc19cc3 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1979,7 +1979,8 @@ int verify_params_with_device_context(struct ibv_context *context,
current_dev != BLUEFIELD &&
current_dev != BLUEFIELD2 &&
current_dev != BLUEFIELD3 &&
- current_dev != EFA)
+ current_dev != EFA &&
+ current_dev != HNS)
{
if (!user_param->use_old_post_send)
{
--
2.30.0

29
0003-Perftest-Add-new-HNS-roce-device-ROH-to-support-new_.patch Normal file
View File

@ -0,0 +1,29 @@
From 1c7d57d794609887f3084408630814c8f02fad1f Mon Sep 17 00:00:00 2001
From: Chengchang Tang <tangchengchang@huawei.com>
Date: Mon, 7 Nov 2022 16:09:42 +0800
Subject: Perftest: Add new HNS roce device (ROH) to support new_io
Add ROH device id.
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
---
src/perftest_parameters.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 73a8df5..a048a5a 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -1858,6 +1858,9 @@ enum ctx_device ib_dev_name(struct ibv_context *context)
case 41510 : dev_fname = HNS; break;
case 41512 : dev_fname = HNS; break;
case 41519 : dev_fname = HNS; break;
+ case 41511 : dev_fname = HNS; break;
+ case 41516 : dev_fname = HNS; break;
+ case 41517 : dev_fname = HNS; break;
default : dev_fname = UNKNOWN;
}
}
--
2.30.0

33
0004-Perftest-Increase-max-inline-size-to-support-larger-.patch Normal file
View File

@ -0,0 +1,33 @@
From 18152fd789f6be3eaf82684206a3ce86149180fe Mon Sep 17 00:00:00 2001
From: Chengchang Tang <tangchengchang@huawei.com>
Date: Tue, 8 Nov 2022 11:48:35 +0800
Subject: Perftest: Increase max inline size to support larger inline tests
For some hns roce devices, a maximum of 1024 bytes of inline is
supported.
So, increate the MAX_INLINE and MAX_INLINE_UD to support related test.
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
---
src/perftest_parameters.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
index 717a154..25be929 100755
--- a/src/perftest_parameters.h
+++ b/src/perftest_parameters.h
@@ -178,8 +178,8 @@
#define UC_MAX_RX (16000)
#define MIN_CQ_MOD (1)
#define MAX_CQ_MOD (1024)
-#define MAX_INLINE (912)
-#define MAX_INLINE_UD (884)
+#define MAX_INLINE (1024)
+#define MAX_INLINE_UD (1024)
#define MIN_EQ_NUM (0)
#define MAX_EQ_NUM (2048)
--
2.30.0

144
0005-Perftest-replace-rand-with-getrandom-during-MR-buffe.patch Normal file
View File

@ -0,0 +1,144 @@
From cfe0f71c2382f95e809f7edefa052e7ac9a41f8e Mon Sep 17 00:00:00 2001
From: Chengchang Tang <tangchengchang@huawei.com>
Date: Thu, 28 Apr 2022 15:59:42 +0800
Subject: Perftest: replace rand() with getrandom() during MR buffer
initialization
rand() has very poor performance in some OS.
ib_send_bw will spend a lot of time during MR initialization when
testing large packects in above scenario.
test has been done:
"""
int main(int argc, char *argv[])
{
char *a = malloc(HUGE_MR_SIZE * sizeof(char));
unsigned int i;
char *tmp = a;
int ret;
srand(time(NULL));
if (a == NULL)
exit(1);
if (argc <= 1)
goto fall_back;
for (i = HUGE_MR_SIZE; i > 0;) {
ret = getrandom(tmp, i, 0);
if (ret < 0)
goto fall_back;
tmp += ret;
i -= ret;
}
goto out;
fall_back:
for(i = 0; i < HUGE_MR_SIZE; i++)
a[i] = (char)rand();
out:
free(a);
return 0;
}
time ./a.out
real 5m35.033s
user 5m33.546s
sys 0m0.918s
time ./a.out 1
real 0m6.454s
user 0m0.000s
sys 0m6.449s
"""
As shown in the test above, getrandom() has a much better performance,
so replace rand() with it.
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
---
configure.ac | 1 +
src/perftest_resources.c | 31 ++++++++++++++++++++++++++-----
2 files changed, 27 insertions(+), 5 deletions(-)
diff --git a/configure.ac b/configure.ac
index 0c310f9..007957b 100755
--- a/configure.ac
+++ b/configure.ac
@@ -60,6 +60,7 @@ AC_PROG_LIBTOOL
AC_PROG_RANLIB
AC_HEADER_STDC
AC_CHECK_HEADERS([infiniband/verbs.h],,[AC_MSG_ERROR([ibverbs header files not found])])
+AC_CHECK_HEADERS([sys/random.h],,[AC_MSG_ERROR([getrandom available])])
AC_CHECK_LIB([ibverbs], [ibv_get_device_list], [], [AC_MSG_ERROR([libibverbs not found])])
AC_CHECK_LIB([rdmacm], [rdma_create_event_channel], [], AC_MSG_ERROR([librdmacm-devel not found]))
AC_CHECK_LIB([ibumad], [umad_init], [LIBUMAD=-libumad], AC_MSG_ERROR([libibumad not found]))
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index dc19cc3..334bede 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -22,6 +22,9 @@
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
+#ifdef HAVE_SYS_RANDOM_H
+#include <sys/random.h>
+#endif
#ifdef HAVE_SRD
#include <infiniband/efadv.h>
#endif
@@ -1716,12 +1719,33 @@ int create_cqs(struct pingpong_context *ctx, struct perftest_parameters *user_pa
return ret;
}
+static void random_data(char *buf, int buff_size)
+{
+ int i;
+#ifdef HAVE_SYS_RANDOM_H
+ char *tmp = buf;
+ int ret;
+
+ for(i = buff_size; i > 0;) {
+ ret = getrandom(tmp, i, 0);
+ if(ret < 0)
+ goto fall_back;
+ tmp += ret;
+ i -= ret;
+ }
+ return;
+fall_back:
+#endif
+ srand(time(NULL));
+ for (i = 0; i < buff_size; i++)
+ buf[i] = (char)rand();
+}
+
/******************************************************************************
*
******************************************************************************/
int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *user_param, int qp_index)
{
- int i;
int flags = IBV_ACCESS_LOCAL_WRITE;
@@ -1860,13 +1884,10 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
#ifdef HAVE_CUDA
if (!user_param->use_cuda) {
#endif
- srand(time(NULL));
if (user_param->verb == WRITE && user_param->tst == LAT) {
memset(ctx->buf[qp_index], 0, ctx->buff_size);
} else {
- for (i = 0; i < ctx->buff_size; i++) {
- ((char*)ctx->buf[qp_index])[i] = (char)rand();
- }
+ random_data(ctx->buf[qp_index], ctx->buff_size);
}
#ifdef HAVE_CUDA
}
--
2.30.0

16
perftest.spec
View File

@ -1,12 +1,18 @@
Name: perftest
Version: 4.5
Release: 1
Release: 2
License: GPLv2 or BSD
Summary: RDMA Performance Testing Tools
Url: https://github.com/linux-rdma/perftest
Source: https://github.com/linux-rdma/perftest/releases/download/v4.5-0.12/perftest-4.5-0.12.ge93c538.tar.gz
BuildRequires: gcc libibverbs-devel >= 1.2.0 librdmacm-devel >= 1.0.21 libibumad-devel >= 1.3.10.2
Patch1: 0001-Perftest-Fix-verification-of-max_inline_data-for-_cr.patch
Patch2: 0002-Perftest-Add-support-for-HNS.patch
Patch3: 0003-Perftest-Add-new-HNS-roce-device-ROH-to-support-new_.patch
Patch4: 0004-Perftest-Increase-max-inline-size-to-support-larger-.patch
Patch5: 0005-Perftest-replace-rand-with-getrandom-during-MR-buffe.patch
BuildRequires: automake gcc libibverbs-devel >= 1.2.0 librdmacm-devel >= 1.0.21 libibumad-devel >= 1.3.10.2
BuildRequires: pciutils-devel
Obsoletes: openib-perftest < 1.3
@ -30,6 +36,12 @@ done
%_bindir/*
%changelog
* Mon Nov 07 2022 tangchengchang <tangchengchang@huawei.com> - 4.5-2
- Type: requirement
- ID: NA
- SUG: NA
- DESC: Add hns support and fixes some bug in perftest
* Tue Jan 18 2022 SimpleUpdate Robot <tc@openeuler.org> - 4.5-1
- Upgrade to version 4.5