From 357ca1208050ee9a12b81fa0ea11f556e2970fab Mon Sep 17 00:00:00 2001 From: yinbin6 Date: Fri, 7 Jun 2024 17:23:48 +0800 Subject: [PATCH] make-rpc_msg_max-recv_ring_size-configurable (cherry picked from commit 6fb4a1e35bbe0b316cbce7480c02a79f8f15e73b) --- ..._msg_max-recv_ring_size-configurable.patch | 169 ++++++++++++++++++ ...mp-while-event-count-exceed-maxevent.patch | 46 +++++ gazelle.spec | 9 +- 3 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 0183-make-rpc_msg_max-recv_ring_size-configurable.patch create mode 100644 0184-EPOLL-fix-coredump-while-event-count-exceed-maxevent.patch diff --git a/0183-make-rpc_msg_max-recv_ring_size-configurable.patch b/0183-make-rpc_msg_max-recv_ring_size-configurable.patch new file mode 100644 index 0000000..ae786ca --- /dev/null +++ b/0183-make-rpc_msg_max-recv_ring_size-configurable.patch @@ -0,0 +1,169 @@ +From 55ed3c6aaccf320a7d3240753a5aabe400ac4bd3 Mon Sep 17 00:00:00 2001 +From: yinbin6 +Date: Fri, 7 Jun 2024 17:06:50 +0800 +Subject: [PATCH] make rpc_msg_max recv_ring_size-configurable + + +diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c +index 9efdbaa..0e5fbf3 100644 +--- a/src/lstack/core/lstack_cfg.c ++++ b/src/lstack/core/lstack_cfg.c +@@ -68,6 +68,7 @@ static int32_t parse_nic_read_number(void); + static int32_t parse_tcp_conn_count(void); + static int32_t parse_mbuf_count_per_conn(void); + static int32_t parse_send_ring_size(void); ++static int32_t parse_recv_ring_size(void); + static int32_t parse_num_process(void); + static int32_t parse_process_numa(void); + static int32_t parse_process_index(void); +@@ -83,6 +84,7 @@ static int32_t parse_nic_txqueue_size(void); + static int32_t parse_stack_thread_mode(void); + static int32_t parse_nic_vlan_mode(void); + static int32_t parse_defaule_nonblock_mode(void); ++static int32_t parse_rpc_msg_max(void); + + #define PARSE_ARG(_arg, _arg_string, _default_val, _min_val, _max_val, _ret) \ + do { \ +@@ -132,6 +134,7 @@ static struct config_vector_t g_config_tbl[] = { + { "rpc_number", parse_rpc_number }, + { "nic_read_number", parse_nic_read_number }, + { "send_ring_size", parse_send_ring_size }, ++ { "recv_ring_size", parse_recv_ring_size }, + { "num_process", parse_num_process }, + { "process_numa", parse_process_numa }, + { "process_idx", parse_process_index }, +@@ -146,6 +149,7 @@ static struct config_vector_t g_config_tbl[] = { + { "stack_thread_mode", parse_stack_thread_mode }, + { "nic_vlan_mode", parse_nic_vlan_mode }, + { "nonblock_mode", parse_defaule_nonblock_mode }, ++ { "rpc_msg_max", parse_rpc_msg_max }, + { NULL, NULL } + }; + +@@ -908,6 +912,14 @@ static int32_t parse_send_ring_size(void) + return ret; + } + ++static int32_t parse_recv_ring_size(void) ++{ ++ int32_t ret; ++ /* recv ring size default value is 128 */ ++ PARSE_ARG(g_config_params.recv_ring_size, "recv_ring_size", 128, 1, SOCK_RECV_RING_SIZE_MAX, ret); ++ return ret; ++} ++ + static int32_t parse_mbuf_count_per_conn(void) + { + int32_t ret; +@@ -1356,3 +1368,15 @@ static int32_t parse_defaule_nonblock_mode(void) + } + return ret; + } ++ ++static int32_t parse_rpc_msg_max(void) ++{ ++ int32_t ret; ++ PARSE_ARG(g_config_params.rpc_msg_max, "rpc_msg_max", 4096, 1, 8192, ret); ++ if (ret != 0) { ++ LSTACK_PRE_LOG(LSTACK_ERR, "cfg: invalid rpc msg max value %d ret=%d. only support 1~8192\n", ++ g_config_params.rpc_msg_max, ret); ++ } ++ return ret; ++} ++ +diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c +index 30bd827..b98ba84 100644 +--- a/src/lstack/core/lstack_thread_rpc.c ++++ b/src/lstack/core/lstack_thread_rpc.c +@@ -13,6 +13,7 @@ + #include + + #include "lstack_log.h" ++#include "lstack_cfg.h" + #include "lstack_dpdk.h" + #include "lstack_rpc_proc.h" + #include "lstack_stack_stat.h" +@@ -71,7 +72,7 @@ static struct rpc_msg *rpc_msg_alloc(rpc_msg_func func) + exit(-1); + } + +- g_rpc_pool->mempool = create_mempool("rpc_pool", RPC_MSG_MAX, sizeof(struct rpc_msg), ++ g_rpc_pool->mempool = create_mempool("rpc_pool", get_global_cfg_params()->rpc_msg_max, sizeof(struct rpc_msg), + 0, rte_gettid()); + if (g_rpc_pool->mempool == NULL) { + LSTACK_LOG(INFO, LSTACK, "rpc_pool create failed, errno is %d\n", errno); +diff --git a/src/lstack/include/lstack_cfg.h b/src/lstack/include/lstack_cfg.h +index a00e47a..94878de 100644 +--- a/src/lstack/include/lstack_cfg.h ++++ b/src/lstack/include/lstack_cfg.h +@@ -110,6 +110,7 @@ struct cfg_params { + struct secondary_attach_arg sec_attach_arg; + char unix_socket_filename[NAME_MAX]; + uint16_t send_ring_size; ++ uint16_t recv_ring_size; + bool tuple_filter; + int8_t bond_mode; + int32_t bond_miimon; +@@ -119,6 +120,7 @@ struct cfg_params { + struct cfg_nic_params nic; + bool stack_mode_rtc; + bool nonblock_mode; ++ uint32_t rpc_msg_max; + }; + + struct cfg_params *get_global_cfg_params(void); +diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h +index 93fa40c..c210ab9 100644 +--- a/src/lstack/include/lstack_protocol_stack.h ++++ b/src/lstack/include/lstack_protocol_stack.h +@@ -25,8 +25,9 @@ + #include "lstack_ethdev.h" + #include "gazelle_opt.h" + +-#define SOCK_RECV_RING_SIZE (128) ++#define SOCK_RECV_RING_SIZE (get_global_cfg_params()->recv_ring_size) + #define SOCK_RECV_FREE_THRES (32) ++#define SOCK_RECV_RING_SIZE_MAX (2048) + #define SOCK_SEND_RING_SIZE_MAX (2048) + #define SOCK_SEND_REPLENISH_THRES (16) + #define WAKEUP_MAX_NUM (32) +diff --git a/src/lstack/include/lstack_thread_rpc.h b/src/lstack/include/lstack_thread_rpc.h +index 8e97c11..0c51848 100644 +--- a/src/lstack/include/lstack_thread_rpc.h ++++ b/src/lstack/include/lstack_thread_rpc.h +@@ -26,8 +26,6 @@ + #define MSG_ARG_4 (4) + #define RPM_MSG_ARG_SIZE (5) + +-#define RPC_MSG_MAX 4096 +-#define RPC_MSG_MASK (RPC_MSG_MAX - 1) + typedef struct lockless_queue rpc_queue; + + struct rpc_stats { +diff --git a/src/lstack/lstack.conf b/src/lstack/lstack.conf +index a7f4e75..4c3784f 100644 +--- a/src/lstack/lstack.conf ++++ b/src/lstack/lstack.conf +@@ -22,8 +22,12 @@ tcp_conn_count = 1500 + mbuf_count_per_conn = 170 + + # send ring size, default is 32, max is 2048 ++# if udp pktlen exceeds 45952(32 * 1436)B, send_ring_size must be at least 64. + send_ring_size = 32 + ++#recv ring size, default is 128, max is 2048 ++recv_ring_size = 128 ++ + #protocol stack thread per loop params + #read data form protocol stack into recv_ring + read_connect_number = 4 +@@ -67,3 +71,6 @@ nic_vlan_mode=-1 + bond_mode=-1 + #bond slave mac, separated by ; , only support 2 slave mac + #bond_slave_mac="aa:bb:cc:dd:ee:ff;gg:hh:ii:jj:kk:ll" ++ ++#maximum number of rpc memory pools ++rpc_msg_max=4096 +-- +2.33.0 + diff --git a/0184-EPOLL-fix-coredump-while-event-count-exceed-maxevent.patch b/0184-EPOLL-fix-coredump-while-event-count-exceed-maxevent.patch new file mode 100644 index 0000000..b80b273 --- /dev/null +++ b/0184-EPOLL-fix-coredump-while-event-count-exceed-maxevent.patch @@ -0,0 +1,46 @@ +From e3c12e9d73678b3c5445f770c929e035ef1c4997 Mon Sep 17 00:00:00 2001 +From: yinbin6 +Date: Fri, 7 Jun 2024 11:13:08 +0800 +Subject: [PATCH] EPOLL: fix coredump while event count exceed maxevents + +--- + src/lstack/api/lstack_epoll.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/src/lstack/api/lstack_epoll.c b/src/lstack/api/lstack_epoll.c +index c8a2e43..28a3dd9 100644 +--- a/src/lstack/api/lstack_epoll.c ++++ b/src/lstack/api/lstack_epoll.c +@@ -488,7 +488,14 @@ int32_t epoll_lwip_event_nolock(struct wakeup_poll *wakeup, struct epoll_event * + list_del_node_null(node); + continue; + } +- ++ ++ if (event_num >= maxevents) { ++ /* move list head after the current node, and start traversing from this node next time */ ++ list_del_node_null(&wakeup->event_list); ++ list_add_node(node, &wakeup->event_list); ++ break; ++ } ++ + events[event_num].events = sock->events & sock->epoll_events; + events[event_num].data = sock->ep_data; + event_num++; +@@ -504,13 +511,6 @@ int32_t epoll_lwip_event_nolock(struct wakeup_poll *wakeup, struct epoll_event * + list_del_node_null(node); + sock->epoll_events = 0; + } +- +- if (event_num >= maxevents) { +- /* move list head after the current node, and start traversing from this node next time */ +- list_del_node_null(&wakeup->event_list); +- list_add_node(node, &wakeup->event_list); +- break; +- } + } + + return event_num; +-- +2.33.0 + diff --git a/gazelle.spec b/gazelle.spec index c92c448..ae80c96 100644 --- a/gazelle.spec +++ b/gazelle.spec @@ -2,7 +2,7 @@ Name: gazelle Version: 1.0.2 -Release: 39 +Release: 40 Summary: gazelle is a high performance user-mode stack License: MulanPSL-2.0 URL: https://gitee.com/openeuler/gazelle @@ -199,7 +199,8 @@ Patch9179: 0179-dfx-fix-gazellectl-x-for-bond.patch Patch9180: 0180-change-gazelle_stat_lstack_proto-from-u16-to-u64.patch Patch9181: 0181-memary-error-fix-some-memary-error.patch Patch9182: 0182-bond-remove-bond-initialization-code-in-dpdk_ethdev_.patch - +Patch9183: 0183-make-rpc_msg_max-recv_ring_size-configurable.patch +Patch9184: 0184-EPOLL-fix-coredump-while-event-count-exceed-maxevent.patch %description %{name} is a high performance user-mode stack. @@ -240,6 +241,10 @@ install -Dpm 0640 %{_builddir}/%{name}-%{version}/src/ltran/ltran.conf %{b %config(noreplace) %{conf_path}/ltran.conf %changelog +* Fri June 7 2024 yinbin6 - 1.0.2-40 +- make rpc_msg_max recv_ring_size configurable +- EPOLL: fix coredump while eventcount exceed maxevent + * Fri May 31 2024 yinbin6 - 1.0.2-39 - bond:remove bond initialization code in dpdk_ethdev_init