packages/gazelle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sync epoll: distinguish add/del_sock_event and add/del_sock_event_nolock

Browse Source
This commit is contained in:
hankangkang 2023-10-30 15:25:11 +08:00
parent 1bd8aa78c5
commit 55cc6675f1
3 changed files with 350 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
0016-cfg-nic-rx-tx-queue-size-configure.patch Normal file
View File

@ -0,0 +1,140 @@
From 4adbb49fe002f097683c3c2155387d6fe7d59952 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Mon, 9 Oct 2023 14:21:41 +0800
Subject: [PATCH] cfg: nic rx/tx queue size configure
---
src/common/gazelle_opt.h | 4 ++--
src/lstack/core/lstack_cfg.c | 20 ++++++++++++++++++++
src/lstack/core/lstack_dpdk.c | 4 ++--
src/lstack/include/lstack_cfg.h | 6 ++++++
src/lstack/include/lstack_protocol_stack.h | 2 +-
src/lstack/lstack.conf | 3 +++
6 files changed, 34 insertions(+), 5 deletions(-)
diff --git a/src/common/gazelle_opt.h b/src/common/gazelle_opt.h
index 7b6aa19..7b855f9 100644
--- a/src/common/gazelle_opt.h
+++ b/src/common/gazelle_opt.h
@@ -42,8 +42,8 @@
#define VDEV_TX_QUEUE_SZ DEFAULT_RING_SIZE
#define FREE_RX_QUEUE_SZ DPDK_PKT_BURST_SIZE
-#define RTE_TEST_TX_DESC_DEFAULT 2048
-#define RTE_TEST_RX_DESC_DEFAULT 4096
+#define NIC_QUEUE_SIZE_MAX 8192
+#define NIC_QUEUE_SIZE_MIN 512
#define TCP_CONN_COUNT 1500
#define MBUF_COUNT_PER_CONN 170
diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c
index 9fc16dd..0eca86e 100644
--- a/src/lstack/core/lstack_cfg.c
+++ b/src/lstack/core/lstack_cfg.c
@@ -74,6 +74,8 @@ static int32_t parse_use_bond4(void);
static int32_t parse_bond4_slave_mac(void);
static int32_t parse_use_sockmap(void);
static int32_t parse_udp_enable(void);
+static int32_t parse_nic_rxqueue_size(void);
+static int32_t parse_nic_txqueue_size(void);
#define PARSE_ARG(_arg, _arg_string, _default_val, _min_val, _max_val, _ret) \
do { \
@@ -131,6 +133,8 @@ static struct config_vector_t g_config_tbl[] = {
{ "bond4_slave_mac", parse_bond4_slave_mac },
{ "use_sockmap", parse_use_sockmap },
{ "udp_enable", parse_udp_enable },
+ { "nic_rxqueue_size", parse_nic_rxqueue_size},
+ { "nic_txqueue_size", parse_nic_txqueue_size},
{ NULL, NULL }
};
@@ -1163,3 +1167,19 @@ static int32_t parse_use_sockmap(void)
PARSE_ARG(g_config_params.use_sockmap, "use_sockmap", 0, 0, 1, ret);
return ret;
}
+
+static int32_t parse_nic_rxqueue_size(void)
+{
+ int32_t ret;
+ PARSE_ARG(g_config_params.nic.rxqueue_size, "nic_rxqueue_size", 4096,
+ NIC_QUEUE_SIZE_MIN, NIC_QUEUE_SIZE_MAX, ret);
+ return ret;
+}
+
+static int32_t parse_nic_txqueue_size(void)
+{
+ int32_t ret;
+ PARSE_ARG(g_config_params.nic.txqueue_size, "nic_txqueue_size", 2048,
+ NIC_QUEUE_SIZE_MIN, NIC_QUEUE_SIZE_MAX, ret);
+ return ret;
+}
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index 902dadc..a53e85c 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -370,8 +370,8 @@ static struct eth_params *alloc_eth_params(uint16_t port_id, uint16_t nb_queues)
eth_params->port_id = port_id;
eth_params->nb_queues = nb_queues;
- eth_params->nb_rx_desc = RTE_TEST_RX_DESC_DEFAULT;
- eth_params->nb_tx_desc = RTE_TEST_TX_DESC_DEFAULT;
+ eth_params->nb_rx_desc = get_global_cfg_params()->nic.rxqueue_size;
+ eth_params->nb_tx_desc = get_global_cfg_params()->nic.txqueue_size;
eth_params->conf.link_speeds = ETH_LINK_SPEED_AUTONEG;
eth_params->conf.txmode.mq_mode = ETH_MQ_TX_NONE;
eth_params->conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
diff --git a/src/lstack/include/lstack_cfg.h b/src/lstack/include/lstack_cfg.h
index 52a1abd..e48a501 100644
--- a/src/lstack/include/lstack_cfg.h
+++ b/src/lstack/include/lstack_cfg.h
@@ -59,6 +59,11 @@ struct secondary_attach_arg {
char file_prefix[PATH_MAX];
};
+struct cfg_nic_params {
+ uint32_t rxqueue_size;
+ uint32_t txqueue_size;
+};
+
struct cfg_params {
ip4_addr_t host_addr;
ip4_addr_t netmask;
@@ -110,6 +115,7 @@ struct cfg_params {
uint8_t bond4_slave2_mac_addr[ETHER_ADDR_LEN];
bool use_sockmap;
bool udp_enable;
+ struct cfg_nic_params nic;
};
struct cfg_params *get_global_cfg_params(void);
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index b7c18d3..a4f6ac2 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -69,7 +69,7 @@ struct protocol_stack {
uint32_t rx_ring_used;
uint32_t tx_ring_used;
- struct rte_mbuf *pkts[RTE_TEST_RX_DESC_DEFAULT];
+ struct rte_mbuf *pkts[NIC_QUEUE_SIZE_MAX];
struct list_node recv_list;
struct list_node same_node_recv_list; /* used for same node processes communication */
struct list_node wakeup_list;
diff --git a/src/lstack/lstack.conf b/src/lstack/lstack.conf
index fa7e284..ad574e1 100644
--- a/src/lstack/lstack.conf
+++ b/src/lstack/lstack.conf
@@ -34,6 +34,9 @@ rpc_number = 4
#read nic pkts number
nic_read_number = 128
+nic_rxqueue_size = 4096
+nic_txqueue_size = 2048
+
#each cpu core start a protocol stack thread.
num_cpus="2"
--
2.27.0

203
0017-epoll-distinguish-add-del_sock_event-and-add-del_soc.patch Normal file
View File

@ -0,0 +1,203 @@
From 506a252ff94be857ab89b1c1d4aea0ef66ed9582 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Mon, 9 Oct 2023 10:36:50 +0800
Subject: [PATCH] epoll: distinguish add/del_sock_event and
add/del_sock_event_nolock
---
src/lstack/api/lstack_epoll.c | 59 +++++++++++++++++--------
src/lstack/core/lstack_lwip.c | 47 ++++----------------
src/lstack/include/posix/lstack_epoll.h | 6 +++
3 files changed, 56 insertions(+), 56 deletions(-)
diff --git a/src/lstack/api/lstack_epoll.c b/src/lstack/api/lstack_epoll.c
index 55ca4fe..7c40792 100644
--- a/src/lstack/api/lstack_epoll.c
+++ b/src/lstack/api/lstack_epoll.c
@@ -46,37 +46,60 @@ static void update_epoll_max_stack(struct wakeup_poll *wakeup);
static void change_epollfd_kernel_thread(struct wakeup_poll *wakeup, struct protocol_stack *old_stack,
struct protocol_stack *new_stack);
+static inline void add_wakeup_to_stack_wakeuplist(struct wakeup_poll *wakeup, struct protocol_stack *stack)
+{
+ if (list_is_null(&wakeup->wakeup_list[stack->stack_idx])) {
+ list_add_node(&stack->wakeup_list, &wakeup->wakeup_list[stack->stack_idx]);
+ }
+}
+
+void add_sock_event_nolock(struct lwip_sock *sock, uint32_t event)
+{
+ struct wakeup_poll *wakeup = sock->wakeup;
+
+ if (wakeup == NULL || wakeup->type == WAKEUP_CLOSE || (event & sock->epoll_events) == 0) {
+ return;
+ }
+ sock->events |= (event == EPOLLERR) ? (EPOLLIN | EPOLLERR) : (event & sock->epoll_events);
+ if (list_is_null(&sock->event_list)) {
+ list_add_node(&wakeup->event_list, &sock->event_list);
+ }
+ return;
+}
+
void add_sock_event(struct lwip_sock *sock, uint32_t event)
{
struct wakeup_poll *wakeup = sock->wakeup;
+ struct protocol_stack *stack = sock->stack;
if (wakeup == NULL || wakeup->type == WAKEUP_CLOSE || (event & sock->epoll_events) == 0) {
return;
}
if (wakeup->type == WAKEUP_EPOLL) {
pthread_spin_lock(&wakeup->event_list_lock);
-
- /* app thread have read/write, event is outdated */
- if (event == EPOLLIN && sock->conn->state != NETCONN_LISTEN && !NETCONN_IS_DATAIN(sock)) {
- pthread_spin_unlock(&wakeup->event_list_lock);
- return;
- }
- if (event == EPOLLOUT && !NETCONN_IS_OUTIDLE(sock)) {
- pthread_spin_unlock(&wakeup->event_list_lock);
- return;
- }
-
- sock->events |= (event == EPOLLERR) ? (EPOLLIN | EPOLLERR) : (event & sock->epoll_events);
- if (list_is_null(&sock->event_list)) {
- list_add_node(&wakeup->event_list, &sock->event_list);
- }
+ add_sock_event_nolock(sock, event);
pthread_spin_unlock(&wakeup->event_list_lock);
}
- struct protocol_stack *stack = sock->stack;
- if (list_is_null(&wakeup->wakeup_list[stack->stack_idx])) {
- list_add_node(&stack->wakeup_list, &wakeup->wakeup_list[stack->stack_idx]);
+ add_wakeup_to_stack_wakeuplist(wakeup, stack);
+ return;
+}
+
+void del_sock_event_nolock(struct lwip_sock *sock, uint32_t event)
+{
+ sock->events &= ~event;
+
+ if (sock->events == 0) {
+ list_del_node_null(&sock->event_list);
}
+ return;
+}
+
+void del_sock_event(struct lwip_sock *sock, uint32_t event)
+{
+ pthread_spin_lock(&sock->wakeup->event_list_lock);
+ del_sock_event_nolock(sock, event);
+ pthread_spin_unlock(&sock->wakeup->event_list_lock);
}
void wakeup_stack_epoll(struct protocol_stack *stack)
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index 9ab8446..a98b1b8 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -317,22 +317,6 @@ void do_lwip_get_from_sendring_over(struct lwip_sock *sock)
sock->stack->stats.write_lwip_cnt++;
}
-static inline void del_data_out_event(struct lwip_sock *sock)
-{
- pthread_spin_lock(&sock->wakeup->event_list_lock);
-
- /* check again avoid cover event add in stack thread */
- if (!NETCONN_IS_OUTIDLE(sock)) {
- sock->events &= ~EPOLLOUT;
-
- if (sock->events == 0) {
- list_del_node_null(&sock->event_list);
- }
- }
-
- pthread_spin_unlock(&sock->wakeup->event_list_lock);
-}
-
static ssize_t do_app_write(struct pbuf *pbufs[], void *buf, size_t len, uint32_t write_num)
{
ssize_t send_len = 0;
@@ -617,8 +601,9 @@ static ssize_t do_lwip_fill_sendring(struct lwip_sock *sock, const void *buf, si
wakeup->stat.app_write_cnt += write_num;
}
- if (wakeup && wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLOUT)) {
- del_data_out_event(sock);
+ if (wakeup && wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLOUT)
+ && !NETCONN_IS_OUTIDLE(sock)) {
+ del_sock_event(sock, EPOLLOUT);
}
END:
@@ -801,22 +786,6 @@ static inline void notice_stack_send(struct lwip_sock *sock, int32_t fd, int32_t
}
}
-static inline void del_data_in_event(struct lwip_sock *sock)
-{
- pthread_spin_lock(&sock->wakeup->event_list_lock);
-
- /* check again avoid cover event add in stack thread */
- if (!NETCONN_IS_DATAIN(sock)) {
- sock->events &= ~EPOLLIN;
-
- if (sock->events == 0) {
- list_del_node_null(&sock->event_list);
- }
- }
-
- pthread_spin_unlock(&sock->wakeup->event_list_lock);
-}
-
/* process on same node use ring to recv data */
ssize_t gazelle_same_node_ring_recv(struct lwip_sock *sock, const void *buf, size_t len, int32_t flags)
{
@@ -847,8 +816,9 @@ ssize_t gazelle_same_node_ring_recv(struct lwip_sock *sock, const void *buf, siz
END:
/* rte_ring_count reduce lock */
- if (sock->wakeup && sock->wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLIN)) {
- del_data_in_event(sock);
+ if (sock->wakeup && sock->wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLIN)
+ && (!NETCONN_IS_DATAIN(sock))) {
+ del_sock_event(sock, EPOLLIN);
}
return act_len;
}
@@ -1042,8 +1012,9 @@ ssize_t do_lwip_read_from_stack(int32_t fd, void *buf, size_t len, int32_t flags
}
/* rte_ring_count reduce lock */
- if (sock->wakeup && sock->wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLIN)) {
- del_data_in_event(sock);
+ if (sock->wakeup && sock->wakeup->type == WAKEUP_EPOLL && (sock->events & EPOLLIN)
+ && (!NETCONN_IS_DATAIN(sock))) {
+ del_sock_event(sock, EPOLLIN);
}
if (pbuf && addr && addrlen) {
diff --git a/src/lstack/include/posix/lstack_epoll.h b/src/lstack/include/posix/lstack_epoll.h
index d6c81a7..699a951 100644
--- a/src/lstack/include/posix/lstack_epoll.h
+++ b/src/lstack/include/posix/lstack_epoll.h
@@ -63,8 +63,14 @@ struct wakeup_poll {
struct netconn;
struct lwip_sock;
+
void add_sock_event(struct lwip_sock *sock, uint32_t event);
+void add_sock_event_nolock(struct lwip_sock *sock, uint32_t event);
+void del_sock_event(struct lwip_sock *sock, uint32_t event);
+void del_sock_event_nolock(struct lwip_sock *sock, uint32_t event);
+
void wakeup_stack_epoll(struct protocol_stack *stack);
+
int32_t lstack_epoll_create(int32_t size);
int32_t lstack_epoll_create1(int32_t flags);
int32_t lstack_epoll_ctl(int32_t epfd, int32_t op, int32_t fd, struct epoll_event *event);
--
2.27.0

8
gazelle.spec
View File

@ -2,7 +2,7 @@
Name: gazelle
Version: 1.0.2
Release: 6
Release: 7
Summary: gazelle is a high performance user-mode stack
License: MulanPSL-2.0
URL: https://gitee.com/openeuler/gazelle
@ -31,6 +31,8 @@ Patch9012: 0012-kernelevent-kernel-event-thread-report-kernel-events.patch
Patch9013: 0013-fix-bond-port-reta_size-init.patch
Patch9014: 0014-init-remove-sync-sem-between-lstack-thread-and-main-.patch
Patch9015: 0015-lstack_lwip-external-api-start-with-do_lwip_-prefix.patch
Patch9016: 0016-cfg-nic-rx-tx-queue-size-configure.patch
Patch9017: 0017-epoll-distinguish-add-del_sock_event-and-add-del_soc.patch
%description
%{name} is a high performance user-mode stack.
@ -72,6 +74,10 @@ install -Dpm 0640 %{_builddir}/%{name}-%{version}/src/ltran/ltran.conf %{b
%config(noreplace) %{conf_path}/ltran.conf
%changelog
* Mon Oct 30 2023 hantwofish <hankangkang5@huawei.com> - 1.0.2-7
- epoll: distinguish add/del_sock_event and add/del_sock_event_nolock
- cfg: nic rx/tx queue size configure
* Sat Oct 14 2023 yinbin6 <yinbin8@huawei.com> - 1.0.2-6
- update lwip version buildrequire