packages/gazelle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
gazelle/0263-add-interrupt-mode-support.patch
jiangheng 6452f9d06f sync af_xdp: set rlimit unlimit when gazelle init 
(cherry picked from commit 3d3a36ef0ac3f8d75b2cd95c60428d0c7fc1c403)
2024-10-09 10:42:28 +08:00

978 lines
34 KiB
Diff
Raw Permalink Blame History

From 2cfaac4ec2a653af30d7368fbc1c2bdec4fa8bdf Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Sun, 29 Sep 2024 14:41:19 +0800
Subject: [PATCH] add interrupt mode support
---
src/common/gazelle_dfx_msg.h | 10 +
src/lstack/core/lstack_cfg.c | 25 ++
src/lstack/core/lstack_dpdk.c | 27 +-
src/lstack/core/lstack_init.c | 5 +
src/lstack/core/lstack_interrupt.c | 346 ++++++++++++++++++++++++
src/lstack/core/lstack_protocol_stack.c | 24 +-
src/lstack/core/lstack_stack_stat.c | 6 +
src/lstack/core/lstack_thread_rpc.c | 10 +-
src/lstack/core/lstack_virtio.c | 9 +
src/lstack/include/lstack_cfg.h | 1 +
src/lstack/include/lstack_interrupt.h | 34 +++
src/lstack/include/lstack_thread_rpc.h | 12 +-
src/lstack/include/lstack_tx_cache.h | 1 +
src/lstack/include/lstack_virtio.h | 2 +-
src/lstack/netif/lstack_tx_cache.c | 27 +-
src/lstack/netif/lstack_vdev.c | 2 +
src/ltran/ltran_dfx.c | 34 +++
17 files changed, 559 insertions(+), 16 deletions(-)
create mode 100644 src/lstack/core/lstack_interrupt.c
create mode 100644 src/lstack/include/lstack_interrupt.h
diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h
index 266c239..1a89e65 100644
--- a/src/common/gazelle_dfx_msg.h
+++ b/src/common/gazelle_dfx_msg.h
@@ -60,6 +60,7 @@ enum GAZELLE_STAT_MODE {
GAZELLE_STAT_LSTACK_SHOW_XSTATS,
GAZELLE_STAT_LSTACK_SHOW_AGGREGATE,
GAZELLE_STAT_LSTACK_SHOW_NIC_FEATURES,
+ GAZELLE_STAT_LSTACK_SHOW_INTR,
#ifdef GAZELLE_FAULT_INJECT_ENABLE
GAZELLE_STAT_FAULT_INJECT_SET,
@@ -345,6 +346,14 @@ struct nic_eth_features {
uint64_t tx_offload;
};
+struct interrupt_stats {
+ uint64_t virtio_user_event_cnt;
+ uint64_t nic_event_cnt;
+ uint64_t remote_event_cnt;
+ uint64_t local_event_cnt;
+ uint64_t timeout_event_cnt;
+};
+
struct gazelle_stack_dfx_data {
/* indicates whether the current message is the last */
uint32_t eof;
@@ -362,6 +371,7 @@ struct gazelle_stack_dfx_data {
struct nic_eth_xstats nic_xstats;
struct nic_eth_features nic_features;
struct gazelle_stat_lstack_proto proto_data;
+ struct interrupt_stats intr_stats;
#ifdef GAZELLE_FAULT_INJECT_ENABLE
struct gazelle_fault_inject_data inject;
diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c
index 659a2a7..f239b60 100644
--- a/src/lstack/core/lstack_cfg.c
+++ b/src/lstack/core/lstack_cfg.c
@@ -85,6 +85,7 @@ static int32_t parse_defaule_nonblock_mode(void);
static int32_t parse_rpc_msg_max(void);
static int32_t parse_send_cache_mode(void);
static int32_t parse_flow_bifurcation(void);
+static int32_t parse_stack_interrupt(void);
#define PARSE_ARG(_arg, _arg_string, _default_val, _min_val, _max_val, _ret) \
do { \
@@ -152,6 +153,7 @@ static struct config_vector_t g_config_tbl[] = {
{ "rpc_msg_max", parse_rpc_msg_max },
{ "send_cache_mode", parse_send_cache_mode },
{ "flow_bifurcation", parse_flow_bifurcation},
+ { "stack_interrupt", parse_stack_interrupt},
{ NULL, NULL }
};
@@ -1383,3 +1385,26 @@ static int32_t parse_flow_bifurcation(void)
PARSE_ARG(g_config_params.flow_bifurcation, "flow_bifurcation", 0, 0, 1, ret);
return ret;
}
+
+static int32_t parse_stack_interrupt(void)
+{
+ int32_t ret;
+ PARSE_ARG(g_config_params.stack_interrupt, "stack_interrupt", false, false, true, ret);
+ if (ret != 0) {
+ LSTACK_PRE_LOG(LSTACK_ERR, "cfg: invalid enable intr value %d. only support 0 or 1\n", \
+ g_config_params.stack_interrupt);
+ }
+
+ if (g_config_params.stack_interrupt == true) {
+ if (g_config_params.stack_mode_rtc == true) {
+ LSTACK_PRE_LOG(LSTACK_ERR, "rtc mode not support interrupt mode now.\n");
+ return -1;
+ }
+ if (g_config_params.bond_mode >= 0) {
+ LSTACK_PRE_LOG(LSTACK_ERR, "bond mode not support interrupt mode.\n");
+ return -1;
+ }
+ }
+
+ return ret;
+}
diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index 1fe0f0a..530332b 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -258,8 +258,8 @@ struct rte_mempool *create_mempool(const char *name, uint32_t count, uint32_t si
int32_t create_shared_ring(struct protocol_stack *stack)
{
- rpc_queue_init(&stack->rpc_queue);
- rpc_queue_init(&stack->dfx_rpc_queue);
+ rpc_queue_init(&stack->rpc_queue, stack->queue_id);
+ rpc_queue_init(&stack->dfx_rpc_queue, stack->queue_id);
if (use_ltran()) {
stack->rx_ring = gazelle_ring_create_fast("RING_RX", VDEV_RX_QUEUE_SZ, RING_F_SP_ENQ | RING_F_SC_DEQ);
@@ -439,6 +439,7 @@ static int eth_params_init(struct eth_params *eth_params, uint16_t port_id, uint
eth_params->conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
/* used for tcp port alloc */
eth_params->reta_mask = dev_info.reta_size - 1;
+ eth_params->conf.intr_conf.rxq = get_global_cfg_params()->stack_interrupt;
eth_params_checksum(&eth_params->conf, &dev_info);
@@ -630,10 +631,11 @@ static int32_t dpdk_ethdev_setup(const struct eth_params *eth_params, uint16_t i
int32_t dpdk_ethdev_start(void)
{
+ int i;
int32_t ret;
const struct protocol_stack_group *stack_group = get_protocol_stack_group();
- for (int32_t i = 0; i < get_global_cfg_params()->tot_queue_num; i++) {
+ for (i = 0; i < get_global_cfg_params()->tot_queue_num; i++) {
ret = dpdk_ethdev_setup(stack_group->eth_params, i);
if (ret < 0) {
LSTACK_LOG(ERR, LSTACK, "dpdk_ethdev_setup fail queueid=%d, ret=%d\n", i, ret);
@@ -647,6 +649,14 @@ int32_t dpdk_ethdev_start(void)
return ret;
}
+ /* after rte_eth_dev_start */
+ for (i = 0; i < get_global_cfg_params()->tot_queue_num; i++) {
+ struct intr_dpdk_event_args intr_arg;
+ intr_arg.port_id = stack_group->eth_params->port_id;
+ intr_arg.queue_id = i;
+ intr_register(i, INTR_DPDK_EVENT, &intr_arg);
+ }
+
return 0;
}
@@ -799,10 +809,21 @@ int init_dpdk_ethdev(void)
}
port_id = rte_eth_bond_primary_get(get_protocol_stack_group()->port_id);
} else {
+ struct rte_eth_dev_info dev_info;
port_id = ethdev_port_id(cfg->mac_addr);
if (port_id < 0) {
return -1;
}
+
+ if (rte_eth_dev_info_get(port_id, &dev_info) < 0) {
+ return -1;
+ }
+ if (strcmp(dev_info.driver_name, "net_hinic") == 0 &&
+ get_global_cfg_params()->stack_interrupt == true) {
+ LSTACK_LOG(ERR, LSTACK, "hinic not support interrupt mode\n");
+ return -1;
+ }
+
ret = dpdk_ethdev_init(port_id);
if (ret != 0) {
LSTACK_LOG(ERR, LSTACK, "dpdk_ethdev_init failed, port id=%d\n", port_id);
diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c
index 37264a1..5e405ee 100644
--- a/src/lstack/core/lstack_init.c
+++ b/src/lstack/core/lstack_init.c
@@ -45,6 +45,7 @@
#include "lstack_preload.h"
#include "lstack_wrap.h"
#include "lstack_flow.h"
+#include "lstack_interrupt.h"
static void check_process_start(void)
{
@@ -289,6 +290,10 @@ __attribute__((constructor)) void gazelle_network_init(void)
LSTACK_EXIT(1, "stack_group_init failed\n");
}
+ if (intr_init() < 0) {
+ LSTACK_EXIT(1, "intr init failed\n");
+ }
+
if (!use_ltran()) {
if (init_dpdk_ethdev() != 0) {
LSTACK_EXIT(1, "init_dpdk_ethdev failed\n");
diff --git a/src/lstack/core/lstack_interrupt.c b/src/lstack/core/lstack_interrupt.c
new file mode 100644
index 0000000..26823cd
--- /dev/null
+++ b/src/lstack/core/lstack_interrupt.c
@@ -0,0 +1,346 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+* http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#include <sys/un.h>
+#include <sys/socket.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <securec.h>
+
+#include <rte_interrupts.h>
+#include <rte_ethdev.h>
+
+#include <lwip/lwipgz_posix_api.h>
+#include <lwip/lwipopts.h>
+#include <lwip/arch/sys_arch.h>
+
+#include "common/dpdk_common.h"
+#include "common/gazelle_opt.h"
+#include "common/gazelle_dfx_msg.h"
+#include "lstack_log.h"
+#include "lstack_cfg.h"
+#include "lstack_interrupt.h"
+
+#define INTR_MAX_EVENT_NUM 8
+
+struct intr_dpdk_event {
+ int rte_epfd;
+#define INTR_PORT_NUM 2
+#define INTR_INVALID_PORT 65535
+ uint16_t port_id[INTR_PORT_NUM]; /* 0: nic port id, 1: virtio_user port id */
+ uint16_t queue_id[INTR_PORT_NUM];
+};
+
+struct intr_local_event {
+ bool (*get_event) (uint16_t stack_id);
+};
+
+struct intr_remote_event {
+ int event_fd;
+};
+
+struct intr_policy {
+#define INTR_LOOP_TIMES 5
+ uint8_t no_event_cnt;
+};
+
+struct intr_config {
+ int epoll_fd; /* used for epoll */
+ uint16_t stack_id;
+ bool in_wait;
+
+ struct intr_dpdk_event dpdk_event;
+ struct intr_local_event local_event;
+ struct intr_remote_event remote_event;
+
+ struct intr_policy policy;
+ struct interrupt_stats stats;
+};
+
+static struct intr_config g_intr_configs[PROTOCOL_STACK_MAX] = {0};
+
+static inline struct intr_config *intr_config_get(uint16_t stack_id)
+{
+ return &g_intr_configs[stack_id];
+}
+
+int intr_init(void)
+{
+ int stack_id;
+ struct cfg_params *cfg = get_global_cfg_params();
+ if (!cfg->stack_interrupt) {
+ return 0;
+ }
+
+ for (stack_id = 0; stack_id < cfg->num_queue; stack_id++) {
+ struct intr_config *intr_config = intr_config_get(stack_id);
+ intr_config->epoll_fd = posix_api->epoll_create_fn(1);
+ if (intr_config->epoll_fd < 0) {
+ LSTACK_LOG(ERR, LSTACK, "epoll create fd fialed, errno is %d\n", errno);
+ return -1;
+ }
+
+ for (int i = 0; i < INTR_PORT_NUM; i++) {
+ intr_config->dpdk_event.port_id[i] = INTR_INVALID_PORT;
+ }
+
+ if (intr_register(stack_id, INTR_REMOTE_EVENT, NULL) < 0) {
+ LSTACK_LOG(ERR, LSTACK, "register intr failed\n");
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static inline int add_fd_to_epoll(int fd, int epoll_fd)
+{
+ struct epoll_event event ;
+ event.data.fd = fd ;
+ event.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP ;
+ int ret = posix_api->epoll_ctl_fn(epoll_fd, EPOLL_CTL_ADD, fd, &event);
+ if (ret < 0) {
+ LSTACK_LOG(ERR, LSTACK, "add fd %d to epoll fd %d failed errno:%d ret=%d.\n",
+ fd, epoll_fd, errno, ret);
+ return ret;
+ }
+
+ return ret ;
+}
+
+static inline int intr_local_event_register(struct intr_config *config, void *priv)
+{
+ config->local_event.get_event = priv;
+ return 0;
+}
+
+static int intr_dpdk_event_register(struct intr_config *config, void *priv)
+{
+ struct intr_dpdk_event_args *arg = priv;
+ int i;
+
+ if (arg == NULL) {
+ return -1;
+ }
+
+ if (config->dpdk_event.rte_epfd <= 0) {
+ config->dpdk_event.rte_epfd = posix_api->epoll_create_fn(1);
+ if (config->dpdk_event.rte_epfd < 0) {
+ LSTACK_LOG(ERR, LSTACK, "epoll create fd fialed, errno is %d\n", errno);
+ return -1;
+ }
+ if (add_fd_to_epoll(config->dpdk_event.rte_epfd, config->epoll_fd) < 0) {
+ return -1;
+ }
+ }
+
+ for (i = 0; i < INTR_PORT_NUM; i++) {
+ if (config->dpdk_event.port_id[i] == INTR_INVALID_PORT) {
+ config->dpdk_event.port_id[i] = arg->port_id;
+ break;
+ }
+ }
+ config->dpdk_event.queue_id[i] = arg->queue_id;
+
+ int data = ((arg->port_id) << CHAR_BIT) | arg->queue_id;
+ if (rte_eth_dev_rx_intr_ctl_q(arg->port_id, arg->queue_id, config->dpdk_event.rte_epfd,
+ RTE_INTR_EVENT_ADD, (void *)((uintptr_t)data)) < 0) {
+ LSTACK_LOG(ERR, LSTACK, "rte_eth_dev_rx_intr_ctl_q failed, port(%d), queue(%d)\n",
+ arg->port_id, arg->queue_id);
+ return -1;
+ }
+ return 0;
+}
+
+static int intr_remote_event_register(struct intr_config *config, void *priv)
+{
+ struct intr_remote_event *remote_event = &config->remote_event;
+ if (remote_event->event_fd > 0) {
+ return 0;
+ }
+
+ remote_event->event_fd = posix_api->eventfd_fn(0, 0);
+ if (remote_event->event_fd < 0) {
+ LSTACK_LOG(ERR, LSTACK, "event fd create failed\n");
+ return -1;
+ }
+ return add_fd_to_epoll(remote_event->event_fd, config->epoll_fd);
+}
+
+int intr_register(uint16_t stack_id, enum intr_type type, void *priv)
+{
+ struct cfg_params *cfg = get_global_cfg_params();
+ if (!cfg->stack_interrupt) {
+ return 0;
+ }
+
+ struct intr_config *config = intr_config_get(stack_id);
+ switch (type) {
+ case INTR_DPDK_EVENT:
+ return intr_dpdk_event_register(config, priv);
+ case INTR_REMOTE_EVENT:
+ return intr_remote_event_register(config, priv);
+ case INTR_LOCAL_EVENT:
+ return intr_local_event_register(config, priv);
+ default:
+ return -1;
+ }
+ return 0;
+}
+
+static inline void intr_remote_event_enable(struct intr_config *config)
+{
+ eventfd_t eventfd_num = 1;
+ if (__atomic_load_n(&config->in_wait, __ATOMIC_ACQUIRE)) {
+ posix_api->write_fn(config->remote_event.event_fd, &eventfd_num, sizeof(eventfd_t));
+ }
+}
+
+static inline void intr_remote_event_disable(struct intr_config *config)
+{
+ eventfd_t read_num;
+ posix_api->read_fn(config->remote_event.event_fd, &read_num, sizeof(eventfd_t));
+}
+
+static inline bool intr_local_event(struct intr_config *config)
+{
+ return config->local_event.get_event(config->stack_id);
+}
+
+void intr_wakeup(uint16_t stack_id, enum intr_type type)
+{
+ if (!get_global_cfg_params()->stack_interrupt) {
+ return;
+ }
+
+ struct intr_config *config = intr_config_get(stack_id);
+ switch (type) {
+ case INTR_REMOTE_EVENT:
+ intr_remote_event_enable(config);
+ break;
+ default:
+ break;
+ }
+}
+
+static inline void intr_dpdk_event_enable(struct intr_config *config)
+{
+ int i;
+ int ret = 0;
+ struct intr_dpdk_event *dpdk_event = &config->dpdk_event;
+
+ for (i = 0; i < INTR_PORT_NUM; i++) {
+ if (dpdk_event->port_id[i] != INTR_INVALID_PORT) {
+ ret = rte_eth_dev_rx_intr_enable(dpdk_event->port_id[i], dpdk_event->queue_id[i]);
+ if (ret != 0) {
+ LSTACK_LOG(ERR, LSTACK, "port(%d) queue(%d) enable interrupt failed\n",
+ dpdk_event->port_id[i], dpdk_event->queue_id[i]);
+ return;
+ }
+ }
+ }
+}
+
+static inline void intr_dpdk_event_disable(struct intr_config *config)
+{
+ int i, n;
+ void *data;
+ uint16_t port_id;
+ uint16_t queue_id;
+ struct intr_dpdk_event *dpdk_event = &config->dpdk_event;
+ struct rte_epoll_event event[INTR_MAX_EVENT_NUM];
+
+ n = rte_epoll_wait(dpdk_event->rte_epfd, event, INTR_MAX_EVENT_NUM, 1);
+ for (i = 0; i < n; i++) {
+ data = event[i].epdata.data;
+ port_id = ((uintptr_t)data) >> CHAR_BIT;
+ queue_id = ((uintptr_t)data) & RTE_LEN2MASK(CHAR_BIT, uint8_t);
+
+ rte_eth_dev_rx_intr_disable(port_id, queue_id);
+
+ if (port_id == dpdk_event->port_id[0]) {
+ config->stats.nic_event_cnt++;
+ } else {
+ config->stats.virtio_user_event_cnt++;
+ }
+ }
+}
+
+static inline void intr_policy_clear(struct intr_config *config)
+{
+ config->policy.no_event_cnt = 0;
+}
+
+static inline bool intr_policy(struct intr_config *config)
+{
+ if (config->policy.no_event_cnt++ < INTR_LOOP_TIMES) {
+ return true;
+ }
+ config->policy.no_event_cnt = 0;
+ return false;
+}
+
+static inline void intr_block(uint16_t stack_id, uint32_t timeout)
+{
+ struct epoll_event events[INTR_MAX_EVENT_NUM];
+ struct intr_config *intr_config = intr_config_get(stack_id);
+
+ /* in_wait need in here to avoid competion problem with remote event */
+ __atomic_store_n(&intr_config->in_wait, true, __ATOMIC_RELEASE);
+ if (intr_local_event(intr_config)) {
+ intr_config->stats.local_event_cnt++;
+ __atomic_store_n(&intr_config->in_wait, false, __ATOMIC_RELEASE);
+ return;
+ }
+
+ intr_dpdk_event_enable(intr_config);
+
+ int32_t event_cnt = posix_api->epoll_wait_fn(intr_config->epoll_fd, events, INTR_MAX_EVENT_NUM, timeout);
+ __atomic_store_n(&intr_config->in_wait, false, __ATOMIC_RELEASE);
+ for (int i = 0; i < event_cnt; i++) {
+ if (events[i].data.fd == intr_config->dpdk_event.rte_epfd) {
+ intr_dpdk_event_disable(intr_config);
+ } else if (events[i].data.fd == intr_config->remote_event.event_fd) {
+ intr_remote_event_disable(intr_config);
+ intr_config->stats.remote_event_cnt++;
+ } else {
+ LSTACK_LOG(ERR, LSTACK, "unknow fd have event.\n");
+ }
+ }
+
+ if (event_cnt < 0) {
+ intr_config->stats.timeout_event_cnt++;
+ }
+}
+
+void intr_wait(uint16_t stack_id, uint32_t timeout)
+{
+ struct intr_config *intr_config = intr_config_get(stack_id);
+
+ if (intr_policy(intr_config)) {
+ return;
+ }
+
+ intr_block(stack_id, timeout);
+
+ intr_policy_clear(intr_config);
+}
+
+int intr_stats_get(uint16_t stack_id, void *ptr, int len)
+{
+ struct intr_config *config = intr_config_get(stack_id);
+ if (len < sizeof(struct interrupt_stats)) {
+ return -1;
+ }
+
+ return memcpy_s(ptr, len, &config->stats, sizeof(struct interrupt_stats));
+}
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index d03b744..e412c3b 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -34,6 +34,8 @@
#include "lstack_virtio.h"
#include "lstack_protocol_stack.h"
+#include "lstack_interrupt.h"
+
#if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0)
#include <rte_kni.h>
#endif
@@ -480,13 +482,18 @@ int stack_polling(unsigned wakeup_tick)
uint32_t rpc_number = cfg->rpc_number;
uint32_t read_connect_number = cfg->read_connect_number;
struct protocol_stack *stack = get_protocol_stack();
+ uint32_t timeout;
/* 2: one dfx consumes two rpc */
rpc_poll_msg(&stack->dfx_rpc_queue, 2);
force_quit = rpc_poll_msg(&stack->rpc_queue, rpc_number);
eth_dev_poll();
- sys_timer_run();
+ timeout = sys_timer_run();
+ if (cfg->stack_interrupt) {
+ intr_wait(stack->stack_idx, timeout);
+ }
+
if (cfg->low_power_mod != 0) {
low_power_idling(stack);
}
@@ -496,6 +503,7 @@ int stack_polling(unsigned wakeup_tick)
}
do_lwip_read_recvlist(stack, read_connect_number);
+
if ((wakeup_tick & 0xf) == 0) {
wakeup_stack_epoll(stack);
if (get_global_cfg_params()->send_cache_mode) {
@@ -533,6 +541,19 @@ int stack_polling(unsigned wakeup_tick)
return force_quit;
}
+static bool stack_local_event_get(uint16_t stack_id)
+{
+ struct protocol_stack *stack = g_stack_group.stacks[stack_id];
+ if (!lockless_queue_empty(&stack->dfx_rpc_queue.queue) ||
+ !lockless_queue_empty(&stack->rpc_queue.queue) ||
+ !list_head_empty(&stack->recv_list) ||
+ !list_head_empty(&stack->wakeup_list) ||
+ tx_cache_count(stack->queue_id)) {
+ return true;
+ }
+ return false;
+}
+
static void* gazelle_stack_thread(void *arg)
{
struct thread_params *t_params = (struct thread_params*) arg;
@@ -541,6 +562,7 @@ static void* gazelle_stack_thread(void *arg)
unsigned wakeup_tick = 0;
stack = stack_thread_init(arg);
+ intr_register(stack->stack_idx, INTR_LOCAL_EVENT, stack_local_event_get);
free(arg);
if (stack == NULL) {
LSTACK_LOG(ERR, LSTACK, "stack_thread_init failed queue_id=%hu\n", queue_id);
diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c
index b6619f6..8efa5ab 100644
--- a/src/lstack/core/lstack_stack_stat.c
+++ b/src/lstack/core/lstack_stack_stat.c
@@ -325,6 +325,12 @@ static void get_stack_dfx_data(struct gazelle_stack_dfx_data *dfx, struct protoc
LSTACK_LOG(ERR, LSTACK, "memcpy_s err ret=%d \n", ret);
}
break;
+ case GAZELLE_STAT_LSTACK_SHOW_INTR:
+ ret = intr_stats_get(stack->stack_idx, &dfx->data.intr_stats, sizeof(dfx->data.intr_stats));
+ if (ret != EOK) {
+ LSTACK_LOG(ERR, LSTACK, "memcpy_s err ret=%d \n", ret);
+ }
+ break;
case GAZELLE_STAT_LSTACK_SHOW_VIRTIO:
ret = memcpy_s(&dfx->data.virtio, sizeof(dfx->data.virtio), virtio_instance_get(),
sizeof(*(virtio_instance_get())));
diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c
index b4a5953..7f77c12 100644
--- a/src/lstack/core/lstack_thread_rpc.c
+++ b/src/lstack/core/lstack_thread_rpc.c
@@ -100,7 +100,8 @@ __rte_always_inline
static void rpc_async_call(rpc_queue *queue, struct rpc_msg *msg)
{
msg->sync_flag = 0;
- lockless_queue_mpsc_push(queue, &msg->queue_node);
+ lockless_queue_mpsc_push(&queue->queue, &msg->queue_node);
+ intr_wakeup(queue->queue_id, INTR_REMOTE_EVENT);
}
__rte_always_inline
@@ -111,7 +112,8 @@ static int rpc_sync_call(rpc_queue *queue, struct rpc_msg *msg)
pthread_spin_trylock(&msg->lock);
msg->sync_flag = 1;
- lockless_queue_mpsc_push(queue, &msg->queue_node);
+ lockless_queue_mpsc_push(&queue->queue, &msg->queue_node);
+ intr_wakeup(queue->queue_id, INTR_REMOTE_EVENT);
// waiting stack unlock
pthread_spin_lock(&msg->lock);
@@ -123,7 +125,7 @@ static int rpc_sync_call(rpc_queue *queue, struct rpc_msg *msg)
int rpc_msgcnt(rpc_queue *queue)
{
- return lockless_queue_count(queue);
+ return lockless_queue_count(&queue->queue);
}
static struct rpc_msg *rpc_msg_alloc_except(rpc_func_t func)
@@ -159,7 +161,7 @@ int rpc_poll_msg(rpc_queue *queue, int max_num)
struct rpc_msg *msg;
while (max_num--) {
- lockless_queue_node *node = lockless_queue_mpsc_pop(queue);
+ lockless_queue_node *node = lockless_queue_mpsc_pop(&queue->queue);
if (node == NULL) {
break;
}
diff --git a/src/lstack/core/lstack_virtio.c b/src/lstack/core/lstack_virtio.c
index fefb06d..75a23f2 100644
--- a/src/lstack/core/lstack_virtio.c
+++ b/src/lstack/core/lstack_virtio.c
@@ -20,6 +20,7 @@
#include "lstack_cfg.h"
#include "lstack_log.h"
#include "lstack_port_map.h"
+#include "lstack_interrupt.h"
#include "lstack_virtio.h"
#include "securec.h"
@@ -274,6 +275,7 @@ static int virtio_port_init(uint16_t port)
return retval;
}
+ port_conf.intr_conf.rxq = get_global_cfg_params()->stack_interrupt;
retval = rte_eth_dev_configure(port, rx_queue_num, tx_queue_num, &port_conf);
if (retval != 0) {
LSTACK_LOG(ERR, LSTACK, "rte_eth_dev_configure failed retval=%d\n", retval);
@@ -297,6 +299,13 @@ static int virtio_port_init(uint16_t port)
LSTACK_LOG(ERR, LSTACK, "rte_eth_rx_queue_setup failed (queue %u) retval=%d \n", q, retval);
return retval;
}
+
+ if (port_conf.intr_conf.rxq) {
+ struct intr_dpdk_event_args intr_arg;
+ intr_arg.port_id = port;
+ intr_arg.queue_id = q;
+ intr_register(q, INTR_DPDK_EVENT, &intr_arg);
+ }
}
return 0;
}
diff --git a/src/lstack/include/lstack_cfg.h b/src/lstack/include/lstack_cfg.h
index 5e2d6fc..071492d 100644
--- a/src/lstack/include/lstack_cfg.h
+++ b/src/lstack/include/lstack_cfg.h
@@ -118,6 +118,7 @@ struct cfg_params {
bool stack_mode_rtc;
bool listen_shadow; // true:listen in all stack thread. false:listen in one stack thread.
+ bool stack_interrupt;
uint32_t read_connect_number;
uint32_t nic_read_number;
diff --git a/src/lstack/include/lstack_interrupt.h b/src/lstack/include/lstack_interrupt.h
new file mode 100644
index 0000000..1be7d07
--- /dev/null
+++ b/src/lstack/include/lstack_interrupt.h
@@ -0,0 +1,34 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
+* gazelle is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+* http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+#ifndef __LSTACK_INTERRUPT_H__
+#define __LSTACK_INTERRUPT_H__
+
+enum intr_type {
+ INTR_DPDK_EVENT = 0,
+ INTR_LOCAL_EVENT,
+ INTR_REMOTE_EVENT,
+};
+
+struct intr_dpdk_event_args {
+ uint16_t port_id;
+ uint16_t queue_id;
+};
+
+int intr_init(void);
+int intr_register(uint16_t stack_id, enum intr_type type, void *priv);
+void intr_wakeup(uint16_t stack_id, enum intr_type type);
+void intr_wait(uint16_t stack_id, uint32_t timeout);
+int intr_stats_get(uint16_t stack_id, void *ptr, int len);
+
+#endif
+
diff --git a/src/lstack/include/lstack_thread_rpc.h b/src/lstack/include/lstack_thread_rpc.h
index 6f8e03e..c284d29 100644
--- a/src/lstack/include/lstack_thread_rpc.h
+++ b/src/lstack/include/lstack_thread_rpc.h
@@ -17,6 +17,7 @@
#include <rte_mempool.h>
#include "lstack_lockless_queue.h"
+#include "lstack_interrupt.h"
#define MSG_ARG_0 (0)
#define MSG_ARG_1 (1)
@@ -25,7 +26,11 @@
#define MSG_ARG_4 (4)
#define RPM_MSG_ARG_SIZE (5)
-typedef struct lockless_queue rpc_queue;
+typedef struct rpc_queue rpc_queue;
+struct rpc_queue {
+ struct lockless_queue queue;
+ uint16_t queue_id;
+};
struct rpc_stats {
uint16_t call_null;
@@ -60,9 +65,10 @@ struct rpc_msg {
lockless_queue_node queue_node;
};
-static inline void rpc_queue_init(rpc_queue *queue)
+static inline void rpc_queue_init(rpc_queue *queue, uint16_t queue_id)
{
- lockless_queue_init(queue);
+ lockless_queue_init(&queue->queue);
+ queue->queue_id = queue_id;
}
struct rpc_stats *rpc_stats_get(void);
int rpc_msgcnt(rpc_queue *queue);
diff --git a/src/lstack/include/lstack_tx_cache.h b/src/lstack/include/lstack_tx_cache.h
index 04e9e35..ccd9c17 100644
--- a/src/lstack/include/lstack_tx_cache.h
+++ b/src/lstack/include/lstack_tx_cache.h
@@ -15,5 +15,6 @@
int tx_cache_init(uint16_t queue_id, void *priv, struct lstack_dev_ops *dev_ops);
int tx_cache_send(uint16_t queue_id);
+int tx_cache_count(uint16_t queue_id);
#endif /* _LSTACK_TX_CACHE_H_ */
diff --git a/src/lstack/include/lstack_virtio.h b/src/lstack/include/lstack_virtio.h
index 5298dbe..745c86b 100644
--- a/src/lstack/include/lstack_virtio.h
+++ b/src/lstack/include/lstack_virtio.h
@@ -50,4 +50,4 @@ int virtio_port_create(int lstack_net_port);
struct virtio_instance* virtio_instance_get(void);
bool virtio_distribute_pkg_to_kernel(uint16_t dst_port);
-#endif
\ No newline at end of file
+#endif
diff --git a/src/lstack/netif/lstack_tx_cache.c b/src/lstack/netif/lstack_tx_cache.c
index 9a48307..17d3c3b 100644
--- a/src/lstack/netif/lstack_tx_cache.c
+++ b/src/lstack/netif/lstack_tx_cache.c
@@ -58,22 +58,40 @@ int tx_cache_init(uint16_t queue_id, void *priv, struct lstack_dev_ops *dev_ops)
return 0;
}
+int tx_cache_count(uint16_t queue_id)
+{
+ struct tx_cache *tx_cache = g_tx_cache[queue_id];
+ if (tx_cache == NULL) {
+ return 0;
+ }
+ uint32_t start = tx_cache->send_start;
+ uint32_t end = tx_cache->send_end;
+ uint32_t count = (end - start) & TX_CACHE_MASK;
+ uint32_t capacity = TX_CACHE_MAX - 1;
+
+ return (count > capacity) ? capacity : count;
+}
+
int tx_cache_send(uint16_t queue_id)
{
+ uint32_t send_num;
+ uint32_t sent_pkts = 0;
+ uint32_t start;
+ uint32_t end;
+
struct tx_cache *tx_cache = g_tx_cache[queue_id];
if (tx_cache == NULL) {
LSTACK_LOG(ERR, LSTACK, "queue(%d) tx cache get failed\n", queue_id);
return 0;
}
- uint32_t send_num = tx_cache->send_end - tx_cache->send_start;
+ send_num = tx_cache_count(queue_id);
if (send_num == 0) {
return 0;
}
- uint32_t start = tx_cache->send_start & TX_CACHE_MASK;
- uint32_t end = tx_cache->send_end & TX_CACHE_MASK;
- uint32_t sent_pkts = 0;
+ start = tx_cache->send_start & TX_CACHE_MASK;
+ end = tx_cache->send_end & TX_CACHE_MASK;
if (start < end) {
sent_pkts = g_tx_cache_dev_ops.tx_xmit(tx_cache->priv, &tx_cache->send_pkts[start], send_num);
} else {
@@ -85,6 +103,7 @@ int tx_cache_send(uint16_t queue_id)
}
tx_cache->send_start += sent_pkts;
+
return sent_pkts;
}
diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c
index e1a63a7..290046e 100644
--- a/src/lstack/netif/lstack_vdev.c
+++ b/src/lstack/netif/lstack_vdev.c
@@ -38,6 +38,8 @@
#include "lstack_port_map.h"
#include "lstack_virtio.h"
+#include "lstack_interrupt.h"
+
/* INUSE_TX_PKTS_WATERMARK < VDEV_RX_QUEUE_SZ;
* USE_RX_PKTS_WATERMARK < FREE_RX_QUEUE_SZ.
* less, means more available mbuf.
diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c
index 7fa117a..f6d1148 100644
--- a/src/ltran/ltran_dfx.c
+++ b/src/ltran/ltran_dfx.c
@@ -138,6 +138,7 @@ static void gazelle_print_lstack_xstats(void *buf, const struct gazelle_stat_msg
static void gazelle_print_lstack_aggregate(void *buf, const struct gazelle_stat_msg_request *req_msg);
static void gazelle_print_lstack_nic_features(void *buf, const struct gazelle_stat_msg_request *req_msg);
static void gazelle_print_lstack_stat_proto(void *buf, const struct gazelle_stat_msg_request *req_msg);
+static void gazelle_print_lstack_stat_intr(void *buf, const struct gazelle_stat_msg_request *req_msg);
#ifdef GAZELLE_FAULT_INJECT_ENABLE
static void gazelle_print_fault_inject_set_status(void *buf, const struct gazelle_stat_msg_request *req_msg);
@@ -172,6 +173,7 @@ static struct gazelle_dfx_list g_gazelle_dfx_tbl[] = {
{GAZELLE_STAT_LSTACK_SHOW_AGGREGATE, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_aggregate},
{GAZELLE_STAT_LSTACK_SHOW_NIC_FEATURES, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_nic_features},
{GAZELLE_STAT_LSTACK_SHOW_PROTOCOL, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_stat_proto},
+ {GAZELLE_STAT_LSTACK_SHOW_INTR, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_stat_intr},
#ifdef GAZELLE_FAULT_INJECT_ENABLE
{GAZELLE_STAT_FAULT_INJECT_SET, sizeof(struct gazelle_stack_dfx_data), gazelle_print_fault_inject_set_status},
@@ -1136,6 +1138,17 @@ static void gazelle_print_lstack_stat_proto_core(const struct gazelle_stack_dfx_
printf("rterr: %lu\n", proto->rterr);
}
+static void gazelle_print_lstack_stat_intr_core(const struct gazelle_stack_dfx_data *stat,
+ const struct interrupt_stats *intr_stats)
+{
+ printf("\n------ stack tid: %6u ------\n", stat->tid);
+ printf("nic_event_cnt: %lu\n", intr_stats->nic_event_cnt);
+ printf("virtio_user_event_cnt: %lu\n", intr_stats->virtio_user_event_cnt);
+ printf("local_event_cnt: %lu\n", intr_stats->local_event_cnt);
+ printf("remote_event_cnt: %lu\n", intr_stats->remote_event_cnt);
+ printf("timeout_event_cnt: %lu\n", intr_stats->timeout_event_cnt);
+}
+
static void gazelle_print_lstack_stat_snmp(void *buf, const struct gazelle_stat_msg_request *req_msg)
{
int32_t ret;
@@ -1155,6 +1168,25 @@ static void gazelle_print_lstack_stat_snmp(void *buf, const struct gazelle_stat_
} while (true);
}
+static void gazelle_print_lstack_stat_intr(void *buf, const struct gazelle_stat_msg_request *req_msg)
+{
+ int32_t ret;
+ struct gazelle_stack_dfx_data *stat = (struct gazelle_stack_dfx_data *)buf;
+ struct interrupt_stats *intr_stats = &stat->data.intr_stats;
+
+ printf("Statistics of lstack interrupt:\n");
+ do {
+ gazelle_print_lstack_stat_intr_core(stat, intr_stats);
+ if (stat->eof != 0) {
+ break;
+ }
+ ret = dfx_stat_read_from_ltran(buf, sizeof(struct gazelle_stack_dfx_data), req_msg->stat_mode);
+ if (ret != GAZELLE_OK) {
+ return;
+ }
+ } while (true);
+}
+
static void gazelle_print_lstack_stat_proto(void *buf, const struct gazelle_stat_msg_request *req_msg)
{
int32_t ret;
@@ -1579,6 +1611,8 @@ static int32_t parse_dfx_lstack_show_args(int32_t argc, char *argv[], struct gaz
req_msg[cmd_index++].stat_mode = GAZELLE_STAT_MODE_MAX;
} else if (strcmp(param, "snmp") == 0 || strcmp(param, "-s") == 0) {
req_msg[cmd_index++].stat_mode = GAZELLE_STAT_LSTACK_SHOW_SNMP;
+ } else if (strcmp(param, "intr") == 0 || strcmp(param, "-I") == 0) {
+ req_msg[cmd_index++].stat_mode = GAZELLE_STAT_LSTACK_SHOW_INTR;
} else if (strcmp(param, "virtio") == 0 || strcmp(param, "-v") == 0) {
req_msg[cmd_index++].stat_mode = GAZELLE_STAT_LSTACK_SHOW_VIRTIO;
} else if (strcmp(param, "connect") == 0 || strcmp(param, "-c") == 0) {
--
2.33.0
Reference in New Issue View Git Blame Copy Permalink