From 2cfaac4ec2a653af30d7368fbc1c2bdec4fa8bdf Mon Sep 17 00:00:00 2001 From: jiangheng Date: Sun, 29 Sep 2024 14:41:19 +0800 Subject: [PATCH] add interrupt mode support --- src/common/gazelle_dfx_msg.h | 10 + src/lstack/core/lstack_cfg.c | 25 ++ src/lstack/core/lstack_dpdk.c | 27 +- src/lstack/core/lstack_init.c | 5 + src/lstack/core/lstack_interrupt.c | 346 ++++++++++++++++++++++++ src/lstack/core/lstack_protocol_stack.c | 24 +- src/lstack/core/lstack_stack_stat.c | 6 + src/lstack/core/lstack_thread_rpc.c | 10 +- src/lstack/core/lstack_virtio.c | 9 + src/lstack/include/lstack_cfg.h | 1 + src/lstack/include/lstack_interrupt.h | 34 +++ src/lstack/include/lstack_thread_rpc.h | 12 +- src/lstack/include/lstack_tx_cache.h | 1 + src/lstack/include/lstack_virtio.h | 2 +- src/lstack/netif/lstack_tx_cache.c | 27 +- src/lstack/netif/lstack_vdev.c | 2 + src/ltran/ltran_dfx.c | 34 +++ 17 files changed, 559 insertions(+), 16 deletions(-) create mode 100644 src/lstack/core/lstack_interrupt.c create mode 100644 src/lstack/include/lstack_interrupt.h diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h index 266c239..1a89e65 100644 --- a/src/common/gazelle_dfx_msg.h +++ b/src/common/gazelle_dfx_msg.h @@ -60,6 +60,7 @@ enum GAZELLE_STAT_MODE { GAZELLE_STAT_LSTACK_SHOW_XSTATS, GAZELLE_STAT_LSTACK_SHOW_AGGREGATE, GAZELLE_STAT_LSTACK_SHOW_NIC_FEATURES, + GAZELLE_STAT_LSTACK_SHOW_INTR, #ifdef GAZELLE_FAULT_INJECT_ENABLE GAZELLE_STAT_FAULT_INJECT_SET, @@ -345,6 +346,14 @@ struct nic_eth_features { uint64_t tx_offload; }; +struct interrupt_stats { + uint64_t virtio_user_event_cnt; + uint64_t nic_event_cnt; + uint64_t remote_event_cnt; + uint64_t local_event_cnt; + uint64_t timeout_event_cnt; +}; + struct gazelle_stack_dfx_data { /* indicates whether the current message is the last */ uint32_t eof; @@ -362,6 +371,7 @@ struct gazelle_stack_dfx_data { struct nic_eth_xstats nic_xstats; struct nic_eth_features nic_features; struct gazelle_stat_lstack_proto proto_data; + struct interrupt_stats intr_stats; #ifdef GAZELLE_FAULT_INJECT_ENABLE struct gazelle_fault_inject_data inject; diff --git a/src/lstack/core/lstack_cfg.c b/src/lstack/core/lstack_cfg.c index 659a2a7..f239b60 100644 --- a/src/lstack/core/lstack_cfg.c +++ b/src/lstack/core/lstack_cfg.c @@ -85,6 +85,7 @@ static int32_t parse_defaule_nonblock_mode(void); static int32_t parse_rpc_msg_max(void); static int32_t parse_send_cache_mode(void); static int32_t parse_flow_bifurcation(void); +static int32_t parse_stack_interrupt(void); #define PARSE_ARG(_arg, _arg_string, _default_val, _min_val, _max_val, _ret) \ do { \ @@ -152,6 +153,7 @@ static struct config_vector_t g_config_tbl[] = { { "rpc_msg_max", parse_rpc_msg_max }, { "send_cache_mode", parse_send_cache_mode }, { "flow_bifurcation", parse_flow_bifurcation}, + { "stack_interrupt", parse_stack_interrupt}, { NULL, NULL } }; @@ -1383,3 +1385,26 @@ static int32_t parse_flow_bifurcation(void) PARSE_ARG(g_config_params.flow_bifurcation, "flow_bifurcation", 0, 0, 1, ret); return ret; } + +static int32_t parse_stack_interrupt(void) +{ + int32_t ret; + PARSE_ARG(g_config_params.stack_interrupt, "stack_interrupt", false, false, true, ret); + if (ret != 0) { + LSTACK_PRE_LOG(LSTACK_ERR, "cfg: invalid enable intr value %d. only support 0 or 1\n", \ + g_config_params.stack_interrupt); + } + + if (g_config_params.stack_interrupt == true) { + if (g_config_params.stack_mode_rtc == true) { + LSTACK_PRE_LOG(LSTACK_ERR, "rtc mode not support interrupt mode now.\n"); + return -1; + } + if (g_config_params.bond_mode >= 0) { + LSTACK_PRE_LOG(LSTACK_ERR, "bond mode not support interrupt mode.\n"); + return -1; + } + } + + return ret; +} diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c index 1fe0f0a..530332b 100644 --- a/src/lstack/core/lstack_dpdk.c +++ b/src/lstack/core/lstack_dpdk.c @@ -258,8 +258,8 @@ struct rte_mempool *create_mempool(const char *name, uint32_t count, uint32_t si int32_t create_shared_ring(struct protocol_stack *stack) { - rpc_queue_init(&stack->rpc_queue); - rpc_queue_init(&stack->dfx_rpc_queue); + rpc_queue_init(&stack->rpc_queue, stack->queue_id); + rpc_queue_init(&stack->dfx_rpc_queue, stack->queue_id); if (use_ltran()) { stack->rx_ring = gazelle_ring_create_fast("RING_RX", VDEV_RX_QUEUE_SZ, RING_F_SP_ENQ | RING_F_SC_DEQ); @@ -439,6 +439,7 @@ static int eth_params_init(struct eth_params *eth_params, uint16_t port_id, uint eth_params->conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE; /* used for tcp port alloc */ eth_params->reta_mask = dev_info.reta_size - 1; + eth_params->conf.intr_conf.rxq = get_global_cfg_params()->stack_interrupt; eth_params_checksum(ð_params->conf, &dev_info); @@ -630,10 +631,11 @@ static int32_t dpdk_ethdev_setup(const struct eth_params *eth_params, uint16_t i int32_t dpdk_ethdev_start(void) { + int i; int32_t ret; const struct protocol_stack_group *stack_group = get_protocol_stack_group(); - for (int32_t i = 0; i < get_global_cfg_params()->tot_queue_num; i++) { + for (i = 0; i < get_global_cfg_params()->tot_queue_num; i++) { ret = dpdk_ethdev_setup(stack_group->eth_params, i); if (ret < 0) { LSTACK_LOG(ERR, LSTACK, "dpdk_ethdev_setup fail queueid=%d, ret=%d\n", i, ret); @@ -647,6 +649,14 @@ int32_t dpdk_ethdev_start(void) return ret; } + /* after rte_eth_dev_start */ + for (i = 0; i < get_global_cfg_params()->tot_queue_num; i++) { + struct intr_dpdk_event_args intr_arg; + intr_arg.port_id = stack_group->eth_params->port_id; + intr_arg.queue_id = i; + intr_register(i, INTR_DPDK_EVENT, &intr_arg); + } + return 0; } @@ -799,10 +809,21 @@ int init_dpdk_ethdev(void) } port_id = rte_eth_bond_primary_get(get_protocol_stack_group()->port_id); } else { + struct rte_eth_dev_info dev_info; port_id = ethdev_port_id(cfg->mac_addr); if (port_id < 0) { return -1; } + + if (rte_eth_dev_info_get(port_id, &dev_info) < 0) { + return -1; + } + if (strcmp(dev_info.driver_name, "net_hinic") == 0 && + get_global_cfg_params()->stack_interrupt == true) { + LSTACK_LOG(ERR, LSTACK, "hinic not support interrupt mode\n"); + return -1; + } + ret = dpdk_ethdev_init(port_id); if (ret != 0) { LSTACK_LOG(ERR, LSTACK, "dpdk_ethdev_init failed, port id=%d\n", port_id); diff --git a/src/lstack/core/lstack_init.c b/src/lstack/core/lstack_init.c index 37264a1..5e405ee 100644 --- a/src/lstack/core/lstack_init.c +++ b/src/lstack/core/lstack_init.c @@ -45,6 +45,7 @@ #include "lstack_preload.h" #include "lstack_wrap.h" #include "lstack_flow.h" +#include "lstack_interrupt.h" static void check_process_start(void) { @@ -289,6 +290,10 @@ __attribute__((constructor)) void gazelle_network_init(void) LSTACK_EXIT(1, "stack_group_init failed\n"); } + if (intr_init() < 0) { + LSTACK_EXIT(1, "intr init failed\n"); + } + if (!use_ltran()) { if (init_dpdk_ethdev() != 0) { LSTACK_EXIT(1, "init_dpdk_ethdev failed\n"); diff --git a/src/lstack/core/lstack_interrupt.c b/src/lstack/core/lstack_interrupt.c new file mode 100644 index 0000000..26823cd --- /dev/null +++ b/src/lstack/core/lstack_interrupt.c @@ -0,0 +1,346 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +* gazelle is licensed under the Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +* PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "common/dpdk_common.h" +#include "common/gazelle_opt.h" +#include "common/gazelle_dfx_msg.h" +#include "lstack_log.h" +#include "lstack_cfg.h" +#include "lstack_interrupt.h" + +#define INTR_MAX_EVENT_NUM 8 + +struct intr_dpdk_event { + int rte_epfd; +#define INTR_PORT_NUM 2 +#define INTR_INVALID_PORT 65535 + uint16_t port_id[INTR_PORT_NUM]; /* 0: nic port id, 1: virtio_user port id */ + uint16_t queue_id[INTR_PORT_NUM]; +}; + +struct intr_local_event { + bool (*get_event) (uint16_t stack_id); +}; + +struct intr_remote_event { + int event_fd; +}; + +struct intr_policy { +#define INTR_LOOP_TIMES 5 + uint8_t no_event_cnt; +}; + +struct intr_config { + int epoll_fd; /* used for epoll */ + uint16_t stack_id; + bool in_wait; + + struct intr_dpdk_event dpdk_event; + struct intr_local_event local_event; + struct intr_remote_event remote_event; + + struct intr_policy policy; + struct interrupt_stats stats; +}; + +static struct intr_config g_intr_configs[PROTOCOL_STACK_MAX] = {0}; + +static inline struct intr_config *intr_config_get(uint16_t stack_id) +{ + return &g_intr_configs[stack_id]; +} + +int intr_init(void) +{ + int stack_id; + struct cfg_params *cfg = get_global_cfg_params(); + if (!cfg->stack_interrupt) { + return 0; + } + + for (stack_id = 0; stack_id < cfg->num_queue; stack_id++) { + struct intr_config *intr_config = intr_config_get(stack_id); + intr_config->epoll_fd = posix_api->epoll_create_fn(1); + if (intr_config->epoll_fd < 0) { + LSTACK_LOG(ERR, LSTACK, "epoll create fd fialed, errno is %d\n", errno); + return -1; + } + + for (int i = 0; i < INTR_PORT_NUM; i++) { + intr_config->dpdk_event.port_id[i] = INTR_INVALID_PORT; + } + + if (intr_register(stack_id, INTR_REMOTE_EVENT, NULL) < 0) { + LSTACK_LOG(ERR, LSTACK, "register intr failed\n"); + return -1; + } + } + + return 0; +} + +static inline int add_fd_to_epoll(int fd, int epoll_fd) +{ + struct epoll_event event ; + event.data.fd = fd ; + event.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP ; + int ret = posix_api->epoll_ctl_fn(epoll_fd, EPOLL_CTL_ADD, fd, &event); + if (ret < 0) { + LSTACK_LOG(ERR, LSTACK, "add fd %d to epoll fd %d failed errno:%d ret=%d.\n", + fd, epoll_fd, errno, ret); + return ret; + } + + return ret ; +} + +static inline int intr_local_event_register(struct intr_config *config, void *priv) +{ + config->local_event.get_event = priv; + return 0; +} + +static int intr_dpdk_event_register(struct intr_config *config, void *priv) +{ + struct intr_dpdk_event_args *arg = priv; + int i; + + if (arg == NULL) { + return -1; + } + + if (config->dpdk_event.rte_epfd <= 0) { + config->dpdk_event.rte_epfd = posix_api->epoll_create_fn(1); + if (config->dpdk_event.rte_epfd < 0) { + LSTACK_LOG(ERR, LSTACK, "epoll create fd fialed, errno is %d\n", errno); + return -1; + } + if (add_fd_to_epoll(config->dpdk_event.rte_epfd, config->epoll_fd) < 0) { + return -1; + } + } + + for (i = 0; i < INTR_PORT_NUM; i++) { + if (config->dpdk_event.port_id[i] == INTR_INVALID_PORT) { + config->dpdk_event.port_id[i] = arg->port_id; + break; + } + } + config->dpdk_event.queue_id[i] = arg->queue_id; + + int data = ((arg->port_id) << CHAR_BIT) | arg->queue_id; + if (rte_eth_dev_rx_intr_ctl_q(arg->port_id, arg->queue_id, config->dpdk_event.rte_epfd, + RTE_INTR_EVENT_ADD, (void *)((uintptr_t)data)) < 0) { + LSTACK_LOG(ERR, LSTACK, "rte_eth_dev_rx_intr_ctl_q failed, port(%d), queue(%d)\n", + arg->port_id, arg->queue_id); + return -1; + } + return 0; +} + +static int intr_remote_event_register(struct intr_config *config, void *priv) +{ + struct intr_remote_event *remote_event = &config->remote_event; + if (remote_event->event_fd > 0) { + return 0; + } + + remote_event->event_fd = posix_api->eventfd_fn(0, 0); + if (remote_event->event_fd < 0) { + LSTACK_LOG(ERR, LSTACK, "event fd create failed\n"); + return -1; + } + return add_fd_to_epoll(remote_event->event_fd, config->epoll_fd); +} + +int intr_register(uint16_t stack_id, enum intr_type type, void *priv) +{ + struct cfg_params *cfg = get_global_cfg_params(); + if (!cfg->stack_interrupt) { + return 0; + } + + struct intr_config *config = intr_config_get(stack_id); + switch (type) { + case INTR_DPDK_EVENT: + return intr_dpdk_event_register(config, priv); + case INTR_REMOTE_EVENT: + return intr_remote_event_register(config, priv); + case INTR_LOCAL_EVENT: + return intr_local_event_register(config, priv); + default: + return -1; + } + return 0; +} + +static inline void intr_remote_event_enable(struct intr_config *config) +{ + eventfd_t eventfd_num = 1; + if (__atomic_load_n(&config->in_wait, __ATOMIC_ACQUIRE)) { + posix_api->write_fn(config->remote_event.event_fd, &eventfd_num, sizeof(eventfd_t)); + } +} + +static inline void intr_remote_event_disable(struct intr_config *config) +{ + eventfd_t read_num; + posix_api->read_fn(config->remote_event.event_fd, &read_num, sizeof(eventfd_t)); +} + +static inline bool intr_local_event(struct intr_config *config) +{ + return config->local_event.get_event(config->stack_id); +} + +void intr_wakeup(uint16_t stack_id, enum intr_type type) +{ + if (!get_global_cfg_params()->stack_interrupt) { + return; + } + + struct intr_config *config = intr_config_get(stack_id); + switch (type) { + case INTR_REMOTE_EVENT: + intr_remote_event_enable(config); + break; + default: + break; + } +} + +static inline void intr_dpdk_event_enable(struct intr_config *config) +{ + int i; + int ret = 0; + struct intr_dpdk_event *dpdk_event = &config->dpdk_event; + + for (i = 0; i < INTR_PORT_NUM; i++) { + if (dpdk_event->port_id[i] != INTR_INVALID_PORT) { + ret = rte_eth_dev_rx_intr_enable(dpdk_event->port_id[i], dpdk_event->queue_id[i]); + if (ret != 0) { + LSTACK_LOG(ERR, LSTACK, "port(%d) queue(%d) enable interrupt failed\n", + dpdk_event->port_id[i], dpdk_event->queue_id[i]); + return; + } + } + } +} + +static inline void intr_dpdk_event_disable(struct intr_config *config) +{ + int i, n; + void *data; + uint16_t port_id; + uint16_t queue_id; + struct intr_dpdk_event *dpdk_event = &config->dpdk_event; + struct rte_epoll_event event[INTR_MAX_EVENT_NUM]; + + n = rte_epoll_wait(dpdk_event->rte_epfd, event, INTR_MAX_EVENT_NUM, 1); + for (i = 0; i < n; i++) { + data = event[i].epdata.data; + port_id = ((uintptr_t)data) >> CHAR_BIT; + queue_id = ((uintptr_t)data) & RTE_LEN2MASK(CHAR_BIT, uint8_t); + + rte_eth_dev_rx_intr_disable(port_id, queue_id); + + if (port_id == dpdk_event->port_id[0]) { + config->stats.nic_event_cnt++; + } else { + config->stats.virtio_user_event_cnt++; + } + } +} + +static inline void intr_policy_clear(struct intr_config *config) +{ + config->policy.no_event_cnt = 0; +} + +static inline bool intr_policy(struct intr_config *config) +{ + if (config->policy.no_event_cnt++ < INTR_LOOP_TIMES) { + return true; + } + config->policy.no_event_cnt = 0; + return false; +} + +static inline void intr_block(uint16_t stack_id, uint32_t timeout) +{ + struct epoll_event events[INTR_MAX_EVENT_NUM]; + struct intr_config *intr_config = intr_config_get(stack_id); + + /* in_wait need in here to avoid competion problem with remote event */ + __atomic_store_n(&intr_config->in_wait, true, __ATOMIC_RELEASE); + if (intr_local_event(intr_config)) { + intr_config->stats.local_event_cnt++; + __atomic_store_n(&intr_config->in_wait, false, __ATOMIC_RELEASE); + return; + } + + intr_dpdk_event_enable(intr_config); + + int32_t event_cnt = posix_api->epoll_wait_fn(intr_config->epoll_fd, events, INTR_MAX_EVENT_NUM, timeout); + __atomic_store_n(&intr_config->in_wait, false, __ATOMIC_RELEASE); + for (int i = 0; i < event_cnt; i++) { + if (events[i].data.fd == intr_config->dpdk_event.rte_epfd) { + intr_dpdk_event_disable(intr_config); + } else if (events[i].data.fd == intr_config->remote_event.event_fd) { + intr_remote_event_disable(intr_config); + intr_config->stats.remote_event_cnt++; + } else { + LSTACK_LOG(ERR, LSTACK, "unknow fd have event.\n"); + } + } + + if (event_cnt < 0) { + intr_config->stats.timeout_event_cnt++; + } +} + +void intr_wait(uint16_t stack_id, uint32_t timeout) +{ + struct intr_config *intr_config = intr_config_get(stack_id); + + if (intr_policy(intr_config)) { + return; + } + + intr_block(stack_id, timeout); + + intr_policy_clear(intr_config); +} + +int intr_stats_get(uint16_t stack_id, void *ptr, int len) +{ + struct intr_config *config = intr_config_get(stack_id); + if (len < sizeof(struct interrupt_stats)) { + return -1; + } + + return memcpy_s(ptr, len, &config->stats, sizeof(struct interrupt_stats)); +} diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c index d03b744..e412c3b 100644 --- a/src/lstack/core/lstack_protocol_stack.c +++ b/src/lstack/core/lstack_protocol_stack.c @@ -34,6 +34,8 @@ #include "lstack_virtio.h" #include "lstack_protocol_stack.h" +#include "lstack_interrupt.h" + #if RTE_VERSION < RTE_VERSION_NUM(23, 11, 0, 0) #include #endif @@ -480,13 +482,18 @@ int stack_polling(unsigned wakeup_tick) uint32_t rpc_number = cfg->rpc_number; uint32_t read_connect_number = cfg->read_connect_number; struct protocol_stack *stack = get_protocol_stack(); + uint32_t timeout; /* 2: one dfx consumes two rpc */ rpc_poll_msg(&stack->dfx_rpc_queue, 2); force_quit = rpc_poll_msg(&stack->rpc_queue, rpc_number); eth_dev_poll(); - sys_timer_run(); + timeout = sys_timer_run(); + if (cfg->stack_interrupt) { + intr_wait(stack->stack_idx, timeout); + } + if (cfg->low_power_mod != 0) { low_power_idling(stack); } @@ -496,6 +503,7 @@ int stack_polling(unsigned wakeup_tick) } do_lwip_read_recvlist(stack, read_connect_number); + if ((wakeup_tick & 0xf) == 0) { wakeup_stack_epoll(stack); if (get_global_cfg_params()->send_cache_mode) { @@ -533,6 +541,19 @@ int stack_polling(unsigned wakeup_tick) return force_quit; } +static bool stack_local_event_get(uint16_t stack_id) +{ + struct protocol_stack *stack = g_stack_group.stacks[stack_id]; + if (!lockless_queue_empty(&stack->dfx_rpc_queue.queue) || + !lockless_queue_empty(&stack->rpc_queue.queue) || + !list_head_empty(&stack->recv_list) || + !list_head_empty(&stack->wakeup_list) || + tx_cache_count(stack->queue_id)) { + return true; + } + return false; +} + static void* gazelle_stack_thread(void *arg) { struct thread_params *t_params = (struct thread_params*) arg; @@ -541,6 +562,7 @@ static void* gazelle_stack_thread(void *arg) unsigned wakeup_tick = 0; stack = stack_thread_init(arg); + intr_register(stack->stack_idx, INTR_LOCAL_EVENT, stack_local_event_get); free(arg); if (stack == NULL) { LSTACK_LOG(ERR, LSTACK, "stack_thread_init failed queue_id=%hu\n", queue_id); diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c index b6619f6..8efa5ab 100644 --- a/src/lstack/core/lstack_stack_stat.c +++ b/src/lstack/core/lstack_stack_stat.c @@ -325,6 +325,12 @@ static void get_stack_dfx_data(struct gazelle_stack_dfx_data *dfx, struct protoc LSTACK_LOG(ERR, LSTACK, "memcpy_s err ret=%d \n", ret); } break; + case GAZELLE_STAT_LSTACK_SHOW_INTR: + ret = intr_stats_get(stack->stack_idx, &dfx->data.intr_stats, sizeof(dfx->data.intr_stats)); + if (ret != EOK) { + LSTACK_LOG(ERR, LSTACK, "memcpy_s err ret=%d \n", ret); + } + break; case GAZELLE_STAT_LSTACK_SHOW_VIRTIO: ret = memcpy_s(&dfx->data.virtio, sizeof(dfx->data.virtio), virtio_instance_get(), sizeof(*(virtio_instance_get()))); diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c index b4a5953..7f77c12 100644 --- a/src/lstack/core/lstack_thread_rpc.c +++ b/src/lstack/core/lstack_thread_rpc.c @@ -100,7 +100,8 @@ __rte_always_inline static void rpc_async_call(rpc_queue *queue, struct rpc_msg *msg) { msg->sync_flag = 0; - lockless_queue_mpsc_push(queue, &msg->queue_node); + lockless_queue_mpsc_push(&queue->queue, &msg->queue_node); + intr_wakeup(queue->queue_id, INTR_REMOTE_EVENT); } __rte_always_inline @@ -111,7 +112,8 @@ static int rpc_sync_call(rpc_queue *queue, struct rpc_msg *msg) pthread_spin_trylock(&msg->lock); msg->sync_flag = 1; - lockless_queue_mpsc_push(queue, &msg->queue_node); + lockless_queue_mpsc_push(&queue->queue, &msg->queue_node); + intr_wakeup(queue->queue_id, INTR_REMOTE_EVENT); // waiting stack unlock pthread_spin_lock(&msg->lock); @@ -123,7 +125,7 @@ static int rpc_sync_call(rpc_queue *queue, struct rpc_msg *msg) int rpc_msgcnt(rpc_queue *queue) { - return lockless_queue_count(queue); + return lockless_queue_count(&queue->queue); } static struct rpc_msg *rpc_msg_alloc_except(rpc_func_t func) @@ -159,7 +161,7 @@ int rpc_poll_msg(rpc_queue *queue, int max_num) struct rpc_msg *msg; while (max_num--) { - lockless_queue_node *node = lockless_queue_mpsc_pop(queue); + lockless_queue_node *node = lockless_queue_mpsc_pop(&queue->queue); if (node == NULL) { break; } diff --git a/src/lstack/core/lstack_virtio.c b/src/lstack/core/lstack_virtio.c index fefb06d..75a23f2 100644 --- a/src/lstack/core/lstack_virtio.c +++ b/src/lstack/core/lstack_virtio.c @@ -20,6 +20,7 @@ #include "lstack_cfg.h" #include "lstack_log.h" #include "lstack_port_map.h" +#include "lstack_interrupt.h" #include "lstack_virtio.h" #include "securec.h" @@ -274,6 +275,7 @@ static int virtio_port_init(uint16_t port) return retval; } + port_conf.intr_conf.rxq = get_global_cfg_params()->stack_interrupt; retval = rte_eth_dev_configure(port, rx_queue_num, tx_queue_num, &port_conf); if (retval != 0) { LSTACK_LOG(ERR, LSTACK, "rte_eth_dev_configure failed retval=%d\n", retval); @@ -297,6 +299,13 @@ static int virtio_port_init(uint16_t port) LSTACK_LOG(ERR, LSTACK, "rte_eth_rx_queue_setup failed (queue %u) retval=%d \n", q, retval); return retval; } + + if (port_conf.intr_conf.rxq) { + struct intr_dpdk_event_args intr_arg; + intr_arg.port_id = port; + intr_arg.queue_id = q; + intr_register(q, INTR_DPDK_EVENT, &intr_arg); + } } return 0; } diff --git a/src/lstack/include/lstack_cfg.h b/src/lstack/include/lstack_cfg.h index 5e2d6fc..071492d 100644 --- a/src/lstack/include/lstack_cfg.h +++ b/src/lstack/include/lstack_cfg.h @@ -118,6 +118,7 @@ struct cfg_params { bool stack_mode_rtc; bool listen_shadow; // true:listen in all stack thread. false:listen in one stack thread. + bool stack_interrupt; uint32_t read_connect_number; uint32_t nic_read_number; diff --git a/src/lstack/include/lstack_interrupt.h b/src/lstack/include/lstack_interrupt.h new file mode 100644 index 0000000..1be7d07 --- /dev/null +++ b/src/lstack/include/lstack_interrupt.h @@ -0,0 +1,34 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +* gazelle is licensed under the Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +* PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + +#ifndef __LSTACK_INTERRUPT_H__ +#define __LSTACK_INTERRUPT_H__ + +enum intr_type { + INTR_DPDK_EVENT = 0, + INTR_LOCAL_EVENT, + INTR_REMOTE_EVENT, +}; + +struct intr_dpdk_event_args { + uint16_t port_id; + uint16_t queue_id; +}; + +int intr_init(void); +int intr_register(uint16_t stack_id, enum intr_type type, void *priv); +void intr_wakeup(uint16_t stack_id, enum intr_type type); +void intr_wait(uint16_t stack_id, uint32_t timeout); +int intr_stats_get(uint16_t stack_id, void *ptr, int len); + +#endif + diff --git a/src/lstack/include/lstack_thread_rpc.h b/src/lstack/include/lstack_thread_rpc.h index 6f8e03e..c284d29 100644 --- a/src/lstack/include/lstack_thread_rpc.h +++ b/src/lstack/include/lstack_thread_rpc.h @@ -17,6 +17,7 @@ #include #include "lstack_lockless_queue.h" +#include "lstack_interrupt.h" #define MSG_ARG_0 (0) #define MSG_ARG_1 (1) @@ -25,7 +26,11 @@ #define MSG_ARG_4 (4) #define RPM_MSG_ARG_SIZE (5) -typedef struct lockless_queue rpc_queue; +typedef struct rpc_queue rpc_queue; +struct rpc_queue { + struct lockless_queue queue; + uint16_t queue_id; +}; struct rpc_stats { uint16_t call_null; @@ -60,9 +65,10 @@ struct rpc_msg { lockless_queue_node queue_node; }; -static inline void rpc_queue_init(rpc_queue *queue) +static inline void rpc_queue_init(rpc_queue *queue, uint16_t queue_id) { - lockless_queue_init(queue); + lockless_queue_init(&queue->queue); + queue->queue_id = queue_id; } struct rpc_stats *rpc_stats_get(void); int rpc_msgcnt(rpc_queue *queue); diff --git a/src/lstack/include/lstack_tx_cache.h b/src/lstack/include/lstack_tx_cache.h index 04e9e35..ccd9c17 100644 --- a/src/lstack/include/lstack_tx_cache.h +++ b/src/lstack/include/lstack_tx_cache.h @@ -15,5 +15,6 @@ int tx_cache_init(uint16_t queue_id, void *priv, struct lstack_dev_ops *dev_ops); int tx_cache_send(uint16_t queue_id); +int tx_cache_count(uint16_t queue_id); #endif /* _LSTACK_TX_CACHE_H_ */ diff --git a/src/lstack/include/lstack_virtio.h b/src/lstack/include/lstack_virtio.h index 5298dbe..745c86b 100644 --- a/src/lstack/include/lstack_virtio.h +++ b/src/lstack/include/lstack_virtio.h @@ -50,4 +50,4 @@ int virtio_port_create(int lstack_net_port); struct virtio_instance* virtio_instance_get(void); bool virtio_distribute_pkg_to_kernel(uint16_t dst_port); -#endif \ No newline at end of file +#endif diff --git a/src/lstack/netif/lstack_tx_cache.c b/src/lstack/netif/lstack_tx_cache.c index 9a48307..17d3c3b 100644 --- a/src/lstack/netif/lstack_tx_cache.c +++ b/src/lstack/netif/lstack_tx_cache.c @@ -58,22 +58,40 @@ int tx_cache_init(uint16_t queue_id, void *priv, struct lstack_dev_ops *dev_ops) return 0; } +int tx_cache_count(uint16_t queue_id) +{ + struct tx_cache *tx_cache = g_tx_cache[queue_id]; + if (tx_cache == NULL) { + return 0; + } + uint32_t start = tx_cache->send_start; + uint32_t end = tx_cache->send_end; + uint32_t count = (end - start) & TX_CACHE_MASK; + uint32_t capacity = TX_CACHE_MAX - 1; + + return (count > capacity) ? capacity : count; +} + int tx_cache_send(uint16_t queue_id) { + uint32_t send_num; + uint32_t sent_pkts = 0; + uint32_t start; + uint32_t end; + struct tx_cache *tx_cache = g_tx_cache[queue_id]; if (tx_cache == NULL) { LSTACK_LOG(ERR, LSTACK, "queue(%d) tx cache get failed\n", queue_id); return 0; } - uint32_t send_num = tx_cache->send_end - tx_cache->send_start; + send_num = tx_cache_count(queue_id); if (send_num == 0) { return 0; } - uint32_t start = tx_cache->send_start & TX_CACHE_MASK; - uint32_t end = tx_cache->send_end & TX_CACHE_MASK; - uint32_t sent_pkts = 0; + start = tx_cache->send_start & TX_CACHE_MASK; + end = tx_cache->send_end & TX_CACHE_MASK; if (start < end) { sent_pkts = g_tx_cache_dev_ops.tx_xmit(tx_cache->priv, &tx_cache->send_pkts[start], send_num); } else { @@ -85,6 +103,7 @@ int tx_cache_send(uint16_t queue_id) } tx_cache->send_start += sent_pkts; + return sent_pkts; } diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c index e1a63a7..290046e 100644 --- a/src/lstack/netif/lstack_vdev.c +++ b/src/lstack/netif/lstack_vdev.c @@ -38,6 +38,8 @@ #include "lstack_port_map.h" #include "lstack_virtio.h" +#include "lstack_interrupt.h" + /* INUSE_TX_PKTS_WATERMARK < VDEV_RX_QUEUE_SZ; * USE_RX_PKTS_WATERMARK < FREE_RX_QUEUE_SZ. * less, means more available mbuf. diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c index 7fa117a..f6d1148 100644 --- a/src/ltran/ltran_dfx.c +++ b/src/ltran/ltran_dfx.c @@ -138,6 +138,7 @@ static void gazelle_print_lstack_xstats(void *buf, const struct gazelle_stat_msg static void gazelle_print_lstack_aggregate(void *buf, const struct gazelle_stat_msg_request *req_msg); static void gazelle_print_lstack_nic_features(void *buf, const struct gazelle_stat_msg_request *req_msg); static void gazelle_print_lstack_stat_proto(void *buf, const struct gazelle_stat_msg_request *req_msg); +static void gazelle_print_lstack_stat_intr(void *buf, const struct gazelle_stat_msg_request *req_msg); #ifdef GAZELLE_FAULT_INJECT_ENABLE static void gazelle_print_fault_inject_set_status(void *buf, const struct gazelle_stat_msg_request *req_msg); @@ -172,6 +173,7 @@ static struct gazelle_dfx_list g_gazelle_dfx_tbl[] = { {GAZELLE_STAT_LSTACK_SHOW_AGGREGATE, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_aggregate}, {GAZELLE_STAT_LSTACK_SHOW_NIC_FEATURES, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_nic_features}, {GAZELLE_STAT_LSTACK_SHOW_PROTOCOL, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_stat_proto}, + {GAZELLE_STAT_LSTACK_SHOW_INTR, sizeof(struct gazelle_stack_dfx_data), gazelle_print_lstack_stat_intr}, #ifdef GAZELLE_FAULT_INJECT_ENABLE {GAZELLE_STAT_FAULT_INJECT_SET, sizeof(struct gazelle_stack_dfx_data), gazelle_print_fault_inject_set_status}, @@ -1136,6 +1138,17 @@ static void gazelle_print_lstack_stat_proto_core(const struct gazelle_stack_dfx_ printf("rterr: %lu\n", proto->rterr); } +static void gazelle_print_lstack_stat_intr_core(const struct gazelle_stack_dfx_data *stat, + const struct interrupt_stats *intr_stats) +{ + printf("\n------ stack tid: %6u ------\n", stat->tid); + printf("nic_event_cnt: %lu\n", intr_stats->nic_event_cnt); + printf("virtio_user_event_cnt: %lu\n", intr_stats->virtio_user_event_cnt); + printf("local_event_cnt: %lu\n", intr_stats->local_event_cnt); + printf("remote_event_cnt: %lu\n", intr_stats->remote_event_cnt); + printf("timeout_event_cnt: %lu\n", intr_stats->timeout_event_cnt); +} + static void gazelle_print_lstack_stat_snmp(void *buf, const struct gazelle_stat_msg_request *req_msg) { int32_t ret; @@ -1155,6 +1168,25 @@ static void gazelle_print_lstack_stat_snmp(void *buf, const struct gazelle_stat_ } while (true); } +static void gazelle_print_lstack_stat_intr(void *buf, const struct gazelle_stat_msg_request *req_msg) +{ + int32_t ret; + struct gazelle_stack_dfx_data *stat = (struct gazelle_stack_dfx_data *)buf; + struct interrupt_stats *intr_stats = &stat->data.intr_stats; + + printf("Statistics of lstack interrupt:\n"); + do { + gazelle_print_lstack_stat_intr_core(stat, intr_stats); + if (stat->eof != 0) { + break; + } + ret = dfx_stat_read_from_ltran(buf, sizeof(struct gazelle_stack_dfx_data), req_msg->stat_mode); + if (ret != GAZELLE_OK) { + return; + } + } while (true); +} + static void gazelle_print_lstack_stat_proto(void *buf, const struct gazelle_stat_msg_request *req_msg) { int32_t ret; @@ -1579,6 +1611,8 @@ static int32_t parse_dfx_lstack_show_args(int32_t argc, char *argv[], struct gaz req_msg[cmd_index++].stat_mode = GAZELLE_STAT_MODE_MAX; } else if (strcmp(param, "snmp") == 0 || strcmp(param, "-s") == 0) { req_msg[cmd_index++].stat_mode = GAZELLE_STAT_LSTACK_SHOW_SNMP; + } else if (strcmp(param, "intr") == 0 || strcmp(param, "-I") == 0) { + req_msg[cmd_index++].stat_mode = GAZELLE_STAT_LSTACK_SHOW_INTR; } else if (strcmp(param, "virtio") == 0 || strcmp(param, "-v") == 0) { req_msg[cmd_index++].stat_mode = GAZELLE_STAT_LSTACK_SHOW_VIRTIO; } else if (strcmp(param, "connect") == 0 || strcmp(param, "-c") == 0) { -- 2.33.0