From ac2c6340d9e08b94bf99682e12ab5d5836d2140d Mon Sep 17 00:00:00 2001
From: wu-changsheng <[email protected]>
Date: Mon, 31 Oct 2022 22:07:26 +0800
Subject: [PATCH 5/7] optimize variable access

---
 src/common/gazelle_opt.h                   |   1 +
 src/lstack/api/lstack_wrap.c               |   8 +-
 src/lstack/core/lstack_lwip.c              | 102 +++++++++------------
 src/lstack/core/lstack_protocol_stack.c    |  24 +++--
 src/lstack/core/lstack_thread_rpc.c        |   2 +
 src/lstack/include/lstack_protocol_stack.h |   1 +
 src/lstack/netif/lstack_ethdev.c           |   5 +-
 src/lstack/netif/lstack_vdev.c             |   7 +-
 8 files changed, 67 insertions(+), 83 deletions(-)

diff --git a/src/common/gazelle_opt.h b/src/common/gazelle_opt.h
index 011553c..8ab40ed 100644
--- a/src/common/gazelle_opt.h
+++ b/src/common/gazelle_opt.h
@@ -31,6 +31,7 @@
 #define GAZELLE_MBUFF_PRIV_SIZE     (sizeof(uint64_t) * 2)
 
 #define DEFAULT_RING_SIZE           (512)
+#define DEFAULT_RING_MASK           (511)
 #define DEFAULT_BACKUP_RING_SIZE_FACTOR   (16)
 
 #define VDEV_RX_QUEUE_SZ            DEFAULT_RING_SIZE
diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c
index 9672d3d..1c7a722 100644
--- a/src/lstack/api/lstack_wrap.c
+++ b/src/lstack/api/lstack_wrap.c
@@ -46,6 +46,10 @@ bool select_thread_path(void);
 
 static enum KERNEL_LWIP_PATH select_path(int fd)
 {
+    if (!select_thread_path()) {
+        return PATH_KERNEL;
+    }
+
     if (unlikely(posix_api == NULL)) {
         /* posix api maybe call before gazelle init */
         if (posix_api_init() != 0) {
@@ -54,10 +58,6 @@ static enum KERNEL_LWIP_PATH select_path(int fd)
         return PATH_KERNEL;
     }
 
-    if (!select_thread_path()) {
-        return PATH_KERNEL;
-    }
-
     if (unlikely(posix_api->ues_posix)) {
         return PATH_KERNEL;
     }
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index 3a1eb81..f924ee7 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -37,8 +37,6 @@
 #define HALF_DIVISOR                    (2)
 #define USED_IDLE_WATERMARK             (VDEV_IDLE_QUEUE_SZ >> 2)
 
-static int32_t lwip_alloc_pbufs(pbuf_layer layer, uint16_t length, pbuf_type type, void **pbufs, uint32_t num);
-
 static void free_ring_pbuf(struct rte_ring *ring)
 {
     void *pbufs[SOCK_RECV_RING_SIZE];
@@ -85,17 +83,38 @@ static void reset_sock_data(struct lwip_sock *sock)
     sock->recv_lastdata = NULL;
 }
 
-static void replenish_send_idlembuf(struct rte_ring *ring)
+static struct pbuf *init_mbuf_to_pbuf(struct rte_mbuf *mbuf, pbuf_layer layer, uint16_t length, pbuf_type type)
+{
+    struct pbuf_custom *pbuf_custom = mbuf_to_pbuf(mbuf);
+    pbuf_custom->custom_free_function = gazelle_free_pbuf;
+
+    void *data = rte_pktmbuf_mtod(mbuf, void *);
+    struct pbuf *pbuf = pbuf_alloced_custom(layer, length, type, pbuf_custom, data, MAX_PACKET_SZ);
+    if (pbuf) {
+        pbuf->ol_flags = 0;
+        pbuf->l2_len = 0;
+        pbuf->l3_len = 0;
+    }
+
+    return pbuf;
+}
+
+static void replenish_send_idlembuf(struct protocol_stack *stack, struct rte_ring *ring)
 {
     void *pbuf[SOCK_SEND_RING_SIZE];
 
     uint32_t replenish_cnt = gazelle_ring_free_count(ring);
 
     uint32_t alloc_num = LWIP_MIN(replenish_cnt, RING_SIZE(SOCK_SEND_RING_SIZE));
-    if (lwip_alloc_pbufs(PBUF_TRANSPORT, TCP_MSS, PBUF_RAM, (void **)pbuf, alloc_num) != 0) {
+    if (rte_pktmbuf_alloc_bulk(stack->tx_pktmbuf_pool, (struct rte_mbuf **)pbuf, alloc_num) != 0) {
+        stack->stats.tx_allocmbuf_fail++;
         return;
     }
 
+    for (uint32_t i = 0; i < alloc_num; i++) {
+        pbuf[i] = init_mbuf_to_pbuf(pbuf[i], PBUF_TRANSPORT, TCP_MSS, PBUF_RAM);
+    }
+
     uint32_t num = gazelle_ring_sp_enqueue(ring, pbuf, alloc_num);
     for (uint32_t i = num; i < alloc_num; i++) {
         pbuf_free(pbuf[i]);
@@ -126,7 +145,7 @@ void gazelle_init_sock(int32_t fd)
         LSTACK_LOG(ERR, LSTACK, "sock_send create failed. errno: %d.\n", rte_errno);
         return;
     }
-    replenish_send_idlembuf(sock->send_ring);
+    replenish_send_idlembuf(stack, sock->send_ring);
 
     sock->stack = stack;
     sock->stack->conn_num++;
@@ -183,46 +202,17 @@ int32_t gazelle_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs,
     return 0;
 }
 
-static struct pbuf *init_mbuf_to_pbuf(struct rte_mbuf *mbuf, pbuf_layer layer, uint16_t length, pbuf_type type)
-{
-    struct pbuf_custom *pbuf_custom = mbuf_to_pbuf(mbuf);
-    pbuf_custom->custom_free_function = gazelle_free_pbuf;
-
-    void *data = rte_pktmbuf_mtod(mbuf, void *);
-    struct pbuf *pbuf = pbuf_alloced_custom(layer, length, type, pbuf_custom, data, MAX_PACKET_SZ);
-    if (pbuf) {
-        pbuf->ol_flags = 0;
-        pbuf->l2_len = 0;
-        pbuf->l3_len = 0;
-    }
-
-    return pbuf;
-}
-
-static int32_t lwip_alloc_pbufs(pbuf_layer layer, uint16_t length, pbuf_type type, void **bufs, uint32_t num)
-{
-    int32_t ret = rte_pktmbuf_alloc_bulk(get_protocol_stack()->tx_pktmbuf_pool, (struct rte_mbuf **)bufs, num);
-    if (ret != 0) {
-        get_protocol_stack()->stats.tx_allocmbuf_fail++;
-        return -1;
-    }
-
-    for (uint32_t i = 0; i < num; i++) {
-        bufs[i] = init_mbuf_to_pbuf(bufs[i], layer, length, type);
-    }
-
-    return 0;
-}
-
 struct pbuf *lwip_alloc_pbuf(pbuf_layer layer, uint16_t length, pbuf_type type)
 {
-    struct pbuf *pbuf;
+    struct rte_mbuf *mbuf;
+    struct protocol_stack *stack = get_protocol_stack();
 
-    if (lwip_alloc_pbufs(layer, length, type, (void **)&pbuf, 1) != 0) {
+    if (rte_pktmbuf_alloc_bulk(stack->tx_pktmbuf_pool, &mbuf, 1) != 0) {
+        stack->stats.tx_allocmbuf_fail++;
         return NULL;
     }
 
-    return pbuf;
+    return init_mbuf_to_pbuf(mbuf, layer, length, type);
 }
 
 struct pbuf *write_lwip_data(struct lwip_sock *sock, uint16_t remain_size, uint8_t *apiflags)
@@ -324,7 +314,7 @@ ssize_t write_stack_data(struct lwip_sock *sock, const void *buf, size_t len)
     return send_len;
 }
 
-static void do_lwip_send(int32_t fd, struct lwip_sock *sock, int32_t flags)
+static void do_lwip_send(struct protocol_stack *stack, int32_t fd, struct lwip_sock *sock, int32_t flags)
 {
     /* send all send_ring, so len set lwip send max. */
     ssize_t len = lwip_send(fd, sock, UINT16_MAX, flags);
@@ -335,7 +325,7 @@ static void do_lwip_send(int32_t fd, struct lwip_sock *sock, int32_t flags)
     }
 
     if (gazelle_ring_readable_count(sock->send_ring) < SOCK_SEND_REPLENISH_THRES) {
-        replenish_send_idlembuf(sock->send_ring);
+        replenish_send_idlembuf(stack, sock->send_ring);
     }
 
     if ((sock->epoll_events & EPOLLOUT) && NETCONN_IS_OUTIDLE(sock)) {
@@ -347,8 +337,7 @@ void stack_send(struct rpc_msg *msg)
 {
     int32_t fd = msg->args[MSG_ARG_0].i;
     int32_t flags = msg->args[MSG_ARG_2].i;
-
-    struct protocol_stack *stack = get_protocol_stack();
+    struct protocol_stack *stack = (struct protocol_stack *)msg->args[MSG_ARG_3].p;
 
     struct lwip_sock *sock = get_socket(fd);
     if (sock == NULL) {
@@ -363,7 +352,7 @@ void stack_send(struct rpc_msg *msg)
         return;
     }
 
-    do_lwip_send(fd, sock, flags);
+    do_lwip_send(stack, fd, sock, flags);
 
     /* have remain data add sendlist */
     if (NETCONN_IS_DATAOUT(sock)) {
@@ -392,7 +381,7 @@ void send_stack_list(struct protocol_stack *stack, uint32_t send_max)
             continue;
         }
 
-        do_lwip_send(sock->conn->socket, sock, 0);
+        do_lwip_send(stack, sock->conn->socket, sock, 0);
 
         if (!NETCONN_IS_DATAOUT(sock)) {
             list_del_node_null(&sock->send_list);
@@ -542,11 +531,7 @@ ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags)
         return 0;
     }
 
-    struct lwip_sock *sock = get_socket(fd);
-    if (sock == NULL) {
-        GAZELLE_RETURN(EINVAL);
-    }
-
+    struct lwip_sock *sock = get_socket_by_fd(fd);
     ssize_t send = write_stack_data(sock, buf, len);
     if (send <= 0) {
         return send;
@@ -562,11 +547,7 @@ ssize_t sendmsg_to_stack(int32_t s, const struct msghdr *message, int32_t flags)
     int32_t ret;
     int32_t i;
     ssize_t buflen = 0;
-
-    struct lwip_sock *sock = get_socket(s);
-    if (sock == NULL) {
-        GAZELLE_RETURN(EINVAL);
-    }
+    struct lwip_sock *sock = get_socket_by_fd(s);
 
     if (check_msg_vaild(message)) {
         GAZELLE_RETURN(EINVAL);
@@ -635,17 +616,16 @@ ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags)
     struct pbuf *pbuf = NULL;
     ssize_t recvd = 0;
     uint16_t copy_len;
-
-    struct lwip_sock *sock = get_socket(fd);
-    if (sock == NULL) {
-        LSTACK_LOG(ERR, LSTACK, "get_socket null fd %d.\n", fd);
-        GAZELLE_RETURN(EINVAL);
-    }
+    struct lwip_sock *sock = get_socket_by_fd(fd);
 
     if (sock->errevent > 0 && !NETCONN_IS_DATAIN(sock)) {
         return 0;
     }
 
+    if (recv_left > UINT16_MAX) {
+        recv_left = UINT16_MAX;
+    }
+
     while (recv_left > 0) {
         if (sock->recv_lastdata) {
             pbuf = sock->recv_lastdata;
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index 41dda89..2759d7d 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -36,9 +36,9 @@
 #include "posix/lstack_epoll.h"
 #include "lstack_stack_stat.h"
 
-#define READ_LIST_MAX                   128
-#define SEND_LIST_MAX                   128
-#define HANDLE_RPC_MSG_MAX              128
+#define READ_LIST_MAX                   32 
+#define SEND_LIST_MAX                   32 
+#define HANDLE_RPC_MSG_MAX              32 
 #define KERNEL_EVENT_100us              100
 
 static PER_THREAD struct protocol_stack *g_stack_p = NULL;
@@ -164,7 +164,7 @@ void low_power_idling(struct protocol_stack *stack)
         last_cycle_ts = sys_now();
     }
 
-    uint64_t now_pkts = get_protocol_stack()->stats.rx;
+    uint64_t now_pkts = stack->stats.rx;
     uint32_t now_ts = sys_now();
     if (((now_ts - last_cycle_ts) > LSTACK_LPM_DETECT_MS) ||
         ((now_pkts - last_cycle_pkts) >= LSTACK_LPM_PKTS_IN_DETECT)) {
@@ -258,7 +258,9 @@ static void* gazelle_kernelevent_thread(void *arg)
     uint16_t queue_id = *(uint16_t *)arg;
     struct protocol_stack *stack = get_protocol_stack_group()->stacks[queue_id];
 
-    bind_to_stack_numa(stack);
+    if (get_global_cfg_params()->app_bind_numa) {
+        bind_to_stack_numa(stack);
+    }
 
     LSTACK_LOG(INFO, LSTACK, "kernelevent_%02hu start\n", queue_id);
 
@@ -420,6 +422,7 @@ static void* gazelle_stack_thread(void *arg)
 {
     uint16_t queue_id = *(uint16_t *)arg;
     bool use_ltran_flag = use_ltran();
+    uint32_t wakeup_tick = 0;
 
     struct protocol_stack *stack = stack_thread_init(queue_id);
     if (stack == NULL) {
@@ -442,9 +445,11 @@ static void* gazelle_stack_thread(void *arg)
 
         send_stack_list(stack, SEND_LIST_MAX);
 
-        wakeup_kernel_event(stack);
-
-        wakeup_stack_epoll(stack);
+        if ((wakeup_tick & 0xf) == 0) {
+            wakeup_kernel_event(stack);
+            wakeup_stack_epoll(stack);
+        }
+        wakeup_tick++;
 
         sys_timer_run();
 
@@ -523,8 +528,9 @@ int32_t init_protocol_stack(void)
 void stack_arp(struct rpc_msg *msg)
 {
     struct rte_mbuf *mbuf = (struct rte_mbuf *)msg->args[MSG_ARG_0].p;
+    struct protocol_stack *stack = (struct protocol_stack*)msg->args[MSG_ARG_1].p;
 
-    eth_dev_recv(mbuf, NULL);
+    eth_dev_recv(mbuf, stack);
 }
 
 void stack_socket(struct rpc_msg *msg)
diff --git a/src/lstack/core/lstack_thread_rpc.c b/src/lstack/core/lstack_thread_rpc.c
index db1de5a..295baf3 100644
--- a/src/lstack/core/lstack_thread_rpc.c
+++ b/src/lstack/core/lstack_thread_rpc.c
@@ -234,6 +234,7 @@ int32_t rpc_call_arp(struct protocol_stack *stack, struct rte_mbuf *mbuf)
 
     msg->self_release = 0;
     msg->args[MSG_ARG_0].p = mbuf;
+    msg->args[MSG_ARG_1].p = stack;
 
     rpc_call(&stack->rpc_queue, msg);
 
@@ -451,6 +452,7 @@ int32_t rpc_call_send(int fd, const void *buf, size_t len, int flags)
     msg->args[MSG_ARG_0].i = fd;
     msg->args[MSG_ARG_1].size = len;
     msg->args[MSG_ARG_2].i = flags;
+    msg->args[MSG_ARG_3].p = stack;
     msg->self_release = 0;
 
     rpc_call(&stack->rpc_queue, msg);
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index cc2cfb9..fed1882 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -49,6 +49,7 @@ struct protocol_stack {
     struct rte_ring *reg_ring;
     struct rte_ring *wakeup_ring;
     struct reg_ring_msg *reg_buf;
+    uint32_t reg_head;
 
     volatile bool low_power;
     lockless_queue rpc_queue __rte_cache_aligned;
diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c
index 5ddc0db..3abed5e 100644
--- a/src/lstack/netif/lstack_ethdev.c
+++ b/src/lstack/netif/lstack_ethdev.c
@@ -39,9 +39,6 @@ void eth_dev_recv(struct rte_mbuf *mbuf, struct protocol_stack *stack)
     struct pbuf *prev = NULL;
     struct pbuf *head = NULL;
     struct pbuf_custom *pc = NULL;
-    if (!stack) {
-        stack = get_protocol_stack();
-    }
     struct rte_mbuf *m = mbuf;
     uint16_t len, pkt_len;
 
@@ -81,7 +78,7 @@ void eth_dev_recv(struct rte_mbuf *mbuf, struct protocol_stack *stack)
     }
 }
 
-#define READ_PKTS_MAX   128
+#define READ_PKTS_MAX   32
 int32_t eth_dev_poll(void)
 {
     uint32_t nr_pkts;
diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c
index f9fa5a3..1c148e1 100644
--- a/src/lstack/netif/lstack_vdev.c
+++ b/src/lstack/netif/lstack_vdev.c
@@ -113,7 +113,6 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple)
     uint32_t sent_pkts = 0;
     void *free_buf[VDEV_REG_QUEUE_SZ];
     struct reg_ring_msg *tmp_buf = NULL;
-    static PER_THREAD uint32_t head = 0;
     const uint32_t tbegin = sys_now();
     struct protocol_stack *stack = get_protocol_stack();
 
@@ -124,6 +123,7 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple)
         }
     }
 
+    uint32_t reg_index = stack->reg_head++ & DEFAULT_RING_MASK;
     do {
         (void)gazelle_ring_sc_dequeue(stack->reg_ring, free_buf, VDEV_REG_QUEUE_SZ);
 
@@ -131,7 +131,7 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple)
             continue;
         }
 
-        tmp_buf = &stack->reg_buf[head];
+        tmp_buf = &stack->reg_buf[reg_index];
         tmp_buf->type = type;
         tmp_buf->tid = get_stack_tid();
         ret = memcpy_s(&tmp_buf->qtuple, sizeof(*qtuple), qtuple, sizeof(struct gazelle_quintuple));
@@ -144,9 +144,6 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple)
         sent_pkts = gazelle_ring_sp_enqueue(stack->reg_ring, free_buf, 1);
     } while ((sent_pkts < 1) && (ENQUEUE_RING_RETRY_TIMEOUT > sys_now() - tbegin) && get_register_state());
 
-    if (sent_pkts == 1) {
-        head = (head + 1) % VDEV_REG_QUEUE_SZ;
-    }
     return (int32_t)sent_pkts;
 }
 
-- 
2.23.0