From f9cb1b86511fac1c3a7f11fdaa4c9c20dc889068 Mon Sep 17 00:00:00 2001
From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
Date: Sun, 9 Oct 2022 19:00:38 +0800
Subject: [PATCH 01/35] use epoll instead of select for wait_exit_fifo

Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
---
 .../modules/service/service_container.c       | 73 ++++++++++++++-----
 src/utils/cutils/mainloop.c                   |  5 ++
 src/utils/cutils/mainloop.h                   |  4 +
 3 files changed, 62 insertions(+), 20 deletions(-)

diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
index 2f688f57..2b3c8794 100644
--- a/src/daemon/modules/service/service_container.c
+++ b/src/daemon/modules/service/service_container.c
@@ -52,6 +52,7 @@
 #include "runtime_api.h"
 #include "error.h"
 #include "io_handler.h"
+#include "mainloop.h"
 #include "constants.h"
 #include "event_type.h"
 #include "utils_array.h"
@@ -662,37 +663,69 @@ static int verify_mounts(const container_t *cont)
     return 0;
 }
 
-static void wait_exit_fifo(const char *id, const int exit_fifo_fd)
+static int wait_exit_fifo_epoll_callback(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
 {
-    int nret = 0;
+    int ret = EPOLL_LOOP_HANDLE_CLOSE;
     int exit_code = 0;
-    const int WAIT_TIMEOUT = 3;
-    fd_set set;
-    struct timeval timeout;
+    char *container_id = cbdata;
 
-    FD_ZERO(&set);
-    FD_SET(exit_fifo_fd, &set);
+    if (util_read_nointr(fd, &exit_code, sizeof(int)) <= 0) {
+        ERROR("Failed to read exit fifo fd for container %s", container_id);
+        return ret;
+    }
 
-    timeout.tv_sec = WAIT_TIMEOUT;
-    timeout.tv_usec = 0;
+    ERROR("The container %s 's monitor on fd %d has exited: %d", container_id, fd, exit_code);
+    return ret;
+}
 
-    nret = select(exit_fifo_fd + 1, &set, NULL, NULL, &timeout);
-    if (nret < 0) {
-        ERROR("Wait containers %s 's monitor on fd %d error: %s", id, exit_fifo_fd,
-              errno ? strerror(errno) : "");
+static void wait_exit_fifo_timeout_callback(void *cbdata)
+{
+    char *container_id = NULL;
+
+    if (cbdata == NULL) {
+        ERROR("Invalid cbdata");
         return;
     }
+    container_id = (char *)cbdata;
 
-    if (nret == 0) {
-        // timeout
-        // maybe monitor still cleanup cgroup and processes,
-        // or monitor doesn't step in cleanup at all
-        ERROR("Wait containers %s 's monitor on fd %d timeout", id, exit_fifo_fd);
+    // timeout
+    // maybe monitor still cleanup cgroup and processes,
+    // or monitor doesn't step in cleanup at all
+    ERROR("Wait container %s 's monitor timeout", container_id);
+}
+
+static void wait_exit_fifo(const char *id, const int exit_fifo_fd)
+{
+    int nret = 0;
+    const int WAIT_TIMEOUT = 3000;
+    char *container_id = NULL;
+    struct epoll_descr descr = { 0 };
+
+    nret = epoll_loop_open(&descr);
+    if (nret != 0) {
+        ERROR("Failed to create epoll for container %s", id);
         return;
     }
 
-    (void)util_read_nointr(exit_fifo_fd, &exit_code, sizeof(int));
-    ERROR("The container %s 's monitor on fd %d has exited: %d", id, exit_fifo_fd, exit_code);
+    container_id = util_strdup_s(id);
+    nret = epoll_loop_add_handler(&descr, exit_fifo_fd, wait_exit_fifo_epoll_callback, container_id);
+    if (nret != 0) {
+        ERROR("Failed to add epoll handler for container %s", id);
+        goto out;
+    }
+
+    descr.timeout_cb = wait_exit_fifo_timeout_callback;
+    descr.timeout_cbdata = container_id;
+    nret = epoll_loop(&descr, WAIT_TIMEOUT);
+    if (nret != 0) {
+        ERROR("Wait container %s 's monitor on fd %d error: %s", id, exit_fifo_fd,
+              errno ? strerror(errno) : "");
+        goto out;
+    }
+
+out:
+    free(container_id);
+    epoll_loop_close(&descr);
 }
 
 static int do_start_container(container_t *cont, const char *console_fifos[], bool reset_rm, pid_ppid_info_t *pid_info)
diff --git a/src/utils/cutils/mainloop.c b/src/utils/cutils/mainloop.c
index cf0b41a3..1028087a 100644
--- a/src/utils/cutils/mainloop.c
+++ b/src/utils/cutils/mainloop.c
@@ -56,6 +56,9 @@ int epoll_loop(struct epoll_descr *descr, int t)
         }
 
         if (ep_fds == 0 && t != 0) {
+            if (descr->timeout_cb != NULL) {
+                descr->timeout_cb(descr->timeout_cbdata);
+            }
             goto out;
         }
 
@@ -139,6 +142,8 @@ int epoll_loop_open(struct epoll_descr *descr)
     }
 
     linked_list_init(&(descr->handler_list));
+    descr->timeout_cb = NULL;
+    descr->timeout_cbdata = NULL;
     return 0;
 }
 
diff --git a/src/utils/cutils/mainloop.h b/src/utils/cutils/mainloop.h
index b9099a74..5124b33a 100644
--- a/src/utils/cutils/mainloop.h
+++ b/src/utils/cutils/mainloop.h
@@ -18,9 +18,13 @@
 #include <stdint.h>
 #include "linked_list.h"
 
+typedef void (*epoll_timeout_callback_t)(void *data);
+
 struct epoll_descr {
     int fd;
     struct linked_list handler_list;
+    epoll_timeout_callback_t timeout_cb;
+    void *timeout_cbdata;
 };
 
 #define EPOLL_LOOP_HANDLE_CONTINUE 0
-- 
2.37.3