!114 iSulad: sync with upstream iSulad

From: @lifeng2221dd1 Reviewed-by: @duguhaotian Signed-off-by: @duguhaotian
2021-02-03 09:22:56 +08:00 · 2021-02-03 09:22:56 +08:00 · 7e6aa593a5
commit 7e6aa593a5
parent e082dcf3ff a1c2183364
18 changed files with 3015 additions and 10 deletions
--- a/0010-fix-ramdom-coredump-if-pull-failed.patch
+++ b/0010-fix-ramdom-coredump-if-pull-failed.patch
@ -0,0 +1,52 @@
+From 7e04901403a0053f67eae6c9bb58764b529c0bd8 Mon Sep 17 00:00:00 2001
+From: WangFengTu <wangfengtu@huawei.com>
+Date: Tue, 19 Jan 2021 16:57:17 +0800
+Subject: [PATCH 10/26] fix ramdom coredump if pull failed
+
+Signed-off-by: WangFengTu <wangfengtu@huawei.com>
+---
+ src/daemon/modules/image/oci/registry/registry.c | 13 +++++++++++--
+ 1 file changed, 11 insertions(+), 2 deletions(-)
+
+diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c
+index 3fba2039..1bb91d0f 100644
+--- a/src/daemon/modules/image/oci/registry/registry.c
+++ b/src/daemon/modules/image/oci/registry/registry.c
+@@ -1298,8 +1298,10 @@ static void free_thread_fetch_info(thread_fetch_info *info)
+     return;
+ }
+ 
+-static bool all_fetch_complete(pull_descriptor *desc, int *result)
+static bool all_fetch_complete(pull_descriptor *desc, thread_fetch_info *infos, int *result)
+ {
+    int i = 0;
+
+     if (!desc->config.complete) {
+         return false;
+     }
+@@ -1314,6 +1316,13 @@ static bool all_fetch_complete(pull_descriptor *desc, int *result)
+         return false;
+     }
+ 
+    // wait all fetch threads completed
+    for (i = 0; i < desc->layers_len; i++) {
+        if (infos[i].use && !infos[i].notified) {
+            return false;
+        }
+    }
+
+     if (desc->cancel) {
+         *result = -1;
+     }
+@@ -1584,7 +1593,7 @@ static int fetch_all(pull_descriptor *desc)
+ 
+     // wait until all pulled or cancelled
+     mutex_lock(&g_shared->mutex);
+-    while (!all_fetch_complete(desc, &result)) {
+    while (!all_fetch_complete(desc, infos, &result)) {
+         ts.tv_sec = time(NULL) + DEFAULT_WAIT_TIMEOUT; // avoid wait forever
+         cond_ret = pthread_cond_timedwait(&g_shared->cond, &g_shared->mutex, &ts);
+         if (cond_ret != 0 && cond_ret != ETIMEDOUT) {
+-- 
+2.25.1
+
--- a/0011-shim-optimize-io-stream.patch
+++ b/0011-shim-optimize-io-stream.patch
@ -0,0 +1,257 @@
+From bbf3f17765483e2e87e96e975c1d85bb5250c8f2 Mon Sep 17 00:00:00 2001
+From: gaohuatao <gaohuatao@huawei.com>
+Date: Wed, 20 Jan 2021 10:13:14 +0800
+Subject: [PATCH 11/26] shim: optimize io stream
+
+Signed-off-by: gaohuatao <gaohuatao@huawei.com>
+---
+ src/cmd/isulad-shim/common.c           | 25 ++++++++++++++
+ src/cmd/isulad-shim/common.h           |  2 ++
+ src/cmd/isulad-shim/process.c          | 48 ++++++++++++++++----------
+ src/cmd/isulad-shim/process.h          |  6 ++--
+ src/cmd/isulad-shim/terminal.c         |  2 +-
+ test/cmd/isulad-shim/isulad-shim_ut.cc |  2 +-
+ 6 files changed, 61 insertions(+), 24 deletions(-)
+
+diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c
+index 23aa33cd..324d72a1 100644
+--- a/src/cmd/isulad-shim/common.c
+++ b/src/cmd/isulad-shim/common.c
+@@ -84,6 +84,31 @@ ssize_t write_nointr(int fd, const void *buf, size_t count)
+     return nret;
+ }
+ 
+ssize_t write_nointr_in_total(int fd, const char *buf, size_t count)
+{
+    ssize_t nret = 0;
+    ssize_t nwritten;
+
+    if (buf == NULL) {
+        return -1;
+    }
+
+    for (nwritten = 0; nwritten < count;) {
+        nret = write(fd, buf + nwritten, count - nwritten);
+        if (nret < 0) {
+            if (errno == EINTR || errno == EAGAIN) {
+                continue;
+            } else {
+                return nret;
+            }
+        } else {
+            nwritten += nret;
+        }
+    }
+
+    return nwritten;
+}
+
+ bool file_exists(const char *f)
+ {
+     struct stat buf;
+diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h
+index b83d72ba..8c6ea7ba 100644
+--- a/src/cmd/isulad-shim/common.h
+++ b/src/cmd/isulad-shim/common.h
+@@ -52,6 +52,8 @@ extern "C" {
+ ssize_t read_nointr(int fd, void *buf, size_t count);
+ ssize_t write_nointr(int fd, const void *buf, size_t count);
+ 
+ssize_t write_nointr_in_total(int fd, const char *buf, size_t count);
+
+ char *read_text_file(const char *path);
+ 
+ bool file_exists(const char *f);
+diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
+index 606a3df7..3ac739b9 100644
+--- a/src/cmd/isulad-shim/process.c
+++ b/src/cmd/isulad-shim/process.c
+@@ -44,15 +44,6 @@
+ 
+ extern int g_log_fd;
+ 
+-typedef int (*epoll_loop_callback_t)(int fd, uint32_t event, void *data);
+-
+-struct epoll_loop_handler {
+-    epoll_loop_callback_t cb;
+-    int epfd;
+-    int cbfd;
+-    void *cbdata;
+-};
+-
+ static shim_client_process_state *load_process()
+ {
+     parser_error err = NULL;
+@@ -243,7 +234,7 @@ static void remove_io_dispatch(io_thread_t *io_thd, int from, int to)
+     pthread_mutex_unlock(&(ioc->mutex));
+ }
+ 
+-static void *task_io_copy(void *data)
+static void *do_io_copy(void *data)
+ {
+     io_thread_t *io_thd = (io_thread_t *)data;
+     if (io_thd == NULL || io_thd->ioc == NULL) {
+@@ -278,7 +269,7 @@ static void *task_io_copy(void *data)
+                     shim_write_container_log_file(io_thd->terminal, ioc->id == stdid_out ? "stdout" : "stderr", buf,
+                                                   r_count);
+                 } else {
+-                    int w_count = write_nointr(fn->fd, buf, r_count);
+                    int w_count = write_nointr_in_total(fn->fd, buf, r_count);
+                     if (w_count < 0) {
+                         /* When any error occurs, remove the write fd */
+                         remove_io_dispatch(io_thd, -1, fn->fd);
+@@ -287,7 +278,11 @@ static void *task_io_copy(void *data)
+             }
+         }
+ 
+-        if (io_thd->shutdown) {
+        /*
+         In the case of stdout and stderr, maybe numbers of read bytes are not the last msg in pipe.
+         So, when the value of r_count is larger than zero, we need to try reading again to avoid loss msgs.
+        */
+        if (io_thd->shutdown && r_count <= 0) {
+             break;
+         }
+     }
+@@ -301,7 +296,7 @@ static void *task_io_copy(void *data)
+     return NULL;
+ }
+ 
+-static void do_io_copy(int fd, uint32_t event, void *data)
+static void sem_post_inotify_io_copy(int fd, uint32_t event, void *data)
+ {
+     io_thread_t *thd = (io_thread_t *)data;
+     if (thd->ioc == NULL || fd != thd->ioc->fd_from) {
+@@ -318,7 +313,7 @@ static void do_io_copy(int fd, uint32_t event, void *data)
+     return;
+ }
+ 
+-static int process_io_start(process_t *p, int std_id)
+static int create_io_copy_thread(process_t *p, int std_id)
+ {
+     int ret = SHIM_ERR;
+     io_thread_t *io_thd = NULL;
+@@ -351,7 +346,7 @@ static int process_io_start(process_t *p, int std_id)
+ 
+     p->io_threads[std_id] = io_thd;
+ 
+-    ret = pthread_create(&(io_thd->tid), NULL, task_io_copy, io_thd);
+    ret = pthread_create(&(io_thd->tid), NULL, do_io_copy, io_thd);
+     if (ret != SHIM_OK) {
+         write_message(g_log_fd, ERR_MSG, "thread io copy create failed:%d", SHIM_SYS_ERR(errno));
+         goto failure;
+@@ -380,7 +375,7 @@ static int start_io_copy_threads(process_t *p)
+ 
+     /* 3 threads for stdin, stdout and stderr */
+     for (i = 0; i < 3; i++) {
+-        ret = process_io_start(p, i);
+        ret = create_io_copy_thread(p, i);
+         if (ret != SHIM_OK) {
+             return SHIM_ERR;
+         }
+@@ -405,6 +400,20 @@ static void destroy_io_thread(process_t *p, int std_id)
+     p->io_threads[std_id] = NULL;
+ }
+ 
+/*
+    std_id: channel type
+    isulad_stdio: one side of the isulad fifo file
+    fd: one side of the shim io pipe
+    ---------------------------------------------------------------
+    | CHANNEL |    iSulad Fifo Side     | Flow Direction |   fd   |
+    ---------------------------------------------------------------
+    |  STDIN  |        READ             |      -->       |  WRITE |
+    ---------------------------------------------------------------
+    |  STDOUT |        WRITE            |      <--       |  READ  |
+    ---------------------------------------------------------------
+    |  STDERR |        WRITE            |      <--       |  READ  |
+    ---------------------------------------------------------------
+*/
+ static int connect_to_isulad(process_t *p, int std_id, const char *isulad_stdio, int fd)
+ {
+     mode_t mode;
+@@ -501,7 +510,7 @@ out:
+     return NULL;
+ }
+ 
+-static void *task_io_loop(void *data)
+static void *io_epoll_loop(void *data)
+ {
+     process_t *p = (process_t *)data;
+     int wait_fds = 0;
+@@ -526,7 +535,7 @@ static void *task_io_loop(void *data)
+ 
+         for (i = 0; i < wait_fds; i++) {
+             io_thread_t *thd_io = (io_thread_t *)evs[i].data.ptr;
+-            do_io_copy(thd_io->ioc->fd_from, evs[i].events, thd_io);
+            sem_post_inotify_io_copy(thd_io->ioc->fd_from, evs[i].events, thd_io);
+         }
+     }
+ }
+@@ -702,6 +711,7 @@ static int open_generic_io(process_t *p)
+ {
+     int ret = SHIM_ERR;
+ 
+    // io: in: w  out/err: r
+     stdio_t *io = initialize_io(p);
+     if (io == NULL) {
+         return SHIM_ERR;
+@@ -858,7 +868,7 @@ int process_io_init(process_t *p)
+     int ret = SHIM_ERR;
+ 
+     pthread_t tid_loop;
+-    ret = pthread_create(&tid_loop, NULL, task_io_loop, p);
+    ret = pthread_create(&tid_loop, NULL, io_epoll_loop, p);
+     if (ret != SHIM_OK) {
+         return SHIM_SYS_ERR(errno);
+     }
+diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h
+index 17704320..c17a20b1 100644
+--- a/src/cmd/isulad-shim/process.h
+++ b/src/cmd/isulad-shim/process.h
+@@ -66,13 +66,13 @@ typedef struct process {
+     char *id;
+     char *bundle;
+     char *runtime;
+-    char *console_sock_path;
+    char *console_sock_path;// pty socket path
+     int io_loop_fd;
+     int exit_fd;
+     int ctr_pid;
+     log_terminal *terminal;
+-    stdio_t *stdio;
+-    stdio_t *shim_io;
+    stdio_t *stdio;// shim to on runtime side, in:r out/err: w
+    stdio_t *shim_io; // shim io on isulad side, in: w  out/err: r
+     io_thread_t *io_threads[3];// stdin,stdout,stderr
+     shim_client_process_state *state;
+     sem_t sem_mainloop;
+diff --git a/src/cmd/isulad-shim/terminal.c b/src/cmd/isulad-shim/terminal.c
+index 989f20d8..ac39539a 100644
+--- a/src/cmd/isulad-shim/terminal.c
+++ b/src/cmd/isulad-shim/terminal.c
+@@ -38,7 +38,7 @@ static ssize_t shim_write_nointr_lock(log_terminal *terminal, const void *buf, s
+     ssize_t ret;
+ 
+     (void)pthread_rwlock_wrlock(&terminal->log_terminal_rwlock);
+-    ret = write_nointr(terminal->fd, buf, count);
+    ret = write_nointr_in_total(terminal->fd, buf, count);
+     (void)pthread_rwlock_unlock(&terminal->log_terminal_rwlock);
+ 
+     return ret;
+diff --git a/test/cmd/isulad-shim/isulad-shim_ut.cc b/test/cmd/isulad-shim/isulad-shim_ut.cc
+index d512f0bc..34ecd452 100644
+--- a/test/cmd/isulad-shim/isulad-shim_ut.cc
+++ b/test/cmd/isulad-shim/isulad-shim_ut.cc
+@@ -79,7 +79,7 @@ TEST_F(IsuladShimUnitTest, test_read_write_nointr)
+ 
+     fd_wr = open_no_inherit(test_file.c_str(), O_CREAT | O_RDWR | O_APPEND | O_SYNC, 0640);
+     EXPECT_GT(fd_wr, 0);
+-    nwrite = write_nointr(fd_wr, test_string.c_str(), 5);
+    nwrite = write_nointr_in_total(fd_wr, test_string.c_str(), 5);
+     EXPECT_EQ(nwrite, 5);
+     fd_rd = open(test_file.c_str(), O_RDONLY);
+     nread = read_nointr(fd_rd, buf, 32);
+-- 
+2.25.1
+
--- a/0012-add-CI-to-test-shim-io.patch
+++ b/0012-add-CI-to-test-shim-io.patch
@ -0,0 +1,412 @@
+From f3f4c25792721bc130aec31deea9473d5283dfc6 Mon Sep 17 00:00:00 2001
+From: gaohuatao <gaohuatao@huawei.com>
+Date: Wed, 20 Jan 2021 10:13:35 +0800
+Subject: [PATCH 12/26] add CI to test shim io
+
+Signed-off-by: gaohuatao <gaohuatao@huawei.com>
+---
+ .../container_cases/bigdata_stream_runc.sh    | 392 ++++++++++++++++++
+ 1 file changed, 392 insertions(+)
+ create mode 100755 CI/test_cases/container_cases/bigdata_stream_runc.sh
+
+diff --git a/CI/test_cases/container_cases/bigdata_stream_runc.sh b/CI/test_cases/container_cases/bigdata_stream_runc.sh
+new file mode 100755
+index 00000000..203bdd98
+--- /dev/null
+++ b/CI/test_cases/container_cases/bigdata_stream_runc.sh
+@@ -0,0 +1,392 @@
+#!/bin/bash
+#
+# attributes: isulad basic container stream exec start attach 
+# concurrent: NA
+# spend time: 6
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: gaohuatao
+##- @Create: 2021-01-19
+#######################################################################
+declare -r curr_path=$(dirname $(readlink -f "$0"))
+source ../helpers.sh
+
+function set_up()
+{
+	local ret=0
+	local image="busybox"
+	local test="set_up => (${FUNCNAME[@]})"
+
+	msg_info "${test} starting..."
+
+	check_valgrind_log
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - memory leak before current testcase, please check...." && return ${FAILURE}
+
+	start_isulad_without_valgrind
+
+	isula pull ${image}
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE}
+
+	isula images | grep busybox
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
+
+	CID=$(isula run -itd --runtime runc ${image} sh)
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
+
+	isula exec -it $CID dd if=/dev/zero of=test_500M bs=1M count=500
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create bigdata" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function record_origin_status()
+{
+	origin_isulad_cpu_usage=$(ps -o %cpu -p $(cat /var/run/isulad.pid) | sed -n '2p')
+	msg_info "origin isulad cpu usage: $origin_isulad_cpu_usage"
+
+	lxc_monitor_pid=$(ps aux | grep "lxc monitor" | grep $CID | awk '{print $2}')
+	origin_lxc_monitor_cpu_usage=$(ps -o %cpu -p $lxc_monitor_pid | sed -n '2p')
+	msg_info "origin lxc monitor cpu usage: $origin_lxc_monitor_cpu_usage"
+
+	rm -rf /iocopy_stream_data_*
+}
+
+function check_last_status()
+{
+	local ret=0
+	sleep 5
+	ps -T -p $(cat /var/run/isulad.pid) | grep IoCopy
+	[[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - IOCopy Thread residue" && ((ret++))
+
+	last_isulad_cpu_usage=$(ps -o %cpu -p $(cat /var/run/isulad.pid) | sed -n '2p')
+	allowable_isulad_cpu_usage=$(echo "$origin_isulad_cpu_usage*2" | bc)
+	if [[ $(echo "$allowable_isulad_cpu_usage < 80.0" | bc) -eq 1 ]]; then
+		allowable_isulad_cpu_usage=80.0
+	fi
+	msg_info "allowable isulad cpu usage: $allowable_isulad_cpu_usage"
+	if [[ $(echo "$last_isulad_cpu_usage > $allowable_isulad_cpu_usage" | bc) -eq 1 ]]; then
+		msg_err "${FUNCNAME[0]}:${LINENO} - Process exception: endless loop or residual thread" && ((ret++))
+		ps -o %cpu -p $(cat /var/run/isulad.pid)
+	fi
+
+	lxc_monintor_pid=$(ps aux | grep "lxc monitor" | grep $CID | awk '{print $2}')
+	last_lxc_monitor_cpu_usage=$(ps -o %cpu -p $lxc_monintor_pid | sed -n '2p')
+	allowable_lxc_monitor_cpu_usage=$(echo "$origin_lxc_monitor_cpu_usage*2" | bc)
+	if [[ $(echo "$allowable_lxc_monitor_cpu_usage < 1.0" | bc) -eq 1 ]]; then
+		allowable_lxc_monitor_cpu_usage=1.0
+	fi
+	msg_info "allowable lxc_monitor cpu usage: $allowable_lxc_monitor_cpu_usage"
+	if [[ $(echo "$last_lxc_monitor_cpu_usage > $allowable_lxc_monitor_cpu_usage" | bc) -eq 1 ]]; then
+		msg_err "${FUNCNAME[0]}:${LINENO} - Process exception: endless loop or residual thread" && ((ret++))
+		ps -o %cpu -p $lxc_monintor_pid
+	fi
+
+	lxc_attach_process_number=$(ps aux | grep lxc-attach | grep $CID | wc -l)
+	if [[ $lxc_attach_process_number -ne 0 ]]; then
+		msg_err "${FUNCNAME[0]}:${LINENO} - lxc_attach process residual" && ((ret++))
+		ps aux | grep lxc-attach | grep $CID
+	fi
+
+	client_pid=$(pidof isula)
+	if [[ -n "$client_pid" ]]; then
+		msg_err "${FUNCNAME[0]}:${LINENO} - client not exit!!" && ((ret++))
+	fi
+
+	ps aux | grep "cat test_" | grep -v "grep"
+	if [[ $? -eq 0 ]]; then
+		msg_err "${FUNCNAME[0]}:${LINENO} - business process residual" && ((ret++))
+	fi
+
+	return ${ret}
+}
+
+function test_cat_bigdata()
+{
+	local ret=0
+	local test="test_concurrent_bigdata_stream => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+	declare -a pids
+
+	for index in $(seq 1 5); do
+		nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M_$index &
+		pids[${#pids[@]}]=$!
+	done
+	wait ${pids[*]// /|}
+
+	for index in $(seq 1 5); do
+		ls -l /tmp/iocopy_stream_data_500M_$index
+		total_size=$(stat -c"%s" /tmp/iocopy_stream_data_500M_$index)
+		[[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++))
+		rm -f /tmp/iocopy_stream_data_500M_$index
+	done
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function test_stream_with_stop_client()
+{
+	local ret=0
+	local test="test_stream_with_stop_client => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+
+	nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M &
+	pid=$!
+	sleep 2
+	kill -19 $pid
+	sleep 5
+	kill -18 $pid
+
+	wait $pid
+
+	ls -l /tmp/iocopy_stream_data_500M
+	total_size=$(stat -c"%s" /tmp/iocopy_stream_data_500M)
+	[[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++))
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function test_stream_with_kill_client()
+{
+	local ret=0
+	local test="test_stream_with_kill_client => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+
+	nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M &
+	pid=$!
+	sleep 5
+	kill -9 $pid
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function test_stream_with_stop_attach()
+{
+	local ret=0
+	local test="test_stream_with_stop_attach => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+
+	nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M &
+    exec_pid=$!
+	sleep 2
+	pid=$(ps aux | grep lxc-attach | grep $CID |grep "cat test_500M" | awk '{print $2}')
+	kill -19 $pid
+	sleep 3
+	kill -18 $pid
+
+	wait $exec_pid
+
+	ls -l /tmp/iocopy_stream_data_500M
+	total_size=$(stat -c"%s" /tmp/iocopy_stream_data_500M)
+	[[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++))
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function test_stream_with_kill_attach()
+{
+	local ret=0
+	local test="test_stream_with_kill_client => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+
+	nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M &
+	sleep 3
+	pid=$(ps aux | grep lxc-attach | grep $CID |grep "cat test_500M" | awk '{print $2}')
+	kill -9 $pid
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function test_stream_with_stop_lxc_monitor()
+{
+	local ret=0
+	local test="test_stream_with_stop_lxc_monitor => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+
+	nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M &
+    exec_pid=$!
+	sleep 2
+	pid=$(ps aux | grep "lxc monitor" | grep $CID  | awk '{print $2}')
+	kill -19 $pid
+	sleep 3
+	kill -18 $pid
+
+	wait $exec_pid
+
+	ls -l /tmp/iocopy_stream_data_500M
+	total_size=$(stat -c"%s" /tmp/iocopy_stream_data_500M)
+	[[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++))
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function test_stream_with_kill_lxc_monitor()
+{
+	local ret=0
+	local test="test_stream_with_kill_lxc_monitor => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+
+	nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M &
+	sleep 3
+	pid=$(ps aux | grep "lxc monitor" | grep $CID  | awk '{print $2}')
+	kill -9 $pid
+	sleep 3
+
+	isula start $CID
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to start container: $CID" && ((ret++))
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function test_stream_with_stop_isulad()
+{
+	local ret=0
+	local test="test_stream_with_stop_isulad => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+
+	nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M &
+    pid=$!
+	sleep 2
+	kill -19 $(cat /var/run/isulad.pid)
+	sleep 3
+	kill -18 $(cat /var/run/isulad.pid) 
+
+	wait $pid 
+
+	ls -l /tmp/iocopy_stream_data_500M
+	total_size=$(stat -c"%s" /tmp/iocopy_stream_data_500M)
+	[[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++))
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function test_stream_with_kill_isulad()
+{
+	local ret=0
+	local test="test_stream_with_kill_isulad => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+
+	nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M &
+	sleep 3
+    isulad_pid=$(cat /var/run/isulad.pid)
+	kill -9 $isulad_pid
+	sleep 1
+
+    check_isulad_stopped $isulad_pid
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - isulad still alive" && ((ret++))
+
+	start_isulad_without_valgrind
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function tear_down()
+{
+	local ret=0
+	isula rm -f $CID
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container: $CID" && ((ret++))
+
+	rm -rf //tmp/iocopy_stream_data_*
+
+	stop_isulad_without_valgrind
+
+	return ${ret}
+}
+
+function test_memory_leak_with_bigdata_stream()
+{
+	local ret=0
+	local image="busybox"
+	local test="test_memory_leak_with_bigdata_stream => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+
+	start_isulad_with_valgrind
+
+	CID=$(isula run -itd ${image} sh)
+
+	isula exec -it $CID dd if=/dev/zero of=test_100M bs=1M count=100
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to create bigdata" && ((ret++))
+
+	isula exec -it $CID cat test_100M > /tmp/iocopy_stream_data_100M
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to cat bigdata from container" && ((ret++))
+
+	rm -rf /tmp/iocopy_stream_data_100M
+
+	isula rm -f $CID
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container" && ((ret++))
+
+	check_valgrind_log
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - memory leak, please check...." && ((ret++))
+
+
+	start_isulad_with_valgrind
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+declare -i ans=0
+
+set_up || ((ans++))
+
+record_origin_status
+test_cat_bigdata || ((ans++))
+test_stream_with_stop_client || ((ans++))
+test_stream_with_kill_client || ((ans++))
+test_stream_with_stop_attach || ((ans++))
+test_stream_with_kill_attach || ((ans++))
+test_stream_with_stop_lxc_monitor || ((ans++))
+test_stream_with_kill_lxc_monitor || ((ans++))
+test_stream_with_stop_isulad || ((ans++))
+test_stream_with_kill_isulad || ((ans++))
+tear_down || ((ans++))
+
+test_memory_leak_with_bigdata_stream || ((ans++))
+
+show_result ${ans} "${curr_path}/${0}"
+-- 
+2.25.1
+
--- a/0013-CI-add-testcase-for-exec-without-pty.patch
+++ b/0013-CI-add-testcase-for-exec-without-pty.patch
@ -0,0 +1,106 @@
+From 0eedc0354deb5616fe7e3308547d475af01d7cc3 Mon Sep 17 00:00:00 2001
+From: Li Feng <lifeng2221dd1@zoho.com.cn>
+Date: Wed, 20 Jan 2021 14:50:43 +0800
+Subject: [PATCH 13/26] CI: add testcase for exec without pty
+
+Signed-off-by: Li Feng <lifeng2221dd1@zoho.com.cn>
+---
+ .../container_cases/bigdata_stream.sh         | 28 +++++++++++++++++++
+ .../container_cases/bigdata_stream_runc.sh    | 28 +++++++++++++++++++
+ 2 files changed, 56 insertions(+)
+
+diff --git a/CI/test_cases/container_cases/bigdata_stream.sh b/CI/test_cases/container_cases/bigdata_stream.sh
+index d7dd2d18..768e9703 100755
+--- a/CI/test_cases/container_cases/bigdata_stream.sh
+++ b/CI/test_cases/container_cases/bigdata_stream.sh
+@@ -138,6 +138,33 @@ function test_concurrent_bigdata_stream()
+ 	return ${ret}
+ }
+ 
+function test_concurrent_bigdata_stream_without_pty()
+{
+	local ret=0
+	local test="test_concurrent_bigdata_stream => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+	declare -a pids
+
+	for index in $(seq 1 5); do
+		nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M_$index &
+		pids[${#pids[@]}]=$!
+	done
+	wait ${pids[*]// /|}
+
+	for index in $(seq 1 5); do
+		ls -l /tmp/iocopy_stream_data_500M_$index
+		total_size=$(stat -c"%s" /tmp/iocopy_stream_data_500M_$index)
+		[[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++))
+		rm -f /tmp/iocopy_stream_data_500M_$index
+	done
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+ function test_more_concurrent_stream()
+ {
+ 	local ret=0
+@@ -432,6 +459,7 @@ set_up || ((ans++))
+ 
+ record_origin_status
+ test_concurrent_bigdata_stream || ((ans++))
+test_concurrent_bigdata_stream_without_pty || ((ans++))
+ test_more_concurrent_stream || ((ans++))
+ test_stream_with_stop_client || ((ans++))
+ test_stream_with_kill_client || ((ans++))
+diff --git a/CI/test_cases/container_cases/bigdata_stream_runc.sh b/CI/test_cases/container_cases/bigdata_stream_runc.sh
+index 203bdd98..f6c2ee94 100755
+--- a/CI/test_cases/container_cases/bigdata_stream_runc.sh
+++ b/CI/test_cases/container_cases/bigdata_stream_runc.sh
+@@ -118,6 +118,33 @@ function test_cat_bigdata()
+ 	msg_info "${test} starting..."
+ 	declare -a pids
+ 
+	for index in $(seq 1 5); do
+		nohup isula exec -it $CID cat test_500M > /tmp/iocopy_stream_data_500M_$index &
+		pids[${#pids[@]}]=$!
+	done
+	wait ${pids[*]// /|}
+
+	for index in $(seq 1 5); do
+		ls -l /tmp/iocopy_stream_data_500M_$index
+		total_size=$(stat -c"%s" /tmp/iocopy_stream_data_500M_$index)
+		[[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++))
+		rm -f /tmp/iocopy_stream_data_500M_$index
+	done
+
+	check_last_status
+	[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - abnormal status" && ((ret++))
+
+	msg_info "${test} finished with return ${ret}..."
+	return ${ret}
+}
+
+function test_cat_bigdata_without_pty()
+{
+	local ret=0
+	local test="test_concurrent_bigdata_stream => (${FUNCNAME[@]})"
+	msg_info "${test} starting..."
+	declare -a pids
+
+ 	for index in $(seq 1 5); do
+ 		nohup isula exec $CID cat test_500M > /tmp/iocopy_stream_data_500M_$index &
+ 		pids[${#pids[@]}]=$!
+@@ -377,6 +404,7 @@ set_up || ((ans++))
+ 
+ record_origin_status
+ test_cat_bigdata || ((ans++))
+test_cat_bigdata_without_pty || ((ans++))
+ test_stream_with_stop_client || ((ans++))
+ test_stream_with_kill_client || ((ans++))
+ test_stream_with_stop_attach || ((ans++))
+-- 
+2.25.1
+
--- a/0014-adapt-for-sparse-file-when-tar-file.patch
+++ b/0014-adapt-for-sparse-file-when-tar-file.patch
@ -0,0 +1,69 @@
+From 7e9b7b16c76785c15fd1465d7985a0919848f786 Mon Sep 17 00:00:00 2001
+From: WangFengTu <wangfengtu@huawei.com>
+Date: Thu, 21 Jan 2021 18:44:52 +0800
+Subject: [PATCH 14/26] adapt for sparse file when tar file
+
+archive_read_data_block can not process sparse file
+correctly, use archive_read_data instead.
+
+Signed-off-by: WangFengTu <wangfengtu@huawei.com>
+---
+ src/utils/tar/util_archive.c | 30 +++++++++++++++++++++---------
+ 1 file changed, 21 insertions(+), 9 deletions(-)
+
+diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c
+index 7a28286a..1b9553c9 100644
+--- a/src/utils/tar/util_archive.c
+++ b/src/utils/tar/util_archive.c
+@@ -699,27 +699,39 @@ out:
+ static int copy_data_between_archives(struct archive *ar, struct archive *aw)
+ {
+     int ret = ARCHIVE_FAILED;
+-    const void *buff = NULL;
+-    size_t size;
+-    int64_t offset;
+    char *buff = NULL;
+    ssize_t size = 0;
+
+    buff = util_common_calloc_s(ARCHIVE_BLOCK_SIZE);
+    if (buff == NULL) {
+        ERROR("out of memory");
+        fprintf(stderr, "out of memory");
+        return ARCHIVE_FAILED;
+    }
+ 
+     for (;;) {
+-        ret = archive_read_data_block(ar, &buff, &size, &offset);
+-        if (ret == ARCHIVE_EOF) {
+-            return ARCHIVE_OK;
+        size = archive_read_data(ar, buff, ARCHIVE_BLOCK_SIZE);
+        if (size == 0) {
+            ret = ARCHIVE_OK;
+            goto out;
+         }
+-        if (ret < ARCHIVE_OK) {
+        if (size < 0) {
+             ERROR("tar archive read result %d, error: %s", ret, archive_error_string(ar));
+             fprintf(stderr, "tar archive read result %d, error: %s", ret, archive_error_string(ar));
+-            return ret;
+            ret = ARCHIVE_FAILED;
+            goto out;
+         }
+         ret = archive_write_data(aw, buff, size);
+         if (ret < ARCHIVE_OK) {
+             ERROR("tar archive write result %d, error: %s", ret, archive_error_string(aw));
+             fprintf(stderr, "tar archive write result %d, error: %s", ret, archive_error_string(aw));
+-            return ret;
+            goto out;
+         }
+     }
+
+out:
+    free(buff);
+    return ret;
+ }
+ 
+ int update_entry_for_hardlink(map_t *map_link, struct archive_entry *entry, const char *src_base, const char *dst_base)
+-- 
+2.25.1
+
--- a/0015-driver-do-not-unlock-and-destroy-lock-when-clean-up.patch
+++ b/0015-driver-do-not-unlock-and-destroy-lock-when-clean-up.patch
@ -0,0 +1,29 @@
+From bba60af5e275a24ab6ae11943ce48ff71524c494 Mon Sep 17 00:00:00 2001
+From: Li Feng <lifeng2221dd1@zoho.com.cn>
+Date: Mon, 25 Jan 2021 11:31:54 +0800
+Subject: [PATCH 15/26] driver: do not unlock and destroy lock when clean up
+
+Signed-off-by: Li Feng <lifeng2221dd1@zoho.com.cn>
+---
+ .../image/oci/storage/layer_store/graphdriver/driver.c        | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c
+index f2df4f8f..b41132ea 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c
+@@ -517,9 +517,9 @@ int graphdriver_cleanup(void)
+     g_graphdriver->home = NULL;
+     free(g_graphdriver->backing_fs);
+     g_graphdriver->backing_fs = NULL;
+-    driver_unlock();
+-    pthread_rwlock_destroy(&(g_graphdriver->rwlock));
+     g_graphdriver = NULL;
+    // notes, do not driver_unlock and destroy the lock, becase the other threads may wait for it
+    // if we unlock and destroy the lock, may cause the lock failure, and result to coredump
+ 
+ out:
+     return ret;
+-- 
+2.25.1
+
--- a/0016-driver-do-not-set-g_graphdriver-to-NULL.patch
+++ b/0016-driver-do-not-set-g_graphdriver-to-NULL.patch
@ -0,0 +1,32 @@
+From d1fbada9a7b520830d8a0c31263aadba97b2dd9d Mon Sep 17 00:00:00 2001
+From: Li Feng <lifeng2221dd1@zoho.com.cn>
+Date: Mon, 25 Jan 2021 15:01:35 +0800
+Subject: [PATCH 16/26] driver: do not set g_graphdriver to NULL
+
+Signed-off-by: Li Feng <lifeng2221dd1@zoho.com.cn>
+---
+ .../image/oci/storage/layer_store/graphdriver/driver.c    | 8 +-------
+ 1 file changed, 1 insertion(+), 7 deletions(-)
+
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c
+index b41132ea..6b1e0922 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c
+@@ -512,13 +512,7 @@ int graphdriver_cleanup(void)
+         driver_unlock();
+         goto out;
+     }
+-
+-    free((char *)g_graphdriver->home);
+-    g_graphdriver->home = NULL;
+-    free(g_graphdriver->backing_fs);
+-    g_graphdriver->backing_fs = NULL;
+-    g_graphdriver = NULL;
+-    // notes, do not driver_unlock and destroy the lock, becase the other threads may wait for it
+    // notes, do not call driver_unlock and destroy the lock, becase the other threads may wait for it
+     // if we unlock and destroy the lock, may cause the lock failure, and result to coredump
+ 
+ out:
+-- 
+2.25.1
+
--- a/0017-ignore-error-if-get-ip-failed.patch
+++ b/0017-ignore-error-if-get-ip-failed.patch
@ -0,0 +1,39 @@
+From 4f2951681dbe583e80af91d808292aad8cceb599 Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Thu, 28 Jan 2021 14:04:54 +0800
+Subject: [PATCH 17/26] ignore error if get ip failed
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ .../entry/cri/cri_pod_sandbox_manager_service_impl.cc       | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+index 27768852..a668d60b 100644
+--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+@@ -989,12 +989,12 @@ void PodSandboxManagerServiceImpl::GetIPs(const std::string &podSandboxID, const
+     }
+ 
+     if (inspect->network_settings != nullptr && inspect->network_settings->ip_address != nullptr) {
+-        WARN("Use container inspect ip info: %s", error.GetCMessage());
+-        error.Clear();
+        WARN("Use container inspect ip info");
+         ips.push_back(inspect->network_settings->ip_address);
+     }
+ 
+     WARN("Failed to read pod IP from plugin/docker: %s", error.GetCMessage());
+    error.Clear();
+ }
+ 
+ void PodSandboxManagerServiceImpl::SetSandboxStatusNetwork(const container_inspect *inspect,
+@@ -1214,4 +1214,4 @@ void PodSandboxManagerServiceImpl::PortForward(const runtime::v1alpha2::PortForw
+     // This feature is temporarily not supported
+ }
+ 
+-} // namespace CRI
+\ No newline at end of file
+} // namespace CRI
+-- 
+2.25.1
+
--- a/0018-GC-add-log-container-info-when-add-into-gc.patch
+++ b/0018-GC-add-log-container-info-when-add-into-gc.patch
@ -0,0 +1,35 @@
+From 72e8e3163524455768986a7496ccfc5ce384fade Mon Sep 17 00:00:00 2001
+From: Li Feng <lifeng2221dd1@zoho.com.cn>
+Date: Fri, 29 Jan 2021 15:33:56 +0800
+Subject: [PATCH 18/26] GC: add log container info when add into gc
+
+Signed-off-by: Li Feng <lifeng2221dd1@zoho.com.cn>
+---
+ src/daemon/modules/container/container_gc/containers_gc.c | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/src/daemon/modules/container/container_gc/containers_gc.c b/src/daemon/modules/container/container_gc/containers_gc.c
+index 6b1c392c..5924aaa3 100644
+--- a/src/daemon/modules/container/container_gc/containers_gc.c
+++ b/src/daemon/modules/container/container_gc/containers_gc.c
+@@ -203,6 +203,8 @@ int gc_add_container(const char *id, const char *runtime, const pid_ppid_info_t
+         return -1;
+     }
+ 
+    EVENT("Event: {Object: GC, Type: Add container %s with pid %u into garbage collector}", id, pid_info->pid);
+
+     newnode = util_common_calloc_s(sizeof(struct linked_list));
+     if (newnode == NULL) {
+         CRIT("Memory allocation error.");
+@@ -481,6 +483,8 @@ static void gc_container_process(struct linked_list *it)
+ 
+         gc_containers_unlock();
+ 
+        EVENT("Event: {Object: GC, Type: Delete container %s with pid %u from garbage collector}", id, pid);
+
+         /* apply restart policy for the container after gc */
+         apply_restart_policy_after_gc(id);
+ 
+-- 
+2.25.1
+
--- a/0019-log-use-the-same-function-to-init-log-in-export-paus.patch
+++ b/0019-log-use-the-same-function-to-init-log-in-export-paus.patch
@ -0,0 +1,146 @@
+From 171cb932bbbbfc5816ceb65223f1d5e733c79d8e Mon Sep 17 00:00:00 2001
+From: Li Feng <lifeng2221dd1@zoho.com.cn>
+Date: Sat, 30 Jan 2021 10:38:11 +0800
+Subject: [PATCH 19/26] log: use the same function to init log in
+ export/pause/resume
+
+Signed-off-by: Li Feng <lifeng2221dd1@zoho.com.cn>
+---
+ src/cmd/isula/extend/export.c | 19 +++++++++----------
+ src/cmd/isula/extend/pause.c  | 20 +++++++++-----------
+ src/cmd/isula/extend/resume.c | 17 +++++++----------
+ 3 files changed, 25 insertions(+), 31 deletions(-)
+
+diff --git a/src/cmd/isula/extend/export.c b/src/cmd/isula/extend/export.c
+index 476cf775..ea9b9c11 100644
+--- a/src/cmd/isula/extend/export.c
+++ b/src/cmd/isula/extend/export.c
+@@ -76,15 +76,7 @@ int cmd_export_main(int argc, const char **argv)
+     char file[PATH_MAX] = { 0 };
+     struct isula_libutils_log_config lconf = { 0 };
+ 
+-    lconf.name = argv[0];
+-    lconf.quiet = true;
+-    lconf.driver = "stdout";
+-    lconf.file = NULL;
+-    lconf.priority = "ERROR";
+-    if (isula_libutils_log_enable(&lconf)) {
+-        COMMAND_ERROR("Export: log init failed");
+-        exit(ECOMMON);
+-    }
+    isula_libutils_default_log_config(argv[0], &lconf);
+ 
+     command_t cmd;
+     if (client_arguments_init(&g_cmd_export_args)) {
+@@ -92,7 +84,9 @@ int cmd_export_main(int argc, const char **argv)
+         exit(ECOMMON);
+     }
+     g_cmd_export_args.progname = argv[0];
+-    struct command_option options[] = { COMMON_OPTIONS(g_cmd_export_args) EXPORT_OPTIONS(g_cmd_export_args) };
+    struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_export_args)
+        EXPORT_OPTIONS(g_cmd_export_args)
+    };
+ 
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_export_desc,
+                  g_cmd_export_usage);
+@@ -100,6 +94,11 @@ int cmd_export_main(int argc, const char **argv)
+         exit(EINVALIDARGS);
+     }
+ 
+    if (isula_libutils_log_enable(&lconf)) {
+        COMMAND_ERROR("log init failed");
+        exit(ECOMMON);
+    }
+
+     if (g_cmd_export_args.argc != 1) {
+         COMMAND_ERROR("Export requires exactly 1 container name");
+         exit(EINVALIDARGS);
+diff --git a/src/cmd/isula/extend/pause.c b/src/cmd/isula/extend/pause.c
+index 4d508e7b..c12eaa8c 100644
+--- a/src/cmd/isula/extend/pause.c
+++ b/src/cmd/isula/extend/pause.c
+@@ -71,23 +71,16 @@ int cmd_pause_main(int argc, const char **argv)
+     int i = 0;
+     int status = 0;
+     struct isula_libutils_log_config lconf = { 0 };
+-
+-    lconf.name = argv[0];
+-    lconf.quiet = true;
+-    lconf.file = NULL;
+-    lconf.priority = "ERROR";
+-    lconf.driver = "stdout";
+-    if (isula_libutils_log_enable(&lconf)) {
+-        COMMAND_ERROR("log init failed");
+-        exit(ECOMMON);
+-    }
+     command_t cmd;
+
+    isula_libutils_default_log_config(argv[0], &lconf);
+
+     if (client_arguments_init(&g_cmd_pause_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_pause_args.progname = argv[0];
+-    struct command_option options[] = { COMMON_OPTIONS(g_cmd_pause_args) };
+    struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_pause_args) };
+ 
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_pause_desc,
+                  g_cmd_pause_usage);
+@@ -95,6 +88,11 @@ int cmd_pause_main(int argc, const char **argv)
+         exit(EINVALIDARGS);
+     }
+ 
+    if (isula_libutils_log_enable(&lconf)) {
+        COMMAND_ERROR("log init failed");
+        exit(ECOMMON);
+    }
+
+     if (g_cmd_pause_args.argc == 0) {
+         COMMAND_ERROR("Pause requires at least 1 container names");
+         exit(EINVALIDARGS);
+diff --git a/src/cmd/isula/extend/resume.c b/src/cmd/isula/extend/resume.c
+index c3c43760..7a25ee68 100644
+--- a/src/cmd/isula/extend/resume.c
+++ b/src/cmd/isula/extend/resume.c
+@@ -71,15 +71,7 @@ int cmd_resume_main(int argc, const char **argv)
+     int status = 0;
+     struct isula_libutils_log_config lconf = { 0 };
+ 
+-    lconf.name = argv[0];
+-    lconf.quiet = true;
+-    lconf.driver = "stdout";
+-    lconf.file = NULL;
+-    lconf.priority = "ERROR";
+-    if (isula_libutils_log_enable(&lconf)) {
+-        COMMAND_ERROR("Resume: log init failed");
+-        exit(ECOMMON);
+-    }
+    isula_libutils_default_log_config(argv[0], &lconf);
+ 
+     command_t cmd;
+     if (client_arguments_init(&g_cmd_resume_args)) {
+@@ -87,7 +79,7 @@ int cmd_resume_main(int argc, const char **argv)
+         exit(ECOMMON);
+     }
+     g_cmd_resume_args.progname = argv[0];
+-    struct command_option options[] = { COMMON_OPTIONS(g_cmd_resume_args) };
+    struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_resume_args) };
+ 
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_resume_desc,
+                  g_cmd_resume_usage);
+@@ -95,6 +87,11 @@ int cmd_resume_main(int argc, const char **argv)
+         exit(EINVALIDARGS);
+     }
+ 
+    if (isula_libutils_log_enable(&lconf)) {
+        COMMAND_ERROR("log init failed");
+        exit(ECOMMON);
+    }
+
+     if (g_cmd_resume_args.argc == 0) {
+         COMMAND_ERROR("Pause requires at least 1 container names");
+         exit(EINVALIDARGS);
+-- 
+2.25.1
+
--- a/0020-init-log-config-should-before-command-parse.patch
+++ b/0020-init-log-config-should-before-command-parse.patch
@ -0,0 +1,623 @@
+From 20a6562ea0a6c50bdc6a863067eeaf7fa04909d0 Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Sat, 30 Jan 2021 14:46:13 +0800
+Subject: [PATCH 20/26] init log config should before command parse
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ src/cmd/isula/base/create.c         | 2 +-
+ src/cmd/isula/base/kill.c           | 2 +-
+ src/cmd/isula/base/rename.c         | 2 +-
+ src/cmd/isula/base/restart.c        | 2 +-
+ src/cmd/isula/base/rm.c             | 2 +-
+ src/cmd/isula/base/run.c            | 2 +-
+ src/cmd/isula/base/start.c          | 2 +-
+ src/cmd/isula/base/stop.c           | 2 +-
+ src/cmd/isula/extend/events.c       | 2 +-
+ src/cmd/isula/extend/stats.c        | 2 +-
+ src/cmd/isula/extend/update.c       | 2 +-
+ src/cmd/isula/images/images.c       | 2 +-
+ src/cmd/isula/images/import.c       | 2 +-
+ src/cmd/isula/images/load.c         | 2 +-
+ src/cmd/isula/images/login.c        | 2 +-
+ src/cmd/isula/images/logout.c       | 2 +-
+ src/cmd/isula/images/pull.c         | 2 +-
+ src/cmd/isula/images/rmi.c          | 2 +-
+ src/cmd/isula/images/tag.c          | 2 +-
+ src/cmd/isula/information/info.c    | 2 +-
+ src/cmd/isula/information/inspect.c | 2 +-
+ src/cmd/isula/information/logs.c    | 2 +-
+ src/cmd/isula/information/ps.c      | 2 +-
+ src/cmd/isula/information/top.c     | 2 +-
+ src/cmd/isula/information/version.c | 2 +-
+ src/cmd/isula/information/wait.c    | 2 +-
+ src/cmd/isula/stream/attach.c       | 2 +-
+ src/cmd/isula/stream/cp.c           | 2 +-
+ src/cmd/isula/stream/exec.c         | 2 +-
+ src/cmd/isula/volume/list.c         | 2 +-
+ src/cmd/isula/volume/prune.c        | 2 +-
+ src/cmd/isula/volume/remove.c       | 2 +-
+ 32 files changed, 32 insertions(+), 32 deletions(-)
+
+diff --git a/src/cmd/isula/base/create.c b/src/cmd/isula/base/create.c
+index 12f0a8be..a531fc0e 100644
+--- a/src/cmd/isula/base/create.c
+++ b/src/cmd/isula/base/create.c
+@@ -1538,6 +1538,7 @@ int cmd_create_main(int argc, const char **argv)
+             g_cmd_create_args) COMMON_OPTIONS(g_cmd_create_args)
+     };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_create_desc,
+                  g_cmd_create_usage);
+     if (command_parse_args(&cmd, &g_cmd_create_args.argc, &g_cmd_create_args.argv) ||
+@@ -1545,7 +1546,6 @@ int cmd_create_main(int argc, const char **argv)
+         nret = EINVALIDARGS;
+         goto out;
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("log init failed");
+         exit(ECOMMON);
+diff --git a/src/cmd/isula/base/kill.c b/src/cmd/isula/base/kill.c
+index b8e85459..7c8bf95c 100644
+--- a/src/cmd/isula/base/kill.c
+++ b/src/cmd/isula/base/kill.c
+@@ -92,12 +92,12 @@ int cmd_kill_main(int argc, const char **argv)
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_kill_args)
+         KILL_OPTIONS(g_cmd_kill_args)
+     };
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_kill_desc,
+                  g_cmd_kill_usage);
+     if (command_parse_args(&cmd, &g_cmd_kill_args.argc, &g_cmd_kill_args.argv)) {
+         exit(ECOMMON);
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("log init failed\n");
+         exit(ECOMMON);
+diff --git a/src/cmd/isula/base/rename.c b/src/cmd/isula/base/rename.c
+index aafc9bda..c5cf5bb6 100644
+--- a/src/cmd/isula/base/rename.c
+++ b/src/cmd/isula/base/rename.c
+@@ -67,12 +67,12 @@ int cmd_rename_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_rename_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_rename_args)) {
+         COMMAND_ERROR("client arguments init failed\n");
+         exit(ECOMMON);
+     }
+     g_cmd_rename_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_rename_desc,
+                  g_cmd_rename_usage);
+     if (command_parse_args(&cmd, &g_cmd_rename_args.argc, &g_cmd_rename_args.argv)) {
+diff --git a/src/cmd/isula/base/restart.c b/src/cmd/isula/base/restart.c
+index cedfaec1..877b0ff3 100644
+--- a/src/cmd/isula/base/restart.c
+++ b/src/cmd/isula/base/restart.c
+@@ -73,12 +73,12 @@ int cmd_restart_main(int argc, const char **argv)
+         RESTART_OPTIONS(g_cmd_restart_args)
+     };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_restart_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_restart_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_restart_desc,
+                  g_cmd_restart_usage);
+ 
+diff --git a/src/cmd/isula/base/rm.c b/src/cmd/isula/base/rm.c
+index 9cd0cbd0..13bd4644 100644
+--- a/src/cmd/isula/base/rm.c
+++ b/src/cmd/isula/base/rm.c
+@@ -118,12 +118,12 @@ int cmd_delete_main(int argc, const char **argv)
+         DELETE_OPTIONS(g_cmd_delete_args)
+     };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_delete_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_delete_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_delete_desc,
+                  g_cmd_delete_usage);
+     if (command_parse_args(&cmd, &g_cmd_delete_args.argc, &g_cmd_delete_args.argv)) {
+diff --git a/src/cmd/isula/base/run.c b/src/cmd/isula/base/run.c
+index 0c82af02..a6068709 100644
+--- a/src/cmd/isula/base/run.c
+++ b/src/cmd/isula/base/run.c
+@@ -148,13 +148,13 @@ int cmd_run_main(int argc, const char **argv)
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_run_args) CREATE_OPTIONS(g_cmd_run_args)
+         CREATE_EXTEND_OPTIONS(g_cmd_run_args) RUN_OPTIONS(g_cmd_run_args)
+     };
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_run_desc,
+                  g_cmd_run_usage);
+     if (command_parse_args(&cmd, &g_cmd_run_args.argc, &g_cmd_run_args.argv) || run_checker(&g_cmd_run_args)) {
+         exit(EINVALIDARGS);
+     }
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("log init failed");
+         exit(ECOMMON);
+diff --git a/src/cmd/isula/base/start.c b/src/cmd/isula/base/start.c
+index 104c24ae..5e55524f 100644
+--- a/src/cmd/isula/base/start.c
+++ b/src/cmd/isula/base/start.c
+@@ -327,12 +327,12 @@ int cmd_start_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_start_args) START_OPTIONS(g_cmd_start_args)};
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_start_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_start_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_start_desc,
+                  g_cmd_start_usage);
+     if (command_parse_args(&cmd, &g_cmd_start_args.argc, &g_cmd_start_args.argv)) {
+diff --git a/src/cmd/isula/base/stop.c b/src/cmd/isula/base/stop.c
+index e52db79c..6a901c42 100644
+--- a/src/cmd/isula/base/stop.c
+++ b/src/cmd/isula/base/stop.c
+@@ -78,12 +78,12 @@ int cmd_stop_main(int argc, const char **argv)
+         STOP_OPTIONS(g_cmd_stop_args)
+     };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_stop_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_stop_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_stop_desc,
+                  g_cmd_stop_usage);
+     if (command_parse_args(&cmd, &g_cmd_stop_args.argc, &g_cmd_stop_args.argv)) {
+diff --git a/src/cmd/isula/extend/events.c b/src/cmd/isula/extend/events.c
+index 9dbd7774..7094ac0d 100644
+--- a/src/cmd/isula/extend/events.c
+++ b/src/cmd/isula/extend/events.c
+@@ -214,12 +214,12 @@ int cmd_events_main(int argc, const char **argv)
+         COMMON_OPTIONS(g_cmd_events_args)
+     };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_events_desc,
+                  g_cmd_events_usage);
+     if (command_parse_args(&cmd, &g_cmd_events_args.argc, &g_cmd_events_args.argv)) {
+         exit(EINVALIDARGS);
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("Events: log init failed");
+         exit(ECOMMON);
+diff --git a/src/cmd/isula/extend/stats.c b/src/cmd/isula/extend/stats.c
+index 655332b8..334f859e 100644
+--- a/src/cmd/isula/extend/stats.c
+++ b/src/cmd/isula/extend/stats.c
+@@ -272,9 +272,9 @@ int cmd_stats_main(int argc, const char **argv)
+         exit(ECOMMON);
+     }
+     g_cmd_stats_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_stats_desc,
+                  g_cmd_stats_usage);
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (command_parse_args(&cmd, &g_cmd_stats_args.argc, &g_cmd_stats_args.argv)) {
+         exit(EINVALIDARGS);
+     }
+diff --git a/src/cmd/isula/extend/update.c b/src/cmd/isula/extend/update.c
+index da472b0b..42cb8f21 100644
+--- a/src/cmd/isula/extend/update.c
+++ b/src/cmd/isula/extend/update.c
+@@ -149,12 +149,12 @@ int cmd_update_main(int argc, const char **argv)
+         COMMON_OPTIONS(g_cmd_update_args)
+     };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_update_args)) {
+         COMMAND_ERROR("client arguments init failed\n");
+         exit(ECOMMON);
+     }
+     g_cmd_update_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_update_desc,
+                  g_cmd_update_usage);
+     if (command_parse_args(&cmd, &g_cmd_update_args.argc, &g_cmd_update_args.argv) ||
+diff --git a/src/cmd/isula/images/images.c b/src/cmd/isula/images/images.c
+index f60e7500..3d538aa5 100644
+--- a/src/cmd/isula/images/images.c
+++ b/src/cmd/isula/images/images.c
+@@ -324,12 +324,12 @@ int cmd_images_main(int argc, const char **argv)
+                COMMON_OPTIONS(g_cmd_images_args)
+     };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_images_desc,
+                  g_cmd_images_usage);
+     if (command_parse_args(&cmd, &g_cmd_images_args.argc, &g_cmd_images_args.argv)) {
+         exit(exit_code);
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("Images: log init failed");
+         exit(exit_code);
+diff --git a/src/cmd/isula/images/import.c b/src/cmd/isula/images/import.c
+index 2dcc6486..dbacb604 100644
+--- a/src/cmd/isula/images/import.c
+++ b/src/cmd/isula/images/import.c
+@@ -88,12 +88,12 @@ int cmd_import_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_import_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_import_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_import_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_import_desc,
+                  g_cmd_import_usage);
+     if (command_parse_args(&cmd, &g_cmd_import_args.argc, &g_cmd_import_args.argv)) {
+diff --git a/src/cmd/isula/images/load.c b/src/cmd/isula/images/load.c
+index 0fb8014e..688edd02 100644
+--- a/src/cmd/isula/images/load.c
+++ b/src/cmd/isula/images/load.c
+@@ -124,12 +124,12 @@ int cmd_load_main(int argc, const char **argv)
+ #endif
+     };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_load_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_load_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_load_desc,
+                  g_cmd_load_usage);
+     if (command_parse_args(&cmd, &g_cmd_load_args.argc, &g_cmd_load_args.argv)) {
+diff --git a/src/cmd/isula/images/login.c b/src/cmd/isula/images/login.c
+index 92550352..0c0c149b 100644
+--- a/src/cmd/isula/images/login.c
+++ b/src/cmd/isula/images/login.c
+@@ -184,13 +184,13 @@ int cmd_login_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { COMMON_OPTIONS(g_cmd_login_args) LOGIN_OPTIONS(g_cmd_login_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_login_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_login_args.progname = argv[0];
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_login_desc,
+                  g_cmd_login_usage);
+     if (command_parse_args(&cmd, &g_cmd_login_args.argc, &g_cmd_login_args.argv)) {
+diff --git a/src/cmd/isula/images/logout.c b/src/cmd/isula/images/logout.c
+index 8efec1e3..45f28509 100644
+--- a/src/cmd/isula/images/logout.c
+++ b/src/cmd/isula/images/logout.c
+@@ -80,13 +80,13 @@ int cmd_logout_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { COMMON_OPTIONS(g_cmd_logout_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_logout_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_logout_args.progname = argv[0];
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_logout_desc,
+                  g_cmd_logout_usage);
+     if (command_parse_args(&cmd, &g_cmd_logout_args.argc, &g_cmd_logout_args.argv)) {
+diff --git a/src/cmd/isula/images/pull.c b/src/cmd/isula/images/pull.c
+index b72b0302..3ba7a715 100644
+--- a/src/cmd/isula/images/pull.c
+++ b/src/cmd/isula/images/pull.c
+@@ -79,13 +79,13 @@ int cmd_pull_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { COMMON_OPTIONS(g_cmd_pull_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_pull_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_pull_args.progname = argv[0];
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_pull_desc,
+                  g_cmd_pull_usage);
+     if (command_parse_args(&cmd, &g_cmd_pull_args.argc, &g_cmd_pull_args.argv)) {
+diff --git a/src/cmd/isula/images/rmi.c b/src/cmd/isula/images/rmi.c
+index 53ea7343..5b07c866 100644
+--- a/src/cmd/isula/images/rmi.c
+++ b/src/cmd/isula/images/rmi.c
+@@ -83,12 +83,12 @@ int cmd_rmi_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_rmi_args) RMI_OPTIONS(g_cmd_rmi_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_rmi_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_rmi_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_rmi_desc,
+                  g_cmd_rmi_usage);
+     if (command_parse_args(&cmd, &g_cmd_rmi_args.argc, &g_cmd_rmi_args.argv)) {
+diff --git a/src/cmd/isula/images/tag.c b/src/cmd/isula/images/tag.c
+index e5a86708..8f399520 100644
+--- a/src/cmd/isula/images/tag.c
+++ b/src/cmd/isula/images/tag.c
+@@ -78,12 +78,12 @@ int cmd_tag_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_tag_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_tag_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_tag_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_tag_desc,
+                  g_cmd_tag_usage);
+     if (command_parse_args(&cmd, &g_cmd_tag_args.argc, &g_cmd_tag_args.argv)) {
+diff --git a/src/cmd/isula/information/info.c b/src/cmd/isula/information/info.c
+index d6f6f7be..d3dd194a 100644
+--- a/src/cmd/isula/information/info.c
+++ b/src/cmd/isula/information/info.c
+@@ -160,12 +160,12 @@ int cmd_info_main(int argc, const char **argv)
+     g_cmd_info_args.progname = argv[0];
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_info_args) };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_info_desc,
+                  g_cmd_info_usage);
+     if (command_parse_args(&cmd, &g_cmd_info_args.argc, &g_cmd_info_args.argv) != 0) {
+         exit(EINVALIDARGS);
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf) != 0) {
+         COMMAND_ERROR("Info: log init failed");
+         exit(ECOMMON);
+diff --git a/src/cmd/isula/information/inspect.c b/src/cmd/isula/information/inspect.c
+index 8ddb032e..d575b3db 100644
+--- a/src/cmd/isula/information/inspect.c
+++ b/src/cmd/isula/information/inspect.c
+@@ -967,12 +967,12 @@ int cmd_inspect_main(int argc, const char **argv)
+                COMMON_OPTIONS(g_cmd_inspect_args)
+     };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_inspect_desc,
+                  g_cmd_inspect_usage);
+     if (command_parse_args(&cmd, &g_cmd_inspect_args.argc, &g_cmd_inspect_args.argv)) {
+         exit(EINVALIDARGS);
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("log init failed");
+         exit(ECOMMON);
+diff --git a/src/cmd/isula/information/logs.c b/src/cmd/isula/information/logs.c
+index 2ddd16e6..5f3951ed 100644
+--- a/src/cmd/isula/information/logs.c
+++ b/src/cmd/isula/information/logs.c
+@@ -109,12 +109,12 @@ static int cmd_logs_init(int argc, const char **argv)
+         COMMON_OPTIONS(g_cmd_logs_args)
+     };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_logs_desc,
+                  g_cmd_logs_usage);
+     if (command_parse_args(&cmd, &g_cmd_logs_args.argc, &g_cmd_logs_args.argv)) {
+         return EINVALIDARGS;
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("log init failed\n");
+         g_cmd_logs_args.name = g_cmd_logs_args.argv[0];
+diff --git a/src/cmd/isula/information/ps.c b/src/cmd/isula/information/ps.c
+index 125353a6..e1f8f75a 100644
+--- a/src/cmd/isula/information/ps.c
+++ b/src/cmd/isula/information/ps.c
+@@ -986,12 +986,12 @@ int cmd_list_main(int argc, const char **argv)
+                COMMON_OPTIONS(g_cmd_list_args)
+     };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_list_desc,
+                  g_cmd_list_usage);
+     if (command_parse_args(&cmd, &g_cmd_list_args.argc, &g_cmd_list_args.argv)) {
+         exit(EINVALIDARGS);
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("PS: log init failed");
+         exit(ECOMMON);
+diff --git a/src/cmd/isula/information/top.c b/src/cmd/isula/information/top.c
+index 5d0e3f0f..eeb47892 100644
+--- a/src/cmd/isula/information/top.c
+++ b/src/cmd/isula/information/top.c
+@@ -120,12 +120,12 @@ int cmd_top_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_top_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_top_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_top_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_top_desc,
+                  g_cmd_top_usage);
+ 
+diff --git a/src/cmd/isula/information/version.c b/src/cmd/isula/information/version.c
+index 62ee5643..46f73cab 100644
+--- a/src/cmd/isula/information/version.c
+++ b/src/cmd/isula/information/version.c
+@@ -98,12 +98,12 @@ int cmd_version_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_version_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_version_args)) {
+         COMMAND_ERROR("client arguments init failed\n");
+         exit(ECOMMON);
+     }
+     g_cmd_version_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_version_desc,
+                  g_cmd_version_usage);
+     if (command_parse_args(&cmd, &g_cmd_version_args.argc, &g_cmd_version_args.argv)) {
+diff --git a/src/cmd/isula/information/wait.c b/src/cmd/isula/information/wait.c
+index b39c4953..aaa8f20e 100644
+--- a/src/cmd/isula/information/wait.c
+++ b/src/cmd/isula/information/wait.c
+@@ -91,12 +91,12 @@ int cmd_wait_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_wait_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_wait_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_wait_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_wait_desc,
+                  g_cmd_wait_usage);
+     if (command_parse_args(&cmd, &g_cmd_wait_args.argc, &g_cmd_wait_args.argv)) {
+diff --git a/src/cmd/isula/stream/attach.c b/src/cmd/isula/stream/attach.c
+index f0a77a16..6dac2a0c 100644
+--- a/src/cmd/isula/stream/attach.c
+++ b/src/cmd/isula/stream/attach.c
+@@ -172,12 +172,12 @@ static int attach_cmd_init(int argc, const char **argv)
+     g_cmd_attach_args.progname = argv[0];
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_attach_args) };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_attach_desc,
+                  g_cmd_attach_usage);
+     if (command_parse_args(&cmd, &g_cmd_attach_args.argc, &g_cmd_attach_args.argv)) {
+         return EINVALIDARGS;
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("log init failed");
+         return ECOMMON;
+diff --git a/src/cmd/isula/stream/cp.c b/src/cmd/isula/stream/cp.c
+index e954ed3d..b869741f 100644
+--- a/src/cmd/isula/stream/cp.c
+++ b/src/cmd/isula/stream/cp.c
+@@ -327,12 +327,12 @@ int cmd_cp_main(int argc, const char **argv)
+     g_cmd_cp_args.progname = argv[0];
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_cp_args) };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_cp_desc,
+                  g_cmd_cp_usage);
+     if (command_parse_args(&cmd, &g_cmd_cp_args.argc, &g_cmd_cp_args.argv)) {
+         exit(EINVALIDARGS);
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("cp: log init failed");
+         exit(ECOMMON);
+diff --git a/src/cmd/isula/stream/exec.c b/src/cmd/isula/stream/exec.c
+index 559a9d0f..d1d57268 100644
+--- a/src/cmd/isula/stream/exec.c
+++ b/src/cmd/isula/stream/exec.c
+@@ -198,12 +198,12 @@ static int exec_cmd_init(int argc, const char **argv)
+         EXEC_OPTIONS(g_cmd_exec_args)
+     };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_exec_args)) {
+         COMMAND_ERROR("client arguments init failed\n");
+         exit(ECOMMON);
+     }
+     g_cmd_exec_args.progname = argv[0];
+    isula_libutils_default_log_config(argv[0], &lconf);
+     command_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_exec_desc,
+                  g_cmd_exec_usage);
+ 
+diff --git a/src/cmd/isula/volume/list.c b/src/cmd/isula/volume/list.c
+index f58abc0f..23fe2027 100644
+--- a/src/cmd/isula/volume/list.c
+++ b/src/cmd/isula/volume/list.c
+@@ -145,12 +145,12 @@ int cmd_volume_ls_main(int argc, const char **argv)
+         COMMON_OPTIONS(g_cmd_volume_ls_args)
+     };
+ 
+    isula_libutils_default_log_config(argv[0], &lconf);
+     subcommand_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_volume_ls_desc,
+                     g_cmd_volume_ls_usage);
+     if (command_parse_args(&cmd, &g_cmd_volume_ls_args.argc, &g_cmd_volume_ls_args.argv)) {
+         exit(exit_code);
+     }
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (isula_libutils_log_enable(&lconf)) {
+         COMMAND_ERROR("volume ls: log init failed");
+         exit(exit_code);
+diff --git a/src/cmd/isula/volume/prune.c b/src/cmd/isula/volume/prune.c
+index e9d628d1..2a3bca3e 100644
+--- a/src/cmd/isula/volume/prune.c
+++ b/src/cmd/isula/volume/prune.c
+@@ -90,12 +90,12 @@ int cmd_volume_prune_main(int argc, const char **argv)
+         PRUNE_OPTIONS(g_cmd_volume_prune_args)
+     };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_volume_prune_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_volume_prune_args.progname = util_string_join(" ", argv, 2);
+    isula_libutils_default_log_config(argv[0], &lconf);
+     subcommand_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_volume_prune_desc,
+                     g_cmd_volume_prune_usage);
+ 
+diff --git a/src/cmd/isula/volume/remove.c b/src/cmd/isula/volume/remove.c
+index 2d10a002..71194722 100644
+--- a/src/cmd/isula/volume/remove.c
+++ b/src/cmd/isula/volume/remove.c
+@@ -80,12 +80,12 @@ int cmd_volume_rm_main(int argc, const char **argv)
+     command_t cmd;
+     struct command_option options[] = { LOG_OPTIONS(lconf) COMMON_OPTIONS(g_cmd_volume_rm_args) };
+ 
+-    isula_libutils_default_log_config(argv[0], &lconf);
+     if (client_arguments_init(&g_cmd_volume_rm_args)) {
+         COMMAND_ERROR("client arguments init failed");
+         exit(ECOMMON);
+     }
+     g_cmd_volume_rm_args.progname = util_string_join(" ", argv, 2);
+    isula_libutils_default_log_config(argv[0], &lconf);
+     subcommand_init(&cmd, options, sizeof(options) / sizeof(options[0]), argc, (const char **)argv, g_cmd_volume_rm_desc,
+                     g_cmd_volume_rm_usage);
+     if (command_parse_args(&cmd, &g_cmd_volume_rm_args.argc, &g_cmd_volume_rm_args.argv)) {
+-- 
+2.25.1
+
--- a/0021-spec-add-verify-for-device-cgroup-access-mode.patch
+++ b/0021-spec-add-verify-for-device-cgroup-access-mode.patch
@ -0,0 +1,102 @@
+From 82d59974b5fcb0abfa2f488801e7d9ed2f93a718 Mon Sep 17 00:00:00 2001
+From: Li Feng <lifeng2221dd1@zoho.com.cn>
+Date: Sat, 30 Jan 2021 14:22:16 +0800
+Subject: [PATCH 21/26] spec: add verify for device cgroup access mode
+
+Signed-off-by: Li Feng <lifeng2221dd1@zoho.com.cn>
+---
+ src/daemon/modules/spec/verify.c | 27 +++++++++++++++++++++++++++
+ src/utils/cutils/utils_verify.c  | 26 ++++++++++++++++++++++++--
+ 2 files changed, 51 insertions(+), 2 deletions(-)
+
+diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c
+index 053a57b3..a3156579 100644
+--- a/src/daemon/modules/spec/verify.c
+++ b/src/daemon/modules/spec/verify.c
+@@ -1064,6 +1064,26 @@ static int adapt_resources_memory(const sysinfo_t *sysinfo, defs_resources_memor
+     return adapt_memory_swap(sysinfo, &(memory->limit), &(memory->swap));
+ }
+ 
+/* verify resources device */
+static int verify_resources_device(defs_resources *resources)
+{
+    int ret = 0;
+    size_t i = 0;
+
+    for (i = 0; i < resources->devices_len; i++) {
+        if (!util_valid_device_mode(resources->devices[i]->access)) {
+            ERROR("Invalid device mode \"%s\" for device \"%ld %ld\"", resources->devices[i]->access,
+                  resources->devices[i]->major, resources->devices[i]->minor);
+            isulad_set_error_message("Invalid device mode \"%s\" for device \"%ld %ld\"", resources->devices[i]->access,
+                                     resources->devices[i]->major, resources->devices[i]->minor);
+            ret = -1;
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
+ /* verify linux resources */
+ static int verify_linux_resources(const sysinfo_t *sysinfo, defs_resources *resources)
+ {
+@@ -1104,6 +1124,13 @@ static int verify_linux_resources(const sysinfo_t *sysinfo, defs_resources *reso
+             goto out;
+         }
+     }
+    // device
+    if (resources->devices != NULL) {
+        ret = verify_resources_device(resources);
+        if (ret != 0) {
+            goto out;
+        }
+    }
+ out:
+     return ret;
+ }
+diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c
+index 5a18e664..58191685 100644
+--- a/src/utils/cutils/utils_verify.c
+++ b/src/utils/cutils/utils_verify.c
+@@ -184,14 +184,36 @@ bool util_validate_socket(const char *socket)
+ bool util_valid_device_mode(const char *mode)
+ {
+     size_t i = 0;
+    int r_count = 0;
+    int w_count = 0;
+    int m_count = 0;
+ 
+     if (mode == NULL || !strcmp(mode, "")) {
+         return false;
+     }
+ 
+     for (i = 0; i < strlen(mode); i++) {
+-        if (mode[i] != 'r' && mode[i] != 'w' && mode[i] != 'm') {
+-            return false;
+        switch (mode[i]) {
+            case 'r':
+                if (r_count != 0) {
+                    return false;
+                }
+                r_count++;
+                break;
+            case 'w':
+                if (w_count != 0) {
+                    return false;
+                }
+                w_count++;
+                break;
+            case 'm':
+                if (m_count != 0) {
+                    return false;
+                }
+                m_count++;
+                break;
+            default:
+                return false;
+         }
+     }
+ 
+-- 
+2.25.1
+
--- a/0022-log-change-log-level-from-warn-to-error.patch
+++ b/0022-log-change-log-level-from-warn-to-error.patch
@ -0,0 +1,26 @@
+From 87e886b239a932f37679f12fe2920d1b36e92985 Mon Sep 17 00:00:00 2001
+From: Li Feng <lifeng2221dd1@zoho.com.cn>
+Date: Sat, 30 Jan 2021 16:12:29 +0800
+Subject: [PATCH 22/26] log: change log level from warn to error
+
+Signed-off-by: Li Feng <lifeng2221dd1@zoho.com.cn>
+---
+ src/daemon/modules/service/service_container.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
+index a4a2414c..5eab8ccf 100644
+--- a/src/daemon/modules/service/service_container.c
+++ b/src/daemon/modules/service/service_container.c
+@@ -1288,7 +1288,7 @@ static int force_kill(container_t *cont)
+     }
+     ret = container_wait_stop(cont, 90);
+     if (ret != 0) {
+-        WARN("Container(%s) stuck for 90 seconds, try to kill the monitor of container", id);
+        ERROR("Container(%s) stuck for 90 seconds, try to kill the monitor of container", id);
+         ret = send_signal_to_process(cont->state->state->p_pid, cont->state->state->p_start_time, stop_signal, SIGKILL);
+         if (ret != 0) {
+             ERROR("Container stuck for 90 seconds and failed to kill the monitor of container, "
+-- 
+2.25.1
+
--- a/0023-Fix-create-env-path-dir-if-dir-exist.patch
+++ b/0023-Fix-create-env-path-dir-if-dir-exist.patch
@ -0,0 +1,27 @@
+From 2bd45202ef01260a2181270012c4781afd5cccba Mon Sep 17 00:00:00 2001
+From: zhangsong234 <zhangsong34@huawei.com>
+Date: Mon, 1 Feb 2021 09:25:51 +0800
+Subject: [PATCH 23/26] Fix create env path dir if dir exist
+
+---
+ src/daemon/modules/service/service_container.c | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
+index 0c1f234e..5fbb06a2 100644
+--- a/src/daemon/modules/service/service_container.c
+++ b/src/daemon/modules/service/service_container.c
+@@ -291,7 +291,9 @@ static int create_env_path_dir(const char *env_path)
+         free(dir);
+         return 0;
+     }
+-    ret = util_mkdir_p(dir, DEFAULT_SECURE_DIRECTORY_MODE);
+    if (!util_dir_exists(dir)) {
+        ret = util_mkdir_p(dir, DEFAULT_SECURE_DIRECTORY_MODE);
+    }
+     free(dir);
+     return ret;
+ }
+-- 
+2.25.1
+
--- a/0024-iSulad-calculate-memusage-with-used-total_inactive_f.patch
+++ b/0024-iSulad-calculate-memusage-with-used-total_inactive_f.patch
@ -0,0 +1,287 @@
+From c5aeb37655533ce84161f237ed6175153891d9e0 Mon Sep 17 00:00:00 2001
+From: Li Feng <lifeng2221dd1@zoho.com.cn>
+Date: Thu, 28 Jan 2021 19:30:44 +0800
+Subject: [PATCH 24/26] iSulad: calculate memusage with used -
+ total_inactive_file
+
+On cgroup v1 host, the result is `mem.used - mem.["total_inactive_file"]` .
+
+This definition is consistent with cadvisor and containerd/CRI.
+https://github.com/google/cadvisor/commit/307d1b1cb320fef66fab02db749f07a459245451
+https://github.com/containerd/cri/commit/6b8846cdf8b8c98c1d965313d66bc8489166059a
+
+Signed-off-by: Li Feng <lifeng2221dd1@zoho.com.cn>
+---
+ src/api/services/containers/container.proto   |  1 +
+ .../connect/grpc/grpc_containers_client.cc    |  4 +-
+ src/client/connect/protocol_type.h            |  1 +
+ src/cmd/isula/extend/stats.c                  | 16 +++++-
+ .../grpc/grpc_containers_service_private.cc   |  1 +
+ .../cri/cri_container_manager_service_impl.cc | 49 +++++++++----------
+ .../executor/container_cb/execution_extend.c  |  1 +
+ src/daemon/modules/api/runtime_api.h          |  1 +
+ .../modules/runtime/engines/lcr/lcr_engine.c  |  1 +
+ 9 files changed, 47 insertions(+), 28 deletions(-)
+
+diff --git a/src/api/services/containers/container.proto b/src/api/services/containers/container.proto
+index 36010860..efd085a1 100644
+--- a/src/api/services/containers/container.proto
+++ b/src/api/services/containers/container.proto
+@@ -85,6 +85,7 @@ message Container_info {
+ 	string status = 15;
+ 	uint64 cache = 16;
+ 	uint64 cache_total = 17;
+	uint64 inactive_file_total = 18;
+ }
+ 
+ message Event {
+diff --git a/src/client/connect/grpc/grpc_containers_client.cc b/src/client/connect/grpc/grpc_containers_client.cc
+index 9bb66b4b..ccde59a4 100644
+--- a/src/client/connect/grpc/grpc_containers_client.cc
+++ b/src/client/connect/grpc/grpc_containers_client.cc
+@@ -1688,6 +1688,7 @@ public:
+                 }
+                 response->container_stats[i].cache = gresponse->containers(i).cache();
+                 response->container_stats[i].cache_total = gresponse->containers(i).cache_total();
+                response->container_stats[i].inactive_file_total = gresponse->containers(i).inactive_file_total();
+             }
+             response->container_num = static_cast<size_t>(size);
+         }
+@@ -1981,7 +1982,8 @@ public:
+     explicit CopyToContainerWriteToServerTask(
+         const struct io_read_wrapper *reader,
+         std::shared_ptr<ClientReaderWriter<CopyToContainerRequest, CopyToContainerResponse>> stream)
+-        : m_reader(reader), m_stream(std::move(std::move(stream)))
+        : m_reader(reader)
+        , m_stream(std::move(std::move(stream)))
+     {
+     }
+     ~CopyToContainerWriteToServerTask() = default;
+diff --git a/src/client/connect/protocol_type.h b/src/client/connect/protocol_type.h
+index 6cbecf66..32f55b51 100644
+--- a/src/client/connect/protocol_type.h
+++ b/src/client/connect/protocol_type.h
+@@ -172,6 +172,7 @@ struct isula_container_info {
+     // Cache usage
+     uint64_t cache;
+     uint64_t cache_total;
+    uint64_t inactive_file_total;
+ };
+ 
+ struct isula_inspect_request {
+diff --git a/src/cmd/isula/extend/stats.c b/src/cmd/isula/extend/stats.c
+index 334f859e..03544325 100644
+--- a/src/cmd/isula/extend/stats.c
+++ b/src/cmd/isula/extend/stats.c
+@@ -76,6 +76,18 @@ static void stats_print_header(void)
+     printf(TERMNORM);
+ }
+ 
+// workingset = usage - total_inactive_file
+static uint64_t memory_get_working_set(const struct isula_container_info *stats)
+{
+    uint64_t workingset = stats->mem_used;
+
+    if (stats->inactive_file_total < stats->mem_used) {
+        workingset = stats->mem_used - stats->inactive_file_total;
+    }
+
+    return workingset;
+}
+
+ static void stats_print(const struct isula_container_info *stats)
+ {
+ #define SHORTIDLEN 12
+@@ -130,8 +142,10 @@ static void stats_print(const struct isula_container_info *stats)
+     if (strlen(short_id) > SHORTIDLEN) {
+         short_id[SHORTIDLEN] = '\0';
+     }
+    // workingset = usage - total_inactive_file
+    uint64_t workingset = memory_get_working_set(stats);
+     printf("%-16s %-10.2f %-26s %-10.2f %-26s %-10llu", short_id, cpu_percent, mem_str,
+-           stats->mem_limit ? ((double)stats->mem_used / stats->mem_limit) * PERCENT : 0.00, iosb_str,
+           stats->mem_limit ? ((double)workingset / stats->mem_limit) * PERCENT : 0.00, iosb_str,
+            (unsigned long long)stats->pids_current);
+     free(short_id);
+ }
+diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service_private.cc b/src/daemon/entry/connect/grpc/grpc_containers_service_private.cc
+index ac87a20b..8e19f978 100644
+--- a/src/daemon/entry/connect/grpc/grpc_containers_service_private.cc
+++ b/src/daemon/entry/connect/grpc/grpc_containers_service_private.cc
+@@ -790,6 +790,7 @@ void ContainerServiceImpl::stats_response_to_grpc(const container_stats_response
+             }
+             stats->set_cache(response->container_stats[i]->cache);
+             stats->set_cache_total(response->container_stats[i]->cache_total);
+            stats->set_inactive_file_total(response->container_stats[i]->inactive_file_total);
+         }
+     }
+     gresponse->set_cc(response->cc);
+diff --git a/src/daemon/entry/cri/cri_container_manager_service_impl.cc b/src/daemon/entry/cri/cri_container_manager_service_impl.cc
+index 812469ee..6c6569a6 100644
+--- a/src/daemon/entry/cri/cri_container_manager_service_impl.cc
+++ b/src/daemon/entry/cri/cri_container_manager_service_impl.cc
+@@ -126,9 +126,8 @@ auto ContainerManagerServiceImpl::PackCreateContainerHostConfigSecurityContext(
+     return 0;
+ }
+ 
+-auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig(const runtime::v1alpha2::ContainerConfig
+-                                                                    &containerConfig,
+-                                                                    Errors &error) -> host_config *
+auto ContainerManagerServiceImpl::GenerateCreateContainerHostConfig(
+    const runtime::v1alpha2::ContainerConfig &containerConfig, Errors &error) -> host_config *
+ {
+     host_config *hostconfig = (host_config *)util_common_calloc_s(sizeof(host_config));
+     if (hostconfig == nullptr) {
+@@ -294,11 +293,11 @@ cleanup:
+     return nullptr;
+ }
+ 
+-container_create_request *ContainerManagerServiceImpl::GenerateCreateContainerRequest(
+-    const std::string &realPodSandboxID,
+-    const runtime::v1alpha2::ContainerConfig &containerConfig,
+-    const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig,
+-    const std::string &podSandboxRuntime, Errors &error)
+container_create_request *
+ContainerManagerServiceImpl::GenerateCreateContainerRequest(const std::string &realPodSandboxID,
+                                                            const runtime::v1alpha2::ContainerConfig &containerConfig,
+                                                            const runtime::v1alpha2::PodSandboxConfig &podSandboxConfig,
+                                                            const std::string &podSandboxRuntime, Errors &error)
+ {
+     struct parser_context ctx {
+         OPT_GEN_SIMPLIFY, 0
+@@ -333,8 +332,7 @@ container_create_request *ContainerManagerServiceImpl::GenerateCreateContainerRe
+         hostconfig->cgroup_parent = util_strdup_s(podSandboxConfig.linux().cgroup_parent().c_str());
+     }
+ 
+-    custom_config = GenerateCreateContainerCustomConfig(realPodSandboxID, containerConfig,
+-                                                        podSandboxConfig, error);
+    custom_config = GenerateCreateContainerCustomConfig(realPodSandboxID, containerConfig, podSandboxConfig, error);
+     if (error.NotEmpty()) {
+         goto cleanup;
+     }
+@@ -611,7 +609,6 @@ void ContainerManagerServiceImpl::ListContainersToGRPC(container_list_response *
+     }
+ }
+ 
+-
+ void ContainerManagerServiceImpl::ListContainers(const runtime::v1alpha2::ContainerFilter *filter,
+                                                  std::vector<std::unique_ptr<runtime::v1alpha2::Container>> *containers,
+                                                  Errors &error)
+@@ -681,9 +678,8 @@ auto ContainerManagerServiceImpl::PackContainerStatsFilter(const runtime::v1alph
+     return 0;
+ }
+ 
+-void ContainerManagerServiceImpl::PackContainerStatsAttributes(const char *id,
+-                                                               std::unique_ptr<runtime::v1alpha2::ContainerStats> &container,
+-                                                               Errors &error)
+void ContainerManagerServiceImpl::PackContainerStatsAttributes(
+    const char *id, std::unique_ptr<runtime::v1alpha2::ContainerStats> &container, Errors &error)
+ {
+     if (id == nullptr) {
+         return;
+@@ -779,7 +775,11 @@ void ContainerManagerServiceImpl::ContainerStatsToGRPC(
+                                           container);
+ 
+         if (response->container_stats[i]->mem_used != 0u) {
+-            container->mutable_memory()->mutable_working_set_bytes()->set_value(response->container_stats[i]->mem_used);
+            uint64_t workingset = response->container_stats[i]->mem_used;
+            if (response->container_stats[i]->inactive_file_total < response->container_stats[i]->mem_used) {
+                workingset = response->container_stats[i]->mem_used - response->container_stats[i]->inactive_file_total;
+            }
+            container->mutable_memory()->mutable_working_set_bytes()->set_value(workingset);
+         }
+ 
+         if (response->container_stats[i]->cpu_use_nanos != 0u) {
+@@ -838,9 +838,8 @@ cleanup:
+     free_container_stats_response(response);
+ }
+ 
+-void ContainerManagerServiceImpl::PackContainerImageToStatus(container_inspect *inspect,
+-                                                             std::unique_ptr<runtime::v1alpha2::ContainerStatus> &contStatus,
+-                                                             Errors &error)
+void ContainerManagerServiceImpl::PackContainerImageToStatus(
+    container_inspect *inspect, std::unique_ptr<runtime::v1alpha2::ContainerStatus> &contStatus, Errors &error)
+ {
+     if (inspect->config == nullptr) {
+         return;
+@@ -854,9 +853,9 @@ void ContainerManagerServiceImpl::PackContainerImageToStatus(container_inspect *
+     return;
+ }
+ 
+-void ContainerManagerServiceImpl::UpdateBaseStatusFromInspect(container_inspect *inspect, int64_t &createdAt,
+-                                                              int64_t &startedAt, int64_t &finishedAt,
+-                                                              std::unique_ptr<runtime::v1alpha2::ContainerStatus> &contStatus)
+void ContainerManagerServiceImpl::UpdateBaseStatusFromInspect(
+    container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, int64_t &finishedAt,
+    std::unique_ptr<runtime::v1alpha2::ContainerStatus> &contStatus)
+ {
+     runtime::v1alpha2::ContainerState state { runtime::v1alpha2::CONTAINER_UNKNOWN };
+     std::string reason;
+@@ -1198,7 +1197,6 @@ auto ContainerManagerServiceImpl::InspectContainerState(const std::string &Id, E
+     return inspect_data;
+ }
+ 
+-
+ auto ContainerManagerServiceImpl::ValidateExecRequest(const runtime::v1alpha2::ExecRequest &req, Errors &error) -> int
+ {
+     if (req.container_id().empty()) {
+@@ -1260,8 +1258,8 @@ void ContainerManagerServiceImpl::Exec(const runtime::v1alpha2::ExecRequest &req
+     resp->set_url(url);
+ }
+ 
+-auto ContainerManagerServiceImpl::ValidateAttachRequest(const runtime::v1alpha2::AttachRequest &req,
+-                                                        Errors &error) -> int
+auto ContainerManagerServiceImpl::ValidateAttachRequest(const runtime::v1alpha2::AttachRequest &req, Errors &error)
+-> int
+ {
+     if (req.container_id().empty()) {
+         error.SetError("missing required container id!");
+@@ -1286,8 +1284,7 @@ auto ContainerManagerServiceImpl::ValidateAttachRequest(const runtime::v1alpha2:
+ }
+ 
+ void ContainerManagerServiceImpl::Attach(const runtime::v1alpha2::AttachRequest &req,
+-                                         runtime::v1alpha2::AttachResponse *resp,
+-                                         Errors &error)
+                                         runtime::v1alpha2::AttachResponse *resp, Errors &error)
+ {
+     if (ValidateAttachRequest(req, error) != 0) {
+         return;
+diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c
+index 40f24d29..2d5c6bed 100644
+--- a/src/daemon/executor/container_cb/execution_extend.c
+++ b/src/daemon/executor/container_cb/execution_extend.c
+@@ -268,6 +268,7 @@ static container_info *get_container_stats(const container_t *cont,
+     info->status = util_strdup_s(container_state_to_string(container_state_get_status(cont->state)));
+     info->cache = einfo->cache;
+     info->cache_total = einfo->cache_total;
+    info->inactive_file_total = einfo->inactive_file_total;
+ 
+     if (copy_map_labels(cont->common_config->config, &map_labels) != 0) {
+         ret = -1;
+diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h
+index f11228b6..dde21b91 100644
+--- a/src/daemon/modules/api/runtime_api.h
+++ b/src/daemon/modules/api/runtime_api.h
+@@ -61,6 +61,7 @@ struct runtime_container_resources_stats_info {
+     /* Cache usage */
+     uint64_t cache;
+     uint64_t cache_total;
+    uint64_t inactive_file_total;
+ };
+ 
+ typedef struct _rt_create_params_t {
+diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c
+index 691bfaa8..350f6497 100644
+--- a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c
+++ b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c
+@@ -164,6 +164,7 @@ static void copy_container_resources_stats(const struct lcr_container_state *lcs
+     rs_stats->kmem_limit = lcs->kmem_limit;
+     rs_stats->cache = lcs->cache;
+     rs_stats->cache_total = lcs->cache_total;
+    rs_stats->inactive_file_total = lcs->inactive_file_total;
+ }
+ 
+ /* get container cgroup resources */
+-- 
+2.25.1
+
--- a/0025-fix-container-exit-health-check-residue-and-multiple.patch
+++ b/0025-fix-container-exit-health-check-residue-and-multiple.patch
@ -0,0 +1,677 @@
+From 012b3f94279b0c6d193d510aa211b977a38e7c24 Mon Sep 17 00:00:00 2001
+From: wujing <jing.woo@outlook.com>
+Date: Fri, 22 Jan 2021 17:13:16 +0800
+Subject: [PATCH 25/26] fix container exit health check residue and multiple
+ health checks
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ .../executor/container_cb/execution_extend.c  |   7 +-
+ src/daemon/modules/api/container_api.h        |   2 +
+ .../container/container_events_handler.c      |   1 -
+ .../container/health_check/health_check.c     | 274 +++++++++++++-----
+ .../modules/service/service_container.c       |   3 +-
+ test/mocks/health_check_mock.cc               |   8 +
+ test/mocks/health_check_mock.h                |   1 +
+ .../execution_extend/execution_extend_ut.cc   |   6 +
+ 8 files changed, 220 insertions(+), 82 deletions(-)
+
+diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c
+index 40f24d29..01a0466f 100644
+--- a/src/daemon/executor/container_cb/execution_extend.c
+++ b/src/daemon/executor/container_cb/execution_extend.c
+@@ -532,6 +532,7 @@ static int do_resume_container(container_t *cont)
+ 
+     params.rootpath = cont->root_path;
+     params.state = cont->state_path;
+
+     if (runtime_resume(id, cont->runtime, &params)) {
+         ERROR("Failed to resume container:%s", id);
+         ret = -1;
+@@ -716,7 +717,11 @@ static int do_pause_container(container_t *cont)
+ 
+     params.rootpath = cont->root_path;
+     params.state = cont->state_path;
+
+    container_stop_health_checks(cont->common_config->id);
+
+     if (runtime_pause(id, cont->runtime, &params)) {
+        container_update_health_monitor(cont->common_config->id);
+         ERROR("Failed to pause container:%s", id);
+         ret = -1;
+         goto out;
+@@ -724,8 +729,6 @@ static int do_pause_container(container_t *cont)
+ 
+     container_state_set_paused(cont->state);
+ 
+-    container_update_health_monitor(cont->common_config->id);
+-
+     if (container_state_to_disk(cont)) {
+         ERROR("Failed to save container \"%s\" to disk", id);
+         ret = -1;
+diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h
+index 83216cf3..3b7f2889 100644
+--- a/src/daemon/modules/api/container_api.h
+++ b/src/daemon/modules/api/container_api.h
+@@ -49,6 +49,8 @@ typedef struct health_check_manager {
+     pthread_mutex_t mutex;
+     bool init_mutex;
+     health_check_monitor_status_t monitor_status;
+    // Used to wait for the health check minotor thread to close
+    bool monitor_exist;
+ } health_check_manager_t;
+ 
+ typedef struct _container_state_t_ {
+diff --git a/src/daemon/modules/container/container_events_handler.c b/src/daemon/modules/container/container_events_handler.c
+index 1283f99c..994c11cc 100644
+--- a/src/daemon/modules/container/container_events_handler.c
+++ b/src/daemon/modules/container/container_events_handler.c
+@@ -156,7 +156,6 @@ static int container_state_changed(container_t *cont, const struct isulad_events
+                 container_state_set_stopped(cont->state, (int)events->exit_status);
+                 container_wait_stop_cond_broadcast(cont);
+                 plugin_event_container_post_stop(cont);
+-                container_stop_health_checks(cont->common_config->id);
+             }
+ 
+             auto_remove = !should_restart && cont->hostconfig != NULL && cont->hostconfig->auto_remove;
+diff --git a/src/daemon/modules/container/health_check/health_check.c b/src/daemon/modules/container/health_check/health_check.c
+index 04467938..c6ccbbf2 100644
+--- a/src/daemon/modules/container/health_check/health_check.c
+++ b/src/daemon/modules/container/health_check/health_check.c
+@@ -73,24 +73,47 @@ static char *get_health_status(container_state_t *s)
+     return status;
+ }
+ 
+-static void set_health_status(container_state_t *s, const char *new)
+static void set_health_status(container_t *cont, const char *new)
+ {
+-    if (s == NULL || new == NULL) {
+    if (cont->state == NULL || new == NULL) {
+         return;
+     }
+-    container_state_lock(s);
+-    free(s->state->health->status);
+-    s->state->health->status = util_strdup_s(new);
+-    container_state_unlock(s);
+
+    container_state_lock(cont->state);
+    free(cont->state->state->health->status);
+    cont->state->state->health->status = util_strdup_s(new);
+    container_state_unlock(cont->state);
+
+    if (container_state_to_disk(cont)) {
+        WARN("Failed to save container \"%s\" to disk", cont->common_config->id);
+    }
+ }
+ 
+-static void set_monitor_idle_status(health_check_manager_t *health)
+static void init_monitor_idle_status(health_check_manager_t *health)
+ {
+     container_health_check_lock(health);
+     health->monitor_status = MONITOR_IDLE;
+     container_health_check_unlock(health);
+ }
+ 
+static int transfer_monitor_idle_status(health_check_manager_t *health)
+{
+    int ret = 0;
+
+    container_health_check_lock(health);
+    // When the minitor status is MONITOR_STOP, it cann't be set to minitor status
+    if (health->monitor_status == MONITOR_STOP) {
+        ret = -1;
+        goto out;
+    }
+
+    health->monitor_status = MONITOR_IDLE;
+
+out:
+    container_health_check_unlock(health);
+    return ret;
+}
+
+ static void set_monitor_stop_status(health_check_manager_t *health)
+ {
+     container_health_check_lock(health);
+@@ -98,11 +121,21 @@ static void set_monitor_stop_status(health_check_manager_t *health)
+     container_health_check_unlock(health);
+ }
+ 
+-static void set_monitor_interval_timeout_status(health_check_manager_t *health)
+static int transfer_monitor_interval_timeout_status(health_check_manager_t *health)
+ {
+    int ret = 0;
+
+     container_health_check_lock(health);
+    // When the minitor status is MONITOR_STOP, it cann't be set to minitor status
+    if (health->monitor_status == MONITOR_STOP) {
+        ret = -1;
+        goto out;
+    }
+     health->monitor_status = MONITOR_INTERVAL;
+
+out:
+     container_health_check_unlock(health);
+    return ret;
+ }
+ 
+ static health_check_monitor_status_t get_health_check_monitor_state(health_check_manager_t *health)
+@@ -116,18 +149,35 @@ static health_check_monitor_status_t get_health_check_monitor_state(health_check
+     return ret;
+ }
+ 
+-static void close_health_check_monitor(const container_t *cont)
+static void set_monitor_exist_flag(health_check_manager_t *health, bool closed)
+{
+    container_health_check_lock(health);
+    health->monitor_exist = closed;
+    container_health_check_unlock(health);
+}
+
+static bool get_monitor_exist_flag(health_check_manager_t *health)
+{
+    bool ret;
+
+    container_health_check_lock(health);
+    ret = health->monitor_exist;
+    container_health_check_unlock(health);
+
+    return ret;
+}
+
+static void close_health_check_monitor(container_t *cont)
+ {
+     if (cont == NULL || cont->health_check == NULL) {
+         return;
+     }
+-    set_monitor_stop_status(cont->health_check);
+-    set_health_status(cont->state, UNHEALTHY);
+-}
+ 
+-static void open_health_check_monitor(health_check_manager_t *health)
+-{
+-    set_monitor_interval_timeout_status(health);
+    set_monitor_stop_status(cont->health_check);
+    // ensure that the monitor process exits
+    while (get_monitor_exist_flag(cont->health_check)) {
+        util_usleep_nointerupt(500);
+    }
+ }
+ 
+ // Called when the container is being stopped (whether because the health check is
+@@ -160,6 +210,7 @@ void health_check_manager_free(health_check_manager_t *health_check)
+     if (health_check->init_mutex) {
+         pthread_mutex_destroy(&health_check->mutex);
+     }
+
+     free(health_check);
+ }
+ 
+@@ -183,6 +234,8 @@ static health_check_manager_t *health_check_manager_new()
+ 
+     health_check->monitor_status = MONITOR_IDLE;
+ 
+    health_check->monitor_exist = false;
+
+     return health_check;
+ cleanup:
+     health_check_manager_free(health_check);
+@@ -320,6 +373,43 @@ out:
+     return ret;
+ }
+ 
+static void *stop_container_on_unhealthy(void *arg)
+{
+    int ret = 0;
+    char *container_id = NULL;
+    container_t *cont = NULL;
+
+    if (arg == NULL) {
+        ERROR("Invalid input arguments");
+        return NULL;
+    }
+    container_id = (char *)arg;
+
+    ret = pthread_detach(pthread_self());
+    if (ret != 0) {
+        CRIT("Set thread detach fail");
+    }
+
+    prctl(PR_SET_NAME, "ExitOnUnhealthy");
+
+    cont = containers_store_get(container_id);
+    if (cont == NULL) {
+        ERROR("Failed to get container info");
+        goto out;
+    }
+
+    // kill container when exit on unhealthy flag is set
+    ret = stop_container(cont, 3, true, false);
+    if (ret != 0) {
+        ERROR("Could not stop running container %s, cannot remove", cont->common_config->id);
+    }
+
+out:
+    free(container_id);
+    container_unref(cont);
+    return NULL;
+}
+
+ static int handle_increment_streak(container_t *cont, int retries)
+ {
+     int ret = 0;
+@@ -328,18 +418,19 @@ static int handle_increment_streak(container_t *cont, int retries)
+     health = cont->state->state->health;
+     health->failing_streak++;
+     if (health->failing_streak >= retries) {
+-        set_health_status(cont->state, UNHEALTHY);
+        set_health_status(cont, UNHEALTHY);
+         if (cont->common_config->config->healthcheck->exit_on_unhealthy) {
+-            // kill container when exit on unhealthy flag is set
+-            ret = stop_container(cont, 3, true, false);
+-            if (ret != 0) {
+-                isulad_try_set_error_message("Could not stop running container %s, cannot remove",
+-                                             cont->common_config->id);
+-                ERROR("Could not stop running container %s, cannot remove", cont->common_config->id);
+            pthread_t stop_container_tid = { 0 };
+            char *container_id = util_strdup_s(cont->common_config->id);
+            if (pthread_create(&stop_container_tid, NULL, stop_container_on_unhealthy,
+                               (void *)container_id)) {
+                free(container_id);
+                ERROR("Failed to create thread to exec health check");
+                 ret = -1;
+             }
+         }
+     }
+
+     return ret;
+ }
+ 
+@@ -442,7 +533,7 @@ static int handle_probe_result(const char *container_id, const defs_health_log_e
+ 
+     if (result->exit_code == EXIT_STATUS_HEALTHY) {
+         health->failing_streak = 0;
+-        set_health_status(cont->state, HEALTHY);
+        set_health_status(cont, HEALTHY);
+     } else {
+         if (handle_unhealthy_case(cont, result, retries)) {
+             ERROR("failed to handle unhealthy case");
+@@ -457,10 +548,7 @@ static int handle_probe_result(const char *container_id, const defs_health_log_e
+         // note: event
+         EVENT("EVENT: {Object: %s, health_status: %s}", cont->common_config->id, current);
+     }
+-    if (container_state_to_disk(cont)) {
+-        ERROR("Failed to save container \"%s\" to disk", cont->common_config->id);
+-        ret = -1;
+-    }
+
+ out:
+     free(old_state);
+     free(current);
+@@ -499,10 +587,9 @@ static void health_check_exec_success_handle(const container_exec_response *cont
+ 
+ // exec the healthcheck command in the container.
+ // Returns the exit code and probe output (if any)
+-void *health_check_run(void *arg)
+static void health_check_run(const char *container_id)
+ {
+     int ret = 0;
+-    char *container_id = NULL;
+     char **cmd_slice = NULL;
+     char output[REV_BUF_SIZE] = { 0 };
+     char timebuffer[TIME_STR_SIZE] = { 0 };
+@@ -514,13 +601,6 @@ void *health_check_run(void *arg)
+     defs_health_log_element *result = NULL;
+     container_config *config = NULL;
+ 
+-    if (arg == NULL) {
+-        ERROR("Invalid input arguments");
+-        return NULL;
+-    }
+-
+-    container_id = util_strdup_s((char *)arg);
+-
+     cont = containers_store_get(container_id);
+     if (cont == NULL) {
+         ERROR("Failed to get container info");
+@@ -590,14 +670,10 @@ void *health_check_run(void *arg)
+ 
+ out:
+     util_free_array(cmd_slice);
+-    free(container_id);
+-    container_id = NULL;
+     free_defs_health_log_element(result);
+     free_container_exec_request(container_req);
+     free_container_exec_response(container_res);
+     container_unref(cont);
+-    DAEMON_CLEAR_ERRMSG();
+-    return NULL;
+ }
+ 
+ // Get a suitable probe implementation for the container's healthcheck configuration.
+@@ -623,41 +699,81 @@ static health_probe_t get_probe(const container_t *cont)
+     }
+ }
+ 
+static bool valid_container_status_for_health_check(const char *container_id)
+{
+    bool bret = true;
+    const char *id = NULL;
+    container_t *cont = NULL;
+
+    cont = containers_store_get(container_id);
+    if (cont == NULL) {
+        ERROR("No such container:%s", container_id);
+        bret = false;
+        goto out;
+    }
+
+    id = cont->common_config->id;
+
+    if (!container_is_running(cont->state)) {
+        ERROR("Container %s is not running.", id);
+        bret = false;
+        goto out;
+    }
+
+    if (container_is_paused(cont->state)) {
+        ERROR("Container %s is paused.", id);
+        bret = false;
+        goto out;
+    }
+
+    if (container_is_restarting(cont->state)) {
+        ERROR("Container %s is restarting.", id);
+        bret = false;
+        goto out;
+    }
+
+out:
+    container_unref(cont);
+    return bret;
+}
+
+ static int do_monitor_interval(const char *container_id, health_check_manager_t *health_check,
+                                types_timestamp_t *start_timestamp)
+ {
+     int ret = 0;
+-    pthread_t exec_tid = { 0 };
+ 
+-    if (pthread_create(&exec_tid, NULL, health_check_run, (void *)container_id)) {
+-        ERROR("Failed to create thread to exec health check");
+    if (!valid_container_status_for_health_check(container_id)) {
+        ERROR("Invalid container status for health check");
+         ret = -1;
+         goto out;
+     }
+-    if (pthread_join(exec_tid, NULL) != 0) {
+-        ERROR("Failed to run health check thread");
+
+    health_check_run(container_id);
+
+    if (transfer_monitor_idle_status(health_check) != 0) {
+         ret = -1;
+         goto out;
+     }
+-    if (get_health_check_monitor_state(health_check) == MONITOR_STOP) {
+-        ret = 0;
+-        goto out;
+-    }
+-    set_monitor_idle_status(health_check);
+     if (util_get_now_time_stamp(start_timestamp) == false) {
+         ERROR("Failed to get time stamp");
+         ret = -1;
+         goto out;
+     }
+
+ out:
+     return ret;
+ }
+ 
+-static int do_monitor_default(int64_t probe_interval, health_check_manager_t *health_check,
+static int do_monitor_default(const char *container_id, int64_t probe_interval, health_check_manager_t *health_check,
+                               const types_timestamp_t *start_timestamp, types_timestamp_t *last_timestamp)
+ {
+     int64_t time_interval = 0;
+ 
+    if (!valid_container_status_for_health_check(container_id)) {
+        ERROR("Invalid container status for health check");
+        return -1;
+    }
+
+     if (util_get_now_time_stamp(last_timestamp) == false) {
+         ERROR("Failed to get time stamp");
+         return -1;
+@@ -668,13 +784,14 @@ static int do_monitor_default(int64_t probe_interval, health_check_manager_t *he
+         return -1;
+     }
+ 
+-    if (time_interval >= probe_interval) {
+-        set_monitor_interval_timeout_status(health_check);
+    if (time_interval >= probe_interval && transfer_monitor_interval_timeout_status(health_check) != 0) {
+        return -1;
+     }
+     util_usleep_nointerupt(500);
+ 
+     return 0;
+ }
+
+ // Run the container's monitoring thread until notified via "stop".
+ // There is never more than one monitor thread running per container at a time.
+ static void *health_check_monitor(void *arg)
+@@ -689,14 +806,17 @@ static void *health_check_monitor(void *arg)
+         ERROR("Container id is empty");
+         return NULL;
+     }
+-    container_id = util_strdup_s((char *)arg);
+
+    container_id = (char *)arg;
+
+    prctl(PR_SET_NAME, "HealthCheck");
+ 
+     cont = containers_store_get(container_id);
+     if (cont == NULL) {
+         ERROR("Failed to get container info");
+         goto out;
+     }
+-
+    set_monitor_exist_flag(cont->health_check, true);
+     if (util_get_now_time_stamp(&start_timestamp) == false) {
+         ERROR("Failed to monitor start time stamp");
+         goto out;
+@@ -704,7 +824,7 @@ static void *health_check_monitor(void *arg)
+     probe_interval = (cont->common_config->config->healthcheck->interval == 0) ?
+                      DEFAULT_PROBE_INTERVAL :
+                      cont->common_config->config->healthcheck->interval;
+-    set_monitor_idle_status(cont->health_check);
+
+     while (true) {
+         switch (get_health_check_monitor_state(cont->health_check)) {
+             case MONITOR_STOP:
+@@ -712,30 +832,35 @@ static void *health_check_monitor(void *arg)
+                 goto out;
+             /* fall-through */
+             case MONITOR_INTERVAL:
+-                if (do_monitor_interval(container_id, cont->health_check, &start_timestamp)) {
+                if (do_monitor_interval(container_id, cont->health_check, &start_timestamp) != 0) {
+                     goto out;
+                 }
+                 break;
+             case MONITOR_IDLE:
+             /* fall-through */
+             default:
+-                if (do_monitor_default(probe_interval, cont->health_check, &start_timestamp, &last_timestamp)) {
+                if (do_monitor_default(container_id, probe_interval, cont->health_check,
+                                       &start_timestamp, &last_timestamp) != 0) {
+                     goto out;
+                 }
+                 break;
+         }
+     }
+
+ out:
+     free(container_id);
+     container_id = NULL;
+    //  unhealthy when the monitor has stopped for compatibility reasons
+    set_health_status(cont, UNHEALTHY);
+    // post semaphore, indicating that the minitor process has exited
+    set_monitor_exist_flag(cont->health_check, false);
+     container_unref(cont);
+     DAEMON_CLEAR_ERRMSG();
+     return NULL;
+ }
+ 
+ // Ensure the health-check monitor is running or not, depending on the current
+-// state of the container.
+-// Called from monitor.go, with c locked.
+// state of the container. Called from monitor, with c locked.
+ void container_update_health_monitor(const char *container_id)
+ {
+     bool want_running = false;
+@@ -746,6 +871,7 @@ void container_update_health_monitor(const char *container_id)
+     if (container_id == NULL) {
+         return;
+     }
+
+     cont = containers_store_get(container_id);
+     if (cont == NULL) {
+         ERROR("Failed to get container info");
+@@ -756,13 +882,18 @@ void container_update_health_monitor(const char *container_id)
+     if (health == NULL) {
+         goto out;
+     }
+
+     probe = get_probe(cont);
+-    want_running = cont->state->state->running && !cont->state->state->paused && probe != HEALTH_NONE;
+ 
+    want_running = container_is_running(cont->state) && !container_is_paused(cont->state) && probe != HEALTH_NONE;
+     if (want_running) {
+-        open_health_check_monitor(cont->health_check);
+         pthread_t monitor_tid = { 0 };
+-        if (pthread_create(&monitor_tid, NULL, health_check_monitor, (void *)container_id)) {
+        char *cid = util_strdup_s(container_id);
+        // ensured that the health check monitor process is stopped
+        close_health_check_monitor(cont);
+        init_monitor_idle_status(cont->health_check);
+        if (pthread_create(&monitor_tid, NULL, health_check_monitor, (void *)cid)) {
+            free(cid);
+             ERROR("Failed to create thread to monitor health check...");
+             goto out;
+         }
+@@ -779,8 +910,7 @@ out:
+ }
+ 
+ // Reset the health state for a newly-started, restarted or restored container.
+-// initHealthMonitor is called from monitor.go and we should never be running
+-// two instances at once.
+// initHealthMonitor is called from monitor and we should never be running two instances at once.
+ // Note: Called with container locked.
+ void container_init_health_monitor(const char *id)
+ {
+@@ -809,14 +939,9 @@ void container_init_health_monitor(const char *id)
+     if (get_probe(cont) == HEALTH_NONE) {
+         goto out;
+     }
+-    // This is needed in case we're auto-restarting
+-    container_stop_health_checks(cont->common_config->id);
+-    if (cont->state == NULL || cont->state->state == NULL) {
+-        goto out;
+-    }
+ 
+     if (cont->state->state->health != NULL) {
+-        set_health_status(cont->state, HEALTH_STARTING);
+        set_health_status(cont, HEALTH_STARTING);
+         cont->state->state->health->failing_streak = 0;
+     } else {
+         cont->state->state->health = util_common_calloc_s(sizeof(defs_health));
+@@ -824,12 +949,7 @@ void container_init_health_monitor(const char *id)
+             ERROR("out of memory");
+             goto out;
+         }
+-        set_health_status(cont->state, HEALTH_STARTING);
+-    }
+-
+-    if (container_state_to_disk(cont)) {
+-        ERROR("Failed to save container \"%s\" to disk", id);
+-        goto out;
+        set_health_status(cont, HEALTH_STARTING);
+     }
+ 
+     container_update_health_monitor(id);
+diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
+index a4a2414c..e96a94d0 100644
+--- a/src/daemon/modules/service/service_container.c
+++ b/src/daemon/modules/service/service_container.c
+@@ -1247,8 +1247,6 @@ static int kill_with_signal(container_t *cont, uint32_t signal)
+         goto out;
+     }
+ 
+-    container_stop_health_checks(id);
+-
+     ret = send_signal_to_process(cont->state->state->pid, cont->state->state->start_time, stop_signal, signal);
+     if (ret != 0) {
+         ERROR("Failed to send signal to container %s with signal %u", id, signal);
+@@ -1353,6 +1351,7 @@ int stop_container(container_t *cont, int timeout, bool force, bool restart)
+             goto out;
+         }
+     }
+
+ out:
+     if (restart) {
+         cont->hostconfig->auto_remove = cont->hostconfig->auto_remove_bak;
+diff --git a/test/mocks/health_check_mock.cc b/test/mocks/health_check_mock.cc
+index 879e4d9c..4347a04e 100644
+--- a/test/mocks/health_check_mock.cc
+++ b/test/mocks/health_check_mock.cc
+@@ -31,3 +31,11 @@ void container_update_health_monitor(const char *container_id)
+     }
+     return;
+ }
+
+void container_stop_health_checks(const char *container_id)
+{
+    if (g_health_check_mock != nullptr) {
+        return g_health_check_mock->ContainerStopHealthCheck(container_id);
+    }
+    return;
+}
+\ No newline at end of file
+diff --git a/test/mocks/health_check_mock.h b/test/mocks/health_check_mock.h
+index 7891f53c..ab8e20b0 100644
+--- a/test/mocks/health_check_mock.h
+++ b/test/mocks/health_check_mock.h
+@@ -22,6 +22,7 @@
+ class MockHealthCheck {
+ public:
+     MOCK_METHOD1(UpdateHealthMonitor, void(const char *container_id));
+    MOCK_METHOD1(ContainerStopHealthCheck, void(const char *container_id));
+ };
+ 
+ void MockHealthCheck_SetMock(MockHealthCheck* mock);
+diff --git a/test/services/execution/execute/execution_extend/execution_extend_ut.cc b/test/services/execution/execute/execution_extend/execution_extend_ut.cc
+index 2dc67814..03872340 100644
+--- a/test/services/execution/execute/execution_extend/execution_extend_ut.cc
+++ b/test/services/execution/execute/execution_extend/execution_extend_ut.cc
+@@ -204,6 +204,11 @@ void invokeStateSetPaused(container_state_t *s)
+     return;
+ }
+ 
+void invokeContainerStopHealthCheck(const char *container_id)
+{
+    return;
+}
+
+ TEST_F(ExecutionExtendUnitTest, test_container_extend_callback_init_pause)
+ {
+     service_container_callback_t cb;
+@@ -220,6 +225,7 @@ TEST_F(ExecutionExtendUnitTest, test_container_extend_callback_init_pause)
+     EXPECT_CALL(m_containerState, IsRestarting(_)).WillRepeatedly(Invoke(invokeIsRestarting));
+     EXPECT_CALL(m_containerUnix, ContainerToDisk(_)).WillRepeatedly(Invoke(invokeContainerToDisk));
+     EXPECT_CALL(m_containerUnix, ContainerStateToDisk(_)).WillRepeatedly(Invoke(invokeContainerStateToDisk));
+    EXPECT_CALL(m_healthCheck, ContainerStopHealthCheck(_)).WillRepeatedly(Invoke(invokeContainerStopHealthCheck));
+     container_extend_callback_init(&cb);
+     ASSERT_EQ(cb.pause(request, &response), 0);
+     testing::Mock::VerifyAndClearExpectations(&m_runtime);
+-- 
+2.25.1
+
--- a/0026-CI-supplementary-testcase-for-health-check-monitor.patch
+++ b/0026-CI-supplementary-testcase-for-health-check-monitor.patch
@ -0,0 +1,63 @@
+From 39e9ae73804880f523d83db6c8ad5d25d8bd79ed Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Mon, 1 Feb 2021 16:58:35 +0800
+Subject: [PATCH 26/26] CI: supplementary testcase for health check monitor
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ CI/test_cases/container_cases/health_check.sh | 33 +++++++++++++++++++
+ 1 file changed, 33 insertions(+)
+
+diff --git a/CI/test_cases/container_cases/health_check.sh b/CI/test_cases/container_cases/health_check.sh
+index eeb2749d..cc934fd8 100755
+--- a/CI/test_cases/container_cases/health_check.sh
+++ b/CI/test_cases/container_cases/health_check.sh
+@@ -160,6 +160,37 @@ function test_health_check_timeout()
+     return ${ret}
+ }
+ 
+function test_health_check_monitor()
+{
+    local ret=0
+    local image="busybox"
+    local test="health check monitor test => (${FUNCNAME[@]})"
+
+    msg_info "${test} starting..."
+
+    isula images | grep ${image}
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
+
+    isula rm -f $(isula ps -qa)
+
+    container_name="health_check_monitor"
+    isula run -itd -n ${container_name} --health-cmd="sleep 3" --health-interval 3s  busybox
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
+
+    isula stop -t 0 ${container_name} && isula start ${container_name} && \
+        isula stop -t 0 ${container_name} && isula start ${container_name}
+
+    health_check_monitor_count=$(ps -T -p $(cat /var/run/isulad.pid) | grep HealthCheck | wc -l)
+    [[ ${health_check_monitor_count} -ne 1 ]] && \
+        msg_err "${FUNCNAME[0]}:${LINENO} - multiple health check monitor thread container: ${container_name}" && ((ret++))
+
+    isula rm -f ${container_name}
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to remove container: ${container_name}" && ((ret++))
+
+    msg_info "${test} finished with return ${ret}..."
+    return ${ret}
+}
+
+ declare -i ans=0
+ 
+ test_health_check_paraments || ((ans++))
+@@ -168,5 +199,7 @@ test_health_check_normally || ((ans++))
+ 
+ test_health_check_timeout || ((ans++))
+ 
+test_health_check_monitor || ((ans++))
+
+ show_result ${ans} "${curr_path}/${0}"
+ 
+-- 
+2.25.1
+
--- a/iSulad.spec
+++ b/iSulad.spec
@ -1,5 +1,5 @@
 %global _version 2.0.8
-%global _release 20210118.195254.git077e10f2
+%global _release 20210202.153251.gite082dcf3
 %global is_systemd 1

 Name:      iSulad
@ -12,15 +12,32 @@ Source:    https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar
 BuildRoot: {_tmppath}/iSulad-%{version}
 ExclusiveArch:  x86_64 aarch64

-Patch1:	0001-make-thread-detach-to-avoid-resource-leak.patch
-Patch2:	0002-devmapper-fix-udev-wait-thread-resource-leak.patch
-Patch3:	0003-clean-code-fix-clean-code.patch
-Patch4:	0004-judge-isula-load-file-exists.patch
-Patch5:	0005-modify-image_load.sh-CI-to-test-file-not-exist.patch
-Patch6:	0006-do-not-pause-container-when-copy.patch
-Patch7:	0007-add-testcases-for-isula-cp.patch
-Patch8:	0008-image_cb-rename-the-function-isula_-docker_-to-do_.patch
-Patch9:	0009-fix-small-probability-of-coredump-in-CRI-streaming-s.patch
+Patch1: 0001-make-thread-detach-to-avoid-resource-leak.patch
+Patch2: 0002-devmapper-fix-udev-wait-thread-resource-leak.patch
+Patch3: 0003-clean-code-fix-clean-code.patch
+Patch4: 0004-judge-isula-load-file-exists.patch
+Patch5: 0005-modify-image_load.sh-CI-to-test-file-not-exist.patch
+Patch6: 0006-do-not-pause-container-when-copy.patch
+Patch7: 0007-add-testcases-for-isula-cp.patch
+Patch8: 0008-image_cb-rename-the-function-isula_-docker_-to-do_.patch
+Patch9: 0009-fix-small-probability-of-coredump-in-CRI-streaming-s.patch
+Patch10: 0010-fix-ramdom-coredump-if-pull-failed.patch
+Patch11: 0011-shim-optimize-io-stream.patch
+Patch12: 0012-add-CI-to-test-shim-io.patch
+Patch13: 0013-CI-add-testcase-for-exec-without-pty.patch
+Patch14: 0014-adapt-for-sparse-file-when-tar-file.patch
+Patch15: 0015-driver-do-not-unlock-and-destroy-lock-when-clean-up.patch
+Patch16: 0016-driver-do-not-set-g_graphdriver-to-NULL.patch
+Patch17: 0017-ignore-error-if-get-ip-failed.patch
+Patch18: 0018-GC-add-log-container-info-when-add-into-gc.patch
+Patch19: 0019-log-use-the-same-function-to-init-log-in-export-paus.patch
+Patch20: 0020-init-log-config-should-before-command-parse.patch
+Patch21: 0021-spec-add-verify-for-device-cgroup-access-mode.patch
+Patch22: 0022-log-change-log-level-from-warn-to-error.patch
+Patch23: 0023-Fix-create-env-path-dir-if-dir-exist.patch
+Patch24: 0024-iSulad-calculate-memusage-with-used-total_inactive_f.patch
+Patch25: 0025-fix-container-exit-health-check-residue-and-multiple.patch
+Patch26: 0026-CI-supplementary-testcase-for-health-check-monitor.patch

 %ifarch x86_64 aarch64
 Provides:       libhttpclient.so()(64bit)
@ -223,6 +240,12 @@ fi
 %endif

 %changelog
+* Tue Feb 2 2021 lifeng <lifeng68@huawei.com> - 2.0.8-20210202.153251.gite082dcf3
+- Type: sync from upstream
+- ID: NA
+- SUG: NA
+- DESC: update from master
+
 * Mon Jan 18 2020 lifeng <lifeng68@huawei.com> - 2.0.8-20210118.195254.git077e10f2
 - Type: sync from upstream
 - ID: NA