!510 update from upstream

From: @zh_xiaoyu Reviewed-by: @duguhaotian Signed-off-by: @duguhaotian
2022-12-17 07:55:00 +00:00 · 2022-12-17 07:55:00 +00:00 · 2af626978c
commit 2af626978c
parent 4b5149fd11 c2bb1cf604
13 changed files with 2771 additions and 1 deletions
--- a/0055-retry-call-runtime-ops.patch
+++ b/0055-retry-call-runtime-ops.patch
@ -0,0 +1,139 @@
+From 3ffaa7ae39d1af1d7c5aae976f1d6017ad76dee9 Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Wed, 30 Nov 2022 17:33:16 +0800
+Subject: [PATCH 55/65] retry call runtime ops
+
+1. add retry macro;
+2. retry call runtime cleanup ops;
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ src/daemon/modules/plugin/plugin.c            | 21 +++----------------
+ .../modules/runtime/isula/isula_rt_ops.c      | 14 +++++++++++--
+ src/utils/cutils/utils.c                      |  1 +
+ src/utils/cutils/utils.h                      | 19 +++++++++++++++++
+ 4 files changed, 35 insertions(+), 20 deletions(-)
+
+diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c
+index 725bca5b..18035518 100644
+--- a/src/daemon/modules/plugin/plugin.c
+++ b/src/daemon/modules/plugin/plugin.c
+@@ -71,7 +71,7 @@
+ // suffix is '.sock'
+ #define PLUGIN_SOCKET_FILE_SUFFIX_LEN 5
+ 
+-#define PLUGIN_ACTIVATE_MAX_RETRY 3
+#define PLUGIN_ACTIVATE_MAX_RETRY 5
+ 
+ #ifndef RestHttpHead
+ #define RestHttpHead "http://localhost"
+@@ -403,22 +403,6 @@ out:
+     return -1;
+ }
+ 
+-static int pm_activate_plugin_with_retry(plugin_t *plugin, size_t retry)
+-{
+-    size_t i = 0;
+-    int err = 0;
+-
+-    for (i = 0; i < retry; i++) {
+-        err = pm_activate_plugin(plugin);
+-        if (!err) {
+-            return 0;
+-        }
+-        sleep((unsigned int)i + 1);
+-    }
+-
+-    return err;
+-}
+-
+ static void pm_rdlock(void)
+ {
+     int errcode;
+@@ -500,7 +484,8 @@ static int pm_register_plugin(const char *name, const char *addr)
+         ERROR("alloc plugin failed");
+         goto failed;
+     }
+-    err = pm_activate_plugin_with_retry(plugin, PLUGIN_ACTIVATE_MAX_RETRY);
+
+    DO_RETYR_CALL(PLUGIN_ACTIVATE_MAX_RETRY, 1000000, err, pm_activate_plugin, plugin);
+     if (err != 0) {
+         ERROR("active plugin failed");
+         goto failed;
+diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+index c9667ee5..bfe7de08 100644
+--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+@@ -946,6 +946,7 @@ int rt_isula_restart(const char *name, const char *runtime, const rt_restart_par
+ int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_params_t *params)
+ {
+     char workdir[PATH_MAX] = { 0 };
+    int nret;
+ 
+     if (id == NULL || runtime == NULL || params == NULL) {
+         ERROR("nullptr arguments not allowed");
+@@ -966,8 +967,17 @@ int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_
+         shim_kill_force(workdir);
+     }
+ 
+-    (void)runtime_call_kill_force(workdir, runtime, id);
+-    (void)runtime_call_delete_force(workdir, runtime, id);
+    // retry 10 count call runtime kill, every call sleep 1s
+    DO_RETYR_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id);
+    if (nret != 0) {
+        WARN("call runtime force kill failed");
+    }
+
+    // retry 10 count call runtime delete, every call sleep 1s
+    DO_RETYR_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id);
+    if (nret != 0) {
+        WARN("call runtime force delete failed");
+    }
+ 
+     if (util_recursive_rmdir(workdir, 0) != 0) {
+         ERROR("failed rmdir -r shim workdir");
+diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c
+index 9f5deaf9..a154c52a 100644
+--- a/src/utils/cutils/utils.c
+++ b/src/utils/cutils/utils.c
+@@ -1216,6 +1216,7 @@ void util_usleep_nointerupt(unsigned long usec)
+         request = remain;
+     } while (ret == -1 && errno == EINTR);
+ }
+
+ int util_generate_random_str(char *id, size_t len)
+ {
+     int fd = -1;
+diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h
+index 27cfc902..0a9535a1 100644
+--- a/src/utils/cutils/utils.h
+++ b/src/utils/cutils/utils.h
+@@ -381,6 +381,25 @@ defs_map_string_object * dup_map_string_empty_object(defs_map_string_object *src
+ 
+ int convert_v2_runtime(const char *runtime, char *binary);
+ 
+/**
+ * retry_cnt: max count of call cb;
+ * interval_us: how many us to sleep, after call cb;
+ * cb: retry call function;
+ * return:
+ *  0 is cb successful at least once;
+ *  1 is all cb are failure;
+*/
+#define DO_RETYR_CALL(retry_cnt, interval_us, ret, cb, ...) do {    \
+        size_t i = 0;                                               \
+        for(; i < retry_cnt; i++) {                                 \
+            ret = cb(__VA_ARGS__);                                  \
+            if (ret == 0) {                                         \
+                break;                                              \
+            }                                                       \
+            util_usleep_nointerupt(interval_us);                    \
+        }                                                           \
+    } while(0)
+
+ #ifdef __cplusplus
+ }
+ #endif
+-- 
+2.25.1
+
--- a/0056-add-ut-test-for-retry-macro.patch
+++ b/0056-add-ut-test-for-retry-macro.patch
@ -0,0 +1,50 @@
+From 61a06b548e05edb3892eb08e1028ef71b41ee332 Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Fri, 2 Dec 2022 15:35:52 +0800
+Subject: [PATCH 56/65] add ut test for retry macro
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ test/cutils/utils_utils/utils_utils_ut.cc | 27 +++++++++++++++++++++++
+ 1 file changed, 27 insertions(+)
+
+diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc
+index 531947d2..5bd98d47 100644
+--- a/test/cutils/utils_utils/utils_utils_ut.cc
+++ b/test/cutils/utils_utils/utils_utils_ut.cc
+@@ -269,4 +269,31 @@ TEST(utils_utils, test_convert_v2_runtime)
+     ASSERT_EQ(convert_v2_runtime(nullptr, buff), -1);
+     ASSERT_EQ(convert_v2_runtime(valid_str.c_str(), nullptr), -1);
+     ASSERT_EQ(convert_v2_runtime(valid_str.c_str(), buff), 0);
+}
+
+int global_total = 0;
+int retry_call_test(int success_idx) {
+    if (global_total == success_idx) {
+        return 0;
+    }
+    global_total++;
+    return -1;
+}
+
+TEST(utils_utils, test_do_retry_call)
+{
+    int nret;
+
+    global_total = 0;
+    DO_RETYR_CALL(10, 100, nret, retry_call_test, 0);
+    ASSERT_EQ(nret, 0);
+    ASSERT_EQ(global_total, 0);
+    global_total = 0;
+    DO_RETYR_CALL(10, 100, nret, retry_call_test, 5);
+    ASSERT_EQ(nret, 0);
+    ASSERT_EQ(global_total, 5);
+    global_total = 0;
+    DO_RETYR_CALL(10, 100, nret, retry_call_test, 11);
+    ASSERT_EQ(global_total, 10);
+    ASSERT_EQ(nret, -1);
+ }
+\ No newline at end of file
+-- 
+2.25.1
+
--- a/0057-1749-set-inspect_container-timeout.patch
+++ b/0057-1749-set-inspect_container-timeout.patch
@ -0,0 +1,98 @@
+From 3e51dc0746c5e0692f54cf54d7aa6beb3b13d799 Mon Sep 17 00:00:00 2001
+From: ger202 <huangsong14@huawei.com>
+Date: Wed, 7 Dec 2022 03:21:17 +0000
+Subject: [PATCH 57/65] !1749 set inspect_container timeout * update function
+ inspect_container timeout
+
+---
+ src/cmd/isula/information/top.c     | 4 +++-
+ src/cmd/isula/stream/attach.c       | 5 ++++-
+ src/cmd/isula/stream/exec.c         | 4 +++-
+ src/daemon/entry/cri/cri_helpers.cc | 3 +--
+ src/utils/cutils/utils.h            | 3 +++
+ 5 files changed, 14 insertions(+), 5 deletions(-)
+
+diff --git a/src/cmd/isula/information/top.c b/src/cmd/isula/information/top.c
+index 603f97ba..e55ec43b 100644
+--- a/src/cmd/isula/information/top.c
+++ b/src/cmd/isula/information/top.c
+@@ -29,7 +29,9 @@
+ const char g_cmd_top_desc[] = "Display the running processes of a container";
+ const char g_cmd_top_usage[] = "top [OPTIONS] CONTAINER [ps OPTIONS]";
+ 
+-struct client_arguments g_cmd_top_args = {};
+struct client_arguments g_cmd_top_args = {
+    .time = INSPECT_TIMEOUT_SEC,
+};
+ static void client_top_info_server(const struct isula_top_response *response)
+ {
+     size_t i;
+diff --git a/src/cmd/isula/stream/attach.c b/src/cmd/isula/stream/attach.c
+index f260a76e..7fb1046a 100644
+--- a/src/cmd/isula/stream/attach.c
+++ b/src/cmd/isula/stream/attach.c
+@@ -48,7 +48,9 @@ sem_t g_attach_waitopen_sem;
+ sem_t g_attach_waitexit_sem;
+ #endif
+ 
+-struct client_arguments g_cmd_attach_args = { 0 };
+struct client_arguments g_cmd_attach_args = { 
+    .time = INSPECT_TIMEOUT_SEC,
+};
+ 
+ 
+ static int check_tty(bool tty, struct termios *oldtios, bool *reset_tty)
+@@ -116,6 +118,7 @@ int inspect_container(const struct client_arguments *args, container_inspect **i
+     }
+ 
+     inspect_request.name = args->name;
+    inspect_request.timeout = args->time;
+     ops = get_connect_client_ops();
+     if (ops == NULL || !ops->container.inspect) {
+         COMMAND_ERROR("Unimplemented ops");
+diff --git a/src/cmd/isula/stream/exec.c b/src/cmd/isula/stream/exec.c
+index 2d0d37da..bd8bd49a 100644
+--- a/src/cmd/isula/stream/exec.c
+++ b/src/cmd/isula/stream/exec.c
+@@ -42,7 +42,9 @@ const char g_cmd_exec_usage[] = "exec [OPTIONS] CONTAINER COMMAND [ARG...]";
+ sem_t g_command_waitopen_sem;
+ sem_t g_command_waitexit_sem;
+ 
+-struct client_arguments g_cmd_exec_args = {};
+struct client_arguments g_cmd_exec_args = {
+    .time = INSPECT_TIMEOUT_SEC,
+};
+ 
+ static int fill_exec_request(const struct client_arguments *args, const struct command_fifo_config *fifos,
+                              struct isula_exec_request *request)
+diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc
+index 7df759e1..64cea7ba 100644
+--- a/src/daemon/entry/cri/cri_helpers.cc
+++ b/src/daemon/entry/cri/cri_helpers.cc
+@@ -720,8 +720,7 @@ out:
+ auto InspectContainer(const std::string &Id, Errors &err, bool with_host_config) -> container_inspect *
+ {
+     container_inspect *inspect_data { nullptr };
+-
+-    inspect_data = inspect_container((const char *)Id.c_str(), 0, with_host_config);
+    inspect_data = inspect_container((const char *)Id.c_str(), INSPECT_TIMEOUT_SEC, with_host_config);
+     if (inspect_data == nullptr) {
+         err.Errorf("Failed to call inspect service %s", Id.c_str());
+     }
+diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h
+index 0a9535a1..4518e3ac 100644
+--- a/src/utils/cutils/utils.h
+++ b/src/utils/cutils/utils.h
+@@ -123,6 +123,9 @@ int malloc_trim(size_t pad);
+ 
+ #define TIME_STR_SIZE 512
+ 
+// client inspect container timeout
+#define INSPECT_TIMEOUT_SEC 120
+
+ // native umask value
+ #define ANNOTATION_UMAKE_KEY "native.umask"
+ #define UMASK_NORMAL "normal"
+-- 
+2.25.1
+
--- a/0058-1757-add-adaption-code-for-musl.patch
+++ b/0058-1757-add-adaption-code-for-musl.patch
@ -0,0 +1,364 @@
+From 484852b127dab5f5548ed34d5bb668b18e4dc99e Mon Sep 17 00:00:00 2001
+From: zhongtao <taozh97@163.com>
+Date: Fri, 9 Dec 2022 07:01:31 +0000
+Subject: [PATCH 58/65] !1757 add adaption code for musl. * add adaption code
+ for musl.
+
+---
+ cmake/checker.cmake                                 |  2 +-
+ cmake/options.cmake                                 | 12 ++++++++++++
+ cmake/set_build_flags.cmake                         |  5 +++++
+ src/CMakeLists.txt                                  |  6 +++---
+ src/daemon/common/selinux_label.c                   |  2 +-
+ src/daemon/common/sysinfo.c                         |  4 ++--
+ src/daemon/executor/container_cb/execution_stream.c |  2 +-
+ src/daemon/modules/image/CMakeLists.txt             |  2 +-
+ src/daemon/modules/image/image.c                    |  6 ++++--
+ src/daemon/modules/image/image_rootfs_handler.c     | 12 ++++++------
+ src/utils/cutils/CMakeLists.txt                     |  2 +-
+ src/utils/cutils/utils.c                            | 13 +++++++++----
+ src/utils/cutils/utils.h                            |  2 +-
+ src/utils/cutils/utils_verify.c                     |  8 ++++----
+ 14 files changed, 51 insertions(+), 27 deletions(-)
+
+diff --git a/cmake/checker.cmake b/cmake/checker.cmake
+index cbcfc929..fea4f925 100644
+--- a/cmake/checker.cmake
+++ b/cmake/checker.cmake
+@@ -54,7 +54,7 @@ find_library(CRYPTO_LIBRARY crypto
+     HINTS ${PC_CRYPTO_LIBDIR} ${PC_LIBCRYPTO_LIBRARY_DIRS})
+ _CHECK(CRYPTO_LIBRARY "CRYPTO_LIBRARY-NOTFOUND" "libcrypto.so")
+ 
+-if (ANDROID)
+if (ANDROID OR MUSL)
+     # check libssl
+     find_library(LIBSSL_LIBRARY ssl)
+     _CHECK(CRYPTO_LIBRARY "LIBSSL_LIBRARY-NOTFOUND" "libssl.so")
+diff --git a/cmake/options.cmake b/cmake/options.cmake
+index 7a141fef..0d44868a 100644
+--- a/cmake/options.cmake
+++ b/cmake/options.cmake
+@@ -115,6 +115,18 @@ if (ENABLE_SUP_GROUPS)
+     message("${Green}--  Enable sup groups${ColourReset}")
+ endif()
+ 
+option(MUSL "available for musl" OFF)
+if (MUSL)
+    add_definitions(-D__MUSL__)
+    message("${Green}--  Available for MUSL${ColourReset}")
+endif()
+
+option(ANDROID "available for android" OFF)
+if (ANDROID)
+    add_definitions(-D__ANDROID__)
+    message("${Green}--  Available for ANDROID${ColourReset}")
+endif()
+
+ if (NOT RUNPATH)
+     set(RUNPATH "/var/run")
+ endif()
+diff --git a/cmake/set_build_flags.cmake b/cmake/set_build_flags.cmake
+index fa6f38c0..89c9468c 100644
+--- a/cmake/set_build_flags.cmake
+++ b/cmake/set_build_flags.cmake
+@@ -9,6 +9,11 @@ endif()
+ set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-E -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -Wtrampolines -shared -pthread")
+ set(CMAKE_EXE_LINKER_FLAGS "-Wl,-E -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -Wtrampolines -pie -rdynamic")
+ 
+if (NOT DISABLE_WERROR)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
+endif()
+
+ if (ISULAD_GCOV)
+     set(CMAKE_C_FLAGS_DEBUG "-Wall -fprofile-arcs -ftest-coverage")
+     set(CMAKE_CXX_FLAGS_DEBUG "-Wall -fprofile-arcs -ftest-coverage")
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index 65bcb978..f3dd3c19 100644
+--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
+@@ -102,7 +102,7 @@ add_executable(isula
+     )
+ target_include_directories(isula PUBLIC ${ISULA_INCS} ${SHARED_INCS})
+ target_link_libraries(isula libisula ${LIBYAJL_LIBRARY})
+-if (ANDROID)
+if (ANDROID OR MUSL)
+     target_link_libraries(isula ${LIBSSL_LIBRARY})
+ else()
+     target_link_libraries(isula -lpthread)
+@@ -115,7 +115,7 @@ add_executable(isulad-shim
+     )
+ target_include_directories(isulad-shim PUBLIC ${ISULAD_SHIM_INCS} ${SHARED_INCS})
+ target_link_libraries(isulad-shim libisulad_tools)
+-if (ANDROID)
+if (ANDROID OR MUSL)
+     target_link_libraries(isulad-shim ${LIBSSL_LIBRARY} ${LIBYAJL_LIBRARY})
+ else()
+     target_link_libraries(isulad-shim -lpthread)
+@@ -181,7 +181,7 @@ else()
+ endif()
+ 
+ target_link_libraries(isulad libisulad_tools libhttpclient -ldl)
+-if (ANDROID)
+if (ANDROID OR MUSL)
+     target_link_libraries(isulad ${LIBSSL_LIBRARY} ${LIBYAJL_LIBRARY})
+ else()
+     target_link_libraries(isulad -lpthread)
+diff --git a/src/daemon/common/selinux_label.c b/src/daemon/common/selinux_label.c
+index 145e4b6e..24294780 100644
+--- a/src/daemon/common/selinux_label.c
+++ b/src/daemon/common/selinux_label.c
+@@ -592,7 +592,7 @@ static int container_label(char **process_label, char **file_label)
+         return 0;
+     }
+ 
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__)
+     lxc_path = ISULAD_DAEMON_CONTAINER_CONTEXTS;
+ #else
+     lxc_path = selinux_lxc_contexts_path();
+diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c
+index d52f8767..e7b3807b 100644
+--- a/src/daemon/common/sysinfo.c
+++ b/src/daemon/common/sysinfo.c
+@@ -1308,7 +1308,7 @@ out:
+     return ret;
+ }
+ 
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__)
+ static bool cgroup2_no_controller()
+ {
+     char *controllers_str = NULL;
+@@ -1336,7 +1336,7 @@ static int make_sure_cgroup2_isulad_path_exist()
+         return -1;
+     }
+ 
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__)
+     if (cgroup2_no_controller()) {
+         DEBUG("no cgroup controller found");
+         return 0;
+diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c
+index 9af6fb5c..ebb9ee2b 100644
+--- a/src/daemon/executor/container_cb/execution_stream.c
+++ b/src/daemon/executor/container_cb/execution_stream.c
+@@ -63,7 +63,7 @@
+ #include "utils_file.h"
+ #include "utils_verify.h"
+ 
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__)
+ #define SIG_CANCEL_SIGNAL     SIGUSR1
+ #define PTHREAD_CANCEL_ENABLE    1
+ #define PTHREAD_CANCEL_DISABLE    0
+diff --git a/src/daemon/modules/image/CMakeLists.txt b/src/daemon/modules/image/CMakeLists.txt
+index 329d2937..6f9e9936 100644
+--- a/src/daemon/modules/image/CMakeLists.txt
+++ b/src/daemon/modules/image/CMakeLists.txt
+@@ -117,7 +117,7 @@ target_link_libraries(${LIB_ISULAD_IMG}
+     ${ZLIB_LIBRARY}
+     libhttpclient)
+ 
+-if (NOT ANDROID)
+if ((NOT ANDROID) AND (NOT MUSL))
+     target_link_libraries(${LIB_ISULAD_IMG} -lpthread)
+ endif()
+ 
+diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c
+index fb0db361..8fb226aa 100644
+--- a/src/daemon/modules/image/image.c
+++ b/src/daemon/modules/image/image.c
+@@ -1773,7 +1773,8 @@ int im_container_export(const im_export_request *request)
+ #endif
+ 
+ #ifdef ENABLE_OCI_IMAGE
+-char *im_get_rootfs_dir(const im_get_rf_dir_request *request) {
+char *im_get_rootfs_dir(const im_get_rf_dir_request *request)
+{
+     char *dir = NULL;
+     struct bim *bim = NULL;
+ 
+@@ -1801,7 +1802,8 @@ out:
+     return dir;
+ }
+ #else
+-char *im_get_rootfs_dir(const im_get_rf_dir_request *request) {
+char *im_get_rootfs_dir(const im_get_rf_dir_request *request)
+{
+     return NULL;
+ }
+ #endif
+diff --git a/src/daemon/modules/image/image_rootfs_handler.c b/src/daemon/modules/image/image_rootfs_handler.c
+index a76363d0..c3964b2c 100644
+--- a/src/daemon/modules/image/image_rootfs_handler.c
+++ b/src/daemon/modules/image/image_rootfs_handler.c
+@@ -87,7 +87,7 @@ static int proc_by_fpasswd(FILE *f_passwd, const char *user, defs_process_user *
+     struct passwd *pwbufp = NULL;
+ 
+     if (f_passwd != NULL) {
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__) 
+         errval = util_getpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp);
+ #else
+         errval = fgetpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp);
+@@ -105,7 +105,7 @@ static int proc_by_fpasswd(FILE *f_passwd, const char *user, defs_process_user *
+                 *matched_username = util_strdup_s(pwbufp->pw_name);
+                 break;
+             }
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__) 
+             errval = util_getpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp);
+ #else
+             errval = fgetpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp);
+@@ -215,7 +215,7 @@ static int do_proc_by_froup(FILE *f_group, const char *group, defs_process_user
+         return 0;
+     }
+ 
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__) 
+     errval = util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp);
+ #else
+     errval = fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp);
+@@ -226,7 +226,7 @@ static int do_proc_by_froup(FILE *f_group, const char *group, defs_process_user
+             if (search_group_list(gbufp, matched_username, puser) != 0) {
+                 return -1;
+             }
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__) 
+             errval = util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp);
+ #else
+             errval = fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp);
+@@ -240,7 +240,7 @@ static int do_proc_by_froup(FILE *f_group, const char *group, defs_process_user
+             puser->gid = gbufp->gr_gid;
+             *groupcnt = 1;
+         }
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__) 
+         errval = util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp);
+ #else
+         errval = fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp);
+@@ -378,7 +378,7 @@ static int get_additional_groups(char **additional_groups, size_t additional_gro
+     struct group *gbufp = NULL;
+     struct group *groups = NULL;
+ 
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__) 
+     while (f_group != NULL && util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp) == 0) {
+ #else
+     while (f_group != NULL && fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp) == 0) {
+diff --git a/src/utils/cutils/CMakeLists.txt b/src/utils/cutils/CMakeLists.txt
+index 30414d91..50a17f60 100644
+--- a/src/utils/cutils/CMakeLists.txt
+++ b/src/utils/cutils/CMakeLists.txt
+@@ -2,7 +2,7 @@
+ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_cutils_srcs)
+ add_subdirectory(map)
+ 
+-if (NOT ANDROID)
+if ((NOT ANDROID) AND (NOT MUSL))
+     list(REMOVE_ITEM local_cutils_srcs "${CMAKE_CURRENT_SOURCE_DIR}/utils_pwgr.c")
+ endif()
+ 
+diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c
+index a154c52a..b1db045a 100644
+--- a/src/utils/cutils/utils.c
+++ b/src/utils/cutils/utils.c
+@@ -16,7 +16,7 @@
+ #define _GNU_SOURCE
+ #include "utils.h"
+ #include <errno.h>
+-#ifndef __ANDROID__
+#if !defined (__ANDROID__) && !defined(__MUSL__)
+ #include <execinfo.h>
+ #endif
+ #include <stdint.h>
+@@ -47,7 +47,7 @@
+ #include "utils_string.h"
+ #include "utils_verify.h"
+ 
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__)
+ int mallopt(int param, int value)
+ {
+     return 1;
+@@ -427,7 +427,12 @@ proc_t *util_stat2proc(const char *s, size_t len)
+ 
+     /* parse these two strings separately, skipping the leading "(". */
+     /* comm[16] in kernel */
+    /* https://www.openwall.com/lists/musl/2013/11/15/5: musl's sscanf("%15c",cmd) requires exactly 15 characters; anything shorter is a matching failure. */
+#ifdef __MUSL__
+    num = sscanf(s, "%d (%15s", &p->pid, p->cmd);
+#else
+     num = sscanf(s, "%d (%15c", &p->pid, p->cmd);
+#endif
+     if (num != 2) {
+         ERROR("Call sscanf error: %s", errno ? strerror(errno) : "");
+         free(p);
+@@ -851,8 +856,8 @@ out:
+ 
+ char **util_get_backtrace(void)
+ {
+-#ifdef __ANDROID__
+-    /* android has no backtrace */
+#if defined (__ANDROID__) || defined(__MUSL__)
+    /* android and musl has no backtrace */
+     return NULL;
+ #else
+ #define BACKTRACE_SIZE 16
+diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h
+index 4518e3ac..72cab9f2 100644
+--- a/src/utils/cutils/utils.h
+++ b/src/utils/cutils/utils.h
+@@ -47,7 +47,7 @@
+ extern "C" {
+ #endif
+ 
+-#ifdef __ANDROID__
+#if defined (__ANDROID__) || defined(__MUSL__)
+ #define M_TRIM_THRESHOLD    -1
+ #define M_TOP_PAD           -2
+ #define M_MMAP_THRESHOLD    -3
+diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c
+index 9ed33bf3..713e72c3 100644
+--- a/src/utils/cutils/utils_verify.c
+++ b/src/utils/cutils/utils_verify.c
+@@ -563,7 +563,7 @@ bool util_valid_value_false(const char *value)
+ 
+ bool util_valid_rw_mode(const char *mode)
+ {
+-    if (mode == NULL){
+    if (mode == NULL) {
+         return false;
+     }
+ 
+@@ -572,7 +572,7 @@ bool util_valid_rw_mode(const char *mode)
+ 
+ bool util_valid_label_mode(const char *mode)
+ {
+-    if (mode == NULL){
+    if (mode == NULL) {
+         return false;
+     }
+ 
+@@ -581,7 +581,7 @@ bool util_valid_label_mode(const char *mode)
+ 
+ bool util_valid_copy_mode(const char *mode)
+ {
+-    if (mode == NULL){
+    if (mode == NULL) {
+         return false;
+     }
+     return !strcmp(mode, "nocopy");
+@@ -708,7 +708,7 @@ int util_valid_env(const char *env, char **dst)
+     int ret = 0;
+     char *value = NULL;
+ 
+-    if (dst == NULL){
+    if (dst == NULL) {
+         ERROR("NULL dst");
+         return -1;
+     }
+-- 
+2.25.1
+
--- a/0059-deleting-broken-rootfs.patch
+++ b/0059-deleting-broken-rootfs.patch
@ -0,0 +1,880 @@
+From edb570b8720aed234bf2c17642f5e6caba2c726a Mon Sep 17 00:00:00 2001
+From: "Neil.wrz" <wangrunze13@huawei.com>
+Date: Tue, 6 Dec 2022 19:23:33 -0800
+Subject: [PATCH 59/65] deleting broken rootfs
+
+Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
+---
+ cmake/options.cmake                           |  6 ++
+ src/cmd/isulad/main.c                         | 15 ++++
+ src/daemon/modules/api/image_api.h            |  2 +
+ src/daemon/modules/api/leftover_cleanup_api.h | 36 +++++++++
+ src/daemon/modules/container/CMakeLists.txt   |  2 +
+ src/daemon/modules/container/container_unix.c |  2 -
+ .../leftover_cleanup/clean_context.c          | 78 +++++++++++++++++++
+ .../leftover_cleanup/clean_context.h          | 41 ++++++++++
+ .../container/leftover_cleanup/cleanup.c      | 46 +++++------
+ .../container/leftover_cleanup/cleanup.h      | 12 ++-
+ .../leftover_cleanup/leftover_cleanup_api.c   | 71 +++++++++++++++++
+ .../leftover_cleanup/oci_rootfs_clean.c       | 34 +++++++-
+ .../leftover_cleanup/oci_rootfs_clean.h       |  4 +-
+ src/daemon/modules/image/image.c              | 48 ++++++++++++
+ src/daemon/modules/image/oci/oci_image.c      | 10 +++
+ src/daemon/modules/image/oci/oci_image.h      |  1 +
+ .../oci/storage/layer_store/layer_store.c     |  2 +
+ .../oci/storage/rootfs_store/rootfs_store.c   | 13 +++-
+ .../modules/image/oci/storage/storage.c       | 32 ++++++++
+ .../modules/image/oci/storage/storage.h       |  3 +
+ test/image/oci/storage/rootfs/CMakeLists.txt  |  2 +
+ 21 files changed, 422 insertions(+), 38 deletions(-)
+ create mode 100644 src/daemon/modules/api/leftover_cleanup_api.h
+ create mode 100644 src/daemon/modules/container/leftover_cleanup/clean_context.c
+ create mode 100644 src/daemon/modules/container/leftover_cleanup/clean_context.h
+ create mode 100644 src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c
+
+diff --git a/cmake/options.cmake b/cmake/options.cmake
+index 0d44868a..1e63a485 100644
+--- a/cmake/options.cmake
+++ b/cmake/options.cmake
+@@ -115,6 +115,12 @@ if (ENABLE_SUP_GROUPS)
+     message("${Green}--  Enable sup groups${ColourReset}")
+ endif()
+ 
+option(DISABLE_CLEANUP "disable cleanup module" OFF)
+if (DISABLE_CLEANUP STREQUAL "ON")
+    add_definitions(-DDISABLE_CLEANUP)
+    message("${Green}--  Disable cleanup module")
+endif()
+
+ option(MUSL "available for musl" OFF)
+ if (MUSL)
+     add_definitions(-D__MUSL__)
+diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
+index d4f984d5..9b664bee 100644
+--- a/src/cmd/isulad/main.c
+++ b/src/cmd/isulad/main.c
+@@ -73,6 +73,9 @@
+ #include "utils_string.h"
+ #include "utils_verify.h"
+ #include "volume_api.h"
+#ifndef DISABLE_CLEANUP
+#include "leftover_cleanup_api.h"
+#endif
+ #include "opt_log.h"
+ 
+ #ifdef GRPC_CONNECTOR
+@@ -1235,6 +1238,14 @@ static int isulad_server_init_common()
+         goto out;
+     }
+ 
+#ifndef DISABLE_CLEANUP
+    // to cleanup leftover, init clean module before other modules.
+    if (clean_module_init() != 0) {
+        ERROR("Failed to init clean module");
+        goto out;
+    }
+#endif
+
+     if (volume_init(args->json_confs->graph) != 0) {
+         ERROR("Failed to init volume");
+         goto out;
+@@ -1451,6 +1462,10 @@ static int start_daemon_threads(char **msg)
+         goto out;
+     }
+ 
+#ifndef DISABLE_CLEANUP
+    clean_module_do_clean();
+#endif
+
+     ret = 0;
+ out:
+     return ret;
+diff --git a/src/daemon/modules/api/image_api.h b/src/daemon/modules/api/image_api.h
+index a1c6084a..b8d7fd5b 100644
+--- a/src/daemon/modules/api/image_api.h
+++ b/src/daemon/modules/api/image_api.h
+@@ -244,6 +244,8 @@ int im_umount_container_rootfs(const char *image_type, const char *image_name, c
+ 
+ int im_remove_container_rootfs(const char *image_type, const char *container_id);
+ 
+int im_remove_broken_rootfs(const char *image_type, const char *container_id);
+
+ int im_merge_image_config(const char *image_type, const char *image_name, container_config *container_spec);
+ 
+ int im_get_user_conf(const char *image_type, const char *basefs, host_config *hc, const char *userstr,
+diff --git a/src/daemon/modules/api/leftover_cleanup_api.h b/src/daemon/modules/api/leftover_cleanup_api.h
+new file mode 100644
+index 00000000..32a41258
+--- /dev/null
+++ b/src/daemon/modules/api/leftover_cleanup_api.h
+@@ -0,0 +1,36 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-12-7
+ * Description: provide cleanup functions
+ *********************************************************************************/
+#ifndef DAEMON_MODULES_API_LEFTOVER_CLEANUP_API_H
+#define DAEMON_MODULES_API_LEFTOVER_CLEANUP_API_H
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+typedef enum {
+    BROKEN_ROOTFS = 1
+} cleanup_ctx_data_t;
+
+int clean_module_init();
+
+void clean_module_fill_ctx(cleanup_ctx_data_t data_type, void *data);
+
+void clean_module_do_clean();
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif
+diff --git a/src/daemon/modules/container/CMakeLists.txt b/src/daemon/modules/container/CMakeLists.txt
+index def602c7..38c3d88b 100644
+--- a/src/daemon/modules/container/CMakeLists.txt
+++ b/src/daemon/modules/container/CMakeLists.txt
+@@ -5,7 +5,9 @@ add_subdirectory(supervisor)
+ add_subdirectory(health_check)
+ add_subdirectory(container_gc)
+ add_subdirectory(restart_manager)
+IF (NOT DISABLE_CLEANUP)
+ add_subdirectory(leftover_cleanup)
+ENDIF()
+ 
+ set(MANAGER_SRCS
+     ${local_manager_srcs}
+diff --git a/src/daemon/modules/container/container_unix.c b/src/daemon/modules/container/container_unix.c
+index 88c4bf51..9910b3c8 100644
+--- a/src/daemon/modules/container/container_unix.c
+++ b/src/daemon/modules/container/container_unix.c
+@@ -46,7 +46,6 @@
+ #include "utils_string.h"
+ #include "volume_api.h"
+ #include "namespace.h"
+-#include "cleanup.h"
+ 
+ static int parse_container_log_configs(container_t *cont);
+ 
+@@ -1279,7 +1278,6 @@ int container_module_init(char **msg)
+     }
+ 
+     containers_restore();
+-    clean_leftover();
+ 
+     if (start_gchandler()) {
+         *msg = "Failed to start garbage collecotor handler";
+diff --git a/src/daemon/modules/container/leftover_cleanup/clean_context.c b/src/daemon/modules/container/leftover_cleanup/clean_context.c
+new file mode 100644
+index 00000000..6ccc39ed
+--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/clean_context.c
+@@ -0,0 +1,78 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide cleanup definition
+ *********************************************************************************/
+#include "clean_context.h"
+#include "linked_list.h"
+#include "utils.h"
+#include "isula_libutils/log.h"
+
+struct clean_ctx *clean_ctx_init()
+{
+    struct clean_ctx *ctx = util_common_calloc_s(sizeof(struct clean_ctx));
+    if (ctx == NULL) {
+        ERROR("Out of memory");
+        return NULL;
+    }
+
+    linked_list_init(&(ctx->broken_rootfs_list));
+    ctx->inited = true;
+
+    return ctx;
+}
+
+void clean_ctx_destroy(struct clean_ctx *ctx)
+{
+    struct linked_list *it = NULL;
+    struct linked_list *next = NULL;
+    char *id = NULL;
+
+    if (ctx == NULL) {
+        return;
+    }
+
+    if (!ctx->inited) {
+        free(ctx);
+        return;
+    }
+
+    linked_list_for_each_safe(it, &(ctx->broken_rootfs_list), next) {
+        id = (char *)it->elem;
+        linked_list_del(it);
+        free(id);
+        free(it);
+        it = NULL;
+    }
+
+    free(ctx);
+}
+
+void clean_ctx_fill_broken_rootfs(struct clean_ctx *ctx, const char *id)
+{
+    struct linked_list *new_node = NULL;
+    char *broken_id = NULL;
+
+    if (!ctx->inited) {
+        return;
+    }
+
+    new_node = util_common_calloc_s(sizeof(struct linked_list));
+    if (new_node == NULL) {
+        ERROR("Out of memory, broken rootfs %s not added", id);
+        return;
+    }
+
+    broken_id = util_strdup_s(id);
+    linked_list_add_elem(new_node, broken_id);
+    linked_list_add_tail(&ctx->broken_rootfs_list, new_node);
+}
+\ No newline at end of file
+diff --git a/src/daemon/modules/container/leftover_cleanup/clean_context.h b/src/daemon/modules/container/leftover_cleanup/clean_context.h
+new file mode 100644
+index 00000000..fcac8df8
+--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/clean_context.h
+@@ -0,0 +1,41 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide cleanup definition
+ *********************************************************************************/
+#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEAN_CONTEXT_H
+#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEAN_CONTEXT_H
+
+#include "linked_list.h"
+#include "utils.h"
+#include "isula_libutils/log.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+struct clean_ctx {
+    bool inited;
+    struct linked_list broken_rootfs_list;
+};
+
+struct clean_ctx *clean_ctx_init();
+
+void clean_ctx_destroy(struct clean_ctx *ctx);
+
+void clean_ctx_fill_broken_rootfs(struct clean_ctx *ctx, const char *id);
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif
+\ No newline at end of file
+diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c
+index 9ce1dd0c..664988b5 100644
+--- a/src/daemon/modules/container/leftover_cleanup/cleanup.c
+++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c
+@@ -31,12 +31,16 @@ static struct cleaners *create_cleaners()
+     return ret;
+ }
+ 
+-static void destroy_cleaners(struct cleaners *clns)
+void destroy_cleaners(struct cleaners *clns)
+ {
+     struct linked_list *it = NULL;
+     struct linked_list *next = NULL;
+     struct clean_node *c_node = NULL;
+ 
+    if (clns == NULL) {
+        return;
+    }
+
+     linked_list_for_each_safe(it, &(clns->cleaner_list), next) {
+         c_node = (struct clean_node *)it->elem;
+         linked_list_del(it);
+@@ -80,25 +84,31 @@ static int default_cleaner()
+     return 0;
+ }
+ 
+-static struct cleaners *cleaner_init()
+struct cleaners *cleaners_init()
+ {
+     int ret = 0;
+     struct cleaners *clns = create_cleaners();
+-    
+
+     if (clns == NULL) {
+         return NULL;
+     }
+ 
+     ret = add_clean_node(clns, default_cleaner, "default clean");
+     if (ret != 0) {
+-        ERROR("add default_cleaner error");
+        ERROR("Add default_cleaner error");
+         return clns;
+     }
+ 
+ #ifdef ENABLE_OCI_IMAGE
+    ret = add_clean_node(clns, oci_broken_rootfs_cleaner, "clean broken rootfs");
+    if (ret != 0) {
+        ERROR("Clean broken rootfs failed");
+        return clns;
+    }
+
+     ret = add_clean_node(clns, oci_rootfs_cleaner, "clean rootfs");
+     if (ret != 0) {
+-        ERROR("add oci_rootfs_cleaner error");
+        ERROR("Add oci_rootfs_cleaner error");
+         return clns;
+     }
+ #endif
+@@ -106,7 +116,7 @@ static struct cleaners *cleaner_init()
+     return clns;
+ }
+ 
+-static void do_clean(struct cleaners * clns)
+void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx)
+ {
+     struct linked_list *it = NULL;
+     struct linked_list *next = NULL;
+@@ -114,31 +124,11 @@ static void do_clean(struct cleaners * clns)
+ 
+     linked_list_for_each_safe(it, &(clns->cleaner_list), next) {
+         c_node = (struct clean_node *)it->elem;
+-        if (c_node->cleaner() != 0) {
+-            ERROR("failed to clean for: %s", c_node->desc);
+        if (c_node->cleaner(ctx) != 0) {
+            ERROR("Failed to clean for: %s", c_node->desc);
+         } else {
+             DEBUG("do clean success for: %s", c_node->desc);
+             clns->done_clean++;
+         }
+     }
+ }
+-
+-void clean_leftover()
+-{
+-    struct cleaners *clns = cleaner_init();
+-
+-    if (clns == NULL) {
+-        ERROR("failed to clean leftovers, because cleaner init error");
+-        return;
+-    }
+-
+-    do_clean(clns);
+-
+-    if (clns->count == clns->done_clean) {
+-        DEBUG("all clean up success");
+-    } else {
+-        ERROR("Aim to do %d clean, %d clean sucess\n", clns->count, clns->done_clean);
+-    }
+-
+-    destroy_cleaners(clns);
+-}
+\ No newline at end of file
+diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.h b/src/daemon/modules/container/leftover_cleanup/cleanup.h
+index efae99d0..8dd5e9bd 100644
+--- a/src/daemon/modules/container/leftover_cleanup/cleanup.h
+++ b/src/daemon/modules/container/leftover_cleanup/cleanup.h
+@@ -12,19 +12,20 @@
+  * Create: 2022-10-31
+  * Description: provide cleanup definition
+  *********************************************************************************/
+-#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_H
+-#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_H
+#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEANERS_H
+#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEANERS_H
+ 
+ #include <stdlib.h>
+ 
+ #include "linked_list.h"
+ #include "isula_libutils/log.h"
+#include "clean_context.h"
+ 
+ #if defined(__cplusplus) || defined(c_plusplus)
+ extern "C" {
+ #endif
+ 
+-typedef int clean_func_t(void);
+typedef int clean_func_t(struct clean_ctx *ctx);
+ 
+ struct clean_node {
+     const char *desc;
+@@ -38,8 +39,11 @@ struct cleaners {
+     struct linked_list cleaner_list;
+ };
+ 
+struct cleaners *cleaners_init();
+ 
+-void clean_leftover();
+void destroy_cleaners(struct cleaners *clns);
+
+void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx);
+ 
+ 
+ #if defined(__cplusplus) || defined(c_plusplus)
+diff --git a/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c
+new file mode 100644
+index 00000000..7bdaef22
+--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c
+@@ -0,0 +1,71 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide cleanup functions
+ *********************************************************************************/
+#include "utils.h"
+#include "leftover_cleanup_api.h"
+#include "cleanup.h"
+#include "clean_context.h"
+
+struct clean_ctx *g_clean_ctx = NULL;
+struct cleaners *g_clns = NULL;
+
+int clean_module_init()
+{
+    // create cleaners and clean_ctx
+    g_clns = cleaners_init();
+    if (g_clns == NULL) {
+        ERROR("Failed to init clean module");
+        return -1;
+    }
+
+    g_clean_ctx = clean_ctx_init();
+    if (g_clean_ctx == NULL) {
+        ERROR("Failed to init clean module");
+        destroy_cleaners(g_clns);
+        return -1;
+    }
+
+    return 0;
+}
+
+void clean_module_fill_ctx(cleanup_ctx_data_t data_type, void *data)
+{
+    switch (data_type) {
+        case BROKEN_ROOTFS:
+            clean_ctx_fill_broken_rootfs(g_clean_ctx, data);
+            break;
+    }
+}
+
+void clean_module_do_clean()
+{
+    if (g_clns == NULL || g_clean_ctx == NULL) {
+        return;
+    }
+
+    cleaners_do_clean(g_clns, g_clean_ctx);
+
+    if (g_clns->count == g_clns->done_clean) {
+        DEBUG("all clean up success");
+    } else {
+        ERROR("Aim to do %d clean, %d clean sucess\n", g_clns->count, g_clns->done_clean);
+    }
+
+    destroy_cleaners(g_clns);
+    clean_ctx_destroy(g_clean_ctx);
+
+    g_clns = NULL;
+    g_clean_ctx = NULL;
+}
+
+diff --git a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c
+index fbef4ce0..b2205569 100644
+--- a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c
+++ b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c
+@@ -19,6 +19,7 @@
+ #include "image_api.h"
+ #include "utils_file.h"
+ #include "utils.h"
+#include "linked_list.h"
+ 
+ struct cb_result {
+     int clean_err_cnt;
+@@ -45,7 +46,7 @@ static bool walk_dir_cb(const char *path_name, const struct dirent *sub_dir, voi
+ }
+ 
+ 
+-int oci_rootfs_cleaner(void)
+int oci_rootfs_cleaner(struct clean_ctx *ctx)
+ {
+     struct cb_result res = { 0 };
+     im_get_rf_dir_request request = { 0 };
+@@ -67,7 +68,36 @@ int oci_rootfs_cleaner(void)
+ 
+     if (res.clean_err_cnt == 0) {
+         return 0;
+-    } 
+    }
+ 
+     return -1;
+ }
+
+int oci_broken_rootfs_cleaner(struct clean_ctx *ctx)
+{
+    int rm_fail_cnt = 0;
+    struct linked_list *it = NULL;
+    struct linked_list *next = NULL;
+    char *id = NULL;
+
+    if (ctx == NULL) {
+        return -1;
+    }
+
+    linked_list_for_each_safe(it, &(ctx->broken_rootfs_list), next) {
+        id = (char *)it->elem;
+        if (im_remove_broken_rootfs(IMAGE_TYPE_OCI, id) != 0) {
+            ERROR("Failed to clean broken rootfs %s", id);
+            rm_fail_cnt++;
+        } else {
+            EVENT("clean broken rootfs succeed %s", id);
+        }
+    }
+
+    if (rm_fail_cnt != 0) {
+        DEBUG("can't clean some broken rootfs, %d left", rm_fail_cnt);
+        return -1;
+    }
+
+    return 0;
+}
+\ No newline at end of file
+diff --git a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h
+index 8dff351f..88920fa9 100644
+--- a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h
+++ b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h
+@@ -21,7 +21,9 @@
+ extern "C" {
+ #endif
+ 
+-int oci_rootfs_cleaner(void);
+int oci_rootfs_cleaner(struct clean_ctx *ctx);
+
+int oci_broken_rootfs_cleaner(struct clean_ctx *ctx);
+ 
+ #if defined(__cplusplus) || defined(c_plusplus)
+ }
+diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c
+index 8fb226aa..6fac7cc4 100644
+--- a/src/daemon/modules/image/image.c
+++ b/src/daemon/modules/image/image.c
+@@ -50,6 +50,7 @@ struct bim_ops {
+     int (*export_rf)(const im_export_request *request);
+     char *(*resolve_image_name)(const char *image_name);
+     char *(*get_dir_rf)(void);
+    int (*delete_broken_rf)(const im_delete_rootfs_request *request);
+ 
+     /* merge image config ops */
+     int (*merge_conf)(const char *img_name, container_config *container_spec);
+@@ -132,6 +133,7 @@ static const struct bim_ops g_embedded_ops = {
+     .mount_rf = embedded_mount_rf,
+     .umount_rf = embedded_umount_rf,
+     .delete_rf = embedded_delete_rf,
+    .delete_broken_rf = NULL,
+     .export_rf = NULL,
+     .get_dir_rf = NULL,
+ 
+@@ -167,6 +169,7 @@ static const struct bim_ops g_oci_ops = {
+     .mount_rf = oci_mount_rf,
+     .umount_rf = oci_umount_rf,
+     .delete_rf = oci_delete_rf,
+    .delete_broken_rf = oci_delete_broken_rf,
+     .export_rf = oci_export_rf,
+     .get_dir_rf = oci_get_dir_rf,
+ 
+@@ -201,6 +204,7 @@ static const struct bim_ops g_ext_ops = {
+     .mount_rf = ext_mount_rf,
+     .umount_rf = ext_umount_rf,
+     .delete_rf = ext_delete_rf,
+    .delete_broken_rf = NULL,
+     .export_rf = NULL,
+     .get_dir_rf = NULL,
+ 
+@@ -464,6 +468,50 @@ out:
+     return ret;
+ }
+ 
+int im_remove_broken_rootfs(const char *image_type, const char *container_id)
+{
+    int ret = 0;
+    im_delete_rootfs_request *request = NULL;
+    struct bim *bim = NULL;
+
+    if (container_id == NULL || image_type == NULL) {
+        ERROR("Invalid input arguments");
+        return -1;
+    }
+
+    bim = bim_get(image_type, NULL, NULL, container_id);
+    if (bim == NULL) {
+        ERROR("Failed to init bim when deleting broken rootfs %s", container_id);
+        return -1;
+    }
+
+    if (bim->ops->delete_broken_rf == NULL) {
+        ERROR("Unimplements delete in %s", bim->type);
+        ret = -1;
+        goto out;
+    }
+
+    request = util_common_calloc_s(sizeof(im_delete_rootfs_request));
+    if (request == NULL) {
+        ERROR("Out of memory");
+        ret = -1;
+        goto out;
+    }
+    request->name_id = util_strdup_s(container_id);
+
+    ret = bim->ops->delete_broken_rf(request);
+    if (ret != 0) {
+        ERROR("Failed to delete rootfs for container %s", container_id);
+        ret = -1;
+        goto out;
+    }
+
+out:
+    bim_put(bim);
+    free_im_delete_request(request);
+    return ret;
+}
+
+ int im_remove_container_rootfs(const char *image_type, const char *container_id)
+ {
+     int ret = 0;
+diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c
+index e951adb4..0fc0b7fe 100644
+--- a/src/daemon/modules/image/oci/oci_image.c
+++ b/src/daemon/modules/image/oci/oci_image.c
+@@ -368,6 +368,16 @@ out:
+     return ret;
+ }
+ 
+int oci_delete_broken_rf(const im_delete_rootfs_request *request)
+{
+    if (request == NULL) {
+        ERROR("Request is NULL");
+        return -1;
+    }
+
+    return storage_broken_rw_layer_delete(request->name_id);
+}
+
+ int oci_delete_rf(const im_delete_rootfs_request *request)
+ {
+     if (request == NULL) {
+diff --git a/src/daemon/modules/image/oci/oci_image.h b/src/daemon/modules/image/oci/oci_image.h
+index aeeb3b65..cd7da336 100644
+--- a/src/daemon/modules/image/oci/oci_image.h
+++ b/src/daemon/modules/image/oci/oci_image.h
+@@ -53,6 +53,7 @@ int oci_merge_conf_rf(const char *img_name, container_config *container_spec);
+ int oci_mount_rf(const im_mount_request *request);
+ int oci_umount_rf(const im_umount_request *request);
+ int oci_delete_rf(const im_delete_rootfs_request *request);
+int oci_delete_broken_rf(const im_delete_rootfs_request *request);
+ int oci_export_rf(const im_export_request *request);
+ char *oci_get_dir_rf(void);
+ int oci_container_filesystem_usage(const im_container_fs_usage_request *request, imagetool_fs_info **fs_usage);
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+index e563a8ef..b9ab0d65 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+@@ -557,6 +557,7 @@ static int update_layer_datas(const char *id, const struct layer_opts *opts, lay
+ 
+     slayer->id = util_strdup_s(id);
+     slayer->parent = util_strdup_s(opts->parent);
+    slayer->writable = opts->writable;
+     if (opts->opts != NULL) {
+         slayer->mountlabel = util_strdup_s(opts->opts->mount_label);
+     }
+@@ -1434,6 +1435,7 @@ static void copy_json_to_layer(const layer_t *jl, struct layer *l)
+         l->mount_point = util_strdup_s(jl->smount_point->path);
+         l->mount_count = jl->smount_point->count;
+     }
+    l->writable = jl->slayer->writable;
+ }
+ 
+ int layer_store_list(struct layer_list *resp)
+diff --git a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c
+index 97cc39e8..c6e305a0 100644
+--- a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c
+++ b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c
+@@ -38,6 +38,10 @@
+ #include "utils_string.h"
+ #include "utils_timestamp.h"
+ 
+#ifndef DISABLE_CLEANUP
+#include "leftover_cleanup_api.h"
+#endif
+
+ #define CONTAINER_JSON "container.json"
+ 
+ typedef struct rootfs_store {
+@@ -170,6 +174,7 @@ static int append_container_by_directory(const char *container_dir)
+ 
+     nret = snprintf(container_path, sizeof(container_path), "%s/%s", container_dir, CONTAINER_JSON);
+     if (nret < 0 || (size_t)nret >= sizeof(container_path)) {
+        // snprintf error, not append, but outside should not delete the rootfs
+         ERROR("Failed to get container path");
+         return -1;
+     }
+@@ -182,6 +187,7 @@ static int append_container_by_directory(const char *container_dir)
+     }
+ 
+     if (do_append_container(c) != 0) {
+        // append error should not return -1, outside should not remove rootfs
+         ERROR("Failed to append container");
+         ret = -1;
+         goto out;
+@@ -199,6 +205,7 @@ static int get_containers_from_json()
+ {
+     int ret = 0;
+     int nret;
+    int append_ret = 0;
+     char **container_dirs = NULL;
+     size_t container_dirs_num = 0;
+     size_t i;
+@@ -231,7 +238,11 @@ static int get_containers_from_json()
+             continue;
+         }
+ 
+-        if (append_container_by_directory(container_path) != 0) {
+        append_ret = append_container_by_directory(container_path);
+        if (append_ret != 0) {
+#ifndef DISABLE_CLEANUP
+            clean_module_fill_ctx(BROKEN_ROOTFS, (void *)container_dirs[i]);
+#endif
+             ERROR("Found container path but load json failed: %s, deleting...", container_path);
+             if (util_recursive_rmdir(container_path, 0) != 0) {
+                 ERROR("Failed to delete rootfs directory : %s", container_path);
+diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c
+index 6cb4a51b..829ea8d0 100644
+--- a/src/daemon/modules/image/oci/storage/storage.c
+++ b/src/daemon/modules/image/oci/storage/storage.c
+@@ -306,6 +306,38 @@ struct layer *storage_layer_get(const char *layer_id)
+     return layer_store_lookup(layer_id);
+ }
+ 
+int storage_broken_rw_layer_delete(const char *layer_id)
+{
+    int ret = 0;
+    struct layer *layer_info = NULL;
+
+    if (layer_id == NULL) {
+        return -1;
+    }
+
+    layer_info = layer_store_lookup(layer_id);
+    if (layer_info == NULL) {
+        ERROR("Failed to get layer info for layer %s", layer_id);
+        return -1;
+    }
+
+    if (!layer_info->writable) {
+        ERROR("Broken rootfs should only delete rw layer, layer %s is ro layer", layer_id);
+        ret = -1;
+        goto out;
+    }
+
+    // delete rootfs and rw layer, rw layer has the same name as rootfs
+    if (layer_store_delete(layer_info->id) != 0) {
+        ERROR("Can't delete layer of broken rootfs");
+        ret = -1;
+    }
+
+out:
+    free_layer(layer_info);
+    return ret;
+}
+
+ void free_layer(struct layer *ptr)
+ {
+     if (ptr == NULL) {
+diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h
+index 5914adec..3ec47959 100644
+--- a/src/daemon/modules/image/oci/storage/storage.h
+++ b/src/daemon/modules/image/oci/storage/storage.h
+@@ -42,6 +42,7 @@ struct layer {
+     int64_t compress_size;
+     char *uncompressed_digest;
+     int64_t uncompress_size;
+    bool writable;
+ };
+ 
+ struct layer_list {
+@@ -163,6 +164,8 @@ struct layer_list *storage_layers_get_by_compress_digest(const char *digest);
+ 
+ struct layer *storage_layer_get(const char *layer_id);
+ 
+int storage_broken_rw_layer_delete(const char *layer_id);
+
+ int storage_layer_try_repair_lowers(const char *layer_id, const char *last_layer_id);
+ 
+ void free_layer(struct layer *l);
+diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt
+index 8da8196b..5bf568f9 100644
+--- a/test/image/oci/storage/rootfs/CMakeLists.txt
+++ b/test/image/oci/storage/rootfs/CMakeLists.txt
+@@ -2,6 +2,8 @@ project(iSulad_UT)
+ 
+ SET(EXE storage_rootfs_ut)
+ 
+add_definitions(-DDISABLE_CLEANUP)
+
+ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_regex.c
+-- 
+2.25.1
+
--- a/0060-1761-fix-leftover-devicemapper-mnt-dir.patch
+++ b/0060-1761-fix-leftover-devicemapper-mnt-dir.patch
@ -0,0 +1,175 @@
+From 9b41a8d28d3ffbe33eb84d8e254b603012b22b34 Mon Sep 17 00:00:00 2001
+From: DriedYellowPeach <wangrunze13@huawei.com>
+Date: Sun, 11 Dec 2022 11:12:55 +0000
+Subject: [PATCH 60/65] !1761 fix leftover devicemapper mnt dir * fix leftover
+ devicemapper mnt dir
+
+---
+ .../graphdriver/devmapper/deviceset.c         | 37 +++++++++++++
+ .../graphdriver/devmapper/deviceset.h         |  2 +
+ .../graphdriver/devmapper/driver_devmapper.c  | 55 ++++++++++++-------
+ .../oci/storage/layer_store/layer_store.c     | 11 +++-
+ 4 files changed, 84 insertions(+), 21 deletions(-)
+
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c
+index 4dadc336..b157510a 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c
+@@ -486,6 +486,43 @@ out:
+     return exist;
+ }
+ 
+// return true if find the metadata
+// or the argument is wrong
+// or can't decide
+bool has_metadata(const char *hash, struct device_set *devset)
+{
+    char metadata_file[PATH_MAX] = { 0 };
+    char *metadata_path = NULL;
+    bool ret = true;
+    int nret = 0;
+
+    if (hash == NULL) {
+        return true;
+    }
+
+    metadata_path = metadata_dir(devset);
+    if (metadata_path == NULL) {
+        ERROR("Failed to get meta data directory");
+        goto out;
+    }
+
+    nret = snprintf(metadata_file, sizeof(metadata_file), "%s/%s", metadata_path, util_valid_str(hash) ? hash : "base");
+    if (nret < 0 || (size_t)nret >= sizeof(metadata_file)) {
+        ERROR("Failed to snprintf metadata file path with hash:%s, path is too long", hash);
+        goto out;
+    }
+
+    if (!util_file_exists(metadata_file)) {
+        WARN("No such file:%s, need not to load", metadata_file);
+        ret = false;
+        goto out;
+    }
+
+out:
+    free(metadata_path);
+    return ret;
+}
+
+ static image_devmapper_device_info *load_metadata(const struct device_set *devset, const char *hash)
+ {
+     image_devmapper_device_info *info = NULL;
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.h
+index c11eece6..ec985e40 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.h
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.h
+@@ -83,6 +83,8 @@ int delete_device(const char *hash, bool sync_delete, struct device_set *devset)
+ 
+ int export_device_metadata(struct device_metadata *dev_metadata, const char *hash, struct device_set *devset);
+ 
+bool has_metadata(const char *hash, struct device_set *devset);
+
+ struct status *device_set_status(struct device_set *devset);
+ 
+ void free_devmapper_status(struct status *st);
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c
+index dd231bd6..c83d3e54 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c
+@@ -144,28 +144,11 @@ int devmapper_create_ro(const char *id, const char *parent, const struct graphdr
+     return do_create(id, parent, driver, create_opts);
+ }
+ 
+-// Remove removes a device with a given id, unmounts the filesystem.
+-int devmapper_rm_layer(const char *id, const struct graphdriver *driver)
+static int devmapper_try_rm_layer_mnt(const char *id, const struct graphdriver *driver)
+ {
+    int ret = 0;
+     char *mnt_parent_dir = NULL;
+     char *mnt_point_dir = NULL;
+-    int ret = 0;
+-
+-    if (!util_valid_str(id) || driver == NULL) {
+-        ERROR("invalid argument");
+-        return -1;
+-    }
+-
+-    if (!has_device(id, driver->devset)) {
+-        DEBUG("Device with id:%s is not exist", id);
+-        goto out;
+-    }
+-
+-    if (delete_device(id, false, driver->devset) != 0) {
+-        ERROR("failed to remove device %s", id);
+-        ret = -1;
+-        goto out;
+-    }
+ 
+     mnt_parent_dir = util_path_join(driver->home, "mnt");
+     if (mnt_parent_dir == NULL) {
+@@ -193,6 +176,40 @@ out:
+     return ret;
+ }
+ 
+// Remove removes a device with a given id, unmounts the filesystem.
+int devmapper_rm_layer(const char *id, const struct graphdriver *driver)
+{
+    if (!util_valid_str(id) || driver == NULL) {
+        ERROR("invalid argument");
+        return -1;
+    }
+
+    if (!has_device(id, driver->devset)) {
+        DEBUG("Device with id:%s is not exist", id);
+        if (!has_metadata(id, driver->devset)) {
+            // this means metadata is lost
+            // if we can rm mnt, then the layer is removed
+            EVENT("try clean lost metadata and its mnt: %s", id);
+            return devmapper_try_rm_layer_mnt(id, driver);
+        }
+        // if has_metadata and not rm successfully, return -1
+        // so next start up of isulad will retry delete the layer.
+        return -1;
+    }
+
+    if (delete_device(id, false, driver->devset) != 0) {
+        ERROR("failed to remove device %s", id);
+        return -1;
+    }
+
+    if (devmapper_try_rm_layer_mnt(id, driver) != 0) {
+        ERROR("failed to remove mnt dir of Device: %s", id);
+        return -1;
+    }
+
+    return 0;
+}
+
+ // devmapper_mount_layer mounts a device with given id into the root filesystem
+ char *devmapper_mount_layer(const char *id, const struct graphdriver *driver,
+                             const struct driver_mount_opts *mount_opts)
+diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+index b9ab0d65..60aaff22 100644
+--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+@@ -1813,8 +1813,15 @@ static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_d
+ 
+ remove_invalid_dir:
+     (void)graphdriver_umount_layer(sub_dir->d_name);
+-    (void)graphdriver_rm_layer(sub_dir->d_name);
+-    (void)util_recursive_rmdir(tmpdir, 0);
+    // layer not removed successfully, we can't remove layer.json
+    if (graphdriver_rm_layer(sub_dir->d_name) != 0) {
+        ERROR("failed to rm layer: %s when handing invalid rootfs", sub_dir->d_name);
+        goto free_out;
+    }
+    ERROR("tmpdir is %s", tmpdir);
+    if (util_recursive_rmdir(tmpdir, 0) != 0) {
+        ERROR("failed to rm rootfs dir: %s when handing invalid rootfs", tmpdir);
+    }
+ 
+ free_out:
+     free(rpath);
+-- 
+2.25.1
+
--- a/0061-check-file-system-ro-before-merge-network-for-syscon.patch
+++ b/0061-check-file-system-ro-before-merge-network-for-syscon.patch
@ -0,0 +1,113 @@
+From fa7428d0baf2c310c852b1ece41736b21ea441f9 Mon Sep 17 00:00:00 2001
+From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
+Date: Fri, 9 Dec 2022 17:26:10 +0800
+Subject: [PATCH 61/65] check file system ro before merge network for
+ syscontainer
+
+Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
+---
+ .../executor/container_cb/execution_network.c | 29 +++++++++++++++++++
+ src/utils/cutils/utils_fs.c                   | 27 +++++++++++++++++
+ src/utils/cutils/utils_fs.h                   |  1 +
+ 3 files changed, 57 insertions(+)
+
+diff --git a/src/daemon/executor/container_cb/execution_network.c b/src/daemon/executor/container_cb/execution_network.c
+index bbc35e80..b738d02f 100644
+--- a/src/daemon/executor/container_cb/execution_network.c
+++ b/src/daemon/executor/container_cb/execution_network.c
+@@ -761,9 +761,38 @@ out:
+     return ret;
+ }
+ 
+static int check_readonly_fs_for_etc(const char *rootfs, bool *ro)
+{
+    char *path = NULL;
+
+    if (util_realpath_in_scope(rootfs, "/etc", &path) < 0) {
+        SYSERROR("Failed to get real path '/etc' under rootfs '%s'", rootfs);
+        isulad_set_error_message("Failed to get real path '/etc' under rootfs '%s'", rootfs);
+        return -1;
+    }
+
+    *ro = util_check_readonly_fs(path);
+
+    free(path);
+    return 0;
+}
+
+// modify network file in rootfs
+// make sure network file saved if rootfs migrate to another host
+ static int merge_network_for_syscontainer(const host_config *host_spec, const char *rootfs, const char *hostname)
+ {
+     int ret = 0;
+    bool ro = false;
+
+    if (check_readonly_fs_for_etc(rootfs, &ro) != 0) {
+        ERROR("Failed to check network path");
+        return -1;
+    }
+
+    if (ro) {
+        WARN("Readonly filesystem for etc under %s. Skip merge network for syscontainer", rootfs);
+        return 0;
+    }
+ 
+     ret = write_hostname_to_file(rootfs, hostname);
+     if (ret) {
+diff --git a/src/utils/cutils/utils_fs.c b/src/utils/cutils/utils_fs.c
+index 218e2fe8..e7165f26 100644
+--- a/src/utils/cutils/utils_fs.c
+++ b/src/utils/cutils/utils_fs.c
+@@ -22,6 +22,7 @@
+ #include <stdlib.h>
+ #include <unistd.h>
+ #include <sys/types.h>
+#include <sys/statvfs.h>
+ #include <linux/magic.h>
+ #include <sys/statfs.h>
+ #include <dirent.h>
+@@ -620,3 +621,29 @@ child_out:
+ cleanup:
+     return ret;
+ }
+
+bool util_check_readonly_fs(const char *path)
+{
+    int i;
+    const int max_retry = 10;
+    struct statfs fsbuf;
+
+    for (i = 0; i < max_retry; i++) {
+        if (statfs(path, &fsbuf) == 0) {
+            break;
+        }
+        if (errno == EINTR) {
+            continue;
+        }
+
+        ERROR("Stat fs failed: %s", strerror(errno));
+        return false;
+    }
+
+    if (i >= max_retry) {
+        ERROR("Too much interrupted");
+        return false;
+    }
+
+    return (fsbuf.f_flags & ST_RDONLY) != 0;
+}
+diff --git a/src/utils/cutils/utils_fs.h b/src/utils/cutils/utils_fs.h
+index 6ab6b78e..6ad64a1a 100644
+--- a/src/utils/cutils/utils_fs.h
+++ b/src/utils/cutils/utils_fs.h
+@@ -38,6 +38,7 @@ int util_ensure_mounted_as(const char *dst, const char *mntopts);
+ int util_mount_from(const char *base, const char *src, const char *dst, const char *mtype, const char *mntopts);
+ typedef int (*mount_info_call_back_t)(const char *, const char *);
+ bool util_deal_with_mount_info(mount_info_call_back_t cb, const char *);
+bool util_check_readonly_fs(const char *path);
+ #ifdef __cplusplus
+ }
+ #endif
+-- 
+2.25.1
+
--- a/0062-isulad-shim-wait-for-all-child-process.patch
+++ b/0062-isulad-shim-wait-for-all-child-process.patch
@ -0,0 +1,134 @@
+From a2ed1886a091d1edfb54fdec38db277e0d0e29f5 Mon Sep 17 00:00:00 2001
+From: "Neil.wrz" <wangrunze13@huawei.com>
+Date: Tue, 13 Dec 2022 01:34:12 -0800
+Subject: [PATCH 62/65] isulad shim wait for all child process
+
+Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
+---
+ src/cmd/isulad-shim/process.c                  | 18 ++++++++++++++++++
+ src/daemon/modules/plugin/plugin.c             |  2 +-
+ .../modules/runtime/isula/isula_rt_ops.c       |  4 ++--
+ src/utils/cutils/utils.h                       |  2 +-
+ test/cutils/utils_utils/utils_utils_ut.cc      |  6 +++---
+ 5 files changed, 25 insertions(+), 7 deletions(-)
+
+diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
+index c8ce7a44..1fc95525 100644
+--- a/src/cmd/isulad-shim/process.c
+++ b/src/cmd/isulad-shim/process.c
+@@ -39,6 +39,7 @@
+ #include "terminal.h"
+ #include "utils_array.h"
+ #include "utils_string.h"
+#include "utils.h"
+ 
+ #define MAX_EVENTS 100
+ #define DEFAULT_IO_COPY_BUF (16 * 1024)
+@@ -1206,10 +1207,20 @@ out:
+     return ret;
+ }
+ 
+static int try_wait_all_child() {
+    if (waitpid(-1, NULL, WNOHANG) == -1 && errno == ECHILD) {
+        // all child handled
+        return 0;
+    }
+
+    return 1;
+}
+
+ int process_signal_handle_routine(process_t *p)
+ {
+     int ret = SHIM_ERR;
+     bool exit_shim = false;
+    int nret = 0;
+     int i;
+ 
+     for (;;) {
+@@ -1235,6 +1246,13 @@ int process_signal_handle_routine(process_t *p)
+         }
+         if (exit_shim) {
+             process_kill_all(p);
+            
+            // wait atmost 120 seconds
+            DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child, 0);
+            if (nret != 0) {
+                write_message(g_log_fd, ERR_MSG, "Failed to wait all child after 120 seconds");
+            }
+
+             process_delete(p);
+             if (p->exit_fd > 0) {
+                 (void)write_nointr(p->exit_fd, &status, sizeof(int));
+diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c
+index 18035518..53afeeaf 100644
+--- a/src/daemon/modules/plugin/plugin.c
+++ b/src/daemon/modules/plugin/plugin.c
+@@ -485,7 +485,7 @@ static int pm_register_plugin(const char *name, const char *addr)
+         goto failed;
+     }
+ 
+-    DO_RETYR_CALL(PLUGIN_ACTIVATE_MAX_RETRY, 1000000, err, pm_activate_plugin, plugin);
+    DO_RETRY_CALL(PLUGIN_ACTIVATE_MAX_RETRY, 1000000, err, pm_activate_plugin, plugin);
+     if (err != 0) {
+         ERROR("active plugin failed");
+         goto failed;
+diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+index bfe7de08..dd1bb4e8 100644
+--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+@@ -968,13 +968,13 @@ int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_
+     }
+ 
+     // retry 10 count call runtime kill, every call sleep 1s
+-    DO_RETYR_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id);
+    DO_RETRY_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id);
+     if (nret != 0) {
+         WARN("call runtime force kill failed");
+     }
+ 
+     // retry 10 count call runtime delete, every call sleep 1s
+-    DO_RETYR_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id);
+    DO_RETRY_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id);
+     if (nret != 0) {
+         WARN("call runtime force delete failed");
+     }
+diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h
+index 72cab9f2..fec6d879 100644
+--- a/src/utils/cutils/utils.h
+++ b/src/utils/cutils/utils.h
+@@ -392,7 +392,7 @@ int convert_v2_runtime(const char *runtime, char *binary);
+  *  0 is cb successful at least once;
+  *  1 is all cb are failure;
+ */
+-#define DO_RETYR_CALL(retry_cnt, interval_us, ret, cb, ...) do {    \
+#define DO_RETRY_CALL(retry_cnt, interval_us, ret, cb, ...) do {    \
+         size_t i = 0;                                               \
+         for(; i < retry_cnt; i++) {                                 \
+             ret = cb(__VA_ARGS__);                                  \
+diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc
+index 5bd98d47..c8f38717 100644
+--- a/test/cutils/utils_utils/utils_utils_ut.cc
+++ b/test/cutils/utils_utils/utils_utils_ut.cc
+@@ -285,15 +285,15 @@ TEST(utils_utils, test_do_retry_call)
+     int nret;
+ 
+     global_total = 0;
+-    DO_RETYR_CALL(10, 100, nret, retry_call_test, 0);
+    DO_RETRY_CALL(10, 100, nret, retry_call_test, 0);
+     ASSERT_EQ(nret, 0);
+     ASSERT_EQ(global_total, 0);
+     global_total = 0;
+-    DO_RETYR_CALL(10, 100, nret, retry_call_test, 5);
+    DO_RETRY_CALL(10, 100, nret, retry_call_test, 5);
+     ASSERT_EQ(nret, 0);
+     ASSERT_EQ(global_total, 5);
+     global_total = 0;
+-    DO_RETYR_CALL(10, 100, nret, retry_call_test, 11);
+    DO_RETRY_CALL(10, 100, nret, retry_call_test, 11);
+     ASSERT_EQ(global_total, 10);
+     ASSERT_EQ(nret, -1);
+ }
+\ No newline at end of file
+-- 
+2.25.1
+
--- a/0063-When-run-options-rm-is-set-delete-the-stoped-contain.patch
+++ b/0063-When-run-options-rm-is-set-delete-the-stoped-contain.patch
@ -0,0 +1,59 @@
+From 33fb34f3c864161fb24de77b72e157327e1e620c Mon Sep 17 00:00:00 2001
+From: zhongtao <taozh97@163.com>
+Date: Thu, 15 Dec 2022 15:55:34 +0800
+Subject: [PATCH 63/65] When run options rm is set, delete the stoped
+ container's fifo directory.
+
+Signed-off-by: zhongtao <taozh97@163.com>
+---
+ src/cmd/isula/base/run.c | 27 +++++++++++++++++++++++++++
+ 1 file changed, 27 insertions(+)
+
+diff --git a/src/cmd/isula/base/run.c b/src/cmd/isula/base/run.c
+index 73486c95..24863120 100644
+--- a/src/cmd/isula/base/run.c
+++ b/src/cmd/isula/base/run.c
+@@ -42,6 +42,29 @@ struct client_arguments g_cmd_run_args = {
+     .pull = "missing"
+ };
+ 
+/*
+ * --rm option will result in no time to delete the directory created by the client, resulting in residue. 
+ * This function is used to delete the client's fifo file and home directory.
+ */
+static void delete_client_fifo_and_home_dir(const char *rundir, const char *name,
+                                            struct command_fifo_config *console_fifos)
+{
+    char client_fifo_home_dir[PATH_MAX] = { 0 };
+    int nret = 0;
+
+    nret = snprintf(client_fifo_home_dir, sizeof(client_fifo_home_dir), "%s/%s/", rundir, name);
+    if (nret < 0 || (size_t)nret >= sizeof(client_fifo_home_dir)) {
+        ERROR("Client fifo home path:%s/%s/ is too long.", rundir, name);
+        return;
+    }
+
+    delete_command_fifo(console_fifos);
+
+    if (util_recursive_rmdir(client_fifo_home_dir, 0)) {
+        WARN("Failed to delete client fifo home path:%s", client_fifo_home_dir);
+    }
+}
+
+ static int local_cmd_start(const struct client_arguments *args)
+ {
+     int ret = 0;
+@@ -66,6 +89,10 @@ static int local_cmd_start(const struct client_arguments *args)
+     client_wait_fifo_exit(args);
+ 
+ free_out:
+    if (args->custom_conf.auto_remove && !args->detach) {
+        delete_client_fifo_and_home_dir(CLIENT_RUNDIR, args->name, console_fifos);
+        console_fifos = NULL;
+    }
+     client_restore_console(reset_tty, &oldtios, console_fifos);
+     return ret;
+ }
+-- 
+2.25.1
+
--- a/0064-recheck-kill-command-exit-status.patch
+++ b/0064-recheck-kill-command-exit-status.patch
@ -0,0 +1,153 @@
+From a8c344ad4400ed876aaa4f53f7c992a7ad1c1580 Mon Sep 17 00:00:00 2001
+From: "Neil.wrz" <wangrunze13@huawei.com>
+Date: Wed, 14 Dec 2022 20:49:34 -0800
+Subject: [PATCH 64/65] recheck kill command exit status
+
+Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
+---
+ .../modules/runtime/isula/isula_rt_ops.c      | 70 +++++++++++++++----
+ 1 file changed, 57 insertions(+), 13 deletions(-)
+
+diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+index dd1bb4e8..dee57263 100644
+--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+@@ -54,6 +54,9 @@
+ #define RESIZE_DATA_SIZE 100
+ #define PID_WAIT_TIME 120
+ 
+// handle string from stderr output.
+typedef int(*handle_output_callback_t)(const char *output);
+
+ static void copy_process(shim_client_process_state *p, defs_process *dp)
+ {
+     p->args = dp->args;
+@@ -584,7 +587,7 @@ out:
+ }
+ 
+ static int runtime_call_simple(const char *workdir, const char *runtime, const char *subcmd, const char **opts,
+-                               size_t opts_len, const char *id)
+                               size_t opts_len, const char *id, handle_output_callback_t cb)
+ {
+     runtime_exec_info rei = { 0 };
+     char *stdout = NULL;
+@@ -596,24 +599,65 @@ static int runtime_call_simple(const char *workdir, const char *runtime, const c
+     if (!util_exec_cmd(runtime_exec_func, &rei, NULL, &stdout, &stderr)) {
+         ERROR("call runtime %s failed stderr %s", subcmd, stderr);
+         ret = -1;
+-        goto out;
+        // additional handler for the stderr, 
+        // this intend to change the ret val of this function
+        // for example, if output string contains some specific content, 
+        // we consider the runtime call simple succeeded, 
+        // even if the process exit with failure.
+        if (stderr != NULL && cb != NULL) {
+            ret = cb(stderr);
+        }
+     }
+ 
+-out:
+     UTIL_FREE_AND_SET_NULL(stdout);
+     UTIL_FREE_AND_SET_NULL(stderr);
+     return ret;
+ }
+ 
+-static int runtime_call_kill_force(const char *workdir, const char *runtime, const char *id)
+// oci runtime return -1 if the container 'does not exist'
+// if output contains 'does not exist', means nothing to kill, return 0
+// this will change the exit status of kill command
+static int kill_output_check(const char *output)
+ {
+-    return runtime_call_simple(workdir, runtime, "kill", NULL, 0, id);
+    char *pattern = "does not exist";
+
+    if (output == NULL) {
+        return -1;
+    }
+
+    // container not exist, kill success, return 0
+    if (util_strings_contains_word(output, pattern)) {
+        return 0;
+    }
+
+    // kill failed, return -1
+    return -1;
+}
+
+// kill success or kill_output_check succeed return 0, DO_RETRY_CALL will break;
+// if kill failed, recheck on shim alive, if not alive, kill succeed,  still return 0;
+// else, return -1, DO_RETRY_CALL will call this again; 
+static int runtime_call_kill_and_check(const char *workdir, const char *runtime, const char *id)
+{
+    int ret = -1;
+
+    // kill succeed, return 0; kill_output_check succeed, return 0;
+    ret = runtime_call_simple(workdir, runtime, "kill", NULL, 0, id, kill_output_check);
+    if (ret == 0) {
+        return 0;
+    }
+
+    if (!shim_alive(workdir)) {
+        ret = 0;
+    }
+
+    return ret;
+ }
+ 
+ static int runtime_call_delete_force(const char *workdir, const char *runtime, const char *id)
+ {
+     const char *opts[1] = { "--force" };
+-    return runtime_call_simple(workdir, runtime, "delete", opts, 1, id);
+    return runtime_call_simple(workdir, runtime, "delete", opts, 1, id, NULL);
+ }
+ 
+ #define ExitSignalOffset 128
+@@ -919,7 +963,7 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t
+     pid_info->ppid = shim_pid;
+     pid_info->pstart_time = p_proc->start_time;
+ 
+-    if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id) != 0) {
+    if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) {
+         ERROR("call runtime start id failed");
+         ret = -1;
+         goto out;
+@@ -967,14 +1011,14 @@ int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_
+         shim_kill_force(workdir);
+     }
+ 
+-    // retry 10 count call runtime kill, every call sleep 1s
+-    DO_RETRY_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id);
+    // retry 10 count call runtime kill, every call sleep 0.5s
+    DO_RETRY_CALL(10, 500000, nret, runtime_call_kill_and_check, workdir, runtime, id);
+     if (nret != 0) {
+         WARN("call runtime force kill failed");
+     }
+ 
+-    // retry 10 count call runtime delete, every call sleep 1s
+-    DO_RETRY_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id);
+    // retry 10 count call runtime delete, every call sleep 0.1s
+    DO_RETRY_CALL(10, 100000, nret, runtime_call_delete_force, workdir, runtime, id);
+     if (nret != 0) {
+         WARN("call runtime force delete failed");
+     }
+@@ -1204,7 +1248,7 @@ int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t
+         return -1;
+     }
+ 
+-    return runtime_call_simple(workdir, runtime, "pause", NULL, 0, id);
+    return runtime_call_simple(workdir, runtime, "pause", NULL, 0, id, NULL);
+ }
+ 
+ int rt_isula_resume(const char *id, const char *runtime, const rt_resume_params_t *params)
+@@ -1221,7 +1265,7 @@ int rt_isula_resume(const char *id, const char *runtime, const rt_resume_params_
+         return -1;
+     }
+ 
+-    return runtime_call_simple(workdir, runtime, "resume", NULL, 0, id);
+    return runtime_call_simple(workdir, runtime, "resume", NULL, 0, id, NULL);
+ }
+ 
+ int rt_isula_listpids(const char *name, const char *runtime, const rt_listpids_params_t *params, rt_listpids_out_t *out)
+-- 
+2.25.1
+
--- a/0065-start-sandbox-before-setup-network-by-default.patch
+++ b/0065-start-sandbox-before-setup-network-by-default.patch
@ -0,0 +1,430 @@
+From a13e021620c62f32dfb1fd5242a3cf43c1d163b8 Mon Sep 17 00:00:00 2001
+From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
+Date: Wed, 14 Dec 2022 10:50:49 +0800
+Subject: [PATCH 65/65] start sandbox before setup network by default
+
+Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
+---
+ src/daemon/entry/cri/cni_network_plugin.cc    |  43 +++---
+ src/daemon/entry/cri/cni_network_plugin.h     |   2 +
+ src/daemon/entry/cri/cri_helpers.cc           |  12 ++
+ src/daemon/entry/cri/cri_helpers.h            |   3 +
+ .../cri_pod_sandbox_manager_service_impl.cc   | 135 +++++++++++-------
+ .../cri_pod_sandbox_manager_service_impl.h    |   2 +-
+ src/daemon/entry/cri/cri_security_context.cc  |   3 +-
+ src/daemon/modules/spec/specs.c               |   2 +-
+ src/utils/cutils/utils_file.c                 |   2 +-
+ 9 files changed, 135 insertions(+), 69 deletions(-)
+
+diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc
+index c850bc32..976a21a4 100644
+--- a/src/daemon/entry/cri/cni_network_plugin.cc
+++ b/src/daemon/entry/cri/cni_network_plugin.cc
+@@ -479,6 +479,27 @@ cleanup:
+     return result;
+ }
+ 
+auto CniNetworkPlugin::GetNetNSPath(const std::string &id, const std::map<std::string, std::string> &annotations,
+                                    Errors &err) -> std::string
+{
+    std::string netnsPath;
+
+    if (CRIHelpers::SetupNetworkFront(annotations)) {
+        auto iter = annotations.find(CRIHelpers::Constants::POD_SANDBOX_KEY);
+        if (iter == annotations.end()) {
+            ERROR("Failed to find sandbox key from annotations");
+            return netnsPath;
+        }
+        return iter->second;
+    }
+
+    netnsPath = GetNetNS(id, err);
+    if (err.NotEmpty()) {
+        ERROR("CNI failed to retrieve network namespace path: %s", err.GetCMessage());
+    }
+
+    return netnsPath;
+}
+ 
+ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName,
+                                 const std::string &id, const std::map<std::string, std::string> &annotations,
+@@ -489,12 +510,7 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name,
+         return;
+     }
+ 
+-    auto iter = annotations.find(CRIHelpers::Constants::POD_SANDBOX_KEY);
+-    if (iter == annotations.end()) {
+-        ERROR("Failed to find sandbox key from annotations");
+-        return;
+-    }
+-    const std::string netnsPath = iter->second;
+    std::string netnsPath = GetNetNSPath(id, annotations, err);
+     if (netnsPath.length() == 0) {
+         ERROR("Failed to get network namespace path");
+         return;
+@@ -600,21 +616,14 @@ void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &nam
+     }
+     Errors tmpErr;
+ 
+-    auto iter = annotations.find(CRIHelpers::Constants::POD_SANDBOX_KEY);
+-    if (iter == annotations.end()) {
+-        ERROR("Failed to find sandbox key from annotations");
+-        return;
+-    }
+-    std::string netnsPath = iter->second;
+-    if (netnsPath.length() == 0) {
+-        ERROR("Failed to get network namespace path");
+-        return;
+-    }
+    std::string netnsPath = GetNetNSPath(id, annotations, err);
+ 
+     // When netns file does not exist, netnsPath is assigned to an
+     // empty string so that lxc can handle the path properly
+-    if (!util_file_exists(netnsPath.c_str())) {
+    if (!util_file_exists(netnsPath.c_str()) || err.NotEmpty()) {
+        ERROR("Failed to get network namespace path, maybe podsandbox '%s' has been stopped", id.c_str());
+         netnsPath = "";
+        err.Clear();
+     }
+ 
+     RLockNetworkMap(err);
+diff --git a/src/daemon/entry/cri/cni_network_plugin.h b/src/daemon/entry/cri/cni_network_plugin.h
+index 8d51a94d..434222b5 100644
+--- a/src/daemon/entry/cri/cni_network_plugin.h
+++ b/src/daemon/entry/cri/cni_network_plugin.h
+@@ -118,6 +118,8 @@ public:
+ private:
+     auto GetNetNS(const std::string &podSandboxID, Errors &err) -> std::string;
+ 
+    auto GetNetNSPath(const std::string &id, const std::map<std::string, std::string> &annotations,
+                      Errors &err) -> std::string;
+ 
+ private:
+     virtual void PlatformInit(Errors &error);
+diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc
+index 64cea7ba..711196ba 100644
+--- a/src/daemon/entry/cri/cri_helpers.cc
+++ b/src/daemon/entry/cri/cri_helpers.cc
+@@ -49,6 +49,8 @@ const std::string Constants::DOCKER_IMAGEID_PREFIX { "docker://" };
+ const std::string Constants::DOCKER_PULLABLE_IMAGEID_PREFIX { "docker-pullable://" };
+ const std::string Constants::RUNTIME_READY { "RuntimeReady" };
+ const std::string Constants::NETWORK_READY { "NetworkReady" };
+// Kata 2.x need create network namespace and setup network befoce run podsandbox
+const std::string Constants::NETWORK_SETUP_ANNOTATION_KEY { "cri.sandbox.network.setup.v2" };
+ const std::string Constants::POD_CHECKPOINT_KEY { "cri.sandbox.isulad.checkpoint" };
+ const std::string Constants::CONTAINER_TYPE_ANNOTATION_KEY { "io.kubernetes.cri.container-type" };
+ const std::string Constants::CONTAINER_NAME_ANNOTATION_KEY { "io.kubernetes.cri.container-name" };
+@@ -1009,4 +1011,14 @@ out:
+     return runtime_val;
+ }
+ 
+bool SetupNetworkFront(const std::map<std::string, std::string> &annotations)
+{
+    auto iter = annotations.find(CRIHelpers::Constants::NETWORK_SETUP_ANNOTATION_KEY);
+    if (iter == annotations.end()) {
+        return false;
+    }
+
+    return iter->second == std::string("true");
+}
+
+ } // namespace CRIHelpers
+diff --git a/src/daemon/entry/cri/cri_helpers.h b/src/daemon/entry/cri/cri_helpers.h
+index b3bfafe4..d50759ad 100644
+--- a/src/daemon/entry/cri/cri_helpers.h
+++ b/src/daemon/entry/cri/cri_helpers.h
+@@ -48,6 +48,7 @@ public:
+     static const std::string DOCKER_PULLABLE_IMAGEID_PREFIX;
+     static const std::string RUNTIME_READY;
+     static const std::string NETWORK_READY;
+    static const std::string NETWORK_SETUP_ANNOTATION_KEY;
+     static const std::string POD_CHECKPOINT_KEY;
+     static const size_t MAX_CHECKPOINT_KEY_LEN { 250 };
+     static const std::string CONTAINER_TYPE_ANNOTATION_KEY;
+@@ -151,6 +152,8 @@ void StopContainer(service_executor_t *cb, const std::string &containerID, int64
+ char *GenerateExecSuffix();
+ 
+ char *cri_runtime_convert(const char *runtime);
+
+bool SetupNetworkFront(const std::map<std::string, std::string> &annotations);
+ }; // namespace CRIHelpers
+ 
+ #endif // DAEMON_ENTRY_CRI_CRI_HELPERS_H
+diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+index f7b3119d..4c245763 100644
+--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+@@ -99,10 +99,20 @@ void PodSandboxManagerServiceImpl::ApplySandboxResources(const runtime::v1alpha2
+ }
+ 
+ 
+-void PodSandboxManagerServiceImpl::SetHostConfigDefaultValue(host_config *hc)
+void PodSandboxManagerServiceImpl::SetHostConfigDefaultValue(const google::protobuf::Map<std::string, std::string> &annotations, 
+                                                             host_config *hc)
+ {
+     free(hc->network_mode);
+-    hc->network_mode = util_strdup_s(CRI::Constants::namespaceModeFile.c_str());
+
+    auto iter = annotations.find(CRIHelpers::Constants::NETWORK_SETUP_ANNOTATION_KEY);
+    // set network mode file when kata 2.x
+    if (iter != annotations.end() && iter->second == std::string("true")) {
+        hc->network_mode = util_strdup_s(CRI::Constants::namespaceModeFile.c_str());
+        return;
+    }
+
+    // default network mode is empty string
+    hc->network_mode = util_strdup_s("");
+ }
+ 
+ void PodSandboxManagerServiceImpl::MakeSandboxIsuladConfig(const runtime::v1alpha2::PodSandboxConfig &c,
+@@ -168,7 +178,7 @@ void PodSandboxManagerServiceImpl::MakeSandboxIsuladConfig(const runtime::v1alph
+         custom_config->hostname = util_strdup_s(c.hostname().c_str());
+     }
+ 
+-    SetHostConfigDefaultValue(hc);
+    SetHostConfigDefaultValue(c.annotations(), hc);
+ 
+     if (c.has_linux()) {
+         ApplySandboxLinuxOptions(c.linux(), hc, custom_config, error);
+@@ -487,7 +497,6 @@ void PodSandboxManagerServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2::
+ {
+     std::map<std::string, std::string> stdAnnos;
+     std::map<std::string, std::string> networkOptions;
+-    char* sandbox_key = get_sandbox_key(inspect_data);
+ 
+     // Setup sandbox files
+     if (config.has_dns_config() && inspect_data->resolv_conf_path != nullptr) {
+@@ -495,32 +504,36 @@ void PodSandboxManagerServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2::
+         SetupSandboxFiles(inspect_data->resolv_conf_path, config, error);
+         if (error.NotEmpty()) {
+             ERROR("failed to setup sandbox files");
+-            goto cleanup;
+            return;
+         }
+     }
+     // Do not invoke network plugins if in hostNetwork mode.
+     if (config.linux().security_context().namespace_options().network() == runtime::v1alpha2::NamespaceMode::NODE) {
+-        goto cleanup;
+        return;
+     }
+ 
+     // Setup networking for the sandbox.
+     CRIHelpers::ProtobufAnnoMapToStd(config.annotations(), stdAnnos);
+     stdAnnos[CRIHelpers::Constants::POD_CHECKPOINT_KEY] = jsonCheckpoint;
+     networkOptions["UID"] = config.metadata().uid();
+-    if (sandbox_key == NULL) {
+-        goto cleanup;
+
+    if (namespace_is_file(inspect_data->host_config->network_mode)) {
+        char *sandbox_key = get_sandbox_key(inspect_data);
+        if (sandbox_key == nullptr) {
+            ERROR("Empty sandbox key");
+            error.SetError("Empty sandbox key");
+            return;
+        }
+        stdAnnos.insert(std::pair<std::string, std::string>(CRIHelpers::Constants::POD_SANDBOX_KEY,
+                        std::string(sandbox_key)));
+        free(sandbox_key);
+     }
+-    stdAnnos.insert(std::pair<std::string, std::string>(CRIHelpers::Constants::POD_SANDBOX_KEY, sandbox_key));
+     m_pluginManager->SetUpPod(config.metadata().namespace_(), config.metadata().name(),
+                               Network::DEFAULT_NETWORK_INTERFACE_NAME, response_id, stdAnnos, networkOptions, error);
+     if (error.NotEmpty()) {
+         ERROR("SetupPod failed: %s", error.GetCMessage());
+-        StopContainerHelper(response_id, error);
+-        goto cleanup;
+     }
+ 
+-cleanup:
+-    free(sandbox_key);
+     return;
+ }
+ 
+@@ -568,8 +581,9 @@ auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSan
+         ERROR("Failed to retrieve inspect data");
+         goto cleanup;
+     }
+-    netnsPath = get_sandbox_key(inspect_data);
+
+     if (namespace_is_file(inspect_data->host_config->network_mode)) {
+        netnsPath = get_sandbox_key(inspect_data);
+         if (!util_file_exists(netnsPath) || util_mount_namespace(netnsPath) != 0) {
+             error.Errorf("Failed to mount network namespace");
+             ERROR("Failed to mount network namespace");
+@@ -578,9 +592,11 @@ auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSan
+     }
+ 
+     // Step 5: Setup networking for the sandbox.
+-    SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, error);
+-    if (error.NotEmpty()) {
+-        goto cleanup;
+    if (namespace_is_file(inspect_data->host_config->network_mode)) {
+        SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, error);
+        if (error.NotEmpty()) {
+            goto cleanup;
+        }
+     }
+ 
+     // Step 6: Start the sandbox container.
+@@ -589,6 +605,15 @@ auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSan
+         goto cleanup;
+     }
+ 
+    // If netns mode is not file, setup network after start sandbox container
+    if (!namespace_is_file(inspect_data->host_config->network_mode)) {
+        SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, error);
+        if (error.NotEmpty()) {
+            StopContainerHelper(response_id, error);
+            goto cleanup;
+        }
+    }
+
+ cleanup:
+     if (error.Empty()) {
+         SetNetworkReady(response_id, true, error);
+@@ -723,44 +748,58 @@ auto PodSandboxManagerServiceImpl::ClearCniNetwork(const std::string &realSandbo
+                                                    /*error*/) -> int
+ {
+     Errors networkErr;
+-    container_inspect* inspect_data = nullptr;
+    container_inspect *inspect_data = nullptr;
+    char *netnsPath = nullptr;
+
+    if (hostNetwork) {
+        return 0;
+    }
+ 
+     bool ready = GetNetworkReady(realSandboxID, networkErr);
+-    if (!hostNetwork && (ready || networkErr.NotEmpty())) {
+-        Errors pluginErr;
+-
+-        // hostNetwork has indicated network mode which render host config unnecessary
+-        // so that with_host_config is set to be false.
+-        inspect_data = CRIHelpers::InspectContainer(realSandboxID, pluginErr, false);
+-        if (pluginErr.NotEmpty()) {
+-            ERROR("Failed to inspect container");
+-        }
+    if (!ready && networkErr.Empty()) {
+        WARN("Network not ready");
+        return 0;
+    }
+ 
+-        char* netnsPath = get_sandbox_key(inspect_data);
+-        if (netnsPath == nullptr) {
+-            ERROR("Failed to get network namespace path");
+-            return 0;
+-        }
+    Errors pluginErr;
+    inspect_data = CRIHelpers::InspectContainer(realSandboxID, pluginErr, true);
+    if (pluginErr.NotEmpty()) {
+        ERROR("Failed to inspect container");
+        // not return and make sure teardown network
+    }
+ 
+-        stdAnnos.insert(std::pair<std::string, std::string>(CRIHelpers::Constants::POD_SANDBOX_KEY, netnsPath));
+-        m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos,
+-                                     pluginErr);
+-        if (pluginErr.NotEmpty()) {
+-            WARN("TearDownPod cni network failed: %s", pluginErr.GetCMessage());
+-            errlist.push_back(pluginErr.GetMessage());
+    if (inspect_data != nullptr && namespace_is_file(inspect_data->host_config->network_mode)) {
+        netnsPath = get_sandbox_key(inspect_data);
+        if (netnsPath == nullptr) {
+            ERROR("Get sandbox key failed");
+            // not return and make sure teardown network
+         } else {
+-            INFO("TearDownPod cni network: success");
+-            SetNetworkReady(realSandboxID, false, pluginErr);
+-            if (pluginErr.NotEmpty()) {
+-                WARN("set network ready: %s", pluginErr.GetCMessage());
+-            }
+-            // umount netns when cni removed network successfully
+-            if (util_umount_namespace(netnsPath) != 0) {
+-                ERROR("Failed to umount directory %s:%s", netnsPath, strerror(errno));
+-            }
+            stdAnnos.insert(std::pair<std::string, std::string>(CRIHelpers::Constants::POD_SANDBOX_KEY,
+                                                    std::string(netnsPath)));
+         }
+-        free(netnsPath);
+     }
+
+    m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos,
+                                    pluginErr);
+    if (pluginErr.NotEmpty()) {
+        WARN("TearDownPod cni network failed: %s", pluginErr.GetCMessage());
+        errlist.push_back(pluginErr.GetMessage());
+        goto out;
+    }
+
+    INFO("TearDownPod cni network: success");
+    SetNetworkReady(realSandboxID, false, pluginErr);
+    if (pluginErr.NotEmpty()) {
+        WARN("set network ready: %s", pluginErr.GetCMessage());
+    }
+    // umount netns when cni removed network successfully
+    if (inspect_data != nullptr && namespace_is_file(inspect_data->host_config->network_mode) &&
+        util_umount_namespace(netnsPath) != 0) {
+        ERROR("Failed to umount directory %s:%s", netnsPath, strerror(errno));
+    }
+
+out:
+    free(netnsPath);
+     free_container_inspect(inspect_data);
+     return 0;
+ }
+diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h
+index 68a9d919..f7c0aa00 100644
+--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h
+++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h
+@@ -66,7 +66,7 @@ private:
+                                                const std::string &image, std::string &jsonCheckpoint,
+                                                const std::string &runtimeHandler,
+                                                Errors &error) -> container_create_request *;
+-    void SetHostConfigDefaultValue(host_config *hc);
+    void SetHostConfigDefaultValue(const google::protobuf::Map<std::string, std::string> &annotations, host_config *hc);
+     void MakeSandboxIsuladConfig(const runtime::v1alpha2::PodSandboxConfig &c, host_config *hc,
+                                  container_config *custom_config, Errors &error);
+     void ApplySandboxLinuxOptions(const runtime::v1alpha2::LinuxPodSandboxConfig &lc, host_config *hc,
+diff --git a/src/daemon/entry/cri/cri_security_context.cc b/src/daemon/entry/cri/cri_security_context.cc
+index 8484108c..1cd9287d 100644
+--- a/src/daemon/entry/cri/cri_security_context.cc
+++ b/src/daemon/entry/cri/cri_security_context.cc
+@@ -162,7 +162,8 @@ static void ModifyHostNetworkOptionForSandbox(const runtime::v1alpha2::Namespace
+         free(hostConfig->uts_mode);
+         hostConfig->uts_mode = util_strdup_s(CRI::Constants::namespaceModeHost.c_str());
+     }
+-    // Note: default networkMode is file
+
+    // Set networkMode to file or "" in SetHostConfigDefaultValue
+ }
+ 
+ static void ModifyContainerNamespaceOptions(const runtime::v1alpha2::NamespaceOption &nsOpts,
+diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c
+index ec6385a6..138a0989 100644
+--- a/src/daemon/modules/spec/specs.c
+++ b/src/daemon/modules/spec/specs.c
+@@ -1460,7 +1460,7 @@ static int merge_share_network_namespace(oci_runtime_spec *oci_spec, const host_
+     int ret = 0;
+     char *ns_path = NULL;
+ 
+-    if (host_spec->network_mode == NULL) {
+    if (host_spec->network_mode == NULL || strlen(host_spec->network_mode) == 0) {
+         return 0;
+     }
+ 
+diff --git a/src/utils/cutils/utils_file.c b/src/utils/cutils/utils_file.c
+index f06f4d49..7b1d5cb6 100644
+--- a/src/utils/cutils/utils_file.c
+++ b/src/utils/cutils/utils_file.c
+@@ -85,7 +85,7 @@ bool util_file_exists(const char *f)
+     struct stat buf;
+     int nret;
+ 
+-    if (f == NULL) {
+    if (f == NULL || strlen(f) == 0) {
+         return false;
+     }
+ 
+-- 
+2.25.1
+
--- a/0066-compatibility-for-manage-pods-which-created-by-old-i.patch
+++ b/0066-compatibility-for-manage-pods-which-created-by-old-i.patch
@ -0,0 +1,157 @@
+From 4ad92d9524915bc83294627afe6954f43587e450 Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Thu, 15 Dec 2022 20:42:59 +0800
+Subject: [PATCH 66/66] compatibility for manage pods which created by old
+ iSulad
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ .../cri_pod_sandbox_manager_service_impl.cc   | 12 +++-
+ src/daemon/entry/cri/naming.cc                | 66 +++++++++++++++----
+ src/daemon/entry/cri/naming.h                 |  2 +-
+ 3 files changed, 66 insertions(+), 14 deletions(-)
+
+diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+index 4c245763..3e89008a 100644
+--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+@@ -1188,7 +1188,11 @@ void PodSandboxManagerServiceImpl::PodSandboxStatusToGRPC(
+ 
+     CRIHelpers::ExtractLabels(inspect->config->labels, *podStatus->mutable_labels());
+     CRIHelpers::ExtractAnnotations(inspect->config->annotations, *podStatus->mutable_annotations());
+-    CRINaming::ParseSandboxName(podStatus->annotations(), *podStatus->mutable_metadata(), error);
+    std::string name;
+    if (inspect->name != nullptr) {
+        name = std::string(inspect->name);
+    }
+    CRINaming::ParseSandboxName(name, podStatus->annotations(), *podStatus->mutable_metadata(), error);
+     if (error.NotEmpty()) {
+         return;
+     }
+@@ -1301,7 +1305,11 @@ void PodSandboxManagerServiceImpl::ListPodSandboxToGRPC(
+ 
+         CRIHelpers::ExtractAnnotations(response->containers[i]->annotations, *pod->mutable_annotations());
+ 
+-        CRINaming::ParseSandboxName(pod->annotations(), *pod->mutable_metadata(), error);
+        std::string name;
+        if (response->containers[i]->name != nullptr) {
+            name = std::string(response->containers[i]->name);
+        }
+        CRINaming::ParseSandboxName(name, pod->annotations(), *pod->mutable_metadata(), error);
+ 
+         if (filterOutReadySandboxes && pod->state() == runtime::v1alpha2::SANDBOX_READY) {
+             continue;
+diff --git a/src/daemon/entry/cri/naming.cc b/src/daemon/entry/cri/naming.cc
+index 682f2e52..54a14a81 100644
+--- a/src/daemon/entry/cri/naming.cc
+++ b/src/daemon/entry/cri/naming.cc
+@@ -26,6 +26,33 @@
+ #include "utils.h"
+ 
+ namespace CRINaming {
+static int parseName(const std::string &name, std::vector<std::string> &items, unsigned int &attempt, Errors &err)
+{
+    std::istringstream f(name);
+    std::string part;
+
+    while (getline(f, part, CRI::Constants::nameDelimiterChar)) {
+        items.push_back(part);
+    }
+
+    if (items.size() != 6) {
+        err.Errorf("failed to parse the sandbox name: %s", name.c_str());
+        return -1;
+    }
+
+    if (items[0] != CRI::Constants::kubePrefix) {
+        err.Errorf("container is not managed by kubernetes: %s", name.c_str());
+        return -1;
+    }
+
+    if (util_safe_uint(items[5].c_str(), &attempt)) {
+        err.Errorf("failed to parse the sandbox name %s: %s", name.c_str(), strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+ std::string MakeSandboxName(const runtime::v1alpha2::PodSandboxMetadata &metadata)
+ {
+     std::string sname;
+@@ -44,9 +71,12 @@ std::string MakeSandboxName(const runtime::v1alpha2::PodSandboxMetadata &metadat
+     return sname;
+ }
+ 
+-void ParseSandboxName(const google::protobuf::Map<std::string, std::string> &annotations,
+void ParseSandboxName(const std::string &name, const google::protobuf::Map<std::string, std::string> &annotations,
+                       runtime::v1alpha2::PodSandboxMetadata &metadata, Errors &err)
+ {
+    // need check uid and attemp 2 items
+    int needSetUidOrAttemp = 2;
+
+     if (annotations.count(CRIHelpers::Constants::SANDBOX_NAME_ANNOTATION_KEY) == 0) {
+         err.Errorf("annotation don't contains the sandbox name, failed to parse it");
+         return;
+@@ -57,21 +87,35 @@ void ParseSandboxName(const google::protobuf::Map<std::string, std::string> &ann
+         return;
+     }
+ 
+-    if (annotations.count(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY) == 0) {
+-        err.Errorf("annotation don't contains the sandbox uid, failed to parse it");
+-        return;
+    metadata.set_name(annotations.at(CRIHelpers::Constants::SANDBOX_NAME_ANNOTATION_KEY));
+    metadata.set_namespace_(annotations.at(CRIHelpers::Constants::SANDBOX_NAMESPACE_ANNOTATION_KEY));
+
+    if (annotations.count(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY) != 0) {
+        metadata.set_uid(annotations.at(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY));
+        needSetUidOrAttemp--;
+    }
+
+    if (annotations.count(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY) != 0) {
+        auto sandboxAttempt = annotations.at(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY);
+        metadata.set_attempt(static_cast<google::protobuf::uint32>(std::stoul(sandboxAttempt)));
+        needSetUidOrAttemp--;
+     }
+ 
+-    if (annotations.count(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY) == 0) {
+-        err.Errorf("annotation don't contains the sandbox attempt, failed to parse it");
+    if (needSetUidOrAttemp == 0) {
+         return;
+     }
+ 
+-    metadata.set_name(annotations.at(CRIHelpers::Constants::SANDBOX_NAME_ANNOTATION_KEY));
+-    metadata.set_namespace_(annotations.at(CRIHelpers::Constants::SANDBOX_NAMESPACE_ANNOTATION_KEY));
+-    metadata.set_uid(annotations.at(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY));
+-    auto sandboxAttempt = annotations.at(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY);
+-    metadata.set_attempt(static_cast<google::protobuf::uint32>(std::stoul(sandboxAttempt)));
+    // get uid and attempt from name,
+    // compatibility to new iSulad manage pods created by old version iSulad
+    // maybe should remove in next version of iSulad
+    std::vector<std::string> items;
+    unsigned int attempt;
+
+    if (parseName(name, items, attempt, err) != 0) {
+        return;
+    }
+    metadata.set_uid(items[4]);
+    metadata.set_attempt(static_cast<google::protobuf::uint32>(attempt));
+ }
+ 
+ std::string MakeContainerName(const runtime::v1alpha2::PodSandboxConfig &s, const runtime::v1alpha2::ContainerConfig &c)
+diff --git a/src/daemon/entry/cri/naming.h b/src/daemon/entry/cri/naming.h
+index 7eab41d3..f2d51a98 100644
+--- a/src/daemon/entry/cri/naming.h
+++ b/src/daemon/entry/cri/naming.h
+@@ -26,7 +26,7 @@ std::string MakeSandboxName(const runtime::v1alpha2::PodSandboxMetadata &metadat
+ std::string MakeContainerName(const runtime::v1alpha2::PodSandboxConfig &s,
+                               const runtime::v1alpha2::ContainerConfig &c);
+ 
+-void ParseSandboxName(const google::protobuf::Map<std::string, std::string> &annotations,
+void ParseSandboxName(const std::string &name, const google::protobuf::Map<std::string, std::string> &annotations,
+                       runtime::v1alpha2::PodSandboxMetadata &metadata, Errors &err);
+ 
+ void ParseContainerName(const google::protobuf::Map<std::string, std::string> &annotations,
+-- 
+2.25.1
+
--- a/iSulad.spec
+++ b/iSulad.spec
@ -1,5 +1,5 @@
 %global _version 2.0.17
-%global _release 11
+%global _release 12
 %global is_systemd 1
 %global enable_shimv2 1
 %global is_embedded 1
@ -67,6 +67,18 @@ Patch0051: 0051-remove-chmod-751-permission-for-dirs-by-engine-when-.patch
 Patch0052: 0052-add-console-ut.patch
 Patch0053: 0053-fix-additional-gids-for-exec-user.patch
 Patch0054: 0054-add-CI-for-additional-gid.patch
+Patch0055: 0055-retry-call-runtime-ops.patch
+Patch0056: 0056-add-ut-test-for-retry-macro.patch
+Patch0057: 0057-1749-set-inspect_container-timeout.patch
+Patch0058: 0058-1757-add-adaption-code-for-musl.patch
+Patch0059: 0059-deleting-broken-rootfs.patch
+Patch0060: 0060-1761-fix-leftover-devicemapper-mnt-dir.patch
+Patch0061: 0061-check-file-system-ro-before-merge-network-for-syscon.patch
+Patch0062: 0062-isulad-shim-wait-for-all-child-process.patch
+Patch0063: 0063-When-run-options-rm-is-set-delete-the-stoped-contain.patch
+Patch0064: 0064-recheck-kill-command-exit-status.patch
+Patch0065: 0065-start-sandbox-before-setup-network-by-default.patch
+Patch0066: 0066-compatibility-for-manage-pods-which-created-by-old-i.patch

 %ifarch x86_64 aarch64
 Provides:       libhttpclient.so()(64bit)
@ -293,6 +305,12 @@ fi
 %endif

 %changelog
+* Fri Dec 16 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-12
+- Type: bugfix
+- ID: NA
+- SUG: NA
+- DESC: update from upstream
+
 * Tue Dec 06 2022 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.0.17-11
 - Type: bugfix
 - ID: NA