update from upstream to update cri

Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
2023-05-11 15:17:12 +08:00 · 2023-05-11 15:17:12 +08:00 · 3e24926d4b
commit 3e24926d4b
parent 80bd003010
55 changed files with 12 additions and 13441 deletions
--- a/0001-modify-dependence-from-lcr-to-libisula.patch
+++ b/0001-modify-dependence-from-lcr-to-libisula.patch
@ -1,206 +0,0 @@
 From d3d13f374bb341e7e4d389feb1780e43fec91945 Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Tue, 7 Feb 2023 14:27:51 +0800
 Subject: [PATCH 01/53] modify dependence from lcr to libisula
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 cmake/checker.cmake                  |  2 +-
 iSulad.spec                          | 10 ++++-----
 isulad.pc.in                         |  6 +++---
 src/CMakeLists.txt                   | 32 ++++++++++++++--------------
 src/daemon/modules/runtime/runtime.c | 14 ++----------
 5 files changed, 27 insertions(+), 37 deletions(-)
 diff --git a/cmake/checker.cmake b/cmake/checker.cmake
 index 0a4b6fb6..a1ca9360 100644
 --- a/cmake/checker.cmake
 +++ b/cmake/checker.cmake
@@ -92,7 +92,7 @@ if (ENABLE_SELINUX)
 endif()
 # check iSula libutils
 -pkg_check_modules(PC_ISULA_LIBUTILS REQUIRED "lcr")
 +pkg_check_modules(PC_ISULA_LIBUTILS REQUIRED "libisula")
 find_path(ISULA_LIBUTILS_INCLUDE_DIR isula_libutils/log.h
 	HINTS ${PC_ISULA_LIBUTILS_INCLUDEDIR} ${PC_ISULA_LIBUTILS_INCLUDE_DIRS})
 _CHECK(ISULA_LIBUTILS_INCLUDE_DIR "ISULA_LIBUTILS_INCLUDE_DIR-NOTFOUND" "isula_libutils/log.h")
 diff --git a/iSulad.spec b/iSulad.spec
 index 79275885..3cff9c25 100644
 --- a/iSulad.spec
 +++ b/iSulad.spec
@@ -16,7 +16,7 @@ ExclusiveArch:  x86_64 aarch64
 %ifarch x86_64 aarch64
 Provides:       libhttpclient.so()(64bit)
 -Provides:       libisula.so()(64bit)
 +Provides:       libisula_client.so()(64bit)
 Provides:       libisulad_img.so()(64bit)
 Provides:       libisulad_tools.so()(64bit)
 %endif
@@ -42,14 +42,14 @@ BuildRequires: lib-shim-v2-devel
 Requires:      lib-shim-v2
 %endif
 -BuildRequires: cmake gcc-c++ lxc-devel lcr-devel yajl-devel
 +BuildRequires: cmake gcc-c++ lxc-devel lcr-devel yajl-devel libisula-devel
 BuildRequires: grpc-plugins grpc-devel protobuf-devel
 BuildRequires: libcurl-devel libarchive-devel device-mapper-devel
 BuildRequires: http-parser-devel
 BuildRequires: libselinux-devel libwebsockets-devel
 BuildRequires: systemd-devel git
 -Requires:      lcr lxc
 +Requires:      libisula lxc
 Requires:      grpc libcurl http-parser
 Requires:      libselinux libwebsockets libarchive device-mapper
 Requires:      systemd
@@ -78,13 +78,13 @@ cd build
 rm -rf %{buildroot}
 cd build
 install -d $RPM_BUILD_ROOT/%{_libdir}
 -install -m 0644 ./src/libisula.so             %{buildroot}/%{_libdir}/libisula.so
 +install -m 0644 ./src/libisula_client.so             %{buildroot}/%{_libdir}/libisula_client.so
 install -m 0644 ./src/utils/http/libhttpclient.so  %{buildroot}/%{_libdir}/libhttpclient.so
 chrpath -d ./src/libisulad_tools.so
 install -m 0644 ./src/libisulad_tools.so  %{buildroot}/%{_libdir}/libisulad_tools.so
 chrpath -d ./src/daemon/modules/image/libisulad_img.so
 install -m 0644 ./src/daemon/modules/image/libisulad_img.so   %{buildroot}/%{_libdir}/libisulad_img.so
 -chmod +x %{buildroot}/%{_libdir}/libisula.so
 +chmod +x %{buildroot}/%{_libdir}/libisula_client.so
 chmod +x %{buildroot}/%{_libdir}/libhttpclient.so
 chmod +x %{buildroot}/%{_libdir}/libisulad_img.so
 diff --git a/isulad.pc.in b/isulad.pc.in
 index 695ba364..016e406d 100644
 --- a/isulad.pc.in
 +++ b/isulad.pc.in
@@ -3,10 +3,10 @@ libdir=@CMAKE_INSTALL_PREFIX@/lib
 localstatedir=@CMAKE_INSTALL_PREFIX@/var
 includedir=@CMAKE_INSTALL_PREFIX@/include
 -Name: libisula
 -Description: light-weighted container runtime daemon library
 +Name: libisula_client
 +Description: light-weighted container client library
 Version: @ISULAD_VERSION@
 URL: iSulad
 -Libs: -L@CMAKE_INSTALL_PREFIX@/lib -lisula
 +Libs: -L@CMAKE_INSTALL_PREFIX@/lib -lisula_client
 Cflags: -I@CMAKE_INSTALL_PREFIX@/include
 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
 index abce1284..8de18082 100644
 --- a/src/CMakeLists.txt
 +++ b/src/CMakeLists.txt
@@ -60,40 +60,40 @@ endif()
 # get all c and header files
 list(REMOVE_DUPLICATES SHARED_INCS)
 -# ------ build libisula ------
 +# ------ build libisula_client ------
 add_subdirectory(client)
 if (OPENSSL_VERIFY)
     list(APPEND CLIENT_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/utils/http/certificate.c)
 endif()
 -add_library(libisula ${LIBTYPE}
 +add_library(libisula_client ${LIBTYPE}
     ${CLIENT_SRCS}
     )
 -target_include_directories(libisula PUBLIC
 +target_include_directories(libisula_client PUBLIC
     ${SHARED_INCS}
     ${CLIENT_INCS}
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/http
     )
 -# set libisula FLAGS
 -set_target_properties(libisula PROPERTIES PREFIX "")
 +# set libisula_client FLAGS
 +set_target_properties(libisula_client PROPERTIES PREFIX "")
 -target_link_libraries(libisula libisulad_tools)
 +target_link_libraries(libisula_client libisulad_tools)
 if (GRPC_CONNECTOR)
 -    target_link_libraries(libisula -Wl,--as-needed -lstdc++)
 -    target_link_libraries(libisula -Wl,--as-needed ${PROTOBUF_LIBRARY})
 -    target_link_libraries(libisula -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY})
 +    target_link_libraries(libisula_client -Wl,--as-needed -lstdc++)
 +    target_link_libraries(libisula_client -Wl,--as-needed ${PROTOBUF_LIBRARY})
 +    target_link_libraries(libisula_client -Wl,--no-as-needed ${GRPC_PP_REFLECTION_LIBRARY} ${GRPC_PP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY})
     if(ABSL_SYNC_LIB)
 -        target_link_libraries(libisula -Wl,--no-as-needed ${ABSL_SYNC_LIB})
 +        target_link_libraries(libisula_client -Wl,--no-as-needed ${ABSL_SYNC_LIB})
     endif()
 else()
 -    target_link_libraries(libisula -ldl libhttpclient)
 -    set_target_properties(libisula PROPERTIES LINKER_LANGUAGE "C")
 +    target_link_libraries(libisula_client -ldl libhttpclient)
 +    set_target_properties(libisula_client PROPERTIES LINKER_LANGUAGE "C")
 endif()
 -# ------ build libisula finish -----
 +# ------ build libisula_client finish -----
 add_subdirectory(cmd)
 # ------ build isula -------
@@ -101,7 +101,7 @@ add_executable(isula
     ${ISULA_SRCS}
     )
 target_include_directories(isula PUBLIC ${ISULA_INCS} ${SHARED_INCS})
 -target_link_libraries(isula libisula ${LIBYAJL_LIBRARY})
 +target_link_libraries(isula libisula_client ${LIBYAJL_LIBRARY})
 if (ANDROID OR MUSL)
     target_link_libraries(isula ${LIBSSL_LIBRARY})
 else()
@@ -192,7 +192,7 @@ endif()
 if (ISULAD_GCOV)
     target_link_libraries(isula -lgcov)
 -    target_link_libraries(libisula -lgcov)
 +    target_link_libraries(libisula_client -lgcov)
     target_link_libraries(isulad -lgcov)
 endif()
@@ -201,7 +201,7 @@ endif()
 # ------ install binary --------
 install(TARGETS libisulad_tools
     ${INSTALL_TYPE} DESTINATION ${LIB_INSTALL_DIR_DEFAULT} PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
 -install(TARGETS libisula
 +install(TARGETS libisula_client
     ${INSTALL_TYPE} DESTINATION ${LIB_INSTALL_DIR_DEFAULT} PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
 install(TARGETS isula
     RUNTIME DESTINATION bin PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
 diff --git a/src/daemon/modules/runtime/runtime.c b/src/daemon/modules/runtime/runtime.c
 index 23a4d60f..7a3ed87f 100644
 --- a/src/daemon/modules/runtime/runtime.c
 +++ b/src/daemon/modules/runtime/runtime.c
@@ -542,20 +542,10 @@ bool is_default_runtime(const char *name)
 int runtime_init()
 {
 -    int ret = 0;
 -
     if (engines_global_init()) {
         ERROR("Init engines global failed");
 -        ret = -1;
 -        goto out;
 +        return -1;
     }
 -    /* Init default engine, now is lcr */
 -    if (engines_discovery(DEFAULT_RUNTIME_NAME)) {
 -        ERROR("Failed to discovery engine %s", DEFAULT_RUNTIME_NAME);
 -        ret = -1;
 -    }
 -
 -out:
 -    return ret;
 +    return 0;
 }
 -- 
 2.25.1
--- a/0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch
+++ b/0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch
@ -1,243 +0,0 @@
 From 27c3d00f74c5641685d5781fe0c02c5eead92d23 Mon Sep 17 00:00:00 2001
 From: "ilya.kuksenok" <ilya.kuksenok@huawei.com>
 Date: Thu, 2 Feb 2023 14:41:16 +0300
 Subject: [PATCH 02/53] Add unified, memory_swap_limit_in_bytes fields into
 ContainerStats; add unified and memory_swap_limit_in_bytes into
 UpdateCreateConfig add nullptr for unified.
 ---
 src/daemon/common/sysinfo.c                   |  3 +-
 .../cri/cri_container_manager_service.cc      | 19 ++++++++++++
 src/daemon/entry/cri/cri_helpers.cc           | 17 ++++++++++-
 src/daemon/modules/runtime/runtime.c          | 30 +++++++++----------
 4 files changed, 52 insertions(+), 17 deletions(-)
 diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c
 index 8b5768db..38416db4 100644
 --- a/src/daemon/common/sysinfo.c
 +++ b/src/daemon/common/sysinfo.c
@@ -743,7 +743,8 @@ static void check_cgroup_mem(struct layer **layers, bool quiet, cgroup_mem_info_
         return;
     }
 -    meminfo->limit = true;
 +    meminfo->limit = cgroup_enabled(mountpoint, CGROUP_MEMORY_LIMIT);
 +    cgroup_do_log(quiet, !(meminfo->limit), "Your kernel does not support memory limit");
     meminfo->swap = cgroup_enabled(mountpoint, CGROUP_MEMORY_SWAP);
     cgroup_do_log(quiet, !(meminfo->swap), "Your kernel does not support swap memory limit");
 diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc
 index d2f486cf..d044cca8 100644
 --- a/src/daemon/entry/cri/cri_container_manager_service.cc
 +++ b/src/daemon/entry/cri/cri_container_manager_service.cc
@@ -1084,6 +1084,12 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai
     struct parser_context ctx {
         OPT_GEN_SIMPLIFY, 0
     };
 +    json_map_string_string *unified = nullptr;
 +    unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string));
 +    if (unified == nullptr) {
 +        error.SetError("Out of memory");
 +        goto cleanup;
 +    }
     request = (container_update_request *)util_common_calloc_s(sizeof(container_update_request));
     if (request == nullptr) {
         error.SetError("Out of memory");
@@ -1100,6 +1106,18 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai
     hostconfig->cpu_period = resources.cpu_period();
     hostconfig->cpu_quota = resources.cpu_quota();
     hostconfig->cpu_shares = resources.cpu_shares();
 +    hostconfig->memory_swap_limit_in_bytes = resources.memory_swap_limit_in_bytes();
 +
 +    if (!resources.unified().empty()) {
 +        for (auto &iter : resources.unified()) {
 +            if (append_json_map_string_string(unified, iter.first.c_str(), iter.second.c_str()) != 0) {
 +                error.SetError("Failed to append string");
 +                goto cleanup;
 +            }
 +        }
 +    }
 +    hostconfig->unified = unified;
 +    unified = nullptr;
     hostconfig->memory = resources.memory_limit_in_bytes();
     if (!resources.cpuset_cpus().empty()) {
         hostconfig->cpuset_cpus = util_strdup_s(resources.cpuset_cpus().c_str());
@@ -1126,6 +1144,7 @@ cleanup:
     free_container_update_request(request);
     free_container_update_response(response);
     free_host_config(hostconfig);
 +    free_json_map_string_string(unified);
     free(perror);
 }
 diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc
 index ddcc153f..2f6dcf78 100644
 --- a/src/daemon/entry/cri/cri_helpers.cc
 +++ b/src/daemon/entry/cri/cri_helpers.cc
@@ -445,8 +445,23 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc,
             hc->cpuset_mems = util_strdup_s(rOpts.cpuset_mems().c_str());
         }
         hc->oom_score_adj = rOpts.oom_score_adj();
 +        hc->memory_swap_limit_in_bytes = rOpts.memory_swap_limit_in_bytes();
 +        auto *unified = (json_map_string_string *)util_common_calloc_s(sizeof(json_map_string_string));
 +        if (unified == nullptr) {
 +            error.SetError("Out of memory");
 +            return;
 +        }
 +        if (!rOpts.unified().empty()) {
 +            for (auto &iter : rOpts.unified()) {
 +                if (append_json_map_string_string(unified, iter.first.c_str(), iter.second.c_str()) != 0) {
 +                    error.SetError("Failed to append string");
 +                    free_json_map_string_string(unified);
 +                    return;
 +                }
 +            }
 +        }
 +        hc->unified = unified;
     }
 -
     createConfig->open_stdin = config.stdin();
     createConfig->tty = config.tty();
 }
 diff --git a/src/daemon/modules/runtime/runtime.c b/src/daemon/modules/runtime/runtime.c
 index 7a3ed87f..29a64ac1 100644
 --- a/src/daemon/modules/runtime/runtime.c
 +++ b/src/daemon/modules/runtime/runtime.c
@@ -122,7 +122,7 @@ int runtime_create(const char *name, const char *runtime, const rt_create_params
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL) {
 -        ERROR("Invalide arguments for runtime create");
 +        ERROR("Invalid arguments for runtime create");
         ret = -1;
         goto out;
     }
@@ -146,7 +146,7 @@ int runtime_start(const char *name, const char *runtime, const rt_start_params_t
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || pid_info == NULL) {
 -        ERROR("Invalide arguments for runtime start");
 +        ERROR("Invalid arguments for runtime start");
         ret = -1;
         goto out;
     }
@@ -194,7 +194,7 @@ int runtime_restart(const char *name, const char *runtime, const rt_restart_para
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL) {
 -        ERROR("Invalide arguments for runtime restart");
 +        ERROR("Invalid arguments for runtime restart");
         ret = -1;
         goto out;
     }
@@ -218,7 +218,7 @@ int runtime_clean_resource(const char *name, const char *runtime, const rt_clean
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL) {
 -        ERROR("Invalide arguments for runtime clean");
 +        ERROR("Invalid arguments for runtime clean");
         ret = -1;
         goto out;
     }
@@ -242,7 +242,7 @@ int runtime_rm(const char *name, const char *runtime, const rt_rm_params_t *para
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL) {
 -        ERROR("Invalide arguments for runtime rm");
 +        ERROR("Invalid arguments for runtime rm");
         ret = -1;
         goto out;
     }
@@ -267,7 +267,7 @@ int runtime_status(const char *name, const char *runtime, const rt_status_params
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || status == NULL) {
 -        ERROR("Invalide arguments for runtime status");
 +        ERROR("Invalid arguments for runtime status");
         ret = -1;
         goto out;
     }
@@ -292,7 +292,7 @@ int runtime_resources_stats(const char *name, const char *runtime, const rt_stat
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || rs_stats == NULL) {
 -        ERROR("Invalide arguments for runtime stats");
 +        ERROR("Invalid arguments for runtime stats");
         ret = -1;
         goto out;
     }
@@ -316,7 +316,7 @@ int runtime_exec(const char *name, const char *runtime, const rt_exec_params_t *
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || exit_code == NULL) {
 -        ERROR("Invalide arguments for runtime exec");
 +        ERROR("Invalid arguments for runtime exec");
         ret = -1;
         goto out;
     }
@@ -340,7 +340,7 @@ int runtime_pause(const char *name, const char *runtime, const rt_pause_params_t
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || params == NULL) {
 -        ERROR("Invalide arguments for runtime pause");
 +        ERROR("Invalid arguments for runtime pause");
         ret = -1;
         goto out;
     }
@@ -364,7 +364,7 @@ int runtime_resume(const char *name, const char *runtime, const rt_resume_params
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || params == NULL) {
 -        ERROR("Invalide arguments for runtime resume");
 +        ERROR("Invalid arguments for runtime resume");
         ret = -1;
         goto out;
     }
@@ -388,7 +388,7 @@ int runtime_attach(const char *name, const char *runtime, const rt_attach_params
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || params == NULL) {
 -        ERROR("Invalide arguments for runtime attach");
 +        ERROR("Invalid arguments for runtime attach");
         ret = -1;
         goto out;
     }
@@ -412,7 +412,7 @@ int runtime_update(const char *name, const char *runtime, const rt_update_params
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || params == NULL) {
 -        ERROR("Invalide arguments for runtime update");
 +        ERROR("Invalid arguments for runtime update");
         ret = -1;
         goto out;
     }
@@ -447,7 +447,7 @@ int runtime_listpids(const char *name, const char *runtime, const rt_listpids_pa
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || params == NULL || out == NULL) {
 -        ERROR("Invalide arguments for runtime listpids");
 +        ERROR("Invalid arguments for runtime listpids");
         ret = -1;
         goto out;
     }
@@ -471,7 +471,7 @@ int runtime_resize(const char *name, const char *runtime, const rt_resize_params
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || params == NULL) {
 -        ERROR("Invalide arguments for runtime resize");
 +        ERROR("Invalid arguments for runtime resize");
         ret = -1;
         goto out;
     }
@@ -495,7 +495,7 @@ int runtime_exec_resize(const char *name, const char *runtime, const rt_exec_res
     const struct rt_ops *ops = NULL;
     if (name == NULL || runtime == NULL || params == NULL) {
 -        ERROR("Invalide arguments for runtime exec resize");
 +        ERROR("Invalid arguments for runtime exec resize");
         ret = -1;
         goto out;
     }
 -- 
 2.25.1
--- a/0003-Add-macro-for-protoc-cmake.patch
+++ b/0003-Add-macro-for-protoc-cmake.patch
@ -1,135 +0,0 @@
 From 466309bc0aafe61ebed5c71012e28b9912783b60 Mon Sep 17 00:00:00 2001
 From: Xuepeng Xu <xuxuepeng1@huawei.com>
 Date: Thu, 9 Feb 2023 14:32:59 +0800
 Subject: [PATCH 03/53] Add macro for protoc cmake
 Signed-off-by: Xuepeng Xu <xuxuepeng1@huawei.com>
 ---
 cmake/protoc.cmake | 98 +++++++++++++---------------------------------
 1 file changed, 28 insertions(+), 70 deletions(-)
 diff --git a/cmake/protoc.cmake b/cmake/protoc.cmake
 index 5c433e5c..23b8c077 100644
 --- a/cmake/protoc.cmake
 +++ b/cmake/protoc.cmake
@@ -11,89 +11,47 @@ if (ENABLE_NATIVE_NETWORK)
 set(NETWORK_PROTOS_OUT_PATH ${GRPC_OUT_PRE_PATH}/src/api/services/network)
 endif()
 +macro(PROTOC_CPP_GEN proto_name cpp_out_path proto_path)
 +    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/${proto_name} --cpp_out=${cpp_out_path} ${proto_path} ERROR_VARIABLE cpp_err)
 +    if (cpp_err)
 +        message("Parse ${proto_path} failed: ")
 +        message(FATAL_ERROR ${cpp_err})
 +    endif()
 +endmacro(PROTOC_CPP_GEN)
 +
 +macro(PROTOC_GRPC_GEN proto_name grpc_out_path proto_path)
 +    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/${proto_name} --grpc_out=${grpc_out_path} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${proto_path} ERROR_VARIABLE grpc_err)
 +    if (grpc_err)
 +        message("Parse ${proto_path} failed: ")
 +        message(FATAL_ERROR ${grpc_err})
 +    endif()
 +endmacro(PROTOC_GRPC_GEN)
 +
 if (GRPC_CONNECTOR)
     execute_process(COMMAND mkdir -p ${CONTAINER_PROTOS_OUT_PATH})
     execute_process(COMMAND mkdir -p ${IMAGE_PROTOS_OUT_PATH})
     execute_process(COMMAND mkdir -p ${VOLUME_PROTOS_OUT_PATH})
     execute_process(COMMAND mkdir -p ${CRI_PROTOS_OUT_PATH})
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/containers --cpp_out=${CONTAINER_PROTOS_OUT_PATH} 
 -        ${PROTOS_PATH}/containers/container.proto ERROR_VARIABLE containers_err)
 -    if (containers_err)
 -        message("Parse ${PROTOS_PATH}/containers/container.proto failed: ")
 -        message(FATAL_ERROR ${containers_err})
 -    endif()
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/containers --grpc_out=${CONTAINER_PROTOS_OUT_PATH} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/containers/container.proto ERROR_VARIABLE containers_err)
 -    if (containers_err)
 -        message("Parse ${PROTOS_PATH}/containers/container.proto plugin failed: ")
 -        message(FATAL_ERROR ${containers_err})
 -    endif()
 +    PROTOC_CPP_GEN(containers ${CONTAINER_PROTOS_OUT_PATH} ${PROTOS_PATH}/containers/container.proto)
 +    PROTOC_GRPC_GEN(containers ${CONTAINER_PROTOS_OUT_PATH} ${PROTOS_PATH}/containers/container.proto)
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/images --cpp_out=${IMAGE_PROTOS_OUT_PATH} ${PROTOS_PATH}/images/images.proto ERROR_VARIABLE images_err)
 -    if (images_err)
 -        message("Parse ${PROTOS_PATH}/images/images.proto failed: ")
 -        message(FATAL_ERROR ${images_err})
 -    endif()
 +    PROTOC_CPP_GEN(images ${IMAGE_PROTOS_OUT_PATH} ${PROTOS_PATH}/images/images.proto)
 +    PROTOC_GRPC_GEN(images ${IMAGE_PROTOS_OUT_PATH} ${PROTOS_PATH}/images/images.proto)
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/images --grpc_out=${IMAGE_PROTOS_OUT_PATH} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/images/images.proto ERROR_VARIABLE images_err)
 -    if (images_err)
 -        message("Parse ${PROTOS_PATH}/images/images.proto plugin failed: ")
 -        message(FATAL_ERROR ${images_err})
 -    endif()
 +    PROTOC_CPP_GEN(volumes ${VOLUME_PROTOS_OUT_PATH} ${PROTOS_PATH}/volumes/volumes.proto)
 +    PROTOC_GRPC_GEN(volumes ${VOLUME_PROTOS_OUT_PATH} ${PROTOS_PATH}/volumes/volumes.proto)
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/volumes --cpp_out=${VOLUME_PROTOS_OUT_PATH} ${PROTOS_PATH}/volumes/volumes.proto ERROR_VARIABLE volumes_err)
 -    if (volumes_err)
 -        message("Parse ${PROTOS_PATH}/volumes/volumes.proto failed: ")
 -        message(FATAL_ERROR ${volumes_err})
 -    endif()
 +    PROTOC_CPP_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/api.proto)
 +    PROTOC_GRPC_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/api.proto)
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/volumes --grpc_out=${VOLUME_PROTOS_OUT_PATH} --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/volumes/volumes.proto ERROR_VARIABLE volumes_err)
 -    if (volumes_err)
 -        message("Parse ${PROTOS_PATH}/volumes/volumes.proto plugin failed: ")
 -        message(FATAL_ERROR ${volumes_err})
 -    endif()
 -
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --cpp_out=${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/api.proto 
 -        ERROR_VARIABLE cri_err)
 -    if (cri_err)
 -        message("Parse ${PROTOS_PATH}/cri/api.proto failed: ")
 -        message(FATAL_ERROR ${cri_err})
 -    endif()
 -
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --grpc_out=${CRI_PROTOS_OUT_PATH} 
 -        --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/cri/api.proto ERROR_VARIABLE cri_err)
 -    if (cri_err)
 -        message("Parse ${PROTOS_PATH}/cri/api.proto plugin failed: ")
 -        message(FATAL_ERROR ${cri_err})
 -    endif()
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --cpp_out=${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/gogo.proto
 -            ERROR_VARIABLE cri_err)
 -    if (cri_err)
 -        message("Parse ${PROTOS_PATH}/cri/gogo.proto failed: ")
 -        message(FATAL_ERROR ${cri_err})
 -    endif()
 -    execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/cri --grpc_out=${CRI_PROTOS_OUT_PATH}
 -            --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/cri/gogo.proto ERROR_VARIABLE cri_err)
 -    if (cri_err)
 -        message("Parse ${PROTOS_PATH}/cri/gogo.proto plugin failed: ")
 -        message(FATAL_ERROR ${cri_err})
 -    endif()
 +    PROTOC_CPP_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/gogo.proto)
 +    PROTOC_GRPC_GEN(cri ${CRI_PROTOS_OUT_PATH} ${PROTOS_PATH}/cri/gogo.proto)
     if (ENABLE_NATIVE_NETWORK)
         execute_process(COMMAND mkdir -p ${NETWORK_PROTOS_OUT_PATH})
 -        execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/network
 -            --cpp_out=${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto ERROR_VARIABLE network_err)
 -        if (network_err)
 -            message("Parse ${PROTOS_PATH}/network/network.proto failed: ")
 -            message(FATAL_ERROR ${network_err})
 -        endif()
 -
 -        execute_process(COMMAND ${CMD_PROTOC} -I ${PROTOS_PATH}/network --grpc_out=${NETWORK_PROTOS_OUT_PATH}
 -            --plugin=protoc-gen-grpc=${CMD_GRPC_CPP_PLUGIN} ${PROTOS_PATH}/network/network.proto ERROR_VARIABLE network_err)
 -        if (network_err)
 -            message("Parse ${PROTOS_PATH}/network/network.proto plugin failed: ")
 -            message(FATAL_ERROR ${network_err})
 -        endif()
 +        PROTOC_CPP_GEN(network ${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto)
 +        PROTOC_GRPC_GEN(network ${NETWORK_PROTOS_OUT_PATH} ${PROTOS_PATH}/network/network.proto)
     endif()
 endif()
 -- 
 2.25.1
--- a/0004-fix-design-typo.patch
+++ b/0004-fix-design-typo.patch
@ -1,25 +0,0 @@
 From a7df50dc3b51f961f3d2e48dd968cfb115c39fec Mon Sep 17 00:00:00 2001
 From: zhushy <zhushangyuan@foxmail.com>
 Date: Sat, 11 Feb 2023 00:05:53 +0800
 Subject: [PATCH 04/53] fix design typo
 ---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/README.md b/README.md
 index 3d1fc7cb..7e4b6de1 100644
 --- a/README.md
 +++ b/README.md
@@ -70,7 +70,7 @@ CRI interface is implemented based on gRPC. iSulad implemented CRI gRPC Server f
 - [user manual](./docs/manual/README.md)
 -- [desgin docs](./docs/design/README.md)
 +- [design docs](./docs/design/README.md)
 ### Installing
 -- 
 2.25.1
--- a/0005-fix-cpu-rt-review-comments.patch
+++ b/0005-fix-cpu-rt-review-comments.patch
@ -1,44 +0,0 @@
 From 257054b234debb7b1fcafce6f2ec3df828370aed Mon Sep 17 00:00:00 2001
 From: songbuhuang <544824346@qq.com>
 Date: Sun, 12 Feb 2023 15:23:37 +0800
 Subject: [PATCH 05/53] fix cpu rt review comments
 Signed-off-by: songbuhuang <544824346@qq.com>
 ---
 src/daemon/executor/container_cb/execution_create.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
 diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c
 index feaa3064..cc9ae716 100644
 --- a/src/daemon/executor/container_cb/execution_create.c
 +++ b/src/daemon/executor/container_cb/execution_create.c
@@ -1327,7 +1327,7 @@ static int save_container_config_before_create(const char *id, const char *runti
 static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path)
 {
     int ret;
 -    int fd = 0;
 +    int fd = -1;
     ssize_t nwrite;
     char fpath[PATH_MAX] = { 0 };
     char buf[ISULAD_NUMSTRLEN64] = { 0 };
@@ -1342,13 +1342,13 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const
         return -1;
     }
 -    int nret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file);
 -    if (nret < 0 || nret >= sizeof(fpath)) {
 +    ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file);
 +    if (ret < 0 || ret >= sizeof(fpath)) {
         ERROR("Failed to print string");
         return -1;
     }
 -    nret = snprintf(buf, sizeof(buf), "%lld", (long long int)value);
 -    if (nret < 0 || (size_t)nret >= sizeof(buf)) {
 +    ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value);
 +    if (ret < 0 || (size_t)ret >= sizeof(buf)) {
         ERROR("Failed to print string");
         return -1;
     }
 -- 
 2.25.1
--- a/0006-fix-inspect.sh-failed.patch
+++ b/0006-fix-inspect.sh-failed.patch
@ -1,31 +0,0 @@
 From e4993d0e89ca853d74d8b23895de0967b4379441 Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Mon, 13 Feb 2023 17:42:30 +0800
 Subject: [PATCH 06/53] fix inspect.sh failed
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 CI/test_cases/container_cases/inspect.sh | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)
 diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh
 index 0d4ccb02..cde9ea1f 100755
 --- a/CI/test_cases/container_cases/inspect.sh
 +++ b/CI/test_cases/container_cases/inspect.sh
@@ -103,12 +103,7 @@ function test_inspect_spec()
     isula inspect --format='{{.Image}}' $containername 2>&1 | grep "sha256:${image_id}"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${image}" && ((ret++))
 -    if [ -d /sys/fs/cgroup/files ];then
 -        grepval="100"
 -    else
 -        grepval="0"
 -    fi
 -    isula inspect --format='{{json .HostConfig.FilesLimit}}' $containername 2>&1 | grep "$grepval"
 +    isula inspect --format='{{json .HostConfig.FilesLimit}}' $containername 2>&1 | grep 0
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${image}" && ((ret++))
     isula inspect --format='{{json .Config.Env}}' $containername 2>&1 | grep "a=1"
 -- 
 2.25.1
--- a/0007-add-CRI-ContainerStats-Service.patch
+++ b/0007-add-CRI-ContainerStats-Service.patch
@ -1,490 +0,0 @@
 From 9bd02c394110180ac7d7cbe80c1f4abe18146ebb Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Fri, 10 Feb 2023 17:43:11 +0800
 Subject: [PATCH 07/53] add CRI ContainerStats Service
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 .../connect/grpc/runtime_image_service.cc     | 12 ++---
 .../connect/grpc/runtime_runtime_service.cc   | 45 +++++++++++-----
 .../connect/grpc/runtime_runtime_service.h    |  3 ++
 .../cri/cri_container_manager_service.cc      | 54 +++++++++++++++++++
 .../entry/cri/cri_container_manager_service.h |  3 ++
 src/daemon/entry/cri/cri_runtime_service.h    |  3 ++
 .../entry/cri/cri_runtime_service_impl.cc     |  6 +++
 .../entry/cri/cri_runtime_service_impl.h      |  3 ++
 src/daemon/modules/events/collector.c         |  4 +-
 src/daemon/modules/image/image.c              | 16 +++---
 .../modules/image/oci/oci_common_operators.c  |  8 +--
 11 files changed, 125 insertions(+), 32 deletions(-)
 diff --git a/src/daemon/entry/connect/grpc/runtime_image_service.cc b/src/daemon/entry/connect/grpc/runtime_image_service.cc
 index 23447baf..e593a9c6 100644
 --- a/src/daemon/entry/connect/grpc/runtime_image_service.cc
 +++ b/src/daemon/entry/connect/grpc/runtime_image_service.cc
@@ -54,7 +54,7 @@ grpc::Status RuntimeImageServiceImpl::ListImages(grpc::ServerContext *context,
     std::vector<std::unique_ptr<runtime::v1alpha2::Image>> images;
     Errors error;
 -    WARN("Event: {Object: CRI, Type: Listing all images}");
 +    INFO("Event: {Object: CRI, Type: Listing all images}");
     rService->ListImages(request->filter(), &images, error);
     if (!error.Empty()) {
@@ -70,7 +70,7 @@ grpc::Status RuntimeImageServiceImpl::ListImages(grpc::ServerContext *context,
         *image = *(iter->get());
     }
 -    WARN("Event: {Object: CRI, Type: Listed all images}");
 +    INFO("Event: {Object: CRI, Type: Listed all images}");
     return grpc::Status::OK;
 }
@@ -82,7 +82,7 @@ grpc::Status RuntimeImageServiceImpl::ImageStatus(grpc::ServerContext *context,
     std::unique_ptr<runtime::v1alpha2::Image> image_info = nullptr;
     Errors error;
 -    WARN("Event: {Object: CRI, Type: Statusing image %s}", request->image().image().c_str());
 +    INFO("Event: {Object: CRI, Type: Statusing image %s}", request->image().image().c_str());
     image_info = rService->ImageStatus(request->image(), error);
     if (!error.Empty() && !CRIHelpers::IsImageNotFoundError(error.GetMessage())) {
@@ -96,7 +96,7 @@ grpc::Status RuntimeImageServiceImpl::ImageStatus(grpc::ServerContext *context,
         *image = *image_info;
     }
 -    WARN("Event: {Object: CRI, Type: Statused image %s}", request->image().image().c_str());
 +    INFO("Event: {Object: CRI, Type: Statused image %s}", request->image().image().c_str());
     return grpc::Status::OK;
 }
@@ -108,7 +108,7 @@ grpc::Status RuntimeImageServiceImpl::ImageFsInfo(grpc::ServerContext *context,
     std::vector<std::unique_ptr<runtime::v1alpha2::FilesystemUsage>> usages;
     Errors error;
 -    WARN("Event: {Object: CRI, Type: Statusing image fs info}");
 +    INFO("Event: {Object: CRI, Type: Statusing image fs info}");
     rService->ImageFsInfo(&usages, error);
     if (!error.Empty()) {
@@ -125,7 +125,7 @@ grpc::Status RuntimeImageServiceImpl::ImageFsInfo(grpc::ServerContext *context,
         *fs_info = *(iter->get());
     }
 -    WARN("Event: {Object: CRI, Type: Statused image fs info}");
 +    INFO("Event: {Object: CRI, Type: Statused image fs info}");
     return grpc::Status::OK;
 }
 diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc
 index b6f9e751..8fed162b 100644
 --- a/src/daemon/entry/connect/grpc/runtime_runtime_service.cc
 +++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.cc
@@ -171,7 +171,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainers(grpc::ServerContext *cont
 {
     Errors error;
 -    WARN("Event: {Object: CRI, Type: Listing all Container}");
 +    INFO("Event: {Object: CRI, Type: Listing all Container}");
     std::vector<std::unique_ptr<runtime::v1alpha2::Container>> containers;
     m_rService->ListContainers(request->has_filter() ? &request->filter() : nullptr, &containers, error);
@@ -189,7 +189,28 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainers(grpc::ServerContext *cont
         *container = *(iter->get());
     }
 -    WARN("Event: {Object: CRI, Type: Listed all Container}");
 +    INFO("Event: {Object: CRI, Type: Listed all Container}");
 +
 +    return grpc::Status::OK;
 +}
 +
 +grpc::Status RuntimeRuntimeServiceImpl::ContainerStats(grpc::ServerContext *context,
 +                                                       const runtime::v1alpha2::ContainerStatsRequest *request,
 +                                                       runtime::v1alpha2::ContainerStatsResponse *reply)
 +{
 +    Errors error;
 +
 +    INFO("Event: {Object: CRI, Type: Getting Container Stats: %s}", request->container_id().c_str());
 +
 +    std::unique_ptr<runtime::v1alpha2::ContainerStats> contStats =
 +        m_rService->ContainerStats(request->container_id(), error);
 +    if (!error.Empty() || !contStats) {
 +        ERROR("Object: CRI, Type: Failed to get container stats %s", request->container_id().c_str());
 +        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
 +    }
 +    *(reply->mutable_stats()) = *contStats;
 +
 +    INFO("Event: {Object: CRI, Type: Got Container stats: %s}", request->container_id().c_str());
     return grpc::Status::OK;
 }
@@ -200,7 +221,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainerStats(grpc::ServerContext *
 {
     Errors error;
 -    WARN("Event: {Object: CRI, Type: Listing all Container stats}");
 +    INFO("Event: {Object: CRI, Type: Listing all Container stats}");
     std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> containers;
     m_rService->ListContainerStats(request->has_filter() ? &request->filter() : nullptr, &containers, error);
@@ -218,7 +239,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListContainerStats(grpc::ServerContext *
         *container = *(iter->get());
     }
 -    WARN("Event: {Object: CRI, Type: Listed all Container stats}");
 +    INFO("Event: {Object: CRI, Type: Listed all Container stats}");
     return grpc::Status::OK;
 }
@@ -229,7 +250,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con
 {
     Errors error;
 -    WARN("Event: {Object: CRI, Type: Statusing Container: %s}", request->container_id().c_str());
 +    INFO("Event: {Object: CRI, Type: Statusing Container: %s}", request->container_id().c_str());
     std::unique_ptr<runtime::v1alpha2::ContainerStatus> contStatus =
         m_rService->ContainerStatus(request->container_id(), error);
@@ -239,7 +260,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con
     }
     *(reply->mutable_status()) = *contStatus;
 -    WARN("Event: {Object: CRI, Type: Statused Container: %s}", request->container_id().c_str());
 +    INFO("Event: {Object: CRI, Type: Statused Container: %s}", request->container_id().c_str());
     return grpc::Status::OK;
 }
@@ -329,7 +350,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co
 {
     Errors error;
 -    WARN("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str());
 +    INFO("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str());
     std::unique_ptr<runtime::v1alpha2::PodSandboxStatus> podStatus;
     podStatus = m_rService->PodSandboxStatus(request->pod_sandbox_id(), error);
@@ -340,7 +361,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co
     }
     *(reply->mutable_status()) = *podStatus;
 -    WARN("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str());
 +    INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str());
     return grpc::Status::OK;
 }
@@ -351,7 +372,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *cont
 {
     Errors error;
 -    WARN("Event: {Object: CRI, Type: Listing all Pods}");
 +    INFO("Event: {Object: CRI, Type: Listing all Pods}");
     std::vector<std::unique_ptr<runtime::v1alpha2::PodSandbox>> pods;
     m_rService->ListPodSandbox(request->has_filter() ? &request->filter() : nullptr, &pods, error);
@@ -368,7 +389,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ListPodSandbox(grpc::ServerContext *cont
         *pod = *(iter->get());
     }
 -    WARN("Event: {Object: CRI, Type: Listed all Pods}");
 +    INFO("Event: {Object: CRI, Type: Listed all Pods}");
     return grpc::Status::OK;
 }
@@ -460,7 +481,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Status(grpc::ServerContext *context,
 {
     Errors error;
 -    WARN("Event: {Object: CRI, Type: Statusing daemon}");
 +    INFO("Event: {Object: CRI, Type: Statusing daemon}");
     std::unique_ptr<runtime::v1alpha2::RuntimeStatus> status = m_rService->Status(error);
     if (status == nullptr || error.NotEmpty()) {
@@ -469,7 +490,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Status(grpc::ServerContext *context,
     }
     *(reply->mutable_status()) = *status;
 -    WARN("Event: {Object: CRI, Type: Statused daemon}");
 +    INFO("Event: {Object: CRI, Type: Statused daemon}");
     return grpc::Status::OK;
 }
 diff --git a/src/daemon/entry/connect/grpc/runtime_runtime_service.h b/src/daemon/entry/connect/grpc/runtime_runtime_service.h
 index 6e8c1009..cb3c5425 100644
 --- a/src/daemon/entry/connect/grpc/runtime_runtime_service.h
 +++ b/src/daemon/entry/connect/grpc/runtime_runtime_service.h
@@ -52,6 +52,9 @@ public:
                                     const runtime::v1alpha2::ListContainerStatsRequest *request,
                                     runtime::v1alpha2::ListContainerStatsResponse *reply) override;
 +    grpc::Status ContainerStats(grpc::ServerContext *context, const runtime::v1alpha2::ContainerStatsRequest *request,
 +                                runtime::v1alpha2::ContainerStatsResponse *reply) override;
 +
     grpc::Status ContainerStatus(grpc::ServerContext *context, const runtime::v1alpha2::ContainerStatusRequest *request,
                                  runtime::v1alpha2::ContainerStatusResponse *reply) override;
 diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc
 index d044cca8..710556a3 100644
 --- a/src/daemon/entry/cri/cri_container_manager_service.cc
 +++ b/src/daemon/entry/cri/cri_container_manager_service.cc
@@ -893,6 +893,60 @@ cleanup:
     free_container_stats_response(response);
 }
 +auto ContainerManagerService::ContainerStats(const std::string &containerID, Errors &error)
 +-> std::unique_ptr<runtime::v1alpha2::ContainerStats>
 +{
 +    container_stats_request *request { nullptr };
 +    container_stats_response *response { nullptr };
 +    std::unique_ptr<runtime::v1alpha2::ContainerStats> contStats { nullptr };
 +    std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> contStatsVec;
 +
 +    if (containerID.empty()) {
 +        error.SetError("Empty container id");
 +        return nullptr;
 +    }
 +
 +    if (m_cb == nullptr || m_cb->container.stats == nullptr) {
 +        error.SetError("Unimplemented callback");
 +        return nullptr;
 +    }
 +
 +    request = (container_stats_request *)util_common_calloc_s(sizeof(container_stats_request));
 +    if (request == nullptr) {
 +        error.SetError("Out of memory");
 +        return nullptr;
 +    }
 +
 +    request->containers = (char **)util_smart_calloc_s(sizeof(char *), 1);
 +    if (request->containers == nullptr) {
 +        error.SetError("Out of memory");
 +        goto cleanup;
 +    }
 +
 +    request->containers[0] = util_strdup_s(containerID.c_str());
 +    request->containers_len = 1;
 +
 +    if (m_cb->container.stats(request, &response) != 0) {
 +        if (response != nullptr && response->errmsg != nullptr) {
 +            error.SetError(response->errmsg);
 +        } else {
 +            error.SetError("Failed to call stats container callback");
 +        }
 +        goto cleanup;
 +    }
 +
 +    ContainerStatsToGRPC(response, &contStatsVec, error);
 +    if (error.NotEmpty()) {
 +        goto cleanup;
 +    }
 +    contStats = std::move(contStatsVec[0]);
 +
 +cleanup:
 +    free_container_stats_request(request);
 +    free_container_stats_response(response);
 +    return contStats;
 +}
 +
 void ContainerManagerService::PackContainerImageToStatus(
     container_inspect *inspect, std::unique_ptr<runtime::v1alpha2::ContainerStatus> &contStatus, Errors &error)
 {
 diff --git a/src/daemon/entry/cri/cri_container_manager_service.h b/src/daemon/entry/cri/cri_container_manager_service.h
 index 8002b77d..6ec1f21d 100644
 --- a/src/daemon/entry/cri/cri_container_manager_service.h
 +++ b/src/daemon/entry/cri/cri_container_manager_service.h
@@ -53,6 +53,9 @@ public:
                             std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> *containerstats,
                             Errors &error);
 +    auto ContainerStats(const std::string &containerID, Errors &error)
 +    -> std::unique_ptr<runtime::v1alpha2::ContainerStats>;
 +
     auto ContainerStatus(const std::string &containerID, Errors &error)
     -> std::unique_ptr<runtime::v1alpha2::ContainerStatus>;
 diff --git a/src/daemon/entry/cri/cri_runtime_service.h b/src/daemon/entry/cri/cri_runtime_service.h
 index 4727230f..5e4740cb 100644
 --- a/src/daemon/entry/cri/cri_runtime_service.h
 +++ b/src/daemon/entry/cri/cri_runtime_service.h
@@ -49,6 +49,9 @@ public:
                                     std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> *containerstats,
                                     Errors &error) = 0;
 +    virtual auto ContainerStats(const std::string &containerID,
 +                                Errors &error) -> std::unique_ptr<runtime::v1alpha2::ContainerStats> = 0;
 +
     virtual auto ContainerStatus(const std::string &containerID,
                                  Errors &error) -> std::unique_ptr<runtime::v1alpha2::ContainerStatus> = 0;
 diff --git a/src/daemon/entry/cri/cri_runtime_service_impl.cc b/src/daemon/entry/cri/cri_runtime_service_impl.cc
 index 241d4316..c4b84828 100644
 --- a/src/daemon/entry/cri/cri_runtime_service_impl.cc
 +++ b/src/daemon/entry/cri/cri_runtime_service_impl.cc
@@ -70,6 +70,12 @@ void CRIRuntimeServiceImpl::ListContainerStats(
     m_containerManager->ListContainerStats(filter, containerstats, error);
 }
 +auto CRIRuntimeServiceImpl::ContainerStats(const std::string &containerID, Errors &error)
 +-> std::unique_ptr<runtime::v1alpha2::ContainerStats>
 +{
 +    return m_containerManager->ContainerStats(containerID, error);
 +}
 +
 auto CRIRuntimeServiceImpl::ContainerStatus(const std::string &containerID, Errors &error)
 -> std::unique_ptr<runtime::v1alpha2::ContainerStatus>
 {
 diff --git a/src/daemon/entry/cri/cri_runtime_service_impl.h b/src/daemon/entry/cri/cri_runtime_service_impl.h
 index f2e25e42..7355c5cb 100644
 --- a/src/daemon/entry/cri/cri_runtime_service_impl.h
 +++ b/src/daemon/entry/cri/cri_runtime_service_impl.h
@@ -51,6 +51,9 @@ public:
                             std::vector<std::unique_ptr<runtime::v1alpha2::ContainerStats>> *containerstats,
                             Errors &error) override;
 +    auto ContainerStats(const std::string &containerID, Errors &error)
 +    -> std::unique_ptr<runtime::v1alpha2::ContainerStats> override;
 +
     auto ContainerStatus(const std::string &containerID, Errors &error)
     -> std::unique_ptr<runtime::v1alpha2::ContainerStatus> override;
 diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c
 index fb4e4a98..f53ad319 100644
 --- a/src/daemon/modules/events/collector.c
 +++ b/src/daemon/modules/events/collector.c
@@ -484,7 +484,7 @@ static int write_events_log(const struct isulad_events_format *events)
     len = calculate_annaotation_info_len(events);
     if (len == 1) {
 -        WARN("Event: {Object: %s, Type: %s}", events->id, events->opt);
 +        INFO("Event: {Object: %s, Type: %s}", events->id, events->opt);
     } else {
         annotation = (char *)util_common_calloc_s(len);
         if (annotation == NULL) {
@@ -502,7 +502,7 @@ static int write_events_log(const struct isulad_events_format *events)
         }
         (void)strcat(annotation, ")");
 -        WARN("Event: {Object: %s, Type: %s %s}", events->id, events->opt, annotation);
 +        INFO("Event: {Object: %s, Type: %s %s}", events->id, events->opt, annotation);
     }
 out:
 diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c
 index d5fbc8dc..a9b9523e 100644
 --- a/src/daemon/modules/image/image.c
 +++ b/src/daemon/modules/image/image.c
@@ -416,7 +416,7 @@ int im_get_filesystem_info(const char *image_type, im_fs_info_response **respons
         goto out;
     }
 -    WARN("Event: {Object: get image filesystem info, Type: inspecting}");
 +    INFO("Event: {Object: get image filesystem info, Type: inspecting}");
     ret = q->ops->get_filesystem_info(response);
     if (ret != 0) {
         if (response != NULL && *response != NULL) {
@@ -426,7 +426,7 @@ int im_get_filesystem_info(const char *image_type, im_fs_info_response **respons
         }
         goto out;
     }
 -    WARN("Event: {Object: get image filesystem info, Type: inspected}");
 +    INFO("Event: {Object: get image filesystem info, Type: inspected}");
 out:
     return ret;
@@ -466,7 +466,7 @@ int im_get_container_filesystem_usage(const char *image_type, const char *id, im
         request->name_id = util_strdup_s(id);
     }
 -    WARN("Event: {Object: container \'%s\' filesystem info, Type: inspecting}", id != NULL ? id : "");
 +    INFO("Event: {Object: container \'%s\' filesystem info, Type: inspecting}", id != NULL ? id : "");
     ret = q->ops->container_fs_usage(request, &filesystemusage);
     if (ret != 0) {
         ERROR("Failed to get filesystem usage for container %s", id);
@@ -476,7 +476,7 @@ int im_get_container_filesystem_usage(const char *image_type, const char *id, im
     *fs_usage = filesystemusage;
     filesystemusage = NULL;
 -    WARN("Event: {Object: container \'%s\' filesystem info, Type: inspected}", id != NULL ? id : "");
 +    INFO("Event: {Object: container \'%s\' filesystem info, Type: inspected}", id != NULL ? id : "");
 out:
     free_im_container_fs_usage_request(request);
@@ -916,7 +916,7 @@ int im_list_images(const im_list_request *ctx, im_list_response **response)
         return -1;
     }
 -    WARN("Event: {Object: list images, Type: listing}");
 +    INFO("Event: {Object: list images, Type: listing}");
     for (i = 0; i < g_numbims; i++) {
         if (g_bims[i].ops->list_ims == NULL) {
@@ -936,7 +936,7 @@ int im_list_images(const im_list_request *ctx, im_list_response **response)
         images_tmp = NULL;
     }
 -    WARN("Event: {Object: list images, Type: listed}");
 +    INFO("Event: {Object: list images, Type: listed}");
     if (g_isulad_errmsg != NULL) {
         (*response)->errmsg = util_strdup_s(g_isulad_errmsg);
@@ -1671,7 +1671,7 @@ int im_inspect_image(const im_inspect_request *request, im_inspect_response **re
     image_ref = util_strdup_s(request->image.image);
 -    WARN("Event: {Object: %s, Type: image inspecting}", image_ref);
 +    INFO("Event: {Object: %s, Type: image inspecting}", image_ref);
     bim_type = bim_query(image_ref);
     if (bim_type == NULL) {
@@ -1693,7 +1693,7 @@ int im_inspect_image(const im_inspect_request *request, im_inspect_response **re
         goto pack_response;
     }
 -    WARN("Event: {Object: %s, Type: image inspected}", image_ref);
 +    INFO("Event: {Object: %s, Type: image inspected}", image_ref);
 pack_response:
     if (g_isulad_errmsg != NULL) {
 diff --git a/src/daemon/modules/image/oci/oci_common_operators.c b/src/daemon/modules/image/oci/oci_common_operators.c
 index 3d200e09..83cccbe6 100644
 --- a/src/daemon/modules/image/oci/oci_common_operators.c
 +++ b/src/daemon/modules/image/oci/oci_common_operators.c
@@ -429,7 +429,7 @@ int oci_summary_image(im_summary_request *request, im_summary_response *response
         goto pack_response;
     }
 -    WARN("Event: {Object: %s, Type: statusing image summary}", resolved_name);
 +    INFO("Event: {Object: %s, Type: statusing image summary}", resolved_name);
     image_summary = storage_img_get_summary(resolved_name);
     if (image_summary == NULL) {
@@ -442,7 +442,7 @@ int oci_summary_image(im_summary_request *request, im_summary_response *response
     response->image_summary = image_summary;
     image_summary = NULL;
 -    WARN("Event: {Object: %s, Type: statused image summary}", resolved_name);
 +    INFO("Event: {Object: %s, Type: statused image summary}", resolved_name);
 pack_response:
     free(resolved_name);
@@ -493,7 +493,7 @@ int oci_status_image(im_status_request *request, im_status_response *response)
         goto pack_response;
     }
 -    WARN("Event: {Object: %s, Type: statusing image}", resolved_name);
 +    INFO("Event: {Object: %s, Type: statusing image}", resolved_name);
     image_info = storage_img_get(resolved_name);
     if (image_info == NULL) {
@@ -506,7 +506,7 @@ int oci_status_image(im_status_request *request, im_status_response *response)
     response->image_info->image = image_info;
     image_info = NULL;
 -    WARN("Event: {Object: %s, Type: statused image}", resolved_name);
 +    INFO("Event: {Object: %s, Type: statused image}", resolved_name);
 pack_response:
     free(resolved_name);
 -- 
 2.25.1
--- a/0008-fix-isula-cpu-rt-CI.patch
+++ b/0008-fix-isula-cpu-rt-CI.patch
@ -1,183 +0,0 @@
 From edef459d5052dc6d7c29e8a7a48ff4bf1b01bd78 Mon Sep 17 00:00:00 2001
 From: songbuhuang <544824346@qq.com>
 Date: Tue, 14 Feb 2023 14:08:01 +0800
 Subject: [PATCH 08/53] fix isula cpu-rt CI
 Signed-off-by: songbuhuang <544824346@qq.com>
 ---
 CI/test_cases/container_cases/cpu_rt.sh | 102 +++++++++++++++++-------
 1 file changed, 73 insertions(+), 29 deletions(-)
 diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh
 index 3d70c840..353c2d71 100755
 --- a/CI/test_cases/container_cases/cpu_rt.sh
 +++ b/CI/test_cases/container_cases/cpu_rt.sh
@@ -21,12 +21,12 @@
 declare -r curr_path=$(dirname $(readlink -f "$0"))
 source ../helpers.sh
 -function test_cpu_rt_isulad_spec()
 +function test_cpurt_isulad_abnormal()
 {
     local ret=0
     local test="isulad cpu realtime test => (${FUNCNAME[@]})"
 -     msg_info "${test} starting..."
 +    msg_info "${test} starting..."
     isulad --cpu-rt-period xx --cpu-rt-runtime 950000 /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument'
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++))
@@ -38,54 +38,64 @@ function test_cpu_rt_isulad_spec()
     return ${ret}
 }
 -function test_cpu_rt_isula_spec()
 +function test_isula_update_normal()
 {
 -    local ret=0
 -    local image="busybox"
 -    local test="container cpu realtime test => (${FUNCNAME[@]})"
 -
 -    msg_info "${test} starting..."
 -
 -    #start isulad without cpu_rt
 -    start_isulad_without_valgrind
 +    #start isulad with cpu_rt
 +    isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 &
 +    wait_isulad_running
 +    
 +    c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000  ${image} sh`
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 -    isula pull ${image}
 -    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE}
 +    isula update --cpu-rt-period 900000 --cpu-rt-runtime 800000 $c_id
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++))
 -    isula images | grep busybox
 -    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
 +    isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "800000"
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 800000" && ((ret++))
 -    test_isula_run_spec
 +    isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_period_us" | grep "900000"
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_period_us: 900000" && ((ret++))
 -    #start isulad without cpu_rt:isulad cpu.rt_period_us default value is the cpu.rt_period_us of the upper-layer directory,cpu.rt_runtime_us is 0.
 -    isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "failed to write 10000" | grep "cpu.rt_runtime_us: Invalid argument"
 -    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++))
 +    isula rm -f $c_id
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++))
     stop_isulad_without_valgrind
 +    #set cpu-rt to the initial state
 +    isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 &
 +    wait_isulad_running
 +    msg_info "${test} finished with return ${ret}..."
 +    return ${ret}
 +}
 +
 +function test_isula_update_abnormal()
 +{
     #start isulad with cpu_rt
     isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 &
     wait_isulad_running
 -
 -    test_isula_run_spec
     c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000  ${image} sh`
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 -    isula update --cpu-rt-runtime 90000 $c_id
 +    isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++))
 -    isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "90000"
 -    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 90000" && ((ret++))
 +    isula update --cpu-rt-runtime 1000000 $c_id | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument"
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++))
     isula rm -f $c_id
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++))
 +    stop_isulad_without_valgrind
 +    #set cpu-rt to the initial state
 +    isulad --cpu-rt-period 1000000 --cpu-rt-runtime 0 -l DEBUG > /dev/null 2>&1 &
 +    wait_isulad_running
 +
     msg_info "${test} finished with return ${ret}..."
     return ${ret}
 }
 -function test_kernel_without_cpu_rt_spec()
 +function test_kernel_without_cpurt()
 {
     local ret=0
     local image="busybox"
@@ -109,8 +119,24 @@ function test_kernel_without_cpu_rt_spec()
     return ${ret}
 }
 -function test_isula_run_spec()
 +function test_isula_run_abnormal()
 {
 +    local ret=0
 +    local image="busybox"
 +    local test="container cpu realtime test => (${FUNCNAME[@]})"
 +
 +    msg_info "${test} starting..."
 +
 +    #start isulad without cpu_rt
 +    isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 &
 +    wait_isulad_running
 +
 +    isula pull ${image}
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE}
 +
 +    isula images | grep busybox
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
 +
     isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime -1 $image /bin/sh 2>&1 | grep "failed to write -1" | grep "cpu.rt_runtime_us: Invalid argument"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++))
@@ -128,15 +154,33 @@ function test_isula_run_spec()
     isula run -itd --cpu-rt-period 100 --cpu-rt-runtime 10000 $image /bin/sh 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cpu-rt-runtime cannot be higher than cpu-rt-period" && ((ret++))
 +
 +    isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 $image /bin/sh 2>&1 | grep "failed to write 960000" | grep "cpu.rt_runtime_us: Invalid argument"
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++))
 +
 +    stop_isulad_without_valgrind
 +}
 +
 +function test_isula_run_normal()
 +{
 +    isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++))
 +
 +    isula rm -f box
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++))
 +
 }
 declare -i ans=0
 if [ -f "/sys/fs/cgroup/cpu/cpu.rt_runtime_us" ];then
 -    test_cpu_rt_isulad_spec || ((ans++))
 -    test_cpu_rt_isula_spec || ((ans++))
 +    test_isula_run_abnormal || ((ans++))
 +    test_isula_run_normal || ((ans++))
 +    test_cpurt_isulad_abnormal || ((ans++))
 +    test_isula_update_normal || ((ans++))
 +    test_isula_update_abnormal || ((ans++))
 else
 -    test_kernel_without_cpu_rt_spec || ((ans++))
 +    test_kernel_without_cpurt || ((ans++))
 fi
 show_result ${ans} "${curr_path}/${0}"
 -- 
 2.25.1
--- a/0009-fix-cpu-rt-CI.patch
+++ b/0009-fix-cpu-rt-CI.patch
@ -1,105 +0,0 @@
 From 0504a907def3efb4c0ad7eabd5921c97090430af Mon Sep 17 00:00:00 2001
 From: songbuhuang <544824346@qq.com>
 Date: Tue, 14 Feb 2023 15:55:56 +0800
 Subject: [PATCH 09/53] fix cpu-rt CI
 Signed-off-by: songbuhuang <544824346@qq.com>
 ---
 CI/test_cases/container_cases/cpu_rt.sh | 32 +++++++++++++++++++------
 1 file changed, 25 insertions(+), 7 deletions(-)
 diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh
 index 353c2d71..42006bc8 100755
 --- a/CI/test_cases/container_cases/cpu_rt.sh
 +++ b/CI/test_cases/container_cases/cpu_rt.sh
@@ -24,14 +24,14 @@ source ../helpers.sh
 function test_cpurt_isulad_abnormal()
 {
     local ret=0
 -    local test="isulad cpu realtime test => (${FUNCNAME[@]})"
 +    local test="isulad cpu realtime abnormal test => (${FUNCNAME[@]})"
     msg_info "${test} starting..."
 -    isulad --cpu-rt-period xx --cpu-rt-runtime 950000 /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument'
 +    isulad --cpu-rt-period xx --cpu-rt-runtime 950000  2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-period: Invalid argument'
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-period" && ((ret++))
 -    isulad --cpu-rt-period 1000000 --cpu-rt-runtime xx /bin/sh 2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument'
 +    isulad --cpu-rt-period 1000000 --cpu-rt-runtime xx  2>&1 | grep 'Invalid value "xx" for flag --cpu-rt-runtime: Invalid argument'
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++))
     msg_info "${test} finished with return ${ret}..."
@@ -40,6 +40,12 @@ function test_cpurt_isulad_abnormal()
 function test_isula_update_normal()
 {
 +    local ret=0
 +    local image="busybox"
 +    local test="isulad update cpu realtime normal test => (${FUNCNAME[@]})"
 +
 +    msg_info "${test} starting..."
 +
     #start isulad with cpu_rt
     isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 &
     wait_isulad_running
@@ -70,6 +76,10 @@ function test_isula_update_normal()
 function test_isula_update_abnormal()
 {
 +    local ret=0
 +    local image="busybox"
 +    local test="isulad update cpu realtime abnormal test => (${FUNCNAME[@]})"
 +
     #start isulad with cpu_rt
     isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 &
     wait_isulad_running
@@ -77,10 +87,10 @@ function test_isula_update_abnormal()
     c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000  ${image} sh`
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 -    isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period"
 +    isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++))
 -    isula update --cpu-rt-runtime 1000000 $c_id | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument"
 +    isula update --cpu-rt-runtime 1000000 $c_id 2>&1 | grep "updating cgroup cpu.rt_runtime_us to 1000000: Invalid argument"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++))
     isula rm -f $c_id
@@ -158,17 +168,23 @@ function test_isula_run_abnormal()
     isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 960000 $image /bin/sh 2>&1 | grep "failed to write 960000" | grep "cpu.rt_runtime_us: Invalid argument"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid argument for cpu-rt-runtime" && ((ret++))
 -    stop_isulad_without_valgrind
 +    msg_info "${test} finished with return ${ret}..."
 +    return ${ret}
 }
 function test_isula_run_normal()
 {
 +    local ret=0
 +    local image="busybox"
 +
     isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++))
     isula rm -f box
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to rm container ${c_id}" && ((ret++))
 -
 +    
 +    msg_info "${test} finished with return ${ret}..."
 +    return ${ret}
 }
 declare -i ans=0
@@ -183,4 +199,6 @@ else
     test_kernel_without_cpurt || ((ans++))
 fi
 +isula rm -f $(isula ps -aq)
 +
 show_result ${ans} "${curr_path}/${0}"
 -- 
 2.25.1
--- a/0010-fix-cpu-rt-CI.patch
+++ b/0010-fix-cpu-rt-CI.patch
@ -1,54 +0,0 @@
 From 21dc648ef93cd0fb858a408bc843d25a5e20e320 Mon Sep 17 00:00:00 2001
 From: songbuhuang <544824346@qq.com>
 Date: Wed, 15 Feb 2023 16:09:38 +0800
 Subject: [PATCH 10/53] fix cpu-rt CI
 Signed-off-by: songbuhuang <544824346@qq.com>
 ---
 CI/test_cases/container_cases/cpu_rt.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
 diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh
 index 42006bc8..39c0b427 100755
 --- a/CI/test_cases/container_cases/cpu_rt.sh
 +++ b/CI/test_cases/container_cases/cpu_rt.sh
@@ -50,14 +50,14 @@ function test_isula_update_normal()
     isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 &
     wait_isulad_running
 -    c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000  ${image} sh`
 +    c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000  ${image} sh`
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 -    isula update --cpu-rt-period 900000 --cpu-rt-runtime 800000 $c_id
 +    isula update --cpu-rt-period 900000 --cpu-rt-runtime 2000 $c_id
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to update container cpu-rt-runtime" && ((ret++))
 -    isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "800000"
 -    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 800000" && ((ret++))
 +    isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_runtime_us" | grep "2000"
 +    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_runtime_us: 2000" && ((ret++))
     isula exec -it $c_id sh -c "cat /sys/fs/cgroup/cpu/cpu.rt_period_us" | grep "900000"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container cpu.rt_period_us: 900000" && ((ret++))
@@ -84,7 +84,7 @@ function test_isula_update_abnormal()
     isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 -l DEBUG > /dev/null 2>&1 &
     wait_isulad_running
 -    c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 950000  ${image} sh`
 +    c_id=`isula run -itd --cpu-rt-period 1000000 --cpu-rt-runtime 1000  ${image} sh`
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
     isula update --cpu-rt-period 800000 --cpu-rt-runtime 900000 $c_id 2>&1 | grep "Invalid --cpu-rt-runtime: rt runtime cannot be higher than rt period"
@@ -177,7 +177,7 @@ function test_isula_run_normal()
     local ret=0
     local image="busybox"
 -    isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 900000 $image /bin/sh 2>&1
 +    isula run -itd -n box --cpu-rt-period 1000000 --cpu-rt-runtime 1000 $image /bin/sh 2>&1
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container" && ((ret++))
     isula rm -f box
 -- 
 2.25.1
--- a/0011-Bugfix-in-config-and-executor.patch
+++ b/0011-Bugfix-in-config-and-executor.patch
@ -1,52 +0,0 @@
 From 933eceb4545a28dba44c72f183dc7104d0fea714 Mon Sep 17 00:00:00 2001
 From: Xuepeng Xu <xuxuepeng1@huawei.com>
 Date: Wed, 15 Feb 2023 12:19:40 +0800
 Subject: [PATCH 11/53] Bugfix in config and executor
 Signed-off-by: Xuepeng Xu <xuxuepeng1@huawei.com>
 ---
 src/daemon/config/isulad_config.c                   | 2 +-
 src/daemon/executor/container_cb/execution_create.c | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c
 index 917e3eaa..38bf4bf9 100644
 --- a/src/daemon/config/isulad_config.c
 +++ b/src/daemon/config/isulad_config.c
@@ -314,7 +314,7 @@ char *conf_get_routine_rootdir(const char *runtime)
     }
     /* path = conf->rootpath + / + engines + / + runtime + /0 */
 -    if (strlen(conf->json_confs->graph) > (SIZE_MAX - strlen(ENGINE_ROOTPATH_NAME)) - 3) {
 +    if (strlen(conf->json_confs->graph) > (SIZE_MAX - strlen(ENGINE_ROOTPATH_NAME) - strlen(runtime)) - 3) {
         ERROR("Graph path is too long");
         goto out;
     }
 diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c
 index cc9ae716..4cc333fd 100644
 --- a/src/daemon/executor/container_cb/execution_create.c
 +++ b/src/daemon/executor/container_cb/execution_create.c
@@ -533,7 +533,7 @@ static char *try_generate_id()
     int i = 0;
     int max_time = 10;
     char *id = NULL;
 -    char *value = NULL;
 +    container_t *value = NULL;
     id = util_smart_calloc_s(sizeof(char), (CONTAINER_ID_MAX_LEN + 1));
     if (id == NULL) {
@@ -547,9 +547,9 @@ static char *try_generate_id()
             goto err_out;
         }
 -        value = container_name_index_get(id);
 +        value = containers_store_get(id);
         if (value != NULL) {
 -            free(value);
 +            container_unref(value);
             value = NULL;
             continue;
         } else {
 -- 
 2.25.1
--- a/0012-fix-cpu-rt-disable-after-reboot-machine.patch
+++ b/0012-fix-cpu-rt-disable-after-reboot-machine.patch
@ -1,823 +0,0 @@
 From 166edf2093b2c35fe4e479ca4b6568be8c98f907 Mon Sep 17 00:00:00 2001
 From: haozi007 <liuhao27@huawei.com>
 Date: Wed, 15 Feb 2023 17:47:12 +0800
 Subject: [PATCH 12/53] fix cpu-rt disable after reboot machine
 1. ensure parent cgroup cpu-rt of container, should do in start container;
 2. current do in create container, will cause failed of start container with cpu-rt after reboot machine
 Signed-off-by: haozi007 <liuhao27@huawei.com>
 ---
 src/daemon/common/sysinfo.c                   |  46 +++-
 src/daemon/common/sysinfo.h                   |   2 +
 src/daemon/executor/container_cb/execution.c  | 158 ++++++++++++-
 .../executor/container_cb/execution_create.c  | 222 +-----------------
 src/daemon/modules/api/specs_api.h            |   2 +-
 src/daemon/modules/spec/specs.c               |  84 ++++---
 test/specs/specs/specs_ut.cc                  |  40 ++--
 7 files changed, 280 insertions(+), 274 deletions(-)
 diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c
 index 38416db4..7559d653 100644
 --- a/src/daemon/common/sysinfo.c
 +++ b/src/daemon/common/sysinfo.c
@@ -24,8 +24,10 @@
 #include <linux/magic.h>
 #include <sys/stat.h>
 +#include <isula_libutils/auto_cleanup.h>
 +#include <isula_libutils/log.h>
 +
 #include "err_msg.h"
 -#include "isula_libutils/log.h"
 #include "utils.h"
 #include "utils_array.h"
 #include "utils_file.h"
@@ -1627,3 +1629,45 @@ free_out:
     }
     return minfos;
 }
 +
 +char *sysinfo_cgroup_controller_cpurt_mnt_path()
 +{
 +    int nret = 0;
 +    __isula_auto_free char *mnt = NULL;
 +    __isula_auto_free char *root = NULL;
 +    char fpath[PATH_MAX] = { 0 };
 +    sysinfo_t *sysinfo = NULL;
 +
 +    sysinfo = get_sys_info(true);
 +    if (sysinfo == NULL) {
 +        ERROR("Can not get system info");
 +        return NULL;
 +    }
 +
 +    if (!(sysinfo->cgcpuinfo.cpu_rt_period)) {
 +        ERROR("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel");
 +        isulad_set_error_message("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel");
 +        return NULL;
 +    }
 +
 +    nret = find_cgroup_mountpoint_and_root("cpu", &mnt, &root);
 +    if (nret != 0 || mnt == NULL || root == NULL) {
 +        ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'");
 +        isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'");
 +        return NULL;
 +    }
 +
 +    // When iSulad is run inside docker, the root is based of the host cgroup.
 +    // Replace root to "/"
 +    if (strncmp(root, "/docker/", strlen("/docker/")) == 0) {
 +        root[1] = '\0';
 +    }
 +
 +    nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root);
 +    if (nret < 0 || (size_t)nret >= sizeof(fpath)) {
 +        ERROR("Failed to print string");
 +        return NULL;
 +    }
 +
 +    return util_strdup_s(fpath);
 +}
 \ No newline at end of file
 diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h
 index 8468e00a..bbb3c6b5 100644
 --- a/src/daemon/common/sysinfo.h
 +++ b/src/daemon/common/sysinfo.h
@@ -139,6 +139,8 @@ mountinfo_t *find_mount_info(mountinfo_t **minfos, const char *dir);
 void free_mounts_info(mountinfo_t **minfos);
 +char *sysinfo_cgroup_controller_cpurt_mnt_path();
 +
 #ifdef __cplusplus
 }
 #endif
 diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c
 index 7b18a8e1..ed70fc14 100644
 --- a/src/daemon/executor/container_cb/execution.c
 +++ b/src/daemon/executor/container_cb/execution.c
@@ -18,6 +18,12 @@
 #include <pthread.h>
 #include <malloc.h>
 #include <sys/eventfd.h>
 +#include <stdbool.h>
 +#include <stdint.h>
 +#include <stdlib.h>
 +#include <string.h>
 +#include <libgen.h>
 +
 #include <isula_libutils/container_config.h>
 #include <isula_libutils/container_config_v2.h>
 #include <isula_libutils/container_delete_request.h>
@@ -34,13 +40,13 @@
 #include <isula_libutils/container_stop_request.h>
 #include <isula_libutils/container_stop_response.h>
 #include <isula_libutils/json_common.h>
 -#include <stdbool.h>
 -#include <stdint.h>
 -#include <stdlib.h>
 -#include <string.h>
 +#include <isula_libutils/auto_cleanup.h>
 +#include <isula_libutils/log.h>
 -#include "isula_libutils/log.h"
 +#include "isulad_config.h"
 +#include "sysinfo.h"
 #include "container_api.h"
 +#include "specs_api.h"
 #include "execution_extend.h"
 #include "execution_information.h"
 #include "execution_stream.h"
@@ -302,6 +308,135 @@ static void pack_start_response(container_start_response *response, uint32_t cc,
     }
 }
 +static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root,
 +                                      int64_t cpu_rt_period, int64_t cpu_rt_runtime);
 +
 +/* maybe create cpu realtime file */
 +static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path)
 +{
 +    int ret;
 +    __isula_auto_close int fd = -1;
 +    ssize_t nwrite;
 +    char fpath[PATH_MAX] = { 0 };
 +    char buf[ISULAD_NUMSTRLEN64] = { 0 };
 +
 +    if (value == 0) {
 +        return 0;
 +    }
 +
 +    ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE);
 +    if (ret != 0) {
 +        ERROR("Failed to mkdir: %s", path);
 +        return -1;
 +    }
 +
 +    ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file);
 +    if (ret < 0 || ret >= sizeof(fpath)) {
 +        ERROR("Failed to print string");
 +        return -1;
 +    }
 +    ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value);
 +    if (ret < 0 || (size_t)ret >= sizeof(buf)) {
 +        ERROR("Failed to print string");
 +        return -1;
 +    }
 +
 +    fd = util_open(fpath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0700);
 +    if (fd < 0) {
 +        ERROR("Failed to open file: %s: %s", fpath, strerror(errno));
 +        isulad_set_error_message("Failed to open file: %s: %s", fpath, strerror(errno));
 +        return -1;
 +    }
 +    nwrite = util_write_nointr(fd, buf, strlen(buf));
 +    if (nwrite < 0 || nwrite != strlen(buf)) {
 +        ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno));
 +        isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno));
 +        return -1;
 +    }
 +
 +    return 0;
 +}
 +
 +static int recursively_create_cgroup(const char *path, const char *mnt_root, int recursive_depth, int64_t cpu_rt_period,
 +                                     int64_t cpu_rt_runtime)
 +{
 +    int ret = 0;
 +    __isula_auto_free char *dup = NULL;
 +    char *dirpath = NULL;
 +    char fpath[PATH_MAX] = { 0 };
 +
 +    dup = util_strdup_s(path);
 +    dirpath = dirname(dup);
 +    ret = do_init_cpurt_cgroups_path(dirpath, (recursive_depth + 1), mnt_root, cpu_rt_period, cpu_rt_runtime);
 +    if (ret != 0) {
 +        return ret;
 +    }
 +
 +    int nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt_root, path);
 +    if (nret < 0 || (size_t)nret >= sizeof(fpath)) {
 +        ERROR("Failed to print string");
 +        return ret;
 +    }
 +
 +    ret = maybe_create_cpu_realtime_file(cpu_rt_period, "cpu.rt_period_us", fpath);
 +    if (ret != 0) {
 +        return ret;
 +    }
 +
 +    return maybe_create_cpu_realtime_file(cpu_rt_runtime, "cpu.rt_runtime_us", fpath);
 +}
 +
 +/* init cgroups path */
 +static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root,
 +                                      int64_t cpu_rt_period, int64_t cpu_rt_runtime)
 +{
 +    if ((recursive_depth + 1) > MAX_PATH_DEPTH) {
 +        ERROR("Reach the max cgroup depth:%s", path);
 +        return -1;
 +    }
 +
 +    if (path == NULL || strcmp(path, "/") == 0 || strcmp(path, ".") == 0) {
 +        return 0;
 +    }
 +
 +    // Recursively create cgroup to ensure that the system and all parent cgroups have values set
 +    // for the period and runtime as this limits what the children can be set to.
 +    return recursively_create_cgroup(path, mnt_root, recursive_depth, cpu_rt_period, cpu_rt_runtime);
 +}
 +
 +// TODO: maybe we should adapt to cgroup v2
 +static int cpurt_controller_init(const char *id, const host_config *host_spec)
 +{
 +    __isula_auto_free char *mnt_root = NULL;
 +    __isula_auto_free char *cgroups_path = NULL;
 +    char *dirpath = NULL;
 +    int64_t cpu_rt_period = 0;
 +    int64_t cpu_rt_runtime = 0;
 +
 +    cgroups_path = merge_container_cgroups_path(id, host_spec);
 +    if (cgroups_path == NULL || strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) {
 +        return 0;
 +    }
 +
 +    if (conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime)) {
 +        return -1;
 +    }
 +
 +    if (cpu_rt_period == 0 && cpu_rt_runtime == 0) {
 +        return 0;
 +    }
 +
 +    mnt_root = sysinfo_cgroup_controller_cpurt_mnt_path();
 +    if (mnt_root == NULL) {
 +        ERROR("Failed to get cpu rt controller mnt root path");
 +        return -1;
 +    }
 +
 +    dirpath = dirname(cgroups_path);
 +
 +    return do_init_cpurt_cgroups_path(dirpath, 0, mnt_root, cpu_rt_period, cpu_rt_runtime);
 +}
 +
 static int container_start_prepare(container_t *cont, const container_start_request *request, int stdinfd,
                                    struct io_write_wrapper *stdout_handler, struct io_write_wrapper *stderr_handler,
                                    char **fifopath, char *fifos[], int *sync_fd, pthread_t *thread_id)
@@ -314,6 +449,19 @@ static int container_start_prepare(container_t *cont, const container_start_requ
         return -1;
     }
 +    // init cgroup path for cpu_rt_runtime and cpu_rt_period
 +    // we should do this in start container, not create container
 +    // because, in scenarios: 
 +    // 1. enable cpu-rt of isulad;
 +    // 2. then run container with --cpu-rt-runtime
 +    // 3. then reboot machine;
 +    // 4. finally, start before container, it will failed...
 +    // cause of no one to set value into cgroup/isulad/cpu-rt-runtime and cpu-rt-period.
 +    if (cpurt_controller_init(id, cont->hostconfig) != 0) {
 +        isulad_set_error_message("Failed to init controller of cpu-rt for container \"%s\".", id);
 +        return -1;
 +    }
 +
     if (prepare_start_io(cont, request, fifopath, fifos, stdinfd, stdout_handler, stderr_handler, sync_fd, thread_id) !=
         0) {
         return -1;
 diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c
 index 4cc333fd..4abc89c7 100644
 --- a/src/daemon/executor/container_cb/execution_create.c
 +++ b/src/daemon/executor/container_cb/execution_create.c
@@ -19,6 +19,14 @@
 #include <errno.h>
 #include <sys/stat.h>
 #include <malloc.h>
 +#include <limits.h>
 +#include <stdbool.h>
 +#include <stdint.h>
 +#include <stdlib.h>
 +#include <string.h>
 +
 +#include <isula_libutils/log.h>
 +#include <isula_libutils/auto_cleanup.h>
 #include <isula_libutils/container_config.h>
 #include <isula_libutils/container_config_v2.h>
 #include <isula_libutils/defs.h>
@@ -26,14 +34,7 @@
 #include <isula_libutils/isulad_daemon_configs.h>
 #include <isula_libutils/json_common.h>
 #include <isula_libutils/oci_runtime_spec.h>
 -#include <limits.h>
 -#include <stdbool.h>
 -#include <stdint.h>
 -#include <stdlib.h>
 -#include <string.h>
 -#include <libgen.h>
 -#include "isula_libutils/log.h"
 #include "isulad_config.h"
 #include "specs_api.h"
 #include "verify.h"
@@ -58,9 +59,6 @@
 #include "opt_log.h"
 #include "runtime_api.h"
 -static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root,
 -                                      int64_t cpu_rt_period, int64_t cpu_rt_runtime);
 -
 static int create_request_check(const container_create_request *request)
 {
     int ret = 0;
@@ -1323,203 +1321,6 @@ static int save_container_config_before_create(const char *id, const char *runti
     return 0;
 }
 -/* maybe create cpu realtime file */
 -static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const char *path)
 -{
 -    int ret;
 -    int fd = -1;
 -    ssize_t nwrite;
 -    char fpath[PATH_MAX] = { 0 };
 -    char buf[ISULAD_NUMSTRLEN64] = { 0 };
 -
 -    if (value == 0) {
 -        return 0;
 -    }
 -
 -    ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE);
 -    if (ret != 0) {
 -        ERROR("Failed to mkdir: %s", path);
 -        return -1;
 -    }
 -
 -    ret = snprintf(fpath, sizeof(fpath), "%s/%s", path, file);
 -    if (ret < 0 || ret >= sizeof(fpath)) {
 -        ERROR("Failed to print string");
 -        return -1;
 -    }
 -    ret = snprintf(buf, sizeof(buf), "%lld", (long long int)value);
 -    if (ret < 0 || (size_t)ret >= sizeof(buf)) {
 -        ERROR("Failed to print string");
 -        return -1;
 -    }
 -
 -    fd = util_open(fpath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0700);
 -    if (fd < 0) {
 -        ERROR("Failed to open file: %s: %s", fpath, strerror(errno));
 -        isulad_set_error_message("Failed to open file: %s: %s", fpath, strerror(errno));
 -        return -1;
 -    }
 -    nwrite = util_write_nointr(fd, buf, strlen(buf));
 -    if (nwrite < 0) {
 -        ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno));
 -        isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno));
 -        close(fd);
 -        return -1;
 -    }
 -    close(fd);
 -
 -    return 0;
 -}
 -
 -static int recursively_create_cgroup(const char *path, const char *mnt_root, int recursive_depth, int64_t cpu_rt_period,
 -                                     int64_t cpu_rt_runtime)
 -{
 -    int ret = 0;
 -    char *dup = NULL;
 -    char *dirpath = NULL;
 -    char fpath[PATH_MAX] = { 0 };
 -
 -    dup = util_strdup_s(path);
 -    dirpath = dirname(dup);
 -    ret = do_init_cpurt_cgroups_path(dirpath, (recursive_depth + 1), mnt_root, cpu_rt_period, cpu_rt_runtime);
 -    free(dup);
 -    if (ret != 0) {
 -        return ret;
 -    }
 -
 -    int nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt_root, path);
 -    if (nret < 0 || (size_t)nret >= sizeof(fpath)) {
 -        ERROR("Failed to print string");
 -        ret = -1;
 -        goto out;
 -    }
 -
 -    ret = maybe_create_cpu_realtime_file(cpu_rt_period, "cpu.rt_period_us", fpath);
 -    if (ret != 0) {
 -        goto out;
 -    }
 -
 -    ret = maybe_create_cpu_realtime_file(cpu_rt_runtime, "cpu.rt_runtime_us", fpath);
 -    if (ret != 0) {
 -        goto out;
 -    }
 -
 -out:
 -    return ret;
 -}
 -
 -/* init cgroups path */
 -static int do_init_cpurt_cgroups_path(const char *path, int recursive_depth, const char *mnt_root,
 -                                      int64_t cpu_rt_period, int64_t cpu_rt_runtime)
 -{
 -    if ((recursive_depth + 1) > MAX_PATH_DEPTH) {
 -        ERROR("Reach the max cgroup depth:%s", path);
 -        return -1;
 -    }
 -
 -    if (path == NULL || strcmp(path, "/") == 0 || strcmp(path, ".") == 0) {
 -        return 0;
 -    }
 -
 -    // Recursively create cgroup to ensure that the system and all parent cgroups have values set
 -    // for the period and runtime as this limits what the children can be set to.
 -    if (recursively_create_cgroup(path, mnt_root, recursive_depth, cpu_rt_period, cpu_rt_runtime)) {
 -        return -1;
 -    }
 -
 -    return 0;
 -}
 -
 -static char *get_cpurt_controller_mnt_path()
 -{
 -    char *res = NULL;
 -    int nret = 0;
 -    char *mnt = NULL;
 -    char *root = NULL;
 -    char fpath[PATH_MAX] = { 0 };
 -
 -    nret = find_cgroup_mountpoint_and_root("cpu", &mnt, &root);
 -    if (nret != 0 || mnt == NULL || root == NULL) {
 -        ERROR("Can not find cgroup mnt and root path for subsystem 'cpu'");
 -        isulad_set_error_message("Can not find cgroup mnt and root path for subsystem 'cpu'");
 -        goto out;
 -    }
 -
 -    // When iSulad is run inside docker, the root is based of the host cgroup.
 -    // Replace root to "/"
 -    if (strncmp(root, "/docker/", strlen("/docker/")) == 0) {
 -        root[1] = '\0';
 -    }
 -
 -    nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root);
 -    if (nret < 0 || (size_t)nret >= sizeof(fpath)) {
 -        ERROR("Failed to print string");
 -        goto out;
 -    }
 -
 -    res = util_strdup_s(fpath);
 -
 -out:
 -    free(mnt);
 -    free(root);
 -    return res;
 -}
 -
 -static int cpurt_controller_init(const char *cgroups_path)
 -{
 -    int ret = 0;
 -    char *dup = NULL;
 -    char *dirpath = NULL;
 -    int64_t cpu_rt_period = 0;
 -    int64_t cpu_rt_runtime = 0;
 -    sysinfo_t *sysinfo = NULL;
 -    char *mnt_root = NULL;
 -
 -    if (cgroups_path == NULL || strcmp(cgroups_path, "/") == 0 || strcmp(cgroups_path, ".") == 0) {
 -        return 0;
 -    }
 -
 -    if (conf_get_cgroup_cpu_rt(&cpu_rt_period, &cpu_rt_runtime)) {
 -        return -1;
 -    }
 -
 -    if (cpu_rt_period == 0 && cpu_rt_runtime == 0) {
 -        return 0;
 -    }
 -
 -    sysinfo = get_sys_info(true);
 -    if (sysinfo == NULL) {
 -        ERROR("Can not get system info");
 -        ret = -1;
 -        goto out;
 -    }
 -
 -    if (!(sysinfo->cgcpuinfo.cpu_rt_period)) {
 -        ERROR("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel");
 -        isulad_set_error_message("Daemon-scoped cpu-rt-period and cpu-rt-runtime are not supported by kernel");
 -        ret = -1;
 -        goto out;
 -    }
 -
 -    mnt_root = get_cpurt_controller_mnt_path();
 -    if (mnt_root == NULL) {
 -        ERROR("Failed to get cpu rt controller mnt root path");
 -        isulad_set_error_message("Failed to get cpu rt controller mnt root path");
 -        ret = -1;
 -        goto out;
 -    }
 -
 -    dup = util_strdup_s(cgroups_path);
 -    dirpath = dirname(dup);
 -
 -    ret = do_init_cpurt_cgroups_path(dirpath, 0, mnt_root, cpu_rt_period, cpu_rt_runtime);
 -
 -out:
 -    free(mnt_root);
 -    free(dup);
 -    return ret;
 -}
 -
 /*
  * request -> host_spec + container_spec
  * container_spec + image config
@@ -1680,13 +1481,6 @@ int container_create_cb(const container_create_request *request, container_creat
         goto umount_channel;
     }
 -    // init cgroup path for cpu_rt_runtime and cpu_rt_period
 -    if (cpurt_controller_init(oci_spec->linux->cgroups_path) != 0) {
 -        ERROR("Unable to init CPU RT controller %s", oci_spec->linux->cgroups_path);
 -        cc = ISULAD_ERR_EXEC;
 -        goto umount_channel;
 -    }
 -
     if (container_v2_spec_merge_contaner_spec(v2_spec) != 0) {
         ERROR("Failed to merge container settings");
         cc = ISULAD_ERR_EXEC;
 diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h
 index 4c132108..e0a73f55 100644
 --- a/src/daemon/modules/api/specs_api.h
 +++ b/src/daemon/modules/api/specs_api.h
@@ -28,7 +28,7 @@ extern "C" {
 int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_config_v2_common_config *v2_spec,
                     oci_runtime_spec *oci_spec);
 -int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec);
 +char *merge_container_cgroups_path(const char *id, const host_config *host_spec);
 int merge_global_config(oci_runtime_spec *oci_spec);
 oci_runtime_spec *load_oci_config(const char *rootpath, const char *name);
 oci_runtime_spec *default_spec(bool system_container);
 diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c
 index 12d9b96d..857fc3dc 100644
 --- a/src/daemon/modules/spec/specs.c
 +++ b/src/daemon/modules/spec/specs.c
@@ -26,6 +26,7 @@
 #include <isula_libutils/oci_runtime_hooks.h>
 #include <isula_libutils/host_config.h>
 #include <isula_libutils/log.h>
 +#include <isula_libutils/auto_cleanup.h>
 #include <limits.h>
 #include <stdint.h>
@@ -165,36 +166,43 @@ out:
     return ret;
 }
 -static int make_annotations_cgroup_dir(const container_config *container_spec, const host_config *host_spec)
 +static char *do_get_container_cgroup_path(const host_config *host_spec)
 {
 -    int ret = 0;
 -    char cleaned[PATH_MAX] = { 0 };
 -    char *default_cgroup_parent = NULL;
     char *path = NULL;
 -    default_cgroup_parent = conf_get_isulad_cgroup_parent();
     if (host_spec->cgroup_parent != NULL) {
 -        path = host_spec->cgroup_parent;
 -    } else if (default_cgroup_parent != NULL) {
 -        path = default_cgroup_parent;
 +        // first, use user setting
 +        path = util_strdup_s(host_spec->cgroup_parent);
 +    } else {
 +        // second, if user donot set, use setting from daemon config
 +        path = conf_get_isulad_cgroup_parent();
     }
 +
     if (path == NULL) {
 -        path = "/isulad";
 +        // third, all faild, just use default '/isulad'
 +        path = util_strdup_s("/isulad");
     }
 +
 +    return path;
 +}
 +
 +static int make_annotations_cgroup_dir(const container_config *container_spec, const host_config *host_spec)
 +{
 +    char cleaned[PATH_MAX] = { 0 };
 +    __isula_auto_free char *path = NULL;
 +
 +    path = do_get_container_cgroup_path(host_spec);
     if (util_clean_path(path, cleaned, sizeof(cleaned)) == NULL) {
         ERROR("Failed to clean path: %s", path);
 -        ret = -1;
 -        goto out;
 +        return -1;
     }
 +
     if (append_json_map_string_string(container_spec->annotations, "cgroup.dir", cleaned)) {
         ERROR("Realloc annotations failed");
 -        ret = -1;
 -        goto out;
 +        return -1;
     }
 -out:
 -    free(default_cgroup_parent);
 -    return ret;
 +    return 0;
 }
 static int make_annotations_oom_score_adj(const container_config *container_spec, const host_config *host_spec)
@@ -2058,42 +2066,40 @@ out:
     return ret;
 }
 -int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec)
 +char *merge_container_cgroups_path(const char *id, const host_config *host_spec)
 {
 -    int ret = 0;
 -    char *default_cgroup_parent = NULL;
 -    char *path = NULL;
 +    __isula_auto_free char *path = NULL;
 +    if (id == NULL || host_spec == NULL) {
 +        ERROR("Invalid arguments");
 +        return NULL;
 +    }
 +
 +    path = do_get_container_cgroup_path(host_spec);
 +
 +    return util_path_join(path, id);
 +}
 +
 +static int merge_oci_cgroups_path(const char *id, oci_runtime_spec *oci_spec, const host_config *host_spec)
 +{
     if (id == NULL || oci_spec == NULL || host_spec == NULL) {
         ERROR("Invalid arguments");
 -        ret = -1;
 -        goto out;
 +        return -1;
     }
     if (make_sure_oci_spec_linux(oci_spec) != 0) {
         ERROR("Failed to make oci spec linux");
 -        ret = -1;
 -        goto out;
 +        return -1;
     }
 -    default_cgroup_parent = conf_get_isulad_cgroup_parent();
 -    path = default_cgroup_parent;
 -    if (host_spec->cgroup_parent != NULL) {
 -        path = host_spec->cgroup_parent;
 -    }
 +    free(oci_spec->linux->cgroups_path);
 +    oci_spec->linux->cgroups_path = merge_container_cgroups_path(id, host_spec);
 -    if (path == NULL) {
 -        free(oci_spec->linux->cgroups_path);
 -        oci_spec->linux->cgroups_path = util_path_join("/isulad", id);
 -        return 0;
 +    if (oci_spec->linux->cgroups_path == NULL) {
 +        WARN("OCI spec cgroups path is NULL");
     }
 -    free(oci_spec->linux->cgroups_path);
 -    oci_spec->linux->cgroups_path = util_path_join(path, id);
 -
 -out:
 -    free(default_cgroup_parent);
 -    return ret;
 +    return 0;
 }
 int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_config_v2_common_config *v2_spec,
 diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc
 index c4014e2e..96aa1c63 100644
 --- a/test/specs/specs/specs_ut.cc
 +++ b/test/specs/specs/specs_ut.cc
@@ -232,15 +232,16 @@ char *invoke_conf_get_isulad_cgroup_parent()
     return util_strdup_s("/var/lib/isulad/engines/lcr");
 }
 -TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_1)
 +TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_1)
 {
 -    ASSERT_EQ(merge_oci_cgroups_path(nullptr, nullptr, nullptr), -1);
 +    ASSERT_EQ(merge_container_cgroups_path(nullptr, nullptr), nullptr);
 }
 -TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_2)
 +TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_2)
 {
     oci_runtime_spec *oci_spec = nullptr;
     host_config *host_spec = nullptr;
 +    char *merged_cp = nullptr;
     oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));
     ASSERT_TRUE(oci_spec != nullptr);
@@ -250,20 +251,23 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_2)
     EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null));
 -    ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0);
 +    merged_cp = merge_container_cgroups_path("123", host_spec);
 +    ASSERT_NE(merged_cp, nullptr);
 -    ASSERT_STREQ(oci_spec->linux->cgroups_path, "/isulad/123");
 +    ASSERT_STREQ(merged_cp, "/isulad/123");
     free_oci_runtime_spec(oci_spec);
     free_host_config(host_spec);
 +    free(merged_cp);
     testing::Mock::VerifyAndClearExpectations(&m_isulad_conf);
 }
 -TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_3)
 +TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_3)
 {
     oci_runtime_spec *oci_spec = nullptr;
     host_config *host_spec = nullptr;
 +    char *merged_cp = nullptr;
     oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));
     ASSERT_TRUE(oci_spec != nullptr);
@@ -275,20 +279,23 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_3)
     EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent_null));
 -    ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0);
 +    merged_cp = merge_container_cgroups_path("123", host_spec);
 +    ASSERT_NE(merged_cp, nullptr);
 -    ASSERT_STREQ(oci_spec->linux->cgroups_path, "/test/123");
 +    ASSERT_STREQ(merged_cp, "/test/123");
     free_oci_runtime_spec(oci_spec);
     free_host_config(host_spec);
 +    free(merged_cp);
     testing::Mock::VerifyAndClearExpectations(&m_isulad_conf);
 }
 -TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_4)
 +TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_4)
 {
     oci_runtime_spec *oci_spec = nullptr;
     host_config *host_spec = nullptr;
 +    char *merged_cp = nullptr;
     oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));
     ASSERT_TRUE(oci_spec != nullptr);
@@ -298,20 +305,23 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_4)
     EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent));
 -    ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0);
 +    merged_cp = merge_container_cgroups_path("123", host_spec);
 +    ASSERT_NE(merged_cp, nullptr);
 -    ASSERT_STREQ(oci_spec->linux->cgroups_path, "/var/lib/isulad/engines/lcr/123");
 +    ASSERT_STREQ(merged_cp, "/var/lib/isulad/engines/lcr/123");
     free_oci_runtime_spec(oci_spec);
     free_host_config(host_spec);
 +    free(merged_cp);
     testing::Mock::VerifyAndClearExpectations(&m_isulad_conf);
 }
 -TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_5)
 +TEST_F(SpecsUnitTest, test_merge_container_cgroups_path_5)
 {
     oci_runtime_spec *oci_spec = nullptr;
     host_config *host_spec = nullptr;
 +    char *merged_cp = nullptr;
     oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));
     ASSERT_TRUE(oci_spec != nullptr);
@@ -323,12 +333,14 @@ TEST_F(SpecsUnitTest, test_merge_oci_cgroups_path_5)
     EXPECT_CALL(m_isulad_conf, GetCgroupParent()).WillRepeatedly(Invoke(invoke_conf_get_isulad_cgroup_parent));
 -    ASSERT_EQ(merge_oci_cgroups_path("123", oci_spec, host_spec), 0);
 +    merged_cp = merge_container_cgroups_path("123", host_spec);
 +    ASSERT_NE(merged_cp, nullptr);
 -    ASSERT_STREQ(oci_spec->linux->cgroups_path, "/test/123");
 +    ASSERT_STREQ(merged_cp, "/test/123");
     free_oci_runtime_spec(oci_spec);
     free_host_config(host_spec);
 +    free(merged_cp);
     testing::Mock::VerifyAndClearExpectations(&m_isulad_conf);
 }
 -- 
 2.25.1
--- a/0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch
+++ b/0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch
@ -1,465 +0,0 @@
 From af8fb9fcf604775f527b58e1b02f220dffd8ff35 Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Thu, 16 Feb 2023 15:26:10 +0800
 Subject: [PATCH 13/53] fix selinux_label_ut timeout and add timeout for all ut
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 test/cgroup/cpu/CMakeLists.txt                                 | 1 +
 test/cmd/isula/extend/pause/CMakeLists.txt                     | 1 +
 test/cmd/isula/extend/resume/CMakeLists.txt                    | 2 +-
 test/cmd/isula/information/info/CMakeLists.txt                 | 1 +
 test/cmd/isula/information/ps/CMakeLists.txt                   | 1 +
 test/cmd/isula/utils/client_show_format/CMakeLists.txt         | 1 +
 test/cmd/isula/utils/template_string_parse/CMakeLists.txt      | 1 +
 test/cmd/isulad-shim/CMakeLists.txt                            | 1 +
 test/cutils/mainloop/CMakeLists.txt                            | 1 +
 test/cutils/map/CMakeLists.txt                                 | 1 +
 test/cutils/path/CMakeLists.txt                                | 1 +
 test/cutils/util_atomic/CMakeLists.txt                         | 1 +
 test/cutils/utils_aes/CMakeLists.txt                           | 1 +
 test/cutils/utils_array/CMakeLists.txt                         | 1 +
 test/cutils/utils_base64/CMakeLists.txt                        | 1 +
 test/cutils/utils_convert/CMakeLists.txt                       | 1 +
 test/cutils/utils_error/CMakeLists.txt                         | 1 +
 test/cutils/utils_file/CMakeLists.txt                          | 1 +
 test/cutils/utils_filters/CMakeLists.txt                       | 1 +
 test/cutils/utils_fs/CMakeLists.txt                            | 1 +
 test/cutils/utils_mount_spec/CMakeLists.txt                    | 1 +
 test/cutils/utils_namespace/CMakeLists.txt                     | 1 +
 test/cutils/utils_network/CMakeLists.txt                       | 1 +
 test/cutils/utils_pwgr/CMakeLists.txt                          | 1 +
 test/cutils/utils_regex/CMakeLists.txt                         | 1 +
 test/cutils/utils_string/CMakeLists.txt                        | 1 +
 test/cutils/utils_timestamp/CMakeLists.txt                     | 1 +
 test/cutils/utils_utils/CMakeLists.txt                         | 1 +
 test/cutils/utils_verify/CMakeLists.txt                        | 1 +
 test/image/oci/oci_config_merge/CMakeLists.txt                 | 1 +
 test/image/oci/registry/CMakeLists.txt                         | 1 +
 test/image/oci/storage/images/CMakeLists.txt                   | 1 +
 test/image/oci/storage/layers/CMakeLists.txt                   | 2 ++
 test/image/oci/storage/rootfs/CMakeLists.txt                   | 1 +
 test/network/CMakeLists.txt                                    | 1 +
 test/runtime/isula/CMakeLists.txt                              | 1 +
 test/runtime/lcr/CMakeLists.txt                                | 1 +
 .../services/execution/execute/execution_extend/CMakeLists.txt | 1 +
 test/services/execution/spec/CMakeLists.txt                    | 2 ++
 test/services/execution/spec/selinux_label_ut.cc               | 2 ++
 test/specs/specs/CMakeLists.txt                                | 1 +
 test/specs/specs_extend/CMakeLists.txt                         | 1 +
 test/volume/CMakeLists.txt                                     | 3 ++-
 43 files changed, 47 insertions(+), 2 deletions(-)
 diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt
 index 159b0d85..6a8af719 100644
 --- a/test/cgroup/cpu/CMakeLists.txt
 +++ b/test/cgroup/cpu/CMakeLists.txt
@@ -26,3 +26,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cmd/isula/extend/pause/CMakeLists.txt b/test/cmd/isula/extend/pause/CMakeLists.txt
 index af48c23f..6eab5172 100644
 --- a/test/cmd/isula/extend/pause/CMakeLists.txt
 +++ b/test/cmd/isula/extend/pause/CMakeLists.txt
@@ -47,3 +47,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cmd/isula/extend/resume/CMakeLists.txt b/test/cmd/isula/extend/resume/CMakeLists.txt
 index 729c4d3b..0812a66e 100644
 --- a/test/cmd/isula/extend/resume/CMakeLists.txt
 +++ b/test/cmd/isula/extend/resume/CMakeLists.txt
@@ -48,4 +48,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 -
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cmd/isula/information/info/CMakeLists.txt b/test/cmd/isula/information/info/CMakeLists.txt
 index 2f134986..d8d8ddb7 100644
 --- a/test/cmd/isula/information/info/CMakeLists.txt
 +++ b/test/cmd/isula/information/info/CMakeLists.txt
@@ -49,3 +49,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cmd/isula/information/ps/CMakeLists.txt b/test/cmd/isula/information/ps/CMakeLists.txt
 index 81d4202c..436e611b 100644
 --- a/test/cmd/isula/information/ps/CMakeLists.txt
 +++ b/test/cmd/isula/information/ps/CMakeLists.txt
@@ -51,3 +51,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cmd/isula/utils/client_show_format/CMakeLists.txt b/test/cmd/isula/utils/client_show_format/CMakeLists.txt
 index 60d538ad..e19332cf 100644
 --- a/test/cmd/isula/utils/client_show_format/CMakeLists.txt
 +++ b/test/cmd/isula/utils/client_show_format/CMakeLists.txt
@@ -19,3 +19,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cmd/isula/utils/template_string_parse/CMakeLists.txt b/test/cmd/isula/utils/template_string_parse/CMakeLists.txt
 index 32256b6b..25efd25c 100644
 --- a/test/cmd/isula/utils/template_string_parse/CMakeLists.txt
 +++ b/test/cmd/isula/utils/template_string_parse/CMakeLists.txt
@@ -19,3 +19,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt
 index 1c2de232..dc293f6d 100644
 --- a/test/cmd/isulad-shim/CMakeLists.txt
 +++ b/test/cmd/isulad-shim/CMakeLists.txt
@@ -30,3 +30,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/mainloop/CMakeLists.txt b/test/cutils/mainloop/CMakeLists.txt
 index 78e3f18d..61ea7ffc 100644
 --- a/test/cutils/mainloop/CMakeLists.txt
 +++ b/test/cutils/mainloop/CMakeLists.txt
@@ -25,3 +25,4 @@ target_link_libraries(${EXE}
     libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/map/CMakeLists.txt b/test/cutils/map/CMakeLists.txt
 index 4059559f..bd21ee3f 100644
 --- a/test/cutils/map/CMakeLists.txt
 +++ b/test/cutils/map/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/path/CMakeLists.txt b/test/cutils/path/CMakeLists.txt
 index 745258a1..aa0c6c6e 100644
 --- a/test/cutils/path/CMakeLists.txt
 +++ b/test/cutils/path/CMakeLists.txt
@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC
 set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,getcwd -Wl,--wrap,readlink")
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/util_atomic/CMakeLists.txt b/test/cutils/util_atomic/CMakeLists.txt
 index 071b2a04..21b05c46 100644
 --- a/test/cutils/util_atomic/CMakeLists.txt
 +++ b/test/cutils/util_atomic/CMakeLists.txt
@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_aes/CMakeLists.txt b/test/cutils/utils_aes/CMakeLists.txt
 index f7535bb3..c4d175c8 100644
 --- a/test/cutils/utils_aes/CMakeLists.txt
 +++ b/test/cutils/utils_aes/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_array/CMakeLists.txt b/test/cutils/utils_array/CMakeLists.txt
 index 71733e31..afe564bf 100644
 --- a/test/cutils/utils_array/CMakeLists.txt
 +++ b/test/cutils/utils_array/CMakeLists.txt
@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC
 set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,calloc")
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_base64/CMakeLists.txt b/test/cutils/utils_base64/CMakeLists.txt
 index d5b99361..35e6ba04 100644
 --- a/test/cutils/utils_base64/CMakeLists.txt
 +++ b/test/cutils/utils_base64/CMakeLists.txt
@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_convert/CMakeLists.txt b/test/cutils/utils_convert/CMakeLists.txt
 index 30068208..fcf70a7c 100644
 --- a/test/cutils/utils_convert/CMakeLists.txt
 +++ b/test/cutils/utils_convert/CMakeLists.txt
@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_error/CMakeLists.txt b/test/cutils/utils_error/CMakeLists.txt
 index 28016605..9607aeeb 100644
 --- a/test/cutils/utils_error/CMakeLists.txt
 +++ b/test/cutils/utils_error/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_file/CMakeLists.txt b/test/cutils/utils_file/CMakeLists.txt
 index 20317e15..01b2ff47 100644
 --- a/test/cutils/utils_file/CMakeLists.txt
 +++ b/test/cutils/utils_file/CMakeLists.txt
@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_filters/CMakeLists.txt b/test/cutils/utils_filters/CMakeLists.txt
 index 31d3ac25..561ff4ef 100644
 --- a/test/cutils/utils_filters/CMakeLists.txt
 +++ b/test/cutils/utils_filters/CMakeLists.txt
@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_fs/CMakeLists.txt b/test/cutils/utils_fs/CMakeLists.txt
 index 7ff3176a..e909aee6 100644
 --- a/test/cutils/utils_fs/CMakeLists.txt
 +++ b/test/cutils/utils_fs/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_mount_spec/CMakeLists.txt b/test/cutils/utils_mount_spec/CMakeLists.txt
 index 24fb5add..099805d2 100644
 --- a/test/cutils/utils_mount_spec/CMakeLists.txt
 +++ b/test/cutils/utils_mount_spec/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_namespace/CMakeLists.txt b/test/cutils/utils_namespace/CMakeLists.txt
 index 8add4a71..fb535e71 100644
 --- a/test/cutils/utils_namespace/CMakeLists.txt
 +++ b/test/cutils/utils_namespace/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_network/CMakeLists.txt b/test/cutils/utils_network/CMakeLists.txt
 index cf0fc481..22e421ab 100644
 --- a/test/cutils/utils_network/CMakeLists.txt
 +++ b/test/cutils/utils_network/CMakeLists.txt
@@ -20,3 +20,4 @@ target_link_libraries(${EXE}
     libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_pwgr/CMakeLists.txt b/test/cutils/utils_pwgr/CMakeLists.txt
 index 5938991e..34acb92a 100644
 --- a/test/cutils/utils_pwgr/CMakeLists.txt
 +++ b/test/cutils/utils_pwgr/CMakeLists.txt
@@ -15,3 +15,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_regex/CMakeLists.txt b/test/cutils/utils_regex/CMakeLists.txt
 index 3f6410b2..45ba2604 100644
 --- a/test/cutils/utils_regex/CMakeLists.txt
 +++ b/test/cutils/utils_regex/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_string/CMakeLists.txt b/test/cutils/utils_string/CMakeLists.txt
 index 1343f4e6..b9968c43 100644
 --- a/test/cutils/utils_string/CMakeLists.txt
 +++ b/test/cutils/utils_string/CMakeLists.txt
@@ -16,3 +16,4 @@ target_include_directories(${EXE} PUBLIC
 set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_strdup_s -Wl,--wrap,calloc -Wl,--wrap,strcat_s")
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_timestamp/CMakeLists.txt b/test/cutils/utils_timestamp/CMakeLists.txt
 index 38aec640..6da3fcc9 100644
 --- a/test/cutils/utils_timestamp/CMakeLists.txt
 +++ b/test/cutils/utils_timestamp/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_utils/CMakeLists.txt b/test/cutils/utils_utils/CMakeLists.txt
 index 7b3bd546..99a83e7a 100644
 --- a/test/cutils/utils_utils/CMakeLists.txt
 +++ b/test/cutils/utils_utils/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_verify/CMakeLists.txt b/test/cutils/utils_verify/CMakeLists.txt
 index abf9596f..dd9ef78b 100644
 --- a/test/cutils/utils_verify/CMakeLists.txt
 +++ b/test/cutils/utils_verify/CMakeLists.txt
@@ -14,3 +14,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt
 index 88047fde..42cd2e78 100644
 --- a/test/image/oci/oci_config_merge/CMakeLists.txt
 +++ b/test/image/oci/oci_config_merge/CMakeLists.txt
@@ -73,3 +73,4 @@ target_include_directories(${EXE} PUBLIC
 set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_common_calloc_s -Wl,--wrap,util_smart_calloc_s -Wl,--wrap,merge_env")
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt
 index 727a615a..13ed95b2 100644
 --- a/test/image/oci/registry/CMakeLists.txt
 +++ b/test/image/oci/registry/CMakeLists.txt
@@ -63,3 +63,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz libhttpclient)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/image/oci/storage/images/CMakeLists.txt b/test/image/oci/storage/images/CMakeLists.txt
 index b00c5a0e..3e6b69a4 100644
 --- a/test/image/oci/storage/images/CMakeLists.txt
 +++ b/test/image/oci/storage/images/CMakeLists.txt
@@ -45,3 +45,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/image/oci/storage/layers/CMakeLists.txt b/test/image/oci/storage/layers/CMakeLists.txt
 index 3fe8ab7c..952e9483 100644
 --- a/test/image/oci/storage/layers/CMakeLists.txt
 +++ b/test/image/oci/storage/layers/CMakeLists.txt
@@ -66,6 +66,7 @@ target_link_libraries(${DRIVER_EXE}
     -lwebsockets -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -ldevmapper -lz)
 add_test(NAME ${DRIVER_EXE} COMMAND ${DRIVER_EXE}  --gtest_output=xml:${DRIVER_EXE}-Results.xml)
 +set_tests_properties(${DRIVER_EXE} PROPERTIES TIMEOUT 120)
 # storage_layers_ut
 SET(LAYER_EXE storage_layers_ut)
@@ -141,3 +142,4 @@ target_link_libraries(${LAYER_EXE}
     -lwebsockets -lcrypto -lyajl -larchive ${SELINUX_LIBRARY} -ldevmapper -lz)
 add_test(NAME ${LAYER_EXE} COMMAND ${LAYER_EXE} --gtest_output=xml:${LAYER_EXE}-Results.xml)
 +set_tests_properties(${LAYER_EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt
 index 5bf568f9..4d7d3533 100644
 --- a/test/image/oci/storage/rootfs/CMakeLists.txt
 +++ b/test/image/oci/storage/rootfs/CMakeLists.txt
@@ -45,3 +45,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/network/CMakeLists.txt b/test/network/CMakeLists.txt
 index e354bebc..be31fd0e 100644
 --- a/test/network/CMakeLists.txt
 +++ b/test/network/CMakeLists.txt
@@ -86,3 +86,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt
 index 38a454b0..f5821953 100644
 --- a/test/runtime/isula/CMakeLists.txt
 +++ b/test/runtime/isula/CMakeLists.txt
@@ -58,3 +58,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lpthread -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt
 index 6f8f784c..979cbe5a 100644
 --- a/test/runtime/lcr/CMakeLists.txt
 +++ b/test/runtime/lcr/CMakeLists.txt
@@ -58,3 +58,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/services/execution/execute/execution_extend/CMakeLists.txt b/test/services/execution/execute/execution_extend/CMakeLists.txt
 index aaff39ef..8588fac3 100644
 --- a/test/services/execution/execute/execution_extend/CMakeLists.txt
 +++ b/test/services/execution/execute/execution_extend/CMakeLists.txt
@@ -76,3 +76,4 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/services/execution/spec/CMakeLists.txt b/test/services/execution/spec/CMakeLists.txt
 index e1aa680e..d4a9d9b0 100644
 --- a/test/services/execution/spec/CMakeLists.txt
 +++ b/test/services/execution/spec/CMakeLists.txt
@@ -74,4 +74,6 @@ target_include_directories(${MOCK_EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SELINUX_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
 target_link_libraries(${MOCK_EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${SELINUX_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 add_test(NAME ${MOCK_EXE} COMMAND ${MOCK_EXE} --gtest_output=xml:${MOCK_EXE}-Results.xml)
 +set_tests_properties(${MOCK_EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/services/execution/spec/selinux_label_ut.cc b/test/services/execution/spec/selinux_label_ut.cc
 index c0e9ab1c..f9a73cca 100644
 --- a/test/services/execution/spec/selinux_label_ut.cc
 +++ b/test/services/execution/spec/selinux_label_ut.cc
@@ -179,11 +179,13 @@ protected:
     void SetUp() override
     {
         CreateTestedObjects();
 +        selinux_state_init();
     }
     void TearDown() override
     {
         ClearTestedObjects();
 +        selinux_state_free();
     }
 private:
 diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt
 index b730959c..c4b36c5f 100644
 --- a/test/specs/specs/CMakeLists.txt
 +++ b/test/specs/specs/CMakeLists.txt
@@ -83,3 +83,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt
 index 7d05deb4..06f46a37 100644
 --- a/test/specs/specs_extend/CMakeLists.txt
 +++ b/test/specs/specs_extend/CMakeLists.txt
@@ -79,3 +79,4 @@ target_include_directories(${EXE} PUBLIC
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt
 index 6eea3f76..e2045b19 100644
 --- a/test/volume/CMakeLists.txt
 +++ b/test/volume/CMakeLists.txt
@@ -39,4 +39,5 @@ target_include_directories(${EXE} PUBLIC
     )
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
 -add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 \ No newline at end of file
 +add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 +set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 -- 
 2.25.1
--- a/0014-add-retry-for-read-write.patch
+++ b/0014-add-retry-for-read-write.patch
@ -1,360 +0,0 @@
 From 734fca150e1c5da2814a55e0315bde8e828e6e8a Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Fri, 17 Feb 2023 16:07:53 +0800
 Subject: [PATCH 14/53] add retry for read/write
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 src/cmd/isulad-shim/common.c                             | 6 +++---
 src/cmd/isulad-shim/process.c                            | 2 +-
 src/cmd/isulad/main.c                                    | 4 ++--
 src/daemon/common/selinux_label.c                        | 2 +-
 src/daemon/entry/connect/grpc/grpc_containers_service.cc | 9 ++++++---
 src/daemon/entry/cri/sysctl_tools.c                      | 2 +-
 src/daemon/executor/container_cb/execution.c             | 4 ++--
 .../modules/container/container_gc/containers_gc.c       | 3 ++-
 src/daemon/modules/events_sender/event_sender.c          | 2 +-
 src/daemon/modules/image/oci/storage/storage.c           | 4 +++-
 src/daemon/modules/log/log_gather.c                      | 6 +++---
 src/daemon/modules/plugin/plugin.c                       | 2 +-
 src/daemon/modules/runtime/isula/isula_rt_ops.c          | 4 ++--
 src/daemon/modules/service/io_handler.c                  | 2 +-
 src/daemon/modules/service/service_container.c           | 2 +-
 src/utils/cutils/utils.c                                 | 2 +-
 src/utils/cutils/utils_aes.c                             | 2 +-
 src/utils/cutils/utils_file.c                            | 2 +-
 src/utils/tar/util_archive.c                             | 4 ++--
 src/utils/tar/util_gzip.c                                | 2 +-
 20 files changed, 36 insertions(+), 30 deletions(-)
 diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c
 index bb8464bb..0c345187 100644
 --- a/src/cmd/isulad-shim/common.c
 +++ b/src/cmd/isulad-shim/common.c
@@ -196,7 +196,7 @@ int generate_random_str(char *id, size_t len)
     }
     for (i = 0; i < len; i++) {
         int nret;
 -        if (read(fd, &num, sizeof(int)) < 0) {
 +        if (read_nointr(fd, &num, sizeof(int)) < 0) {
             close(fd);
             return SHIM_ERR;
         }
@@ -232,8 +232,8 @@ void write_message(int fd, const char *level, const char *fmt, ...)
     va_end(arg_list);
     snprintf(msg, MAX_MESSAGE_LEN - 1, "{\"level\": \"%s\", \"msg\": \"%s\"}\n", level, buf);
 -    nwrite = write(fd, msg, strlen(msg));
 -    if (nwrite != strlen(msg)) {
 +    nwrite = write_nointr_in_total(fd, msg, strlen(msg));
 +    if (nwrite < 0 || (size_t)nwrite != strlen(msg)) {
         return;
     }
 }
 diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
 index 8a0aa142..02ce3c85 100644
 --- a/src/cmd/isulad-shim/process.c
 +++ b/src/cmd/isulad-shim/process.c
@@ -283,7 +283,7 @@ static void *do_io_copy(void *data)
             break;
         }
 -        int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF);
 +        int r_count = util_read_nointr(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF);
         if (r_count == -1) {
             if (errno == EAGAIN || errno == EINTR) {
                 continue;
 diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
 index b17657c5..a75fb189 100644
 --- a/src/cmd/isulad/main.c
 +++ b/src/cmd/isulad/main.c
@@ -482,8 +482,8 @@ int check_and_save_pid(const char *fn)
         goto out;
     }
 -    len = (int)write(fd, pidbuf, strlen(pidbuf));
 -    if (len < 0) {
 +    len = util_write_nointr(fd, pidbuf, strlen(pidbuf));
 +    if (len < 0 || len != strlen(pidbuf)) {
         ERROR("Failed to write pid to file:%s: %s", fn, strerror(errno));
         ret = -1;
     }
 diff --git a/src/daemon/common/selinux_label.c b/src/daemon/common/selinux_label.c
 index 24294780..173f3acb 100644
 --- a/src/daemon/common/selinux_label.c
 +++ b/src/daemon/common/selinux_label.c
@@ -310,7 +310,7 @@ static int get_random_value(unsigned int range, unsigned int *val)
         return -1;
     }
 -    if (read(fd, &num, sizeof(int)) < 0) {
 +    if (util_read_nointr(fd, &num, sizeof(int)) < 0) {
         ERROR("Failed to read urandom value\n");
         ret = -1;
         goto out;
 diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.cc b/src/daemon/entry/connect/grpc/grpc_containers_service.cc
 index c0210ed9..eb79223b 100644
 --- a/src/daemon/entry/connect/grpc/grpc_containers_service.cc
 +++ b/src/daemon/entry/connect/grpc/grpc_containers_service.cc
@@ -292,7 +292,8 @@ Status ContainerServiceImpl::RemoteStart(ServerContext *context,
                 break;
             }
             const std::string &command = request.stdin();
 -            if (write(read_pipe_fd[1], (void *)(command.c_str()), command.length()) < 0) {
 +            int nret = util_write_nointr_in_total(read_pipe_fd[1], command.c_str(), command.length());
 +            if (nret < 0 || (size_t)nret != command.length()) {
                 ERROR("sub write over!");
                 break;
             }
@@ -407,7 +408,8 @@ public:
             }
             for (int i = 0; i < request.cmd_size(); i++) {
                 std::string command = request.cmd(i);
 -                if (write(m_read_pipe_fd, (void *)(command.c_str()), command.length()) < 0) {
 +                int nret = util_write_nointr_in_total(m_read_pipe_fd, command.c_str(), command.length());
 +                if (nret < 0 || (size_t)nret !=  command.length()) {
                     ERROR("sub write over!");
                     return;
                 }
@@ -629,7 +631,8 @@ Status ContainerServiceImpl::Attach(ServerContext *context, ServerReaderWriter<A
                 break;
             }
             std::string command = request.stdin();
 -            if (write(pipefd[1], (void *)(command.c_str()), command.length()) < 0) {
 +            int nret = util_write_nointr_in_total(pipefd[1], command.c_str(), command.length());
 +            if (nret < 0 || (size_t)nret != command.length()) {
                 ERROR("sub write over!");
                 break;
             }
 diff --git a/src/daemon/entry/cri/sysctl_tools.c b/src/daemon/entry/cri/sysctl_tools.c
 index 257ccf8f..3c558fa1 100644
 --- a/src/daemon/entry/cri/sysctl_tools.c
 +++ b/src/daemon/entry/cri/sysctl_tools.c
@@ -99,7 +99,7 @@ int set_sysctl(const char *sysctl, int new_value, char **err)
         goto free_out;
     }
     rsize = util_write_nointr(fd, buff, strlen(buff));
 -    if (rsize <= 0) {
 +    if (rsize < 0 || (size_t)rsize != strlen(buff)) {
         if (asprintf(err, "Write new value failed: %s", strerror(errno)) < 0) {
             *err = util_strdup_s("Out of memory");
         }
 diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c
 index ed70fc14..198052d3 100644
 --- a/src/daemon/executor/container_cb/execution.c
 +++ b/src/daemon/executor/container_cb/execution.c
@@ -348,7 +348,7 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const
         return -1;
     }
     nwrite = util_write_nointr(fd, buf, strlen(buf));
 -    if (nwrite < 0 || nwrite != strlen(buf)) {
 +    if (nwrite < 0 || (size_t)nwrite != strlen(buf)) {
         ERROR("Failed to write %s to %s: %s", buf, fpath, strerror(errno));
         isulad_set_error_message("Failed to write '%s' to '%s': %s", buf, fpath, strerror(errno));
         return -1;
@@ -451,7 +451,7 @@ static int container_start_prepare(container_t *cont, const container_start_requ
     // init cgroup path for cpu_rt_runtime and cpu_rt_period
     // we should do this in start container, not create container
 -    // because, in scenarios: 
 +    // because, in scenarios:
     // 1. enable cpu-rt of isulad;
     // 2. then run container with --cpu-rt-runtime
     // 3. then reboot machine;
 diff --git a/src/daemon/modules/container/container_gc/containers_gc.c b/src/daemon/modules/container/container_gc/containers_gc.c
 index 8c858a96..9feb6d3c 100644
 --- a/src/daemon/modules/container/container_gc/containers_gc.c
 +++ b/src/daemon/modules/container/container_gc/containers_gc.c
@@ -88,7 +88,8 @@ static int save_gc_config(const char *json_gc_config)
         goto out;
     }
 -    if (write(fd, json_gc_config, strlen(json_gc_config)) == -1) {
 +    nret = util_write_nointr(fd, json_gc_config, strlen(json_gc_config));
 +    if (nret < 0 || (size_t)nret != strlen(json_gc_config)) {
         ERROR("write %s failed: %s", filename, strerror(errno));
         ret = -1;
     }
 diff --git a/src/daemon/modules/events_sender/event_sender.c b/src/daemon/modules/events_sender/event_sender.c
 index 03dcbbf3..a3903f3e 100644
 --- a/src/daemon/modules/events_sender/event_sender.c
 +++ b/src/daemon/modules/events_sender/event_sender.c
@@ -58,7 +58,7 @@ static void isulad_monitor_fifo_send(const struct monitord_msg *msg)
     do {
         ret = util_write_nointr(fd, msg, sizeof(struct monitord_msg));
 -        if (ret != sizeof(struct monitord_msg)) {
 +        if (ret < 0 || (size_t)ret != sizeof(struct monitord_msg)) {
             util_usleep_nointerupt(1000);
         }
     } while (ret != sizeof(struct monitord_msg));
 diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c
 index 829ea8d0..2f4bdf5f 100644
 --- a/src/daemon/modules/image/oci/storage/storage.c
 +++ b/src/daemon/modules/image/oci/storage/storage.c
@@ -1429,6 +1429,7 @@ static int do_add_checked_layer(const char *lid, int fd, map_t *checked_layers)
     bool default_value = true;
     char buf[PATH_MAX] = { 0 };
     int ret = 0;
 +    int nret;
     if (strlen(lid) >= PATH_MAX - 1) {
         ERROR("Invalid layer id: %s", lid);
@@ -1438,7 +1439,8 @@ static int do_add_checked_layer(const char *lid, int fd, map_t *checked_layers)
     (void)memcpy(buf, lid, strlen(lid));
     buf[strlen(lid)] = '\n';
     // save checked layer ids into file
 -    if (util_write_nointr(fd, buf, strlen(lid) + 1) < 0) {
 +    nret = util_write_nointr(fd, buf, strlen(lid) + 1);
 +    if (nret < 0 || (size_t)nret != strlen(lid) + 1) {
         ERROR("Write checked layer data failed: %s", strerror(errno));
         ret = -1;
         goto out;
 diff --git a/src/daemon/modules/log/log_gather.c b/src/daemon/modules/log/log_gather.c
 index 49facaa2..414c9ad1 100644
 --- a/src/daemon/modules/log/log_gather.c
 +++ b/src/daemon/modules/log/log_gather.c
@@ -183,9 +183,9 @@ static int write_into_file(const void *buf, size_t g_log_size)
             return -1;
         }
     }
 -    ret = (int)write(g_log_fd, buf, g_log_size);
 -    if (ret <= 0) {
 -        return ret;
 +    ret = util_write_nointr_in_total(g_log_fd, buf, g_log_size);
 +    if (ret < 0 || (size_t)ret != g_log_size) {
 +        return -1;
     }
     write_size += ret;
 diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c
 index 53afeeaf..1c0af368 100644
 --- a/src/daemon/modules/plugin/plugin.c
 +++ b/src/daemon/modules/plugin/plugin.c
@@ -618,7 +618,7 @@ static int process_plugin_events(int inotify_fd, const char *plugin_dir)
     struct inotify_event *plugin_event = NULL;
     char buffer[8192 + 1] = { 0 };
     int action = 0;
 -    events_length = read(inotify_fd, buffer, 8192);
 +    events_length = util_read_nointr(inotify_fd, buffer, 8192);
     if (events_length <= 0) {
         ERROR("Failed to wait events");
 diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
 index 76e3bcb7..5463bb1b 100644
 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
 +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -1363,8 +1363,8 @@ int rt_isula_exec_resize(const char *id, const char *runtime, const rt_exec_resi
         goto out;
     }
 -    count = write(fd, data, RESIZE_DATA_SIZE);
 -    if (count <= 0) {
 +    count = util_write_nointr(fd, data, strlen(data));
 +    if (count < 0 || (size_t)count != strlen(data)) {
         ERROR("write exec resize data error");
         ret = -1;
         goto out;
 diff --git a/src/daemon/modules/service/io_handler.c b/src/daemon/modules/service/io_handler.c
 index 893733bc..98c763a4 100644
 --- a/src/daemon/modules/service/io_handler.c
 +++ b/src/daemon/modules/service/io_handler.c
@@ -340,7 +340,7 @@ static ssize_t write_to_fd(void *context, const void *data, size_t len)
 {
     ssize_t ret;
     ret = util_write_nointr(*(int *)context, data, len);
 -    if ((ret <= 0) || (ret != (ssize_t)len)) {
 +    if (ret < 0 || (size_t)ret != len) {
         ERROR("Failed to write: %s", strerror(errno));
         return -1;
     }
 diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
 index eeb035a0..cc777411 100644
 --- a/src/daemon/modules/service/service_container.c
 +++ b/src/daemon/modules/service/service_container.c
@@ -345,7 +345,7 @@ static int write_env_content(const char *env_path, const char **env, size_t env_
                 goto out;
             }
             nret = util_write_nointr(fd, env_content, strlen(env_content));
 -            if (nret < 0 || nret != len - 1) {
 +            if (nret < 0 || (size_t)nret != strlen(env_content)) {
                 SYSERROR("Write env file failed");
                 free(env_content);
                 ret = -1;
 diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c
 index de636bcb..f99b28e4 100644
 --- a/src/utils/cutils/utils.c
 +++ b/src/utils/cutils/utils.c
@@ -1251,7 +1251,7 @@ int util_generate_random_str(char *id, size_t len)
     }
     for (i = 0; i < len; i++) {
         int nret;
 -        if (read(fd, &num, sizeof(int)) < 0) {
 +        if (util_read_nointr(fd, &num, sizeof(int)) < 0) {
             ERROR("Failed to read urandom value");
             close(fd);
             return -1;
 diff --git a/src/utils/cutils/utils_aes.c b/src/utils/cutils/utils_aes.c
 index 1e25ecd3..055a9538 100644
 --- a/src/utils/cutils/utils_aes.c
 +++ b/src/utils/cutils/utils_aes.c
@@ -77,7 +77,7 @@ int util_aes_key(const char *key_file, bool create, unsigned char *aeskey)
             goto out;
         }
 -        if (read(fd, aeskey, AES_256_CFB_KEY_LEN) != AES_256_CFB_KEY_LEN) {
 +        if (util_read_nointr(fd, aeskey, AES_256_CFB_KEY_LEN) != AES_256_CFB_KEY_LEN) {
             ERROR("read key file %s failed: %s", key_file, strerror(errno));
             ret = -1;
             goto out;
 diff --git a/src/utils/cutils/utils_file.c b/src/utils/cutils/utils_file.c
 index cdd712a7..34c5b060 100644
 --- a/src/utils/cutils/utils_file.c
 +++ b/src/utils/cutils/utils_file.c
@@ -998,7 +998,7 @@ int util_file2str(const char *filename, char *buf, size_t len)
     if (fd == -1) {
         return -1;
     }
 -    num_read = (int)read(fd, buf, len - 1);
 +    num_read = (int)util_read_nointr(fd, buf, len - 1);
     if (num_read <= 0) {
         num_read = -1;
     } else {
 diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c
 index 2d56d8a7..7ace2924 100644
 --- a/src/utils/tar/util_archive.c
 +++ b/src/utils/tar/util_archive.c
@@ -662,7 +662,7 @@ child_out:
     if (ret != 0) {
         ERROR("Wait archive_untar_handler failed with error:%s", strerror(errno));
         fcntl(pipe_stderr[0], F_SETFL, O_NONBLOCK);
 -        if (read(pipe_stderr[0], errbuf, BUFSIZ) < 0) {
 +        if (util_read_nointr(pipe_stderr[0], errbuf, BUFSIZ) < 0) {
             ERROR("read error message from child failed");
         }
     }
@@ -1057,7 +1057,7 @@ child_out:
     if (ret != 0) {
         ERROR("tar failed");
         fcntl(pipe_for_read[0], F_SETFL, O_NONBLOCK);
 -        if (read(pipe_for_read[0], errbuf, BUFSIZ) < 0) {
 +        if (util_read_nointr(pipe_for_read[0], errbuf, BUFSIZ) < 0) {
             ERROR("read error message from child failed");
         }
     }
 diff --git a/src/utils/tar/util_gzip.c b/src/utils/tar/util_gzip.c
 index 5c34d719..2f4750be 100644
 --- a/src/utils/tar/util_gzip.c
 +++ b/src/utils/tar/util_gzip.c
@@ -212,7 +212,7 @@ int gzip(const char *filename, size_t len)
         return -1;
     }
 -    size_read = read(pipefd[0], buffer, BUFSIZ);
 +    size_read = util_read_nointr(pipefd[0], buffer, BUFSIZ);
     close(pipefd[0]);
     if (size_read) {
 -- 
 2.25.1
--- a/0015-support-pull-image-with-digest.patch
+++ b/0015-support-pull-image-with-digest.patch
@ -1,302 +0,0 @@
 From aaf8dec80eff5390404d7da66dbb229e44c76b12 Mon Sep 17 00:00:00 2001
 From: haozi007 <liuhao27@huawei.com>
 Date: Thu, 16 Feb 2023 18:22:02 +0800
 Subject: [PATCH 15/53] support pull image with digest
 usage: isula pull busybox@sha256:907ca53d7e2947e849b839b1cd258c98fd3916c60f2e6e70c30edbf741ab6754
 Signed-off-by: haozi007 <liuhao27@huawei.com>
 ---
 src/daemon/executor/image_cb/image_cb.c       |  8 ++++
 src/daemon/modules/image/oci/oci_pull.c       | 23 ++++++----
 .../modules/image/oci/registry/registry.c     |  2 +-
 .../oci/storage/image_store/image_store.c     |  7 +++
 src/daemon/modules/image/oci/utils_images.c   | 45 +++++++++++++++----
 src/daemon/modules/image/oci/utils_images.h   |  2 +
 src/utils/cutils/utils_verify.c               | 25 ++++++++---
 src/utils/cutils/utils_verify.h               |  3 ++
 8 files changed, 91 insertions(+), 24 deletions(-)
 diff --git a/src/daemon/executor/image_cb/image_cb.c b/src/daemon/executor/image_cb/image_cb.c
 index 06de7543..124feb21 100644
 --- a/src/daemon/executor/image_cb/image_cb.c
 +++ b/src/daemon/executor/image_cb/image_cb.c
@@ -561,6 +561,14 @@ static int trans_one_image(image_list_images_response *response, size_t image_in
         out_image->name = util_strdup_s(im_image->repo_tags[repo_index]);
     }
 +    if (out_image->name == NULL && im_image->repo_digests != NULL && im_image->repo_digests_len > 0) {
 +        // repo digest must valid, so just get lastest @
 +        char *pod = strrchr(im_image->repo_digests[0], '@');
 +        if (pod != NULL) {
 +            out_image->name = util_sub_string(im_image->repo_digests[0], 0, (size_t)(pod - im_image->repo_digests[0]));
 +        }
 +    }
 +
     out_image->target = util_common_calloc_s(sizeof(image_descriptor));
     if (out_image->target == NULL) {
         ERROR("Out of memory");
 diff --git a/src/daemon/modules/image/oci/oci_pull.c b/src/daemon/modules/image/oci/oci_pull.c
 index 5e774c9e..5b35ca2b 100644
 --- a/src/daemon/modules/image/oci/oci_pull.c
 +++ b/src/daemon/modules/image/oci/oci_pull.c
@@ -117,10 +117,19 @@ static int pull_image(const im_pull_request *request, char **name)
     options->skip_tls_verify = oci_image_data->insecure_skip_verify_enforce;
     insecure_registries = oci_image_data->insecure_registries;
 +    // key of image which save in image-store
 +    options->dest_image_name = oci_normalize_image_name(request->image);
 +
 +    // add default tag if required
 +    with_tag = oci_default_tag(request->image);
 +
     host = oci_get_host(request->image);
     if (host != NULL) {
 -        options->image_name = oci_default_tag(request->image);
 -        options->dest_image_name = oci_normalize_image_name(request->image);
 +        // 1. image_name use for split host/tag/name
 +        // 2. user for tag of log
 +        options->image_name = with_tag;
 +        with_tag = NULL;
 +
         update_option_insecure_registry(options, insecure_registries, host);
         ret = registry_pull(options);
         if (ret != 0) {
@@ -141,13 +150,12 @@ static int pull_image(const im_pull_request *request, char **name)
             }
             host = oci_host_from_mirror(*mirror);
             update_option_insecure_registry(options, insecure_registries, host);
 -            with_tag = oci_default_tag(request->image);
 +            // add current mirror to image name
 +            free(options->image_name);
             options->image_name = oci_add_host(host, with_tag);
 -            free(with_tag);
 -            with_tag = NULL;
             free(host);
             host = NULL;
 -            options->dest_image_name = oci_normalize_image_name(request->image);
 +
             ret = registry_pull(options);
             if (ret != 0) {
                 continue;
@@ -159,10 +167,9 @@ static int pull_image(const im_pull_request *request, char **name)
     *name = util_strdup_s(options->dest_image_name);
 out:
 +    free(with_tag);
     free(host);
 -    host = NULL;
     free_registry_pull_options(options);
 -    options = NULL;
     return ret;
 }
 diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c
 index 143de6e4..62d0c35e 100644
 --- a/src/daemon/modules/image/oci/registry/registry.c
 +++ b/src/daemon/modules/image/oci/registry/registry.c
@@ -1861,7 +1861,7 @@ static int prepare_pull_desc(pull_descriptor *desc, registry_pull_options *optio
     }
     if (!util_valid_image_name(options->dest_image_name)) {
 -        ERROR("Invalid dest image name %s", options->image_name);
 +        ERROR("Invalid dest image name %s", options->dest_image_name);
         isulad_try_set_error_message("Invalid image name");
         return -1;
     }
 diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c
 index 39bda87d..cf1e88ff 100644
 --- a/src/daemon/modules/image/oci/storage/image_store/image_store.c
 +++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c
@@ -1979,6 +1979,7 @@ static int resort_image_names(const char **names, size_t names_len, char **first
                                      MAX_IMAGE_NAME_LENGTH - MAX_IMAGE_DIGEST_LENGTH);
         }
 +        // TODO: maybe should support other digest
         if (prefix != NULL && strcmp(prefix, DIGEST_PREFIX) == 0) {
             if (util_array_append(image_digests, names[i]) != 0) {
                 ERROR("Failed to append image to digest: %s", names[i]);
@@ -2172,6 +2173,7 @@ static int get_image_repo_digests(char ***old_repo_digests, char **image_tags, i
         goto out;
     }
 +    // get repo digest from images which with tag
     if (pack_repo_digest(old_repo_digests, (const char **)image_tags, digest, repo_digests) != 0) {
         ERROR("Failed to pack repo digest");
         ret = -1;
@@ -2194,12 +2196,17 @@ static int pack_image_tags_and_repo_digest(image_t *img, imagetool_image *info)
     char *image_digest = NULL;
     char **repo_digests = NULL;
 +    // get names from image-store names:
 +    // 1. image names with tag;
 +    // 2. image names with digests;
 +    // 3. get first image name, current unused;
     if (resort_image_names((const char **)img->simage->names, img->simage->names_len, &name, &tags, &digests) != 0) {
         ERROR("Failed to resort image names");
         ret = -1;
         goto out;
     }
 +    // update repo digests from tags
     if (get_image_repo_digests(&digests, tags, img, &image_digest, &repo_digests) != 0) {
         ERROR("Failed to get image repo digests");
         ret = -1;
 diff --git a/src/daemon/modules/image/oci/utils_images.c b/src/daemon/modules/image/oci/utils_images.c
 index 9e7bb16f..ad7fe0f4 100644
 --- a/src/daemon/modules/image/oci/utils_images.c
 +++ b/src/daemon/modules/image/oci/utils_images.c
@@ -42,6 +42,26 @@
 // nanos of 2038-01-19T03:14:07, the max valid linux time
 #define MAX_NANOS 2147483647000000000
 +char *oci_image_digest_pos(const char *name)
 +{
 +    char *pos = NULL;
 +
 +    if (name == NULL) {
 +        return NULL;
 +    }
 +
 +    pos = strrchr(name, '@');
 +    if (pos == NULL) {
 +        return NULL;
 +    }
 +
 +    if (util_reg_match(__DIGESTPattern, pos) != 0) {
 +        return NULL;
 +    }
 +
 +    return pos;
 +}
 +
 char *get_last_part(char **parts)
 {
     char *last_part = NULL;
@@ -98,6 +118,7 @@ char *oci_default_tag(const char *name)
     }
     last_part = get_last_part(parts);
 +    // will pass image name with digest and with tag
     if (last_part != NULL && strrchr(last_part, ':') == NULL) {
         add_default_tag = DEFAULT_TAG;
     }
@@ -181,9 +202,9 @@ char *oci_normalize_image_name(const char *name)
     return result;
 }
 -int oci_split_image_name(const char *image_name, char **host, char **name, char **tag)
 +int oci_split_image_name(const char *image_name, char **host, char **name, char **tag_digest)
 {
 -    char *tag_pos = NULL;
 +    char *tag_digest_pos = NULL;
     char *name_pos = NULL;
     char *tmp_image_name = NULL;
@@ -193,18 +214,24 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char
     }
     tmp_image_name = util_strdup_s(image_name);
 -    tag_pos = util_tag_pos(tmp_image_name);
 -    if (tag_pos != NULL) {
 -        *tag_pos = 0;
 -        tag_pos++;
 -        if (tag != NULL) {
 -            *tag = util_strdup_s(tag_pos);
 +
 +    // check digest first
 +    tag_digest_pos = oci_image_digest_pos(tmp_image_name);
 +    if (tag_digest_pos == NULL) {
 +        tag_digest_pos = util_tag_pos(tmp_image_name);
 +    }
 +
 +    if (tag_digest_pos != NULL) {
 +        *tag_digest_pos = '\0';
 +        tag_digest_pos++;
 +        if (tag_digest != NULL) {
 +            *tag_digest = util_strdup_s(tag_digest_pos);
         }
     }
     name_pos = strchr(tmp_image_name, '/');
     if (name_pos != NULL) {
 -        *name_pos = 0;
 +        *name_pos = '\0';
         name_pos++;
         if (name != NULL) {
             *name = util_strdup_s(name_pos);
 diff --git a/src/daemon/modules/image/oci/utils_images.h b/src/daemon/modules/image/oci/utils_images.h
 index daa8c040..97879e41 100644
 --- a/src/daemon/modules/image/oci/utils_images.h
 +++ b/src/daemon/modules/image/oci/utils_images.h
@@ -59,6 +59,8 @@ char *oci_get_isulad_tmpdir(const char *root_dir);
 int makesure_isulad_tmpdir_perm_right(const char *root_dir);
 char *get_hostname_to_strip();
 +char *oci_image_digest_pos(const char *name);
 +
 #ifdef __cplusplus
 }
 #endif
 diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c
 index d39d8da5..5868e890 100644
 --- a/src/utils/cutils/utils_verify.c
 +++ b/src/utils/cutils/utils_verify.c
@@ -359,7 +359,7 @@ cleanup:
 bool util_valid_image_name(const char *name)
 {
     char *copy = NULL;
 -    char *tag_pos = NULL;
 +    char *check_pos = NULL;
     bool bret = false;
     if (name == NULL) {
@@ -372,13 +372,26 @@ bool util_valid_image_name(const char *name)
     }
     copy = util_strdup_s(name);
 -    tag_pos = util_tag_pos(copy);
 -    if (tag_pos != NULL) {
 -        if (util_reg_match(__TagPattern, tag_pos)) {
 +
 +    // 1. first, check digest or not
 +    check_pos = strrchr(copy, '@');
 +    if (check_pos != NULL) {
 +        // image name with digest
 +        if (util_reg_match(__DIGESTPattern, check_pos)) {
             goto cleanup;
         }
 -
 -        *tag_pos = '\0';
 +        *check_pos = '\0';
 +    } else {
 +        // image name without digest
 +        // 2. check tag or not
 +        check_pos = util_tag_pos(copy);
 +        if (check_pos != NULL) {
 +            if (util_reg_match(__TagPattern, check_pos)) {
 +                goto cleanup;
 +            }
 +
 +            *check_pos = '\0';
 +        }
     }
     if (util_reg_match(__NamePattern, copy)) {
 diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h
 index a885250f..ad4466ef 100644
 --- a/src/utils/cutils/utils_verify.h
 +++ b/src/utils/cutils/utils_verify.h
@@ -33,6 +33,9 @@ extern "C" {
     "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])"                             \
     "((\\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(:[0-9]+)?/)?[a-z0-9]" \
     "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?$"
 +
 +#define __DIGESTPattern "@[a-z0-9]+:[a-z0-9]{32,}"
 +
 #define VALID_VOLUME_NAME "[a-zA-Z0-9][a-zA-Z0-9_.-]{1,63}"
 extern const char *g_all_caps[];
 -- 
 2.25.1
--- a/0016-isulad-shim-support-execSync-with-timeout.patch
+++ b/0016-isulad-shim-support-execSync-with-timeout.patch
@ -1,186 +0,0 @@
 From 32dbf764fd5b7f6941c49750b49dbba253bd3234 Mon Sep 17 00:00:00 2001
 From: zhongtao <zhongtao17@huawei.com>
 Date: Mon, 13 Feb 2023 15:36:58 +0800
 Subject: [PATCH 16/53] isulad-shim support execSync with timeout
 Signed-off-by: zhongtao <zhongtao17@huawei.com>
 ---
 .../executor/container_cb/execution_stream.c  |  2 +-
 .../modules/runtime/isula/isula_rt_ops.c      | 37 +++++++++++++++---
 src/utils/cutils/utils.c                      | 39 +++++++++++++++++++
 src/utils/cutils/utils.h                      |  5 +++
 4 files changed, 77 insertions(+), 6 deletions(-)
 diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c
 index fde0335e..1a7353b5 100644
 --- a/src/daemon/executor/container_cb/execution_stream.c
 +++ b/src/daemon/executor/container_cb/execution_stream.c
@@ -161,7 +161,7 @@ static int container_exec_cb(const container_exec_request *request, container_ex
     if (exec_container(cont, request, *response, stdinfd, stdout_handler, stderr_handler) != 0) {
         ret = -1;
 -        goto out;
 +        goto pack_err_response;
     }
     goto out;
 diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
 index 5463bb1b..6f2b4f7d 100644
 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
 +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -205,6 +205,10 @@ static void show_shim_runtime_errlog(const char *workdir)
     char buf1[SHIM_LOG_SIZE] = { 0 };
     char buf2[SHIM_LOG_SIZE] = { 0 };
 +    if (g_isulad_errmsg != NULL) {
 +        return;
 +    }
 +
     get_err_message(buf1, sizeof(buf1), workdir, "shim-log.json");
     get_err_message(buf2, sizeof(buf2), workdir, "log.json");
     ERROR("shim-log: %s", buf1);
@@ -686,8 +690,29 @@ static int status_to_exit_code(int status)
     return exit_code;
 }
 +static int try_wait_pid(pid_t pid)
 +{
 +    if (waitpid(pid, NULL, WNOHANG) == pid) {
 +        return 0;
 +    }
 +
 +    return 1;
 +}
 +
 +static void kill_and_show_err(pid_t pid)
 +{
 +    int nret = 0;
 +    kill(pid, SIGKILL);
 +    // wait atmost 0.5 seconds
 +    DO_RETRY_CALL(5, 100000, nret, try_wait_pid, pid);
 +    if (nret != 0) {
 +        WARN("Fail to wait isulad-shim");
 +    }
 +    isulad_set_error_message("Exec container error;exec timeout");
 +}
 +
 static int shim_create(bool fg, const char *id, const char *workdir, const char *bundle, const char *runtime_cmd,
 -                       int *exit_code)
 +                       int *exit_code, const int64_t timeout)
 {
     pid_t pid = 0;
     int exec_fd[2] = { -1, -1 };
@@ -778,7 +803,7 @@ realexec:
         goto out;
     }
 -    status = util_wait_for_pid_status(pid);
 +    status = util_waitpid_with_timeout(pid, timeout, kill_and_show_err);
     if (status < 0) {
         ERROR("failed wait shim-parent %d exit %s", pid, strerror(errno));
         ret = -1;
@@ -792,7 +817,9 @@ realexec:
 out:
     if (ret != 0) {
         show_shim_runtime_errlog(workdir);
 -        kill(pid, SIGKILL); /* can kill other process? */
 +        if (timeout <= 0) {
 +            kill(pid, SIGKILL); /* can kill other process? */
 +        }
     }
     return ret;
@@ -901,7 +928,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_
     }
     get_runtime_cmd(runtime, &cmd);
 -    ret = shim_create(false, id, workdir, params->bundle, cmd, NULL);
 +    ret = shim_create(false, id, workdir, params->bundle, cmd, NULL, -1);
     if (ret != 0) {
         runtime_call_delete_force(workdir, runtime, id);
         ERROR("%s: failed create shim process", id);
@@ -1173,7 +1200,7 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p
     }
     get_runtime_cmd(runtime, &cmd);
 -    ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code);
 +    ret = shim_create(fg_exec(params), id, workdir, bundle, cmd, exit_code, params->timeout);
     if (ret != 0) {
         ERROR("%s: failed create shim process for exec %s", id, exec_id);
         goto errlog_out;
 diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c
 index f99b28e4..2c4c01e4 100644
 --- a/src/utils/cutils/utils.c
 +++ b/src/utils/cutils/utils.c
@@ -311,6 +311,45 @@ rep:
     return 0;
 }
 +/*
 + * If timeout <= 0, blocking wait pid.
 + * If timeout > 0, non-blocking wait pid with timeout.
 + * When waitpid timeout, calling handle_timeout_callback_t.
 + */
 +int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb)
 +{
 +    int nret = 0;
 +    time_t start_time = time(NULL);
 +    time_t end_time;
 +    double interval;
 +
 +    if (timeout <= 0) {
 +        return util_wait_for_pid_status(pid);
 +    }
 +
 +    for (;;) {
 +        nret = waitpid(pid, NULL, WNOHANG);
 +        if (nret == pid) {
 +            break;
 +        }
 +        if (nret == -1 && errno != EINTR) {
 +            return -1;
 +        }
 +        end_time = time(NULL);
 +        interval = difftime(end_time, start_time);
 +        if (nret == 0 && interval >= timeout) {
 +            INFO("Wait %d timeout", pid);
 +            if (cb != NULL) {
 +                cb(pid);
 +            }
 +            return -1;
 +        }
 +        // sleep some time instead to avoid cpu full running and then retry.
 +        sleep(0.1);
 +    }
 +    return 0;
 +}
 +
 int util_wait_for_pid_status(pid_t pid)
 {
     int st;
 diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h
 index 6261dc05..01107605 100644
 --- a/src/utils/cutils/utils.h
 +++ b/src/utils/cutils/utils.h
@@ -302,6 +302,9 @@ typedef struct _proc_t {
         processor; /* current (or most recent?) CPU */
 } proc_t;
 +// handle waitpid timeout.
 +typedef void(*handle_timeout_callback_t)(pid_t pid);
 +
 struct signame {
     int num;
     const char *name;
@@ -329,6 +332,8 @@ char *util_strdup_s(const char *src);
 int util_wait_for_pid(pid_t pid);
 +int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb);
 +
 void util_contain_errmsg(const char *errmsg, int *exit_code);
 char *util_short_digest(const char *digest);
 -- 
 2.25.1
--- a/0017-Refine-the-commit-info.patch
+++ b/0017-Refine-the-commit-info.patch
@ -1,30 +0,0 @@
 From a1c06194fea99d1011551fd84b1fb1f28b974170 Mon Sep 17 00:00:00 2001
 From: sailorvii <chenw66@chinaunicom.cn>
 Date: Tue, 21 Feb 2023 02:40:50 +0000
 Subject: [PATCH 17/53] Refine the commit info.
 ---
 docs/design/detailed/Network/native_network_design_zh.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 diff --git a/docs/design/detailed/Network/native_network_design_zh.md b/docs/design/detailed/Network/native_network_design_zh.md
 index 0ca0f850..27c10c3e 100644
 --- a/docs/design/detailed/Network/native_network_design_zh.md
 +++ b/docs/design/detailed/Network/native_network_design_zh.md
@@ -299,10 +299,10 @@ cache:::unFinish
 ## 4.2 adaptor模块
 -1. 查看 CRI adapter 模块的设计文档： [CRI_adapter_design](./cni_operator_design_zh.md) 。
 +1. 查看 CRI adapter 模块的设计文档： [CRI_adapter_design](./CRI_adapter_design_zh.md) 。
 2. 查看 native network adapter 模块的设计文档： [native_network_adapter_design](./native_network_adapter_design_zh.md) 。
 ## 4.3 cni-operator模块
 -- 查看 cni operator 模块的设计文档： [cni_operator_design](./cni_operator_design_zh.md) 。
 \ No newline at end of file
 +- 查看 cni operator 模块的设计文档： [cni_operator_design](./cni_operator_design_zh.md) 。
 -- 
 2.25.1
--- a/0018-Refine-typo-of-word-container.patch
+++ b/0018-Refine-typo-of-word-container.patch
@ -1,85 +0,0 @@
 From 53ec87b8c5224b1069bef50d09403c53fb48640f Mon Sep 17 00:00:00 2001
 From: sailorvii <chenw66@chinaunicom.cn>
 Date: Tue, 21 Feb 2023 06:50:21 +0000
 Subject: [PATCH 18/53] Refine typo of word "container".
 ---
 src/daemon/executor/container_cb/execution_create.c | 2 +-
 src/daemon/modules/api/container_api.h              | 2 +-
 src/daemon/modules/container/container_unix.c       | 2 +-
 src/daemon/modules/service/service_network.c        | 6 +++---
 4 files changed, 6 insertions(+), 6 deletions(-)
 diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c
 index 4abc89c7..e8f74f1b 100644
 --- a/src/daemon/executor/container_cb/execution_create.c
 +++ b/src/daemon/executor/container_cb/execution_create.c
@@ -1481,7 +1481,7 @@ int container_create_cb(const container_create_request *request, container_creat
         goto umount_channel;
     }
 -    if (container_v2_spec_merge_contaner_spec(v2_spec) != 0) {
 +    if (container_v2_spec_merge_container_spec(v2_spec) != 0) {
         ERROR("Failed to merge container settings");
         cc = ISULAD_ERR_EXEC;
         goto umount_channel;
 diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h
 index 270d6da6..1511db78 100644
 --- a/src/daemon/modules/api/container_api.h
 +++ b/src/daemon/modules/api/container_api.h
@@ -175,7 +175,7 @@ void container_unlock(container_t *cont);
 char *container_get_env_nolock(const container_t *cont, const char *key);
 -int container_v2_spec_merge_contaner_spec(container_config_v2_common_config *v2_spec);
 +int container_v2_spec_merge_container_spec(container_config_v2_common_config *v2_spec);
 char *container_get_command(const container_t *cont);
 diff --git a/src/daemon/modules/container/container_unix.c b/src/daemon/modules/container/container_unix.c
 index adc11be7..1a252b92 100644
 --- a/src/daemon/modules/container/container_unix.c
 +++ b/src/daemon/modules/container/container_unix.c
@@ -470,7 +470,7 @@ out:
 }
 /* container merge basic v2 spec info */
 -int container_v2_spec_merge_contaner_spec(container_config_v2_common_config *v2_spec)
 +int container_v2_spec_merge_container_spec(container_config_v2_common_config *v2_spec)
 {
     int ret = 0;
     int i = 0;
 diff --git a/src/daemon/modules/service/service_network.c b/src/daemon/modules/service/service_network.c
 index 2e7fa28c..2d5f2f6e 100644
 --- a/src/daemon/modules/service/service_network.c
 +++ b/src/daemon/modules/service/service_network.c
@@ -569,7 +569,7 @@ err_out:
     return NULL;
 }
 -static container_network_settings *dup_contaner_network_settings(const container_network_settings *settings)
 +static container_network_settings *dup_container_network_settings(const container_network_settings *settings)
 {
     char *jstr = NULL;
     container_network_settings *res = NULL;
@@ -1278,7 +1278,7 @@ static int update_container_network_settings(container_t *cont, const cni_anno_p
     bool to_disk = false;
     container_network_settings *backup = NULL;
 -    backup = dup_contaner_network_settings(cont->network_settings);
 +    backup = dup_container_network_settings(cont->network_settings);
     if (backup == NULL) {
         ERROR("Failed to dup container network settings");
         return -1;
@@ -1509,7 +1509,7 @@ static int drop_container_network_settings(container_t *cont)
         return -1;
     }
 -    backup = dup_contaner_network_settings(cont->network_settings);
 +    backup = dup_container_network_settings(cont->network_settings);
     if (backup == NULL) {
         ERROR("Failed to dup container network settings");
         return -1;
 -- 
 2.25.1
--- a/0019-cleancode-for-read-write.patch
+++ b/0019-cleancode-for-read-write.patch
@ -1,54 +0,0 @@
 From f9224d47ddc4193678f7ffe501be144fedff0102 Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Mon, 20 Feb 2023 17:28:33 +0800
 Subject: [PATCH 19/53] cleancode for read/write
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 src/cmd/isulad-shim/process.c                            | 2 +-
 src/cmd/isulad/main.c                                    | 2 +-
 src/daemon/entry/connect/grpc/grpc_containers_service.cc | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
 diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
 index 02ce3c85..8a0aa142 100644
 --- a/src/cmd/isulad-shim/process.c
 +++ b/src/cmd/isulad-shim/process.c
@@ -283,7 +283,7 @@ static void *do_io_copy(void *data)
             break;
         }
 -        int r_count = util_read_nointr(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF);
 +        int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF);
         if (r_count == -1) {
             if (errno == EAGAIN || errno == EINTR) {
                 continue;
 diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
 index a75fb189..0cdbfb53 100644
 --- a/src/cmd/isulad/main.c
 +++ b/src/cmd/isulad/main.c
@@ -483,7 +483,7 @@ int check_and_save_pid(const char *fn)
     }
     len = util_write_nointr(fd, pidbuf, strlen(pidbuf));
 -    if (len < 0 || len != strlen(pidbuf)) {
 +    if (len < 0 || (size_t)len != strlen(pidbuf)) {
         ERROR("Failed to write pid to file:%s: %s", fn, strerror(errno));
         ret = -1;
     }
 diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.cc b/src/daemon/entry/connect/grpc/grpc_containers_service.cc
 index eb79223b..7340c3ed 100644
 --- a/src/daemon/entry/connect/grpc/grpc_containers_service.cc
 +++ b/src/daemon/entry/connect/grpc/grpc_containers_service.cc
@@ -409,7 +409,7 @@ public:
             for (int i = 0; i < request.cmd_size(); i++) {
                 std::string command = request.cmd(i);
                 int nret = util_write_nointr_in_total(m_read_pipe_fd, command.c_str(), command.length());
 -                if (nret < 0 || (size_t)nret !=  command.length()) {
 +                if (nret < 0 || (size_t)nret != command.length()) {
                     ERROR("sub write over!");
                     return;
                 }
 -- 
 2.25.1
--- a/0020-add-crictl-timeout-and-sync-for-CI.patch
+++ b/0020-add-crictl-timeout-and-sync-for-CI.patch
@ -1,127 +0,0 @@
 From 7941e0fcd8d7b8edb303a1661233fd9688c46819 Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Mon, 20 Feb 2023 15:42:40 +0800
 Subject: [PATCH 20/53] add crictl timeout and sync for CI
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 CI/test_cases/container_cases/bigdata_stream.sh      | 7 +++++++
 CI/test_cases/container_cases/bigdata_stream_runc.sh | 4 ++++
 CI/test_cases/helpers.sh                             | 5 ++++-
 3 files changed, 15 insertions(+), 1 deletion(-)
 diff --git a/CI/test_cases/container_cases/bigdata_stream.sh b/CI/test_cases/container_cases/bigdata_stream.sh
 index 7e74d700..3bfc2d50 100755
 --- a/CI/test_cases/container_cases/bigdata_stream.sh
 +++ b/CI/test_cases/container_cases/bigdata_stream.sh
@@ -124,6 +124,7 @@ function test_concurrent_bigdata_stream()
         pids[${#pids[@]}]=$!
     done
     wait ${pids[*]// /|}
 +    sync && sync
     for index in $(seq 1 5); do
         ls -l /home/iocopy_stream_data_500M_$index
@@ -151,6 +152,7 @@ function test_concurrent_bigdata_stream_without_pty()
         pids[${#pids[@]}]=$!
     done
     wait ${pids[*]// /|}
 +    sync && sync
     for index in $(seq 1 5); do
         ls -l /home/iocopy_stream_data_500M_$index
@@ -209,6 +211,7 @@ function test_stream_with_stop_client()
     kill -18 $pid
     wait $pid
 +    sync && sync
     ls -l /home/iocopy_stream_data_500M
     total_size=$(stat -c"%s" /home/iocopy_stream_data_500M)
@@ -254,6 +257,7 @@ function test_stream_with_stop_attach()
     kill -18 $pid
     wait $exec_pid
 +    sync && sync
     ls -l /home/iocopy_stream_data_500M
     total_size=$(stat -c"%s" /home/iocopy_stream_data_500M)
@@ -299,6 +303,7 @@ function test_stream_with_stop_lxc_monitor()
     kill -18 $pid
     wait $exec_pid
 +    sync && sync
     ls -l /home/iocopy_stream_data_500M
     total_size=$(stat -c"%s" /home/iocopy_stream_data_500M)
@@ -347,6 +352,7 @@ function test_stream_with_stop_isulad()
     kill -18 $(cat /var/run/isulad.pid)
     wait $pid
 +    sync && sync
     ls -l /home/iocopy_stream_data_500M
     total_size=$(stat -c"%s" /home/iocopy_stream_data_500M)
@@ -397,6 +403,7 @@ function test_stream_with_runc()
     isula exec -it $RUNCID cat test_500M > /home/iocopy_stream_data_500M
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to cat bigdata" && ((ret++))
 +    sync && sync
     total_size=$(stat -c"%s" /home/iocopy_stream_data_500M)
     [[ $total_size -ne 524288000 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stream iocopy loss data" && ((ret++))
 diff --git a/CI/test_cases/container_cases/bigdata_stream_runc.sh b/CI/test_cases/container_cases/bigdata_stream_runc.sh
 index 6933bef0..3a384cd8 100755
 --- a/CI/test_cases/container_cases/bigdata_stream_runc.sh
 +++ b/CI/test_cases/container_cases/bigdata_stream_runc.sh
@@ -117,6 +117,7 @@ function test_cat_bigdata()
         pids[${#pids[@]}]=$!
     done
     wait ${pids[*]// /|}
 +    sync && sync
     for index in $(seq 1 5); do
         ls -l /home/iocopy_stream_data_500M_$index
@@ -144,6 +145,7 @@ function test_cat_bigdata_without_pty()
         pids[${#pids[@]}]=$!
     done
     wait ${pids[*]// /|}
 +    sync && sync
     for index in $(seq 1 5); do
         ls -l /home/iocopy_stream_data_500M_$index
@@ -173,6 +175,7 @@ function test_stream_with_stop_client()
     kill -18 $pid
     wait $pid
 +    sync && sync
     ls -l /home/iocopy_stream_data_500M
     total_size=$(stat -c"%s" /home/iocopy_stream_data_500M)
@@ -217,6 +220,7 @@ function test_stream_with_stop_isulad()
     kill -18 $(cat /var/run/isulad.pid)
     wait $pid
 +    sync && sync
     ls -l /home/iocopy_stream_data_500M
     total_size=$(stat -c"%s" /home/iocopy_stream_data_500M)
 diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh
 index bba4e7e3..5ea4ff94 100755
 --- a/CI/test_cases/helpers.sh
 +++ b/CI/test_cases/helpers.sh
@@ -69,7 +69,10 @@ function testcontainer() {
 function crictl() {
     CRICTL=$(which crictl)
 -    "$CRICTL" -i unix:///var/run/isulad.sock -r unix:///var/run/isulad.sock "$@"
 +    # Default timeout is 2s.
 +    # In some high IO testcase, isulad handle CRI request time maybe more than 2s.
 +    # And the crictl will print error message "context deadline exceeded"
 +    "$CRICTL" -i unix:///var/run/isulad.sock -r unix:///var/run/isulad.sock --timeout 5s "$@"
 }
 function msg_ok()
 -- 
 2.25.1
--- a/0021-unlock-m_podsLock-if-new-failed.patch
+++ b/0021-unlock-m_podsLock-if-new-failed.patch
@ -1,27 +0,0 @@
 From f6243bb672bca8fd2e32752480aa92dc8f97adc9 Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Wed, 22 Feb 2023 10:43:52 +0800
 Subject: [PATCH 21/53] unlock m_podsLock if new failed
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 src/daemon/entry/cri/network_plugin.cc | 3 +++
 1 file changed, 3 insertions(+)
 diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc
 index f6a155c3..4a119d6b 100644
 --- a/src/daemon/entry/cri/network_plugin.cc
 +++ b/src/daemon/entry/cri/network_plugin.cc
@@ -345,6 +345,9 @@ void PluginManager::Lock(const std::string &fullPodName, Errors &error)
         auto tmpLock = std::unique_ptr<PodLock>(new (std::nothrow) PodLock());
         if (tmpLock == nullptr) {
             error.SetError("Out of memory");
 +            if (pthread_mutex_unlock(&m_podsLock) != 0) {
 +                error.SetError("plugin manager unlock failed");
 +            }
             return;
         }
         lock = tmpLock.get();
 -- 
 2.25.1
--- a/0022-Update-CRI.patch
+++ b/0022-Update-CRI.patch
@ -1,73 +0,0 @@
 From 65c3b3c803128f92113f9f21bf41da1ad56017c8 Mon Sep 17 00:00:00 2001
 From: shijiaqi1 <jiaqi@isrc.iscas.ac.cn>
 Date: Wed, 8 Feb 2023 13:31:36 +0800
 Subject: [PATCH 22/53] Update-CRI
 ---
 .../cri/cri_container_manager_service.cc      | 19 +++++++++++++++++++
 src/daemon/entry/cri/cri_helpers.cc           | 19 +++++++++++++++++++
 2 files changed, 38 insertions(+)
 diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc
 index 710556a3..b02367c8 100644
 --- a/src/daemon/entry/cri/cri_container_manager_service.cc
 +++ b/src/daemon/entry/cri/cri_container_manager_service.cc
@@ -1179,6 +1179,25 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai
     if (!resources.cpuset_mems().empty()) {
         hostconfig->cpuset_mems = util_strdup_s(resources.cpuset_mems().c_str());
     }
 +    if (resources.hugepage_limits_size() != 0) {
 +        hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s(
 +                sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size());
 +        if (hostconfig->hugetlbs == nullptr) {
 +            error.SetError("Out of memory");
 +            return;
 +        }
 +	for (int i = 0; i < resources.hugepage_limits_size(); i++) {
 +            hostconfig->hugetlbs[i] =
 +                    (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element));
 +            if (hostconfig->hugetlbs[i] == nullptr) {
 +                error.SetError("Out of memory");
 +                goto cleanup;
 +            }
 +            hostconfig->hugetlbs[i]->page_size = util_strdup_s(resources.hugepage_limits(i).page_size().c_str());
 +            hostconfig->hugetlbs[i]->limit = resources.hugepage_limits(i).limit();
 +            hostconfig->hugetlbs_len++;
 +        }
 +    }
     request->host_config = host_config_generate_json(hostconfig, &ctx, &perror);
     if (request->host_config == nullptr) {
 diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc
 index 2f6dcf78..6d59ec11 100644
 --- a/src/daemon/entry/cri/cri_helpers.cc
 +++ b/src/daemon/entry/cri/cri_helpers.cc
@@ -461,6 +461,25 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc,
             }
         }
         hc->unified = unified;
 +        if (rOpts.hugepage_limits_size() != 0) {
 +            hc->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s(sizeof(host_config_hugetlbs_element *),
 +                                                                                rOpts.hugepage_limits_size());
 +            if (hc->hugetlbs == nullptr) {
 +                error.SetError("Out of memory");
 +                return;
 +            }
 +            for (int i = 0; i < rOpts.hugepage_limits_size(); i++) {
 +                hc->hugetlbs[i] =
 +                        (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element));
 +                if (hc->hugetlbs[i] == nullptr) {
 +                    error.SetError("Out of memory");
 +                    return;
 +                }
 +                hc->hugetlbs[i]->page_size = util_strdup_s(rOpts.hugepage_limits(i).page_size().c_str());
 +                hc->hugetlbs[i]->limit = rOpts.hugepage_limits(i).limit();
 +                hc->hugetlbs_len++;
 +            }
 +        }
     }
     createConfig->open_stdin = config.stdin();
     createConfig->tty = config.tty();
 -- 
 2.25.1
--- a/0023-add-cgroup-cpu-ut.patch
+++ b/0023-add-cgroup-cpu-ut.patch
@ -1,70 +0,0 @@
 From dd238f80fd754b135c7b0e6a6535d45cf57c0b82 Mon Sep 17 00:00:00 2001
 From: songbuhuang <544824346@qq.com>
 Date: Wed, 22 Feb 2023 16:46:45 +0800
 Subject: [PATCH 23/53] add cgroup cpu ut
 Signed-off-by: songbuhuang <544824346@qq.com>
 ---
 test/cgroup/cpu/CMakeLists.txt   |  2 ++
 test/cgroup/cpu/cgroup_cpu_ut.cc | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+)
 diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt
 index 6a8af719..b619fb59 100644
 --- a/test/cgroup/cpu/CMakeLists.txt
 +++ b/test/cgroup/cpu/CMakeLists.txt
@@ -22,8 +22,10 @@ target_include_directories(${EXE} PUBLIC
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad
 +    ${CMAKE_CURRENT_SOURCE_DIR}/../../mocks
     )
 +set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,util_common_calloc_s")
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lgrpc++ -lprotobuf -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc
 index 032ad656..7d23d014 100644
 --- a/test/cgroup/cpu/cgroup_cpu_ut.cc
 +++ b/test/cgroup/cpu/cgroup_cpu_ut.cc
@@ -16,10 +16,18 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <gtest/gtest.h>
 +#include <gmock/gmock.h>
 #include "daemon_arguments.h"
 #include "isulad_config.h"
 +#include "mock.h"
 +#include "sysinfo.h"
 #include "utils.h"
 +extern "C" {
 +    DECLARE_WRAPPER(util_common_calloc_s, void *, (size_t size));
 +    DEFINE_WRAPPER(util_common_calloc_s, void *, (size_t size), (size));
 +}
 +
 struct service_arguments *new_args(int64_t cpu_rt_period, int64_t cpu_rt_runtime)
 {
     struct service_arguments *args = (struct service_arguments *)util_common_calloc_s(sizeof(struct service_arguments));
@@ -57,4 +65,17 @@ TEST(CgroupCpuUnitTest, test_conf_get_cgroup_cpu_rt)
     ASSERT_EQ(cpu_rt_runtime, 0);
 }
 +TEST(CgroupCpuUnitTest, test_find_cgroup_mountpoint_and_root)
 +{
 +    char *mnt = NULL;
 +    char *root = NULL;
 +    ASSERT_EQ(find_cgroup_mountpoint_and_root(nullptr, &mnt, &root), -1);
 +}
 +TEST(CgroupCpuUnitTest, test_sysinfo_cgroup_controller_cpurt_mnt_path)
 +{
 +    MOCK_SET(util_common_calloc_s, nullptr);
 +    ASSERT_EQ(get_sys_info(true), nullptr);
 +    ASSERT_EQ(sysinfo_cgroup_controller_cpurt_mnt_path(), nullptr);
 +    MOCK_CLEAR(util_common_calloc_s);
 +}
 -- 
 2.25.1
--- a/0024-remove-temp-variables.patch
+++ b/0024-remove-temp-variables.patch
@ -1,179 +0,0 @@
 From e20767fc709bb945eb8e076a57f07d5b70b5f3ab Mon Sep 17 00:00:00 2001
 From: "ilya.kuksenok" <ilya.kuksenok@huawei.com>
 Date: Wed, 22 Feb 2023 16:17:34 +0300
 Subject: [PATCH 24/53] remove temp variables
 ---
 src/api/services/containers/container.proto    |  5 +++++
 .../connect/grpc/grpc_containers_client.cc     |  5 +++++
 src/client/connect/protocol_type.h             |  5 +++++
 .../connect/rest/rest_containers_client.c      |  5 +++++
 .../connect/grpc/container/stats_service.cc    |  6 ++++++
 .../entry/cri/cri_container_manager_service.cc | 18 ++++++++++++++++--
 .../executor/container_cb/execution_extend.c   |  5 +++++
 src/daemon/modules/api/runtime_api.h           |  5 +++++
 .../modules/runtime/engines/lcr/lcr_engine.c   |  6 ++++++
 9 files changed, 58 insertions(+), 2 deletions(-)
 diff --git a/src/api/services/containers/container.proto b/src/api/services/containers/container.proto
 index 13d1367b..230d18b3 100644
 --- a/src/api/services/containers/container.proto
 +++ b/src/api/services/containers/container.proto
@@ -86,6 +86,11 @@ message Container_info {
 	uint64 cache = 16;
 	uint64 cache_total = 17;
 	uint64 inactive_file_total = 18;
 +	uint64 avaliable_bytes = 19;
 +	uint64 usage_bytes = 20;
 +	uint64 rss_bytes = 21;
 +	uint64 page_faults = 22;
 +	uint64 major_page_faults = 23;
 }
 message Event {
 diff --git a/src/client/connect/grpc/grpc_containers_client.cc b/src/client/connect/grpc/grpc_containers_client.cc
 index 1528b2ee..20766049 100644
 --- a/src/client/connect/grpc/grpc_containers_client.cc
 +++ b/src/client/connect/grpc/grpc_containers_client.cc
@@ -1690,6 +1690,11 @@ public:
                 response->container_stats[i].blkio_read = gresponse->containers(i).blkio_read();
                 response->container_stats[i].blkio_write = gresponse->containers(i).blkio_write();
                 response->container_stats[i].mem_used = gresponse->containers(i).mem_used();
 +                response->container_stats[i].avaliable_bytes = gresponse->containers(i).avaliable_bytes();
 +                response->container_stats[i].usage_bytes = gresponse->containers(i).usage_bytes();
 +                response->container_stats[i].rss_bytes = gresponse->containers(i).rss_bytes();
 +                response->container_stats[i].page_faults = gresponse->containers(i).page_faults();
 +                response->container_stats[i].major_page_faults = gresponse->containers(i).major_page_faults();
                 response->container_stats[i].mem_limit = gresponse->containers(i).mem_limit();
                 response->container_stats[i].kmem_used = gresponse->containers(i).kmem_used();
                 response->container_stats[i].kmem_limit = gresponse->containers(i).kmem_limit();
 diff --git a/src/client/connect/protocol_type.h b/src/client/connect/protocol_type.h
 index 3831c7c0..a2d474a6 100644
 --- a/src/client/connect/protocol_type.h
 +++ b/src/client/connect/protocol_type.h
@@ -165,6 +165,11 @@ struct isula_container_info {
     // Memory usage
     uint64_t mem_used;
     uint64_t mem_limit;
 +    uint64_t avaliable_bytes;
 +    uint64_t usage_bytes;
 +    uint64_t rss_bytes;
 +    uint64_t page_faults;
 +    uint64_t major_page_faults;
     // Kernel Memory usage
     uint64_t kmem_used;
     uint64_t kmem_limit;
 diff --git a/src/client/connect/rest/rest_containers_client.c b/src/client/connect/rest/rest_containers_client.c
 index 0a735a64..9eee9d40 100644
 --- a/src/client/connect/rest/rest_containers_client.c
 +++ b/src/client/connect/rest/rest_containers_client.c
@@ -2195,6 +2195,11 @@ static int unpack_container_info_for_stats_response(const container_stats_respon
         // memory usage
         infos[i].mem_used = con_info->mem_used;
         infos[i].mem_limit = con_info->mem_limit;
 +        infos[i].avaliable_bytes = con_info->avaliable_bytes();
 +        infos[i].usage_bytes = con_info->usage_bytes();
 +        infos[i].rss_bytes = con_info->rss_bytes();
 +        infos[i].page_faults = con_info->page_faults();
 +        infos[i].major_page_faults = con_info->major_page_faults();
         // kernel memory usage
         infos[i].kmem_used = con_info->kmem_used;
         infos[i].kmem_limit = con_info->kmem_limit;
 diff --git a/src/daemon/entry/connect/grpc/container/stats_service.cc b/src/daemon/entry/connect/grpc/container/stats_service.cc
 index 6e116b4c..7315fe75 100644
 --- a/src/daemon/entry/connect/grpc/container/stats_service.cc
 +++ b/src/daemon/entry/connect/grpc/container/stats_service.cc
@@ -87,6 +87,12 @@ void ContainerStatsService::FillResponseTogRPC(void *containerRes, StatsResponse
         stats->set_mem_limit(response->container_stats[i]->mem_limit);
         stats->set_kmem_used(response->container_stats[i]->kmem_used);
         stats->set_kmem_limit(response->container_stats[i]->kmem_limit);
 +        stats->set_avaliable_bytes(response->container_stats[i]->avaliable_bytes);
 +        stats->set_usage_bytes(response->container_stats[i]->usage_bytes);
 +        stats->set_mem_used(response->container_stats[i]->mem_used);
 +        stats->set_rss_bytes(response->container_stats[i]->rss_bytes);
 +        stats->set_page_faults(response->container_stats[i]->page_faults);
 +        stats->set_major_page_faults(response->container_stats[i]->major_page_faults);
         if (response->container_stats[i]->name != nullptr) {
             stats->set_name(response->container_stats[i]->name);
         }
 diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc
 index b02367c8..57284593 100644
 --- a/src/daemon/entry/cri/cri_container_manager_service.cc
 +++ b/src/daemon/entry/cri/cri_container_manager_service.cc
@@ -842,8 +842,22 @@ void ContainerManagerService::ContainerStatsToGRPC(
                 response->container_stats[i]->cpu_use_nanos);
             container->mutable_cpu()->set_timestamp(timestamp);
         }
 -
 -        containerstats->push_back(move(container));
 +        if (response->container_stats[i]->avaliable_bytes != 0u) {
 +            container->mutable_memory()->mutable_available_bytes()->set_value(response->container_stats[i]->avaliable_bytes);
 +        }
 +        if (response->container_stats[i]->usage_bytes != 0u) {
 +            container->mutable_memory()->mutable_usage_bytes()->set_value(response->container_stats[i]->usage_bytes);
 +        }
 +        if (response->container_stats[i]->rss_bytes != 0u) {
 +            container->mutable_memory()->mutable_rss_bytes()->set_value(response->container_stats[i]->rss_bytes);
 +        }
 +        if (response->container_stats[i]->page_faults != 0u) {
 +            container->mutable_memory()->mutable_page_faults()->set_value(response->container_stats[i]->page_faults);
 +        }
 +        if (response->container_stats[i]->major_page_faults != 0u) {
 +            container->mutable_memory()->mutable_major_page_faults()->set_value(response->container_stats[i]->major_page_faults);
 +        }
 +        containerstats->push_back(std::move(container));
     }
 }
 diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c
 index b0da705e..67d0845a 100644
 --- a/src/daemon/executor/container_cb/execution_extend.c
 +++ b/src/daemon/executor/container_cb/execution_extend.c
@@ -243,6 +243,11 @@ static container_info *get_container_stats(const container_t *cont,
     info->blkio_write = einfo->blkio_write;
     info->mem_used = einfo->mem_used;
     info->mem_limit = einfo->mem_limit;
 +    info->avaliable_bytes = einfo->avaliable_bytes;
 +    info->usage_bytes = einfo->usage_bytes;
 +    info->rss_bytes = einfo->rss_bytes;
 +    info->page_faults = einfo->page_faults;
 +    info->major_page_faults = einfo->major_page_faults;
     info->kmem_used = einfo->kmem_used;
     info->kmem_limit = einfo->kmem_limit;
 diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h
 index de4136a3..b245ebf9 100644
 --- a/src/daemon/modules/api/runtime_api.h
 +++ b/src/daemon/modules/api/runtime_api.h
@@ -55,6 +55,11 @@ struct runtime_container_resources_stats_info {
     /* Memory usage */
     uint64_t mem_used;
     uint64_t mem_limit;
 +    uint64_t avaliable_bytes;
 +    uint64_t usage_bytes;
 +    uint64_t rss_bytes;
 +    uint64_t page_faults;
 +    uint64_t major_page_faults;
     /* Kernel Memory usage */
     uint64_t kmem_used;
     uint64_t kmem_limit;
 diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c
 index 2ca12545..2f7f73f8 100644
 --- a/src/daemon/modules/runtime/engines/lcr/lcr_engine.c
 +++ b/src/daemon/modules/runtime/engines/lcr/lcr_engine.c
@@ -162,6 +162,12 @@ static void copy_container_resources_stats(const struct lcr_container_state *lcs
     rs_stats->mem_limit = lcs->mem_limit;
     rs_stats->kmem_used = lcs->kmem_used;
     rs_stats->kmem_limit = lcs->kmem_limit;
 +    rs_stats->avaliable_bytes = lcs->avaliable_bytes;
 +    rs_stats->usage_bytes = lcs->usage_bytes;
 +    rs_stats->mem_used = lcs->mem_used;
 +    rs_stats->rss_bytes = lcs->rss_bytes;
 +    rs_stats->page_faults = lcs->page_faults;
 +    rs_stats->major_page_faults = lcs->major_page_faults;
     rs_stats->cache = lcs->cache;
     rs_stats->cache_total = lcs->cache_total;
     rs_stats->inactive_file_total = lcs->inactive_file_total;
 -- 
 2.25.1
--- a/0025-fix-read-member-error-from-struct.patch
+++ b/0025-fix-read-member-error-from-struct.patch
@ -1,34 +0,0 @@
 From 3e71fe5910a1b0c9dbb899e06614103a2cebf0fc Mon Sep 17 00:00:00 2001
 From: haozi007 <liuhao27@huawei.com>
 Date: Fri, 24 Feb 2023 10:25:46 +0800
 Subject: [PATCH 25/53] fix read member error from struct
 Signed-off-by: haozi007 <liuhao27@huawei.com>
 ---
 src/client/connect/rest/rest_containers_client.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
 diff --git a/src/client/connect/rest/rest_containers_client.c b/src/client/connect/rest/rest_containers_client.c
 index 9eee9d40..0ae9a4f6 100644
 --- a/src/client/connect/rest/rest_containers_client.c
 +++ b/src/client/connect/rest/rest_containers_client.c
@@ -2195,11 +2195,11 @@ static int unpack_container_info_for_stats_response(const container_stats_respon
         // memory usage
         infos[i].mem_used = con_info->mem_used;
         infos[i].mem_limit = con_info->mem_limit;
 -        infos[i].avaliable_bytes = con_info->avaliable_bytes();
 -        infos[i].usage_bytes = con_info->usage_bytes();
 -        infos[i].rss_bytes = con_info->rss_bytes();
 -        infos[i].page_faults = con_info->page_faults();
 -        infos[i].major_page_faults = con_info->major_page_faults();
 +        infos[i].avaliable_bytes = con_info->avaliable_bytes;
 +        infos[i].usage_bytes = con_info->usage_bytes;
 +        infos[i].rss_bytes = con_info->rss_bytes;
 +        infos[i].page_faults = con_info->page_faults;
 +        infos[i].major_page_faults = con_info->major_page_faults;
         // kernel memory usage
         infos[i].kmem_used = con_info->kmem_used;
         infos[i].kmem_limit = con_info->kmem_limit;
 -- 
 2.25.1
--- a/0026-Fix-PR-runc.patch
+++ b/0026-Fix-PR-runc.patch
@ -1,108 +0,0 @@
 From 39d7fd140e8b590a925e5cdf8ace20b0161328c8 Mon Sep 17 00:00:00 2001
 From: sailorvii <chenw66@chinaunicom.cn>
 Date: Mon, 27 Feb 2023 02:33:46 +0000
 Subject: [PATCH 26/53] =?UTF-8?q?Fix=20PR=20runc=E8=BF=90=E8=A1=8C?=
 =?UTF-8?q?=E5=A4=B1=E8=B4=A5.?=
 MIME-Version: 1.0
 Content-Type: text/plain; charset=UTF-8
 Content-Transfer-Encoding: 8bit
 The environment NOTIFY_SOCKET is used when runc start but not runc create.
 As the source code: https://github.com/opencontainers/runc/blob/main/start.go#L35.
 So move the related code to the right location.
 ---
 .../modules/runtime/isula/isula_rt_ops.c      | 27 +++++++++----------
 1 file changed, 13 insertions(+), 14 deletions(-)
 diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
 index 6f2b4f7d..60742d42 100644
 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
 +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -760,12 +760,6 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char
             goto realexec;
         }
 -        // clear NOTIFY_SOCKET from the env to adapt runc create
 -        if (unsetenv("NOTIFY_SOCKET") != 0) {
 -            (void)dprintf(exec_fd[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno));
 -            exit(EXIT_FAILURE);
 -        }
 -
         pid = fork();
         if (pid < 0) {
             (void)dprintf(exec_fd[1], "%s: fork shim-process failed %s", id, strerror(errno));
@@ -945,7 +939,7 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t
     char shim_pid_file_name[PATH_MAX] = { 0 };
     pid_t pid = 0;
     pid_t shim_pid = -1;
 -    int ret = 0;
 +    int ret = -1;
     int splice_ret = 0;
     proc_t *proc = NULL;
     proc_t *p_proc = NULL;
@@ -967,28 +961,24 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t
     pid = get_container_process_pid(workdir);
     if (pid < 0) {
 -        ret = -1;
         ERROR("%s: failed wait init pid", id);
         goto out;
     }
     file_read_int(shim_pid_file_name, &shim_pid);
     if (shim_pid < 0) {
 -        ret = -1;
         ERROR("%s: failed to read isulad shim pid", id);
         goto out;
     }
     proc = util_get_process_proc_info(pid);
     if (proc == NULL) {
 -        ret = -1;
         ERROR("%s: failed to read pidinfo", id);
         goto out;
     }
     p_proc = util_get_process_proc_info(shim_pid);
     if (p_proc == NULL) {
 -        ret = -1;
         ERROR("%s: failed to read isulad shim pidinfo", id);
         goto out;
     }
@@ -998,20 +988,29 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t
     pid_info->ppid = shim_pid;
     pid_info->pstart_time = p_proc->start_time;
 +    // clear NOTIFY_SOCKET from the env to adapt runc start
 +    if (unsetenv("NOTIFY_SOCKET") != 0) {
 +        ERROR("%s: unset env NOTIFY_SOCKET failed %s", id);
 +    }
 +
     if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) {
         ERROR("call runtime start id failed");
 -        ret = -1;
         goto out;
     }
 +    ret = 0;
 out:
     if (ret != 0) {
         show_shim_runtime_errlog(workdir);
         shim_kill_force(workdir);
     }
 -    free(proc);
 -    free(p_proc);
 +    if (proc != NULL) {
 +        free(proc);
 +    }
 +    if (p_proc != NULL) {
 +        free(p_proc);
 +    }
     return ret;
 }
 -- 
 2.25.1
--- a/0027-allow-the-paused-container-to-be-stopped.patch
+++ b/0027-allow-the-paused-container-to-be-stopped.patch
@ -1,68 +0,0 @@
 From 70173556a3825870b44cc344e83cdf0fd6d577c5 Mon Sep 17 00:00:00 2001
 From: zhongtao <zhongtao17@huawei.com>
 Date: Thu, 23 Feb 2023 16:08:48 +0800
 Subject: [PATCH 27/53] allow the paused container to be stopped
 Signed-off-by: zhongtao <zhongtao17@huawei.com>
 ---
 src/daemon/entry/cri/cri_container_manager_service.cc | 6 +++---
 src/daemon/entry/cri/cri_helpers.cc                   | 2 +-
 src/daemon/modules/service/service_container.c        | 7 +++----
 3 files changed, 7 insertions(+), 8 deletions(-)
 diff --git a/src/daemon/entry/cri/cri_container_manager_service.cc b/src/daemon/entry/cri/cri_container_manager_service.cc
 index 57284593..1592c0a6 100644
 --- a/src/daemon/entry/cri/cri_container_manager_service.cc
 +++ b/src/daemon/entry/cri/cri_container_manager_service.cc
@@ -1195,14 +1195,14 @@ void ContainerManagerService::UpdateContainerResources(const std::string &contai
     }
     if (resources.hugepage_limits_size() != 0) {
         hostconfig->hugetlbs = (host_config_hugetlbs_element **)util_smart_calloc_s(
 -                sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size());
 +                                   sizeof(host_config_hugetlbs_element *), resources.hugepage_limits_size());
         if (hostconfig->hugetlbs == nullptr) {
             error.SetError("Out of memory");
             return;
         }
 -	for (int i = 0; i < resources.hugepage_limits_size(); i++) {
 +        for (int i = 0; i < resources.hugepage_limits_size(); i++) {
             hostconfig->hugetlbs[i] =
 -                    (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element));
 +                (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element));
             if (hostconfig->hugetlbs[i] == nullptr) {
                 error.SetError("Out of memory");
                 goto cleanup;
 diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc
 index 6d59ec11..c24c8b73 100644
 --- a/src/daemon/entry/cri/cri_helpers.cc
 +++ b/src/daemon/entry/cri/cri_helpers.cc
@@ -470,7 +470,7 @@ void UpdateCreateConfig(container_config *createConfig, host_config *hc,
             }
             for (int i = 0; i < rOpts.hugepage_limits_size(); i++) {
                 hc->hugetlbs[i] =
 -                        (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element));
 +                    (host_config_hugetlbs_element *)util_common_calloc_s(sizeof(host_config_hugetlbs_element));
                 if (hc->hugetlbs[i] == nullptr) {
                     error.SetError("Out of memory");
                     return;
 diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
 index cc777411..d69ee757 100644
 --- a/src/daemon/modules/service/service_container.c
 +++ b/src/daemon/modules/service/service_container.c
@@ -1495,10 +1495,9 @@ int stop_container(container_t *cont, int timeout, bool force, bool restart)
     container_lock(cont);
 -    if (container_is_paused(cont->state)) {
 -        ERROR("Container %s is paused. Unpause the container before stopping or killing", id);
 -        isulad_set_error_message("Container %s is paused. Unpause the container before stopping or killing", id);
 -        ret = -1;
 +    if (!container_is_running(cont->state)) {
 +        INFO("Container %s is already stopped", id);
 +        ret = 0;
         goto out;
     }
 -- 
 2.25.1
--- a/0028-Refine.patch
+++ b/0028-Refine.patch
@ -1,29 +0,0 @@
 From 376677f2f38b3c27b14b7a21aa021ea683a2f0e1 Mon Sep 17 00:00:00 2001
 From: sailorvii <chenw66@chinaunicom.cn>
 Date: Tue, 28 Feb 2023 06:09:48 +0000
 Subject: [PATCH 28/53] Refine.
 ---
 src/daemon/modules/runtime/isula/isula_rt_ops.c | 6 ++++++
 1 file changed, 6 insertions(+)
 diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
 index 60742d42..5d622515 100644
 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
 +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -760,6 +760,12 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char
             goto realexec;
         }
 +        // clear NOTIFY_SOCKET from the env to adapt runc create
 +        if (unsetenv("NOTIFY_SOCKET") != 0) {
 +            (void)dprintf(exec_fd[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno));
 +            exit(EXIT_FAILURE);
 +        }
 +
         pid = fork();
         if (pid < 0) {
             (void)dprintf(exec_fd[1], "%s: fork shim-process failed %s", id, strerror(errno));
 -- 
 2.25.1
--- a/0029-support-isula-update-when-runtime-is-runc.patch
+++ b/0029-support-isula-update-when-runtime-is-runc.patch
@ -1,291 +0,0 @@
 From 7f00006ea65378e7b27049ff3f0eb3fa70e69b09 Mon Sep 17 00:00:00 2001
 From: zhongtao <zhongtao17@huawei.com>
 Date: Thu, 16 Feb 2023 20:20:54 +0800
 Subject: [PATCH 29/53] support isula update when runtime is runc
 Signed-off-by: zhongtao <zhongtao17@huawei.com>
 ---
 .../executor/container_cb/execution_extend.c  |   1 +
 src/daemon/modules/api/runtime_api.h          |   1 +
 .../modules/runtime/isula/isula_rt_ops.c      | 180 ++++++++++++++++--
 3 files changed, 161 insertions(+), 21 deletions(-)
 diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c
 index 67d0845a..2f565d78 100644
 --- a/src/daemon/executor/container_cb/execution_extend.c
 +++ b/src/daemon/executor/container_cb/execution_extend.c
@@ -1134,6 +1134,7 @@ static int do_update_resources(const container_update_request *request, containe
     if (container_is_running(cont->state)) {
         params.rootpath = cont->root_path;
         params.hostconfig = hostconfig;
 +        params.state = cont->state_path;
         if (runtime_update(id, cont->runtime, &params)) {
             ERROR("Update container %s failed", id);
             ret = -1;
 diff --git a/src/daemon/modules/api/runtime_api.h b/src/daemon/modules/api/runtime_api.h
 index b245ebf9..199c9f4b 100644
 --- a/src/daemon/modules/api/runtime_api.h
 +++ b/src/daemon/modules/api/runtime_api.h
@@ -169,6 +169,7 @@ typedef struct _rt_attach_params_t {
 typedef struct _rt_update_params_t {
     const char *rootpath;
     const host_config *hostconfig;
 +    const char *state;
 } rt_update_params_t;
 typedef struct _rt_listpids_params_t {
 diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
 index 6f2b4f7d..41791388 100644
 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
 +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -21,6 +21,7 @@
 #include <limits.h>
 #include <errno.h>
 #include <fcntl.h>
 +#include <isula_libutils/auto_cleanup.h>
 #include <isula_libutils/defs.h>
 #include <isula_libutils/isulad_daemon_configs.h>
 #include <isula_libutils/json_common.h>
@@ -39,6 +40,7 @@
 #include "constants.h"
 #include "isula_libutils/shim_client_process_state.h"
 #include "isula_libutils/shim_client_runtime_stats.h"
 +#include "isula_libutils/shim_client_cgroup_resources.h"
 #include "isula_libutils/oci_runtime_state.h"
 #include "isulad_config.h"
 #include "utils_string.h"
@@ -54,6 +56,9 @@
 #define RESIZE_DATA_SIZE 100
 #define PID_WAIT_TIME 120
 +// file name formats of cgroup resources json
 +#define RESOURCE_FNAME_FORMATS "%s/resources.json"
 +
 // handle string from stderr output.
 typedef int(*handle_output_callback_t)(const char *output);
@@ -229,34 +234,27 @@ bool rt_isula_detect(const char *runtime)
 static int create_process_json_file(const char *workdir, const shim_client_process_state *p)
 {
     struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 };
 -    parser_error perr = NULL;
 -    char *data = NULL;
 +    __isula_auto_free parser_error perr = NULL;
 +    __isula_auto_free char *data = NULL;
     char fname[PATH_MAX] = { 0 };
 -    int retcode = 0;
     if (snprintf(fname, sizeof(fname), "%s/process.json", workdir) < 0) {
 -        ERROR("failed make process.json full path");
 +        ERROR("Failed make process.json full path");
         return -1;
     }
     data = shim_client_process_state_generate_json(p, &ctx, &perr);
     if (data == NULL) {
 -        retcode = -1;
 -        ERROR("failed generate json for process.json error=%s", perr);
 -        goto out;
 +        ERROR("Failed generate json for process.json error=%s", perr);
 +        return -1;
     }
     if (util_write_file(fname, data, strlen(data), DEFAULT_SECURE_FILE_MODE) != 0) {
 -        retcode = -1;
 -        ERROR("failed write process.json");
 -        goto out;
 +        ERROR("Failed write process.json");
 +        return -1;
     }
 -out:
 -    UTIL_FREE_AND_SET_NULL(perr);
 -    UTIL_FREE_AND_SET_NULL(data);
 -
 -    return retcode;
 +    return 0;
 }
 static void get_runtime_cmd(const char *runtime, const char **cmd)
@@ -733,18 +731,18 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char
     runtime_exec_param_dump(params);
     if (snprintf(fpid, sizeof(fpid), "%s/shim-pid", workdir) < 0) {
 -        ERROR("failed make shim-pid full path");
 +        ERROR("Failed make shim-pid full path");
         return -1;
     }
     if (pipe2(exec_fd, O_CLOEXEC) != 0) {
 -        ERROR("failed to create pipe for shim create");
 +        ERROR("Failed to create pipe for shim create");
         return -1;
     }
     pid = fork();
     if (pid < 0) {
 -        ERROR("failed fork for shim parent %s", strerror(errno));
 +        ERROR("Failed fork for shim parent %s", strerror(errno));
         close(exec_fd[0]);
         close(exec_fd[1]);
         return -1;
@@ -1264,13 +1262,153 @@ int rt_isula_attach(const char *id, const char *runtime, const rt_attach_params_
     return -1;
 }
 -int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params)
 +static int to_engine_resources(const host_config *hostconfig, shim_client_cgroup_resources *cr)
 +{
 +    uint64_t period = 0;
 +    int64_t quota = 0;
 +
 +    if (hostconfig == NULL || cr == NULL) {
 +        return -1;
 +    }
 +
 +    cr->block_io = util_common_calloc_s(sizeof(shim_client_cgroup_resources_block_io));
 +    if (cr->block_io == NULL) {
 +        ERROR("Out of memory");
 +        return -1;
 +    }
 +
 +    cr->cpu = util_common_calloc_s(sizeof(shim_client_cgroup_resources_cpu));
 +    if (cr->cpu == NULL) {
 +        ERROR("Out of memory");
 +        return -1;
 +    }
 +
 +    cr->memory = util_common_calloc_s(sizeof(shim_client_cgroup_resources_memory));
 +    if (cr->memory == NULL) {
 +        ERROR("Out of memory");
 +        return -1;
 +    }
 +
 +    cr->block_io->weight = hostconfig->blkio_weight;
 +    cr->cpu->shares = (uint64_t)hostconfig->cpu_shares;
 +    cr->cpu->period = (uint64_t)hostconfig->cpu_period;
 +    cr->cpu->quota = hostconfig->cpu_quota;
 +    cr->cpu->cpus = util_strdup_s(hostconfig->cpuset_cpus);
 +    cr->cpu->mems = util_strdup_s(hostconfig->cpuset_mems);
 +    cr->memory->limit = (uint64_t)hostconfig->memory;
 +    cr->memory->swap = (uint64_t)hostconfig->memory_swap;
 +    cr->memory->reservation = (uint64_t)hostconfig->memory_reservation;
 +    cr->memory->kernel = (uint64_t)hostconfig->kernel_memory;
 +    cr->cpu->realtime_period = hostconfig->cpu_realtime_period;
 +    cr->cpu->realtime_runtime = hostconfig->cpu_realtime_runtime;
 +
 +    // when --cpus=n is set, nano_cpus = n * 1e9.
 +    if (hostconfig->nano_cpus > 0) {
 +        // in the case, period will be set to the default value of 100000(0.1s).
 +        period = (uint64_t)(100 * Time_Milli / Time_Micro);
 +        // set quota = period * n, in order to let container process fully occupy n cpus.
 +        if ((hostconfig->nano_cpus / 1e9)  > (INT64_MAX / (int64_t)period)) {
 +            ERROR("Overflow of quota");
 +            return -1;
 +        }
 +        quota = hostconfig->nano_cpus / 1e9 * (int64_t)period;
 +        cr->cpu->period = period;
 +        cr->cpu->quota = quota;
 +    }
 +
 +    return 0;
 +}
 +
 +static int create_resources_json_file(const char *workdir, const shim_client_cgroup_resources *cr, char *fname,
 +                                      size_t fname_size)
 {
 -    ERROR("isula update not support on isulad-shim");
 -    isulad_set_error_message("isula update not support on isulad-shim");
 +    struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 };
 +    __isula_auto_free parser_error perr = NULL;
 +    __isula_auto_free char *data = NULL;
 +
 +    if (snprintf(fname, fname_size, RESOURCE_FNAME_FORMATS, workdir) < 0) {
 +        ERROR("Failed make resources.json full path");
 +        return -1;
 +    }
 +
 +    data = shim_client_cgroup_resources_generate_json(cr, &ctx, &perr);
 +    if (data == NULL) {
 +        return -1;
 +    }
 +
 +    if (util_write_file(fname, data, strlen(data), DEFAULT_SECURE_FILE_MODE) != 0) {
 +        return -1;
 +    }
 +
 +    return 0;
 +}
 +
 +// show std error msg, always return -1.
 +static int show_stderr(const char *err)
 +{
 +    isulad_set_error_message(err);
     return -1;
 }
 +int rt_isula_update(const char *id, const char *runtime, const rt_update_params_t *params)
 +{
 +    int ret = 0;
 +    char workdir[PATH_MAX] = { 0 };
 +    char resources_fname[PATH_MAX] = { 0 };
 +    const char *opts[2] = { 0 };
 +    shim_client_cgroup_resources *cr = NULL;
 +
 +    if (id == NULL || runtime == NULL || params == NULL) {
 +        ERROR("Nullptr arguments not allowed");
 +        return -1;
 +    }
 +
 +    ret = snprintf(workdir, sizeof(workdir), "%s/%s/update", params->state, id);
 +    if (ret < 0) {
 +        ERROR("Failed join update full path");
 +        return ret;
 +    }
 +
 +    ret = util_mkdir_p(workdir, DEFAULT_SECURE_DIRECTORY_MODE);
 +    if (ret < 0) {
 +        ERROR("Failed mkdir update workdir %s", workdir);
 +        return ret;
 +    }
 +
 +    cr = util_common_calloc_s(sizeof(shim_client_cgroup_resources));
 +    if (cr == NULL) {
 +        ERROR("Out of memory");
 +        goto del_out;
 +    }
 +
 +    ret = to_engine_resources(params->hostconfig, cr);
 +    if (ret < 0) {
 +        ERROR("Failed to get resources for update");
 +        goto del_out;
 +    }
 +
 +    ret = create_resources_json_file(workdir, cr, resources_fname, sizeof(resources_fname));
 +    if (ret != 0) {
 +        ERROR("%s: failed create update json file", id);
 +        goto del_out;
 +    }
 +
 +    opts[0] = "--resources";
 +    opts[1] = resources_fname;
 +
 +    if (runtime_call_simple(workdir, runtime, "update", opts, 2, id, show_stderr) != 0) {
 +        ERROR("Call runtime update id failed");
 +        ret = -1;
 +    }
 +
 +del_out:
 +    if (util_recursive_rmdir(workdir, 0)) {
 +        ERROR("Rmdir %s failed", workdir);
 +    }
 +    free_shim_client_cgroup_resources(cr);
 +    return ret;
 +}
 +
 int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t *params)
 {
     char workdir[PATH_MAX] = { 0 };
 -- 
 2.25.1
--- a/0030-Refine-as-others-feedback.patch
+++ b/0030-Refine-as-others-feedback.patch
@ -1,41 +0,0 @@
 From 4900996ca52e46795eb25b05ce50519c7c71ae38 Mon Sep 17 00:00:00 2001
 From: sailorvii <chenw66@chinaunicom.cn>
 Date: Thu, 2 Mar 2023 09:04:57 +0000
 Subject: [PATCH 30/53] Refine as others' feedback.
 ---
 src/daemon/modules/runtime/isula/isula_rt_ops.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
 diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
 index 5d622515..7ec3fc49 100644
 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
 +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -473,6 +473,12 @@ static void runtime_exec_func(void *arg)
         _exit(EXIT_FAILURE);
     }
 +    // clear NOTIFY_SOCKET from the env to adapt runc start
 +    if (strcmp(rei->subcmd, "start") == 0 && unsetenv("NOTIFY_SOCKET") != 0) {
 +        dprintf(STDERR_FILENO, "unset env NOTIFY_SOCKET failed %s", strerror(errno));
 +        _exit(EXIT_FAILURE);
 +    }
 +
     execvp(rei->cmd, rei->params);
     dprintf(STDERR_FILENO, "exec %s %s %s failed", rei->cmd, rei->subcmd, rei->id);
     _exit(EXIT_FAILURE);
@@ -994,11 +1000,6 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t
     pid_info->ppid = shim_pid;
     pid_info->pstart_time = p_proc->start_time;
 -    // clear NOTIFY_SOCKET from the env to adapt runc start
 -    if (unsetenv("NOTIFY_SOCKET") != 0) {
 -        ERROR("%s: unset env NOTIFY_SOCKET failed %s", id);
 -    }
 -
     if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) {
         ERROR("call runtime start id failed");
         goto out;
 -- 
 2.25.1
--- a/0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch
+++ b/0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch
@ -1,134 +0,0 @@
 From a87e8aeea252e5aec9e1dea0daf99562eb86b092 Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Thu, 2 Mar 2023 20:05:49 +0800
 Subject: [PATCH 31/53] fix CRI SetupPod and TearDownPod deadlock
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 src/daemon/entry/cri/cni_network_plugin.cc    | 14 +++++++----
 .../cri/cri_pod_sandbox_manager_service.cc    |  2 ++
 src/daemon/entry/cri/network_plugin.cc        | 25 +++++++++++++------
 3 files changed, 29 insertions(+), 12 deletions(-)
 diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc
 index 1bce13f6..0fe095b6 100644
 --- a/src/daemon/entry/cri/cni_network_plugin.cc
 +++ b/src/daemon/entry/cri/cni_network_plugin.cc
@@ -120,8 +120,10 @@ auto CniNetworkPlugin::Name() const -> const std::string &
 void CniNetworkPlugin::CheckInitialized(Errors &err)
 {
 -    RLockNetworkMap(err);
 -    if (err.NotEmpty()) {
 +    Errors tmpErr;
 +    RLockNetworkMap(tmpErr);
 +    if (tmpErr.NotEmpty()) {
 +        err.AppendError(tmpErr.GetCMessage());
         return;
     }
@@ -129,9 +131,10 @@ void CniNetworkPlugin::CheckInitialized(Errors &err)
         err.SetError("cni config uninitialized");
     }
 -    UnlockNetworkMap(err);
 -    if (err.NotEmpty()) {
 -        WARN("Unable to update cni config: %s", err.GetCMessage());
 +    UnlockNetworkMap(tmpErr);
 +    if (tmpErr.NotEmpty()) {
 +        WARN("Unable to update cni config: %s", tmpErr.GetCMessage());
 +        err.AppendError(tmpErr.GetCMessage());
     }
 }
@@ -623,6 +626,7 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name,
         return;
     }
 +    err.Clear();
     RLockNetworkMap(err);
     if (err.NotEmpty()) {
         ERROR("%s", err.GetCMessage());
 diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc
 index 3d183861..ee523b9c 100644
 --- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc
 +++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service.cc
@@ -705,6 +705,7 @@ auto PodSandboxManagerService::GetRealSandboxIDToStop(const std::string &podSand
         if (status->linux().namespaces().has_options()) {
             hostNetwork = (status->linux().namespaces().options().network() == runtime::v1alpha2::NamespaceMode::NODE);
         }
 +        // if metadata is invalid, don't return -1 and continue stopping pod
         if (status->has_metadata()) {
             name = status->metadata().name();
             ns = status->metadata().namespace_();
@@ -831,6 +832,7 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID,
         }
         stdAnnos.insert(std::pair<std::string, std::string>(CRIHelpers::Constants::POD_SANDBOX_KEY, netnsPath));
 +        pluginErr.Clear();
         m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos,
                                      pluginErr);
         if (pluginErr.NotEmpty()) {
 diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc
 index 4a119d6b..e55db4ea 100644
 --- a/src/daemon/entry/cri/network_plugin.cc
 +++ b/src/daemon/entry/cri/network_plugin.cc
@@ -425,20 +425,26 @@ void PluginManager::SetUpPod(const std::string &ns, const std::string &name, con
         return;
     }
 +    Errors tmpErr;
     std::string fullName = name + "_" + ns;
 -    Lock(fullName, error);
 -    if (error.NotEmpty()) {
 +    Lock(fullName, tmpErr);
 +    if (tmpErr.NotEmpty()) {
 +        error.AppendError(tmpErr.GetCMessage());
         return;
     }
     INFO("Calling network plugin %s to set up pod %s", m_plugin->Name().c_str(), fullName.c_str());
 -    Errors tmpErr;
     m_plugin->SetUpPod(ns, name, interfaceName, podSandboxID, annotations, options, network_settings_json, tmpErr);
     if (tmpErr.NotEmpty()) {
         error.Errorf("NetworkPlugin %s failed to set up pod %s network: %s", m_plugin->Name().c_str(), fullName.c_str(),
                      tmpErr.GetCMessage());
     }
 -    Unlock(fullName, error);
 +
 +    tmpErr.Clear();
 +    Unlock(fullName, tmpErr);
 +    if (tmpErr.NotEmpty()) {
 +        error.AppendError(tmpErr.GetCMessage());
 +    }
 }
 void PluginManager::TearDownPod(const std::string &ns, const std::string &name, const std::string &interfaceName,
@@ -447,8 +453,9 @@ void PluginManager::TearDownPod(const std::string &ns, const std::string &name,
 {
     Errors tmpErr;
     std::string fullName = name + "_" + ns;
 -    Lock(fullName, error);
 -    if (error.NotEmpty()) {
 +    Lock(fullName, tmpErr);
 +    if (tmpErr.NotEmpty()) {
 +        error.AppendError(tmpErr.GetCMessage());
         return;
     }
     if (m_plugin == nullptr) {
@@ -462,7 +469,11 @@ void PluginManager::TearDownPod(const std::string &ns, const std::string &name,
                      fullName.c_str(), tmpErr.GetCMessage());
     }
 unlock:
 -    Unlock(fullName, error);
 +    tmpErr.Clear();
 +    Unlock(fullName, tmpErr);
 +    if (tmpErr.NotEmpty()) {
 +        error.AppendError(tmpErr.GetCMessage());
 +    }
 }
 void NoopNetworkPlugin::Init(const std::string &hairpinMode, const std::string &nonMasqueradeCIDR, int mtu,
 -- 
 2.25.1
--- a/0032-remote-layer-store-demo.patch
+++ b/0032-remote-layer-store-demo.patch
--- a/0033-add-ci-for-remote-ro.patch
+++ b/0033-add-ci-for-remote-ro.patch
@ -1,119 +0,0 @@
 From 6311ccd6e367f965da4dc1b4c9efb4bf43275f64 Mon Sep 17 00:00:00 2001
 From: Neil <wrz750726@gmail.com>
 Date: Sun, 5 Mar 2023 12:23:29 +0000
 Subject: [PATCH 33/53] add ci for remote ro
 Signed-off-by: Neil <wangrunze13@huawei.com>
 ---
 CI/make-and-install.sh                        |  4 +-
 .../container_cases/test_data/daemon.json     |  1 +
 CI/test_cases/image_cases/ro_separate.sh      | 69 +++++++++++++++++++
 3 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 CI/test_cases/image_cases/ro_separate.sh
 diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh
 index 3dbff480..fa9c2250 100755
 --- a/CI/make-and-install.sh
 +++ b/CI/make-and-install.sh
@@ -103,9 +103,9 @@ rm -rf build
 mkdir build
 cd build
 if [[ ${enable_gcov} -ne 0 ]]; then
 -  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON ..
 +  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
 else
 -  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON ..
 +  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
 fi
 make -j $(nproc)
 make install
 diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json
 index aa88c9da..2664c6b2 100644
 --- a/CI/test_cases/container_cases/test_data/daemon.json
 +++ b/CI/test_cases/container_cases/test_data/daemon.json
@@ -19,6 +19,7 @@
     "hook-spec": "/etc/default/isulad/hooks/default.json",
     "start-timeout": "2m",
     "storage-driver": "overlay2",
 +    "storage-enable-remote-layer": false,
     "storage-opts": [
         "overlay2.override_kernel_check=true"
     ],
 diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh
 new file mode 100644
 index 00000000..47e04abb
 --- /dev/null
 +++ b/CI/test_cases/image_cases/ro_separate.sh
@@ -0,0 +1,69 @@
 +#!/bin/bash
 +#
 +# attributes: isulad basic image
 +# concurrent: NA
 +# spend time: 22
 +
 +#######################################################################
 +##- Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
 +# - iSulad licensed under the Mulan PSL v2.
 +# - You can use this software according to the terms and conditions of the Mulan PSL v2.
 +# - You may obtain a copy of Mulan PSL v2 at:
 +# -     http://license.coscl.org.cn/MulanPSL2
 +# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
 +# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
 +# - PURPOSE.
 +# - See the Mulan PSL v2 for more details.
 +##- @Description:CI
 +##- @Author: wangrunze
 +##- @Create: 2023-03-03
 +#######################################################################
 +
 +declare -r curr_path=$(dirname $(readlink -f "$0"))
 +source ../helpers.sh
 +single_image="${curr_path}/busybox.tar"
 +
 +function test_separate_ro()
 +{
 +  local ret=0
 +  local test="isula separate ro test => (${FUNCNAME[@]})"
 +
 +  msg_info "${test} starting..."
 +
 +  sed -i 's/"storage-enable-remote-layer": false/"storage-enable-remote-layer": true/' /etc/isulad/daemon.json
 +  start_isulad_with_valgrind
 +  wait_isulad_running
 +
 +  isula rmi busybox
 +
 +  isula pull busybox
 +  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - pull image failed" && ((ret++))
 +
 +  isula run -tid --name test_separate busybox /bin/sh
 +  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++))
 +
 +  isula stop test_separate
 +  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++))
 +
 +  isula rmi busybox
 +  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++))
 +
 +  isula load -i $single_image
 +  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - load image failed: ${rootfs_tar}" && ((ret++))
 +
 +  check_valgrind_log
 +  [[ $? -ne 0 ]] && msg_err "separate ro test - memory leak, please check...." && ((ret++))
 +
 +  sed -i 's/"storage-enable-remote-layer": true/"storage-enable-remote-layer": false/' /etc/isulad/daemon.json
 +  start_isulad_with_valgrind
 +  wait_isulad_running
 +
 +  msg_info "${test} finished with return ${ret}..."
 +  return ${ret}
 +}
 +
 +declare -i ans=0
 +
 +test_separate_ro || ((ans++))
 +
 +show_result ${ans} "${curr_path}/${0}"
 -- 
 2.25.1
--- a/0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch
+++ b/0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch
@ -1,26 +0,0 @@
 From 9663e62598570d16c0e8a70be4341ff72663b8df Mon Sep 17 00:00:00 2001
 From: zhongtao <zhongtao17@huawei.com>
 Date: Mon, 6 Mar 2023 09:51:17 +0800
 Subject: [PATCH 34/53] change sleep() to usleep() to avoid lossing of accuracy
 Signed-off-by: zhongtao <zhongtao17@huawei.com>
 ---
 src/utils/cutils/utils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c
 index 2c4c01e4..983d81d8 100644
 --- a/src/utils/cutils/utils.c
 +++ b/src/utils/cutils/utils.c
@@ -345,7 +345,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c
             return -1;
         }
         // sleep some time instead to avoid cpu full running and then retry.
 -        sleep(0.1);
 +        usleep(100);
     }
     return 0;
 }
 -- 
 2.25.1
--- a/0035-fix-compile-error-when-not-enable-remote-ro.patch
+++ b/0035-fix-compile-error-when-not-enable-remote-ro.patch
@ -1,62 +0,0 @@
 From 6a0b11ae6584ee2eefff9bd20c96bc60582ccb6b Mon Sep 17 00:00:00 2001
 From: "Neil.wrz" <wangrunze13@huawei.com>
 Date: Sun, 5 Mar 2023 18:55:40 -0800
 Subject: [PATCH 35/53] fix compile error when not enable remote ro
 Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
 ---
 .../modules/image/oci/storage/image_store/CMakeLists.txt      | 3 +++
 .../modules/image/oci/storage/layer_store/CMakeLists.txt      | 4 ++++
 .../storage/layer_store/graphdriver/overlay2/CMakeLists.txt   | 3 +++
 3 files changed, 10 insertions(+)
 diff --git a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt
 index ecf21caa..7d4fb77c 100644
 --- a/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt
 +++ b/src/daemon/modules/image/oci/storage/image_store/CMakeLists.txt
@@ -1,5 +1,8 @@
 # get current directory sources files
 aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_image_store_srcs)
 +IF (NOT ENABLE_REMOTE_LAYER_STORE)
 +list(REMOVE_ITEM local_image_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/image_remote_impl.c")
 +ENDIF()
 set(IMAGE_STORE_SRCS
     ${local_image_store_srcs}
 diff --git a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt
 index f964f709..e04b4ad7 100644
 --- a/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt
 +++ b/src/daemon/modules/image/oci/storage/layer_store/CMakeLists.txt
@@ -1,5 +1,8 @@
 # get current directory sources files
 aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_layer_store_srcs)
 +IF (NOT ENABLE_REMOTE_LAYER_STORE)
 +list(REMOVE_ITEM local_layer_store_srcs "${CMAKE_CURRENT_SOURCE_DIR}/layer_remote_impl.c")
 +ENDIF()
 add_subdirectory(graphdriver)
 set(LAYER_STORE_SRCS
@@ -7,6 +10,7 @@ set(LAYER_STORE_SRCS
     ${GRAPHDRIVER_SRCS}
     PARENT_SCOPE
     )
 +
 set(LAYER_STORE_INCS
     ${CMAKE_CURRENT_SOURCE_DIR}
     ${GRAPHDRIVER_INCS}
 diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt
 index ceed16b7..dd4e82aa 100644
 --- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt
 +++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/CMakeLists.txt
@@ -1,5 +1,8 @@
 # get current directory sources files
 aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_overlay2_srcs)
 +IF (NOT ENABLE_REMOTE_LAYER_STORE)
 +list(REMOVE_ITEM local_overlay2_srcs "${CMAKE_CURRENT_SOURCE_DIR}/overlay_remote_impl.c")
 +ENDIF()
 set(OVERLAY2_SRCS
     ${local_overlay2_srcs}
 -- 
 2.25.1
--- a/0036-adapt-to-repo-of-openeuler-url-changed.patch
+++ b/0036-adapt-to-repo-of-openeuler-url-changed.patch
@ -1,33 +0,0 @@
 From 42a46f64e533ea0a89f95bef80a421fe06a5bfa2 Mon Sep 17 00:00:00 2001
 From: haozi007 <liuhao27@huawei.com>
 Date: Mon, 6 Mar 2023 14:38:58 +0800
 Subject: [PATCH 36/53] adapt to repo of openeuler url changed
 Signed-off-by: haozi007 <liuhao27@huawei.com>
 ---
 CI/pr-gateway.sh | 8 ++++++++
 1 file changed, 8 insertions(+)
 diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh
 index c715fc39..7c696de3 100755
 --- a/CI/pr-gateway.sh
 +++ b/CI/pr-gateway.sh
@@ -18,7 +18,15 @@ if [ $# -eq 1 ]; then
     tbranch=$1
 fi
 +sed -i "s#http://repo.openeuler.org#https://repo.huaweicloud.com/openeuler#g" /etc/yum.repos.d/openEuler.repo
 +
 +dnf update -y
 +
 dnf install -y gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath
 +if [ $? -ne 0 ]; then
 +    echo "install dependences failed"
 +    exit 1
 +fi
 # dnf install -y cargo rust rust-packaging
 -- 
 2.25.1
--- a/0037-change-goto-branch.patch
+++ b/0037-change-goto-branch.patch
@ -1,27 +0,0 @@
 From 66532035178723175ad9522201440372316dc914 Mon Sep 17 00:00:00 2001
 From: zhongtao <zhongtao17@huawei.com>
 Date: Mon, 6 Mar 2023 17:54:55 +0800
 Subject: [PATCH 37/53] change goto branch
 Signed-off-by: zhongtao <zhongtao17@huawei.com>
 ---
 src/daemon/executor/container_cb/execution_stream.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
 diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c
 index 1a7353b5..7e928cf7 100644
 --- a/src/daemon/executor/container_cb/execution_stream.c
 +++ b/src/daemon/executor/container_cb/execution_stream.c
@@ -161,7 +161,8 @@ static int container_exec_cb(const container_exec_request *request, container_ex
     if (exec_container(cont, request, *response, stdinfd, stdout_handler, stderr_handler) != 0) {
         ret = -1;
 -        goto pack_err_response;
 +        // pack err response in exec_container, there is no need to pack here.
 +        goto out;
     }
     goto out;
 -- 
 2.25.1
--- a/0038-CI-not-enable-remote-ro-for-ut.patch
+++ b/0038-CI-not-enable-remote-ro-for-ut.patch
@ -1,41 +0,0 @@
 From c414e2226542660579e14b9401fc28bc91b709c7 Mon Sep 17 00:00:00 2001
 From: "Neil.wrz" <wangrunze13@huawei.com>
 Date: Mon, 6 Mar 2023 18:59:43 -0800
 Subject: [PATCH 38/53] CI not enable remote ro for ut
 Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
 ---
 CI/make-and-install.sh                   | 2 +-
 CI/test_cases/image_cases/ro_separate.sh | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)
 diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh
 index fa9c2250..faeaf005 100755
 --- a/CI/make-and-install.sh
 +++ b/CI/make-and-install.sh
@@ -103,7 +103,7 @@ rm -rf build
 mkdir build
 cd build
 if [[ ${enable_gcov} -ne 0 ]]; then
 -  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
 +  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_UT=ON -DENABLE_METRICS=ON ..
 else
   cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON ..
 fi
 diff --git a/CI/test_cases/image_cases/ro_separate.sh b/CI/test_cases/image_cases/ro_separate.sh
 index 47e04abb..df45e120 100644
 --- a/CI/test_cases/image_cases/ro_separate.sh
 +++ b/CI/test_cases/image_cases/ro_separate.sh
@@ -45,6 +45,9 @@ function test_separate_ro()
   isula stop test_separate
   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++))
 +  isula rm test_separate
 +  [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove container failed" && ((ret++))
 +
   isula rmi busybox
   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - remove image failed" && ((ret++))
 -- 
 2.25.1
--- a/0039-use-auto-free-to-proc_t.patch
+++ b/0039-use-auto-free-to-proc_t.patch
@ -1,43 +0,0 @@
 From 0a68a53ccb6582384dad478d4197ec9386306027 Mon Sep 17 00:00:00 2001
 From: zhongtao <zhongtao17@huawei.com>
 Date: Mon, 6 Mar 2023 10:56:16 +0800
 Subject: [PATCH 39/53] use auto free to proc_t
 Signed-off-by: zhongtao <zhongtao17@huawei.com>
 ---
 src/daemon/modules/runtime/isula/isula_rt_ops.c | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)
 diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
 index 536e36b2..5ec0e639 100644
 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
 +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -951,8 +951,8 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t
     pid_t shim_pid = -1;
     int ret = -1;
     int splice_ret = 0;
 -    proc_t *proc = NULL;
 -    proc_t *p_proc = NULL;
 +    __isula_auto_free proc_t *proc = NULL;
 +    __isula_auto_free proc_t *p_proc = NULL;
     if (id == NULL || runtime == NULL || params == NULL || pid_info == NULL) {
         ERROR("nullptr arguments not allowed");
@@ -1009,14 +1009,6 @@ out:
         show_shim_runtime_errlog(workdir);
         shim_kill_force(workdir);
     }
 -
 -    if (proc != NULL) {
 -        free(proc);
 -    }
 -    if (p_proc != NULL) {
 -        free(p_proc);
 -    }
 -
     return ret;
 }
 -- 
 2.25.1
--- a/0040-modifying-cpurt-file-permissions.patch
+++ b/0040-modifying-cpurt-file-permissions.patch
@ -1,72 +0,0 @@
 From 2d9b9d88f3150027609fe984930af1dcf06dfd00 Mon Sep 17 00:00:00 2001
 From: songbuhuang <544824346@qq.com>
 Date: Thu, 9 Mar 2023 18:32:19 +0800
 Subject: [PATCH 40/53] modifying cpurt file permissions
 Signed-off-by: songbuhuang <544824346@qq.com>
 ---
 src/common/constants.h                       | 4 ++++
 src/daemon/common/sysinfo.c                  | 3 +--
 src/daemon/executor/container_cb/execution.c | 3 ++-
 3 files changed, 7 insertions(+), 3 deletions(-)
 diff --git a/src/common/constants.h b/src/common/constants.h
 index b43d8c80..06015f1e 100644
 --- a/src/common/constants.h
 +++ b/src/common/constants.h
@@ -22,6 +22,10 @@ extern "C" {
 /* mode of file and directory */
 +#define DEFAULT_CGROUP_FILE_MODE 0644
 +
 +#define DEFAULT_CGROUP_DIR_MODE 0755
 +
 #define DEFAULT_SECURE_FILE_MODE 0640
 #define DEFAULT_SECURE_DIRECTORY_MODE 0750
 diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c
 index 7559d653..baf53510 100644
 --- a/src/daemon/common/sysinfo.c
 +++ b/src/daemon/common/sysinfo.c
@@ -27,6 +27,7 @@
 #include <isula_libutils/auto_cleanup.h>
 #include <isula_libutils/log.h>
 +#include "constants.h"
 #include "err_msg.h"
 #include "utils.h"
 #include "utils_array.h"
@@ -73,8 +74,6 @@
 #define CGROUP_MOUNTPOINT "/sys/fs/cgroup"
 #define CGROUP_ISULAD_PATH CGROUP_MOUNTPOINT"/isulad"
 -#define DEFAULT_CGROUP_DIR_MODE 0755
 -#define DEFAULT_CGROUP_FILE_MODE 0644
 #define CGROUP2_CONTROLLERS_PATH CGROUP_MOUNTPOINT"/cgroup.controllers"
 #define CGROUP2_SUBTREE_CONTROLLER_PATH CGROUP_MOUNTPOINT"/cgroup.subtree_control"
 #define CGROUP2_CPUSET_CPUS_EFFECTIVE_PATH CGROUP_MOUNTPOINT"/cpuset.cpus.effective"
 diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c
 index 198052d3..c2a0bdb9 100644
 --- a/src/daemon/executor/container_cb/execution.c
 +++ b/src/daemon/executor/container_cb/execution.c
@@ -46,6 +46,7 @@
 #include "isulad_config.h"
 #include "sysinfo.h"
 #include "container_api.h"
 +#include "constants.h"
 #include "specs_api.h"
 #include "execution_extend.h"
 #include "execution_information.h"
@@ -324,7 +325,7 @@ static int maybe_create_cpu_realtime_file(int64_t value, const char *file, const
         return 0;
     }
 -    ret = util_mkdir_p(path, CONFIG_DIRECTORY_MODE);
 +    ret = util_mkdir_p(path, DEFAULT_CGROUP_DIR_MODE);
     if (ret != 0) {
         ERROR("Failed to mkdir: %s", path);
         return -1;
 -- 
 2.25.1
--- a/0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch
+++ b/0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch
@ -1,131 +0,0 @@
 From d0b0baa3f2624b6de0ca92c051c154f0cff43f1a Mon Sep 17 00:00:00 2001
 From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 Date: Tue, 14 Mar 2023 10:33:38 +0800
 Subject: [PATCH 41/53] use CURLOPT_XFERINFOFUNCTION instead of deprecated
 CURLOPT_PROGRESSFUNCTION since curl 7.32.0
 Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
 ---
 .../modules/image/oci/registry/http_request.c | 12 +++++++++++
 src/utils/http/http.c                         | 21 ++++++++++++++-----
 src/utils/http/http.h                         |  7 +++++++
 test/image/oci/registry/registry_ut.cc        |  8 +++++++
 4 files changed, 43 insertions(+), 5 deletions(-)
 diff --git a/src/daemon/modules/image/oci/registry/http_request.c b/src/daemon/modules/image/oci/registry/http_request.c
 index f29c2017..ce8b7667 100644
 --- a/src/daemon/modules/image/oci/registry/http_request.c
 +++ b/src/daemon/modules/image/oci/registry/http_request.c
@@ -691,6 +691,16 @@ static int progress(void *p, double dltotal, double dlnow, double ultotal, doubl
     return 0;
 }
 +static int xfer(void *p, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
 +{
 +    bool *cancel = p;
 +    if (*cancel) {
 +        // return nonzero code means abort transition
 +        return -1;
 +    }
 +    return 0;
 +}
 +
 int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file,
                       resp_data_type type, CURLcode *errcode)
 {
@@ -721,6 +731,8 @@ int http_request_file(pull_descriptor *desc, const char *url, const char **custo
     options->show_progress = 1;
     options->progressinfo = &desc->cancel;
     options->progress_info_op = progress;
 +    options->xferinfo = &desc->cancel;
 +    options->xferinfo_op = xfer;
     options->timeout = true;
     ret = setup_common_options(desc, options, url, custom_headers);
 diff --git a/src/utils/http/http.c b/src/utils/http/http.c
 index bf163d86..986f1f0d 100644
 --- a/src/utils/http/http.c
 +++ b/src/utils/http/http.c
@@ -219,12 +219,23 @@ static void http_custom_general_options(CURL *curl_handle, const struct http_get
     /* disable progress meter, set to 0L to enable and disable debug output */
     if (options->show_progress == 0) {
         curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L);
 -    } else if (options->show_progress && options->progressinfo && options->progress_info_op) {
 -        curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, options->progress_info_op);
 -        /* pass the struct pointer into the progress function */
 -        curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, options->progressinfo);
 -        curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L);
     } else {
 +    /* libcurl support option CURLOPT_XFERINFOFUNCTION when version >= 7.32.0
 +     * #define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|(z))
 +     * CURL_VERSION_BITS(7,32,0) = 0x072000 */
 +#if (LIBCURL_VERSION_NUM >= 0x072000)
 +        if (options->xferinfo && options->xferinfo_op) {
 +            curl_easy_setopt(curl_handle, CURLOPT_XFERINFOFUNCTION, options->xferinfo_op);
 +            /* pass the struct pointer into the progress function */
 +            curl_easy_setopt(curl_handle, CURLOPT_XFERINFODATA, options->xferinfo);
 +        }
 +#else
 +        if (options->progressinfo && options->progress_info_op) {
 +            curl_easy_setopt(curl_handle, CURLOPT_PROGRESSFUNCTION, options->progress_info_op);
 +            /* pass the struct pointer into the progress function */
 +            curl_easy_setopt(curl_handle, CURLOPT_PROGRESSDATA, options->progressinfo);
 +        }
 +#endif
         curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L);
     }
 diff --git a/src/utils/http/http.h b/src/utils/http/http.h
 index 343d92c3..cdd6d64f 100644
 --- a/src/utils/http/http.h
 +++ b/src/utils/http/http.h
@@ -17,6 +17,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 +#include <curl/curl.h>
 #ifdef __cplusplus
 extern "C" {
@@ -25,6 +26,9 @@ extern "C" {
 typedef int(*progress_info_func)(void *p,
                                  double dltotal, double dlnow,
                                  double ultotal, double ulnow);
 +typedef int(*xferinfo_func)(void *p,
 +                            curl_off_t dltotal, curl_off_t dlnow,
 +                            curl_off_t ultotal, curl_off_t ulnow);
 struct http_get_options {
     unsigned with_head : 1, /* if set, means write output with response HEADER */
@@ -77,6 +81,9 @@ struct http_get_options {
     void *progressinfo;
     progress_info_func progress_info_op;
 +
 +    void *xferinfo;
 +    xferinfo_func xferinfo_op;
 };
 #define HTTP_RES_OK                 0
 diff --git a/test/image/oci/registry/registry_ut.cc b/test/image/oci/registry/registry_ut.cc
 index 8d9ea92b..f4f8a763 100644
 --- a/test/image/oci/registry/registry_ut.cc
 +++ b/test/image/oci/registry/registry_ut.cc
@@ -221,6 +221,14 @@ int invokeHttpRequestV2(const char *url, struct http_get_options *options, long
             if (options->progress_info_op(options->progressinfo, 0, 0, 0, 0) != 0) {
                 return -1;
             }
 +
 +            cancel = (bool *)options->xferinfo;
 +            while (!(*cancel)) {
 +                sleep(1); // schedule out to let cancel variable set to be true
 +            }
 +            if (options->xferinfo_op(options->xferinfo, 0, 0, 0, 0) != 0) {
 +                return -1;
 +            }
         }
     } else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/blobs/sha256:91f30d77")) {
         if (retry) {
 -- 
 2.25.1
--- a/0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch
+++ b/0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch
@ -1,99 +0,0 @@
 From a08e511eb6ee1955bb62e774190dfe7a7599fbdd Mon Sep 17 00:00:00 2001
 From: "Neil.wrz" <wangrunze13@huawei.com>
 Date: Tue, 7 Mar 2023 23:59:56 -0800
 Subject: [PATCH 42/53] bugfix remote ro try add or remove image/layer twice
 Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
 ---
 .../image/oci/storage/image_store/image_store.c    | 14 ++++++++++++++
 .../oci/storage/layer_store/layer_remote_impl.c    |  2 +-
 .../image/oci/storage/layer_store/layer_store.c    | 11 +++++++++++
 3 files changed, 26 insertions(+), 1 deletion(-)
 diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c
 index caff3705..84187ded 100644
 --- a/src/daemon/modules/image/oci/storage/image_store/image_store.c
 +++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c
@@ -3668,6 +3668,11 @@ int append_image_by_directory_with_lock(const char *id)
         return -1;
     }
 +    if (map_search(g_image_store->byid, (void *)id) != NULL ) {
 +        DEBUG("remote image already exist, not added: %s", id);
 +        goto out;
 +    }
 +
     nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id);
     if (nret < 0 || (size_t)nret >= sizeof(image_path)) {
         ERROR("Failed to get image path");
@@ -3675,6 +3680,8 @@ int append_image_by_directory_with_lock(const char *id)
     }
     ret = append_image_by_directory(image_path);
 +
 +out:
     image_store_unlock();
     return ret;
@@ -3689,7 +3696,14 @@ int remove_image_from_memory_with_lock(const char *id)
         return -1;
     }
 +    if (map_search(g_image_store->byid, (void *)id) == NULL) {
 +        DEBUG("remote image already remvoed, don't delete twice: %s", id);
 +        goto out;
 +    }
 +
     ret = remove_image_from_memory(id);
 +
 +out:
     image_store_unlock();
     return ret;
 diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
 index d03fc20b..d676458c 100644
 --- a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
 +++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
@@ -175,7 +175,7 @@ static int remote_support_add(void *data)
         }
         if (add_one_remote_layer(data, array_added[i]) != 0) {
 -            ERROR("Failed to add remote overlay layer: %s", array_added[i]);
 +            ERROR("Failed to add remote layer: %s", array_added[i]);
             ret = -1;
         }
     }
 diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
 index c00c3356..e88067bc 100644
 --- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
 +++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
@@ -1852,6 +1852,11 @@ int load_one_layer(const char *id)
         return -1;
     }
 +    if (map_search(g_metadata.by_id, (void *)id) != NULL) {
 +        DEBUG("remote layer already exist, not added: %s", id);
 +        goto unlock_out;
 +    }
 +
     tl = load_one_layer_from_json(id);
     if (tl == NULL) {
         ret = -1;
@@ -2482,8 +2487,14 @@ int remove_memory_stores_with_lock(const char *id)
         ERROR("Failed to lock layer store when handle: %s", id);
         return -1;
     }
 +    if (map_search(g_metadata.by_id, (void *)id) == NULL) {
 +        DEBUG("remote layer already removed, don't delete: %s", id);
 +        goto unlock_out;
 +    }
     ret = remove_memory_stores(id);
 +
 +unlock_out:
     layer_store_unlock();
     return ret;
 -- 
 2.25.1
--- a/0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch
+++ b/0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch
@ -1,34 +0,0 @@
 From c8702b62fd6016a96794d74abcae2e74551a9c07 Mon Sep 17 00:00:00 2001
 From: "Neil.wrz" <wangrunze13@huawei.com>
 Date: Tue, 14 Mar 2023 20:32:23 -0700
 Subject: [PATCH 43/53] bugfix can't delete layers under dir overlay-layers
 Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
 ---
 .../modules/image/oci/storage/layer_store/layer_store.c   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
 diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
 index e88067bc..4edd0cad 100644
 --- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
 +++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
@@ -1409,10 +1409,14 @@ static int do_delete_layer(const char *id)
     }
 #ifdef ENABLE_REMOTE_LAYER_STORE
 -    if (l->slayer->writable) {
 +    if (!g_enable_remote_layer) {
         ret = layer_store_remove_layer(l->slayer->id);
     } else {
 -        ret = remote_layer_remove_ro_dir(l->slayer->id);
 +        if (l->slayer->writable) {
 +            ret = layer_store_remove_layer(l->slayer->id);
 +        } else {
 +            ret = remote_layer_remove_ro_dir(l->slayer->id);
 +        }
     }
 #else
     ret = layer_store_remove_layer(l->slayer->id);
 -- 
 2.25.1
--- a/0044-doc-add-document-about-support-remote-ro-directory.patch
+++ b/0044-doc-add-document-about-support-remote-ro-directory.patch
@ -1,396 +0,0 @@
 From 6911c68b96f3a0fc35b40e5c387da51cd84e5461 Mon Sep 17 00:00:00 2001
 From: "Neil.wrz" <wangrunze13@huawei.com>
 Date: Mon, 13 Feb 2023 03:34:31 -0800
 Subject: [PATCH 44/53] doc: add document about support remote ro directory
 ---
 docs/design/README.md                         |   4 +-
 docs/design/README_zh.md                      |   6 +-
 .../design/detailed/Image/remote_ro_design.md | 333 ++++++++++++++++++
 3 files changed, 340 insertions(+), 3 deletions(-)
 create mode 100644 docs/design/detailed/Image/remote_ro_design.md
 diff --git a/docs/design/README.md b/docs/design/README.md
 index a9d357b4..cf29c0a1 100644
 --- a/docs/design/README.md
 +++ b/docs/design/README.md
@@ -34,6 +34,8 @@ This section contains some design documents for users who want to learn more abo
 - You can see how the isula search are designed in [image_search_design](./detailed/Image/image_search_design_zh.md)
 +- You can see how the remote ro are designed in [remote_ro_design](./detailed/Image/remote_ro_design.md)
 +
 ## Network
 - You can see how the cni operator modules are designed in [cni_operator_design](./detailed/Network/cni_operator_design.md).
@@ -54,4 +56,4 @@ This section contains some design documents for users who want to learn more abo
 ## Volume
 -- You can see how the local volume modules are designed in [local_volume_design](./detailed/Volume/local_volume_design.md).
 \ No newline at end of file
 +- You can see how the local volume modules are designed in [local_volume_design](./detailed/Volume/local_volume_design.md).
 diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md
 index f51930e4..1f1c94b5 100644
 --- a/docs/design/README_zh.md
 +++ b/docs/design/README_zh.md
@@ -32,7 +32,9 @@
 - 查看 registry 模块的设计文档： [registry_degisn](./detailed/Image/registry_degisn_zh.md) 。
 -- 查看 isula search 的设计文档：[image_search_design](./detailed/Image/image_search_design_zh.md)
 +- 查看 isula search 的设计文档：[image_search_design](./detailed/Image/image_search_design_zh.md) 。
 +
 +- 查看 ro目录分离的设计文档： [remote_ro_design](./detailed/Image/remote_ro_design.md) 。
 ## Network
@@ -54,4 +56,4 @@
 ## Volume
 -- 查看 local volume 模块的设计文档： [local_volume_design](./detailed/Volume/local_volume_design_zh.md).
 \ No newline at end of file
 +- 查看 local volume 模块的设计文档： [local_volume_design](./detailed/Volume/local_volume_design_zh.md).
 diff --git a/docs/design/detailed/Image/remote_ro_design.md b/docs/design/detailed/Image/remote_ro_design.md
 new file mode 100644
 index 00000000..fee33835
 --- /dev/null
 +++ b/docs/design/detailed/Image/remote_ro_design.md
@@ -0,0 +1,333 @@
 +| Author | 王润泽                 |
 +| ------ | ---------------------- |
 +| Date   | 2023-2-13              |
 +| Email  | wangrunze13@huawei.com |
 +
 +# 1. 方案目标
 +目标有两个：
 +1. 把isulad当前的layer store里的RO层分离出来，把RW layer和RO layer分开到不同到目录存储。
 +2. isulad在运行时如果在相关目录里恢复了正确的镜像数据(image和layer数据), 可实现不重启isulad, 直接使用新恢复的镜像。如果移除当前没有容器正在使用的镜像数据，可实现不重启isulad，isulad更新当前管理的镜像列表去除该镜像。只考虑完全正确的新增和删除。
 +
 +
 +## 1.1 用法说明
 +通过源码编译打开编译选项来开启功能，cmake添加`cmake -DENABLE_REMOTE_LAYER_STORE=ON ..`, 然后`make -j`即可。启动iSulad之前，还需要在配置文件`/etc/isulad/daemon.json`里面添加`"storage-enable-remote-layer": true`来打开开关。
 +
 +# 2. 总体设计
 +
 +*Modules Dependencies*
 +```
 +=> New Added Module:
 +
 +   +===================================+
 +   |    Remote Supporter Module        |
 +   +===================================+
 +   |                                   |
 +   |    +-------------------------+    |                       
 +   |    |   maintainer submod     |    |                       
 +   |    +-------------------------+    |                       
 +   |    |   global data initer    |    |                       
 +   |    |  symbol link maintainer |    |                       
 +   |    |   global data getter    |    |                       
 +   |    +-------------------------+    |                       
 +   |                                   |
 +   |    +-------------------------+    |
 +   |    |    Supporter submod     |    |
 +   |    +-------------------------+    |
 +   |    |   supporter interface   |    |
 +   |    |  overlay supporter impl |    |
 +   |    |   layer supporter impl  |    |
 +   |    |   image supporter impl  |    |
 +   |    |  remote refresh thread  |    |
 +   |    +-------------------------+    |
 +   +-----------------------------------+
 +
 +
 +=> Modified Modules:
 +
 +   +===================================+
 +   |          Storage Module           |
 +   +===================================+
 +   |                                   |
 +   |    +-------------------------+    |                       
 +   |    |   Image Store submod    |    |                       
 +   |    |     Added Functions     |    |                       
 +   |    +-------------------------+    |                       
 +   |    |   add image in memory   |    |                       
 +   |    |  delete image in memory |    |                       
 +   |    |   get image top layer   |    |                       
 +   |    |   valid image manifest  |    |                       
 +   |    +-------------------------+    |                       
 +   |                                   |
 +   |    +-------------------------+    |
 +   |    |    Layer Store submod   |    |
 +   |    |     Added Functions     |    |                       
 +   |    +-------------------------+    |
 +   |    |   add layer in memory   |    |
 +   |    |  delete layer in memory |    |
 +   |    +-------------------------+    |
 +   |                                   |
 +   |    +-------------------------+    |
 +   |    |  Driver Overlay submod  |    |
 +   |    |     Added Functions     |    |                       
 +   |    +-------------------------+    |
 +   |    |          -              |    |
 +   |    +-------------------------+    |
 +   +-----------------------------------+
 +
 +
 +=>  Modules Dependencies:      
 +
 +                          +-------------------------+          +---------------+                                           
 +                          |    Supporter submod     |          |   Image Store |                                           
 +                          +-------------------------+    +----▶|    submod     ---------+                                  
 +                          |   supporter interface   |    |     +---------------+        |                                  
 +                          |  overlay supporter impl -----+                              |                                  
 +                          |   layer supporter impl  -----+                              |                                  
 ++----------------+        |   image supporter impl  |    |     +----------------+       |                                  
 +| storage module |-------▶|  remote refresh thread  |    +----▶|    Layer Store |       | init    +-----------------------+
 ++----------------+        +-------------------------+          |     submod     --------+--------▶|   maintainer submod   |
 +                                     |                         +----------------+       |         +-----------------------+
 +                                     |                                                  |                     ▲            
 +                                     |                                                  |                     |            
 +                                     |                         +----------------+       |                     |            
 +                                     |                         |    Layer Store |       |                     |            
 +                                     |                         |     submod     --------+                     |            
 +                                     |                         +----------------+                             |            
 +                                     |                                                                        |            
 +                                     +------------------------------------------------------------------------+               
 +                                                                 get global data                                       
 +```
 +
 +
 +总体来说有两部分的功能：
 +- iSulad原有的image storage适配分离的RO目录结构，*分离的RO目录*可用于远程挂载
 +- iSulad实例同步内存数据，镜像数据和layer数据*定期更新*，不通过`isula pull` 和 `isula rmi` 等命令，直接通过分离目录里面的数据来更新镜像数据。
 +
 +*分离RO目录*
 +修改前后storage目录结构对比：
 +
 +```
 +old:
 +overlay-layer
 +├── b703587
 +│    └── layer.json
 +└── b64792c
 +     └── layer.json 
 +     └── b64792.tar.gz
 +
 +new:
 +overlay-layer
 +├── b64792c -> ../RO/b64792c
 +├── b703587
 +│   └── layer.json
 +└── RO
 +    └── b64792c
 +        └── layer.json
 +        └── b64792.tar.gz
 +```
 +
 +以overlay-layers目录为例，创建新layer时，如果是只读层，就把层数据放到RO目录下，在RO上层目录创建软连接指向真实数据。删除layer时需要额外删除软连接。
 +
 +
 +*定期更新*
 +定期更新通过启动一个线程周期扫描`overlay`, `overlay-layers`, `overlay-image`这三个目录，通过比较当前时刻与上一时刻的目录差异，来获取镜像和层的删减情况，进而同步isulad的storage内存数据和维护软链接。
 +
 +```
 ++---------------------+       +---------------------+           +---------------------+           +-----------------------+
 +| refresh thread loop |       | overlay remote impl |           |  layer remote impl  |           |   image remote impl   |
 ++---------------------+       +---------------------+           +---------------------+           +-----------------------+
 +        |                              |                                  |                                  |
 +        | refresh start                |                                  |                                  |
 +        |-----------------------------▶|                                  |                                  |
 +        |                              | overlay dir scan                 |                                  |
 +        |                              |                                  |                                  |
 +        |                              | to added layers                  |                                  |
 +        |                              | memory and symlink add           |                                  |
 +        |                              | to deleted layers                |                                  |
 +        |                              | memory and symlink del           |                                  |
 +        |                              | valid overlay layers             |                                  |
 +        |                              |---------------------------------▶|                                  |
 +        |                              |         next scan                |                                  |
 +        |                              |                                  |                                  |
 +        |                              |                                  |                                  |
 +        |                              |                                  |                                  |
 +        |                              |                                  | overlay-layers dir scan          |
 +        |                              |    check overlay layer ready     |                                  |
 +        |                              |◀---------------------------------| to added layers                  |
 +        |                              |---------------------------------▶| filter invalid layers            |
 +        |                              |           result                 | memory and symlink add           |
 +        |                              |                                  | to deleted layers                |
 +        |                              |                                  | memory and symlink del           |
 +        |                              |                                  | valid overlay layers             |
 +        |                              |                                  |---------------------------------▶|
 +        |                              |                                  |        next scan                 |
 +        |                              |                                  |                                  |
 +        |                              |                                  |                                  |
 +        |                              |                                  |                                  |
 +        |                              |                                  |                                  | overlay-image dir scan
 +        |                              |                                  |       check layers ready         |
 +        |                              |                                  |◀---------------------------------| to added images        
 +        |                              |                                  |---------------------------------▶| filter invalid images   
 +        |                              |                                  |           result                 | memory add images   
 +        |                              |                                  |                                  | to deleted images   
 +        |                              |                                  |                                  | memory del images   
 +        |                              |                                  |                                  |
 +        |◀---------------------------------------------------------------------------------------------------|
 +        | refresh end                  |                                  |                                  |
 +        |                              |                                  |                                  |
 ++---------------------+       +---------------------+          +---------------------+            +-----------------------+
 +| refresh thread loop |       | image remote module |          | layer remote module |            | overlay remote module |
 ++---------------------+       +---------------------+          +---------------------+            +-----------------------+
 +
 +```
 +
 +# 3. 接口描述
 +
 +```c
 +// 初始化remote模块里的layer data
 +int remote_layer_init(const char *root_dir);
 +
 +// 初始化remote模块里的overlay data
 +int remote_overlay_init(const char *driver_home);
 +
 +// 清理remote模块的资源
 +void remote_maintain_cleanup();
 +
 +// 启动 定期更新的thread
 +int start_refresh_thread(void);
 +
 +// 创建新layer目录
 +int remote_layer_build_ro_dir(const char *id);
 +
 +// 创建新overlay目录
 +int remote_overlay_build_ro_dir(const char *id);
 +
 +// 删除layer目录
 +int remote_layer_remove_ro_dir(const char *id);
 +
 +// 删除overlay目录
 +int remote_overlay_remove_ro_dir(const char *id);
 +```
 +
 +# 4. 详细设计
 +分离RO目录的关键在于适配原来的代码逻辑，原先的代码在操作镜像和层的时候，不管是RO层还是RW层，从创建到删除都是在当前目录下进行的，这就是我们额外创建一个软连接的作用:
 +- RO目录的作用是为了支持远程挂载
 +- 软连接的作用是模拟原来的目录结构
 +
 +这样以来，image module的逻辑几乎不需要改动，除了以下几点需要注意：
 +- 创建和删除的时候需要处理一个额外的资源：软连接，之前只需要关注目录即可，现在如果创建的是只读层，就需要额外创建软连接，如果删除的是只读层，就需要额外删除软连接
 +- 以`overlay-layers`目录为例，isulad启动时会以正则规则扫描当前目录下的子目录是否合法，所以需要屏蔽`RO`目录
 +
 +定时刷新的逻辑如下：
 +以`overlay-image`目录的刷新为例，通过维护两个集合`new` 和 `old`, 这两个集合初始都为空，通过扫描目录里面所有的子目录，把合法的image id 加入`new`集合， 通过计算两个集合的差， 在集合`new`里面存在而在集合`old`里面不存在的id则为新增加的镜像， 在集合`old`里面存在而在集合`new`里面不存在的id则为删除的镜像。处理新增加的镜像还需要额外的一个判断，就是判断镜像的层数据是否已经加载，如果没加载则该镜像本轮不加载。`overlay-layers` 和 `overlay` 目录的处理逻辑类似。
 +
 +
 +*可能的使用场景*
 +一个可能的使用场景就是通过远程文件共享(nfs)让多台启动的isulad实例共享某些只读的数据；具体来说，在两个host A和B上都启动了iSulad, 如果A pull或者load了镜像busybox, 那么B上的isulad同样可以使用这个镜像。
 +
 +```
 +operations:
 +
 ++--------------------+           +--------------------+           +--------------------+
 +| isula pull busybox |           | without pull       |           | without pull       |
 +| isula pull nginx   |           | isula run busybox  |           | isula run ngxinx   |
 +| isula pull ...     |           | isula run ...      |           | isula run centos   |
 ++--------------------+           +--------------------+           +--------------------+
 +    |                                |                                |
 +    ▼                                ▼                                ▼
 ++======================+         +======================+         +======================+
 +| isulad on Host A     |         | isulad on Host B     |         | isulad on Host C     |
 ++======================+         +======================+         +======================+
 +| image store module   |         | image store module   |         | image store module   |
 ++----------------------+         +----------------------+         +----------------------+
 +| refresh thread off   |         | refresh thread on    |         | refresh thread on    |
 ++----------------------+         +----------------------+         +----------------------+
 +| local rw | remote ro |         | local rw | remote ro |         | local rw | remote ro |
 ++----------------------+         +----------------------+         +----------------------+
 +                |                               |                             |            
 +                | enable nfs                    | mounted on                  | mounted on
 +                ▼                               ▼                             ▼           
 +            +=====================================================================+
 +            |                    nfs directory over network                       |
 +            +=====================================================================+
 +            |                          image store                                |
 +            +---------------------------------------------------------------------+
 +            |   image      |   image     |   image   |   image     |   image      | 
 +            |   busybox    |   nginx     |   my-app  |   ubuntu    |   centos     | 
 +            |   4MB        |   100MB     |   1.2GB   |   5MB       |   6MB        | 
 +            +---------------------------------------------------------------------+
 +            |                          layers store                               |
 +            +---------------------------------------------------------------------+
 +            |     layer            |      layer            |        layer         |
 +            |   05c361054          |    8a9d75caad         |      789d605ac       |
 +            +---------------------------------------------------------------------+
 +```
 +
 +*同步问题*
 +共享资源并发使用发生竞争的条件是：`two process access the same resource concurrently and at least one of the access is a writer`。这里的共享资源有：
 +
 +```
 ++============================+         +=====================================+
 +|      Sharing Resource      |         |          Storage                    |
 ++============================+         +=====================================+
 +| read-only overlay layers   | mounted | /var/lib/isulad/overlay/RO          |
 ++----------------------------+ ======▶ +-------------------------------------+
 +| reald-only layers metadata | shared  | /var/lib/isulad/overlay-layers/RO   |
 ++----------------------------+         +-------------------------------------+
 +| reald-only images          |         | /var/lib/isulad/overlay-images      |
 ++----------------------------+         +-------------------------------------+
 +
 +```
 +而分布在不同host上的isulad进程通过网络共享这些资源，如果不考虑新增删除的情况，不会出现资源竞争，所有的节点都是reader。
 +
 +对于主节点pull镜像，其他节点使用镜像的情况，主节点是writer，其他节点是reader。这时候可能出现的问题是主节点pull镜像的流程没有完全结束，但是其他节点开始使用这个不完整的镜像。对于这个问题的解决方案是通过扫描image目录来新增镜像，通过这种方式能确保一定该新增镜像的信息完整。
 +
 +```
 ++---------------------+         +--------------------+         +-----------------------+         +-----------------------+
 +|    registry         |         | image module       |         | layer store module    |         | driver overlay module |
 ++---------------------+         +--------------------+         +-----------------------+         +-----------------------+
 +        |                                |                                 |                                 |
 +        |                                |                                 |                                 |
 +        | registry_pull                  |                                 |                                 |
 +        | fetch_manifest                 |                                 |                                 |
 +        | check reuse and fetch          |                                 |                                 |
 +        | +----------------+             |                                 |                                 |
 +        | | register_layer |             |                                 |                                 |
 +        | +----------------+             |                                 |                                 |
 +        |-----------------------------------------------------------------▶|                                 |
 +        |                                |                                 | layer_store_create              |
 +        |                                |                                 |--------------------------------▶|
 +        |                                |                                 |                                 | driver_create_layer
 +        |                                |                                 |                                 | +--------------------+ 
 +        |                                |                                 |                                 | | setup overlay dir  |
 +        |                                |                                 |                                 | +--------------------+
 +        |                                |                                 |                                 | driver_create_layer done
 +        |                                |                                 |◀--------------------------------|
 +        |                                |                                 | +------------+                  |
 +        |                                |                                 | | save layer |                  |
 +        |                                |                                 | +------------+                  |
 +        |                                |                                 | layer create done               |
 +        |◀-----------------------------------------------------------------|                                 |
 +        | all layer setup                |                                 |                                 |
 +        | +----------------+             |                                 |                                 |
 +        | | register image |             |                                 |                                 |
 +        | +----------------+             |                                 |                                 |
 +        |-------------------------------▶|                                 |                                 |
 +        |                                | storage_img_create              |                                 |
 +        |                                | set img top layer               |                                 |
 +        |                                | img create                      |                                 |
 +        |                                | +------------+                  |                                 |
 +        |                                | | save image |                  |                                 |
 +        |                                | +------------+                  |                                 |
 +        |                                | create image done               |                                 |
 +        |◀-------------------------------|                                 |                                 |
 +        | pull img done                  |                                 |                                 |
 +        |                                |                                 |                                 |
 +        |                                |                                 |                                 |
 ++---------------------+         +--------------------+         +-----------------------+         +-----------------------+
 +|    registry         |         | image module       |         | layer store module    |         | driver overlay module |
 ++---------------------+         +--------------------+         +-----------------------+         +-----------------------+
 +
 +```
 +
 +至于主节点删除镜像的情况，主节点是writer，其他节点是reader，可能出现的情况是其他节点还有容器在使用镜像的时候，镜像被删除，但是根据需求场景暂不处理这种情况。其他的处理与新增镜像相同，依然以image dir作为入口，扫描发现删除的镜像。删除时需要关注layer和overlay目录下的软链接。
 -- 
 2.25.1
--- a/0045-Refine-a-minor-log-message.patch
+++ b/0045-Refine-a-minor-log-message.patch
@ -1,34 +0,0 @@
 From 9b7e37018612f7443a0f8f966106995da31cc292 Mon Sep 17 00:00:00 2001
 From: sailorvii <chenw66@chinaunicom.cn>
 Date: Fri, 17 Mar 2023 06:34:37 +0000
 Subject: [PATCH 45/53] Refine a minor log message.
 ---
 src/daemon/modules/network/native/adaptor_native.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 diff --git a/src/daemon/modules/network/native/adaptor_native.c b/src/daemon/modules/network/native/adaptor_native.c
 index 02ff3642..8bc386d1 100644
 --- a/src/daemon/modules/network/native/adaptor_native.c
 +++ b/src/daemon/modules/network/native/adaptor_native.c
@@ -1764,8 +1764,7 @@ bool has_connected_container(native_network *network)
         message[pos - 1] = '\0';
     }
 -    ERROR("network %s has connected containers [ %s ]", network->conflist->list->name, message);
 -    isulad_set_error_message("network %s has connected containers [ %s ]", network->conflist->list->name, message);
 +    INFO("network %s has connected containers [ %s ]", network->conflist->list->name, message);
 out:
     native_network_unlock(network);
@@ -1923,6 +1922,7 @@ int native_config_remove(const char *name, char **res_name)
     }
     if (has_connected_container(network)) {
 +        isulad_set_error_message("network %s has connected containers", network->conflist->list->name);
         ret = -1;
         goto out;
     }
 -- 
 2.25.1
--- a/0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch
+++ b/0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch
@ -1,44 +0,0 @@
 From ea8962c6ee3ed05cfabe84c58c47cf56e3388242 Mon Sep 17 00:00:00 2001
 From: zhongtao <zhongtao17@huawei.com>
 Date: Mon, 20 Mar 2023 14:17:00 +0800
 Subject: [PATCH 46/53] modify the return value of the
 util_waitpid_with_timeout to status
 Signed-off-by: zhongtao <zhongtao17@huawei.com>
 ---
 src/utils/cutils/utils.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
 diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c
 index 983d81d8..3500d8f8 100644
 --- a/src/utils/cutils/utils.c
 +++ b/src/utils/cutils/utils.c
@@ -319,6 +319,7 @@ rep:
 int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_callback_t cb)
 {
     int nret = 0;
 +    int st;
     time_t start_time = time(NULL);
     time_t end_time;
     double interval;
@@ -328,7 +329,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c
     }
     for (;;) {
 -        nret = waitpid(pid, NULL, WNOHANG);
 +        nret = waitpid(pid, &st, WNOHANG);
         if (nret == pid) {
             break;
         }
@@ -347,7 +348,7 @@ int util_waitpid_with_timeout(pid_t pid, const int64_t timeout, handle_timeout_c
         // sleep some time instead to avoid cpu full running and then retry.
         usleep(100);
     }
 -    return 0;
 +    return st;
 }
 int util_wait_for_pid_status(pid_t pid)
 -- 
 2.25.1
--- a/0047-fix-util_getgrent_r-overflow.patch
+++ b/0047-fix-util_getgrent_r-overflow.patch
--- a/0048-add-ut-for-runc.patch
+++ b/0048-add-ut-for-runc.patch
@ -1,188 +0,0 @@
 From c20a580a7d8c6a8a16273513c6d11edd827b90b0 Mon Sep 17 00:00:00 2001
 From: zhongtao <zhongtao17@huawei.com>
 Date: Thu, 23 Feb 2023 14:39:23 +0800
 Subject: [PATCH 48/53] add ut for runc
 Signed-off-by: zhongtao <zhongtao17@huawei.com>
 ---
 .../modules/runtime/isula/isula_rt_ops.c      |  5 ++
 test/cutils/utils_utils/CMakeLists.txt        |  2 +
 test/cutils/utils_utils/utils_utils_ut.cc     | 61 +++++++++++++++++++
 test/runtime/isula/isula_rt_ops_ut.cc         | 56 +++++++++++++++++
 4 files changed, 124 insertions(+)
 diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
 index 5ec0e639..fe0f227c 100644
 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
 +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -1553,6 +1553,11 @@ out:
 int rt_isula_kill(const char *id, const char *runtime, const rt_kill_params_t *params)
 {
 +    if (id == NULL || runtime == NULL || params == NULL || params->pid < 0) {
 +        ERROR("Invalid arguments not allowed");
 +        return -1;
 +    }
 +
     if (util_process_alive(params->pid, params->start_time) == false) {
         if (params->signal == params->stop_signal || params->signal == SIGKILL) {
             WARN("Process %d is not alive", params->pid);
 diff --git a/test/cutils/utils_utils/CMakeLists.txt b/test/cutils/utils_utils/CMakeLists.txt
 index 99a83e7a..6d276390 100644
 --- a/test/cutils/utils_utils/CMakeLists.txt
 +++ b/test/cutils/utils_utils/CMakeLists.txt
@@ -12,6 +12,8 @@ target_include_directories(${EXE} PUBLIC
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils
     )
 +
 +set_target_properties(${EXE} PROPERTIES LINK_FLAGS "-Wl,--wrap,waitpid")
 target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
 set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
 diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc
 index 5308351a..18f0a506 100644
 --- a/test/cutils/utils_utils/utils_utils_ut.cc
 +++ b/test/cutils/utils_utils/utils_utils_ut.cc
@@ -15,6 +15,44 @@
 #include <gtest/gtest.h>
 #include "utils.h"
 +#include "mock.h"
 +
 +static pid_t test_pid = -1;
 +
 +extern "C" {
 +    DECLARE_WRAPPER_V(waitpid, pid_t, (__pid_t pid, int *stat_loc, int options));
 +    DEFINE_WRAPPER_V(waitpid, pid_t, (__pid_t pid, int *stat_loc, int options),(pid, stat_loc, options));
 +}
 +
 +static pid_t waitpid_none_zero(__pid_t pid, int *stat_loc, int options)
 +{
 +    *stat_loc = 256;
 +    return test_pid;
 +}
 +
 +static pid_t waitpid_zero(__pid_t pid, int *stat_loc, int options)
 +{
 +    *stat_loc = 0;
 +    return test_pid;
 +}
 +
 +#define ExitSignalOffset 128
 +static int status_to_exit_code(int status)
 +{
 +    int exit_code = 0;
 +
 +    if (WIFEXITED(status)) {
 +        exit_code = WEXITSTATUS(status);
 +    } else {
 +        exit_code = -1;
 +    }
 +    if (WIFSIGNALED(status)) {
 +        int signal;
 +        signal = WTERMSIG(status);
 +        exit_code = ExitSignalOffset + signal;
 +    }
 +    return exit_code;
 +}
 TEST(utils_utils, test_util_mem_realloc)
 {
@@ -284,4 +322,27 @@ TEST(utils_utils, test_do_retry_call)
     DO_RETRY_CALL(10, 100, nret, retry_call_test, 11);
     ASSERT_EQ(global_total, 10);
     ASSERT_EQ(nret, -1);
 +}
 +
 +TEST(utils_utils, test_util_waitpid_with_timeout)
 +{
 +    int64_t timeout = 2;
 +    pid_t pid = getpid();
 +    int status = 0;
 +
 +    test_pid = pid;
 +    MOCK_SET_V(waitpid, waitpid_none_zero);
 +    status = util_waitpid_with_timeout(test_pid, timeout, nullptr);
 +    ASSERT_EQ(status, 256);
 +    ASSERT_EQ(status_to_exit_code(status), 1);
 +    MOCK_CLEAR(waitpid);
 +
 +    MOCK_SET_V(waitpid, waitpid_zero);
 +    status = util_waitpid_with_timeout(test_pid, timeout, nullptr);
 +    ASSERT_EQ(status, 0);
 +    ASSERT_EQ(status_to_exit_code(status), 0);
 +    MOCK_CLEAR(waitpid);
 +
 +    ASSERT_EQ(util_waitpid_with_timeout(pid, timeout, nullptr), -1);
 +
 }
 \ No newline at end of file
 diff --git a/test/runtime/isula/isula_rt_ops_ut.cc b/test/runtime/isula/isula_rt_ops_ut.cc
 index 03c213a5..f37e62a0 100644
 --- a/test/runtime/isula/isula_rt_ops_ut.cc
 +++ b/test/runtime/isula/isula_rt_ops_ut.cc
@@ -163,3 +163,59 @@ TEST_F(IsulaRtOpsUnitTest, test_rt_isula_exec_resize)
     close(fd);
     ASSERT_EQ(system(rm_path.c_str()), 0);
 }
 +
 +TEST_F(IsulaRtOpsUnitTest, test_rt_isula_update)
 +{
 +    rt_update_params_t params = {};
 +    ASSERT_EQ(rt_isula_update(nullptr, nullptr, nullptr), -1);
 +
 +    ASSERT_EQ(rt_isula_update("123", nullptr, nullptr), -1);
 +    ASSERT_EQ(rt_isula_update("123", "runtime", nullptr), -1);
 +    ASSERT_EQ(rt_isula_update("123", "runtime", &params), -1);
 +}
 +
 +TEST_F(IsulaRtOpsUnitTest, test_rt_isula_pause)
 +{
 +    rt_pause_params_t params = {};
 +    ASSERT_EQ(rt_isula_pause(nullptr, nullptr, nullptr), -1);
 +
 +    ASSERT_EQ(rt_isula_pause("123", nullptr, nullptr), -1);
 +    ASSERT_EQ(rt_isula_pause("123", "runtime", nullptr), -1);
 +    ASSERT_EQ(rt_isula_pause("123", "runtime", &params), -1);
 +}
 +
 +TEST_F(IsulaRtOpsUnitTest, test_rt_isula_resume)
 +{
 +    rt_resume_params_t params = {};
 +    ASSERT_EQ(rt_isula_resume(nullptr, nullptr, nullptr), -1);
 +
 +    ASSERT_EQ(rt_isula_resume("123", nullptr, nullptr), -1);
 +    ASSERT_EQ(rt_isula_resume("123", "runtime", nullptr), -1);
 +    ASSERT_EQ(rt_isula_resume("123", "runtime", &params), -1);
 +}
 +
 +TEST_F(IsulaRtOpsUnitTest, test_rt_isula_resources_stats)
 +{
 +    rt_stats_params_t params = {};
 +    struct runtime_container_resources_stats_info stats = {};
 +
 +    ASSERT_EQ(rt_isula_resources_stats(nullptr, nullptr, nullptr, nullptr), -1);
 +
 +    ASSERT_EQ(rt_isula_resources_stats("123", nullptr, nullptr, nullptr), -1);
 +    ASSERT_EQ(rt_isula_resources_stats("123", "runtime", nullptr, nullptr), -1);
 +    ASSERT_EQ(rt_isula_resources_stats("123", "runtime", &params, nullptr), -1);
 +    params.state = "/var/run/isulad/runtime";
 +    ASSERT_EQ(rt_isula_resources_stats("123", "runtime", &params, &stats), -1);
 +}
 +
 +TEST_F(IsulaRtOpsUnitTest, test_rt_isula_kill)
 +{
 +    rt_kill_params_t kill_params = {
 +        .pid = -1,
 +    };
 +    ASSERT_EQ(rt_isula_kill(nullptr, nullptr, nullptr), -1);
 +
 +    ASSERT_EQ(rt_isula_kill("123", nullptr, nullptr), -1);
 +    ASSERT_EQ(rt_isula_kill("123", "runtime", nullptr), -1);
 +    ASSERT_EQ(rt_isula_kill("123", "runtime", &kill_params), -1);
 +}
 \ No newline at end of file
 -- 
 2.25.1
--- a/0049-add-runc-doc.patch
+++ b/0049-add-runc-doc.patch
--- a/0050-fix-isula_rt_ops_ut-bugs.patch
+++ b/0050-fix-isula_rt_ops_ut-bugs.patch
@ -1,28 +0,0 @@
 From e2382f841c7bc3215793fdd8ce29132871281810 Mon Sep 17 00:00:00 2001
 From: zhongtao <zhongtao17@huawei.com>
 Date: Wed, 22 Mar 2023 08:02:51 +0800
 Subject: [PATCH 50/53] fix isula_rt_ops_ut bugs
 Signed-off-by: zhongtao <zhongtao17@huawei.com>
 ---
 test/runtime/isula/isula_rt_ops_ut.cc | 3 +++
 1 file changed, 3 insertions(+)
 diff --git a/test/runtime/isula/isula_rt_ops_ut.cc b/test/runtime/isula/isula_rt_ops_ut.cc
 index f37e62a0..9e014ac3 100644
 --- a/test/runtime/isula/isula_rt_ops_ut.cc
 +++ b/test/runtime/isula/isula_rt_ops_ut.cc
@@ -211,7 +211,10 @@ TEST_F(IsulaRtOpsUnitTest, test_rt_isula_resources_stats)
 TEST_F(IsulaRtOpsUnitTest, test_rt_isula_kill)
 {
     rt_kill_params_t kill_params = {
 +        .signal = SIGTERM,
 +        .stop_signal = SIGKILL,
         .pid = -1,
 +        .start_time = 12345,
     };
     ASSERT_EQ(rt_isula_kill(nullptr, nullptr, nullptr), -1);
 -- 
 2.25.1
--- a/0051-refactor-remote-ro-code.patch
+++ b/0051-refactor-remote-ro-code.patch
--- a/0052-add-ci-for-runc.patch
+++ b/0052-add-ci-for-runc.patch
--- a/0053-bugfix-when-refresh-can-t-load-or-pull-images.patch
+++ b/0053-bugfix-when-refresh-can-t-load-or-pull-images.patch
@ -1,319 +0,0 @@
 From 9d6df0b3065867d5ca1a597bedb10eab5a1c9235 Mon Sep 17 00:00:00 2001
 From: "Neil.wrz" <wangrunze13@huawei.com>
 Date: Mon, 20 Mar 2023 23:47:25 -0700
 Subject: [PATCH 53/53] bugfix when refresh can't load or pull images
 Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
 ---
 src/daemon/modules/image/oci/oci_image.c      | 105 +++++++++++++++++-
 .../remote_layer_support/remote_support.c     |  34 +++++-
 .../remote_layer_support/remote_support.h     |   4 +-
 .../modules/image/oci/storage/storage.c       |   2 +-
 .../modules/image/oci/storage/storage.h       |   2 +
 5 files changed, 143 insertions(+), 4 deletions(-)
 diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c
 index fa92a861..40e9a88f 100644
 --- a/src/daemon/modules/image/oci/oci_image.c
 +++ b/src/daemon/modules/image/oci/oci_image.c
@@ -44,6 +44,39 @@
 struct oci_image_module_data g_oci_image_module_data = { 0 };
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +// intend to make remote refresh and oci ops exlusive
 +static bool g_enable_remote;
 +static pthread_rwlock_t g_remote_lock = PTHREAD_RWLOCK_INITIALIZER;
 +
 +static inline bool oci_remote_lock(pthread_rwlock_t *remote_lock, bool writable)
 +{
 +    int nret = 0;
 +
 +    if (writable) {
 +        nret = pthread_rwlock_wrlock(remote_lock);
 +    } else {
 +        nret = pthread_rwlock_rdlock(remote_lock);
 +    }
 +    if (nret != 0) {
 +        ERROR("Lock memory store failed: %s", strerror(nret));
 +        return false;
 +    }
 +
 +    return true;
 +}
 +
 +static inline void oci_remote_unlock(pthread_rwlock_t *remote_lock)
 +{
 +    int nret = 0;
 +
 +    nret = pthread_rwlock_unlock(remote_lock);
 +    if (nret != 0) {
 +        FATAL("Unlock memory store failed: %s", strerror(nret));
 +    }
 +}
 +#endif
 +
 static void free_oci_image_data(void)
 {
     free(g_oci_image_module_data.root_dir);
@@ -220,6 +253,7 @@ static int storage_module_init_helper(const isulad_daemon_configs *args)
 #ifdef ENABLE_REMOTE_LAYER_STORE
     storage_opts->enable_remote_layer = args->storage_enable_remote_layer;
 +    storage_opts->remote_lock = &g_remote_lock;
 #endif
     if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts,
@@ -303,6 +337,10 @@ int oci_init(const isulad_daemon_configs *args)
         goto out;
     }
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +    g_enable_remote = args->storage_enable_remote_layer;
 +#endif
 +
     if (storage_module_init_helper(args) != 0) {
         ret = -1;
         goto out;
@@ -321,6 +359,7 @@ void oci_exit()
 int oci_pull_rf(const im_pull_request *request, im_pull_response *response)
 {
 +    int ret = 0;
     if (request == NULL || request->image == NULL || response == NULL) {
         ERROR("Invalid NULL param");
         return -1;
@@ -331,8 +370,24 @@ int oci_pull_rf(const im_pull_request *request, im_pull_response *response)
         isulad_try_set_error_message("Invalid image name: %s", request->image);
         return -1;
     }
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +    // read lock here because pull have exclusive access against remote refresh
 +    // pull can work concurrently with other oci operations.
 +    if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
 +        ERROR("Failed to lock oci remote lock when load image");
 +        return -1;
 +    }
 +#endif
 +
 +    ret = oci_do_pull_image(request, response);
 +
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +    if (g_enable_remote) {
 +        oci_remote_unlock(&g_remote_lock);
 +    }
 +#endif
 -    return oci_do_pull_image(request, response);
 +    return ret;
 }
 int oci_prepare_rf(const im_prepare_request *request, char **real_rootfs)
@@ -441,6 +496,15 @@ int oci_rmi(const im_rmi_request *request)
         return -1;
     }
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +    // read lock here because load have exclusive access against remote refresh
 +    // load can work concurrently with other oci operations.
 +    if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
 +        ERROR("Failed to lock oci remote lock when load image");
 +        return -1;
 +    }
 +#endif
 +
     if (!util_valid_image_name(request->image.image)) {
         ERROR("Invalid image name: %s", request->image.image);
         isulad_try_set_error_message("Invalid image name: %s", request->image.image);
@@ -502,6 +566,11 @@ int oci_rmi(const im_rmi_request *request)
     }
 out:
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +    if (g_enable_remote) {
 +        oci_remote_unlock(&g_remote_lock);
 +    }
 +#endif
     free(real_image_name);
     free(image_ID);
     util_free_array_by_len(image_names, image_names_len);
@@ -527,7 +596,24 @@ int oci_import(const im_import_request *request, char **id)
         goto err_out;
     }
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +    // read lock here because import have exclusive access against remote refresh
 +    // import can work concurrently with other oci operations.
 +    if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
 +        ERROR("Failed to lock oci remote lock when load image");
 +        ret = -1;
 +        goto err_out;
 +    }
 +#endif
 +
     ret = oci_do_import(request->file, dest_name, id);
 +
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +    if (g_enable_remote) {
 +        oci_remote_unlock(&g_remote_lock);
 +    }
 +#endif
 +
     if (ret != 0) {
         goto err_out;
     }
@@ -677,7 +763,24 @@ int oci_load_image(const im_load_request *request)
         goto out;
     }
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +    // read lock here because load have exclusive access against remote refresh
 +    // load can work concurrently with other oci operations.
 +    if (g_enable_remote && !oci_remote_lock(&g_remote_lock, false)) {
 +        ERROR("Failed to lock oci remote lock when load image");
 +        ret = -1;
 +        goto out;
 +    }
 +#endif
 +
     ret = oci_do_load(request);
 +
 +#ifdef ENABLE_REMOTE_LAYER_STORE
 +    if (g_enable_remote) {
 +        oci_remote_unlock(&g_remote_lock);
 +    }
 +#endif
 +
     if (ret != 0) {
         ERROR("Failed to load image");
         goto out;
 diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
 index 3c7d0f54..7d457755 100644
 --- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
 +++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
@@ -24,10 +24,38 @@ struct supporters {
     struct remote_image_data *image_data;
     struct remote_layer_data *layer_data;
     struct remote_overlay_data *overlay_data;
 +    pthread_rwlock_t *remote_lock;
 };
 static struct supporters supporters;
 +static inline bool remote_refresh_lock(pthread_rwlock_t *remote_lock, bool writable)
 +{
 +    int nret = 0;
 +
 +    if (writable) {
 +        nret = pthread_rwlock_wrlock(remote_lock);
 +    } else {
 +        nret = pthread_rwlock_rdlock(remote_lock);
 +    }
 +    if (nret != 0) {
 +        ERROR("Lock memory store failed: %s", strerror(nret));
 +        return false;
 +    }
 +
 +    return true;
 +}
 +
 +static inline void remote_refresh_unlock(pthread_rwlock_t *remote_lock)
 +{
 +    int nret = 0;
 +
 +    nret = pthread_rwlock_unlock(remote_lock);
 +    if (nret != 0) {
 +        FATAL("Unlock memory store failed: %s", strerror(nret));
 +    }
 +}
 +
 static void *remote_refresh_ro_symbol_link(void *arg)
 {
     struct supporters *refresh_supporters = (struct supporters *)arg;
@@ -37,16 +65,18 @@ static void *remote_refresh_ro_symbol_link(void *arg)
         util_usleep_nointerupt(5 * 1000 * 1000);
         DEBUG("remote refresh start\n");
 +        remote_refresh_lock(supporters.remote_lock, true);
         remote_overlay_refresh(refresh_supporters->overlay_data);
         remote_layer_refresh(refresh_supporters->layer_data);
         remote_image_refresh(refresh_supporters->image_data);
 +        remote_refresh_unlock(supporters.remote_lock);
         DEBUG("remote refresh end\n");
     }
     return NULL;
 }
 -int remote_start_refresh_thread(void)
 +int remote_start_refresh_thread(pthread_rwlock_t *remote_lock)
 {
     int res = 0;
     pthread_t a_thread;
@@ -67,6 +97,8 @@ int remote_start_refresh_thread(void)
         goto free_out;
     }
 +    supporters.remote_lock = remote_lock;
 +
     res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters);
     if (res != 0) {
         CRIT("Thread creation failed");
 diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
 index 892a9155..30e3ebb0 100644
 --- a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
 +++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
@@ -16,6 +16,8 @@
 #ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H
 #define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H
 +#include <pthread.h>
 +
 #include "linked_list.h"
 #include "map.h"
 #include "ro_symlink_maintain.h"
@@ -64,7 +66,7 @@ void remote_overlay_refresh(struct remote_overlay_data *data);
 bool remote_overlay_layer_valid(const char *layer_id);
 // start refresh remote
 -int remote_start_refresh_thread(void);
 +int remote_start_refresh_thread(pthread_rwlock_t *remote_lock);
 // extra map utils
 char **remote_deleted_layers(const map_t *old, const map_t *new_l);
 diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c
 index f9830ac3..836ccf4d 100644
 --- a/src/daemon/modules/image/oci/storage/storage.c
 +++ b/src/daemon/modules/image/oci/storage/storage.c
@@ -1874,7 +1874,7 @@ int storage_module_init(struct storage_module_init_options *opts)
     }
 #ifdef ENABLE_REMOTE_LAYER_STORE
 -    if (opts->enable_remote_layer && remote_start_refresh_thread() != 0) {
 +    if (opts->enable_remote_layer && remote_start_refresh_thread(opts->remote_lock) != 0) {
         ERROR("Failed to start remote refresh thread");
     }
 #endif
 diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h
 index 7404ee54..df9fd761 100644
 --- a/src/daemon/modules/image/oci/storage/storage.h
 +++ b/src/daemon/modules/image/oci/storage/storage.h
@@ -18,6 +18,7 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include <stddef.h>
 +#include <pthread.h>
 #include <isula_libutils/imagetool_image.h>
 #include <isula_libutils/json_common.h>
@@ -72,6 +73,7 @@ struct storage_module_init_options {
     bool integration_check;
 #ifdef ENABLE_REMOTE_LAYER_STORE
     bool enable_remote_layer;
 +    pthread_rwlock_t *remote_lock;
 #endif
 };
 -- 
 2.25.1
--- a/iSulad.spec
+++ b/iSulad.spec
@ -1,5 +1,5 @@
-%global _version 2.1.1
+%global _version 2.1.2
-%global _release 6
+%global _release 1
 %global is_systemd 1
 %global enable_shimv2 1
 %global is_embedded 1
@ -13,60 +13,6 @@ URL:       https://gitee.com/openeuler/iSulad
 Source:    https://gitee.com/openeuler/iSulad/repository/archive/v%{version}.tar.gz
 BuildRoot: {_tmppath}/iSulad-%{version}
 Patch0001:	0001-modify-dependence-from-lcr-to-libisula.patch
 Patch0002:	0002-Add-unified-memory_swap_limit_in_bytes-fields-into-C.patch
 Patch0003:	0003-Add-macro-for-protoc-cmake.patch
 Patch0004:	0004-fix-design-typo.patch
 Patch0005:	0005-fix-cpu-rt-review-comments.patch
 Patch0006:	0006-fix-inspect.sh-failed.patch
 Patch0007:	0007-add-CRI-ContainerStats-Service.patch
 Patch0008:	0008-fix-isula-cpu-rt-CI.patch
 Patch0009:	0009-fix-cpu-rt-CI.patch
 Patch0010:	0010-fix-cpu-rt-CI.patch
 Patch0011:	0011-Bugfix-in-config-and-executor.patch
 Patch0012:	0012-fix-cpu-rt-disable-after-reboot-machine.patch
 Patch0013:	0013-fix-selinux_label_ut-timeout-and-add-timeout-for-all.patch
 Patch0014:	0014-add-retry-for-read-write.patch
 Patch0015:	0015-support-pull-image-with-digest.patch
 Patch0016:	0016-isulad-shim-support-execSync-with-timeout.patch
 Patch0017:	0017-Refine-the-commit-info.patch
 Patch0018:	0018-Refine-typo-of-word-container.patch
 Patch0019:	0019-cleancode-for-read-write.patch
 Patch0020:	0020-add-crictl-timeout-and-sync-for-CI.patch
 Patch0021:	0021-unlock-m_podsLock-if-new-failed.patch
 Patch0022:	0022-Update-CRI.patch
 Patch0023:	0023-add-cgroup-cpu-ut.patch
 Patch0024:	0024-remove-temp-variables.patch
 Patch0025:	0025-fix-read-member-error-from-struct.patch
 Patch0026:	0026-Fix-PR-runc.patch
 Patch0027:	0027-allow-the-paused-container-to-be-stopped.patch
 Patch0028:	0028-Refine.patch
 Patch0029:	0029-support-isula-update-when-runtime-is-runc.patch
 Patch0030:	0030-Refine-as-others-feedback.patch
 Patch0031:	0031-fix-CRI-SetupPod-and-TearDownPod-deadlock.patch
 Patch0032:	0032-remote-layer-store-demo.patch
 Patch0033:	0033-add-ci-for-remote-ro.patch
 Patch0034:	0034-change-sleep-to-usleep-to-avoid-lossing-of-accuracy.patch
 Patch0035:	0035-fix-compile-error-when-not-enable-remote-ro.patch
 Patch0036:	0036-adapt-to-repo-of-openeuler-url-changed.patch
 Patch0037:	0037-change-goto-branch.patch
 Patch0038:	0038-CI-not-enable-remote-ro-for-ut.patch
 Patch0039:	0039-use-auto-free-to-proc_t.patch
 Patch0040:	0040-modifying-cpurt-file-permissions.patch
 Patch0041:	0041-use-CURLOPT_XFERINFOFUNCTION-instead-of-deprecated-C.patch
 Patch0042:	0042-bugfix-remote-ro-try-add-or-remove-image-layer-twice.patch
 Patch0043:	0043-bugfix-can-t-delete-layers-under-dir-overlay-layers.patch
 Patch0044:	0044-doc-add-document-about-support-remote-ro-directory.patch
 Patch0045:	0045-Refine-a-minor-log-message.patch
 Patch0046:	0046-modify-the-return-value-of-the-util_waitpid_with_tim.patch
 Patch0047:	0047-fix-util_getgrent_r-overflow.patch
 Patch0048:	0048-add-ut-for-runc.patch
 Patch0049:	0049-add-runc-doc.patch
 Patch0050:	0050-fix-isula_rt_ops_ut-bugs.patch
 Patch0051:	0051-refactor-remote-ro-code.patch
 Patch0052:	0052-add-ci-for-runc.patch
 Patch0053:	0053-bugfix-when-refresh-can-t-load-or-pull-images.patch
 %ifarch x86_64 aarch64
 Provides:       libhttpclient.so()(64bit)
 Provides:       libisula_client.so()(64bit)
@ -133,15 +79,15 @@ mkdir -p build
 cd build
 %if 0%{?enable_shimv2}
 %if %{defined openeuler}
-%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON -DENABLE_UT=ON ../
+%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON -DENABLE_UT=ON -DENABLE_GRPC_REMOTE_ACCESS=ON ../
 %else
-%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON ../
+%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_SHIM_V2=ON -DENABLE_GRPC_REMOTE_ACCESS=ON ../
 %endif
 %else
 %if %{defined openeuler}
-%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_UT=ON ../
+%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_UT=ON -DENABLE_GRPC_REMOTE_ACCESS=ON ../
 %else
-%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr ../
+%cmake -DDEBUG=ON -DCMAKE_SKIP_RPATH=TRUE -DLIB_INSTALL_DIR=%{_libdir} -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_GRPC_REMOTE_ACCESS=ON ../
 %endif
 %endif
 %make_build
@ -308,6 +254,12 @@ fi
 %endif
 %changelog
 * Thu May 11 2023 zhangxiaoyu <zhangxiaoyu58@huawei.com> - 2.1.2-1
 - Type: bugfix
 - ID: NA
 - SUG: NA
 - DESC: update from upstream to update cri
 * Fri Mar 24 2023 wangrunze <wangrunze13@huawei.com> - 2.1.1-6
 - Type: bugfix
 - ID: NA
--- a/v2.1.2.tar.gz
+++ b/v2.1.2.tar.gz