diff --git a/0055-retry-call-runtime-ops.patch b/0055-retry-call-runtime-ops.patch new file mode 100644 index 0000000..f2e6bb8 --- /dev/null +++ b/0055-retry-call-runtime-ops.patch @@ -0,0 +1,139 @@ +From 3ffaa7ae39d1af1d7c5aae976f1d6017ad76dee9 Mon Sep 17 00:00:00 2001 +From: haozi007 +Date: Wed, 30 Nov 2022 17:33:16 +0800 +Subject: [PATCH 55/65] retry call runtime ops + +1. add retry macro; +2. retry call runtime cleanup ops; + +Signed-off-by: haozi007 +--- + src/daemon/modules/plugin/plugin.c | 21 +++---------------- + .../modules/runtime/isula/isula_rt_ops.c | 14 +++++++++++-- + src/utils/cutils/utils.c | 1 + + src/utils/cutils/utils.h | 19 +++++++++++++++++ + 4 files changed, 35 insertions(+), 20 deletions(-) + +diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c +index 725bca5b..18035518 100644 +--- a/src/daemon/modules/plugin/plugin.c ++++ b/src/daemon/modules/plugin/plugin.c +@@ -71,7 +71,7 @@ + // suffix is '.sock' + #define PLUGIN_SOCKET_FILE_SUFFIX_LEN 5 + +-#define PLUGIN_ACTIVATE_MAX_RETRY 3 ++#define PLUGIN_ACTIVATE_MAX_RETRY 5 + + #ifndef RestHttpHead + #define RestHttpHead "http://localhost" +@@ -403,22 +403,6 @@ out: + return -1; + } + +-static int pm_activate_plugin_with_retry(plugin_t *plugin, size_t retry) +-{ +- size_t i = 0; +- int err = 0; +- +- for (i = 0; i < retry; i++) { +- err = pm_activate_plugin(plugin); +- if (!err) { +- return 0; +- } +- sleep((unsigned int)i + 1); +- } +- +- return err; +-} +- + static void pm_rdlock(void) + { + int errcode; +@@ -500,7 +484,8 @@ static int pm_register_plugin(const char *name, const char *addr) + ERROR("alloc plugin failed"); + goto failed; + } +- err = pm_activate_plugin_with_retry(plugin, PLUGIN_ACTIVATE_MAX_RETRY); ++ ++ DO_RETYR_CALL(PLUGIN_ACTIVATE_MAX_RETRY, 1000000, err, pm_activate_plugin, plugin); + if (err != 0) { + ERROR("active plugin failed"); + goto failed; +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index c9667ee5..bfe7de08 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -946,6 +946,7 @@ int rt_isula_restart(const char *name, const char *runtime, const rt_restart_par + int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_params_t *params) + { + char workdir[PATH_MAX] = { 0 }; ++ int nret; + + if (id == NULL || runtime == NULL || params == NULL) { + ERROR("nullptr arguments not allowed"); +@@ -966,8 +967,17 @@ int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_ + shim_kill_force(workdir); + } + +- (void)runtime_call_kill_force(workdir, runtime, id); +- (void)runtime_call_delete_force(workdir, runtime, id); ++ // retry 10 count call runtime kill, every call sleep 1s ++ DO_RETYR_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id); ++ if (nret != 0) { ++ WARN("call runtime force kill failed"); ++ } ++ ++ // retry 10 count call runtime delete, every call sleep 1s ++ DO_RETYR_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id); ++ if (nret != 0) { ++ WARN("call runtime force delete failed"); ++ } + + if (util_recursive_rmdir(workdir, 0) != 0) { + ERROR("failed rmdir -r shim workdir"); +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index 9f5deaf9..a154c52a 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -1216,6 +1216,7 @@ void util_usleep_nointerupt(unsigned long usec) + request = remain; + } while (ret == -1 && errno == EINTR); + } ++ + int util_generate_random_str(char *id, size_t len) + { + int fd = -1; +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 27cfc902..0a9535a1 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -381,6 +381,25 @@ defs_map_string_object * dup_map_string_empty_object(defs_map_string_object *src + + int convert_v2_runtime(const char *runtime, char *binary); + ++/** ++ * retry_cnt: max count of call cb; ++ * interval_us: how many us to sleep, after call cb; ++ * cb: retry call function; ++ * return: ++ * 0 is cb successful at least once; ++ * 1 is all cb are failure; ++*/ ++#define DO_RETYR_CALL(retry_cnt, interval_us, ret, cb, ...) do { \ ++ size_t i = 0; \ ++ for(; i < retry_cnt; i++) { \ ++ ret = cb(__VA_ARGS__); \ ++ if (ret == 0) { \ ++ break; \ ++ } \ ++ util_usleep_nointerupt(interval_us); \ ++ } \ ++ } while(0) ++ + #ifdef __cplusplus + } + #endif +-- +2.25.1 + diff --git a/0056-add-ut-test-for-retry-macro.patch b/0056-add-ut-test-for-retry-macro.patch new file mode 100644 index 0000000..f66b457 --- /dev/null +++ b/0056-add-ut-test-for-retry-macro.patch @@ -0,0 +1,50 @@ +From 61a06b548e05edb3892eb08e1028ef71b41ee332 Mon Sep 17 00:00:00 2001 +From: haozi007 +Date: Fri, 2 Dec 2022 15:35:52 +0800 +Subject: [PATCH 56/65] add ut test for retry macro + +Signed-off-by: haozi007 +--- + test/cutils/utils_utils/utils_utils_ut.cc | 27 +++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + +diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc +index 531947d2..5bd98d47 100644 +--- a/test/cutils/utils_utils/utils_utils_ut.cc ++++ b/test/cutils/utils_utils/utils_utils_ut.cc +@@ -269,4 +269,31 @@ TEST(utils_utils, test_convert_v2_runtime) + ASSERT_EQ(convert_v2_runtime(nullptr, buff), -1); + ASSERT_EQ(convert_v2_runtime(valid_str.c_str(), nullptr), -1); + ASSERT_EQ(convert_v2_runtime(valid_str.c_str(), buff), 0); ++} ++ ++int global_total = 0; ++int retry_call_test(int success_idx) { ++ if (global_total == success_idx) { ++ return 0; ++ } ++ global_total++; ++ return -1; ++} ++ ++TEST(utils_utils, test_do_retry_call) ++{ ++ int nret; ++ ++ global_total = 0; ++ DO_RETYR_CALL(10, 100, nret, retry_call_test, 0); ++ ASSERT_EQ(nret, 0); ++ ASSERT_EQ(global_total, 0); ++ global_total = 0; ++ DO_RETYR_CALL(10, 100, nret, retry_call_test, 5); ++ ASSERT_EQ(nret, 0); ++ ASSERT_EQ(global_total, 5); ++ global_total = 0; ++ DO_RETYR_CALL(10, 100, nret, retry_call_test, 11); ++ ASSERT_EQ(global_total, 10); ++ ASSERT_EQ(nret, -1); + } +\ No newline at end of file +-- +2.25.1 + diff --git a/0057-1749-set-inspect_container-timeout.patch b/0057-1749-set-inspect_container-timeout.patch new file mode 100644 index 0000000..83f73f7 --- /dev/null +++ b/0057-1749-set-inspect_container-timeout.patch @@ -0,0 +1,98 @@ +From 3e51dc0746c5e0692f54cf54d7aa6beb3b13d799 Mon Sep 17 00:00:00 2001 +From: ger202 +Date: Wed, 7 Dec 2022 03:21:17 +0000 +Subject: [PATCH 57/65] !1749 set inspect_container timeout * update function + inspect_container timeout + +--- + src/cmd/isula/information/top.c | 4 +++- + src/cmd/isula/stream/attach.c | 5 ++++- + src/cmd/isula/stream/exec.c | 4 +++- + src/daemon/entry/cri/cri_helpers.cc | 3 +-- + src/utils/cutils/utils.h | 3 +++ + 5 files changed, 14 insertions(+), 5 deletions(-) + +diff --git a/src/cmd/isula/information/top.c b/src/cmd/isula/information/top.c +index 603f97ba..e55ec43b 100644 +--- a/src/cmd/isula/information/top.c ++++ b/src/cmd/isula/information/top.c +@@ -29,7 +29,9 @@ + const char g_cmd_top_desc[] = "Display the running processes of a container"; + const char g_cmd_top_usage[] = "top [OPTIONS] CONTAINER [ps OPTIONS]"; + +-struct client_arguments g_cmd_top_args = {}; ++struct client_arguments g_cmd_top_args = { ++ .time = INSPECT_TIMEOUT_SEC, ++}; + static void client_top_info_server(const struct isula_top_response *response) + { + size_t i; +diff --git a/src/cmd/isula/stream/attach.c b/src/cmd/isula/stream/attach.c +index f260a76e..7fb1046a 100644 +--- a/src/cmd/isula/stream/attach.c ++++ b/src/cmd/isula/stream/attach.c +@@ -48,7 +48,9 @@ sem_t g_attach_waitopen_sem; + sem_t g_attach_waitexit_sem; + #endif + +-struct client_arguments g_cmd_attach_args = { 0 }; ++struct client_arguments g_cmd_attach_args = { ++ .time = INSPECT_TIMEOUT_SEC, ++}; + + + static int check_tty(bool tty, struct termios *oldtios, bool *reset_tty) +@@ -116,6 +118,7 @@ int inspect_container(const struct client_arguments *args, container_inspect **i + } + + inspect_request.name = args->name; ++ inspect_request.timeout = args->time; + ops = get_connect_client_ops(); + if (ops == NULL || !ops->container.inspect) { + COMMAND_ERROR("Unimplemented ops"); +diff --git a/src/cmd/isula/stream/exec.c b/src/cmd/isula/stream/exec.c +index 2d0d37da..bd8bd49a 100644 +--- a/src/cmd/isula/stream/exec.c ++++ b/src/cmd/isula/stream/exec.c +@@ -42,7 +42,9 @@ const char g_cmd_exec_usage[] = "exec [OPTIONS] CONTAINER COMMAND [ARG...]"; + sem_t g_command_waitopen_sem; + sem_t g_command_waitexit_sem; + +-struct client_arguments g_cmd_exec_args = {}; ++struct client_arguments g_cmd_exec_args = { ++ .time = INSPECT_TIMEOUT_SEC, ++}; + + static int fill_exec_request(const struct client_arguments *args, const struct command_fifo_config *fifos, + struct isula_exec_request *request) +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index 7df759e1..64cea7ba 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -720,8 +720,7 @@ out: + auto InspectContainer(const std::string &Id, Errors &err, bool with_host_config) -> container_inspect * + { + container_inspect *inspect_data { nullptr }; +- +- inspect_data = inspect_container((const char *)Id.c_str(), 0, with_host_config); ++ inspect_data = inspect_container((const char *)Id.c_str(), INSPECT_TIMEOUT_SEC, with_host_config); + if (inspect_data == nullptr) { + err.Errorf("Failed to call inspect service %s", Id.c_str()); + } +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 0a9535a1..4518e3ac 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -123,6 +123,9 @@ int malloc_trim(size_t pad); + + #define TIME_STR_SIZE 512 + ++// client inspect container timeout ++#define INSPECT_TIMEOUT_SEC 120 ++ + // native umask value + #define ANNOTATION_UMAKE_KEY "native.umask" + #define UMASK_NORMAL "normal" +-- +2.25.1 + diff --git a/0058-1757-add-adaption-code-for-musl.patch b/0058-1757-add-adaption-code-for-musl.patch new file mode 100644 index 0000000..4854425 --- /dev/null +++ b/0058-1757-add-adaption-code-for-musl.patch @@ -0,0 +1,364 @@ +From 484852b127dab5f5548ed34d5bb668b18e4dc99e Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 9 Dec 2022 07:01:31 +0000 +Subject: [PATCH 58/65] !1757 add adaption code for musl. * add adaption code + for musl. + +--- + cmake/checker.cmake | 2 +- + cmake/options.cmake | 12 ++++++++++++ + cmake/set_build_flags.cmake | 5 +++++ + src/CMakeLists.txt | 6 +++--- + src/daemon/common/selinux_label.c | 2 +- + src/daemon/common/sysinfo.c | 4 ++-- + src/daemon/executor/container_cb/execution_stream.c | 2 +- + src/daemon/modules/image/CMakeLists.txt | 2 +- + src/daemon/modules/image/image.c | 6 ++++-- + src/daemon/modules/image/image_rootfs_handler.c | 12 ++++++------ + src/utils/cutils/CMakeLists.txt | 2 +- + src/utils/cutils/utils.c | 13 +++++++++---- + src/utils/cutils/utils.h | 2 +- + src/utils/cutils/utils_verify.c | 8 ++++---- + 14 files changed, 51 insertions(+), 27 deletions(-) + +diff --git a/cmake/checker.cmake b/cmake/checker.cmake +index cbcfc929..fea4f925 100644 +--- a/cmake/checker.cmake ++++ b/cmake/checker.cmake +@@ -54,7 +54,7 @@ find_library(CRYPTO_LIBRARY crypto + HINTS ${PC_CRYPTO_LIBDIR} ${PC_LIBCRYPTO_LIBRARY_DIRS}) + _CHECK(CRYPTO_LIBRARY "CRYPTO_LIBRARY-NOTFOUND" "libcrypto.so") + +-if (ANDROID) ++if (ANDROID OR MUSL) + # check libssl + find_library(LIBSSL_LIBRARY ssl) + _CHECK(CRYPTO_LIBRARY "LIBSSL_LIBRARY-NOTFOUND" "libssl.so") +diff --git a/cmake/options.cmake b/cmake/options.cmake +index 7a141fef..0d44868a 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -115,6 +115,18 @@ if (ENABLE_SUP_GROUPS) + message("${Green}-- Enable sup groups${ColourReset}") + endif() + ++option(MUSL "available for musl" OFF) ++if (MUSL) ++ add_definitions(-D__MUSL__) ++ message("${Green}-- Available for MUSL${ColourReset}") ++endif() ++ ++option(ANDROID "available for android" OFF) ++if (ANDROID) ++ add_definitions(-D__ANDROID__) ++ message("${Green}-- Available for ANDROID${ColourReset}") ++endif() ++ + if (NOT RUNPATH) + set(RUNPATH "/var/run") + endif() +diff --git a/cmake/set_build_flags.cmake b/cmake/set_build_flags.cmake +index fa6f38c0..89c9468c 100644 +--- a/cmake/set_build_flags.cmake ++++ b/cmake/set_build_flags.cmake +@@ -9,6 +9,11 @@ endif() + set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-E -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -Wtrampolines -shared -pthread") + set(CMAKE_EXE_LINKER_FLAGS "-Wl,-E -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -Wtrampolines -pie -rdynamic") + ++if (NOT DISABLE_WERROR) ++ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") ++endif() ++ + if (ISULAD_GCOV) + set(CMAKE_C_FLAGS_DEBUG "-Wall -fprofile-arcs -ftest-coverage") + set(CMAKE_CXX_FLAGS_DEBUG "-Wall -fprofile-arcs -ftest-coverage") +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 65bcb978..f3dd3c19 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -102,7 +102,7 @@ add_executable(isula + ) + target_include_directories(isula PUBLIC ${ISULA_INCS} ${SHARED_INCS}) + target_link_libraries(isula libisula ${LIBYAJL_LIBRARY}) +-if (ANDROID) ++if (ANDROID OR MUSL) + target_link_libraries(isula ${LIBSSL_LIBRARY}) + else() + target_link_libraries(isula -lpthread) +@@ -115,7 +115,7 @@ add_executable(isulad-shim + ) + target_include_directories(isulad-shim PUBLIC ${ISULAD_SHIM_INCS} ${SHARED_INCS}) + target_link_libraries(isulad-shim libisulad_tools) +-if (ANDROID) ++if (ANDROID OR MUSL) + target_link_libraries(isulad-shim ${LIBSSL_LIBRARY} ${LIBYAJL_LIBRARY}) + else() + target_link_libraries(isulad-shim -lpthread) +@@ -181,7 +181,7 @@ else() + endif() + + target_link_libraries(isulad libisulad_tools libhttpclient -ldl) +-if (ANDROID) ++if (ANDROID OR MUSL) + target_link_libraries(isulad ${LIBSSL_LIBRARY} ${LIBYAJL_LIBRARY}) + else() + target_link_libraries(isulad -lpthread) +diff --git a/src/daemon/common/selinux_label.c b/src/daemon/common/selinux_label.c +index 145e4b6e..24294780 100644 +--- a/src/daemon/common/selinux_label.c ++++ b/src/daemon/common/selinux_label.c +@@ -592,7 +592,7 @@ static int container_label(char **process_label, char **file_label) + return 0; + } + +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + lxc_path = ISULAD_DAEMON_CONTAINER_CONTEXTS; + #else + lxc_path = selinux_lxc_contexts_path(); +diff --git a/src/daemon/common/sysinfo.c b/src/daemon/common/sysinfo.c +index d52f8767..e7b3807b 100644 +--- a/src/daemon/common/sysinfo.c ++++ b/src/daemon/common/sysinfo.c +@@ -1308,7 +1308,7 @@ out: + return ret; + } + +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + static bool cgroup2_no_controller() + { + char *controllers_str = NULL; +@@ -1336,7 +1336,7 @@ static int make_sure_cgroup2_isulad_path_exist() + return -1; + } + +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + if (cgroup2_no_controller()) { + DEBUG("no cgroup controller found"); + return 0; +diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c +index 9af6fb5c..ebb9ee2b 100644 +--- a/src/daemon/executor/container_cb/execution_stream.c ++++ b/src/daemon/executor/container_cb/execution_stream.c +@@ -63,7 +63,7 @@ + #include "utils_file.h" + #include "utils_verify.h" + +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + #define SIG_CANCEL_SIGNAL SIGUSR1 + #define PTHREAD_CANCEL_ENABLE 1 + #define PTHREAD_CANCEL_DISABLE 0 +diff --git a/src/daemon/modules/image/CMakeLists.txt b/src/daemon/modules/image/CMakeLists.txt +index 329d2937..6f9e9936 100644 +--- a/src/daemon/modules/image/CMakeLists.txt ++++ b/src/daemon/modules/image/CMakeLists.txt +@@ -117,7 +117,7 @@ target_link_libraries(${LIB_ISULAD_IMG} + ${ZLIB_LIBRARY} + libhttpclient) + +-if (NOT ANDROID) ++if ((NOT ANDROID) AND (NOT MUSL)) + target_link_libraries(${LIB_ISULAD_IMG} -lpthread) + endif() + +diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c +index fb0db361..8fb226aa 100644 +--- a/src/daemon/modules/image/image.c ++++ b/src/daemon/modules/image/image.c +@@ -1773,7 +1773,8 @@ int im_container_export(const im_export_request *request) + #endif + + #ifdef ENABLE_OCI_IMAGE +-char *im_get_rootfs_dir(const im_get_rf_dir_request *request) { ++char *im_get_rootfs_dir(const im_get_rf_dir_request *request) ++{ + char *dir = NULL; + struct bim *bim = NULL; + +@@ -1801,7 +1802,8 @@ out: + return dir; + } + #else +-char *im_get_rootfs_dir(const im_get_rf_dir_request *request) { ++char *im_get_rootfs_dir(const im_get_rf_dir_request *request) ++{ + return NULL; + } + #endif +diff --git a/src/daemon/modules/image/image_rootfs_handler.c b/src/daemon/modules/image/image_rootfs_handler.c +index a76363d0..c3964b2c 100644 +--- a/src/daemon/modules/image/image_rootfs_handler.c ++++ b/src/daemon/modules/image/image_rootfs_handler.c +@@ -87,7 +87,7 @@ static int proc_by_fpasswd(FILE *f_passwd, const char *user, defs_process_user * + struct passwd *pwbufp = NULL; + + if (f_passwd != NULL) { +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + errval = util_getpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp); + #else + errval = fgetpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp); +@@ -105,7 +105,7 @@ static int proc_by_fpasswd(FILE *f_passwd, const char *user, defs_process_user * + *matched_username = util_strdup_s(pwbufp->pw_name); + break; + } +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + errval = util_getpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp); + #else + errval = fgetpwent_r(f_passwd, &pw, buf, sizeof(buf), &pwbufp); +@@ -215,7 +215,7 @@ static int do_proc_by_froup(FILE *f_group, const char *group, defs_process_user + return 0; + } + +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + errval = util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); + #else + errval = fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); +@@ -226,7 +226,7 @@ static int do_proc_by_froup(FILE *f_group, const char *group, defs_process_user + if (search_group_list(gbufp, matched_username, puser) != 0) { + return -1; + } +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + errval = util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); + #else + errval = fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); +@@ -240,7 +240,7 @@ static int do_proc_by_froup(FILE *f_group, const char *group, defs_process_user + puser->gid = gbufp->gr_gid; + *groupcnt = 1; + } +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + errval = util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); + #else + errval = fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp); +@@ -378,7 +378,7 @@ static int get_additional_groups(char **additional_groups, size_t additional_gro + struct group *gbufp = NULL; + struct group *groups = NULL; + +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + while (f_group != NULL && util_getgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp) == 0) { + #else + while (f_group != NULL && fgetgrent_r(f_group, &grp, buf, sizeof(buf), &gbufp) == 0) { +diff --git a/src/utils/cutils/CMakeLists.txt b/src/utils/cutils/CMakeLists.txt +index 30414d91..50a17f60 100644 +--- a/src/utils/cutils/CMakeLists.txt ++++ b/src/utils/cutils/CMakeLists.txt +@@ -2,7 +2,7 @@ + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_cutils_srcs) + add_subdirectory(map) + +-if (NOT ANDROID) ++if ((NOT ANDROID) AND (NOT MUSL)) + list(REMOVE_ITEM local_cutils_srcs "${CMAKE_CURRENT_SOURCE_DIR}/utils_pwgr.c") + endif() + +diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c +index a154c52a..b1db045a 100644 +--- a/src/utils/cutils/utils.c ++++ b/src/utils/cutils/utils.c +@@ -16,7 +16,7 @@ + #define _GNU_SOURCE + #include "utils.h" + #include +-#ifndef __ANDROID__ ++#if !defined (__ANDROID__) && !defined(__MUSL__) + #include + #endif + #include +@@ -47,7 +47,7 @@ + #include "utils_string.h" + #include "utils_verify.h" + +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + int mallopt(int param, int value) + { + return 1; +@@ -427,7 +427,12 @@ proc_t *util_stat2proc(const char *s, size_t len) + + /* parse these two strings separately, skipping the leading "(". */ + /* comm[16] in kernel */ ++ /* https://www.openwall.com/lists/musl/2013/11/15/5: musl's sscanf("%15c",cmd) requires exactly 15 characters; anything shorter is a matching failure. */ ++#ifdef __MUSL__ ++ num = sscanf(s, "%d (%15s", &p->pid, p->cmd); ++#else + num = sscanf(s, "%d (%15c", &p->pid, p->cmd); ++#endif + if (num != 2) { + ERROR("Call sscanf error: %s", errno ? strerror(errno) : ""); + free(p); +@@ -851,8 +856,8 @@ out: + + char **util_get_backtrace(void) + { +-#ifdef __ANDROID__ +- /* android has no backtrace */ ++#if defined (__ANDROID__) || defined(__MUSL__) ++ /* android and musl has no backtrace */ + return NULL; + #else + #define BACKTRACE_SIZE 16 +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 4518e3ac..72cab9f2 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -47,7 +47,7 @@ + extern "C" { + #endif + +-#ifdef __ANDROID__ ++#if defined (__ANDROID__) || defined(__MUSL__) + #define M_TRIM_THRESHOLD -1 + #define M_TOP_PAD -2 + #define M_MMAP_THRESHOLD -3 +diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c +index 9ed33bf3..713e72c3 100644 +--- a/src/utils/cutils/utils_verify.c ++++ b/src/utils/cutils/utils_verify.c +@@ -563,7 +563,7 @@ bool util_valid_value_false(const char *value) + + bool util_valid_rw_mode(const char *mode) + { +- if (mode == NULL){ ++ if (mode == NULL) { + return false; + } + +@@ -572,7 +572,7 @@ bool util_valid_rw_mode(const char *mode) + + bool util_valid_label_mode(const char *mode) + { +- if (mode == NULL){ ++ if (mode == NULL) { + return false; + } + +@@ -581,7 +581,7 @@ bool util_valid_label_mode(const char *mode) + + bool util_valid_copy_mode(const char *mode) + { +- if (mode == NULL){ ++ if (mode == NULL) { + return false; + } + return !strcmp(mode, "nocopy"); +@@ -708,7 +708,7 @@ int util_valid_env(const char *env, char **dst) + int ret = 0; + char *value = NULL; + +- if (dst == NULL){ ++ if (dst == NULL) { + ERROR("NULL dst"); + return -1; + } +-- +2.25.1 + diff --git a/0059-deleting-broken-rootfs.patch b/0059-deleting-broken-rootfs.patch new file mode 100644 index 0000000..4e713b7 --- /dev/null +++ b/0059-deleting-broken-rootfs.patch @@ -0,0 +1,880 @@ +From edb570b8720aed234bf2c17642f5e6caba2c726a Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Tue, 6 Dec 2022 19:23:33 -0800 +Subject: [PATCH 59/65] deleting broken rootfs + +Signed-off-by: Neil.wrz +--- + cmake/options.cmake | 6 ++ + src/cmd/isulad/main.c | 15 ++++ + src/daemon/modules/api/image_api.h | 2 + + src/daemon/modules/api/leftover_cleanup_api.h | 36 +++++++++ + src/daemon/modules/container/CMakeLists.txt | 2 + + src/daemon/modules/container/container_unix.c | 2 - + .../leftover_cleanup/clean_context.c | 78 +++++++++++++++++++ + .../leftover_cleanup/clean_context.h | 41 ++++++++++ + .../container/leftover_cleanup/cleanup.c | 46 +++++------ + .../container/leftover_cleanup/cleanup.h | 12 ++- + .../leftover_cleanup/leftover_cleanup_api.c | 71 +++++++++++++++++ + .../leftover_cleanup/oci_rootfs_clean.c | 34 +++++++- + .../leftover_cleanup/oci_rootfs_clean.h | 4 +- + src/daemon/modules/image/image.c | 48 ++++++++++++ + src/daemon/modules/image/oci/oci_image.c | 10 +++ + src/daemon/modules/image/oci/oci_image.h | 1 + + .../oci/storage/layer_store/layer_store.c | 2 + + .../oci/storage/rootfs_store/rootfs_store.c | 13 +++- + .../modules/image/oci/storage/storage.c | 32 ++++++++ + .../modules/image/oci/storage/storage.h | 3 + + test/image/oci/storage/rootfs/CMakeLists.txt | 2 + + 21 files changed, 422 insertions(+), 38 deletions(-) + create mode 100644 src/daemon/modules/api/leftover_cleanup_api.h + create mode 100644 src/daemon/modules/container/leftover_cleanup/clean_context.c + create mode 100644 src/daemon/modules/container/leftover_cleanup/clean_context.h + create mode 100644 src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c + +diff --git a/cmake/options.cmake b/cmake/options.cmake +index 0d44868a..1e63a485 100644 +--- a/cmake/options.cmake ++++ b/cmake/options.cmake +@@ -115,6 +115,12 @@ if (ENABLE_SUP_GROUPS) + message("${Green}-- Enable sup groups${ColourReset}") + endif() + ++option(DISABLE_CLEANUP "disable cleanup module" OFF) ++if (DISABLE_CLEANUP STREQUAL "ON") ++ add_definitions(-DDISABLE_CLEANUP) ++ message("${Green}-- Disable cleanup module") ++endif() ++ + option(MUSL "available for musl" OFF) + if (MUSL) + add_definitions(-D__MUSL__) +diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c +index d4f984d5..9b664bee 100644 +--- a/src/cmd/isulad/main.c ++++ b/src/cmd/isulad/main.c +@@ -73,6 +73,9 @@ + #include "utils_string.h" + #include "utils_verify.h" + #include "volume_api.h" ++#ifndef DISABLE_CLEANUP ++#include "leftover_cleanup_api.h" ++#endif + #include "opt_log.h" + + #ifdef GRPC_CONNECTOR +@@ -1235,6 +1238,14 @@ static int isulad_server_init_common() + goto out; + } + ++#ifndef DISABLE_CLEANUP ++ // to cleanup leftover, init clean module before other modules. ++ if (clean_module_init() != 0) { ++ ERROR("Failed to init clean module"); ++ goto out; ++ } ++#endif ++ + if (volume_init(args->json_confs->graph) != 0) { + ERROR("Failed to init volume"); + goto out; +@@ -1451,6 +1462,10 @@ static int start_daemon_threads(char **msg) + goto out; + } + ++#ifndef DISABLE_CLEANUP ++ clean_module_do_clean(); ++#endif ++ + ret = 0; + out: + return ret; +diff --git a/src/daemon/modules/api/image_api.h b/src/daemon/modules/api/image_api.h +index a1c6084a..b8d7fd5b 100644 +--- a/src/daemon/modules/api/image_api.h ++++ b/src/daemon/modules/api/image_api.h +@@ -244,6 +244,8 @@ int im_umount_container_rootfs(const char *image_type, const char *image_name, c + + int im_remove_container_rootfs(const char *image_type, const char *container_id); + ++int im_remove_broken_rootfs(const char *image_type, const char *container_id); ++ + int im_merge_image_config(const char *image_type, const char *image_name, container_config *container_spec); + + int im_get_user_conf(const char *image_type, const char *basefs, host_config *hc, const char *userstr, +diff --git a/src/daemon/modules/api/leftover_cleanup_api.h b/src/daemon/modules/api/leftover_cleanup_api.h +new file mode 100644 +index 00000000..32a41258 +--- /dev/null ++++ b/src/daemon/modules/api/leftover_cleanup_api.h +@@ -0,0 +1,36 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2022-12-7 ++ * Description: provide cleanup functions ++ *********************************************************************************/ ++#ifndef DAEMON_MODULES_API_LEFTOVER_CLEANUP_API_H ++#define DAEMON_MODULES_API_LEFTOVER_CLEANUP_API_H ++ ++#if defined(__cplusplus) || defined(c_plusplus) ++extern "C" { ++#endif ++ ++typedef enum { ++ BROKEN_ROOTFS = 1 ++} cleanup_ctx_data_t; ++ ++int clean_module_init(); ++ ++void clean_module_fill_ctx(cleanup_ctx_data_t data_type, void *data); ++ ++void clean_module_do_clean(); ++ ++#if defined(__cplusplus) || defined(c_plusplus) ++} ++#endif ++ ++#endif +diff --git a/src/daemon/modules/container/CMakeLists.txt b/src/daemon/modules/container/CMakeLists.txt +index def602c7..38c3d88b 100644 +--- a/src/daemon/modules/container/CMakeLists.txt ++++ b/src/daemon/modules/container/CMakeLists.txt +@@ -5,7 +5,9 @@ add_subdirectory(supervisor) + add_subdirectory(health_check) + add_subdirectory(container_gc) + add_subdirectory(restart_manager) ++IF (NOT DISABLE_CLEANUP) + add_subdirectory(leftover_cleanup) ++ENDIF() + + set(MANAGER_SRCS + ${local_manager_srcs} +diff --git a/src/daemon/modules/container/container_unix.c b/src/daemon/modules/container/container_unix.c +index 88c4bf51..9910b3c8 100644 +--- a/src/daemon/modules/container/container_unix.c ++++ b/src/daemon/modules/container/container_unix.c +@@ -46,7 +46,6 @@ + #include "utils_string.h" + #include "volume_api.h" + #include "namespace.h" +-#include "cleanup.h" + + static int parse_container_log_configs(container_t *cont); + +@@ -1279,7 +1278,6 @@ int container_module_init(char **msg) + } + + containers_restore(); +- clean_leftover(); + + if (start_gchandler()) { + *msg = "Failed to start garbage collecotor handler"; +diff --git a/src/daemon/modules/container/leftover_cleanup/clean_context.c b/src/daemon/modules/container/leftover_cleanup/clean_context.c +new file mode 100644 +index 00000000..6ccc39ed +--- /dev/null ++++ b/src/daemon/modules/container/leftover_cleanup/clean_context.c +@@ -0,0 +1,78 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2022-10-31 ++ * Description: provide cleanup definition ++ *********************************************************************************/ ++#include "clean_context.h" ++#include "linked_list.h" ++#include "utils.h" ++#include "isula_libutils/log.h" ++ ++struct clean_ctx *clean_ctx_init() ++{ ++ struct clean_ctx *ctx = util_common_calloc_s(sizeof(struct clean_ctx)); ++ if (ctx == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ linked_list_init(&(ctx->broken_rootfs_list)); ++ ctx->inited = true; ++ ++ return ctx; ++} ++ ++void clean_ctx_destroy(struct clean_ctx *ctx) ++{ ++ struct linked_list *it = NULL; ++ struct linked_list *next = NULL; ++ char *id = NULL; ++ ++ if (ctx == NULL) { ++ return; ++ } ++ ++ if (!ctx->inited) { ++ free(ctx); ++ return; ++ } ++ ++ linked_list_for_each_safe(it, &(ctx->broken_rootfs_list), next) { ++ id = (char *)it->elem; ++ linked_list_del(it); ++ free(id); ++ free(it); ++ it = NULL; ++ } ++ ++ free(ctx); ++} ++ ++void clean_ctx_fill_broken_rootfs(struct clean_ctx *ctx, const char *id) ++{ ++ struct linked_list *new_node = NULL; ++ char *broken_id = NULL; ++ ++ if (!ctx->inited) { ++ return; ++ } ++ ++ new_node = util_common_calloc_s(sizeof(struct linked_list)); ++ if (new_node == NULL) { ++ ERROR("Out of memory, broken rootfs %s not added", id); ++ return; ++ } ++ ++ broken_id = util_strdup_s(id); ++ linked_list_add_elem(new_node, broken_id); ++ linked_list_add_tail(&ctx->broken_rootfs_list, new_node); ++} +\ No newline at end of file +diff --git a/src/daemon/modules/container/leftover_cleanup/clean_context.h b/src/daemon/modules/container/leftover_cleanup/clean_context.h +new file mode 100644 +index 00000000..fcac8df8 +--- /dev/null ++++ b/src/daemon/modules/container/leftover_cleanup/clean_context.h +@@ -0,0 +1,41 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2022-10-31 ++ * Description: provide cleanup definition ++ *********************************************************************************/ ++#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEAN_CONTEXT_H ++#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEAN_CONTEXT_H ++ ++#include "linked_list.h" ++#include "utils.h" ++#include "isula_libutils/log.h" ++ ++#if defined(__cplusplus) || defined(c_plusplus) ++extern "C" { ++#endif ++ ++struct clean_ctx { ++ bool inited; ++ struct linked_list broken_rootfs_list; ++}; ++ ++struct clean_ctx *clean_ctx_init(); ++ ++void clean_ctx_destroy(struct clean_ctx *ctx); ++ ++void clean_ctx_fill_broken_rootfs(struct clean_ctx *ctx, const char *id); ++ ++#if defined(__cplusplus) || defined(c_plusplus) ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c +index 9ce1dd0c..664988b5 100644 +--- a/src/daemon/modules/container/leftover_cleanup/cleanup.c ++++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c +@@ -31,12 +31,16 @@ static struct cleaners *create_cleaners() + return ret; + } + +-static void destroy_cleaners(struct cleaners *clns) ++void destroy_cleaners(struct cleaners *clns) + { + struct linked_list *it = NULL; + struct linked_list *next = NULL; + struct clean_node *c_node = NULL; + ++ if (clns == NULL) { ++ return; ++ } ++ + linked_list_for_each_safe(it, &(clns->cleaner_list), next) { + c_node = (struct clean_node *)it->elem; + linked_list_del(it); +@@ -80,25 +84,31 @@ static int default_cleaner() + return 0; + } + +-static struct cleaners *cleaner_init() ++struct cleaners *cleaners_init() + { + int ret = 0; + struct cleaners *clns = create_cleaners(); +- ++ + if (clns == NULL) { + return NULL; + } + + ret = add_clean_node(clns, default_cleaner, "default clean"); + if (ret != 0) { +- ERROR("add default_cleaner error"); ++ ERROR("Add default_cleaner error"); + return clns; + } + + #ifdef ENABLE_OCI_IMAGE ++ ret = add_clean_node(clns, oci_broken_rootfs_cleaner, "clean broken rootfs"); ++ if (ret != 0) { ++ ERROR("Clean broken rootfs failed"); ++ return clns; ++ } ++ + ret = add_clean_node(clns, oci_rootfs_cleaner, "clean rootfs"); + if (ret != 0) { +- ERROR("add oci_rootfs_cleaner error"); ++ ERROR("Add oci_rootfs_cleaner error"); + return clns; + } + #endif +@@ -106,7 +116,7 @@ static struct cleaners *cleaner_init() + return clns; + } + +-static void do_clean(struct cleaners * clns) ++void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx) + { + struct linked_list *it = NULL; + struct linked_list *next = NULL; +@@ -114,31 +124,11 @@ static void do_clean(struct cleaners * clns) + + linked_list_for_each_safe(it, &(clns->cleaner_list), next) { + c_node = (struct clean_node *)it->elem; +- if (c_node->cleaner() != 0) { +- ERROR("failed to clean for: %s", c_node->desc); ++ if (c_node->cleaner(ctx) != 0) { ++ ERROR("Failed to clean for: %s", c_node->desc); + } else { + DEBUG("do clean success for: %s", c_node->desc); + clns->done_clean++; + } + } + } +- +-void clean_leftover() +-{ +- struct cleaners *clns = cleaner_init(); +- +- if (clns == NULL) { +- ERROR("failed to clean leftovers, because cleaner init error"); +- return; +- } +- +- do_clean(clns); +- +- if (clns->count == clns->done_clean) { +- DEBUG("all clean up success"); +- } else { +- ERROR("Aim to do %d clean, %d clean sucess\n", clns->count, clns->done_clean); +- } +- +- destroy_cleaners(clns); +-} +\ No newline at end of file +diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.h b/src/daemon/modules/container/leftover_cleanup/cleanup.h +index efae99d0..8dd5e9bd 100644 +--- a/src/daemon/modules/container/leftover_cleanup/cleanup.h ++++ b/src/daemon/modules/container/leftover_cleanup/cleanup.h +@@ -12,19 +12,20 @@ + * Create: 2022-10-31 + * Description: provide cleanup definition + *********************************************************************************/ +-#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_H +-#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_H ++#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEANERS_H ++#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEANERS_H + + #include + + #include "linked_list.h" + #include "isula_libutils/log.h" ++#include "clean_context.h" + + #if defined(__cplusplus) || defined(c_plusplus) + extern "C" { + #endif + +-typedef int clean_func_t(void); ++typedef int clean_func_t(struct clean_ctx *ctx); + + struct clean_node { + const char *desc; +@@ -38,8 +39,11 @@ struct cleaners { + struct linked_list cleaner_list; + }; + ++struct cleaners *cleaners_init(); + +-void clean_leftover(); ++void destroy_cleaners(struct cleaners *clns); ++ ++void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx); + + + #if defined(__cplusplus) || defined(c_plusplus) +diff --git a/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c +new file mode 100644 +index 00000000..7bdaef22 +--- /dev/null ++++ b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c +@@ -0,0 +1,71 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: wangrunze ++ * Create: 2022-10-31 ++ * Description: provide cleanup functions ++ *********************************************************************************/ ++#include "utils.h" ++#include "leftover_cleanup_api.h" ++#include "cleanup.h" ++#include "clean_context.h" ++ ++struct clean_ctx *g_clean_ctx = NULL; ++struct cleaners *g_clns = NULL; ++ ++int clean_module_init() ++{ ++ // create cleaners and clean_ctx ++ g_clns = cleaners_init(); ++ if (g_clns == NULL) { ++ ERROR("Failed to init clean module"); ++ return -1; ++ } ++ ++ g_clean_ctx = clean_ctx_init(); ++ if (g_clean_ctx == NULL) { ++ ERROR("Failed to init clean module"); ++ destroy_cleaners(g_clns); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++void clean_module_fill_ctx(cleanup_ctx_data_t data_type, void *data) ++{ ++ switch (data_type) { ++ case BROKEN_ROOTFS: ++ clean_ctx_fill_broken_rootfs(g_clean_ctx, data); ++ break; ++ } ++} ++ ++void clean_module_do_clean() ++{ ++ if (g_clns == NULL || g_clean_ctx == NULL) { ++ return; ++ } ++ ++ cleaners_do_clean(g_clns, g_clean_ctx); ++ ++ if (g_clns->count == g_clns->done_clean) { ++ DEBUG("all clean up success"); ++ } else { ++ ERROR("Aim to do %d clean, %d clean sucess\n", g_clns->count, g_clns->done_clean); ++ } ++ ++ destroy_cleaners(g_clns); ++ clean_ctx_destroy(g_clean_ctx); ++ ++ g_clns = NULL; ++ g_clean_ctx = NULL; ++} ++ +diff --git a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c +index fbef4ce0..b2205569 100644 +--- a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c ++++ b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c +@@ -19,6 +19,7 @@ + #include "image_api.h" + #include "utils_file.h" + #include "utils.h" ++#include "linked_list.h" + + struct cb_result { + int clean_err_cnt; +@@ -45,7 +46,7 @@ static bool walk_dir_cb(const char *path_name, const struct dirent *sub_dir, voi + } + + +-int oci_rootfs_cleaner(void) ++int oci_rootfs_cleaner(struct clean_ctx *ctx) + { + struct cb_result res = { 0 }; + im_get_rf_dir_request request = { 0 }; +@@ -67,7 +68,36 @@ int oci_rootfs_cleaner(void) + + if (res.clean_err_cnt == 0) { + return 0; +- } ++ } + + return -1; + } ++ ++int oci_broken_rootfs_cleaner(struct clean_ctx *ctx) ++{ ++ int rm_fail_cnt = 0; ++ struct linked_list *it = NULL; ++ struct linked_list *next = NULL; ++ char *id = NULL; ++ ++ if (ctx == NULL) { ++ return -1; ++ } ++ ++ linked_list_for_each_safe(it, &(ctx->broken_rootfs_list), next) { ++ id = (char *)it->elem; ++ if (im_remove_broken_rootfs(IMAGE_TYPE_OCI, id) != 0) { ++ ERROR("Failed to clean broken rootfs %s", id); ++ rm_fail_cnt++; ++ } else { ++ EVENT("clean broken rootfs succeed %s", id); ++ } ++ } ++ ++ if (rm_fail_cnt != 0) { ++ DEBUG("can't clean some broken rootfs, %d left", rm_fail_cnt); ++ return -1; ++ } ++ ++ return 0; ++} +\ No newline at end of file +diff --git a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h +index 8dff351f..88920fa9 100644 +--- a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h ++++ b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h +@@ -21,7 +21,9 @@ + extern "C" { + #endif + +-int oci_rootfs_cleaner(void); ++int oci_rootfs_cleaner(struct clean_ctx *ctx); ++ ++int oci_broken_rootfs_cleaner(struct clean_ctx *ctx); + + #if defined(__cplusplus) || defined(c_plusplus) + } +diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c +index 8fb226aa..6fac7cc4 100644 +--- a/src/daemon/modules/image/image.c ++++ b/src/daemon/modules/image/image.c +@@ -50,6 +50,7 @@ struct bim_ops { + int (*export_rf)(const im_export_request *request); + char *(*resolve_image_name)(const char *image_name); + char *(*get_dir_rf)(void); ++ int (*delete_broken_rf)(const im_delete_rootfs_request *request); + + /* merge image config ops */ + int (*merge_conf)(const char *img_name, container_config *container_spec); +@@ -132,6 +133,7 @@ static const struct bim_ops g_embedded_ops = { + .mount_rf = embedded_mount_rf, + .umount_rf = embedded_umount_rf, + .delete_rf = embedded_delete_rf, ++ .delete_broken_rf = NULL, + .export_rf = NULL, + .get_dir_rf = NULL, + +@@ -167,6 +169,7 @@ static const struct bim_ops g_oci_ops = { + .mount_rf = oci_mount_rf, + .umount_rf = oci_umount_rf, + .delete_rf = oci_delete_rf, ++ .delete_broken_rf = oci_delete_broken_rf, + .export_rf = oci_export_rf, + .get_dir_rf = oci_get_dir_rf, + +@@ -201,6 +204,7 @@ static const struct bim_ops g_ext_ops = { + .mount_rf = ext_mount_rf, + .umount_rf = ext_umount_rf, + .delete_rf = ext_delete_rf, ++ .delete_broken_rf = NULL, + .export_rf = NULL, + .get_dir_rf = NULL, + +@@ -464,6 +468,50 @@ out: + return ret; + } + ++int im_remove_broken_rootfs(const char *image_type, const char *container_id) ++{ ++ int ret = 0; ++ im_delete_rootfs_request *request = NULL; ++ struct bim *bim = NULL; ++ ++ if (container_id == NULL || image_type == NULL) { ++ ERROR("Invalid input arguments"); ++ return -1; ++ } ++ ++ bim = bim_get(image_type, NULL, NULL, container_id); ++ if (bim == NULL) { ++ ERROR("Failed to init bim when deleting broken rootfs %s", container_id); ++ return -1; ++ } ++ ++ if (bim->ops->delete_broken_rf == NULL) { ++ ERROR("Unimplements delete in %s", bim->type); ++ ret = -1; ++ goto out; ++ } ++ ++ request = util_common_calloc_s(sizeof(im_delete_rootfs_request)); ++ if (request == NULL) { ++ ERROR("Out of memory"); ++ ret = -1; ++ goto out; ++ } ++ request->name_id = util_strdup_s(container_id); ++ ++ ret = bim->ops->delete_broken_rf(request); ++ if (ret != 0) { ++ ERROR("Failed to delete rootfs for container %s", container_id); ++ ret = -1; ++ goto out; ++ } ++ ++out: ++ bim_put(bim); ++ free_im_delete_request(request); ++ return ret; ++} ++ + int im_remove_container_rootfs(const char *image_type, const char *container_id) + { + int ret = 0; +diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c +index e951adb4..0fc0b7fe 100644 +--- a/src/daemon/modules/image/oci/oci_image.c ++++ b/src/daemon/modules/image/oci/oci_image.c +@@ -368,6 +368,16 @@ out: + return ret; + } + ++int oci_delete_broken_rf(const im_delete_rootfs_request *request) ++{ ++ if (request == NULL) { ++ ERROR("Request is NULL"); ++ return -1; ++ } ++ ++ return storage_broken_rw_layer_delete(request->name_id); ++} ++ + int oci_delete_rf(const im_delete_rootfs_request *request) + { + if (request == NULL) { +diff --git a/src/daemon/modules/image/oci/oci_image.h b/src/daemon/modules/image/oci/oci_image.h +index aeeb3b65..cd7da336 100644 +--- a/src/daemon/modules/image/oci/oci_image.h ++++ b/src/daemon/modules/image/oci/oci_image.h +@@ -53,6 +53,7 @@ int oci_merge_conf_rf(const char *img_name, container_config *container_spec); + int oci_mount_rf(const im_mount_request *request); + int oci_umount_rf(const im_umount_request *request); + int oci_delete_rf(const im_delete_rootfs_request *request); ++int oci_delete_broken_rf(const im_delete_rootfs_request *request); + int oci_export_rf(const im_export_request *request); + char *oci_get_dir_rf(void); + int oci_container_filesystem_usage(const im_container_fs_usage_request *request, imagetool_fs_info **fs_usage); +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index e563a8ef..b9ab0d65 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -557,6 +557,7 @@ static int update_layer_datas(const char *id, const struct layer_opts *opts, lay + + slayer->id = util_strdup_s(id); + slayer->parent = util_strdup_s(opts->parent); ++ slayer->writable = opts->writable; + if (opts->opts != NULL) { + slayer->mountlabel = util_strdup_s(opts->opts->mount_label); + } +@@ -1434,6 +1435,7 @@ static void copy_json_to_layer(const layer_t *jl, struct layer *l) + l->mount_point = util_strdup_s(jl->smount_point->path); + l->mount_count = jl->smount_point->count; + } ++ l->writable = jl->slayer->writable; + } + + int layer_store_list(struct layer_list *resp) +diff --git a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c +index 97cc39e8..c6e305a0 100644 +--- a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c ++++ b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c +@@ -38,6 +38,10 @@ + #include "utils_string.h" + #include "utils_timestamp.h" + ++#ifndef DISABLE_CLEANUP ++#include "leftover_cleanup_api.h" ++#endif ++ + #define CONTAINER_JSON "container.json" + + typedef struct rootfs_store { +@@ -170,6 +174,7 @@ static int append_container_by_directory(const char *container_dir) + + nret = snprintf(container_path, sizeof(container_path), "%s/%s", container_dir, CONTAINER_JSON); + if (nret < 0 || (size_t)nret >= sizeof(container_path)) { ++ // snprintf error, not append, but outside should not delete the rootfs + ERROR("Failed to get container path"); + return -1; + } +@@ -182,6 +187,7 @@ static int append_container_by_directory(const char *container_dir) + } + + if (do_append_container(c) != 0) { ++ // append error should not return -1, outside should not remove rootfs + ERROR("Failed to append container"); + ret = -1; + goto out; +@@ -199,6 +205,7 @@ static int get_containers_from_json() + { + int ret = 0; + int nret; ++ int append_ret = 0; + char **container_dirs = NULL; + size_t container_dirs_num = 0; + size_t i; +@@ -231,7 +238,11 @@ static int get_containers_from_json() + continue; + } + +- if (append_container_by_directory(container_path) != 0) { ++ append_ret = append_container_by_directory(container_path); ++ if (append_ret != 0) { ++#ifndef DISABLE_CLEANUP ++ clean_module_fill_ctx(BROKEN_ROOTFS, (void *)container_dirs[i]); ++#endif + ERROR("Found container path but load json failed: %s, deleting...", container_path); + if (util_recursive_rmdir(container_path, 0) != 0) { + ERROR("Failed to delete rootfs directory : %s", container_path); +diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c +index 6cb4a51b..829ea8d0 100644 +--- a/src/daemon/modules/image/oci/storage/storage.c ++++ b/src/daemon/modules/image/oci/storage/storage.c +@@ -306,6 +306,38 @@ struct layer *storage_layer_get(const char *layer_id) + return layer_store_lookup(layer_id); + } + ++int storage_broken_rw_layer_delete(const char *layer_id) ++{ ++ int ret = 0; ++ struct layer *layer_info = NULL; ++ ++ if (layer_id == NULL) { ++ return -1; ++ } ++ ++ layer_info = layer_store_lookup(layer_id); ++ if (layer_info == NULL) { ++ ERROR("Failed to get layer info for layer %s", layer_id); ++ return -1; ++ } ++ ++ if (!layer_info->writable) { ++ ERROR("Broken rootfs should only delete rw layer, layer %s is ro layer", layer_id); ++ ret = -1; ++ goto out; ++ } ++ ++ // delete rootfs and rw layer, rw layer has the same name as rootfs ++ if (layer_store_delete(layer_info->id) != 0) { ++ ERROR("Can't delete layer of broken rootfs"); ++ ret = -1; ++ } ++ ++out: ++ free_layer(layer_info); ++ return ret; ++} ++ + void free_layer(struct layer *ptr) + { + if (ptr == NULL) { +diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h +index 5914adec..3ec47959 100644 +--- a/src/daemon/modules/image/oci/storage/storage.h ++++ b/src/daemon/modules/image/oci/storage/storage.h +@@ -42,6 +42,7 @@ struct layer { + int64_t compress_size; + char *uncompressed_digest; + int64_t uncompress_size; ++ bool writable; + }; + + struct layer_list { +@@ -163,6 +164,8 @@ struct layer_list *storage_layers_get_by_compress_digest(const char *digest); + + struct layer *storage_layer_get(const char *layer_id); + ++int storage_broken_rw_layer_delete(const char *layer_id); ++ + int storage_layer_try_repair_lowers(const char *layer_id, const char *last_layer_id); + + void free_layer(struct layer *l); +diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt +index 8da8196b..5bf568f9 100644 +--- a/test/image/oci/storage/rootfs/CMakeLists.txt ++++ b/test/image/oci/storage/rootfs/CMakeLists.txt +@@ -2,6 +2,8 @@ project(iSulad_UT) + + SET(EXE storage_rootfs_ut) + ++add_definitions(-DDISABLE_CLEANUP) ++ + add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_regex.c +-- +2.25.1 + diff --git a/0060-1761-fix-leftover-devicemapper-mnt-dir.patch b/0060-1761-fix-leftover-devicemapper-mnt-dir.patch new file mode 100644 index 0000000..b824328 --- /dev/null +++ b/0060-1761-fix-leftover-devicemapper-mnt-dir.patch @@ -0,0 +1,175 @@ +From 9b41a8d28d3ffbe33eb84d8e254b603012b22b34 Mon Sep 17 00:00:00 2001 +From: DriedYellowPeach +Date: Sun, 11 Dec 2022 11:12:55 +0000 +Subject: [PATCH 60/65] !1761 fix leftover devicemapper mnt dir * fix leftover + devicemapper mnt dir + +--- + .../graphdriver/devmapper/deviceset.c | 37 +++++++++++++ + .../graphdriver/devmapper/deviceset.h | 2 + + .../graphdriver/devmapper/driver_devmapper.c | 55 ++++++++++++------- + .../oci/storage/layer_store/layer_store.c | 11 +++- + 4 files changed, 84 insertions(+), 21 deletions(-) + +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c +index 4dadc336..b157510a 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.c +@@ -486,6 +486,43 @@ out: + return exist; + } + ++// return true if find the metadata ++// or the argument is wrong ++// or can't decide ++bool has_metadata(const char *hash, struct device_set *devset) ++{ ++ char metadata_file[PATH_MAX] = { 0 }; ++ char *metadata_path = NULL; ++ bool ret = true; ++ int nret = 0; ++ ++ if (hash == NULL) { ++ return true; ++ } ++ ++ metadata_path = metadata_dir(devset); ++ if (metadata_path == NULL) { ++ ERROR("Failed to get meta data directory"); ++ goto out; ++ } ++ ++ nret = snprintf(metadata_file, sizeof(metadata_file), "%s/%s", metadata_path, util_valid_str(hash) ? hash : "base"); ++ if (nret < 0 || (size_t)nret >= sizeof(metadata_file)) { ++ ERROR("Failed to snprintf metadata file path with hash:%s, path is too long", hash); ++ goto out; ++ } ++ ++ if (!util_file_exists(metadata_file)) { ++ WARN("No such file:%s, need not to load", metadata_file); ++ ret = false; ++ goto out; ++ } ++ ++out: ++ free(metadata_path); ++ return ret; ++} ++ + static image_devmapper_device_info *load_metadata(const struct device_set *devset, const char *hash) + { + image_devmapper_device_info *info = NULL; +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.h +index c11eece6..ec985e40 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.h ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.h +@@ -83,6 +83,8 @@ int delete_device(const char *hash, bool sync_delete, struct device_set *devset) + + int export_device_metadata(struct device_metadata *dev_metadata, const char *hash, struct device_set *devset); + ++bool has_metadata(const char *hash, struct device_set *devset); ++ + struct status *device_set_status(struct device_set *devset); + + void free_devmapper_status(struct status *st); +diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c +index dd231bd6..c83d3e54 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c +@@ -144,28 +144,11 @@ int devmapper_create_ro(const char *id, const char *parent, const struct graphdr + return do_create(id, parent, driver, create_opts); + } + +-// Remove removes a device with a given id, unmounts the filesystem. +-int devmapper_rm_layer(const char *id, const struct graphdriver *driver) ++static int devmapper_try_rm_layer_mnt(const char *id, const struct graphdriver *driver) + { ++ int ret = 0; + char *mnt_parent_dir = NULL; + char *mnt_point_dir = NULL; +- int ret = 0; +- +- if (!util_valid_str(id) || driver == NULL) { +- ERROR("invalid argument"); +- return -1; +- } +- +- if (!has_device(id, driver->devset)) { +- DEBUG("Device with id:%s is not exist", id); +- goto out; +- } +- +- if (delete_device(id, false, driver->devset) != 0) { +- ERROR("failed to remove device %s", id); +- ret = -1; +- goto out; +- } + + mnt_parent_dir = util_path_join(driver->home, "mnt"); + if (mnt_parent_dir == NULL) { +@@ -193,6 +176,40 @@ out: + return ret; + } + ++// Remove removes a device with a given id, unmounts the filesystem. ++int devmapper_rm_layer(const char *id, const struct graphdriver *driver) ++{ ++ if (!util_valid_str(id) || driver == NULL) { ++ ERROR("invalid argument"); ++ return -1; ++ } ++ ++ if (!has_device(id, driver->devset)) { ++ DEBUG("Device with id:%s is not exist", id); ++ if (!has_metadata(id, driver->devset)) { ++ // this means metadata is lost ++ // if we can rm mnt, then the layer is removed ++ EVENT("try clean lost metadata and its mnt: %s", id); ++ return devmapper_try_rm_layer_mnt(id, driver); ++ } ++ // if has_metadata and not rm successfully, return -1 ++ // so next start up of isulad will retry delete the layer. ++ return -1; ++ } ++ ++ if (delete_device(id, false, driver->devset) != 0) { ++ ERROR("failed to remove device %s", id); ++ return -1; ++ } ++ ++ if (devmapper_try_rm_layer_mnt(id, driver) != 0) { ++ ERROR("failed to remove mnt dir of Device: %s", id); ++ return -1; ++ } ++ ++ return 0; ++} ++ + // devmapper_mount_layer mounts a device with given id into the root filesystem + char *devmapper_mount_layer(const char *id, const struct graphdriver *driver, + const struct driver_mount_opts *mount_opts) +diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +index b9ab0d65..60aaff22 100644 +--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c ++++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c +@@ -1813,8 +1813,15 @@ static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_d + + remove_invalid_dir: + (void)graphdriver_umount_layer(sub_dir->d_name); +- (void)graphdriver_rm_layer(sub_dir->d_name); +- (void)util_recursive_rmdir(tmpdir, 0); ++ // layer not removed successfully, we can't remove layer.json ++ if (graphdriver_rm_layer(sub_dir->d_name) != 0) { ++ ERROR("failed to rm layer: %s when handing invalid rootfs", sub_dir->d_name); ++ goto free_out; ++ } ++ ERROR("tmpdir is %s", tmpdir); ++ if (util_recursive_rmdir(tmpdir, 0) != 0) { ++ ERROR("failed to rm rootfs dir: %s when handing invalid rootfs", tmpdir); ++ } + + free_out: + free(rpath); +-- +2.25.1 + diff --git a/0061-check-file-system-ro-before-merge-network-for-syscon.patch b/0061-check-file-system-ro-before-merge-network-for-syscon.patch new file mode 100644 index 0000000..6a7b9da --- /dev/null +++ b/0061-check-file-system-ro-before-merge-network-for-syscon.patch @@ -0,0 +1,113 @@ +From fa7428d0baf2c310c852b1ece41736b21ea441f9 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Fri, 9 Dec 2022 17:26:10 +0800 +Subject: [PATCH 61/65] check file system ro before merge network for + syscontainer + +Signed-off-by: zhangxiaoyu +--- + .../executor/container_cb/execution_network.c | 29 +++++++++++++++++++ + src/utils/cutils/utils_fs.c | 27 +++++++++++++++++ + src/utils/cutils/utils_fs.h | 1 + + 3 files changed, 57 insertions(+) + +diff --git a/src/daemon/executor/container_cb/execution_network.c b/src/daemon/executor/container_cb/execution_network.c +index bbc35e80..b738d02f 100644 +--- a/src/daemon/executor/container_cb/execution_network.c ++++ b/src/daemon/executor/container_cb/execution_network.c +@@ -761,9 +761,38 @@ out: + return ret; + } + ++static int check_readonly_fs_for_etc(const char *rootfs, bool *ro) ++{ ++ char *path = NULL; ++ ++ if (util_realpath_in_scope(rootfs, "/etc", &path) < 0) { ++ SYSERROR("Failed to get real path '/etc' under rootfs '%s'", rootfs); ++ isulad_set_error_message("Failed to get real path '/etc' under rootfs '%s'", rootfs); ++ return -1; ++ } ++ ++ *ro = util_check_readonly_fs(path); ++ ++ free(path); ++ return 0; ++} ++ ++// modify network file in rootfs ++// make sure network file saved if rootfs migrate to another host + static int merge_network_for_syscontainer(const host_config *host_spec, const char *rootfs, const char *hostname) + { + int ret = 0; ++ bool ro = false; ++ ++ if (check_readonly_fs_for_etc(rootfs, &ro) != 0) { ++ ERROR("Failed to check network path"); ++ return -1; ++ } ++ ++ if (ro) { ++ WARN("Readonly filesystem for etc under %s. Skip merge network for syscontainer", rootfs); ++ return 0; ++ } + + ret = write_hostname_to_file(rootfs, hostname); + if (ret) { +diff --git a/src/utils/cutils/utils_fs.c b/src/utils/cutils/utils_fs.c +index 218e2fe8..e7165f26 100644 +--- a/src/utils/cutils/utils_fs.c ++++ b/src/utils/cutils/utils_fs.c +@@ -22,6 +22,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -620,3 +621,29 @@ child_out: + cleanup: + return ret; + } ++ ++bool util_check_readonly_fs(const char *path) ++{ ++ int i; ++ const int max_retry = 10; ++ struct statfs fsbuf; ++ ++ for (i = 0; i < max_retry; i++) { ++ if (statfs(path, &fsbuf) == 0) { ++ break; ++ } ++ if (errno == EINTR) { ++ continue; ++ } ++ ++ ERROR("Stat fs failed: %s", strerror(errno)); ++ return false; ++ } ++ ++ if (i >= max_retry) { ++ ERROR("Too much interrupted"); ++ return false; ++ } ++ ++ return (fsbuf.f_flags & ST_RDONLY) != 0; ++} +diff --git a/src/utils/cutils/utils_fs.h b/src/utils/cutils/utils_fs.h +index 6ab6b78e..6ad64a1a 100644 +--- a/src/utils/cutils/utils_fs.h ++++ b/src/utils/cutils/utils_fs.h +@@ -38,6 +38,7 @@ int util_ensure_mounted_as(const char *dst, const char *mntopts); + int util_mount_from(const char *base, const char *src, const char *dst, const char *mtype, const char *mntopts); + typedef int (*mount_info_call_back_t)(const char *, const char *); + bool util_deal_with_mount_info(mount_info_call_back_t cb, const char *); ++bool util_check_readonly_fs(const char *path); + #ifdef __cplusplus + } + #endif +-- +2.25.1 + diff --git a/0062-isulad-shim-wait-for-all-child-process.patch b/0062-isulad-shim-wait-for-all-child-process.patch new file mode 100644 index 0000000..73d8c56 --- /dev/null +++ b/0062-isulad-shim-wait-for-all-child-process.patch @@ -0,0 +1,134 @@ +From a2ed1886a091d1edfb54fdec38db277e0d0e29f5 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Tue, 13 Dec 2022 01:34:12 -0800 +Subject: [PATCH 62/65] isulad shim wait for all child process + +Signed-off-by: Neil.wrz +--- + src/cmd/isulad-shim/process.c | 18 ++++++++++++++++++ + src/daemon/modules/plugin/plugin.c | 2 +- + .../modules/runtime/isula/isula_rt_ops.c | 4 ++-- + src/utils/cutils/utils.h | 2 +- + test/cutils/utils_utils/utils_utils_ut.cc | 6 +++--- + 5 files changed, 25 insertions(+), 7 deletions(-) + +diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c +index c8ce7a44..1fc95525 100644 +--- a/src/cmd/isulad-shim/process.c ++++ b/src/cmd/isulad-shim/process.c +@@ -39,6 +39,7 @@ + #include "terminal.h" + #include "utils_array.h" + #include "utils_string.h" ++#include "utils.h" + + #define MAX_EVENTS 100 + #define DEFAULT_IO_COPY_BUF (16 * 1024) +@@ -1206,10 +1207,20 @@ out: + return ret; + } + ++static int try_wait_all_child() { ++ if (waitpid(-1, NULL, WNOHANG) == -1 && errno == ECHILD) { ++ // all child handled ++ return 0; ++ } ++ ++ return 1; ++} ++ + int process_signal_handle_routine(process_t *p) + { + int ret = SHIM_ERR; + bool exit_shim = false; ++ int nret = 0; + int i; + + for (;;) { +@@ -1235,6 +1246,13 @@ int process_signal_handle_routine(process_t *p) + } + if (exit_shim) { + process_kill_all(p); ++ ++ // wait atmost 120 seconds ++ DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child, 0); ++ if (nret != 0) { ++ write_message(g_log_fd, ERR_MSG, "Failed to wait all child after 120 seconds"); ++ } ++ + process_delete(p); + if (p->exit_fd > 0) { + (void)write_nointr(p->exit_fd, &status, sizeof(int)); +diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c +index 18035518..53afeeaf 100644 +--- a/src/daemon/modules/plugin/plugin.c ++++ b/src/daemon/modules/plugin/plugin.c +@@ -485,7 +485,7 @@ static int pm_register_plugin(const char *name, const char *addr) + goto failed; + } + +- DO_RETYR_CALL(PLUGIN_ACTIVATE_MAX_RETRY, 1000000, err, pm_activate_plugin, plugin); ++ DO_RETRY_CALL(PLUGIN_ACTIVATE_MAX_RETRY, 1000000, err, pm_activate_plugin, plugin); + if (err != 0) { + ERROR("active plugin failed"); + goto failed; +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index bfe7de08..dd1bb4e8 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -968,13 +968,13 @@ int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_ + } + + // retry 10 count call runtime kill, every call sleep 1s +- DO_RETYR_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id); ++ DO_RETRY_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id); + if (nret != 0) { + WARN("call runtime force kill failed"); + } + + // retry 10 count call runtime delete, every call sleep 1s +- DO_RETYR_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id); ++ DO_RETRY_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id); + if (nret != 0) { + WARN("call runtime force delete failed"); + } +diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h +index 72cab9f2..fec6d879 100644 +--- a/src/utils/cutils/utils.h ++++ b/src/utils/cutils/utils.h +@@ -392,7 +392,7 @@ int convert_v2_runtime(const char *runtime, char *binary); + * 0 is cb successful at least once; + * 1 is all cb are failure; + */ +-#define DO_RETYR_CALL(retry_cnt, interval_us, ret, cb, ...) do { \ ++#define DO_RETRY_CALL(retry_cnt, interval_us, ret, cb, ...) do { \ + size_t i = 0; \ + for(; i < retry_cnt; i++) { \ + ret = cb(__VA_ARGS__); \ +diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc +index 5bd98d47..c8f38717 100644 +--- a/test/cutils/utils_utils/utils_utils_ut.cc ++++ b/test/cutils/utils_utils/utils_utils_ut.cc +@@ -285,15 +285,15 @@ TEST(utils_utils, test_do_retry_call) + int nret; + + global_total = 0; +- DO_RETYR_CALL(10, 100, nret, retry_call_test, 0); ++ DO_RETRY_CALL(10, 100, nret, retry_call_test, 0); + ASSERT_EQ(nret, 0); + ASSERT_EQ(global_total, 0); + global_total = 0; +- DO_RETYR_CALL(10, 100, nret, retry_call_test, 5); ++ DO_RETRY_CALL(10, 100, nret, retry_call_test, 5); + ASSERT_EQ(nret, 0); + ASSERT_EQ(global_total, 5); + global_total = 0; +- DO_RETYR_CALL(10, 100, nret, retry_call_test, 11); ++ DO_RETRY_CALL(10, 100, nret, retry_call_test, 11); + ASSERT_EQ(global_total, 10); + ASSERT_EQ(nret, -1); + } +\ No newline at end of file +-- +2.25.1 + diff --git a/0063-When-run-options-rm-is-set-delete-the-stoped-contain.patch b/0063-When-run-options-rm-is-set-delete-the-stoped-contain.patch new file mode 100644 index 0000000..fe29a39 --- /dev/null +++ b/0063-When-run-options-rm-is-set-delete-the-stoped-contain.patch @@ -0,0 +1,59 @@ +From 33fb34f3c864161fb24de77b72e157327e1e620c Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Thu, 15 Dec 2022 15:55:34 +0800 +Subject: [PATCH 63/65] When run options rm is set, delete the stoped + container's fifo directory. + +Signed-off-by: zhongtao +--- + src/cmd/isula/base/run.c | 27 +++++++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + +diff --git a/src/cmd/isula/base/run.c b/src/cmd/isula/base/run.c +index 73486c95..24863120 100644 +--- a/src/cmd/isula/base/run.c ++++ b/src/cmd/isula/base/run.c +@@ -42,6 +42,29 @@ struct client_arguments g_cmd_run_args = { + .pull = "missing" + }; + ++/* ++ * --rm option will result in no time to delete the directory created by the client, resulting in residue. ++ * This function is used to delete the client's fifo file and home directory. ++ */ ++static void delete_client_fifo_and_home_dir(const char *rundir, const char *name, ++ struct command_fifo_config *console_fifos) ++{ ++ char client_fifo_home_dir[PATH_MAX] = { 0 }; ++ int nret = 0; ++ ++ nret = snprintf(client_fifo_home_dir, sizeof(client_fifo_home_dir), "%s/%s/", rundir, name); ++ if (nret < 0 || (size_t)nret >= sizeof(client_fifo_home_dir)) { ++ ERROR("Client fifo home path:%s/%s/ is too long.", rundir, name); ++ return; ++ } ++ ++ delete_command_fifo(console_fifos); ++ ++ if (util_recursive_rmdir(client_fifo_home_dir, 0)) { ++ WARN("Failed to delete client fifo home path:%s", client_fifo_home_dir); ++ } ++} ++ + static int local_cmd_start(const struct client_arguments *args) + { + int ret = 0; +@@ -66,6 +89,10 @@ static int local_cmd_start(const struct client_arguments *args) + client_wait_fifo_exit(args); + + free_out: ++ if (args->custom_conf.auto_remove && !args->detach) { ++ delete_client_fifo_and_home_dir(CLIENT_RUNDIR, args->name, console_fifos); ++ console_fifos = NULL; ++ } + client_restore_console(reset_tty, &oldtios, console_fifos); + return ret; + } +-- +2.25.1 + diff --git a/0064-recheck-kill-command-exit-status.patch b/0064-recheck-kill-command-exit-status.patch new file mode 100644 index 0000000..c0be2de --- /dev/null +++ b/0064-recheck-kill-command-exit-status.patch @@ -0,0 +1,153 @@ +From a8c344ad4400ed876aaa4f53f7c992a7ad1c1580 Mon Sep 17 00:00:00 2001 +From: "Neil.wrz" +Date: Wed, 14 Dec 2022 20:49:34 -0800 +Subject: [PATCH 64/65] recheck kill command exit status + +Signed-off-by: Neil.wrz +--- + .../modules/runtime/isula/isula_rt_ops.c | 70 +++++++++++++++---- + 1 file changed, 57 insertions(+), 13 deletions(-) + +diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c +index dd1bb4e8..dee57263 100644 +--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c ++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c +@@ -54,6 +54,9 @@ + #define RESIZE_DATA_SIZE 100 + #define PID_WAIT_TIME 120 + ++// handle string from stderr output. ++typedef int(*handle_output_callback_t)(const char *output); ++ + static void copy_process(shim_client_process_state *p, defs_process *dp) + { + p->args = dp->args; +@@ -584,7 +587,7 @@ out: + } + + static int runtime_call_simple(const char *workdir, const char *runtime, const char *subcmd, const char **opts, +- size_t opts_len, const char *id) ++ size_t opts_len, const char *id, handle_output_callback_t cb) + { + runtime_exec_info rei = { 0 }; + char *stdout = NULL; +@@ -596,24 +599,65 @@ static int runtime_call_simple(const char *workdir, const char *runtime, const c + if (!util_exec_cmd(runtime_exec_func, &rei, NULL, &stdout, &stderr)) { + ERROR("call runtime %s failed stderr %s", subcmd, stderr); + ret = -1; +- goto out; ++ // additional handler for the stderr, ++ // this intend to change the ret val of this function ++ // for example, if output string contains some specific content, ++ // we consider the runtime call simple succeeded, ++ // even if the process exit with failure. ++ if (stderr != NULL && cb != NULL) { ++ ret = cb(stderr); ++ } + } + +-out: + UTIL_FREE_AND_SET_NULL(stdout); + UTIL_FREE_AND_SET_NULL(stderr); + return ret; + } + +-static int runtime_call_kill_force(const char *workdir, const char *runtime, const char *id) ++// oci runtime return -1 if the container 'does not exist' ++// if output contains 'does not exist', means nothing to kill, return 0 ++// this will change the exit status of kill command ++static int kill_output_check(const char *output) + { +- return runtime_call_simple(workdir, runtime, "kill", NULL, 0, id); ++ char *pattern = "does not exist"; ++ ++ if (output == NULL) { ++ return -1; ++ } ++ ++ // container not exist, kill success, return 0 ++ if (util_strings_contains_word(output, pattern)) { ++ return 0; ++ } ++ ++ // kill failed, return -1 ++ return -1; ++} ++ ++// kill success or kill_output_check succeed return 0, DO_RETRY_CALL will break; ++// if kill failed, recheck on shim alive, if not alive, kill succeed, still return 0; ++// else, return -1, DO_RETRY_CALL will call this again; ++static int runtime_call_kill_and_check(const char *workdir, const char *runtime, const char *id) ++{ ++ int ret = -1; ++ ++ // kill succeed, return 0; kill_output_check succeed, return 0; ++ ret = runtime_call_simple(workdir, runtime, "kill", NULL, 0, id, kill_output_check); ++ if (ret == 0) { ++ return 0; ++ } ++ ++ if (!shim_alive(workdir)) { ++ ret = 0; ++ } ++ ++ return ret; + } + + static int runtime_call_delete_force(const char *workdir, const char *runtime, const char *id) + { + const char *opts[1] = { "--force" }; +- return runtime_call_simple(workdir, runtime, "delete", opts, 1, id); ++ return runtime_call_simple(workdir, runtime, "delete", opts, 1, id, NULL); + } + + #define ExitSignalOffset 128 +@@ -919,7 +963,7 @@ int rt_isula_start(const char *id, const char *runtime, const rt_start_params_t + pid_info->ppid = shim_pid; + pid_info->pstart_time = p_proc->start_time; + +- if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id) != 0) { ++ if (runtime_call_simple(workdir, runtime, "start", NULL, 0, id, NULL) != 0) { + ERROR("call runtime start id failed"); + ret = -1; + goto out; +@@ -967,14 +1011,14 @@ int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_ + shim_kill_force(workdir); + } + +- // retry 10 count call runtime kill, every call sleep 1s +- DO_RETRY_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id); ++ // retry 10 count call runtime kill, every call sleep 0.5s ++ DO_RETRY_CALL(10, 500000, nret, runtime_call_kill_and_check, workdir, runtime, id); + if (nret != 0) { + WARN("call runtime force kill failed"); + } + +- // retry 10 count call runtime delete, every call sleep 1s +- DO_RETRY_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id); ++ // retry 10 count call runtime delete, every call sleep 0.1s ++ DO_RETRY_CALL(10, 100000, nret, runtime_call_delete_force, workdir, runtime, id); + if (nret != 0) { + WARN("call runtime force delete failed"); + } +@@ -1204,7 +1248,7 @@ int rt_isula_pause(const char *id, const char *runtime, const rt_pause_params_t + return -1; + } + +- return runtime_call_simple(workdir, runtime, "pause", NULL, 0, id); ++ return runtime_call_simple(workdir, runtime, "pause", NULL, 0, id, NULL); + } + + int rt_isula_resume(const char *id, const char *runtime, const rt_resume_params_t *params) +@@ -1221,7 +1265,7 @@ int rt_isula_resume(const char *id, const char *runtime, const rt_resume_params_ + return -1; + } + +- return runtime_call_simple(workdir, runtime, "resume", NULL, 0, id); ++ return runtime_call_simple(workdir, runtime, "resume", NULL, 0, id, NULL); + } + + int rt_isula_listpids(const char *name, const char *runtime, const rt_listpids_params_t *params, rt_listpids_out_t *out) +-- +2.25.1 + diff --git a/0065-start-sandbox-before-setup-network-by-default.patch b/0065-start-sandbox-before-setup-network-by-default.patch new file mode 100644 index 0000000..84ab494 --- /dev/null +++ b/0065-start-sandbox-before-setup-network-by-default.patch @@ -0,0 +1,430 @@ +From a13e021620c62f32dfb1fd5242a3cf43c1d163b8 Mon Sep 17 00:00:00 2001 +From: zhangxiaoyu +Date: Wed, 14 Dec 2022 10:50:49 +0800 +Subject: [PATCH 65/65] start sandbox before setup network by default + +Signed-off-by: zhangxiaoyu +--- + src/daemon/entry/cri/cni_network_plugin.cc | 43 +++--- + src/daemon/entry/cri/cni_network_plugin.h | 2 + + src/daemon/entry/cri/cri_helpers.cc | 12 ++ + src/daemon/entry/cri/cri_helpers.h | 3 + + .../cri_pod_sandbox_manager_service_impl.cc | 135 +++++++++++------- + .../cri_pod_sandbox_manager_service_impl.h | 2 +- + src/daemon/entry/cri/cri_security_context.cc | 3 +- + src/daemon/modules/spec/specs.c | 2 +- + src/utils/cutils/utils_file.c | 2 +- + 9 files changed, 135 insertions(+), 69 deletions(-) + +diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc +index c850bc32..976a21a4 100644 +--- a/src/daemon/entry/cri/cni_network_plugin.cc ++++ b/src/daemon/entry/cri/cni_network_plugin.cc +@@ -479,6 +479,27 @@ cleanup: + return result; + } + ++auto CniNetworkPlugin::GetNetNSPath(const std::string &id, const std::map &annotations, ++ Errors &err) -> std::string ++{ ++ std::string netnsPath; ++ ++ if (CRIHelpers::SetupNetworkFront(annotations)) { ++ auto iter = annotations.find(CRIHelpers::Constants::POD_SANDBOX_KEY); ++ if (iter == annotations.end()) { ++ ERROR("Failed to find sandbox key from annotations"); ++ return netnsPath; ++ } ++ return iter->second; ++ } ++ ++ netnsPath = GetNetNS(id, err); ++ if (err.NotEmpty()) { ++ ERROR("CNI failed to retrieve network namespace path: %s", err.GetCMessage()); ++ } ++ ++ return netnsPath; ++} + + void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, + const std::string &id, const std::map &annotations, +@@ -489,12 +510,7 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, + return; + } + +- auto iter = annotations.find(CRIHelpers::Constants::POD_SANDBOX_KEY); +- if (iter == annotations.end()) { +- ERROR("Failed to find sandbox key from annotations"); +- return; +- } +- const std::string netnsPath = iter->second; ++ std::string netnsPath = GetNetNSPath(id, annotations, err); + if (netnsPath.length() == 0) { + ERROR("Failed to get network namespace path"); + return; +@@ -600,21 +616,14 @@ void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &nam + } + Errors tmpErr; + +- auto iter = annotations.find(CRIHelpers::Constants::POD_SANDBOX_KEY); +- if (iter == annotations.end()) { +- ERROR("Failed to find sandbox key from annotations"); +- return; +- } +- std::string netnsPath = iter->second; +- if (netnsPath.length() == 0) { +- ERROR("Failed to get network namespace path"); +- return; +- } ++ std::string netnsPath = GetNetNSPath(id, annotations, err); + + // When netns file does not exist, netnsPath is assigned to an + // empty string so that lxc can handle the path properly +- if (!util_file_exists(netnsPath.c_str())) { ++ if (!util_file_exists(netnsPath.c_str()) || err.NotEmpty()) { ++ ERROR("Failed to get network namespace path, maybe podsandbox '%s' has been stopped", id.c_str()); + netnsPath = ""; ++ err.Clear(); + } + + RLockNetworkMap(err); +diff --git a/src/daemon/entry/cri/cni_network_plugin.h b/src/daemon/entry/cri/cni_network_plugin.h +index 8d51a94d..434222b5 100644 +--- a/src/daemon/entry/cri/cni_network_plugin.h ++++ b/src/daemon/entry/cri/cni_network_plugin.h +@@ -118,6 +118,8 @@ public: + private: + auto GetNetNS(const std::string &podSandboxID, Errors &err) -> std::string; + ++ auto GetNetNSPath(const std::string &id, const std::map &annotations, ++ Errors &err) -> std::string; + + private: + virtual void PlatformInit(Errors &error); +diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc +index 64cea7ba..711196ba 100644 +--- a/src/daemon/entry/cri/cri_helpers.cc ++++ b/src/daemon/entry/cri/cri_helpers.cc +@@ -49,6 +49,8 @@ const std::string Constants::DOCKER_IMAGEID_PREFIX { "docker://" }; + const std::string Constants::DOCKER_PULLABLE_IMAGEID_PREFIX { "docker-pullable://" }; + const std::string Constants::RUNTIME_READY { "RuntimeReady" }; + const std::string Constants::NETWORK_READY { "NetworkReady" }; ++// Kata 2.x need create network namespace and setup network befoce run podsandbox ++const std::string Constants::NETWORK_SETUP_ANNOTATION_KEY { "cri.sandbox.network.setup.v2" }; + const std::string Constants::POD_CHECKPOINT_KEY { "cri.sandbox.isulad.checkpoint" }; + const std::string Constants::CONTAINER_TYPE_ANNOTATION_KEY { "io.kubernetes.cri.container-type" }; + const std::string Constants::CONTAINER_NAME_ANNOTATION_KEY { "io.kubernetes.cri.container-name" }; +@@ -1009,4 +1011,14 @@ out: + return runtime_val; + } + ++bool SetupNetworkFront(const std::map &annotations) ++{ ++ auto iter = annotations.find(CRIHelpers::Constants::NETWORK_SETUP_ANNOTATION_KEY); ++ if (iter == annotations.end()) { ++ return false; ++ } ++ ++ return iter->second == std::string("true"); ++} ++ + } // namespace CRIHelpers +diff --git a/src/daemon/entry/cri/cri_helpers.h b/src/daemon/entry/cri/cri_helpers.h +index b3bfafe4..d50759ad 100644 +--- a/src/daemon/entry/cri/cri_helpers.h ++++ b/src/daemon/entry/cri/cri_helpers.h +@@ -48,6 +48,7 @@ public: + static const std::string DOCKER_PULLABLE_IMAGEID_PREFIX; + static const std::string RUNTIME_READY; + static const std::string NETWORK_READY; ++ static const std::string NETWORK_SETUP_ANNOTATION_KEY; + static const std::string POD_CHECKPOINT_KEY; + static const size_t MAX_CHECKPOINT_KEY_LEN { 250 }; + static const std::string CONTAINER_TYPE_ANNOTATION_KEY; +@@ -151,6 +152,8 @@ void StopContainer(service_executor_t *cb, const std::string &containerID, int64 + char *GenerateExecSuffix(); + + char *cri_runtime_convert(const char *runtime); ++ ++bool SetupNetworkFront(const std::map &annotations); + }; // namespace CRIHelpers + + #endif // DAEMON_ENTRY_CRI_CRI_HELPERS_H +diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc +index f7b3119d..4c245763 100644 +--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc ++++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc +@@ -99,10 +99,20 @@ void PodSandboxManagerServiceImpl::ApplySandboxResources(const runtime::v1alpha2 + } + + +-void PodSandboxManagerServiceImpl::SetHostConfigDefaultValue(host_config *hc) ++void PodSandboxManagerServiceImpl::SetHostConfigDefaultValue(const google::protobuf::Map &annotations, ++ host_config *hc) + { + free(hc->network_mode); +- hc->network_mode = util_strdup_s(CRI::Constants::namespaceModeFile.c_str()); ++ ++ auto iter = annotations.find(CRIHelpers::Constants::NETWORK_SETUP_ANNOTATION_KEY); ++ // set network mode file when kata 2.x ++ if (iter != annotations.end() && iter->second == std::string("true")) { ++ hc->network_mode = util_strdup_s(CRI::Constants::namespaceModeFile.c_str()); ++ return; ++ } ++ ++ // default network mode is empty string ++ hc->network_mode = util_strdup_s(""); + } + + void PodSandboxManagerServiceImpl::MakeSandboxIsuladConfig(const runtime::v1alpha2::PodSandboxConfig &c, +@@ -168,7 +178,7 @@ void PodSandboxManagerServiceImpl::MakeSandboxIsuladConfig(const runtime::v1alph + custom_config->hostname = util_strdup_s(c.hostname().c_str()); + } + +- SetHostConfigDefaultValue(hc); ++ SetHostConfigDefaultValue(c.annotations(), hc); + + if (c.has_linux()) { + ApplySandboxLinuxOptions(c.linux(), hc, custom_config, error); +@@ -487,7 +497,6 @@ void PodSandboxManagerServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2:: + { + std::map stdAnnos; + std::map networkOptions; +- char* sandbox_key = get_sandbox_key(inspect_data); + + // Setup sandbox files + if (config.has_dns_config() && inspect_data->resolv_conf_path != nullptr) { +@@ -495,32 +504,36 @@ void PodSandboxManagerServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2:: + SetupSandboxFiles(inspect_data->resolv_conf_path, config, error); + if (error.NotEmpty()) { + ERROR("failed to setup sandbox files"); +- goto cleanup; ++ return; + } + } + // Do not invoke network plugins if in hostNetwork mode. + if (config.linux().security_context().namespace_options().network() == runtime::v1alpha2::NamespaceMode::NODE) { +- goto cleanup; ++ return; + } + + // Setup networking for the sandbox. + CRIHelpers::ProtobufAnnoMapToStd(config.annotations(), stdAnnos); + stdAnnos[CRIHelpers::Constants::POD_CHECKPOINT_KEY] = jsonCheckpoint; + networkOptions["UID"] = config.metadata().uid(); +- if (sandbox_key == NULL) { +- goto cleanup; ++ ++ if (namespace_is_file(inspect_data->host_config->network_mode)) { ++ char *sandbox_key = get_sandbox_key(inspect_data); ++ if (sandbox_key == nullptr) { ++ ERROR("Empty sandbox key"); ++ error.SetError("Empty sandbox key"); ++ return; ++ } ++ stdAnnos.insert(std::pair(CRIHelpers::Constants::POD_SANDBOX_KEY, ++ std::string(sandbox_key))); ++ free(sandbox_key); + } +- stdAnnos.insert(std::pair(CRIHelpers::Constants::POD_SANDBOX_KEY, sandbox_key)); + m_pluginManager->SetUpPod(config.metadata().namespace_(), config.metadata().name(), + Network::DEFAULT_NETWORK_INTERFACE_NAME, response_id, stdAnnos, networkOptions, error); + if (error.NotEmpty()) { + ERROR("SetupPod failed: %s", error.GetCMessage()); +- StopContainerHelper(response_id, error); +- goto cleanup; + } + +-cleanup: +- free(sandbox_key); + return; + } + +@@ -568,8 +581,9 @@ auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSan + ERROR("Failed to retrieve inspect data"); + goto cleanup; + } +- netnsPath = get_sandbox_key(inspect_data); ++ + if (namespace_is_file(inspect_data->host_config->network_mode)) { ++ netnsPath = get_sandbox_key(inspect_data); + if (!util_file_exists(netnsPath) || util_mount_namespace(netnsPath) != 0) { + error.Errorf("Failed to mount network namespace"); + ERROR("Failed to mount network namespace"); +@@ -578,9 +592,11 @@ auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSan + } + + // Step 5: Setup networking for the sandbox. +- SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, error); +- if (error.NotEmpty()) { +- goto cleanup; ++ if (namespace_is_file(inspect_data->host_config->network_mode)) { ++ SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, error); ++ if (error.NotEmpty()) { ++ goto cleanup; ++ } + } + + // Step 6: Start the sandbox container. +@@ -589,6 +605,15 @@ auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSan + goto cleanup; + } + ++ // If netns mode is not file, setup network after start sandbox container ++ if (!namespace_is_file(inspect_data->host_config->network_mode)) { ++ SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, error); ++ if (error.NotEmpty()) { ++ StopContainerHelper(response_id, error); ++ goto cleanup; ++ } ++ } ++ + cleanup: + if (error.Empty()) { + SetNetworkReady(response_id, true, error); +@@ -723,44 +748,58 @@ auto PodSandboxManagerServiceImpl::ClearCniNetwork(const std::string &realSandbo + /*error*/) -> int + { + Errors networkErr; +- container_inspect* inspect_data = nullptr; ++ container_inspect *inspect_data = nullptr; ++ char *netnsPath = nullptr; ++ ++ if (hostNetwork) { ++ return 0; ++ } + + bool ready = GetNetworkReady(realSandboxID, networkErr); +- if (!hostNetwork && (ready || networkErr.NotEmpty())) { +- Errors pluginErr; +- +- // hostNetwork has indicated network mode which render host config unnecessary +- // so that with_host_config is set to be false. +- inspect_data = CRIHelpers::InspectContainer(realSandboxID, pluginErr, false); +- if (pluginErr.NotEmpty()) { +- ERROR("Failed to inspect container"); +- } ++ if (!ready && networkErr.Empty()) { ++ WARN("Network not ready"); ++ return 0; ++ } + +- char* netnsPath = get_sandbox_key(inspect_data); +- if (netnsPath == nullptr) { +- ERROR("Failed to get network namespace path"); +- return 0; +- } ++ Errors pluginErr; ++ inspect_data = CRIHelpers::InspectContainer(realSandboxID, pluginErr, true); ++ if (pluginErr.NotEmpty()) { ++ ERROR("Failed to inspect container"); ++ // not return and make sure teardown network ++ } + +- stdAnnos.insert(std::pair(CRIHelpers::Constants::POD_SANDBOX_KEY, netnsPath)); +- m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, +- pluginErr); +- if (pluginErr.NotEmpty()) { +- WARN("TearDownPod cni network failed: %s", pluginErr.GetCMessage()); +- errlist.push_back(pluginErr.GetMessage()); ++ if (inspect_data != nullptr && namespace_is_file(inspect_data->host_config->network_mode)) { ++ netnsPath = get_sandbox_key(inspect_data); ++ if (netnsPath == nullptr) { ++ ERROR("Get sandbox key failed"); ++ // not return and make sure teardown network + } else { +- INFO("TearDownPod cni network: success"); +- SetNetworkReady(realSandboxID, false, pluginErr); +- if (pluginErr.NotEmpty()) { +- WARN("set network ready: %s", pluginErr.GetCMessage()); +- } +- // umount netns when cni removed network successfully +- if (util_umount_namespace(netnsPath) != 0) { +- ERROR("Failed to umount directory %s:%s", netnsPath, strerror(errno)); +- } ++ stdAnnos.insert(std::pair(CRIHelpers::Constants::POD_SANDBOX_KEY, ++ std::string(netnsPath))); + } +- free(netnsPath); + } ++ ++ m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, ++ pluginErr); ++ if (pluginErr.NotEmpty()) { ++ WARN("TearDownPod cni network failed: %s", pluginErr.GetCMessage()); ++ errlist.push_back(pluginErr.GetMessage()); ++ goto out; ++ } ++ ++ INFO("TearDownPod cni network: success"); ++ SetNetworkReady(realSandboxID, false, pluginErr); ++ if (pluginErr.NotEmpty()) { ++ WARN("set network ready: %s", pluginErr.GetCMessage()); ++ } ++ // umount netns when cni removed network successfully ++ if (inspect_data != nullptr && namespace_is_file(inspect_data->host_config->network_mode) && ++ util_umount_namespace(netnsPath) != 0) { ++ ERROR("Failed to umount directory %s:%s", netnsPath, strerror(errno)); ++ } ++ ++out: ++ free(netnsPath); + free_container_inspect(inspect_data); + return 0; + } +diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h +index 68a9d919..f7c0aa00 100644 +--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h ++++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h +@@ -66,7 +66,7 @@ private: + const std::string &image, std::string &jsonCheckpoint, + const std::string &runtimeHandler, + Errors &error) -> container_create_request *; +- void SetHostConfigDefaultValue(host_config *hc); ++ void SetHostConfigDefaultValue(const google::protobuf::Map &annotations, host_config *hc); + void MakeSandboxIsuladConfig(const runtime::v1alpha2::PodSandboxConfig &c, host_config *hc, + container_config *custom_config, Errors &error); + void ApplySandboxLinuxOptions(const runtime::v1alpha2::LinuxPodSandboxConfig &lc, host_config *hc, +diff --git a/src/daemon/entry/cri/cri_security_context.cc b/src/daemon/entry/cri/cri_security_context.cc +index 8484108c..1cd9287d 100644 +--- a/src/daemon/entry/cri/cri_security_context.cc ++++ b/src/daemon/entry/cri/cri_security_context.cc +@@ -162,7 +162,8 @@ static void ModifyHostNetworkOptionForSandbox(const runtime::v1alpha2::Namespace + free(hostConfig->uts_mode); + hostConfig->uts_mode = util_strdup_s(CRI::Constants::namespaceModeHost.c_str()); + } +- // Note: default networkMode is file ++ ++ // Set networkMode to file or "" in SetHostConfigDefaultValue + } + + static void ModifyContainerNamespaceOptions(const runtime::v1alpha2::NamespaceOption &nsOpts, +diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c +index ec6385a6..138a0989 100644 +--- a/src/daemon/modules/spec/specs.c ++++ b/src/daemon/modules/spec/specs.c +@@ -1460,7 +1460,7 @@ static int merge_share_network_namespace(oci_runtime_spec *oci_spec, const host_ + int ret = 0; + char *ns_path = NULL; + +- if (host_spec->network_mode == NULL) { ++ if (host_spec->network_mode == NULL || strlen(host_spec->network_mode) == 0) { + return 0; + } + +diff --git a/src/utils/cutils/utils_file.c b/src/utils/cutils/utils_file.c +index f06f4d49..7b1d5cb6 100644 +--- a/src/utils/cutils/utils_file.c ++++ b/src/utils/cutils/utils_file.c +@@ -85,7 +85,7 @@ bool util_file_exists(const char *f) + struct stat buf; + int nret; + +- if (f == NULL) { ++ if (f == NULL || strlen(f) == 0) { + return false; + } + +-- +2.25.1 + diff --git a/0066-compatibility-for-manage-pods-which-created-by-old-i.patch b/0066-compatibility-for-manage-pods-which-created-by-old-i.patch new file mode 100644 index 0000000..8a08492 --- /dev/null +++ b/0066-compatibility-for-manage-pods-which-created-by-old-i.patch @@ -0,0 +1,157 @@ +From 4ad92d9524915bc83294627afe6954f43587e450 Mon Sep 17 00:00:00 2001 +From: haozi007 +Date: Thu, 15 Dec 2022 20:42:59 +0800 +Subject: [PATCH 66/66] compatibility for manage pods which created by old + iSulad + +Signed-off-by: haozi007 +--- + .../cri_pod_sandbox_manager_service_impl.cc | 12 +++- + src/daemon/entry/cri/naming.cc | 66 +++++++++++++++---- + src/daemon/entry/cri/naming.h | 2 +- + 3 files changed, 66 insertions(+), 14 deletions(-) + +diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc +index 4c245763..3e89008a 100644 +--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc ++++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc +@@ -1188,7 +1188,11 @@ void PodSandboxManagerServiceImpl::PodSandboxStatusToGRPC( + + CRIHelpers::ExtractLabels(inspect->config->labels, *podStatus->mutable_labels()); + CRIHelpers::ExtractAnnotations(inspect->config->annotations, *podStatus->mutable_annotations()); +- CRINaming::ParseSandboxName(podStatus->annotations(), *podStatus->mutable_metadata(), error); ++ std::string name; ++ if (inspect->name != nullptr) { ++ name = std::string(inspect->name); ++ } ++ CRINaming::ParseSandboxName(name, podStatus->annotations(), *podStatus->mutable_metadata(), error); + if (error.NotEmpty()) { + return; + } +@@ -1301,7 +1305,11 @@ void PodSandboxManagerServiceImpl::ListPodSandboxToGRPC( + + CRIHelpers::ExtractAnnotations(response->containers[i]->annotations, *pod->mutable_annotations()); + +- CRINaming::ParseSandboxName(pod->annotations(), *pod->mutable_metadata(), error); ++ std::string name; ++ if (response->containers[i]->name != nullptr) { ++ name = std::string(response->containers[i]->name); ++ } ++ CRINaming::ParseSandboxName(name, pod->annotations(), *pod->mutable_metadata(), error); + + if (filterOutReadySandboxes && pod->state() == runtime::v1alpha2::SANDBOX_READY) { + continue; +diff --git a/src/daemon/entry/cri/naming.cc b/src/daemon/entry/cri/naming.cc +index 682f2e52..54a14a81 100644 +--- a/src/daemon/entry/cri/naming.cc ++++ b/src/daemon/entry/cri/naming.cc +@@ -26,6 +26,33 @@ + #include "utils.h" + + namespace CRINaming { ++static int parseName(const std::string &name, std::vector &items, unsigned int &attempt, Errors &err) ++{ ++ std::istringstream f(name); ++ std::string part; ++ ++ while (getline(f, part, CRI::Constants::nameDelimiterChar)) { ++ items.push_back(part); ++ } ++ ++ if (items.size() != 6) { ++ err.Errorf("failed to parse the sandbox name: %s", name.c_str()); ++ return -1; ++ } ++ ++ if (items[0] != CRI::Constants::kubePrefix) { ++ err.Errorf("container is not managed by kubernetes: %s", name.c_str()); ++ return -1; ++ } ++ ++ if (util_safe_uint(items[5].c_str(), &attempt)) { ++ err.Errorf("failed to parse the sandbox name %s: %s", name.c_str(), strerror(errno)); ++ return -1; ++ } ++ ++ return 0; ++} ++ + std::string MakeSandboxName(const runtime::v1alpha2::PodSandboxMetadata &metadata) + { + std::string sname; +@@ -44,9 +71,12 @@ std::string MakeSandboxName(const runtime::v1alpha2::PodSandboxMetadata &metadat + return sname; + } + +-void ParseSandboxName(const google::protobuf::Map &annotations, ++void ParseSandboxName(const std::string &name, const google::protobuf::Map &annotations, + runtime::v1alpha2::PodSandboxMetadata &metadata, Errors &err) + { ++ // need check uid and attemp 2 items ++ int needSetUidOrAttemp = 2; ++ + if (annotations.count(CRIHelpers::Constants::SANDBOX_NAME_ANNOTATION_KEY) == 0) { + err.Errorf("annotation don't contains the sandbox name, failed to parse it"); + return; +@@ -57,21 +87,35 @@ void ParseSandboxName(const google::protobuf::Map &ann + return; + } + +- if (annotations.count(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY) == 0) { +- err.Errorf("annotation don't contains the sandbox uid, failed to parse it"); +- return; ++ metadata.set_name(annotations.at(CRIHelpers::Constants::SANDBOX_NAME_ANNOTATION_KEY)); ++ metadata.set_namespace_(annotations.at(CRIHelpers::Constants::SANDBOX_NAMESPACE_ANNOTATION_KEY)); ++ ++ if (annotations.count(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY) != 0) { ++ metadata.set_uid(annotations.at(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY)); ++ needSetUidOrAttemp--; ++ } ++ ++ if (annotations.count(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY) != 0) { ++ auto sandboxAttempt = annotations.at(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY); ++ metadata.set_attempt(static_cast(std::stoul(sandboxAttempt))); ++ needSetUidOrAttemp--; + } + +- if (annotations.count(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY) == 0) { +- err.Errorf("annotation don't contains the sandbox attempt, failed to parse it"); ++ if (needSetUidOrAttemp == 0) { + return; + } + +- metadata.set_name(annotations.at(CRIHelpers::Constants::SANDBOX_NAME_ANNOTATION_KEY)); +- metadata.set_namespace_(annotations.at(CRIHelpers::Constants::SANDBOX_NAMESPACE_ANNOTATION_KEY)); +- metadata.set_uid(annotations.at(CRIHelpers::Constants::SANDBOX_UID_ANNOTATION_KEY)); +- auto sandboxAttempt = annotations.at(CRIHelpers::Constants::SANDBOX_ATTEMPT_ANNOTATION_KEY); +- metadata.set_attempt(static_cast(std::stoul(sandboxAttempt))); ++ // get uid and attempt from name, ++ // compatibility to new iSulad manage pods created by old version iSulad ++ // maybe should remove in next version of iSulad ++ std::vector items; ++ unsigned int attempt; ++ ++ if (parseName(name, items, attempt, err) != 0) { ++ return; ++ } ++ metadata.set_uid(items[4]); ++ metadata.set_attempt(static_cast(attempt)); + } + + std::string MakeContainerName(const runtime::v1alpha2::PodSandboxConfig &s, const runtime::v1alpha2::ContainerConfig &c) +diff --git a/src/daemon/entry/cri/naming.h b/src/daemon/entry/cri/naming.h +index 7eab41d3..f2d51a98 100644 +--- a/src/daemon/entry/cri/naming.h ++++ b/src/daemon/entry/cri/naming.h +@@ -26,7 +26,7 @@ std::string MakeSandboxName(const runtime::v1alpha2::PodSandboxMetadata &metadat + std::string MakeContainerName(const runtime::v1alpha2::PodSandboxConfig &s, + const runtime::v1alpha2::ContainerConfig &c); + +-void ParseSandboxName(const google::protobuf::Map &annotations, ++void ParseSandboxName(const std::string &name, const google::protobuf::Map &annotations, + runtime::v1alpha2::PodSandboxMetadata &metadata, Errors &err); + + void ParseContainerName(const google::protobuf::Map &annotations, +-- +2.25.1 + diff --git a/iSulad.spec b/iSulad.spec index b778b33..e235673 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,5 +1,5 @@ %global _version 2.0.17 -%global _release 11 +%global _release 12 %global is_systemd 1 %global enable_shimv2 1 %global is_embedded 1 @@ -67,6 +67,18 @@ Patch0051: 0051-remove-chmod-751-permission-for-dirs-by-engine-when-.patch Patch0052: 0052-add-console-ut.patch Patch0053: 0053-fix-additional-gids-for-exec-user.patch Patch0054: 0054-add-CI-for-additional-gid.patch +Patch0055: 0055-retry-call-runtime-ops.patch +Patch0056: 0056-add-ut-test-for-retry-macro.patch +Patch0057: 0057-1749-set-inspect_container-timeout.patch +Patch0058: 0058-1757-add-adaption-code-for-musl.patch +Patch0059: 0059-deleting-broken-rootfs.patch +Patch0060: 0060-1761-fix-leftover-devicemapper-mnt-dir.patch +Patch0061: 0061-check-file-system-ro-before-merge-network-for-syscon.patch +Patch0062: 0062-isulad-shim-wait-for-all-child-process.patch +Patch0063: 0063-When-run-options-rm-is-set-delete-the-stoped-contain.patch +Patch0064: 0064-recheck-kill-command-exit-status.patch +Patch0065: 0065-start-sandbox-before-setup-network-by-default.patch +Patch0066: 0066-compatibility-for-manage-pods-which-created-by-old-i.patch %ifarch x86_64 aarch64 Provides: libhttpclient.so()(64bit) @@ -293,6 +305,12 @@ fi %endif %changelog +* Fri Dec 16 2022 zhangxiaoyu - 2.0.17-12 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: update from upstream + * Tue Dec 06 2022 zhangxiaoyu - 2.0.17-11 - Type: bugfix - ID: NA