iSulad/0039-feat-add-container-cleanup-module.patch

From 587aa25d7f44f79c28c3331b81bbc77efb7e1eb1 Mon Sep 17 00:00:00 2001
From: "Neil.wrz" <wangrunze13@huawei.com>
Date: Thu, 27 Oct 2022 00:45:53 -0700
Subject: [PATCH 39/43] feat add container cleanup module

Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
---
 src/daemon/modules/api/image_api.h            |   6 +
 src/daemon/modules/container/CMakeLists.txt   |   3 +
 src/daemon/modules/container/container_unix.c |   2 +
 .../container/leftover_cleanup/CMakeLists.txt |  19 +++
 .../container/leftover_cleanup/cleanup.c      | 135 ++++++++++++++++++
 .../container/leftover_cleanup/cleanup.h      |  49 +++++++
 .../leftover_cleanup/oci_rootfs_clean.c       |  71 +++++++++
 .../leftover_cleanup/oci_rootfs_clean.h       |  30 ++++
 src/daemon/modules/image/image.c              |  26 ++++
 src/daemon/modules/image/oci/oci_image.c      |   5 +
 src/daemon/modules/image/oci/oci_image.h      |   1 +
 .../oci/storage/rootfs_store/rootfs_store.c   |   5 +
 .../oci/storage/rootfs_store/rootfs_store.h   |   3 +
 .../modules/image/oci/storage/storage.c       |   5 +
 .../modules/image/oci/storage/storage.h       |   2 +
 15 files changed, 362 insertions(+)
 create mode 100644 src/daemon/modules/container/leftover_cleanup/CMakeLists.txt
 create mode 100644 src/daemon/modules/container/leftover_cleanup/cleanup.c
 create mode 100644 src/daemon/modules/container/leftover_cleanup/cleanup.h
 create mode 100644 src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c
 create mode 100644 src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h

diff --git a/src/daemon/modules/api/image_api.h b/src/daemon/modules/api/image_api.h
index 10fa7b5b..a1c6084a 100644
--- a/src/daemon/modules/api/image_api.h
+++ b/src/daemon/modules/api/image_api.h
@@ -184,6 +184,10 @@ typedef struct {
     char *name_id;
 } im_export_request;
 
+typedef struct {
+    char *type;
+} im_get_rf_dir_request;
+
 typedef struct {
     char *name_id;
     bool force;
@@ -325,6 +329,8 @@ int im_container_export(const im_export_request *request);
 
 void free_im_export_request(im_export_request *ptr);
 
+char *im_get_rootfs_dir(const im_get_rf_dir_request *request);
+
 int im_resolv_image_name(const char *image_type, const char *image_name, char **resolved_name);
 
 container_inspect_graph_driver *im_graphdriver_get_metadata_by_container_id(const char *id);
diff --git a/src/daemon/modules/container/CMakeLists.txt b/src/daemon/modules/container/CMakeLists.txt
index fd2a0b10..def602c7 100644
--- a/src/daemon/modules/container/CMakeLists.txt
+++ b/src/daemon/modules/container/CMakeLists.txt
@@ -5,6 +5,7 @@ add_subdirectory(supervisor)
 add_subdirectory(health_check)
 add_subdirectory(container_gc)
 add_subdirectory(restart_manager)
+add_subdirectory(leftover_cleanup)
 
 set(MANAGER_SRCS
     ${local_manager_srcs}
@@ -13,6 +14,7 @@ set(MANAGER_SRCS
     ${HEALTH_CHECK_SRCS}
     ${GC_SRCS}
     ${RESTART_MANAGER_SRCS}
+    ${LEFTOVER_CLEANUP_SRCS}
     PARENT_SCOPE
     )
 
@@ -23,5 +25,6 @@ set(MANAGER_INCS
     ${HEALTH_CHECK_INCS}
     ${GC_INCS}
     ${RESTART_MANAGER_INCS}
+    ${LEFTOVER_CLEANUP_INCS}
     PARENT_SCOPE
     )
diff --git a/src/daemon/modules/container/container_unix.c b/src/daemon/modules/container/container_unix.c
index 9910b3c8..88c4bf51 100644
--- a/src/daemon/modules/container/container_unix.c
+++ b/src/daemon/modules/container/container_unix.c
@@ -46,6 +46,7 @@
 #include "utils_string.h"
 #include "volume_api.h"
 #include "namespace.h"
+#include "cleanup.h"
 
 static int parse_container_log_configs(container_t *cont);
 
@@ -1278,6 +1279,7 @@ int container_module_init(char **msg)
     }
 
     containers_restore();
+    clean_leftover();
 
     if (start_gchandler()) {
         *msg = "Failed to start garbage collecotor handler";
diff --git a/src/daemon/modules/container/leftover_cleanup/CMakeLists.txt b/src/daemon/modules/container/leftover_cleanup/CMakeLists.txt
new file mode 100644
index 00000000..225ac38f
--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/CMakeLists.txt
@@ -0,0 +1,19 @@
+# get current directory sources files
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_leftover_cleanup_srcs)
+
+set(LEFTOVER_CLEANUP_SRCS
+    ${local_leftover_cleanup_srcs}
+    PARENT_SCOPE
+    )
+
+set(LEFTOVER_CLEANUP_INCS
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    PARENT_SCOPE
+    )
+
+if (DISABLE_OCI)
+    list(REMOVE_ITEM
+    local_leftover_cleanup_srcs 
+    ${CMAKE_CURRENT_SOURCE_DIR}/oci_rootfs_clean.c
+    )
+endif()
\ No newline at end of file
diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c
new file mode 100644
index 00000000..29fa4bfa
--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c
@@ -0,0 +1,135 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide cleanup functions
+ *********************************************************************************/
+#include "utils.h"
+#include "cleanup.h"
+#include "oci_rootfs_clean.h"
+
+static struct cleaners *create_cleaners()
+{
+    struct cleaners *ret = NULL;
+
+    ret = util_common_calloc_s(sizeof(struct cleaners));
+    if (ret == NULL) {
+        ERROR("Out of memory");
+        return NULL;
+    }
+
+    linked_list_init(&(ret->cleaner_list));
+
+    return ret;
+}
+
+static void destroy_cleaners(struct cleaners *clns)
+{
+    struct linked_list *it = NULL;
+    struct linked_list *next = NULL;
+    struct clean_node *c_node = NULL;
+
+    linked_list_for_each_safe(it, &(clns->cleaner_list), next) {
+        c_node = (struct clean_node *)it->elem;
+        linked_list_del(it);
+        free(c_node);
+        free(it);
+        it = NULL;
+    }
+
+    free(clns);
+}
+
+static int add_clean_node(struct cleaners * clns, clean_func_t f, char * desc)
+{
+    struct linked_list *new_node = NULL;
+    struct clean_node *c_node = NULL;
+
+    new_node = util_common_calloc_s(sizeof(struct linked_list));
+    if (new_node == NULL) {
+        ERROR("Out of memory");
+        return -1;
+    }
+
+    c_node = util_common_calloc_s(sizeof(struct clean_node));
+    if (c_node == NULL) {
+        ERROR("Out of memory");
+        free(new_node);
+        return -1;
+    }
+    c_node->cleaner = f;
+    c_node->desc = desc;
+
+    linked_list_add_elem(new_node, c_node);
+    linked_list_add_tail(&(clns->cleaner_list), new_node);
+    clns->count++;
+
+    return 0;
+}
+
+static int default_cleaner()
+{
+    return 0;
+}
+
+static struct cleaners *cleaner_init()
+{
+    struct cleaners *clns = create_cleaners();
+    
+    if (clns == NULL) {
+        return NULL;
+    }
+
+    add_clean_node(clns, default_cleaner, "default clean");
+#ifdef ENABLE_OCI_IMAGE
+    add_clean_node(clns, oci_rootfs_cleaner, "clean rootfs");
+#endif
+
+    return clns;
+}
+
+static void do_clean(struct cleaners * clns)
+{
+    struct linked_list *it = NULL;
+    struct linked_list *next = NULL;
+    struct clean_node *c_node = NULL;
+    int ret = 0;
+
+    linked_list_for_each_safe(it, &(clns->cleaner_list), next) {
+        c_node = (struct clean_node *)it->elem;
+        if ((ret = c_node->cleaner()) != 0) {
+            ERROR("failed to clean for: %s", c_node->desc);
+        } else {
+            DEBUG("do clean success for: %s", c_node->desc);
+            clns->done_clean++;
+        }
+    }
+}
+
+void clean_leftover()
+{
+    struct cleaners *clns = cleaner_init();
+
+    if (clns == NULL) {
+        ERROR("failed to clean leftovers, because cleaner init error");
+        return;
+    }
+
+    do_clean(clns);
+
+    if (clns->count == clns->done_clean) {
+        DEBUG("all clean up success");
+    } else {
+        ERROR("Aim to do %d clean, %d clean sucess\n", clns->count, clns->done_clean);
+    }
+
+    destroy_cleaners(clns);
+}
\ No newline at end of file
diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.h b/src/daemon/modules/container/leftover_cleanup/cleanup.h
new file mode 100644
index 00000000..26fc1b0b
--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/cleanup.h
@@ -0,0 +1,49 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide cleanup definition
+ *********************************************************************************/
+#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_H
+#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_H
+
+#include <stdlib.h>
+
+#include "linked_list.h"
+#include "isula_libutils/log.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+typedef int clean_func_t(void);
+
+struct clean_node {
+    char *desc;
+    clean_func_t *cleaner;
+    int error_code;
+};
+
+struct cleaners {
+    int count;
+    int done_clean;
+    struct linked_list cleaner_list;
+};
+
+
+void clean_leftover();
+
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c
new file mode 100644
index 00000000..db56870b
--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c
@@ -0,0 +1,71 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide rootfs cleaner functions
+ *********************************************************************************/
+#include <sys/stat.h>
+#include <string.h>
+#include "oci_rootfs_clean.h"
+#include "container_api.h"
+#include "image_api.h"
+#include "utils_file.h"
+#include "utils.h"
+
+struct cb_result {
+    int clean_err_cnt;
+};
+
+static bool walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context)
+{
+    struct cb_result *result = (struct cb_result *)context;
+    container_t *cont = containers_store_get(sub_dir->d_name);
+    int rm_rootfs_ret = 0;
+
+    if (cont != NULL) {
+        return true;
+    }
+
+    INFO("cleaning leftover dir: %s", sub_dir->d_name);
+    rm_rootfs_ret = im_remove_container_rootfs(IMAGE_TYPE_OCI, sub_dir->d_name);
+    if (rm_rootfs_ret != 0) {
+        result->clean_err_cnt++;
+    }
+
+    return true;
+}
+
+
+int oci_rootfs_cleaner(void)
+{
+    struct cb_result res = { 0 };
+    im_get_rf_dir_request request = { 0 };
+    char *rf_dir = NULL;
+    int ret = 0;
+
+    request.type = IMAGE_TYPE_OCI;
+    rf_dir = im_get_rootfs_dir(&request);
+    if (rf_dir == NULL) {
+        return 0;
+    }
+
+    ret = util_scan_subdirs(rf_dir, walk_dir_cb, &res);
+    if (ret != 0) {
+        ERROR("failed to scan subdirs");
+        return -1;
+    }
+
+    if (res.clean_err_cnt == 0) {
+        return 0;
+    } 
+
+    return -1;
+}
diff --git a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h
new file mode 100644
index 00000000..8dff351f
--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h
@@ -0,0 +1,30 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide rootfs cleaner definition
+ *********************************************************************************/
+#ifndef DAEMON_MODULES_CONTAINER_ROOTFS_CLEAN_H
+#define DAEMON_MODULES_CONTAINER_ROOTFS_CLEAN_H
+
+#include "cleanup.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+int oci_rootfs_cleaner(void);
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c
index f487f831..3c395c1f 100644
--- a/src/daemon/modules/image/image.c
+++ b/src/daemon/modules/image/image.c
@@ -49,6 +49,7 @@ struct bim_ops {
     int (*delete_rf)(const im_delete_rootfs_request *request);
     int (*export_rf)(const im_export_request *request);
     char *(*resolve_image_name)(const char *image_name);
+    char *(*get_dir_rf)(void);
 
     /* merge image config ops */
     int (*merge_conf)(const char *img_name, container_config *container_spec);
@@ -132,6 +133,7 @@ static const struct bim_ops g_embedded_ops = {
     .umount_rf = embedded_umount_rf,
     .delete_rf = embedded_delete_rf,
     .export_rf = NULL,
+    .get_dir_rf = NULL,
 
     .merge_conf = embedded_merge_conf,
     .get_user_conf = embedded_get_user_conf,
@@ -166,6 +168,7 @@ static const struct bim_ops g_oci_ops = {
     .umount_rf = oci_umount_rf,
     .delete_rf = oci_delete_rf,
     .export_rf = oci_export_rf,
+    .get_dir_rf = oci_get_dir_rf,
 
     .merge_conf = oci_merge_conf_rf,
     .get_user_conf = oci_get_user_conf,
@@ -199,6 +202,7 @@ static const struct bim_ops g_ext_ops = {
     .umount_rf = ext_umount_rf,
     .delete_rf = ext_delete_rf,
     .export_rf = NULL,
+    .get_dir_rf = NULL,
 
     .merge_conf = ext_merge_conf,
     .get_user_conf = ext_get_user_conf,
@@ -1768,6 +1772,28 @@ int im_container_export(const im_export_request *request)
 }
 #endif
 
+#ifdef ENABLE_OCI_IMAGE
+char *im_get_rootfs_dir(const im_get_rf_dir_request *request) {
+    if (request->type == NULL) {
+        ERROR("Missing image type");
+        return NULL;
+    }
+
+    struct bim *bim = NULL;
+    bim = bim_get(request->type, NULL, NULL, NULL);
+    if (bim->ops->get_dir_rf == NULL) {
+        ERROR("Unimplemnts get rootfs dir in %s", bim->type);
+        return NULL;
+    }
+
+    return bim->ops->get_dir_rf();
+}
+#else
+char *im_get_rootfs_dir(const im_get_rf_dir_request *request) {
+    return NULL;
+}
+#endif
+
 void free_im_export_request(im_export_request *ptr)
 {
     if (ptr == NULL) {
diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c
index 86828f50..e951adb4 100644
--- a/src/daemon/modules/image/oci/oci_image.c
+++ b/src/daemon/modules/image/oci/oci_image.c
@@ -686,6 +686,11 @@ int oci_export_rf(const im_export_request *request)
     return ret;
 }
 
+char *oci_get_dir_rf(void)
+{
+    return storage_rootfs_get_dir();
+}
+
 int oci_login(const im_login_request *request)
 {
     int ret = 0;
diff --git a/src/daemon/modules/image/oci/oci_image.h b/src/daemon/modules/image/oci/oci_image.h
index 64a4d8e8..aeeb3b65 100644
--- a/src/daemon/modules/image/oci/oci_image.h
+++ b/src/daemon/modules/image/oci/oci_image.h
@@ -54,6 +54,7 @@ int oci_mount_rf(const im_mount_request *request);
 int oci_umount_rf(const im_umount_request *request);
 int oci_delete_rf(const im_delete_rootfs_request *request);
 int oci_export_rf(const im_export_request *request);
+char *oci_get_dir_rf(void);
 int oci_container_filesystem_usage(const im_container_fs_usage_request *request, imagetool_fs_info **fs_usage);
 int oci_login(const im_login_request *request);
 int oci_logout(const im_logout_request *request);
diff --git a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c
index 378d1a96..0270f6a7 100644
--- a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c
+++ b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c
@@ -1319,3 +1319,8 @@ out:
     rootfs_store_unlock();
     return ret;
 }
+
+char *rootfs_store_get_data_dir()
+{
+    return g_rootfs_store->dir;
+}
\ No newline at end of file
diff --git a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.h b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.h
index e13f97bc..c23af091 100644
--- a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.h
+++ b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.h
@@ -65,6 +65,9 @@ storage_rootfs *rootfs_store_get_rootfs(const char *id);
 // Return a slice enumerating the known containers.
 int rootfs_store_get_all_rootfs(struct rootfs_list *all_rootfs);
 
+// Return rootfs store data dir
+char *rootfs_store_get_data_dir();
+
 // Free memory of container store, but will not delete the persisted files
 void rootfs_store_free();
 
diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c
index 57b5fb80..6cb4a51b 100644
--- a/src/daemon/modules/image/oci/storage/storage.c
+++ b/src/daemon/modules/image/oci/storage/storage.c
@@ -1868,3 +1868,8 @@ int storage_module_init(struct storage_module_init_options *opts)
 out:
     return ret;
 }
+
+char *storage_rootfs_get_dir()
+{
+    return rootfs_store_get_data_dir();
+}
\ No newline at end of file
diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h
index 1fe29b45..5914adec 100644
--- a/src/daemon/modules/image/oci/storage/storage.h
+++ b/src/daemon/modules/image/oci/storage/storage.h
@@ -181,6 +181,8 @@ char *storage_rootfs_mount(const char *container_id);
 
 int storage_rootfs_umount(const char *container_id, bool force);
 
+char *storage_rootfs_get_dir();
+
 container_inspect_graph_driver *storage_get_metadata_by_container_id(const char *id);
 
 #ifdef __cplusplus
-- 
2.25.1