iSulad/0059-deleting-broken-rootfs.patch

From edb570b8720aed234bf2c17642f5e6caba2c726a Mon Sep 17 00:00:00 2001
From: "Neil.wrz" <wangrunze13@huawei.com>
Date: Tue, 6 Dec 2022 19:23:33 -0800
Subject: [PATCH 59/65] deleting broken rootfs

Signed-off-by: Neil.wrz <wangrunze13@huawei.com>
---
 cmake/options.cmake                           |  6 ++
 src/cmd/isulad/main.c                         | 15 ++++
 src/daemon/modules/api/image_api.h            |  2 +
 src/daemon/modules/api/leftover_cleanup_api.h | 36 +++++++++
 src/daemon/modules/container/CMakeLists.txt   |  2 +
 src/daemon/modules/container/container_unix.c |  2 -
 .../leftover_cleanup/clean_context.c          | 78 +++++++++++++++++++
 .../leftover_cleanup/clean_context.h          | 41 ++++++++++
 .../container/leftover_cleanup/cleanup.c      | 46 +++++------
 .../container/leftover_cleanup/cleanup.h      | 12 ++-
 .../leftover_cleanup/leftover_cleanup_api.c   | 71 +++++++++++++++++
 .../leftover_cleanup/oci_rootfs_clean.c       | 34 +++++++-
 .../leftover_cleanup/oci_rootfs_clean.h       |  4 +-
 src/daemon/modules/image/image.c              | 48 ++++++++++++
 src/daemon/modules/image/oci/oci_image.c      | 10 +++
 src/daemon/modules/image/oci/oci_image.h      |  1 +
 .../oci/storage/layer_store/layer_store.c     |  2 +
 .../oci/storage/rootfs_store/rootfs_store.c   | 13 +++-
 .../modules/image/oci/storage/storage.c       | 32 ++++++++
 .../modules/image/oci/storage/storage.h       |  3 +
 test/image/oci/storage/rootfs/CMakeLists.txt  |  2 +
 21 files changed, 422 insertions(+), 38 deletions(-)
 create mode 100644 src/daemon/modules/api/leftover_cleanup_api.h
 create mode 100644 src/daemon/modules/container/leftover_cleanup/clean_context.c
 create mode 100644 src/daemon/modules/container/leftover_cleanup/clean_context.h
 create mode 100644 src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c

diff --git a/cmake/options.cmake b/cmake/options.cmake
index 0d44868a..1e63a485 100644
--- a/cmake/options.cmake
+++ b/cmake/options.cmake
@@ -115,6 +115,12 @@ if (ENABLE_SUP_GROUPS)
     message("${Green}--  Enable sup groups${ColourReset}")
 endif()
 
+option(DISABLE_CLEANUP "disable cleanup module" OFF)
+if (DISABLE_CLEANUP STREQUAL "ON")
+    add_definitions(-DDISABLE_CLEANUP)
+    message("${Green}--  Disable cleanup module")
+endif()
+
 option(MUSL "available for musl" OFF)
 if (MUSL)
     add_definitions(-D__MUSL__)
diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
index d4f984d5..9b664bee 100644
--- a/src/cmd/isulad/main.c
+++ b/src/cmd/isulad/main.c
@@ -73,6 +73,9 @@
 #include "utils_string.h"
 #include "utils_verify.h"
 #include "volume_api.h"
+#ifndef DISABLE_CLEANUP
+#include "leftover_cleanup_api.h"
+#endif
 #include "opt_log.h"
 
 #ifdef GRPC_CONNECTOR
@@ -1235,6 +1238,14 @@ static int isulad_server_init_common()
         goto out;
     }
 
+#ifndef DISABLE_CLEANUP
+    // to cleanup leftover, init clean module before other modules.
+    if (clean_module_init() != 0) {
+        ERROR("Failed to init clean module");
+        goto out;
+    }
+#endif
+
     if (volume_init(args->json_confs->graph) != 0) {
         ERROR("Failed to init volume");
         goto out;
@@ -1451,6 +1462,10 @@ static int start_daemon_threads(char **msg)
         goto out;
     }
 
+#ifndef DISABLE_CLEANUP
+    clean_module_do_clean();
+#endif
+
     ret = 0;
 out:
     return ret;
diff --git a/src/daemon/modules/api/image_api.h b/src/daemon/modules/api/image_api.h
index a1c6084a..b8d7fd5b 100644
--- a/src/daemon/modules/api/image_api.h
+++ b/src/daemon/modules/api/image_api.h
@@ -244,6 +244,8 @@ int im_umount_container_rootfs(const char *image_type, const char *image_name, c
 
 int im_remove_container_rootfs(const char *image_type, const char *container_id);
 
+int im_remove_broken_rootfs(const char *image_type, const char *container_id);
+
 int im_merge_image_config(const char *image_type, const char *image_name, container_config *container_spec);
 
 int im_get_user_conf(const char *image_type, const char *basefs, host_config *hc, const char *userstr,
diff --git a/src/daemon/modules/api/leftover_cleanup_api.h b/src/daemon/modules/api/leftover_cleanup_api.h
new file mode 100644
index 00000000..32a41258
--- /dev/null
+++ b/src/daemon/modules/api/leftover_cleanup_api.h
@@ -0,0 +1,36 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-12-7
+ * Description: provide cleanup functions
+ *********************************************************************************/
+#ifndef DAEMON_MODULES_API_LEFTOVER_CLEANUP_API_H
+#define DAEMON_MODULES_API_LEFTOVER_CLEANUP_API_H
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+typedef enum {
+    BROKEN_ROOTFS = 1
+} cleanup_ctx_data_t;
+
+int clean_module_init();
+
+void clean_module_fill_ctx(cleanup_ctx_data_t data_type, void *data);
+
+void clean_module_do_clean();
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif
diff --git a/src/daemon/modules/container/CMakeLists.txt b/src/daemon/modules/container/CMakeLists.txt
index def602c7..38c3d88b 100644
--- a/src/daemon/modules/container/CMakeLists.txt
+++ b/src/daemon/modules/container/CMakeLists.txt
@@ -5,7 +5,9 @@ add_subdirectory(supervisor)
 add_subdirectory(health_check)
 add_subdirectory(container_gc)
 add_subdirectory(restart_manager)
+IF (NOT DISABLE_CLEANUP)
 add_subdirectory(leftover_cleanup)
+ENDIF()
 
 set(MANAGER_SRCS
     ${local_manager_srcs}
diff --git a/src/daemon/modules/container/container_unix.c b/src/daemon/modules/container/container_unix.c
index 88c4bf51..9910b3c8 100644
--- a/src/daemon/modules/container/container_unix.c
+++ b/src/daemon/modules/container/container_unix.c
@@ -46,7 +46,6 @@
 #include "utils_string.h"
 #include "volume_api.h"
 #include "namespace.h"
-#include "cleanup.h"
 
 static int parse_container_log_configs(container_t *cont);
 
@@ -1279,7 +1278,6 @@ int container_module_init(char **msg)
     }
 
     containers_restore();
-    clean_leftover();
 
     if (start_gchandler()) {
         *msg = "Failed to start garbage collecotor handler";
diff --git a/src/daemon/modules/container/leftover_cleanup/clean_context.c b/src/daemon/modules/container/leftover_cleanup/clean_context.c
new file mode 100644
index 00000000..6ccc39ed
--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/clean_context.c
@@ -0,0 +1,78 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide cleanup definition
+ *********************************************************************************/
+#include "clean_context.h"
+#include "linked_list.h"
+#include "utils.h"
+#include "isula_libutils/log.h"
+
+struct clean_ctx *clean_ctx_init()
+{
+    struct clean_ctx *ctx = util_common_calloc_s(sizeof(struct clean_ctx));
+    if (ctx == NULL) {
+        ERROR("Out of memory");
+        return NULL;
+    }
+
+    linked_list_init(&(ctx->broken_rootfs_list));
+    ctx->inited = true;
+
+    return ctx;
+}
+
+void clean_ctx_destroy(struct clean_ctx *ctx)
+{
+    struct linked_list *it = NULL;
+    struct linked_list *next = NULL;
+    char *id = NULL;
+
+    if (ctx == NULL) {
+        return;
+    }
+
+    if (!ctx->inited) {
+        free(ctx);
+        return;
+    }
+
+    linked_list_for_each_safe(it, &(ctx->broken_rootfs_list), next) {
+        id = (char *)it->elem;
+        linked_list_del(it);
+        free(id);
+        free(it);
+        it = NULL;
+    }
+
+    free(ctx);
+}
+
+void clean_ctx_fill_broken_rootfs(struct clean_ctx *ctx, const char *id)
+{
+    struct linked_list *new_node = NULL;
+    char *broken_id = NULL;
+
+    if (!ctx->inited) {
+        return;
+    }
+
+    new_node = util_common_calloc_s(sizeof(struct linked_list));
+    if (new_node == NULL) {
+        ERROR("Out of memory, broken rootfs %s not added", id);
+        return;
+    }
+
+    broken_id = util_strdup_s(id);
+    linked_list_add_elem(new_node, broken_id);
+    linked_list_add_tail(&ctx->broken_rootfs_list, new_node);
+}
\ No newline at end of file
diff --git a/src/daemon/modules/container/leftover_cleanup/clean_context.h b/src/daemon/modules/container/leftover_cleanup/clean_context.h
new file mode 100644
index 00000000..fcac8df8
--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/clean_context.h
@@ -0,0 +1,41 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide cleanup definition
+ *********************************************************************************/
+#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEAN_CONTEXT_H
+#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEAN_CONTEXT_H
+
+#include "linked_list.h"
+#include "utils.h"
+#include "isula_libutils/log.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+struct clean_ctx {
+    bool inited;
+    struct linked_list broken_rootfs_list;
+};
+
+struct clean_ctx *clean_ctx_init();
+
+void clean_ctx_destroy(struct clean_ctx *ctx);
+
+void clean_ctx_fill_broken_rootfs(struct clean_ctx *ctx, const char *id);
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c
index 9ce1dd0c..664988b5 100644
--- a/src/daemon/modules/container/leftover_cleanup/cleanup.c
+++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c
@@ -31,12 +31,16 @@ static struct cleaners *create_cleaners()
     return ret;
 }
 
-static void destroy_cleaners(struct cleaners *clns)
+void destroy_cleaners(struct cleaners *clns)
 {
     struct linked_list *it = NULL;
     struct linked_list *next = NULL;
     struct clean_node *c_node = NULL;
 
+    if (clns == NULL) {
+        return;
+    }
+
     linked_list_for_each_safe(it, &(clns->cleaner_list), next) {
         c_node = (struct clean_node *)it->elem;
         linked_list_del(it);
@@ -80,25 +84,31 @@ static int default_cleaner()
     return 0;
 }
 
-static struct cleaners *cleaner_init()
+struct cleaners *cleaners_init()
 {
     int ret = 0;
     struct cleaners *clns = create_cleaners();
-    
+
     if (clns == NULL) {
         return NULL;
     }
 
     ret = add_clean_node(clns, default_cleaner, "default clean");
     if (ret != 0) {
-        ERROR("add default_cleaner error");
+        ERROR("Add default_cleaner error");
         return clns;
     }
 
 #ifdef ENABLE_OCI_IMAGE
+    ret = add_clean_node(clns, oci_broken_rootfs_cleaner, "clean broken rootfs");
+    if (ret != 0) {
+        ERROR("Clean broken rootfs failed");
+        return clns;
+    }
+
     ret = add_clean_node(clns, oci_rootfs_cleaner, "clean rootfs");
     if (ret != 0) {
-        ERROR("add oci_rootfs_cleaner error");
+        ERROR("Add oci_rootfs_cleaner error");
         return clns;
     }
 #endif
@@ -106,7 +116,7 @@ static struct cleaners *cleaner_init()
     return clns;
 }
 
-static void do_clean(struct cleaners * clns)
+void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx)
 {
     struct linked_list *it = NULL;
     struct linked_list *next = NULL;
@@ -114,31 +124,11 @@ static void do_clean(struct cleaners * clns)
 
     linked_list_for_each_safe(it, &(clns->cleaner_list), next) {
         c_node = (struct clean_node *)it->elem;
-        if (c_node->cleaner() != 0) {
-            ERROR("failed to clean for: %s", c_node->desc);
+        if (c_node->cleaner(ctx) != 0) {
+            ERROR("Failed to clean for: %s", c_node->desc);
         } else {
             DEBUG("do clean success for: %s", c_node->desc);
             clns->done_clean++;
         }
     }
 }
-
-void clean_leftover()
-{
-    struct cleaners *clns = cleaner_init();
-
-    if (clns == NULL) {
-        ERROR("failed to clean leftovers, because cleaner init error");
-        return;
-    }
-
-    do_clean(clns);
-
-    if (clns->count == clns->done_clean) {
-        DEBUG("all clean up success");
-    } else {
-        ERROR("Aim to do %d clean, %d clean sucess\n", clns->count, clns->done_clean);
-    }
-
-    destroy_cleaners(clns);
-}
\ No newline at end of file
diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.h b/src/daemon/modules/container/leftover_cleanup/cleanup.h
index efae99d0..8dd5e9bd 100644
--- a/src/daemon/modules/container/leftover_cleanup/cleanup.h
+++ b/src/daemon/modules/container/leftover_cleanup/cleanup.h
@@ -12,19 +12,20 @@
  * Create: 2022-10-31
  * Description: provide cleanup definition
  *********************************************************************************/
-#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_H
-#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_H
+#ifndef DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEANERS_H
+#define DAEMON_MODULES_CONTAINER_LEFTOVER_CLEANUP_CLEANERS_H
 
 #include <stdlib.h>
 
 #include "linked_list.h"
 #include "isula_libutils/log.h"
+#include "clean_context.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif
 
-typedef int clean_func_t(void);
+typedef int clean_func_t(struct clean_ctx *ctx);
 
 struct clean_node {
     const char *desc;
@@ -38,8 +39,11 @@ struct cleaners {
     struct linked_list cleaner_list;
 };
 
+struct cleaners *cleaners_init();
 
-void clean_leftover();
+void destroy_cleaners(struct cleaners *clns);
+
+void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx);
 
 
 #if defined(__cplusplus) || defined(c_plusplus)
diff --git a/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c
new file mode 100644
index 00000000..7bdaef22
--- /dev/null
+++ b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c
@@ -0,0 +1,71 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2022. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2022-10-31
+ * Description: provide cleanup functions
+ *********************************************************************************/
+#include "utils.h"
+#include "leftover_cleanup_api.h"
+#include "cleanup.h"
+#include "clean_context.h"
+
+struct clean_ctx *g_clean_ctx = NULL;
+struct cleaners *g_clns = NULL;
+
+int clean_module_init()
+{
+    // create cleaners and clean_ctx
+    g_clns = cleaners_init();
+    if (g_clns == NULL) {
+        ERROR("Failed to init clean module");
+        return -1;
+    }
+
+    g_clean_ctx = clean_ctx_init();
+    if (g_clean_ctx == NULL) {
+        ERROR("Failed to init clean module");
+        destroy_cleaners(g_clns);
+        return -1;
+    }
+
+    return 0;
+}
+
+void clean_module_fill_ctx(cleanup_ctx_data_t data_type, void *data)
+{
+    switch (data_type) {
+        case BROKEN_ROOTFS:
+            clean_ctx_fill_broken_rootfs(g_clean_ctx, data);
+            break;
+    }
+}
+
+void clean_module_do_clean()
+{
+    if (g_clns == NULL || g_clean_ctx == NULL) {
+        return;
+    }
+
+    cleaners_do_clean(g_clns, g_clean_ctx);
+
+    if (g_clns->count == g_clns->done_clean) {
+        DEBUG("all clean up success");
+    } else {
+        ERROR("Aim to do %d clean, %d clean sucess\n", g_clns->count, g_clns->done_clean);
+    }
+
+    destroy_cleaners(g_clns);
+    clean_ctx_destroy(g_clean_ctx);
+
+    g_clns = NULL;
+    g_clean_ctx = NULL;
+}
+
diff --git a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c
index fbef4ce0..b2205569 100644
--- a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c
+++ b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.c
@@ -19,6 +19,7 @@
 #include "image_api.h"
 #include "utils_file.h"
 #include "utils.h"
+#include "linked_list.h"
 
 struct cb_result {
     int clean_err_cnt;
@@ -45,7 +46,7 @@ static bool walk_dir_cb(const char *path_name, const struct dirent *sub_dir, voi
 }
 
 
-int oci_rootfs_cleaner(void)
+int oci_rootfs_cleaner(struct clean_ctx *ctx)
 {
     struct cb_result res = { 0 };
     im_get_rf_dir_request request = { 0 };
@@ -67,7 +68,36 @@ int oci_rootfs_cleaner(void)
 
     if (res.clean_err_cnt == 0) {
         return 0;
-    } 
+    }
 
     return -1;
 }
+
+int oci_broken_rootfs_cleaner(struct clean_ctx *ctx)
+{
+    int rm_fail_cnt = 0;
+    struct linked_list *it = NULL;
+    struct linked_list *next = NULL;
+    char *id = NULL;
+
+    if (ctx == NULL) {
+        return -1;
+    }
+
+    linked_list_for_each_safe(it, &(ctx->broken_rootfs_list), next) {
+        id = (char *)it->elem;
+        if (im_remove_broken_rootfs(IMAGE_TYPE_OCI, id) != 0) {
+            ERROR("Failed to clean broken rootfs %s", id);
+            rm_fail_cnt++;
+        } else {
+            EVENT("clean broken rootfs succeed %s", id);
+        }
+    }
+
+    if (rm_fail_cnt != 0) {
+        DEBUG("can't clean some broken rootfs, %d left", rm_fail_cnt);
+        return -1;
+    }
+
+    return 0;
+}
\ No newline at end of file
diff --git a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h
index 8dff351f..88920fa9 100644
--- a/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h
+++ b/src/daemon/modules/container/leftover_cleanup/oci_rootfs_clean.h
@@ -21,7 +21,9 @@
 extern "C" {
 #endif
 
-int oci_rootfs_cleaner(void);
+int oci_rootfs_cleaner(struct clean_ctx *ctx);
+
+int oci_broken_rootfs_cleaner(struct clean_ctx *ctx);
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }
diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c
index 8fb226aa..6fac7cc4 100644
--- a/src/daemon/modules/image/image.c
+++ b/src/daemon/modules/image/image.c
@@ -50,6 +50,7 @@ struct bim_ops {
     int (*export_rf)(const im_export_request *request);
     char *(*resolve_image_name)(const char *image_name);
     char *(*get_dir_rf)(void);
+    int (*delete_broken_rf)(const im_delete_rootfs_request *request);
 
     /* merge image config ops */
     int (*merge_conf)(const char *img_name, container_config *container_spec);
@@ -132,6 +133,7 @@ static const struct bim_ops g_embedded_ops = {
     .mount_rf = embedded_mount_rf,
     .umount_rf = embedded_umount_rf,
     .delete_rf = embedded_delete_rf,
+    .delete_broken_rf = NULL,
     .export_rf = NULL,
     .get_dir_rf = NULL,
 
@@ -167,6 +169,7 @@ static const struct bim_ops g_oci_ops = {
     .mount_rf = oci_mount_rf,
     .umount_rf = oci_umount_rf,
     .delete_rf = oci_delete_rf,
+    .delete_broken_rf = oci_delete_broken_rf,
     .export_rf = oci_export_rf,
     .get_dir_rf = oci_get_dir_rf,
 
@@ -201,6 +204,7 @@ static const struct bim_ops g_ext_ops = {
     .mount_rf = ext_mount_rf,
     .umount_rf = ext_umount_rf,
     .delete_rf = ext_delete_rf,
+    .delete_broken_rf = NULL,
     .export_rf = NULL,
     .get_dir_rf = NULL,
 
@@ -464,6 +468,50 @@ out:
     return ret;
 }
 
+int im_remove_broken_rootfs(const char *image_type, const char *container_id)
+{
+    int ret = 0;
+    im_delete_rootfs_request *request = NULL;
+    struct bim *bim = NULL;
+
+    if (container_id == NULL || image_type == NULL) {
+        ERROR("Invalid input arguments");
+        return -1;
+    }
+
+    bim = bim_get(image_type, NULL, NULL, container_id);
+    if (bim == NULL) {
+        ERROR("Failed to init bim when deleting broken rootfs %s", container_id);
+        return -1;
+    }
+
+    if (bim->ops->delete_broken_rf == NULL) {
+        ERROR("Unimplements delete in %s", bim->type);
+        ret = -1;
+        goto out;
+    }
+
+    request = util_common_calloc_s(sizeof(im_delete_rootfs_request));
+    if (request == NULL) {
+        ERROR("Out of memory");
+        ret = -1;
+        goto out;
+    }
+    request->name_id = util_strdup_s(container_id);
+
+    ret = bim->ops->delete_broken_rf(request);
+    if (ret != 0) {
+        ERROR("Failed to delete rootfs for container %s", container_id);
+        ret = -1;
+        goto out;
+    }
+
+out:
+    bim_put(bim);
+    free_im_delete_request(request);
+    return ret;
+}
+
 int im_remove_container_rootfs(const char *image_type, const char *container_id)
 {
     int ret = 0;
diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c
index e951adb4..0fc0b7fe 100644
--- a/src/daemon/modules/image/oci/oci_image.c
+++ b/src/daemon/modules/image/oci/oci_image.c
@@ -368,6 +368,16 @@ out:
     return ret;
 }
 
+int oci_delete_broken_rf(const im_delete_rootfs_request *request)
+{
+    if (request == NULL) {
+        ERROR("Request is NULL");
+        return -1;
+    }
+
+    return storage_broken_rw_layer_delete(request->name_id);
+}
+
 int oci_delete_rf(const im_delete_rootfs_request *request)
 {
     if (request == NULL) {
diff --git a/src/daemon/modules/image/oci/oci_image.h b/src/daemon/modules/image/oci/oci_image.h
index aeeb3b65..cd7da336 100644
--- a/src/daemon/modules/image/oci/oci_image.h
+++ b/src/daemon/modules/image/oci/oci_image.h
@@ -53,6 +53,7 @@ int oci_merge_conf_rf(const char *img_name, container_config *container_spec);
 int oci_mount_rf(const im_mount_request *request);
 int oci_umount_rf(const im_umount_request *request);
 int oci_delete_rf(const im_delete_rootfs_request *request);
+int oci_delete_broken_rf(const im_delete_rootfs_request *request);
 int oci_export_rf(const im_export_request *request);
 char *oci_get_dir_rf(void);
 int oci_container_filesystem_usage(const im_container_fs_usage_request *request, imagetool_fs_info **fs_usage);
diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
index e563a8ef..b9ab0d65 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
@@ -557,6 +557,7 @@ static int update_layer_datas(const char *id, const struct layer_opts *opts, lay
 
     slayer->id = util_strdup_s(id);
     slayer->parent = util_strdup_s(opts->parent);
+    slayer->writable = opts->writable;
     if (opts->opts != NULL) {
         slayer->mountlabel = util_strdup_s(opts->opts->mount_label);
     }
@@ -1434,6 +1435,7 @@ static void copy_json_to_layer(const layer_t *jl, struct layer *l)
         l->mount_point = util_strdup_s(jl->smount_point->path);
         l->mount_count = jl->smount_point->count;
     }
+    l->writable = jl->slayer->writable;
 }
 
 int layer_store_list(struct layer_list *resp)
diff --git a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c
index 97cc39e8..c6e305a0 100644
--- a/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c
+++ b/src/daemon/modules/image/oci/storage/rootfs_store/rootfs_store.c
@@ -38,6 +38,10 @@
 #include "utils_string.h"
 #include "utils_timestamp.h"
 
+#ifndef DISABLE_CLEANUP
+#include "leftover_cleanup_api.h"
+#endif
+
 #define CONTAINER_JSON "container.json"
 
 typedef struct rootfs_store {
@@ -170,6 +174,7 @@ static int append_container_by_directory(const char *container_dir)
 
     nret = snprintf(container_path, sizeof(container_path), "%s/%s", container_dir, CONTAINER_JSON);
     if (nret < 0 || (size_t)nret >= sizeof(container_path)) {
+        // snprintf error, not append, but outside should not delete the rootfs
         ERROR("Failed to get container path");
         return -1;
     }
@@ -182,6 +187,7 @@ static int append_container_by_directory(const char *container_dir)
     }
 
     if (do_append_container(c) != 0) {
+        // append error should not return -1, outside should not remove rootfs
         ERROR("Failed to append container");
         ret = -1;
         goto out;
@@ -199,6 +205,7 @@ static int get_containers_from_json()
 {
     int ret = 0;
     int nret;
+    int append_ret = 0;
     char **container_dirs = NULL;
     size_t container_dirs_num = 0;
     size_t i;
@@ -231,7 +238,11 @@ static int get_containers_from_json()
             continue;
         }
 
-        if (append_container_by_directory(container_path) != 0) {
+        append_ret = append_container_by_directory(container_path);
+        if (append_ret != 0) {
+#ifndef DISABLE_CLEANUP
+            clean_module_fill_ctx(BROKEN_ROOTFS, (void *)container_dirs[i]);
+#endif
             ERROR("Found container path but load json failed: %s, deleting...", container_path);
             if (util_recursive_rmdir(container_path, 0) != 0) {
                 ERROR("Failed to delete rootfs directory : %s", container_path);
diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c
index 6cb4a51b..829ea8d0 100644
--- a/src/daemon/modules/image/oci/storage/storage.c
+++ b/src/daemon/modules/image/oci/storage/storage.c
@@ -306,6 +306,38 @@ struct layer *storage_layer_get(const char *layer_id)
     return layer_store_lookup(layer_id);
 }
 
+int storage_broken_rw_layer_delete(const char *layer_id)
+{
+    int ret = 0;
+    struct layer *layer_info = NULL;
+
+    if (layer_id == NULL) {
+        return -1;
+    }
+
+    layer_info = layer_store_lookup(layer_id);
+    if (layer_info == NULL) {
+        ERROR("Failed to get layer info for layer %s", layer_id);
+        return -1;
+    }
+
+    if (!layer_info->writable) {
+        ERROR("Broken rootfs should only delete rw layer, layer %s is ro layer", layer_id);
+        ret = -1;
+        goto out;
+    }
+
+    // delete rootfs and rw layer, rw layer has the same name as rootfs
+    if (layer_store_delete(layer_info->id) != 0) {
+        ERROR("Can't delete layer of broken rootfs");
+        ret = -1;
+    }
+
+out:
+    free_layer(layer_info);
+    return ret;
+}
+
 void free_layer(struct layer *ptr)
 {
     if (ptr == NULL) {
diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h
index 5914adec..3ec47959 100644
--- a/src/daemon/modules/image/oci/storage/storage.h
+++ b/src/daemon/modules/image/oci/storage/storage.h
@@ -42,6 +42,7 @@ struct layer {
     int64_t compress_size;
     char *uncompressed_digest;
     int64_t uncompress_size;
+    bool writable;
 };
 
 struct layer_list {
@@ -163,6 +164,8 @@ struct layer_list *storage_layers_get_by_compress_digest(const char *digest);
 
 struct layer *storage_layer_get(const char *layer_id);
 
+int storage_broken_rw_layer_delete(const char *layer_id);
+
 int storage_layer_try_repair_lowers(const char *layer_id, const char *last_layer_id);
 
 void free_layer(struct layer *l);
diff --git a/test/image/oci/storage/rootfs/CMakeLists.txt b/test/image/oci/storage/rootfs/CMakeLists.txt
index 8da8196b..5bf568f9 100644
--- a/test/image/oci/storage/rootfs/CMakeLists.txt
+++ b/test/image/oci/storage/rootfs/CMakeLists.txt
@@ -2,6 +2,8 @@ project(iSulad_UT)
 
 SET(EXE storage_rootfs_ut)
 
+add_definitions(-DDISABLE_CLEANUP)
+
 add_executable(${EXE}
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils.c
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils_regex.c
-- 
2.25.1