iSulad/0032-remote-layer-store-demo.patch

From 9072007e7f61f7658baf7c4101126040dc341d0b Mon Sep 17 00:00:00 2001
From: "Neil.wrz" <wangrunze13@huawei.com>
Date: Mon, 9 Jan 2023 17:12:53 -0800
Subject: [PATCH 32/53] remote layer store demo

Signed-off-by: Neil <wangrunze13@huawei.com>
---
 cmake/options.cmake                           |   6 +
 src/daemon/config/isulad_config.c             |   1 +
 src/daemon/modules/image/oci/oci_image.c      |   6 +-
 .../modules/image/oci/storage/CMakeLists.txt  |   6 +
 .../storage/image_store/image_remote_impl.c   | 173 +++++++++
 .../oci/storage/image_store/image_store.c     |  76 +++-
 .../oci/storage/image_store/image_store.h     |  11 +
 .../storage/layer_store/graphdriver/driver.c  |   4 +-
 .../storage/layer_store/graphdriver/driver.h  |   3 +
 .../graphdriver/overlay2/driver_overlay2.c    | 158 +++++++-
 .../graphdriver/overlay2/driver_overlay2.h    |   8 +
 .../overlay2/overlay_remote_impl.c            | 282 ++++++++++++++
 .../storage/layer_store/layer_remote_impl.c   | 219 +++++++++++
 .../oci/storage/layer_store/layer_store.c     | 199 +++++++++-
 .../oci/storage/layer_store/layer_store.h     |  11 +
 .../remote_layer_support/CMakeLists.txt       |  12 +
 .../remote_layer_support/remote_support.c     | 122 ++++++
 .../remote_layer_support/remote_support.h     |  58 +++
 .../ro_symlink_maintain.c                     | 347 ++++++++++++++++++
 .../ro_symlink_maintain.h                     |  52 +++
 .../modules/image/oci/storage/storage.c       |  11 +-
 .../modules/image/oci/storage/storage.h       |   3 +
 22 files changed, 1750 insertions(+), 18 deletions(-)
 create mode 100644 src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c
 create mode 100644 src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c
 create mode 100644 src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
 create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt
 create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
 create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
 create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c
 create mode 100644 src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h

diff --git a/cmake/options.cmake b/cmake/options.cmake
index a357a504..44ccf672 100644
--- a/cmake/options.cmake
+++ b/cmake/options.cmake
@@ -148,6 +148,12 @@ if (DISABLE_CLEANUP STREQUAL "ON")
     message("${Green}--  Disable cleanup module")
 endif()
 
+option(ENABLE_REMOTE_LAYER_STORE "enable remote layer store" OFF)
+if (ENABLE_REMOTE_LAYER_STORE STREQUAL "ON")
+    add_definitions(-DENABLE_REMOTE_LAYER_STORE)
+    message("${Green}--  Enable remote layer store")
+endif()
+
 option(MUSL "available for musl" OFF)
 if (MUSL)
     add_definitions(-D__MUSL__)
diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c
index 38bf4bf9..b7bfe2df 100644
--- a/src/daemon/config/isulad_config.c
+++ b/src/daemon/config/isulad_config.c
@@ -1555,6 +1555,7 @@ static int merge_authorization_conf_into_global(struct service_arguments *args,
 static int merge_storage_conf_into_global(struct service_arguments *args, isulad_daemon_configs *tmp_json_confs)
 {
     override_string_value(&args->json_confs->storage_driver, &tmp_json_confs->storage_driver);
+    args->json_confs->storage_enable_remote_layer = tmp_json_confs->storage_enable_remote_layer;
 
     if (string_array_append(tmp_json_confs->storage_opts, tmp_json_confs->storage_opts_len,
                             &(args->json_confs->storage_opts_len), &(args->json_confs->storage_opts)) != 0) {
diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c
index 50d13cec..fa92a861 100644
--- a/src/daemon/modules/image/oci/oci_image.c
+++ b/src/daemon/modules/image/oci/oci_image.c
@@ -218,6 +218,10 @@ static int storage_module_init_helper(const isulad_daemon_configs *args)
         goto out;
     }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    storage_opts->enable_remote_layer = args->storage_enable_remote_layer;
+#endif
+
     if (util_dup_array_of_strings((const char **)args->storage_opts, args->storage_opts_len, &storage_opts->driver_opts,
                                   &storage_opts->driver_opts_len) != 0) {
         ERROR("Failed to get storage storage opts");
@@ -763,4 +767,4 @@ int oci_search(const im_search_request *request, imagetool_search_result **resul
 
     return ret;
 }
-#endif
\ No newline at end of file
+#endif
diff --git a/src/daemon/modules/image/oci/storage/CMakeLists.txt b/src/daemon/modules/image/oci/storage/CMakeLists.txt
index 06c2f378..24b71450 100644
--- a/src/daemon/modules/image/oci/storage/CMakeLists.txt
+++ b/src/daemon/modules/image/oci/storage/CMakeLists.txt
@@ -3,12 +3,17 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_storage_srcs)
 add_subdirectory(image_store)
 add_subdirectory(layer_store)
 add_subdirectory(rootfs_store)
+IF (ENABLE_REMOTE_LAYER_STORE)
+add_subdirectory(remote_layer_support)
+ENDIF()
+
 
 set(STORAGE_SRCS
     ${local_storage_srcs}
     ${IMAGE_STORE_SRCS}
     ${LAYER_STORE_SRCS}
     ${ROOTFS_STORE_SRCS}
+    ${REMOTE_LAYER_SUPPORT_SRCS}
     PARENT_SCOPE
     )
 
@@ -17,5 +22,6 @@ set(STORAGE_INCS
     ${IMAGE_STORE_INCS}
     ${LAYER_STORE_INCS}
     ${ROOTFS_STORE_INCS}
+    ${REMOTE_LAYER_SUPPORT_INCS}
     PARENT_SCOPE
     )
diff --git a/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c
new file mode 100644
index 00000000..20da8116
--- /dev/null
+++ b/src/daemon/modules/image/oci/storage/image_store/image_remote_impl.c
@@ -0,0 +1,173 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2023-03-03
+ * Description: provide remote image store functions
+ ******************************************************************************/
+#define _GNU_SOURCE
+#include "image_store.h"
+
+#include <isula_libutils/log.h>
+#include <stdio.h>
+
+#include "remote_support.h"
+#include "ro_symlink_maintain.h"
+#include "map.h"
+#include "utils_file.h"
+#include "utils.h"
+#include "layer_store.h"
+#include "utils_array.h"
+
+struct remote_image_data {
+    const char *image_home;
+};
+
+static map_t *image_byid_old = NULL;
+static map_t *image_byid_new = NULL;
+
+static void *remote_support_create(const char *remote_home, const char *remote_ro)
+{
+    struct remote_image_data *data = util_common_calloc_s(sizeof(struct remote_image_data));
+    if (data == NULL) {
+        ERROR("Out of memory");
+        return NULL;
+    }
+    data->image_home = remote_home;
+    image_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
+    image_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
+    return data;
+}
+
+static void remote_support_destroy(void *data)
+{
+    if (data == NULL) {
+        return;
+    }
+
+    map_free(image_byid_old);
+    map_free(image_byid_new);
+
+    free(data);
+    return;
+}
+
+static int remote_support_scan(void *data)
+{
+    int ret = 0;
+    int nret;
+    char **image_dirs = NULL;
+    size_t image_dirs_num = 0;
+    size_t i;
+    char *id_patten = "^[a-f0-9]{64}$";
+    char image_path[PATH_MAX] = { 0x00 };
+    bool exist = true;
+    struct remote_image_data *img_data = (struct remote_image_data *)data;
+
+    ret = util_list_all_subdir(img_data->image_home, &image_dirs);
+    if (ret != 0) {
+        ERROR("Failed to get images directory");
+        goto out;
+    }
+    image_dirs_num = util_array_len((const char **)image_dirs);
+
+    for (i = 0; i < image_dirs_num; i++) {
+        bool valid_v1_image = false;
+
+        if (util_reg_match(id_patten, image_dirs[i]) != 0) {
+            DEBUG("Image's json is placed inside image's data directory, so skip any other file or directory: %s",
+                  image_dirs[i]);
+            continue;
+        }
+
+        nret = snprintf(image_path, sizeof(image_path), "%s/%s", img_data->image_home, image_dirs[i]);
+        if (nret < 0 || (size_t)nret >= sizeof(image_path)) {
+            ERROR("Failed to get image path");
+            continue;
+        }
+
+        if (validate_manifest_schema_version_1(image_path, &valid_v1_image) != 0) {
+            ERROR("Failed to validate manifest schema version 1 format");
+            continue;
+        }
+
+        if (!valid_v1_image) {
+            map_insert(image_byid_new, util_strdup_s(image_dirs[i]), (void *)&exist);
+        }
+    }
+
+out:
+    util_free_array(image_dirs);
+    return ret;
+}
+
+static int remote_support_add(void *data)
+{
+    char **array_added = NULL;
+    char **array_deleted = NULL;
+    char *top_layer = NULL;
+    map_t *tmp_map = NULL;
+    int i = 0;
+    int ret = 0;
+
+    if (data == NULL) {
+        return -1;
+    }
+
+    array_added = added_layers(image_byid_old, image_byid_new);
+    array_deleted = deleted_layers(image_byid_old, image_byid_new);
+
+    for (i = 0; i < util_array_len((const char **)array_added); i++) {
+        top_layer = get_top_layer_from_json(array_added[i]);
+        if (top_layer != NULL && !layer_remote_layer_valid(top_layer)) {
+            ERROR("ERROR not find valid under layer, remoet image:%s not added", array_added[i]);
+            map_remove(image_byid_new, (void *)array_added[i]);
+            continue;
+        }
+
+        if (append_image_by_directory_with_lock(array_added[i]) != 0) {
+            ERROR("Failed to load image into memrory: %s", array_added[i]);
+            ret = -1;
+        }
+    }
+
+    for (i = 0; i < util_array_len((const char **)array_deleted); i++) {
+        if (remove_image_from_memory_with_lock(array_deleted[i]) != 0) {
+            ERROR("Failed to remove remote memory store");
+            ret = -1;
+        }
+    }
+
+    tmp_map = image_byid_old;
+    image_byid_old = image_byid_new;
+    image_byid_new = tmp_map;
+    empty_map(image_byid_new);
+
+    util_free_array(array_added);
+    util_free_array(array_deleted);
+    free(top_layer);
+
+    return ret;
+}
+
+remote_support *image_store_impl_remote_support(void)
+{
+    remote_support *rs = util_common_calloc_s(sizeof(remote_support));
+    if (rs == NULL) {
+        return NULL;
+    }
+
+    rs->create = remote_support_create;
+    rs->destroy = remote_support_destroy;
+    rs->scan_remote_dir = remote_support_scan;
+    rs->load_item = remote_support_add;
+
+    return rs;
+}
diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c
index cf1e88ff..caff3705 100644
--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c
+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c
@@ -49,6 +49,9 @@
 #include "image_type.h"
 #include "linked_list.h"
 #include "utils_verify.h"
+#ifdef ENABLE_REMOTE_LAYER_STORE
+#include "ro_symlink_maintain.h"
+#endif
 
 // the name of the big data item whose contents we consider useful for computing a "digest" of the
 // image, by which we can locate the image later.
@@ -3096,7 +3099,7 @@ out:
     return ret;
 }
 
-static int validate_manifest_schema_version_1(const char *path, bool *valid)
+int validate_manifest_schema_version_1(const char *path, bool *valid)
 {
     int ret = 0;
     int nret;
@@ -3640,6 +3643,10 @@ int image_store_init(struct storage_module_init_options *opts)
         goto out;
     }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    remote_image_init(g_image_store->dir);
+#endif
+
 out:
     if (ret != 0) {
         free_image_store(g_image_store);
@@ -3648,3 +3655,70 @@ out:
     free(root_dir);
     return ret;
 }
+
+#ifdef ENABLE_REMOTE_LAYER_STORE
+int append_image_by_directory_with_lock(const char *id)
+{
+    int ret = 0;
+    int nret = 0;
+    char image_path[PATH_MAX] = { 0x00 };
+
+    if (!image_store_lock(EXCLUSIVE)) {
+        ERROR("Failed to lock remote image store when handle: %s", id);
+        return -1;
+    }
+
+    nret = snprintf(image_path, sizeof(image_path), "%s/%s", g_image_store->dir, id);
+    if (nret < 0 || (size_t)nret >= sizeof(image_path)) {
+        ERROR("Failed to get image path");
+        return -1;
+    }
+
+    ret = append_image_by_directory(image_path);
+    image_store_unlock();
+
+    return ret;
+}
+
+int remove_image_from_memory_with_lock(const char *id)
+{
+    int ret = 0;
+
+    if (!image_store_lock(EXCLUSIVE)) {
+        ERROR("Failed to lock remote image store when handle: %s", id);
+        return -1;
+    }
+
+    ret = remove_image_from_memory(id);
+    image_store_unlock();
+
+    return ret;
+}
+
+char *get_top_layer_from_json(const char *img_id)
+{
+
+    char *ret = NULL;
+    int nret = 0;
+    char image_path[PATH_MAX] = { 0x00 };
+    storage_image *im = NULL;
+    parser_error err = NULL;
+
+    nret = snprintf(image_path, sizeof(image_path), "%s/%s/%s", g_image_store->dir, img_id, IMAGE_JSON);
+    if (nret < 0 || (size_t)nret >= sizeof(image_path)) {
+        ERROR("Failed to get image path");
+        return NULL;
+    }
+
+    im = storage_image_parse_file(image_path, NULL, &err);
+    if (im == NULL) {
+        ERROR("Failed to parse images path: %s", err);
+        return NULL;
+    }
+
+    ret = util_strdup_s(im->layer);
+    free_storage_image(im);
+
+    return ret;
+}
+#endif
diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.h b/src/daemon/modules/image/oci/storage/image_store/image_store.h
index edd4fa2c..c3cb50e3 100644
--- a/src/daemon/modules/image/oci/storage/image_store/image_store.h
+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.h
@@ -28,6 +28,9 @@
 #include "isula_libutils/imagetool_image.h"
 #include "isula_libutils/imagetool_images_list.h"
 #include "isula_libutils/imagetool_image_summary.h"
+#ifdef ENABLE_REMOTE_LAYER_STORE
+#include "remote_support.h"
+#endif
 
 struct storage_module_init_options;
 
@@ -108,6 +111,14 @@ void image_store_free();
 
 imagetool_image_summary *image_store_get_image_summary(const char *id);
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+remote_support *image_store_impl_remote_support();
+int validate_manifest_schema_version_1(const char *path, bool *valid);
+int append_image_by_directory_with_lock(const char *image_dir);
+int remove_image_from_memory_with_lock(const char *id);
+char *get_top_layer_from_json(const char *img_id); /* return top layer id */
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c
index d3b5209a..b83c63b1 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.c
@@ -141,7 +141,9 @@ int graphdriver_init(const struct storage_module_init_options *opts)
                 ret = -1;
                 goto out;
             }
-
+#ifdef ENABLE_REMOTE_LAYER_STORE
+            g_drivers[i].enable_remote_layer = opts->enable_remote_layer;
+#endif
             if (g_drivers[i].ops->init(&g_drivers[i], driver_home, (const char **)opts->driver_opts,
                                        opts->driver_opts_len) != 0) {
                 ret = -1;
diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h
index 7faf70c8..acd847cc 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/driver.h
@@ -90,6 +90,9 @@ struct graphdriver {
     bool support_dtype;
 
     bool support_quota;
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    bool enable_remote_layer;
+#endif
     struct pquota_control *quota_ctrl;
 
     // options for overlay2
diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c
index 330c230a..eedbeef2 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c
@@ -12,6 +12,7 @@
  * Create: 2020-04-02
  * Description: provide overlay2 function definition
  ******************************************************************************/
+#define _GNU_SOURCE
 #include "driver_overlay2.h"
 
 #include <string.h>
@@ -44,6 +45,9 @@
 #include "utils_timestamp.h"
 #include "selinux_label.h"
 #include "err_msg.h"
+#ifdef ENABLE_REMOTE_LAYER_STORE
+#include "ro_symlink_maintain.h"
+#endif
 
 struct io_read_wrapper;
 
@@ -343,6 +347,13 @@ int overlay2_init(struct graphdriver *driver, const char *driver_home, const cha
         return -1;
     }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    if (driver->enable_remote_layer && remote_overlay_init(driver_home) != 0) {
+        ERROR("Failed to init overlay remote");
+        return -1;
+    }
+#endif
+
     driver->home = util_strdup_s(driver_home);
 
     root_dir = util_path_dir(driver_home);
@@ -423,7 +434,7 @@ static int mk_diff_directory(const char *layer_dir)
     int ret = 0;
     char *diff_dir = NULL;
 #ifdef ENABLE_USERNS_REMAP
-    char* userns_remap = conf_get_isulad_userns_remap();
+    char *userns_remap = conf_get_isulad_userns_remap();
 #endif
 
     diff_dir = util_path_join(layer_dir, OVERLAY_LAYER_DIFF);
@@ -538,7 +549,7 @@ static int mk_work_directory(const char *layer_dir)
     int ret = 0;
     char *work_dir = NULL;
 #ifdef ENABLE_USERNS_REMAP
-    char* userns_remap = conf_get_isulad_userns_remap();
+    char *userns_remap = conf_get_isulad_userns_remap();
 #endif
 
     work_dir = util_path_join(layer_dir, OVERLAY_LAYER_WORK);
@@ -575,7 +586,7 @@ static int mk_merged_directory(const char *layer_dir)
     int ret = 0;
     char *merged_dir = NULL;
 #ifdef ENABLE_USERNS_REMAP
-    char* userns_remap = conf_get_isulad_userns_remap();
+    char *userns_remap = conf_get_isulad_userns_remap();
 #endif
 
     merged_dir = util_path_join(layer_dir, OVERLAY_LAYER_MERGED);
@@ -852,13 +863,115 @@ out:
     return ret;
 }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+static int do_create_remote_ro(const char *id, const char *parent, const struct graphdriver *driver,
+                               const struct driver_create_opts *create_opts)
+{
+    int ret = 0;
+    int get_err = 0;
+    char *ro_symlink = NULL;
+    char *ro_home = NULL;
+    char *layer_dir = NULL;
+#ifdef ENABLE_USERNS_REMAP
+    char *userns_remap = conf_get_isulad_userns_remap();
+#endif
+
+    ro_home = util_path_join(driver->home, OVERLAY_RO_DIR);
+    if (ro_home == NULL) {
+        ERROR("Failed to join ro_home");
+        ret = -1;
+        goto out;
+    }
+
+    layer_dir = util_path_join(ro_home, id);
+    if (layer_dir == NULL) {
+        ERROR("Failed to join layer_dir");
+        ret = -1;
+        goto out;
+    }
+
+    ro_symlink = util_path_join(driver->home, id);
+    if (ro_symlink == NULL) {
+        ERROR("Failed to join ro_symlink");
+        ret = -1;
+        goto out;
+    }
+
+    if (layer_dir == NULL) {
+        ERROR("Failed to join layer dir:%s", id);
+        ret = -1;
+        goto out;
+    }
+
+    if (check_parent_valid(parent, driver) != 0) {
+        ret = -1;
+        goto out;
+    }
+
+    if (util_mkdir_p(layer_dir, 0700) != 0) {
+        ERROR("Unable to create layer directory %s.", layer_dir);
+        ret = -1;
+        goto out;
+    }
+
+    // mk symbol link
+    if (symlink(layer_dir, ro_symlink) != 0) {
+        SYSERROR("Unable to create symbol link to layer directory %s", layer_dir);
+        ret = -1;
+        goto err_out;
+    }
+
+#ifdef ENABLE_USERNS_REMAP
+    if (set_file_owner_for_userns_remap(layer_dir, userns_remap) != 0) {
+        ERROR("Unable to change directory %s owner for user remap.", layer_dir);
+        ret = -1;
+        goto out;
+    }
+#endif
+
+    if (create_opts->storage_opt != NULL && create_opts->storage_opt->len != 0) {
+        if (set_layer_quota(layer_dir, create_opts->storage_opt, driver) != 0) {
+            ERROR("Unable to set layer quota %s", layer_dir);
+            ret = -1;
+            goto err_out;
+        }
+    }
+
+    if (mk_sub_directories(id, parent, layer_dir, driver->home) != 0) {
+        ret = -1;
+        goto err_out;
+    }
+
+    goto out;
+
+err_out:
+    if (util_recursive_rmdir(layer_dir, 0)) {
+        ERROR("Failed to delete layer path: %s", layer_dir);
+    }
+
+    // to remove a file
+    if (util_fileself_exists(ro_symlink) && !util_force_remove_file(ro_symlink, &get_err)) {
+        ERROR("Failed to remove symbol link %s", ro_symlink);
+    }
+
+out:
+    free(layer_dir);
+    free(ro_home);
+    free(ro_symlink);
+#ifdef ENABLE_USERNS_REMAP
+    free(userns_remap);
+#endif
+    return ret;
+}
+#endif
+
 static int do_create(const char *id, const char *parent, const struct graphdriver *driver,
                      const struct driver_create_opts *create_opts)
 {
     int ret = 0;
     char *layer_dir = NULL;
 #ifdef ENABLE_USERNS_REMAP
-    char* userns_remap = conf_get_isulad_userns_remap();
+    char *userns_remap = conf_get_isulad_userns_remap();
 #endif
 
     layer_dir = util_path_join(driver->home, id);
@@ -1002,7 +1115,15 @@ int overlay2_create_ro(const char *id, const char *parent, const struct graphdri
         return -1;
     }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    if (driver->enable_remote_layer) {
+        return do_create_remote_ro(id, parent, driver, create_opts);
+    } else {
+        return do_create(id, parent, driver, create_opts);
+    }
+#else
     return do_create(id, parent, driver, create_opts);
+#endif
 }
 
 static char *read_layer_link_file(const char *layer_dir)
@@ -1047,6 +1168,9 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver)
     char *link_id = NULL;
     char link_path[PATH_MAX] = { 0 };
     char clean_path[PATH_MAX] = { 0 };
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    struct stat stat_buf;
+#endif
 
     if (id == NULL || driver == NULL) {
         ERROR("Invalid input arguments");
@@ -1079,11 +1203,34 @@ int overlay2_rm_layer(const char *id, const struct graphdriver *driver)
         }
     }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    if (lstat(layer_dir, &stat_buf) < 0) {
+        SYSERROR("Failed to lstat path: %s", layer_dir);
+        ret = -1;
+        goto out;
+    }
+
+    if (driver->enable_remote_layer && S_ISLNK(stat_buf.st_mode)) {
+        // jusdge if the dir is symlink?
+        if (remote_overlay_remove_ro_dir(id) != 0) {
+            ERROR("Failed to delete symlink to layer dir: %s", layer_dir);
+            ret = -1;
+            goto out;
+        }
+    } else {
+        if (util_recursive_rmdir(layer_dir, 0) != 0) {
+            SYSERROR("Failed to remove layer directory %s", layer_dir);
+            ret = -1;
+            goto out;
+        }
+    }
+#else
     if (util_recursive_rmdir(layer_dir, 0) != 0) {
         SYSERROR("Failed to remove layer directory %s", layer_dir);
         ret = -1;
         goto out;
     }
+#endif
 
 out:
     free(layer_dir);
@@ -1747,7 +1894,7 @@ int overlay2_apply_diff(const char *id, const struct graphdriver *driver, const
     int ret = 0;
 #ifdef ENABLE_USERNS_REMAP
     unsigned int size = 0;
-    char* userns_remap = conf_get_isulad_userns_remap();
+    char *userns_remap = conf_get_isulad_userns_remap();
 #endif
     char *layer_dir = NULL;
     char *layer_diff = NULL;
@@ -2166,3 +2313,4 @@ out:
     free(layer_diff);
     return ret;
 }
+
diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h
index e14271b1..5c1d93fb 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.h
@@ -22,6 +22,9 @@
 #include <stdint.h>
 
 #include "driver.h"
+#ifdef ENABLE_REMOTE_LAYER_STORE
+#include "remote_support.h"
+#endif
 
 struct driver_create_opts;
 struct driver_mount_opts;
@@ -68,6 +71,11 @@ int overlay2_repair_lowers(const char *id, const char *parent, const struct grap
 
 int overlay2_get_layer_fs_info(const char *id, const struct graphdriver *driver, imagetool_fs_info *fs_info);
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+remote_support *overlay_driver_impl_remote_support(void);
+bool overlay_remote_layer_valid(const char *layer_id);
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c
new file mode 100644
index 00000000..a674a00f
--- /dev/null
+++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/overlay_remote_impl.c
@@ -0,0 +1,282 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2023-02-27
+ * Description: provide remote implementation for driver overlay
+ ******************************************************************************/
+#define _GNU_SOURCE
+#include "driver_overlay2.h"
+
+#include <stdio.h>
+
+#include "map.h"
+#include "remote_support.h"
+#include "ro_symlink_maintain.h"
+#include "isula_libutils/log.h"
+#include "utils.h"
+#include "utils_array.h"
+#include "utils_file.h"
+#include "path.h"
+
+#define OVERLAY_LINK_DIR "l"
+#define OVERLAY_LAYER_LINK "link"
+
+struct remote_overlay_data {
+    const char *overlay_home;
+    const char *overlay_ro;
+};
+
+static map_t *overlay_byid_old = NULL;
+static map_t *overlay_byid_new = NULL;
+
+static void *remote_support_create(const char *remote_home, const char *remote_ro)
+{
+    struct remote_overlay_data *data = util_common_calloc_s(sizeof(struct remote_overlay_data));
+    if (data == NULL) {
+        ERROR("Out of memory");
+        return NULL;
+    }
+    data->overlay_home = remote_home;
+    data->overlay_ro = remote_ro;
+    overlay_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
+    overlay_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
+
+    return data;
+}
+
+static void remote_support_destroy(void *data)
+{
+    if (data == NULL) {
+        return;
+    }
+
+    map_free(overlay_byid_old);
+    map_free(overlay_byid_new);
+    free(data);
+}
+
+static bool overlay_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context)
+{
+    bool exist = true;
+    if (!map_insert(overlay_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) {
+        ERROR("can't insert remote layer into map");
+        return false;
+    }
+
+    return true;
+}
+
+static int remote_support_scan(void *data)
+{
+    struct remote_overlay_data *remote_data = data;
+    return util_scan_subdirs(remote_data->overlay_ro, overlay_walk_dir_cb, data);
+}
+
+static int do_diff_symlink(const char *id, char *link_id, const char *driver_home)
+{
+    int ret = 0;
+    int nret = 0;
+    char target_path[PATH_MAX] = { 0 };
+    char link_path[PATH_MAX] = { 0 };
+    char clean_path[PATH_MAX] = { 0 };
+
+    nret = snprintf(target_path, PATH_MAX, "../%s/diff", id);
+    if (nret < 0 || nret >= PATH_MAX) {
+        ERROR("Failed to get target path %s", id);
+        ret = -1;
+        goto out;
+    }
+
+    nret = snprintf(link_path, PATH_MAX, "%s/%s/%s", driver_home, OVERLAY_LINK_DIR, link_id);
+    if (nret < 0 || nret >= PATH_MAX) {
+        ERROR("Failed to get link path %s", link_id);
+        ret = -1;
+        goto out;
+    }
+
+    if (util_clean_path(link_path, clean_path, sizeof(clean_path)) == NULL) {
+        ERROR("failed to get clean path %s", link_path);
+        ret = -1;
+        goto out;
+    }
+
+    if (util_fileself_exists(clean_path) && util_path_remove(clean_path) != 0) {
+        ERROR("failed to remove old symbol link");
+        ret = -1;
+        goto out;
+    }
+
+    nret = symlink(target_path, clean_path);
+    if (nret < 0) {
+        SYSERROR("Failed to create symlink from \"%s\" to \"%s\"", clean_path, target_path);
+        ret = -1;
+        goto out;
+    }
+
+out:
+    return ret;
+}
+
+static int remove_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id)
+{
+    char *ro_symlink = NULL;
+    char clean_path[PATH_MAX] = { 0 };
+    int nret = 0;
+    int ret = 0;
+
+    nret = asprintf(&ro_symlink, "%s/%s", data->overlay_home, overlay_id);
+    if (nret < 0 || nret > PATH_MAX) {
+        SYSERROR("Create layer symbol link path failed");
+        ret = -1;
+        goto out;
+    }
+
+    if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) {
+        ERROR("Failed to clean path: %s", ro_symlink);
+        ret = -1;
+        goto out;
+    }
+
+    if (util_path_remove(clean_path) != 0) {
+        SYSERROR("Failed to remove link path %s", clean_path);
+    }
+
+out:
+    free(ro_symlink);
+    return ret;
+}
+
+static int add_one_remote_overlay_layer(struct remote_overlay_data *data, const char *overlay_id)
+{
+    char *ro_symlink = NULL;
+    char *layer_dir = NULL;
+    char *link_file = NULL;
+    char *diff_symlink = NULL;
+    int ret = 0;
+
+    ro_symlink = util_path_join(data->overlay_home, overlay_id);
+    if (ro_symlink == NULL) {
+        ERROR("Failed to join ro symlink path: %s", overlay_id);
+        ret = -1;
+        goto free_out;
+    }
+
+    layer_dir = util_path_join(data->overlay_ro, overlay_id);
+    if (layer_dir == NULL) {
+        ERROR("Failed to join ro layer dir: %s", overlay_id);
+        ret = -1;
+        goto free_out;
+    }
+
+    // add RO symbol link first
+    if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) {
+        SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir);
+        ret = -1;
+        goto free_out;
+    }
+
+    // maintain link
+    // try read link file in layer_dir
+    // mk symlink between ro_symlink
+    link_file = util_path_join(layer_dir, OVERLAY_LAYER_LINK);
+    if (link_file == NULL) {
+        ERROR("Failed to get layer link file %s", layer_dir);
+        ret = -1;
+        goto free_out;
+    }
+
+    if (!util_fileself_exists(link_file)) {
+        ERROR("link file for layer %s not exist", layer_dir);
+        ret = -1;
+        goto free_out;
+    }
+
+    diff_symlink = util_read_content_from_file(link_file);
+    if (link_file == NULL) {
+        ERROR("Failed to read content from link file of layer %s", layer_dir);
+        ret = -1;
+        goto free_out;
+    }
+
+    if (do_diff_symlink(overlay_id, diff_symlink, data->overlay_home) != 0) {
+        ERROR("Failed to add diff link for layer %s", overlay_id);
+        ret = -1;
+    }
+
+free_out:
+    free(ro_symlink);
+    free(layer_dir);
+    free(link_file);
+    free(diff_symlink);
+
+    return ret;
+}
+
+static int remote_support_add(void *data)
+{
+    int ret = 0;
+    char **array_added = NULL;
+    char **array_deleted = NULL;
+    map_t *tmp_map = NULL;
+    int i = 0;
+
+    if (data == NULL) {
+        return -1;
+    }
+
+    array_added = added_layers(overlay_byid_old, overlay_byid_new);
+    array_deleted = deleted_layers(overlay_byid_old, overlay_byid_new);
+
+    for (i = 0; i < util_array_len((const char **)array_added); i++) {
+        if (add_one_remote_overlay_layer(data, array_added[i]) != 0) {
+            ERROR("Failed to add remote overlay layer: %s", array_added[i]);
+            ret = -1;
+        }
+    }
+
+    for (i = 0; i < util_array_len((const char **)array_deleted); i++) {
+        if (remove_one_remote_overlay_layer(data, array_deleted[i]) != 0) {
+            ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]);
+            ret = -1;
+        }
+    }
+
+    tmp_map = overlay_byid_old;
+    overlay_byid_old = overlay_byid_new;
+    overlay_byid_new = tmp_map;
+    empty_map(overlay_byid_new);
+
+    util_free_array(array_added);
+    util_free_array(array_deleted);
+
+    return ret;
+}
+
+remote_support *overlay_driver_impl_remote_support(void)
+{
+    remote_support *rs = util_common_calloc_s(sizeof(remote_support));
+    if (rs == NULL) {
+        ERROR("Failed to calloc overlay supporter");
+        return NULL;
+    }
+
+    rs->create = remote_support_create;
+    rs->destroy = remote_support_destroy;
+    rs->scan_remote_dir = remote_support_scan;
+    rs->load_item = remote_support_add;
+
+    return rs;
+}
+
+bool overlay_remote_layer_valid(const char *layer_id)
+{
+    return map_search(overlay_byid_old, (void *)layer_id) != NULL;
+}
diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
new file mode 100644
index 00000000..d03fc20b
--- /dev/null
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_remote_impl.c
@@ -0,0 +1,219 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2023-02-27
+ * Description: remote layer store implementation
+ ******************************************************************************/
+#define _GNU_SOURCE
+#include "layer_store.h"
+
+#include <pthread.h>
+#include <isula_libutils/log.h>
+#include <stdio.h>
+
+#include "map.h"
+#include "utils.h"
+#include "remote_support.h"
+#include "ro_symlink_maintain.h"
+#include "path.h"
+#include "driver_overlay2.h"
+
+struct remote_layer_data {
+    const char *layer_home;
+    const char *layer_ro;
+};
+
+static map_t *layer_byid_old = NULL;
+static map_t *layer_byid_new = NULL;
+
+static void *remote_support_create(const char *layer_home, const char *layer_ro)
+{
+    struct remote_layer_data *data = util_common_calloc_s(sizeof(struct remote_layer_data));
+    if (data == NULL) {
+        ERROR("Out of memory");
+        return NULL;
+    }
+    data->layer_home = util_strdup_s(layer_home);
+    data->layer_ro = util_strdup_s(layer_ro);
+    layer_byid_old = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
+    layer_byid_new = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
+
+    return data;
+};
+
+static void remote_support_destroy(void *data)
+{
+    if (data == NULL) {
+        return;
+    }
+
+    map_free(layer_byid_old);
+    map_free(layer_byid_new);
+    free(data);
+}
+
+static bool layer_walk_dir_cb(const char *path_name, const struct dirent *sub_dir, void *context)
+{
+    bool exist = true;
+
+    if (!map_insert(layer_byid_new, util_strdup_s(sub_dir->d_name), (void *)&exist)) {
+        ERROR("can't insert remote layer into map");
+        return false;
+    }
+
+    return true;
+}
+
+static int remote_support_scan(void *data)
+{
+    struct remote_layer_data *remote_data = data;
+    return util_scan_subdirs(remote_data->layer_ro, layer_walk_dir_cb, data);
+}
+
+static int remove_one_remote_layer(struct remote_layer_data *data, char *layer_id)
+{
+    char *ro_symlink = NULL;
+    char clean_path[PATH_MAX] = { 0 };
+    int nret = 0;
+    int ret = 0;
+
+    nret = asprintf(&ro_symlink, "%s/%s", data->layer_home, layer_id);
+    if (nret < 0 || nret > PATH_MAX) {
+        SYSERROR("Create layer symbol link path failed");
+        ret = -1;
+        goto out;
+    }
+
+    if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) {
+        ERROR("Failed to clean path: %s", ro_symlink);
+        ret = -1;
+        goto out;
+    }
+
+    if (util_path_remove(clean_path) != 0) {
+        SYSERROR("Failed to remove link path %s", clean_path);
+    }
+
+    if (remove_memory_stores_with_lock(layer_id) != 0) {
+        ERROR("Failed to remove remote layer store memory");
+        ret = -1;
+    }
+
+out:
+    free(ro_symlink);
+    return ret;
+
+}
+
+static int add_one_remote_layer(struct remote_layer_data *data, char *layer_id)
+{
+    char *ro_symlink = NULL;
+    char *layer_dir = NULL;
+    int ret = 0;
+
+    ro_symlink = util_path_join(data->layer_home, layer_id);
+    layer_dir = util_path_join(data->layer_ro, layer_id);
+
+    if (ro_symlink == NULL) {
+        ERROR("Failed to join ro symlink path: %s", layer_id);
+        ret = -1;
+        goto free_out;
+    }
+
+    if (layer_dir == NULL) {
+        ERROR("Failed to join ro layer dir: %s", layer_id);
+        ret = -1;
+        goto free_out;
+    }
+    // add symbol link first
+    if (!util_fileself_exists(ro_symlink) && symlink(layer_dir, ro_symlink) != 0) {
+        SYSERROR("Unable to create symbol link to layer directory: %s", layer_dir);
+        ret = -1;
+        goto free_out;
+    }
+    // insert layer into memory
+    if (load_one_layer(layer_id) != 0) {
+        ERROR("Failed to load new layer: %s into memory", layer_id);
+        ret = -1;
+    }
+
+free_out:
+    free(ro_symlink);
+    free(layer_dir);
+
+    return ret;
+}
+
+static int remote_support_add(void *data)
+{
+    int ret = 0;
+    char **array_added = NULL;
+    char **array_deleted = NULL;
+    map_t *tmp_map = NULL;
+    int i = 0;
+
+    if (data == NULL) {
+        return -1;
+    }
+
+    array_added = added_layers(layer_byid_old, layer_byid_new);
+    array_deleted = deleted_layers(layer_byid_old, layer_byid_new);
+
+    for (i = 0; i < util_array_len((const char **)array_added); i++) {
+        if (!overlay_remote_layer_valid(array_added[i]) != 0) {
+            map_remove(layer_byid_new, (void *)array_added[i]);
+            ERROR("remote overlay layer current not valid: %s", array_added[i]);
+            continue;
+        }
+
+        if (add_one_remote_layer(data, array_added[i]) != 0) {
+            ERROR("Failed to add remote overlay layer: %s", array_added[i]);
+            ret = -1;
+        }
+    }
+
+    for (i = 0; i < util_array_len((const char **)array_deleted); i++) {
+        if (remove_one_remote_layer(data, array_deleted[i]) != 0) {
+            ERROR("Failed to delete remote overlay layer: %s", array_deleted[i]);
+            ret = -1;
+        }
+    }
+
+    tmp_map = layer_byid_old;
+    layer_byid_old = layer_byid_new;
+    layer_byid_new = tmp_map;
+    empty_map(layer_byid_new);
+
+    util_free_array(array_added);
+    util_free_array(array_deleted);
+
+    return ret;
+}
+
+remote_support *layer_store_impl_remote_support()
+{
+    remote_support *rs = util_common_calloc_s(sizeof(remote_support));
+    if (rs == NULL) {
+        return NULL;
+    }
+
+    rs->create = remote_support_create;
+    rs->destroy = remote_support_destroy;
+    rs->scan_remote_dir = remote_support_scan;
+    rs->load_item = remote_support_add;
+
+    return rs;
+}
+
+bool layer_remote_layer_valid(const char *layer_id)
+{
+    return map_search(layer_byid_old, (void *)layer_id) != NULL;
+}
diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
index 89b4f58c..c00c3356 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.c
@@ -30,7 +30,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
-
 #include <archive.h>
 #include <archive_entry.h>
 
@@ -48,6 +47,10 @@
 #include "http.h"
 #include "utils_base64.h"
 #include "constants.h"
+#include "path.h"
+#ifdef ENABLE_REMOTE_LAYER_STORE
+#include "ro_symlink_maintain.h"
+#endif
 
 #define PAYLOAD_CRC_LEN 12
 
@@ -74,6 +77,9 @@ typedef struct {
 static layer_store_metadata g_metadata;
 static char *g_root_dir;
 static char *g_run_dir;
+#ifdef ENABLE_REMOTE_LAYER_STORE
+static bool g_enable_remote_layer;
+#endif
 
 static inline char *tar_split_path(const char *id);
 static inline char *mountpoint_json_path(const char *id);
@@ -127,7 +133,7 @@ void layer_store_cleanup()
     map_free(g_metadata.by_uncompress_digest);
     g_metadata.by_uncompress_digest = NULL;
 
-    linked_list_for_each_safe(item, &(g_metadata.layers_list), next) {
+    linked_list_for_each_safe (item, &(g_metadata.layers_list), next) {
         linked_list_del(item);
         layer_ref_dec((layer_t *)item->elem);
         free(item);
@@ -158,7 +164,7 @@ static void free_digest_layer_t(digest_layer_t *ptr)
         return;
     }
 
-    linked_list_for_each_safe(item, &(ptr->layer_list), next) {
+    linked_list_for_each_safe (item, &(ptr->layer_list), next) {
         linked_list_del(item);
         free(item->elem);
         item->elem = NULL;
@@ -272,6 +278,10 @@ static bool init_from_conf(const struct storage_module_init_options *conf)
     g_root_dir = tmp_path;
     tmp_path = NULL;
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    g_enable_remote_layer = conf->enable_remote_layer;
+#endif
+
     return true;
 free_out:
     free(g_run_dir);
@@ -609,7 +619,7 @@ static int delete_digest_from_map(map_t *by_digest, const char *digest, const ch
         return 0;
     }
 
-    linked_list_for_each_safe(item, &(old_list->layer_list), next) {
+    linked_list_for_each_safe (item, &(old_list->layer_list), next) {
         char *t_id = (char *)item->elem;
         if (strcmp(t_id, id) == 0) {
             linked_list_del(item);
@@ -722,7 +732,7 @@ static int remove_memory_stores(const char *id)
         }
     }
 
-    linked_list_for_each_safe(item, &(g_metadata.layers_list), next) {
+    linked_list_for_each_safe (item, &(g_metadata.layers_list), next) {
         layer_t *tl = (layer_t *)item->elem;
         if (strcmp(tl->slayer->id, id) != 0) {
             continue;
@@ -1115,10 +1125,25 @@ static int new_layer_by_opts(const char *id, const struct layer_opts *opts)
         ret = -1;
         goto out;
     }
+
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    if (g_enable_remote_layer && !opts->writable) {
+        if (remote_layer_build_ro_dir(id) != 0) {
+            ret = -1;
+            goto out;
+        }
+    } else {
+        if (!build_layer_dir(id)) {
+            ret = -1;
+            goto out;
+        }
+    }
+#else
     if (!build_layer_dir(id)) {
         ret = -1;
         goto out;
     }
+#endif
 
     ret = update_layer_datas(id, opts, l);
     if (ret != 0) {
@@ -1300,7 +1325,15 @@ clear_memory:
 driver_remove:
     if (ret != 0) {
         (void)graphdriver_rm_layer(lid);
+#ifdef ENABLE_REMOTE_LAYER_STORE
+        if (g_enable_remote_layer) {
+            (void)remote_layer_remove_ro_dir(lid);
+        } else {
+            (void)layer_store_remove_layer(lid);
+        }
+#else
         (void)layer_store_remove_layer(lid);
+#endif
     }
 free_out:
     layer_store_unlock();
@@ -1375,7 +1408,15 @@ static int do_delete_layer(const char *id)
         goto free_out;
     }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    if (l->slayer->writable) {
+        ret = layer_store_remove_layer(l->slayer->id);
+    } else {
+        ret = remote_layer_remove_ro_dir(l->slayer->id);
+    }
+#else
     ret = layer_store_remove_layer(l->slayer->id);
+#endif
 
 free_out:
     free(tspath);
@@ -1457,7 +1498,7 @@ int layer_store_list(struct layer_list *resp)
         goto unlock;
     }
 
-    linked_list_for_each_safe(item, &(g_metadata.layers_list), next) {
+    linked_list_for_each_safe (item, &(g_metadata.layers_list), next) {
         layer_t *l = (layer_t *)item->elem;
         resp->layers[i] = util_common_calloc_s(sizeof(struct layer));
         if (resp->layers[i] == NULL) {
@@ -1500,7 +1541,7 @@ static int layers_by_digest_map(map_t *m, const char *digest, struct layer_list
         goto free_out;
     }
 
-    linked_list_for_each_safe(item, &(id_list->layer_list), next) {
+    linked_list_for_each_safe (item, &(id_list->layer_list), next) {
         layer_t *l = NULL;
         resp->layers[i] = util_common_calloc_s(sizeof(struct layer));
         if (resp->layers[i] == NULL) {
@@ -1744,6 +1785,114 @@ out:
     return ret;
 }
 
+static layer_t *load_one_layer_from_json(const char *id)
+{
+    int nret = 0;
+    char *mount_point_path = NULL;
+    char tmpdir[PATH_MAX] = { 0 };
+    char *rpath = NULL;
+    layer_t *l = NULL;
+    bool layer_valid = false;
+
+    nret = snprintf(tmpdir, PATH_MAX, "%s/%s", g_root_dir, id);
+    if (nret < 0 || nret >= PATH_MAX) {
+        ERROR("Sprintf: %s failed", id);
+        goto free_out;
+    }
+
+    mount_point_path = mountpoint_json_path(id);
+    if (mount_point_path == NULL) {
+        ERROR("Out of Memory");
+        goto free_out;
+    }
+
+    rpath = layer_json_path(id);
+    if (rpath == NULL) {
+        ERROR("%s is invalid layer", id);
+        goto free_out;
+    }
+
+    l = load_layer(rpath, mount_point_path);
+    if (l == NULL) {
+        ERROR("load layer: %s failed, remove it", id);
+        goto free_out;
+    }
+
+    if (do_validate_image_layer(tmpdir, l) != 0) {
+        ERROR("%s is invalid image layer", id);
+        goto free_out;
+    }
+
+    if (do_validate_rootfs_layer(l) != 0) {
+        ERROR("%s is invalid rootfs layer", id);
+        goto free_out;
+    }
+
+    layer_valid = true;
+
+free_out:
+    free(rpath);
+    free(mount_point_path);
+    if (!layer_valid) {
+        free_layer_t(l);
+        l = NULL;
+    }
+    // always return true;
+    // if load layer failed, just remove it
+    return l;
+}
+
+int load_one_layer(const char *id)
+{
+    int ret = 0;
+    layer_t *tl = NULL;
+    int i = 0;
+
+    if (!layer_store_lock(true)) {
+        return -1;
+    }
+
+    tl = load_one_layer_from_json(id);
+    if (tl == NULL) {
+        ret = -1;
+        goto unlock_out;
+    }
+
+    if (!map_insert(g_metadata.by_id, (void *)tl->slayer->id, (void *)tl)) {
+        ERROR("Insert id: %s for layer failed", tl->slayer->id);
+        ret = -1;
+        goto unlock_out;
+    }
+
+    for (; i < tl->slayer->names_len; i++) {
+        // this should be done by master isulad
+        // if (remove_name(tl->slayer->names[i])) {
+        //     should_save = true;
+        // }
+        if (!map_insert(g_metadata.by_name, (void *)tl->slayer->names[i], (void *)tl)) {
+            ret = -1;
+            ERROR("Insert name: %s for layer failed", tl->slayer->names[i]);
+            goto unlock_out;
+        }
+    }
+    ret = insert_digest_into_map(g_metadata.by_compress_digest, tl->slayer->compressed_diff_digest, tl->slayer->id);
+    if (ret != 0) {
+        ERROR("update layer: %s compress failed", tl->slayer->id);
+        goto unlock_out;
+    }
+
+    ret = insert_digest_into_map(g_metadata.by_uncompress_digest, tl->slayer->diff_digest, tl->slayer->id);
+    if (ret != 0) {
+        ERROR("update layer: %s uncompress failed", tl->slayer->id);
+        goto unlock_out;
+    }
+
+    ret = 0;
+unlock_out:
+    layer_store_unlock();
+    return ret;
+}
+
 static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_dir, void *context)
 {
 #define LAYER_NAME_LEN 64
@@ -1760,6 +1909,14 @@ static bool load_layer_json_cb(const char *path_name, const struct dirent *sub_d
         goto free_out;
     }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    // skip RO dir
+    // otherwise, RO dir will be treat as invalid layer dir
+    if (strcmp(sub_dir->d_name, REMOTE_RO_LAYER_DIR) == 0) {
+        goto free_out;
+    }
+#endif
+
     if (!util_dir_exists(tmpdir)) {
         // ignore non-dir
         DEBUG("%s is not directory", sub_dir->d_name);
@@ -1846,7 +2003,7 @@ static int load_layers_from_json_files()
         goto unlock_out;
     }
 
-    linked_list_for_each_safe(item, &(g_metadata.layers_list), next) {
+    linked_list_for_each_safe (item, &(g_metadata.layers_list), next) {
         layer_t *tl = (layer_t *)item->elem;
         size_t i = 0;
 
@@ -1953,6 +2110,13 @@ int layer_store_init(const struct storage_module_init_options *conf)
         goto free_out;
     }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    if (g_enable_remote_layer && remote_layer_init(g_root_dir) != 0) {
+        ERROR("Failed to init layer remote");
+        goto free_out;
+    }
+#endif
+
     if (load_layers_from_json_files() != 0) {
         goto free_out;
     }
@@ -2125,7 +2289,7 @@ static tar_split *new_tar_split(layer_t *l, const char *tspath)
     int ret = 0;
     int nret = 0;
     tar_split *ts = NULL;
-    char path[PATH_MAX] = {0};
+    char path[PATH_MAX] = { 0 };
 
     ts = util_common_calloc_s(sizeof(tar_split));
     if (ts == NULL) {
@@ -2308,3 +2472,20 @@ container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char
 {
     return graphdriver_get_metadata(id);
 }
+
+#ifdef ENABLE_REMOTE_LAYER_STORE
+int remove_memory_stores_with_lock(const char *id)
+{
+    int ret = 0;
+
+    if (!layer_store_lock(true)) {
+        ERROR("Failed to lock layer store when handle: %s", id);
+        return -1;
+    }
+
+    ret = remove_memory_stores(id);
+    layer_store_unlock();
+
+    return ret;
+}
+#endif
diff --git a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h
index 94d4bf04..44bd297e 100644
--- a/src/daemon/modules/image/oci/storage/layer_store/layer_store.h
+++ b/src/daemon/modules/image/oci/storage/layer_store/layer_store.h
@@ -23,6 +23,10 @@
 
 #include "storage.h"
 #include "io_wrapper.h"
+#include "map.h"
+#ifdef ENABLE_REMOTE_LAYER_STORE
+#include "remote_support.h"
+#endif
 
 struct io_read_wrapper;
 struct layer_list;
@@ -79,6 +83,13 @@ int layer_store_check(const char *id);
 
 container_inspect_graph_driver *layer_store_get_metadata_by_layer_id(const char *id);
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+remote_support *layer_store_impl_remote_support();
+bool layer_remote_layer_valid(const char *layer_id);
+int load_one_layer(const char *id);
+int remove_memory_stores_with_lock(const char *id);
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt
new file mode 100644
index 00000000..06c78678
--- /dev/null
+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/CMakeLists.txt
@@ -0,0 +1,12 @@
+# get current directory sources files
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_remote_layer_support_srcs)
+
+set(REMOTE_LAYER_SUPPORT_SRCS
+    ${local_remote_layer_support_srcs}
+    PARENT_SCOPE
+    )
+
+set(REMOTE_LAYER_SUPPORT_INCS
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    PARENT_SCOPE
+)
\ No newline at end of file
diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
new file mode 100644
index 00000000..9dc096f7
--- /dev/null
+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.c
@@ -0,0 +1,122 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2023-03-03
+ * Description: provide image store functions
+ ******************************************************************************/
+
+#include "remote_support.h"
+
+#include "layer_store.h"
+#include "image_store.h"
+#include "isula_libutils/log.h"
+#include "driver_overlay2.h"
+#include "utils.h"
+
+remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro)
+{
+    remote_support *handlers = layer_store_impl_remote_support();
+    if (handlers == NULL || handlers->create == NULL) {
+        return NULL;
+    }
+
+    remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter));
+    if (supporter == NULL) {
+        goto err_out;
+    }
+
+    supporter->handlers = handlers;
+    supporter->data = handlers->create(remote_home, remote_ro);
+
+    return supporter;
+
+err_out:
+    free(handlers);
+    free(supporter);
+    return NULL;
+}
+
+remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro)
+{
+    remote_support *handlers = image_store_impl_remote_support();
+    if (handlers == NULL || handlers->create == NULL) {
+        return NULL;
+    }
+
+    remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter));
+    if (supporter == NULL) {
+        goto err_out;
+    }
+
+    supporter->handlers = handlers;
+    supporter->data = handlers->create(remote_home, remote_ro);
+
+    return supporter;
+
+err_out:
+    free(handlers);
+    free(supporter);
+    return NULL;
+}
+
+remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro)
+{
+    remote_support *handlers = overlay_driver_impl_remote_support();
+    if (handlers == NULL || handlers->create == NULL) {
+        return NULL;
+    }
+
+    remote_supporter *supporter = (remote_supporter *)util_common_calloc_s(sizeof(remote_supporter));
+    if (supporter == NULL) {
+        goto err_out;
+    }
+
+    supporter->handlers = handlers;
+    supporter->data = handlers->create(remote_home, remote_ro);
+
+    return supporter;
+
+err_out:
+    free(handlers);
+    free(supporter);
+    return NULL;
+
+}
+
+void destroy_suppoter(remote_supporter *supporter)
+{
+    if (supporter->handlers->destroy == NULL) {
+        ERROR("destroy_supporter operation not supported");
+        return;
+    }
+
+    supporter->handlers->destroy(supporter->data);
+    free(supporter->handlers);
+    free(supporter);
+}
+
+int scan_remote_dir(remote_supporter *supporter)
+{
+    if (supporter->handlers->scan_remote_dir == NULL) {
+        ERROR("scan_remote_dir operation not supported");
+        return -1;
+    }
+    return supporter->handlers->scan_remote_dir(supporter->data);
+}
+
+int load_item(remote_supporter *supporter)
+{
+    if (supporter->handlers->scan_remote_dir == NULL) {
+        ERROR("load_item operation not supported");
+        return -1;
+    }
+    return supporter->handlers->load_item(supporter->data);
+}
diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
new file mode 100644
index 00000000..d1f7af35
--- /dev/null
+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/remote_support.h
@@ -0,0 +1,58 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2023-03-03
+ * Description: provide remote support functions
+ ******************************************************************************/
+
+#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H
+#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_REMOTE_SUPPORT_H
+
+#include "linked_list.h"
+#define REMOTE_RO_LAYER_DIR "RO"
+#define OVERLAY_RO_DIR "RO"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    void *(*create)(const char *remote_home, const char *remote_ro);
+    void (*destroy)(void *data);
+    // populate the list contains all dirs
+    int (*scan_remote_dir)(void *data);
+    // consume the list contains all dirs
+    int (*load_item)(void *data);
+} remote_support;
+
+typedef struct {
+    void *data;
+    remote_support *handlers;
+} remote_supporter;
+
+// RemoteSupport *impl_remote_support();
+remote_supporter *create_image_supporter(const char *remote_home, const char *remote_ro);
+
+remote_supporter *create_layer_supporter(const char *remote_home, const char *remote_ro);
+
+remote_supporter *create_overlay_supporter(const char *remote_home, const char *remote_ro);
+
+void destroy_suppoter(remote_supporter *supporter);
+
+int scan_remote_dir(remote_supporter *supporter);
+
+int load_item(remote_supporter *supporter);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c
new file mode 100644
index 00000000..7df7a221
--- /dev/null
+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c
@@ -0,0 +1,347 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2023-01-12
+ * Description: provide remote symlink maintain functions
+ ******************************************************************************/
+#define _GNU_SOURCE
+#include "ro_symlink_maintain.h"
+
+#include <sys/prctl.h>
+#include <stdio.h>
+#include <pthread.h>
+#include <unistd.h>
+#include "map.h"
+#include "path.h"
+#include "linked_list.h"
+#include "layer_store.h"
+#include "layer.h"
+#include "isula_libutils/log.h"
+#include "image_store.h"
+#include "remote_support.h"
+#include "utils.h"
+#include "utils_file.h"
+#include "stdlib.h"
+
+#define REMOTE_RO_LAYER_DIR "RO"
+
+// overlay-layers and overlay-layers/RO
+static char *image_home;
+
+static char *layer_ro_dir;
+static char *layer_home;
+
+// overlay and overlay/RO
+static char *overlay_ro_dir;
+static char *overlay_home;
+
+struct supporters {
+    remote_supporter *image_supporter;
+    remote_supporter *layer_supporter;
+    remote_supporter *overlay_supporter;
+};
+
+static struct supporters supporters;
+
+int remote_image_init(const char *root_dir)
+{
+    if (root_dir == NULL) {
+        goto out;
+    }
+
+    image_home = util_strdup_s(root_dir);
+    if (image_home == NULL) {
+        ERROR("Failed create path for remote image home");
+        goto out;
+    }
+    return 0;
+
+out:
+    remote_maintain_cleanup();
+    return -1;
+}
+
+int remote_layer_init(const char *root_dir)
+{
+    if (root_dir == NULL) {
+        goto out;
+    }
+
+    layer_home = util_strdup_s(root_dir);
+    layer_ro_dir = util_path_join(root_dir, REMOTE_RO_LAYER_DIR);
+    if (layer_ro_dir == NULL) {
+        ERROR("Failed join path when init remote layer maintainer");
+        goto out;
+    }
+    if (!util_file_exists(layer_ro_dir) && util_mkdir_p(layer_ro_dir, 0700) != 0) {
+        ERROR("Failed to create RO dir under overlay");
+        goto out;
+    }
+
+    return 0;
+
+out:
+    remote_maintain_cleanup();
+    return -1;
+}
+
+int remote_overlay_init(const char *driver_home)
+{
+    if (driver_home == NULL) {
+        goto out;
+    }
+
+    overlay_home = util_strdup_s(driver_home);
+    overlay_ro_dir = util_path_join(driver_home, REMOTE_RO_LAYER_DIR);
+    if (overlay_ro_dir == NULL) {
+        ERROR("Failed to join path when init remote maintainer");
+        goto out;
+    }
+    // build RO dir if not exist
+    if (!util_file_exists(overlay_ro_dir) && util_mkdir_p(overlay_ro_dir, 0700) != 0) {
+        ERROR("Failed to create RO dir under overlay");
+        goto out;
+    }
+
+    return 0;
+
+out:
+    remote_maintain_cleanup();
+    return -1;
+}
+
+void remote_maintain_cleanup(void)
+{
+    free(image_home);
+    image_home = NULL;
+
+    free(layer_home);
+    layer_home = NULL;
+    free(layer_ro_dir);
+    layer_ro_dir = NULL;
+    free(overlay_home);
+
+    overlay_home = NULL;
+    free(overlay_ro_dir);
+    overlay_ro_dir = NULL;
+}
+
+// to maintain the symbol links, add new symbol link and delete invalid symbol link
+// arg is const char *driver_home
+// scanning driver->home/RO/ directory, build symlink in driver->home
+static void *remote_refresh_ro_symbol_link(void *arg)
+{
+    struct supporters *supporters = (struct supporters *)arg;
+    prctl(PR_SET_NAME, "RoLayerRefresh");
+
+    while (true) {
+        util_usleep_nointerupt(5 * 1000 * 1000);
+        DEBUG("remote refresh start\n");
+        scan_remote_dir(supporters->overlay_supporter);
+        load_item(supporters->overlay_supporter);
+        scan_remote_dir(supporters->layer_supporter);
+        load_item(supporters->layer_supporter);
+        scan_remote_dir(supporters->image_supporter);
+        load_item(supporters->image_supporter);
+        DEBUG("remote refresh end\n");
+    }
+    return NULL;
+}
+
+int start_refresh_thread(void)
+{
+    int res = 0;
+    pthread_t a_thread;
+
+    supporters.image_supporter = create_image_supporter(image_home, NULL);
+    if (supporters.image_supporter == NULL) {
+        goto free_out;
+    }
+
+    supporters.layer_supporter = create_layer_supporter(layer_home, layer_ro_dir);
+    if (supporters.layer_supporter == NULL) {
+        goto free_out;
+    }
+
+    supporters.overlay_supporter = create_overlay_supporter(overlay_home, overlay_ro_dir);
+    if (supporters.overlay_supporter == NULL) {
+        goto free_out;
+    }
+
+    res = pthread_create(&a_thread, NULL, remote_refresh_ro_symbol_link, (void *)&supporters);
+    if (res != 0) {
+        CRIT("Thread creation failed");
+        return -1;
+    }
+
+    if (pthread_detach(a_thread) != 0) {
+        SYSERROR("Failed to detach 0x%lx", a_thread);
+        return -1;
+    }
+
+    return 0;
+
+free_out:
+    destroy_suppoter(supporters.image_supporter);
+    destroy_suppoter(supporters.layer_supporter);
+    destroy_suppoter(supporters.overlay_supporter);
+
+    return -1;
+}
+
+static int do_build_ro_dir(const char *home, const char *id)
+{
+    char *ro_symlink = NULL;
+    char *ro_layer_dir = NULL;
+    int nret = 0;
+    // bool ret = true;
+    int ret = 0;
+
+    nret = asprintf(&ro_symlink, "%s/%s", home, id);
+    if (nret < 0 || nret > PATH_MAX) {
+        SYSERROR("Failed create ro layer dir sym link path");
+        return -1;
+    }
+
+    nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id);
+    if (nret < 0 || nret > PATH_MAX) {
+        SYSERROR("Failed to create ro layer dir path");
+        return -1;
+    }
+
+    if (util_mkdir_p(ro_layer_dir, IMAGE_STORE_PATH_MODE) != 0) {
+        ret = -1;
+        ERROR("Failed to create layer direcotry %s", ro_layer_dir);
+        goto out;
+    }
+
+    if (symlink(ro_layer_dir, ro_symlink) != 0) {
+        ret = -1;
+        SYSERROR("Failed to create symlink to layer dir %s", ro_layer_dir);
+        goto err_out;
+    }
+
+    goto out;
+
+err_out:
+    if (util_recursive_rmdir(ro_layer_dir, 0)) {
+        ERROR("Failed to delete layer path: %s", ro_layer_dir);
+    }
+
+out:
+    free(ro_layer_dir);
+    free(ro_symlink);
+    return ret;
+}
+
+int remote_overlay_build_ro_dir(const char *id)
+{
+    return do_build_ro_dir(overlay_home, id);
+}
+
+int remote_layer_build_ro_dir(const char *id)
+{
+    return do_build_ro_dir(layer_home, id);
+}
+
+int do_remove_ro_dir(const char *home, const char *id)
+{
+    char *ro_layer_dir = NULL;
+    char *ro_symlink = NULL;
+    char clean_path[PATH_MAX] = { 0 };
+    int ret = 0;
+    int nret = 0;
+
+    if (id == NULL) {
+        return 0;
+    }
+
+    nret = asprintf(&ro_symlink, "%s/%s", home, id);
+    if (nret < 0 || nret > PATH_MAX) {
+        SYSERROR("Create layer sym link path failed");
+        return -1;
+    }
+
+    if (util_clean_path(ro_symlink, clean_path, sizeof(clean_path)) == NULL) {
+        ERROR("Failed to clean path: %s", ro_symlink);
+        ret = -1;
+        goto out;
+    }
+
+    if (util_path_remove(clean_path) != 0) {
+        SYSERROR("Failed to remove link path %s", clean_path);
+    }
+
+    nret = asprintf(&ro_layer_dir, "%s/%s/%s", home, REMOTE_RO_LAYER_DIR, id);
+    if (nret < 0 || nret > PATH_MAX) {
+        SYSERROR("Create layer json path failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = util_recursive_rmdir(ro_layer_dir, 0);
+
+out:
+    free(ro_layer_dir);
+    free(ro_symlink);
+    return ret;
+}
+
+int remote_layer_remove_ro_dir(const char *id)
+{
+    return do_remove_ro_dir(layer_home, id);
+}
+
+int remote_overlay_remove_ro_dir(const char *id)
+{
+    return do_remove_ro_dir(overlay_home, id);
+}
+
+static char **map_diff(map_t *map_a, map_t *map_b)
+{
+    char **array = NULL;
+    map_itor *itor = map_itor_new(map_a);
+    bool *found = NULL;
+
+    // iter new_map, every item not in old, append them to new_layers
+    for (; map_itor_valid(itor); map_itor_next(itor)) {
+        char *id = map_itor_key(itor);
+        found = map_search(map_b, id);
+        if (found == NULL) {
+            util_array_append(&array, util_strdup_s(id));
+        }
+    }
+
+    map_itor_free(itor);
+
+    return array;
+}
+
+char **deleted_layers(map_t *old, map_t *new)
+{
+    return map_diff(old, new);
+}
+
+char **added_layers(map_t *old, map_t *new)
+{
+    return map_diff(new, old);
+}
+
+int empty_map(map_t *mp)
+{
+    if (mp == NULL) {
+        return -1;
+    }
+
+    map_clear(mp);
+    mp->store->root = mp->store->nil;
+    return 0;
+}
diff --git a/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h
new file mode 100644
index 00000000..25712d40
--- /dev/null
+++ b/src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.h
@@ -0,0 +1,52 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: wangrunze
+ * Create: 2023-01-12
+ * Description: provide remote symlink maintain functions
+ ******************************************************************************/
+#ifndef DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H
+#define DAEMON_MODULES_IMAGE_OCI_STORAGE_LAYER_STORE_REMOTE_LAYER_SUPPORT_RO_SYMLINK_MAINTAIN_H
+
+#include "map.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int remote_image_init(const char *root_dir);
+
+int remote_layer_init(const char *root_dir);
+
+int remote_overlay_init(const char *driver_home);
+
+void remote_maintain_cleanup(void);
+
+int start_refresh_thread(void);
+
+int remote_layer_build_ro_dir(const char *id);
+
+int remote_overlay_build_ro_dir(const char *id);
+
+int remote_layer_remove_ro_dir(const char *id);
+
+int remote_overlay_remove_ro_dir(const char *id);
+
+char **deleted_layers(map_t *old, map_t *new);
+
+char **added_layers(map_t *old, map_t *new);
+
+int empty_map(map_t *mp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/daemon/modules/image/oci/storage/storage.c b/src/daemon/modules/image/oci/storage/storage.c
index 2f4bdf5f..31812a22 100644
--- a/src/daemon/modules/image/oci/storage/storage.c
+++ b/src/daemon/modules/image/oci/storage/storage.c
@@ -42,6 +42,9 @@
 #include "utils_string.h"
 #include "utils_verify.h"
 #include "sha256.h"
+#ifdef ENABLE_REMOTE_LAYER_STORE
+#include "ro_symlink_maintain.h"
+#endif
 
 static pthread_rwlock_t g_storage_rwlock;
 static char *g_storage_run_root;
@@ -1870,6 +1873,12 @@ int storage_module_init(struct storage_module_init_options *opts)
         goto out;
     }
 
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    if (opts->enable_remote_layer && start_refresh_thread() != 0) {
+        ERROR("Failed to start remote refresh thread");
+    }
+#endif
+
     if (restore_images_size() != 0) {
         ERROR("Failed to recal image size");
         ret = -1;
@@ -1906,4 +1915,4 @@ out:
 char *storage_rootfs_get_dir()
 {
     return rootfs_store_get_data_dir();
-}
\ No newline at end of file
+}
diff --git a/src/daemon/modules/image/oci/storage/storage.h b/src/daemon/modules/image/oci/storage/storage.h
index 3ec47959..7404ee54 100644
--- a/src/daemon/modules/image/oci/storage/storage.h
+++ b/src/daemon/modules/image/oci/storage/storage.h
@@ -70,6 +70,9 @@ struct storage_module_init_options {
     char **driver_opts;
     size_t driver_opts_len;
     bool integration_check;
+#ifdef ENABLE_REMOTE_LAYER_STORE
+    bool enable_remote_layer;
+#endif
 };
 
 struct storage_img_create_options {
-- 
2.25.1