From b69da83db290057dde5dbe34e153fb0895e456e2 Mon Sep 17 00:00:00 2001 From: WangFengTu Date: Tue, 29 Dec 2020 10:16:13 +0800 Subject: [PATCH 06/53] do not pause container when copy and use libarchive to do unpack/tar instead of execute tar command. Once not pause container, we need to chroot to container's rootfs first to avoid symlink attrack when copy. Signed-off-by: WangFengTu --- src/cmd/isula/main.c | 20 + src/cmd/isula/stream/cp.c | 3 +- .../executor/container_cb/execution_stream.c | 133 ++-- src/daemon/modules/image/oci/oci_load.c | 8 +- .../graphdriver/devmapper/driver_devmapper.c | 6 +- .../graphdriver/overlay2/driver_overlay2.c | 6 +- src/utils/tar/isulad_tar.c | 405 +----------- src/utils/tar/isulad_tar.h | 10 +- src/utils/tar/util_archive.c | 611 ++++++++++++++++-- src/utils/tar/util_archive.h | 15 +- 10 files changed, 679 insertions(+), 538 deletions(-) diff --git a/src/cmd/isula/main.c b/src/cmd/isula/main.c index d4a66695..a69df5d5 100644 --- a/src/cmd/isula/main.c +++ b/src/cmd/isula/main.c @@ -14,6 +14,7 @@ ******************************************************************************/ #include +#include #include "isula_commands.h" #include "create.h" @@ -202,8 +203,27 @@ struct command g_commands[] = { { NULL, false, NULL, NULL, NULL, NULL } // End of the list }; +static int set_locale() +{ + int ret = 0; + + /* Change from the standard (C) to en_US.UTF-8 locale, so libarchive can handle filename conversions.*/ + if (setlocale(LC_CTYPE, "en_US.UTF-8") == NULL) { + fprintf(stderr, "Could not set locale to en_US.UTF-8:%s", strerror(errno)); + ret = -1; + goto out; + } + +out: + return ret; +} + int main(int argc, char **argv) { + if (set_locale() != 0) { + exit(ECOMMON); + } + if (connect_client_ops_init()) { return ECOMMON; } diff --git a/src/cmd/isula/stream/cp.c b/src/cmd/isula/stream/cp.c index 4ebca2b3..e954ed3d 100644 --- a/src/cmd/isula/stream/cp.c +++ b/src/cmd/isula/stream/cp.c @@ -27,6 +27,7 @@ #include "path.h" #include "isula_connect.h" #include "isulad_tar.h" +#include "util_archive.h" #include "command_parser.h" #include "connect.h" #include "io_wrapper.h" @@ -124,7 +125,7 @@ static int client_copy_from_container(const struct client_arguments *args, const srcinfo->path = util_strdup_s(srcpath); srcinfo->isdir = S_ISDIR(response->stat->mode); - nret = archive_copy_to(&response->reader, false, srcinfo, resolved, &archive_err); + nret = archive_copy_to(&response->reader, srcinfo, resolved, &archive_err); if (nret != 0) { ret = nret; } diff --git a/src/daemon/executor/container_cb/execution_stream.c b/src/daemon/executor/container_cb/execution_stream.c index fde5d41d..7d165fb9 100644 --- a/src/daemon/executor/container_cb/execution_stream.c +++ b/src/daemon/executor/container_cb/execution_stream.c @@ -46,6 +46,7 @@ #include "image_api.h" #include "path.h" #include "isulad_tar.h" +#include "util_archive.h" #include "container_api.h" #include "error.h" #include "isula_libutils/logger_json_file.h" @@ -384,9 +385,18 @@ out: return ret; } +static char *get_tar_path(const char *srcdir, const char *srcbase, const char *container_fs) +{ + if (!util_has_prefix(srcdir, container_fs)) { + ERROR("srcdir %s does not contain %s", srcdir, container_fs); + return NULL; + } + return util_path_join(srcdir + strlen(container_fs), srcbase); +} + static int archive_and_send_copy_data(const stream_func_wrapper *stream, struct isulad_copy_from_container_response *response, const char *resolvedpath, - const char *abspath) + const char *abspath, const char *container_fs) { int ret = -1; int nret; @@ -399,6 +409,7 @@ static int archive_and_send_copy_data(const stream_func_wrapper *stream, char *buf = NULL; char cleaned[PATH_MAX + 2] = { 0 }; struct io_read_wrapper reader = { 0 }; + char *tar_path = NULL; buf = util_common_calloc_s(buf_len); if (buf == NULL) { @@ -422,7 +433,15 @@ static int archive_and_send_copy_data(const stream_func_wrapper *stream, ERROR("split %s failed", abspath); goto cleanup; } - nret = archive_path(srcdir, srcbase, absbase, false, &reader); + + tar_path = get_tar_path(srcdir, srcbase, container_fs); + if (tar_path == NULL) { + goto cleanup; + } + + DEBUG("archive chroot tar stream container_fs(%s) srcdir(%s) relative(%s) srcbase(%s) absbase(%s)", + container_fs, srcdir, tar_path, srcbase, absbase); + nret = archive_chroot_tar_stream(container_fs, tar_path, srcbase, absbase, &reader); if (nret != 0) { ERROR("Archive %s failed", resolvedpath); goto cleanup; @@ -445,6 +464,7 @@ static int archive_and_send_copy_data(const stream_func_wrapper *stream, ret = 0; cleanup: + free(tar_path); free(buf); free(srcdir); free(srcbase); @@ -583,58 +603,6 @@ static container_path_stat *resolve_and_stat_path(const char *rootpath, const ch return stat; } -static int pause_container(const container_t *cont) -{ - int ret = 0; - rt_pause_params_t params = { 0 }; - const char *id = cont->common_config->id; - - params.rootpath = cont->root_path; - params.state = cont->state_path; - if (runtime_pause(id, cont->runtime, ¶ms)) { - ERROR("Failed to pause container:%s", id); - ret = -1; - goto out; - } - - container_state_set_paused(cont->state); - - if (container_state_to_disk(cont)) { - ERROR("Failed to save container \"%s\" to disk", id); - ret = -1; - goto out; - } - -out: - return ret; -} - -static int resume_container(const container_t *cont) -{ - int ret = 0; - rt_resume_params_t params = { 0 }; - const char *id = cont->common_config->id; - - params.rootpath = cont->root_path; - params.state = cont->state_path; - if (runtime_resume(id, cont->runtime, ¶ms)) { - ERROR("Failed to resume container:%s", id); - ret = -1; - goto out; - } - - container_state_reset_paused(cont->state); - - if (container_state_to_disk(cont)) { - ERROR("Failed to save container \"%s\" to disk", id); - ret = -1; - goto out; - } - -out: - return ret; -} - static int copy_from_container_cb(const struct isulad_copy_from_container_request *request, const stream_func_wrapper *stream, char **err) { @@ -645,7 +613,6 @@ static int copy_from_container_cb(const struct isulad_copy_from_container_reques container_path_stat *stat = NULL; container_t *cont = NULL; struct isulad_copy_from_container_response *response = NULL; - bool need_pause = false; DAEMON_CLEAR_ERRMSG(); if (request == NULL || stream == NULL || err == NULL) { @@ -665,19 +632,10 @@ static int copy_from_container_cb(const struct isulad_copy_from_container_reques goto unlock_container; } - need_pause = container_is_running(cont->state) && !container_is_paused(cont->state); - if (need_pause) { - if (pause_container(cont) != 0) { - ERROR("can't copy to a container which is cannot be paused"); - isulad_set_error_message("can't copy to a container which is cannot be paused"); - goto unlock_container; - } - } - nret = im_mount_container_rootfs(cont->common_config->image_type, cont->common_config->image, cont->common_config->id); if (nret != 0) { - goto unpause_container; + goto unlock_container; } stat = resolve_and_stat_path(cont->common_config->base_fs, request->srcpath, &resolvedpath, &abspath); @@ -692,7 +650,7 @@ static int copy_from_container_cb(const struct isulad_copy_from_container_reques goto cleanup_rootfs; } - nret = archive_and_send_copy_data(stream, response, resolvedpath, abspath); + nret = archive_and_send_copy_data(stream, response, resolvedpath, abspath, cont->common_config->base_fs); if (nret < 0) { ERROR("Failed to send archive data"); goto cleanup_rootfs; @@ -705,10 +663,6 @@ cleanup_rootfs: cont->common_config->id) != 0) { WARN("Can not umount rootfs of container: %s", cont->common_config->id); } -unpause_container: - if (need_pause && resume_container(cont) != 0) { - ERROR("can't resume container which has been paused before copy"); - } unlock_container: container_unlock(cont); container_unref(cont); @@ -777,15 +731,16 @@ static ssize_t extract_stream_to_io_read(void *content, void *buf, size_t buf_le return (ssize_t)(copy.data_len); } -int read_and_extract_archive(stream_func_wrapper *stream, const char *resolved_path, const char *transform) +static int read_and_extract_archive(stream_func_wrapper *stream, const char *container_fs, + const char *dstdir_in_container, const char *src_rebase, + const char *dst_rebase) { int ret = -1; char *err = NULL; struct io_read_wrapper content = { 0 }; - content.context = stream; content.read = extract_stream_to_io_read; - ret = archive_untar(&content, false, resolved_path, transform, &err); + ret = archive_chroot_untar_stream(&content, container_fs, dstdir_in_container, src_rebase, dst_rebase, &err); if (ret != 0) { ERROR("Can not untar to container: %s", (err != NULL) ? err : "unknown"); isulad_set_error_message("Can not untar to container: %s", (err != NULL) ? err : "unknown"); @@ -795,7 +750,7 @@ int read_and_extract_archive(stream_func_wrapper *stream, const char *resolved_p } static char *copy_to_container_get_dstdir(const container_t *cont, const container_copy_to_request *request, - char **transform) + char **src_base, char **dst_base) { char *dstdir = NULL; char *error = NULL; @@ -836,7 +791,7 @@ static char *copy_to_container_get_dstdir(const container_t *cont, const contain srcinfo.path = request->src_path; srcinfo.rebase_name = request->src_rebase_name; - dstdir = prepare_archive_copy(&srcinfo, dstinfo, transform, &error); + dstdir = prepare_archive_copy(&srcinfo, dstinfo, src_base, dst_base, &error); if (dstdir == NULL) { if (error == NULL) { ERROR("Can not prepare archive copy"); @@ -930,9 +885,9 @@ static int copy_to_container_cb(const container_copy_to_request *request, stream char *resolvedpath = NULL; char *abspath = NULL; char *dstdir = NULL; - char *transform = NULL; + char *src_base = NULL; + char *dst_base = NULL; container_t *cont = NULL; - bool need_pause = false; DAEMON_CLEAR_ERRMSG(); if (request == NULL || stream == NULL || err == NULL) { @@ -952,22 +907,13 @@ static int copy_to_container_cb(const container_copy_to_request *request, stream goto unlock_container; } - need_pause = container_is_running(cont->state) && !container_is_paused(cont->state); - if (need_pause) { - if (pause_container(cont) != 0) { - ERROR("can't copy to a container which is cannot be paused"); - isulad_set_error_message("can't copy to a container which is cannot be paused"); - goto unlock_container; - } - } - nret = im_mount_container_rootfs(cont->common_config->image_type, cont->common_config->image, cont->common_config->id); if (nret != 0) { - goto unpause_container; + goto unlock_container; } - dstdir = copy_to_container_get_dstdir(cont, request, &transform); + dstdir = copy_to_container_get_dstdir(cont, request, &src_base, &dst_base); if (dstdir == NULL) { goto cleanup_rootfs; } @@ -982,7 +928,8 @@ static int copy_to_container_cb(const container_copy_to_request *request, stream goto cleanup_rootfs; } - nret = read_and_extract_archive(stream, resolvedpath, transform); + nret = read_and_extract_archive(stream, cont->common_config->base_fs, + dstdir, src_base, dst_base); if (nret < 0) { ERROR("Failed to send archive data"); goto cleanup_rootfs; @@ -997,11 +944,6 @@ cleanup_rootfs: WARN("Can not umount rootfs of container: %s", cont->common_config->id); } -unpause_container: - if (need_pause && resume_container(cont) != 0) { - ERROR("can't resume container which has been paused before copy"); - } - unlock_container: container_unlock(cont); container_unref(cont); @@ -1013,7 +955,8 @@ pack_response: free(resolvedpath); free(abspath); free(dstdir); - free(transform); + free(src_base); + free(dst_base); return ret; } diff --git a/src/daemon/modules/image/oci/oci_load.c b/src/daemon/modules/image/oci/oci_load.c index 80647253..a8eecfe9 100644 --- a/src/daemon/modules/image/oci/oci_load.c +++ b/src/daemon/modules/image/oci/oci_load.c @@ -1061,6 +1061,7 @@ int oci_do_load(const im_load_request *request) load_image_t *im = NULL; char *digest = NULL; char *dstdir = NULL; + char *err = NULL; if (request == NULL || request->file == NULL) { ERROR("Invalid input arguments, cannot load image"); @@ -1082,9 +1083,9 @@ int oci_do_load(const im_load_request *request) } options.whiteout_format = NONE_WHITEOUT_FORMATE; - if (archive_unpack(&reader, dstdir, &options) != 0) { - ERROR("Failed to unpack to :%s", dstdir); - isulad_try_set_error_message("Failed to unpack to :%s", dstdir); + if (archive_unpack(&reader, dstdir, &options, &err) != 0) { + ERROR("Failed to unpack to %s: %s", dstdir, err); + isulad_try_set_error_message("Failed to unpack to %s: %s", dstdir, err); ret = -1; goto out; } @@ -1167,5 +1168,6 @@ out: WARN("failed to remove directory %s", dstdir); } free(dstdir); + free(err); return ret; } diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c index f2586f0d..e91ffe05 100644 --- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c +++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c @@ -319,6 +319,7 @@ int devmapper_apply_diff(const char *id, const struct graphdriver *driver, const char *layer_fs = NULL; int ret = 0; struct archive_options options = { 0 }; + char *err = NULL; if (!util_valid_str(id) || driver == NULL || content == NULL) { ERROR("invalid argument to apply diff with id(%s)", id); @@ -340,8 +341,8 @@ int devmapper_apply_diff(const char *id, const struct graphdriver *driver, const } options.whiteout_format = REMOVE_WHITEOUT_FORMATE; - if (archive_unpack(content, layer_fs, &options) != 0) { - ERROR("devmapper: failed to unpack to :%s", layer_fs); + if (archive_unpack(content, layer_fs, &options, &err) != 0) { + ERROR("devmapper: failed to unpack to %s: %s", layer_fs, err); ret = -1; goto out; } @@ -355,6 +356,7 @@ int devmapper_apply_diff(const char *id, const struct graphdriver *driver, const out: free_driver_mount_opts(mount_opts); free(layer_fs); + free(err); return ret; } diff --git a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c index 6cdabe54..659d9d52 100644 --- a/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c +++ b/src/daemon/modules/image/oci/storage/layer_store/graphdriver/overlay2/driver_overlay2.c @@ -1657,6 +1657,7 @@ int overlay2_apply_diff(const char *id, const struct graphdriver *driver, const char *layer_dir = NULL; char *layer_diff = NULL; struct archive_options options = { 0 }; + char *err = NULL; if (id == NULL || driver == NULL || content == NULL) { ERROR("invalid argument"); @@ -1680,14 +1681,15 @@ int overlay2_apply_diff(const char *id, const struct graphdriver *driver, const options.whiteout_format = OVERLAY_WHITEOUT_FORMATE; - ret = archive_unpack(content, layer_diff, &options); + ret = archive_unpack(content, layer_diff, &options, &err); if (ret != 0) { - ERROR("Failed to unpack to :%s", layer_diff); + ERROR("Failed to unpack to %s: %s", layer_diff, err); ret = -1; goto out; } out: + free(err); free(layer_dir); free(layer_diff); return ret; diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c index 5edf2ac3..03277373 100644 --- a/src/utils/tar/isulad_tar.c +++ b/src/utils/tar/isulad_tar.c @@ -31,17 +31,7 @@ #include "isula_libutils/log.h" #include "error.h" #include "isula_libutils/json_common.h" -#include "io_wrapper.h" -#include "utils_file.h" -#include "utils_verify.h" - -#define TAR_MAX_OPTS 50 -#define TAR_CMD "tar" -#define TAR_TRANSFORM_OPT "--transform" -#define TAR_CREATE_OPT "-c" -#define TAR_EXACT_OPT "-x" -#define TAR_CHDIR_OPT "-C" -#define TAR_GZIP_OPT "-z" +#include "util_archive.h" static void set_char_to_separator(char *p) { @@ -126,110 +116,6 @@ int gzip(const char *filename, size_t len) return status; } -struct archive_context { - int stdin_fd; - int stdout_fd; - int stderr_fd; - pid_t pid; -}; - -static ssize_t archive_context_read(void *context, void *buf, size_t len) -{ - struct archive_context *ctx = (struct archive_context *)context; - if (ctx == NULL) { - return -1; - } - if (ctx->stdout_fd >= 0) { - return util_read_nointr(ctx->stdout_fd, buf, len); - } - return 0; -} - -static ssize_t archive_context_write(const void *context, const void *buf, size_t len) -{ - struct archive_context *ctx = (struct archive_context *)context; - if (ctx == NULL) { - return -1; - } - if (ctx->stdin_fd >= 0) { - return util_write_nointr(ctx->stdin_fd, buf, len); - } - return 0; -} - -static int close_wait_pid(struct archive_context *ctx, int *status) -{ - int ret = 0; - - // close stdin and stdout first, this will make sure the process of tar exit. - if (ctx->stdin_fd >= 0) { - close(ctx->stdin_fd); - } - - if (ctx->stdout_fd >= 0) { - close(ctx->stdout_fd); - } - - if (ctx->pid > 0) { - if (waitpid(ctx->pid, status, 0) != ctx->pid) { - ERROR("Failed to wait pid %u", ctx->pid); - ret = -1; - } - } - - return ret; -} - -static int archive_context_close(void *context, char **err) -{ - int ret = 0; - int status = 0; - char *reason = NULL; - ssize_t size_read = 0; - char buffer[BUFSIZ + 1] = { 0 }; - struct archive_context *ctx = (struct archive_context *)context; - char *marshaled = NULL; - - if (ctx == NULL) { - return 0; - } - - ret = close_wait_pid(ctx, &status); - - if (WIFSIGNALED((unsigned int)status)) { - status = WTERMSIG(status); - reason = "signaled"; - } else if (WIFEXITED(status)) { - status = WEXITSTATUS(status); - reason = "exited"; - } else { - reason = "unknown"; - } - - if (ctx->stderr_fd >= 0) { - size_read = util_read_nointr(ctx->stderr_fd, buffer, BUFSIZ); - if (size_read > 0) { - reason = buffer; - marshaled = util_marshal_string(buffer); - if (marshaled == NULL) { - ERROR("Can not marshal json buffer: %s", buffer); - } else { - reason = marshaled; - } - } - close(ctx->stderr_fd); - } - - if (size_read > 0 || status != 0) { - format_errorf(err, "tar exited with status %d: %s", status, reason); - ret = -1; - } - - free(marshaled); - free(ctx); - return ret; -} - static int get_rebase_name(const char *path, const char *real_path, char **resolved_path, char **rebase_name) { int nret; @@ -502,50 +388,8 @@ static bool asserts_directory(const char *path) return util_has_trailing_path_separator(path) || util_specify_current_dir(path); } -static char *format_transform_of_tar(const char *srcbase, const char *dstbase) -{ - char *transform = NULL; - const char *src_escaped = srcbase; - const char *dst_escaped = dstbase; - int nret; - size_t len; - - if (srcbase == NULL || dstbase == NULL) { - return NULL; - } - - // escape "/" by "." to avoid generating leading / in tar archive which is dangerous to host when untar. - // this means tar or untar with leading / is forbidden and may got error, take care of this when coding. - if (strcmp(srcbase, "/") == 0) { - src_escaped = "."; - } - - if (strcmp(dstbase, "/") == 0) { - dst_escaped = "."; - } - - len = strlen(src_escaped) + strlen(dst_escaped) + 5; - if (len > PATH_MAX) { - ERROR("Invalid path length"); - return NULL; - } - - transform = util_common_calloc_s(len); - if (transform == NULL) { - ERROR("Out of memory"); - return NULL; - } - nret = snprintf(transform, len, "s/%s/%s/", src_escaped, dst_escaped); - if (nret < 0 || (size_t)nret >= len) { - ERROR("Failed to print string"); - free(transform); - return NULL; - } - return transform; -} - char *prepare_archive_copy(const struct archive_copy_info *srcinfo, const struct archive_copy_info *dstinfo, - char **transform, char **err) + char **src_base, char **dst_base, char **err) { char *dstdir = NULL; char *srcbase = NULL; @@ -573,7 +417,8 @@ char *prepare_archive_copy(const struct archive_copy_info *srcinfo, const struct free(srcbase); srcbase = util_strdup_s(srcinfo->rebase_name); } - *transform = format_transform_of_tar(srcbase, dstbase); + *src_base = util_strdup_s(srcbase); + *dst_base = util_strdup_s(dstbase); } else if (srcinfo->isdir) { // dst does not exist and src is a directory, untar the content to parent of dest, // and rename basename of src name to dest's basename. @@ -581,7 +426,8 @@ char *prepare_archive_copy(const struct archive_copy_info *srcinfo, const struct free(srcbase); srcbase = util_strdup_s(srcinfo->rebase_name); } - *transform = format_transform_of_tar(srcbase, dstbase); + *src_base = util_strdup_s(srcbase); + *dst_base = util_strdup_s(dstbase); } else if (asserts_directory(dstinfo->path)) { // dst does not exist and is want to be created as a directory, but src is not a directory, report error. format_errorf(err, "no such directory, can not copy file"); @@ -594,7 +440,8 @@ char *prepare_archive_copy(const struct archive_copy_info *srcinfo, const struct free(srcbase); srcbase = util_strdup_s(srcinfo->rebase_name); } - *transform = format_transform_of_tar(srcbase, dstbase); + *src_base = util_strdup_s(srcbase); + *dst_base = util_strdup_s(dstbase); } cleanup: @@ -603,125 +450,14 @@ cleanup: return dstdir; } -static void close_pipe_fd(int pipe_fd[]) -{ - if (pipe_fd[0] != -1) { - close(pipe_fd[0]); - pipe_fd[0] = -1; - } - if (pipe_fd[1] != -1) { - close(pipe_fd[1]); - pipe_fd[1] = -1; - } -} - -int archive_untar(const struct io_read_wrapper *content, bool compression, const char *dstdir, const char *transform, - char **err) -{ - int stdin_pipe[2] = { -1, -1 }; - int stderr_pipe[2] = { -1, -1 }; - int ret = -1; - int cret = 0; - pid_t pid; - struct archive_context *ctx = NULL; - char *buf = NULL; - size_t buf_len = ARCHIVE_BLOCK_SIZE; - ssize_t read_len; - const char *params[TAR_MAX_OPTS] = { NULL }; - - buf = util_common_calloc_s(buf_len); - if (buf == NULL) { - ERROR("Out of memory"); - return -1; - } - - if (pipe(stderr_pipe) != 0) { - ERROR("Failed to create pipe: %s", strerror(errno)); - goto cleanup; - } - if (pipe(stdin_pipe) != 0) { - ERROR("Failed to create pipe: %s", strerror(errno)); - goto cleanup; - } - - pid = fork(); - if (pid == (pid_t) -1) { - ERROR("Failed to fork: %s", strerror(errno)); - goto cleanup; - } - - if (pid == (pid_t)0) { - int i = 0; - // child process, dup2 stderr[1] to stderr, stdout[0] to stdin. - close(stderr_pipe[0]); - dup2(stderr_pipe[1], 2); - close(stdin_pipe[1]); - dup2(stdin_pipe[0], 0); - - params[i++] = TAR_CMD; - params[i++] = TAR_EXACT_OPT; - if (compression) { - params[i++] = TAR_GZIP_OPT; - } - params[i++] = TAR_CHDIR_OPT; - params[i++] = dstdir; - if (transform != NULL) { - params[i++] = TAR_TRANSFORM_OPT; - params[i++] = transform; - } - - execvp(TAR_CMD, (char * const *)params); - - fprintf(stderr, "Failed to exec tar: %s", strerror(errno)); - exit(EXIT_FAILURE); - } - - close(stderr_pipe[1]); - stderr_pipe[1] = -1; - close(stdin_pipe[0]); - stdin_pipe[0] = -1; - - ctx = util_common_calloc_s(sizeof(struct archive_context)); - if (ctx == NULL) { - goto cleanup; - } - - ctx->pid = pid; - ctx->stdin_fd = stdin_pipe[1]; - stdin_pipe[1] = -1; - ctx->stdout_fd = -1; - ctx->stderr_fd = stderr_pipe[0]; - stderr_pipe[0] = -1; - - read_len = content->read(content->context, buf, buf_len); - while (read_len > 0) { - ssize_t writed_len = archive_context_write(ctx, buf, (size_t)read_len); - if (writed_len < 0) { - DEBUG("Tar may exited: %s", strerror(errno)); - break; - } - read_len = content->read(content->context, buf, buf_len); - } - - ret = 0; - -cleanup: - free(buf); - cret = archive_context_close(ctx, err); - ret = (cret != 0) ? cret : ret; - close_pipe_fd(stderr_pipe); - close_pipe_fd(stdin_pipe); - - return ret; -} - -int archive_copy_to(const struct io_read_wrapper *content, bool compression, const struct archive_copy_info *srcinfo, +int archive_copy_to(const struct io_read_wrapper *content, const struct archive_copy_info *srcinfo, const char *dstpath, char **err) { int ret = -1; struct archive_copy_info *dstinfo = NULL; char *dstdir = NULL; - char *transform = NULL; + char *src_base = NULL; + char *dst_base = NULL; dstinfo = copy_info_destination_path(dstpath, err); if (dstinfo == NULL) { @@ -729,128 +465,23 @@ int archive_copy_to(const struct io_read_wrapper *content, bool compression, con return -1; } - dstdir = prepare_archive_copy(srcinfo, dstinfo, &transform, err); + dstdir = prepare_archive_copy(srcinfo, dstinfo, &src_base, &dst_base, err); if (dstdir == NULL) { ERROR("Can not prepare archive copy"); goto cleanup; } - ret = archive_untar(content, compression, dstdir, transform, err); + ret = archive_chroot_untar_stream(content, dstdir, ".", src_base, dst_base, err); cleanup: free_archive_copy_info(dstinfo); free(dstdir); - free(transform); - return ret; -} - -static void close_archive_pipes_fd(int *pipes, size_t pipe_size) -{ - size_t i = 0; - - for (i = 0; i < pipe_size; i++) { - if (pipes[i] >= 0) { - close(pipes[i]); - pipes[i] = -1; - } - } -} - -/* - * Archive file or directory. - * param src : file or directory to compression. - * param compression : using gzip compression or not - * param exclude_base : exclude source basename in the archived file or not - * return : zero if archive success, non-zero if not. - */ -int archive_path(const char *srcdir, const char *srcbase, const char *rebase_name, bool compression, - struct io_read_wrapper *archive_reader) -{ - int stderr_pipe[2] = { -1, -1 }; - int stdout_pipe[2] = { -1, -1 }; - int ret = -1; - pid_t pid; - struct archive_context *ctx = NULL; - char *transform = NULL; - const char *params[TAR_MAX_OPTS] = { NULL }; - - transform = format_transform_of_tar(srcbase, rebase_name); - - if (pipe(stderr_pipe) != 0) { - ERROR("Failed to create pipe: %s", strerror(errno)); - goto free_out; - } - if (pipe(stdout_pipe) != 0) { - ERROR("Failed to create pipe: %s", strerror(errno)); - goto free_out; - } - - pid = fork(); - if (pid == (pid_t) -1) { - ERROR("Failed to fork: %s", strerror(errno)); - goto free_out; - } - - if (pid == (pid_t)0) { - int i = 0; - // child process, dup2 stderr[1] to stderr, stdout[1] to stdout. - close(stderr_pipe[0]); - close(stdout_pipe[0]); - dup2(stderr_pipe[1], 2); - dup2(stdout_pipe[1], 1); - - params[i++] = TAR_CMD; - params[i++] = TAR_CREATE_OPT; - if (compression) { - params[i++] = TAR_GZIP_OPT; - } - params[i++] = TAR_CHDIR_OPT; - params[i++] = srcdir; - if (transform != NULL) { - params[i++] = TAR_TRANSFORM_OPT; - params[i++] = transform; - } - params[i++] = srcbase; - - execvp(TAR_CMD, (char * const *)params); - - fprintf(stderr, "Failed to exec tar: %s", strerror(errno)); - exit(EXIT_FAILURE); - } - - close(stderr_pipe[1]); - stderr_pipe[1] = -1; - close(stdout_pipe[1]); - stdout_pipe[1] = -1; - - ctx = util_common_calloc_s(sizeof(struct archive_context)); - if (ctx == NULL) { - goto free_out; - } - - ctx->stdin_fd = -1; - ctx->stdout_fd = stdout_pipe[0]; - stdout_pipe[0] = -1; - ctx->stderr_fd = stderr_pipe[0]; - stderr_pipe[0] = -1; - ctx->pid = pid; - - archive_reader->close = archive_context_close; - archive_reader->context = ctx; - ctx = NULL; - archive_reader->read = archive_context_read; - - ret = 0; -free_out: - free(transform); - close_archive_pipes_fd(stderr_pipe, 2); - close_archive_pipes_fd(stdout_pipe, 2); - free(ctx); - + free(src_base); + free(dst_base); return ret; } -int tar_resource_rebase(const char *path, const char *rebase, struct io_read_wrapper *archive_reader, char **err) +static int tar_resource_rebase(const char *path, const char *rebase, struct io_read_wrapper *archive_reader, char **err) { int ret = -1; int nret; @@ -868,8 +499,8 @@ int tar_resource_rebase(const char *path, const char *rebase, struct io_read_wra goto cleanup; } - DEBUG("Copying %s from %s", srcbase, srcdir); - nret = archive_path(srcdir, srcbase, rebase, false, archive_reader); + DEBUG("chroot tar stream srcdir(%s) srcbase(%s) rebase(%s)", srcdir, srcbase, rebase); + nret = archive_chroot_tar_stream(srcdir, srcbase, srcbase, rebase, archive_reader); if (nret < 0) { ERROR("Can not archive path: %s", path); goto cleanup; diff --git a/src/utils/tar/isulad_tar.h b/src/utils/tar/isulad_tar.h index e2b78463..c773fe9b 100644 --- a/src/utils/tar/isulad_tar.h +++ b/src/utils/tar/isulad_tar.h @@ -57,19 +57,13 @@ int gzip(const char *filename, size_t len); struct archive_copy_info *copy_info_source_path(const char *path, bool follow_link, char **err); char *prepare_archive_copy(const struct archive_copy_info *srcinfo, const struct archive_copy_info *dstinfo, - char **transform, char **err); + char **src_base, char **dst_base, char **err); int tar_resource(const struct archive_copy_info *info, struct io_read_wrapper *archive_reader, char **err); -int archive_untar(const struct io_read_wrapper *content, bool compression, const char *dstdir, const char *transform, - char **err); - -int archive_copy_to(const struct io_read_wrapper *content, bool compression, const struct archive_copy_info *srcinfo, +int archive_copy_to(const struct io_read_wrapper *content, const struct archive_copy_info *srcinfo, const char *dstpath, char **err); -int archive_path(const char *srcdir, const char *srcbase, const char *rebase_name, bool compression, - struct io_read_wrapper *archive_reader); - #ifdef __cplusplus } #endif diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c index 234e661e..7a28286a 100644 --- a/src/utils/tar/util_archive.c +++ b/src/utils/tar/util_archive.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "stdbool.h" #include "utils.h" @@ -33,11 +34,14 @@ #include "io_wrapper.h" #include "utils_file.h" #include "map.h" +#include "path.h" +#include "error.h" struct archive; struct archive_entry; #define ARCHIVE_READ_BUFFER_SIZE (10 * 1024) +#define ARCHIVE_WRITE_BUFFER_SIZE (10 * 1024) #define TAR_DEFAULT_MODE 0600 #define TAR_DEFAULT_FLAG (O_WRONLY | O_CREAT | O_TRUNC) @@ -45,6 +49,13 @@ struct archive_entry; #define WHITEOUT_META_PREFIX ".wh..wh." #define WHITEOUT_OPAQUEDIR ".wh..wh..opq" +struct archive_context { + int stdin_fd; + int stdout_fd; + int stderr_fd; + pid_t pid; +}; + struct archive_content_data { const struct io_read_wrapper *content; char buff[ARCHIVE_READ_BUFFER_SIZE]; @@ -286,8 +297,104 @@ static whiteout_convert_call_back_t get_whiteout_convert_cb(whiteout_format_type return NULL; } -int archive_unpack_handler(const struct io_read_wrapper *content, const char *dstdir, - const struct archive_options *options) +static char *to_relative_path(const char *path) +{ + char *dst_path = NULL; + + if (path != NULL && path[0] == '/') { + if (strcmp(path, "/") == 0) { + dst_path = util_strdup_s("."); + } else { + dst_path = util_strdup_s(path + 1); + } + } else { + dst_path = util_strdup_s(path); + } + + return dst_path; +} + +static int rebase_pathname(struct archive_entry *entry, const char *src_base, const char *dst_base) +{ + int nret = 0; + const char *pathname = archive_entry_pathname(entry); + char path[PATH_MAX] = { 0 }; + + if (src_base == NULL || dst_base == NULL || !util_has_prefix(pathname, src_base)) { + return 0; + } + + nret = snprintf(path, sizeof(path), "%s%s", dst_base, pathname + strlen(src_base)); + if (nret < 0 || (size_t)nret >= sizeof(path)) { + ERROR("snprintf %s%s failed", dst_base, pathname + strlen(src_base)); + fprintf(stderr, "snprintf %s%s failed", dst_base, pathname + strlen(src_base)); + return -1; + } + + archive_entry_set_pathname(entry, path); + + return 0; +} + +static char *update_entry_for_pathname(struct archive_entry *entry, const char *src_base, const char *dst_base) +{ + char *dst_path = NULL; + const char *pathname = NULL; + + if (rebase_pathname(entry, src_base, dst_base) != 0) { + return NULL; + } + + pathname = archive_entry_pathname(entry); + if (pathname == NULL) { + ERROR("Failed to get archive entry path name"); + fprintf(stderr, "Failed to get archive entry path name"); + return NULL; + } + + // if path in archive is absolute, we need to translate it to relative because + // libarchive can not support absolute path when unpack + dst_path = to_relative_path(pathname); + if (dst_path == NULL) { + ERROR("translate %s to relative path failed", pathname); + fprintf(stderr, "translate %s to relative path failed", pathname); + goto out; + } + + archive_entry_set_pathname(entry, dst_path); +out: + + return dst_path; +} + +static int rebase_hardlink(struct archive_entry *entry, const char *src_base, const char *dst_base) +{ + int nret = 0; + const char *linkname = NULL; + char path[PATH_MAX] = { 0 }; + + linkname = archive_entry_hardlink(entry); + if (linkname == NULL) { + return 0; + } + + if (src_base == NULL || dst_base == NULL || !util_has_prefix(linkname, src_base)) { + return 0; + } + + nret = snprintf(path, sizeof(path), "%s%s", dst_base, linkname + strlen(src_base)); + if (nret < 0 || (size_t)nret >= sizeof(path)) { + ERROR("snprintf %s%s failed", dst_base, linkname + strlen(src_base)); + fprintf(stderr, "snprintf %s%s failed", dst_base, linkname + strlen(src_base)); + return -1; + } + + archive_entry_set_hardlink(entry, path); + + return 0; +} + +int archive_unpack_handler(const struct io_read_wrapper *content, const struct archive_options *options) { int ret = 0; struct archive *a = NULL; @@ -302,6 +409,7 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds unpacked_path_map = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC); if (unpacked_path_map == NULL) { ERROR("Out of memory"); + fprintf(stderr, "Out of memory"); ret = -1; goto out; } @@ -309,6 +417,7 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds mydata = util_common_calloc_s(sizeof(struct archive_content_data)); if (mydata == NULL) { ERROR("Memory out"); + fprintf(stderr, "Memory out"); ret = -1; goto out; } @@ -327,6 +436,7 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds a = archive_read_new(); if (a == NULL) { ERROR("archive read new failed"); + fprintf(stderr, "archive read new failed"); ret = -1; goto out; } @@ -336,6 +446,7 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds ext = archive_write_disk_new(); if (ext == NULL) { ERROR("archive write disk new failed"); + fprintf(stderr, "archive write disk new failed"); ret = -1; goto out; } @@ -345,6 +456,7 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds ret = archive_read_open(a, mydata, NULL, read_content, NULL); if (ret != 0) { SYSERROR("Failed to open archive"); + fprintf(stderr, "Failed to open archive: %s", strerror(errno)); ret = -1; goto out; } @@ -354,7 +466,6 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds for (;;) { free(dst_path); dst_path = NULL; - ret = archive_read_next_header(a, &entry); if (ret == ARCHIVE_EOF) { @@ -363,20 +474,23 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds if (ret != ARCHIVE_OK) { ERROR("Warning reading tar header: %s", archive_error_string(a)); + fprintf(stderr, "Warning reading tar header: %s", archive_error_string(a)); ret = -1; goto out; } - const char *pathname = archive_entry_pathname(entry); - if (pathname == NULL) { - ERROR("Failed to get archive entry path name"); + dst_path = update_entry_for_pathname(entry, options->src_base, options->dst_base); + if (dst_path == NULL) { + ERROR("Failed to update pathname"); + fprintf(stderr, "Failed to update pathname"); ret = -1; goto out; } - dst_path = util_path_join(dstdir, pathname); - if (dst_path == NULL) { - ERROR("Failed to get archive entry dst path %s/%s", dstdir, pathname); + ret = rebase_hardlink(entry, options->src_base, options->dst_base); + if (ret != 0) { + ERROR("Failed to rebase hardlink"); + fprintf(stderr, "Failed to rebase hardlink"); ret = -1; goto out; } @@ -385,22 +499,17 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds continue; } - // if path in archive is absolute, we need to translate it to relative because - // libarchive can not support absolute path when unpack - pathname = archive_entry_pathname(entry); - if (pathname != NULL && pathname[0] == '/') { - archive_entry_set_pathname(entry, pathname + 1); - } - ret = archive_write_header(ext, entry); if (ret != ARCHIVE_OK) { ERROR("Fail to handle tar header: %s", archive_error_string(ext)); + fprintf(stderr, "Fail to handle tar header: %s", archive_error_string(ext)); ret = -1; goto out; } else if (archive_entry_size(entry) > 0) { ret = copy_data(a, ext); if (ret != ARCHIVE_OK) { ERROR("Failed to do copy tar data: %s", archive_error_string(ext)); + fprintf(stderr, "Failed to do copy tar data: %s", archive_error_string(ext)); ret = -1; goto out; } @@ -408,6 +517,7 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds ret = archive_write_finish_entry(ext); if (ret != ARCHIVE_OK) { ERROR("Failed to freeing archive entry: %s\n", archive_error_string(ext)); + fprintf(stderr, "Failed to freeing archive entry: %s\n", archive_error_string(ext)); ret = -1; goto out; } @@ -415,6 +525,7 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const char *ds bool b = true; if (!map_replace(unpacked_path_map, (void *)dst_path, (void *)(&b))) { ERROR("Failed to replace unpacked path map element"); + fprintf(stderr, "Failed to replace unpacked path map element"); ret = -1; goto out; } @@ -433,11 +544,32 @@ out: return ret; } -int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, const struct archive_options *options) +static void close_archive_pipes_fd(int *pipes, size_t pipe_size) +{ + size_t i = 0; + + for (i = 0; i < pipe_size; i++) { + if (pipes[i] >= 0) { + close(pipes[i]); + pipes[i] = -1; + } + } +} + +int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, const struct archive_options *options, + char **errmsg) { int ret = 0; pid_t pid = -1; - int keepfds[] = { -1, -1 }; + int keepfds[] = { -1, -1, -1 }; + int pipe_stderr[2] = { -1, -1 }; + char errbuf[BUFSIZ] = { 0 }; + + if (pipe2(pipe_stderr, O_CLOEXEC) != 0) { + ERROR("Failed to create pipe"); + ret = -1; + goto cleanup; + } pid = fork(); if (pid == (pid_t) -1) { @@ -448,26 +580,37 @@ int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, co if (pid == (pid_t)0) { keepfds[0] = isula_libutils_get_log_fd(); keepfds[1] = *(int *)(content->context); - ret = util_check_inherited_exclude_fds(true, keepfds, 2); + keepfds[2] = pipe_stderr[1]; + ret = util_check_inherited_exclude_fds(true, keepfds, 3); if (ret != 0) { ERROR("Failed to close fds."); + fprintf(stderr, "Failed to close fds."); + ret = -1; + goto child_out; + } + + // child process, dup2 pipe_for_read[1] to stderr, + if (dup2(pipe_stderr[1], 2) < 0) { + ERROR("Dup fd error: %s", strerror(errno)); ret = -1; goto child_out; } if (chroot(dstdir) != 0) { SYSERROR("Failed to chroot to %s", dstdir); + fprintf(stderr, "Failed to chroot to %s: %s", dstdir, strerror(errno)); ret = -1; goto child_out; } if (chdir("/") != 0) { SYSERROR("Failed to chroot to /"); + fprintf(stderr, "Failed to chroot to /: %s", strerror(errno)); ret = -1; goto child_out; } - ret = archive_unpack_handler(content, "/", options); + ret = archive_unpack_handler(content, options); child_out: if (ret != 0) { @@ -476,13 +619,23 @@ child_out: exit(EXIT_SUCCESS); } } + close(pipe_stderr[1]); + pipe_stderr[1] = -1; ret = util_wait_for_pid(pid); if (ret != 0) { ERROR("Wait archive_untar_handler failed"); + fcntl(pipe_stderr[0], F_SETFL, O_NONBLOCK); + if (read(pipe_stderr[0], errbuf, BUFSIZ) < 0) { + ERROR("read error message from child failed"); + } } cleanup: + close_archive_pipes_fd(pipe_stderr, 2); + if (errmsg != NULL && strlen(errbuf) != 0) { + *errmsg = util_strdup_s(errbuf); + } return ret; } @@ -569,19 +722,19 @@ static int copy_data_between_archives(struct archive *ar, struct archive *aw) } } -int update_entry_for_hardlink(map_t *map_link, struct archive_entry *entry) +int update_entry_for_hardlink(map_t *map_link, struct archive_entry *entry, const char *src_base, const char *dst_base) { const char *path = archive_entry_pathname(entry); char *linkname = NULL; unsigned int nlink = archive_entry_nlink(entry); int ino = archive_entry_ino(entry); + const char *hardlink = archive_entry_hardlink(entry); - // hardlink is regular file, not type AE_IFLNK - if (archive_entry_filetype(entry) != AE_IFREG) { - return 0; + if (hardlink != NULL && rebase_hardlink(entry, src_base, dst_base) != 0) { + return -1; } - // no hardlink + // try to use hardlink to reduce tar size if (nlink <= 1) { return 0; } @@ -610,11 +763,12 @@ static void link_kvfree(void *key, void *value) return; } -int tar_handler(struct archive *r, struct archive *w) +int tar_handler(struct archive *r, struct archive *w, const char *src_base, const char *dst_base) { int ret = ARCHIVE_OK; struct archive_entry *entry = NULL; map_t *map_link = NULL; + char *pathname = NULL; map_link = map_new(MAP_INT_STR, MAP_DEFAULT_CMP_FUNC, link_kvfree); if (map_link == NULL) { @@ -636,11 +790,18 @@ int tar_handler(struct archive *r, struct archive *w) break; } - if (update_entry_for_hardlink(map_link, entry) != 0) { + pathname = update_entry_for_pathname(entry, src_base, dst_base); + if (pathname == NULL) { ret = ARCHIVE_FAILED; break; } + free(pathname); + pathname = NULL; + if (update_entry_for_hardlink(map_link, entry, src_base, dst_base) != 0) { + ret = ARCHIVE_FAILED; + break; + } ret = archive_write_header(w, entry); if (ret != ARCHIVE_OK) { ERROR("Fail to write tar header: %s", archive_error_string(w)); @@ -680,7 +841,29 @@ int tar_handler(struct archive *r, struct archive *w) return ret; } -static int tar_all(int fd) +static ssize_t stream_write_data(struct archive *a, void *client_data, const void *buffer, size_t length) +{ + struct io_write_wrapper *writer = (struct io_write_wrapper *)client_data; + size_t written_length = 0; + size_t size = 0; + while (length > written_length) { + if (length - written_length > ARCHIVE_WRITE_BUFFER_SIZE) { + size = ARCHIVE_WRITE_BUFFER_SIZE; + } else { + size = length - written_length; + } + if (!writer->write_func(writer->context, (const char *)buffer + written_length, size)) { + ERROR("write stream failed"); + return -1; + } + written_length += size; + } + + return size; +} + +static int tar_all(const struct io_write_wrapper *writer, const char *tar_dir, + const char *src_base, const char *dst_base) { struct archive *r = NULL; struct archive *w = NULL; @@ -689,12 +872,13 @@ static int tar_all(int fd) r = archive_read_disk_new(); if (r == NULL) { ERROR("archive read disk new failed"); + fprintf(stderr, "archive read disk new failed"); return -1; } archive_read_disk_set_standard_lookup(r); archive_read_disk_set_symlink_physical(r); archive_read_disk_set_behavior(r, ARCHIVE_READDISK_NO_TRAVERSE_MOUNTS); - ret = archive_read_disk_open(r, "."); + ret = archive_read_disk_open(r, tar_dir); if (ret != ARCHIVE_OK) { ERROR("open archive read failed: %s", archive_error_string(r)); fprintf(stderr, "open archive read failed: %s\n", archive_error_string(r)); @@ -704,19 +888,20 @@ static int tar_all(int fd) w = archive_write_new(); if (w == NULL) { ERROR("archive write new failed"); + fprintf(stderr, "archive write new failed"); ret = ARCHIVE_FAILED; goto out; } archive_write_set_format_pax(w); archive_write_set_options(w, "xattrheader=SCHILY"); - ret = archive_write_open_fd(w, fd); + ret = archive_write_open(w, (void*)writer, NULL, stream_write_data, NULL); if (ret != ARCHIVE_OK) { ERROR("open archive write failed: %s", archive_error_string(w)); fprintf(stderr, "open archive write failed: %s\n", archive_error_string(w)); goto out; } - ret = tar_handler(r, w); + ret = tar_handler(r, w, src_base, dst_base); out: archive_free(r); @@ -725,8 +910,14 @@ out: return (ret == ARCHIVE_OK) ? 0 : -1; } +static ssize_t fd_write(void *context, const void *data, size_t len) +{ + return util_write_nointr(*(int*)context, data, len); +} + int archive_chroot_tar(char *path, char *file, char **errmsg) { + struct io_write_wrapper pipe_context = { 0 }; int ret = 0; pid_t pid; int pipe_for_read[2] = { -1, -1 }; @@ -744,8 +935,6 @@ int archive_chroot_tar(char *path, char *file, char **errmsg) if (pid == (pid_t) -1) { ERROR("Failed to fork()"); ret = -1; - close(pipe_for_read[0]); - close(pipe_for_read[1]); goto cleanup; } @@ -788,7 +977,9 @@ int archive_chroot_tar(char *path, char *file, char **errmsg) goto child_out; } - ret = tar_all(fd); + pipe_context.context = (void*)&fd; + pipe_context.write_func = fd_write; + ret = tar_all(&pipe_context, ".", ".", NULL); child_out: @@ -798,6 +989,8 @@ child_out: exit(EXIT_SUCCESS); } } + close(pipe_for_read[1]); + pipe_for_read[1] = -1; ret = util_wait_for_pid(pid); if (ret != 0) { @@ -806,17 +999,357 @@ child_out: if (read(pipe_for_read[0], errbuf, BUFSIZ) < 0) { ERROR("read error message from child failed"); } - close(pipe_for_read[0]); - pipe_for_read[0] = -1; } - close(pipe_for_read[1]); - pipe_for_read[1] = -1; - cleanup: + close_archive_pipes_fd(pipe_for_read, 2); if (errmsg != NULL && strlen(errbuf) != 0) { *errmsg = util_strdup_s(errbuf); } return ret; } + +static ssize_t pipe_read(void *context, void *buf, size_t len) +{ + return util_read_nointr(*(int*)context, buf, len); +} + +static ssize_t archive_context_write(const void *context, const void *buf, size_t len) +{ + struct archive_context *ctx = (struct archive_context *)context; + if (ctx == NULL) { + return -1; + } + if (ctx->stdin_fd >= 0) { + return util_write_nointr(ctx->stdin_fd, buf, len); + } + return 0; +} + +static ssize_t pipe_write(void *context, const void *data, size_t len) +{ + return util_write_nointr(*(int*)context, data, len); +} + +static ssize_t archive_context_read(void *context, void *buf, size_t len) +{ + struct archive_context *ctx = (struct archive_context *)context; + if (ctx == NULL) { + return -1; + } + if (ctx->stdout_fd >= 0) { + return util_read_nointr(ctx->stdout_fd, buf, len); + } + return 0; +} + +static int close_wait_pid(struct archive_context *ctx, int *status) +{ + int ret = 0; + + // close stdin and stdout first, this will make sure the process of tar exit. + if (ctx->stdin_fd >= 0) { + close(ctx->stdin_fd); + } + + if (ctx->stdout_fd >= 0) { + close(ctx->stdout_fd); + } + + if (ctx->pid > 0) { + if (waitpid(ctx->pid, status, 0) != ctx->pid) { + ERROR("Failed to wait pid %u", ctx->pid); + ret = -1; + } + } + + return ret; +} + +static int archive_context_close(void *context, char **err) +{ + int ret = 0; + int status = 0; + char *reason = NULL; + ssize_t size_read = 0; + char buffer[BUFSIZ + 1] = { 0 }; + struct archive_context *ctx = (struct archive_context *)context; + char *marshaled = NULL; + + if (ctx == NULL) { + return 0; + } + + ret = close_wait_pid(ctx, &status); + + if (WIFSIGNALED((unsigned int)status)) { + status = WTERMSIG(status); + reason = "signaled"; + } else if (WIFEXITED(status)) { + status = WEXITSTATUS(status); + reason = "exited"; + } else { + reason = "unknown"; + } + if (ctx->stderr_fd >= 0) { + size_read = util_read_nointr(ctx->stderr_fd, buffer, BUFSIZ); + if (size_read > 0) { + reason = buffer; + marshaled = util_marshal_string(buffer); + if (marshaled == NULL) { + ERROR("Can not marshal json buffer: %s", buffer); + } else { + reason = marshaled; + } + } + close(ctx->stderr_fd); + } + + if (size_read > 0 || status != 0) { + format_errorf(err, "tar exited with status %d: %s", status, reason); + ret = -1; + } + + free(marshaled); + free(ctx); + return ret; +} + +int archive_chroot_untar_stream(const struct io_read_wrapper *context, const char *chroot_dir, + const char *untar_dir, const char *src_base, const char *dst_base, + char **errmsg) +{ + struct io_read_wrapper pipe_context = { 0 }; + int pipe_stream[2] = { -1, -1 }; + int pipe_stderr[2] = { -1, -1 }; + int keepfds[] = { -1, -1, -1 }; + int ret = -1; + int cret = 0; + pid_t pid; + struct archive_context *ctx = NULL; + char *buf = NULL; + size_t buf_len = ARCHIVE_BLOCK_SIZE; + ssize_t read_len; + struct archive_options options = { + .whiteout_format = NONE_WHITEOUT_FORMATE, + .src_base = src_base, + .dst_base = dst_base + }; + + buf = util_common_calloc_s(buf_len); + if (buf == NULL) { + ERROR("Out of memory"); + return -1; + } + + if (pipe(pipe_stderr) != 0) { + ERROR("Failed to create pipe: %s", strerror(errno)); + goto cleanup; + } + if (pipe(pipe_stream) != 0) { + ERROR("Failed to create pipe: %s", strerror(errno)); + goto cleanup; + } + + pid = fork(); + if (pid == (pid_t) -1) { + ERROR("Failed to fork: %s", strerror(errno)); + goto cleanup; + } + + if (pid == (pid_t)0) { + keepfds[0] = isula_libutils_get_log_fd(); + keepfds[1] = pipe_stderr[1]; + keepfds[2] = pipe_stream[0]; + ret = util_check_inherited_exclude_fds(true, keepfds, 3); + if (ret != 0) { + ERROR("Failed to close fds."); + ret = -1; + goto child_out; + } + + // child process, dup2 pipe_stderr[1] to stderr, + if (dup2(pipe_stderr[1], 2) < 0) { + ERROR("Dup fd error: %s", strerror(errno)); + ret = -1; + goto child_out; + } + + if (chroot(chroot_dir) != 0) { + SYSERROR("Failed to chroot to %s", chroot_dir); + ret = -1; + goto child_out; + } + + if (chdir("/") != 0 || chdir(untar_dir) != 0) { + SYSERROR("Failed to chdir to %s", untar_dir); + fprintf(stderr, "Failed to chdir to %s", untar_dir); + ret = -1; + goto child_out; + } + + pipe_context.context = (void*)&pipe_stream[0]; + pipe_context.read = pipe_read; + ret = archive_unpack_handler(&pipe_context, &options); + +child_out: + if (ret != 0) { + exit(EXIT_FAILURE); + } else { + exit(EXIT_SUCCESS); + } + } + + close(pipe_stderr[1]); + pipe_stderr[1] = -1; + close(pipe_stream[0]); + pipe_stream[0] = -1; + + ctx = util_common_calloc_s(sizeof(struct archive_context)); + if (ctx == NULL) { + goto cleanup; + } + + ctx->pid = pid; + ctx->stdin_fd = pipe_stream[1]; + pipe_stream[1] = -1; + ctx->stdout_fd = -1; + ctx->stderr_fd = pipe_stderr[0]; + pipe_stderr[0] = -1; + + read_len = context->read(context->context, buf, buf_len); + while (read_len > 0) { + ssize_t writed_len = archive_context_write(ctx, buf, (size_t)read_len); + if (writed_len < 0) { + DEBUG("Tar may exited: %s", strerror(errno)); + break; + } + read_len = context->read(context->context, buf, buf_len); + } + + ret = 0; + +cleanup: + free(buf); + cret = archive_context_close(ctx, errmsg); + ret = (cret != 0) ? cret : ret; + close_archive_pipes_fd(pipe_stderr, 2); + close_archive_pipes_fd(pipe_stream, 2); + + return ret; +} + +int archive_chroot_tar_stream(const char *chroot_dir, const char *tar_path, const char *src_base, + const char *dst_base, struct io_read_wrapper *reader) +{ + struct io_write_wrapper pipe_context = { 0 }; + int keepfds[] = { -1, -1, -1 }; + int pipe_stderr[2] = { -1, -1 }; + int pipe_stream[2] = { -1, -1 }; + int ret = -1; + pid_t pid; + struct archive_context *ctx = NULL; + + if (pipe(pipe_stderr) != 0) { + ERROR("Failed to create pipe: %s", strerror(errno)); + goto free_out; + } + if (pipe(pipe_stream) != 0) { + ERROR("Failed to create pipe: %s", strerror(errno)); + goto free_out; + } + + pid = fork(); + if (pid == (pid_t) - 1) { + ERROR("Failed to fork: %s", strerror(errno)); + goto free_out; + } + + if (pid == (pid_t)0) { + char *tar_dir_name = NULL; + char *tar_base_name = NULL; + + keepfds[0] = isula_libutils_get_log_fd(); + keepfds[1] = pipe_stderr[1]; + keepfds[2] = pipe_stream[1]; + ret = util_check_inherited_exclude_fds(true, keepfds, 3); + if (ret != 0) { + ERROR("Failed to close fds."); + ret = -1; + goto child_out; + } + + // child process, dup2 pipe_stderr[1] to stderr, + if (dup2(pipe_stderr[1], 2) < 0) { + ERROR("Dup fd error: %s", strerror(errno)); + ret = -1; + goto child_out; + } + + if (chroot(chroot_dir) != 0) { + ERROR("Failed to chroot to %s", chroot_dir); + fprintf(stderr, "Failed to chroot to %s\n", chroot_dir); + ret = -1; + goto child_out; + } + + if (util_split_dir_and_base_name(tar_path, &tar_dir_name, &tar_base_name) != 0) { + ERROR("Failed to split %s", tar_path); + fprintf(stderr, "Failed to split %s\n", tar_path); + ret = -1; + goto child_out; + } + + if (chdir("/") != 0 || chdir(tar_dir_name) != 0) { + ERROR("Failed to chdir to %s", tar_dir_name); + fprintf(stderr, "Failed to chdir to %s\n", tar_dir_name); + ret = -1; + goto child_out; + } + + pipe_context.context = (void*)&pipe_stream[1]; + pipe_context.write_func = pipe_write; + ret = tar_all(&pipe_context, tar_base_name, src_base, dst_base); + +child_out: + free(tar_dir_name); + free(tar_base_name); + + if (ret != 0) { + exit(EXIT_FAILURE); + } else { + exit(EXIT_SUCCESS); + } + } + + close(pipe_stderr[1]); + pipe_stderr[1] = -1; + close(pipe_stream[1]); + pipe_stream[1] = -1; + + ctx = util_common_calloc_s(sizeof(struct archive_context)); + if (ctx == NULL) { + goto free_out; + } + + ctx->stdin_fd = -1; + ctx->stdout_fd = pipe_stream[0]; + pipe_stream[0] = -1; + ctx->stderr_fd = pipe_stderr[0]; + pipe_stderr[0] = -1; + ctx->pid = pid; + + reader->close = archive_context_close; + reader->context = ctx; + ctx = NULL; + reader->read = archive_context_read; + + ret = 0; +free_out: + close_archive_pipes_fd(pipe_stderr, 2); + close_archive_pipes_fd(pipe_stream, 2); + free(ctx); + + return ret; +} diff --git a/src/utils/tar/util_archive.h b/src/utils/tar/util_archive.h index 0e05a363..55fd7683 100644 --- a/src/utils/tar/util_archive.h +++ b/src/utils/tar/util_archive.h @@ -24,6 +24,8 @@ #include "io_wrapper.h" +#define ARCHIVE_BLOCK_SIZE (32 * 1024) + struct io_read_wrapper; #ifdef __cplusplus @@ -38,14 +40,25 @@ typedef enum { struct archive_options { whiteout_format_type whiteout_format; + + // rename archive entry's name from src_base to dst_base + const char *src_base; + const char *dst_base; }; -int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, const struct archive_options *options); +int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, const struct archive_options *options, + char **errmsg); bool valid_archive_format(const char *file); int archive_chroot_tar(char *path, char *file, char **errmsg); +int archive_chroot_tar_stream(const char *chroot_dir, const char *tar_path, const char *src_base, + const char *dst_base, struct io_read_wrapper *content); +int archive_chroot_untar_stream(const struct io_read_wrapper *content, const char *chroot_dir, + const char *untar_dir, const char *src_base, const char *dst_base, + char **errmsg); + #ifdef __cplusplus } #endif -- 2.25.1