packages/iSulad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
iSulad/0024-2170-isula-image-pull.patch

1816 lines
65 KiB
Diff
Raw Normal View History

!638 upgrade from upstream * upgrade from upstream
2023-12-21 02:03:18 +00:00
From 53d551f613bfa8ce0552ca62f964a0584e3665bb Mon Sep 17 00:00:00 2001
From: sailorvii <chenw66@chinaunicom.cn>
Date: Wed, 22 Nov 2023 01:22:04 +0000
Subject: [PATCH 24/64] =?UTF-8?q?!2170=20=E5=A2=9E=E5=8A=A0isula=20image?=
=?UTF-8?q?=20pull=E8=BF=9B=E5=BA=A6=E6=98=BE=E7=A4=BA=20*=20Refine=20some?=
=?UTF-8?q?=20issues.=20*=20Address=20comment=20*=20Address=20comments=20*?=
=?UTF-8?q?=201.=20Address=20comments.=20*=20Address=20comments=20*=20Addr?=
=?UTF-8?q?ess=20comments=20*=20Address=20comments=20*=20Address=20comment?=
=?UTF-8?q?s=20*=20Address=20comments=20*=20Address=20comments=20*=20Addre?=
=?UTF-8?q?ss=20comments=20*=20Address=20comments=20*=20Address=20comments?=
=?UTF-8?q?=20*=20Address=20test=20issue=20*=20Address=20test=20compile=20?=
=?UTF-8?q?issue=20*=20Address=20compile=20issue=20*=20Fix=20compile=20iss?=
=?UTF-8?q?ue=20*=20Address=20comments=20*=20Address=20comments=20*=20Addr?=
=?UTF-8?q?ess=20comments.=20*=20Address=20issuse=20*=20Address=20many=20i?=
=?UTF-8?q?ssues.=20*=20Fix=20some=20minor=20issuses.=20*=20Address=20comm?=
=?UTF-8?q?ents.=20*=20Refine=20as=20Haozi's=20comments=20*=20Fix=20some?=
=?UTF-8?q?=20issues=20by=20Haozi's=20comments.=20*=20Refine=20formats.=20?=
=?UTF-8?q?*=20Add=20process=20bar=20show=20for=20pull=20functions.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
CI/dockerfiles/Dockerfile-fedora | 2 +-
CI/pr-gateway.sh | 2 +-
cmake/checker.cmake | 10 ++
src/CMakeLists.txt | 4 +
src/api/services/images/images.proto | 35 +++-
src/client/connect/grpc/grpc_images_client.cc | 167 ++++++++++++++++--
src/client/connect/protocol_type.h | 1 +
src/cmd/isula/images/pull.c | 26 ++-
.../connect/grpc/grpc_containers_service.h | 4 +
.../entry/connect/grpc/grpc_images_service.cc | 105 ++++++++++-
.../entry/connect/grpc/grpc_images_service.h | 7 +
.../entry/connect/rest/rest_images_service.c | 2 +-
.../v1/v1_cri_image_manager_service_impl.cc | 2 +-
.../v1alpha/cri_image_manager_service_impl.cc | 2 +-
src/daemon/executor/callback.h | 3 +-
src/daemon/executor/image_cb/image_cb.c | 8 +-
src/daemon/modules/api/image_api.h | 5 +-
src/daemon/modules/image/image.c | 9 +-
src/daemon/modules/image/oci/oci_image.c | 4 +-
src/daemon/modules/image/oci/oci_image.h | 2 +-
src/daemon/modules/image/oci/oci_pull.c | 158 +++++++++++++++--
src/daemon/modules/image/oci/oci_pull.h | 2 +-
src/daemon/modules/image/oci/progress.c | 124 +++++++++++++
src/daemon/modules/image/oci/progress.h | 52 ++++++
.../modules/image/oci/registry/http_request.c | 104 ++++++++---
.../modules/image/oci/registry/http_request.h | 2 +-
.../modules/image/oci/registry/registry.c | 3 +-
.../modules/image/oci/registry/registry.h | 2 +
.../image/oci/registry/registry_apiv2.c | 12 +-
src/daemon/modules/image/oci/registry_type.h | 3 +
src/utils/CMakeLists.txt | 3 +
src/utils/http/http.h | 17 +-
src/utils/progress/CMakeLists.txt | 13 ++
src/utils/progress/show.c | 64 +++++++
src/utils/progress/show.h | 34 ++++
test/cutils/CMakeLists.txt | 1 +
test/image/oci/registry/CMakeLists.txt | 1 +
test/image/oci/registry/registry_ut.cc | 16 +-
38 files changed, 912 insertions(+), 99 deletions(-)
create mode 100644 src/daemon/modules/image/oci/progress.c
create mode 100644 src/daemon/modules/image/oci/progress.h
create mode 100644 src/utils/progress/CMakeLists.txt
create mode 100644 src/utils/progress/show.c
create mode 100644 src/utils/progress/show.h
diff --git a/CI/dockerfiles/Dockerfile-fedora b/CI/dockerfiles/Dockerfile-fedora
index bef44377..a105cbb4 100644
--- a/CI/dockerfiles/Dockerfile-fedora
+++ b/CI/dockerfiles/Dockerfile-fedora
@@ -115,7 +115,7 @@ RUN echo "[source.crates-io]" >> ${HOME}/.cargo/config && \
echo "[source.local-registry]" >> ${HOME}/.cargo/config && \
echo "directory = \"vendor\"" >> ${HOME}/.cargo/config
-RUN dnf install -y lcov && dnf clean all
+RUN dnf install -y lcov ncurses-devel && dnf clean all
# install libevhtp
RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \
diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh
index 08bcfc4f..e5bf627e 100755
--- a/CI/pr-gateway.sh
+++ b/CI/pr-gateway.sh
@@ -22,7 +22,7 @@ sed -i "s#http://repo.openeuler.org#https://repo.huaweicloud.com/openeuler#g" /e
dnf update -y
-dnf install -y docbook2X doxygen gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath
+dnf install -y docbook2X doxygen gtest-devel gmock-devel diffutils cmake gcc-c++ yajl-devel patch make libtool libevent-devel libevhtp-devel grpc grpc-plugins grpc-devel protobuf-devel libcurl libcurl-devel sqlite-devel libarchive-devel device-mapper-devel http-parser-devel libseccomp-devel libcap-devel libselinux-devel libwebsockets libwebsockets-devel systemd-devel git chrpath ncurses-devel
if [ $? -ne 0 ]; then
echo "install dependences failed"
exit 1
diff --git a/cmake/checker.cmake b/cmake/checker.cmake
index 358ab4af..cc4a1fc3 100644
--- a/cmake/checker.cmake
+++ b/cmake/checker.cmake
@@ -154,6 +154,16 @@ if (GRPC_CONNECTOR)
_CHECK(WEBSOCKET_INCLUDE_DIR "WEBSOCKET_INCLUDE_DIR-NOTFOUND" libwebsockets.h)
find_library(WEBSOCKET_LIBRARY websockets)
_CHECK(WEBSOCKET_LIBRARY "WEBSOCKET_LIBRARY-NOTFOUND" "libwebsockets.so")
+
+ # check libncurses
+ pkg_check_modules(PC_LIBNCURSES REQUIRED "ncurses")
+ find_path(NCURSES_INCLUDE_DIR curses.h
+ HINTS ${PC_NCURSES_INCLUDEDIR} ${PC_NCURSES_INCLUDE_DIRS})
+ _CHECK(NCURSES_INCLUDE_DIR "NCURSES_INCLUDE_DIR-NOTFOUND" "curses.h")
+
+ find_library(NCURSES_LIBRARY ncurses
+ HINTS ${PC_NCURSES_LIBDIR} ${PC_NCURSES_LIBRARY_DIRS})
+ _CHECK(NCURSES_LIBRARY "NCURSES_LIBRARY-NOTFOUND" "libncurses.so")
endif()
if ((NOT GRPC_CONNECTOR) OR (GRPC_CONNECTOR AND ENABLE_METRICS))
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 8e197b9f..860447de 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -102,6 +102,10 @@ add_executable(isula
)
target_include_directories(isula PUBLIC ${ISULA_INCS} ${SHARED_INCS})
target_link_libraries(isula libisula_client ${LIBYAJL_LIBRARY})
+if (GRPC_CONNECTOR)
+ target_link_libraries(isula ${NCURSES_LIBRARY})
+endif()
+
if (ANDROID OR MUSL)
target_link_libraries(isula ${LIBSSL_LIBRARY})
else()
diff --git a/src/api/services/images/images.proto b/src/api/services/images/images.proto
index 2a34f02d..9f2cb803 100644
--- a/src/api/services/images/images.proto
+++ b/src/api/services/images/images.proto
@@ -30,6 +30,23 @@ service ImagesService {
rpc Tag(TagImageRequest) returns (TagImageResponse);
rpc Import(ImportRequest) returns (ImportResponse);
rpc Search(SearchRequest) returns (SearchResponse);
+ rpc PullImage(PullImageRequest) returns (stream PullImageResponse);
+}
+
+// ImageSpec is an internal representation of an image.
+message ImageSpec {
+ // Container's Image field (e.g. imageID or imageDigest).
+ string image = 1;
+ // Unstructured key-value map holding arbitrary metadata.
+ // ImageSpec Annotations can be used to help the runtime target specific
+ // images in multi-arch images.
+ map<string, string> annotations = 2;
+}
+
+// AuthConfig contains authorization information for connecting to a registry.
+message AuthConfig {
+ string username = 1;
+ string password = 2;
}
message Descriptor {
@@ -152,4 +169,20 @@ message SearchResponse {
repeated SearchImage search_result = 2;
uint32 cc = 3;
string errmsg = 4;
-}
\ No newline at end of file
+}
+
+message PullImageRequest {
+ // Spec of the image.
+ ImageSpec image = 1;
+ // Authentication configuration for pulling the image.
+ AuthConfig auth = 2;
+ bool is_progress_visible = 3;
+}
+
+message PullImageResponse {
+ // Reference to the image in use. For most runtimes, this should be an
+ // image ID or digest.
+ string image_ref = 1;
+ string stream = 2;
+ bytes progress_data = 3;
+}
diff --git a/src/client/connect/grpc/grpc_images_client.cc b/src/client/connect/grpc/grpc_images_client.cc
index 9cc2a174..7a283e8c 100644
--- a/src/client/connect/grpc/grpc_images_client.cc
+++ b/src/client/connect/grpc/grpc_images_client.cc
@@ -13,12 +13,15 @@
* Description: provide grpc container service functions
******************************************************************************/
#include "grpc_images_client.h"
-#include "api.grpc.pb.h"
#include "client_base.h"
#include "images.grpc.pb.h"
+
+#include <isula_libutils/auto_cleanup.h>
+#include <isula_libutils/image_progress.h>
+#include <string>
+#include "show.h"
#include "utils.h"
#include "constants.h"
-#include <string>
using namespace images;
@@ -337,9 +340,9 @@ public:
}
};
-class ImagesPull : public ClientBase<runtime::v1alpha2::ImageService, runtime::v1alpha2::ImageService::Stub,
- isula_pull_request, runtime::v1alpha2::PullImageRequest, isula_pull_response,
- runtime::v1alpha2::PullImageResponse> {
+class ImagesPull : public ClientBase<ImagesService, ImagesService::Stub,
+ isula_pull_request, PullImageRequest,
+ isula_pull_response, PullImageResponse> {
public:
explicit ImagesPull(void *args)
: ClientBase(args)
@@ -347,15 +350,14 @@ public:
}
~ImagesPull() = default;
- auto request_to_grpc(const isula_pull_request *request, runtime::v1alpha2::PullImageRequest *grequest)
+ auto request_to_grpc(const isula_pull_request *request, PullImageRequest *grequest)
-> int override
{
if (request == nullptr) {
return -1;
}
-
if (request->image_name != nullptr) {
- auto *image_spec = new (std::nothrow) runtime::v1alpha2::ImageSpec;
+ auto *image_spec = new (std::nothrow) ImageSpec;
if (image_spec == nullptr) {
return -1;
}
@@ -363,10 +365,12 @@ public:
grequest->set_allocated_image(image_spec);
}
+ grequest->set_is_progress_visible(request->is_progress_visible);
+
return 0;
}
- auto response_from_grpc(runtime::v1alpha2::PullImageResponse *gresponse, isula_pull_response *response)
+ auto response_from_grpc(PullImageResponse *gresponse, isula_pull_response *response)
-> int override
{
if (!gresponse->image_ref().empty()) {
@@ -376,7 +380,7 @@ public:
return 0;
}
- auto check_parameter(const runtime::v1alpha2::PullImageRequest &req) -> int override
+ auto check_parameter(const PullImageRequest &req) -> int override
{
if (req.image().image().empty()) {
ERROR("Missing image name in the request");
@@ -386,10 +390,147 @@ public:
return 0;
}
- auto grpc_call(ClientContext *context, const runtime::v1alpha2::PullImageRequest &req,
- runtime::v1alpha2::PullImageResponse *reply) -> Status override
+ auto run(const struct isula_pull_request *request, struct isula_pull_response *response) -> int override
+ {
+ ClientContext context;
+ PullImageRequest grequest;
+
+#ifdef ENABLE_GRPC_REMOTE_CONNECT
+#ifdef OPENSSL_VERIFY
+ // Set common name from cert.perm
+ char common_name_value[ClientBaseConstants::COMMON_NAME_LEN] = { 0 };
+ int ret = get_common_name_from_tls_cert(m_certFile.c_str(), common_name_value,
+ ClientBaseConstants::COMMON_NAME_LEN);
+ if (ret != 0) {
+ ERROR("Failed to get common name in: %s", m_certFile.c_str());
+ return -1;
+ }
+ context.AddMetadata("username", std::string(common_name_value, strlen(common_name_value)));
+ context.AddMetadata("tls_mode", m_tlsMode);
+#endif
+#endif
+ if (request_to_grpc(request, &grequest) != 0) {
+ ERROR("Failed to transform pull request to grpc");
+ response->server_errono = ISULAD_ERR_INPUT;
+ return -1;
+ }
+
+ auto reader = stub_->PullImage(&context, grequest);
+
+ PullImageResponse gresponse;
+ if (grequest.is_progress_visible()) {
+ while (reader->Read(&gresponse)) {
+ output_progress(gresponse);
+ }
+ } else {
+ reader->Read(&gresponse);
+ WARN("The terminal may not support ANSI Escape code. Display is skipped");
+ }
+ Status status = reader->Finish();
+ if (!status.ok()) {
+ ERROR("Error code: %d: %s", status.error_code(), status.error_message().c_str());
+ unpackStatus(status, response);
+ return -1;
+ }
+ response->image_ref = util_strdup_s(gresponse.image_ref().c_str());
+ return 0;
+ }
+
+private:
+ void output_progress(PullImageResponse &gresponse)
{
- return stub_->PullImage(context, req, reply);
+ __isula_auto_free char *err = nullptr;
+ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 };
+
+ image_progress *progresses = image_progress_parse_data(gresponse.progress_data().c_str(), &ctx, &err);
+ if (progresses == nullptr) {
+ ERROR("Parse image progress error %s", err);
+ return;
+ }
+ show_processes(progresses);
+ }
+
+ void get_printed_value(int64_t value, char *printed)
+ {
+ float float_value = 0.0;
+ const float GB = 1024 * 1024 * 1024;
+ const float MB = 1024 * 1024;
+ const float KB = 1024;
+
+ if ((float)value / GB > 1) {
+ float_value = (float)value / GB;
+ sprintf(printed, "%.2fGB", float_value);
+ } else if ((float)value / MB > 1) {
+ float_value = (float)value / MB;
+ sprintf(printed, "%.2fMB", float_value);
+ } else if ((float)value / KB > 1) {
+ float_value = (float)value / KB;
+ sprintf(printed, "%.2fKB", float_value);
+ } else {
+ sprintf(printed, "%ldB", value);
+ }
+ }
+
+ void display_progress_bar(image_progress_progresses_element *progress_item, int width, bool if_show_all)
+ {
+ float progress = 0.0;
+ int filled_width = 0;
+ const int FLOAT_STRING_SIZE = 64;
+ char total[FLOAT_STRING_SIZE] = {0};
+ char current[FLOAT_STRING_SIZE] = {0};
+ int empty_width = 0;
+
+ if (progress_item->total != 0) {
+ progress = (float)progress_item->current / (float)progress_item->total;
+ }
+ filled_width = (int)(progress * width);
+ empty_width = width - filled_width;
+ get_printed_value(progress_item->total, total);
+ get_printed_value(progress_item->current, current);
+
+ if (if_show_all) {
+ int i = 0;
+
+ printf("%s: [", progress_item->id);
+
+ // Print filled characters
+ for (i = 0; i < filled_width; i++) {
+ printf("=");
+ }
+ printf(">");
+ // Print empty characters
+ for (i = 0; i < empty_width; i++) {
+ printf(" ");
+ }
+
+ printf("] %s/%s", current, total);
+ } else {
+ printf("%s: %s/%s", progress_item->id, current, total);
+ }
+ printf("\n");
+ fflush(stdout);
+ }
+
+ void show_processes(image_progress *progresses)
+ {
+ size_t i = 0;
+ static size_t len = 0;
+ const int TERMINAL_SHOW_WIDTH = 110;
+ const int width = 50; // Width of the progress bars
+
+ if (len != 0) {
+ move_cursor_up(len);
+ }
+ clear_lines_below();
+ len = progresses->progresses_len;
+ int terminal_width = get_terminal_width();
+ bool if_show_all = true;
+ if (terminal_width < TERMINAL_SHOW_WIDTH) {
+ if_show_all = false;
+ }
+ for (i = 0; i < len; i++) {
+ display_progress_bar(progresses->progresses[i], width, if_show_all);
+ }
}
};
diff --git a/src/client/connect/protocol_type.h b/src/client/connect/protocol_type.h
index 4206c50b..2b445c5a 100644
--- a/src/client/connect/protocol_type.h
+++ b/src/client/connect/protocol_type.h
@@ -479,6 +479,7 @@ struct isula_rmi_response {
struct isula_pull_request {
char *image_name;
+ bool is_progress_visible;
};
struct isula_tag_request {
diff --git a/src/cmd/isula/images/pull.c b/src/cmd/isula/images/pull.c
index 548e8d90..9d420778 100644
--- a/src/cmd/isula/images/pull.c
+++ b/src/cmd/isula/images/pull.c
@@ -14,6 +14,10 @@
********************************************************************************/
#include "pull.h"
+#ifdef GRPC_CONNECTOR
+#include <curses.h>
+#include <term.h>
+#endif
#include <stdio.h>
#include <stdlib.h>
@@ -29,6 +33,25 @@ const char g_cmd_pull_usage[] = "pull [OPTIONS] NAME[:TAG]";
struct client_arguments g_cmd_pull_args = {};
+static bool is_terminal_show_supported()
+{
+#ifdef GRPC_CONNECTOR
+ // Initialize the terminfo database
+ setupterm(NULL, STDOUT_FILENO, (int *)0);
+
+ // Query the database for the capability to move the cursor
+ char *cursor_movement = tgetstr("cm", NULL);
+
+ if (cursor_movement != NULL) {
+ return true;
+ } else {
+ return false;
+ }
+#else
+ return false;
+#endif
+}
+
/*
* Pull an image or a repository from a registry
*/
@@ -47,6 +70,7 @@ int client_pull(const struct client_arguments *args)
}
request.image_name = args->image_name;
+ request.is_progress_visible = is_terminal_show_supported();
ops = get_connect_client_ops();
if (ops == NULL || ops->image.pull == NULL) {
@@ -63,8 +87,8 @@ int client_pull(const struct client_arguments *args)
ret = ESERVERERROR;
goto out;
}
- COMMAND_ERROR("Image \"%s\" pulled", response->image_ref);
+ COMMAND_ERROR("Image \"%s\" pulled", response->image_ref);
out:
isula_pull_response_free(response);
return ret;
diff --git a/src/daemon/entry/connect/grpc/grpc_containers_service.h b/src/daemon/entry/connect/grpc/grpc_containers_service.h
index 92428fbe..4a6c584b 100644
--- a/src/daemon/entry/connect/grpc/grpc_containers_service.h
+++ b/src/daemon/entry/connect/grpc/grpc_containers_service.h
@@ -37,6 +37,10 @@ using google::protobuf::Timestamp;
void protobuf_timestamp_to_grpc(types_timestamp_t *timestamp, Timestamp *gtimestamp);
void protobuf_timestamp_from_grpc(types_timestamp_t *timestamp, const Timestamp &gtimestamp);
+bool grpc_is_call_cancelled(void *context);
+bool grpc_add_initial_metadata(void *context, const char *header, const char *val);
+bool grpc_event_write_function(void *writer, void *data);
+
// Implement of containers service
class ContainerServiceImpl final : public ContainerService::Service {
public:
diff --git a/src/daemon/entry/connect/grpc/grpc_images_service.cc b/src/daemon/entry/connect/grpc/grpc_images_service.cc
index 5d3fac6b..406f81a9 100644
--- a/src/daemon/entry/connect/grpc/grpc_images_service.cc
+++ b/src/daemon/entry/connect/grpc/grpc_images_service.cc
@@ -21,9 +21,12 @@
#include <new>
#include <string>
-#include "isula_libutils/log.h"
+#include <isula_libutils/auto_cleanup.h>
+#include <isula_libutils/image_progress.h>
+#include <isula_libutils/log.h>
#include "utils.h"
#include "grpc_server_tls_auth.h"
+#include "grpc_containers_service.h"
int ImagesServiceImpl::image_list_request_from_grpc(const ListImagesRequest *grequest,
image_list_images_request **request)
@@ -596,6 +599,104 @@ Status ImagesServiceImpl::Logout(ServerContext *context, const LogoutRequest *re
return Status::OK;
}
+int ImagesServiceImpl::image_pull_request_from_grpc(const PullImageRequest *grequest,
+ image_pull_image_request **request)
+{
+ auto *tmpreq = (image_pull_image_request *)util_common_calloc_s(sizeof(image_pull_image_request));
+ if (tmpreq == nullptr) {
+ ERROR("Out of memory");
+ return -1;
+ }
+
+ if (!grequest->image().image().empty()) {
+ tmpreq->image_name = util_strdup_s(grequest->image().image().c_str());
+ }
+ tmpreq->is_progress_visible = grequest->is_progress_visible();
+ *request = tmpreq;
+
+ return 0;
+}
+
+void image_pull_progress_to_grpc(const image_progress *progress,
+ PullImageResponse &gresponse)
+{
+ if (progress == nullptr) {
+ ERROR("Invalid parameter");
+ return;
+ }
+
+ gresponse.Clear();
+ __isula_auto_free char *err = nullptr;
+ struct parser_context ctx = { OPT_GEN_SIMPLIFY, 0 };
+ char *data = image_progress_generate_json(progress, &ctx, &err);
+ if (data == nullptr) {
+ ERROR("Failed to generate image progress json: %s", err);
+ return;
+ }
+
+ gresponse.set_progress_data(data, strlen(data));
+ if (progress->image != nullptr) {
+ gresponse.set_image_ref(progress->image);
+ }
+ free(data);
+}
+
+bool grpc_pull_write_function(void *writer, void *data)
+{
+ auto *progress = static_cast<image_progress *>(data);
+ auto *gwriter = static_cast<ServerWriter<PullImageResponse> *>(writer);
+ PullImageResponse gresponse;
+
+ image_pull_progress_to_grpc(progress, gresponse);
+
+ return gwriter->Write(gresponse);
+}
+
+Status ImagesServiceImpl::PullImage(ServerContext *context, const PullImageRequest *request,
+ ServerWriter<PullImageResponse> *writer)
+{
+ prctl(PR_SET_NAME, "RegistryPull");
+
+ int ret = 0;
+ std::string errmsg = "Failed to execute image pull";
+ stream_func_wrapper stream = { 0 };
+ image_pull_image_request *image_req = nullptr;
+ image_pull_image_response *image_res = nullptr;
+
+ if (context == nullptr || request == nullptr || writer == nullptr) {
+ return Status(StatusCode::INVALID_ARGUMENT, "Invalid argument");
+ }
+
+ auto status = GrpcServerTlsAuth::auth(context, "pull");
+ if (!status.ok()) {
+ return status;
+ }
+
+ service_executor_t *cb = get_service_executor();
+ if (cb == nullptr || cb->image.pull == nullptr) {
+ return Status(StatusCode::UNIMPLEMENTED, "Unimplemented callback");
+ }
+
+ ret = image_pull_request_from_grpc(request, &image_req);
+ if (ret != 0) {
+ ERROR("Failed to transform grpc request");
+ return Status(StatusCode::UNKNOWN, "Failed to transform grpc request");
+ }
+
+ stream.context = (void *)context;
+ stream.is_cancelled = &grpc_is_call_cancelled;
+ stream.write_func = &grpc_pull_write_function;
+ stream.writer = (void *)writer;
+
+ ret = cb->image.pull(image_req, &stream, &image_res);
+ free_image_pull_image_request(image_req);
+ free_image_pull_image_response(image_res);
+ if (ret == 0) {
+ return Status::OK;
+ }
+ return Status(StatusCode::UNKNOWN, errmsg);
+}
+
#ifdef ENABLE_IMAGE_SEARCH
int ImagesServiceImpl::search_request_from_grpc(const SearchRequest *grequest, image_search_images_request **request)
{
@@ -723,4 +824,4 @@ Status ImagesServiceImpl::Search(ServerContext *context, const SearchRequest *re
return Status::OK;
}
-#endif
\ No newline at end of file
+#endif
diff --git a/src/daemon/entry/connect/grpc/grpc_images_service.h b/src/daemon/entry/connect/grpc/grpc_images_service.h
index b75075ba..9690f544 100644
--- a/src/daemon/entry/connect/grpc/grpc_images_service.h
+++ b/src/daemon/entry/connect/grpc/grpc_images_service.h
@@ -58,6 +58,9 @@ public:
Status Logout(ServerContext *context, const LogoutRequest *request, LogoutResponse *reply) override;
+ Status PullImage(ServerContext *context, const PullImageRequest *request,
+ ServerWriter<PullImageResponse> *writer) override;
+
#ifdef ENABLE_IMAGE_SEARCH
Status Search(ServerContext *context, const SearchRequest *request, SearchResponse *reply) override;
#endif
@@ -99,6 +102,10 @@ private:
int image_logout_request_from_grpc(const LogoutRequest *grequest, image_logout_request **request);
+ int image_pull_request_from_grpc(const PullImageRequest *grequest, image_pull_image_request **request);
+
+ void image_pull_response_to_grpc(const image_pull_image_response *response, PullImageResponse *gresponse);
+
#ifdef ENABLE_IMAGE_SEARCH
int search_request_from_grpc(const SearchRequest *grequest, image_search_images_request **request);
diff --git a/src/daemon/entry/connect/rest/rest_images_service.c b/src/daemon/entry/connect/rest/rest_images_service.c
index 5a719f83..220de399 100644
--- a/src/daemon/entry/connect/rest/rest_images_service.c
+++ b/src/daemon/entry/connect/rest/rest_images_service.c
@@ -513,7 +513,7 @@ static void rest_image_pull_cb(evhtp_request_t *req, void *arg)
goto out;
}
- (void)cb->image.pull(crequest, &cresponse);
+ (void)cb->image.pull(crequest, NULL, &cresponse);
evhtp_send_image_pull_repsponse(req, cresponse, RESTFUL_RES_OK);
diff --git a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc
index b74834fb..b9cbf24c 100644
--- a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc
+++ b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc
@@ -265,7 +265,7 @@ auto ImageManagerServiceImpl::PullImage(const runtime::v1::ImageSpec &image,
}
request->type = util_strdup_s(IMAGE_TYPE_OCI);
- ret = im_pull_image(request, &response);
+ ret = im_pull_image(request, nullptr, &response);
if (ret != 0) {
if (response != nullptr && response->errmsg != nullptr) {
error.SetError(response->errmsg);
diff --git a/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc
index 3ff79ffc..0b36f007 100644
--- a/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc
+++ b/src/daemon/entry/cri/v1alpha/cri_image_manager_service_impl.cc
@@ -265,7 +265,7 @@ auto ImageManagerServiceImpl::PullImage(const runtime::v1alpha2::ImageSpec &imag
}
request->type = util_strdup_s(IMAGE_TYPE_OCI);
- ret = im_pull_image(request, &response);
+ ret = im_pull_image(request, NULL, &response);
if (ret != 0) {
if (response != nullptr && response->errmsg != nullptr) {
error.SetError(response->errmsg);
diff --git a/src/daemon/executor/callback.h b/src/daemon/executor/callback.h
index c48253a1..b32c6b27 100644
--- a/src/daemon/executor/callback.h
+++ b/src/daemon/executor/callback.h
@@ -285,7 +285,8 @@ typedef struct {
int (*logout)(const image_logout_request *request, image_logout_response **response);
int (*tag)(const image_tag_image_request *request, image_tag_image_response **response);
- int (*pull)(const image_pull_image_request *request, image_pull_image_response **response);
+
+ int (*pull)(const image_pull_image_request *request, stream_func_wrapper *stream, image_pull_image_response **response);
#ifdef ENABLE_IMAGE_SEARCH
int (*search)(const image_search_images_request *request, image_search_images_response **response);
#endif
diff --git a/src/daemon/executor/image_cb/image_cb.c b/src/daemon/executor/image_cb/image_cb.c
index 61fa29db..317cb0a8 100644
--- a/src/daemon/executor/image_cb/image_cb.c
+++ b/src/daemon/executor/image_cb/image_cb.c
@@ -955,12 +955,14 @@ int pull_request_from_rest(const image_pull_image_request *request, im_pull_requ
}
(*im_req)->image = util_strdup_s(request->image_name);
+ (*im_req)->is_progress_visible = request->is_progress_visible;
return 0;
}
/* image pull cb */
-static int image_pull_cb(const image_pull_image_request *request, image_pull_image_response **response)
+static int image_pull_cb(const image_pull_image_request *request, stream_func_wrapper *stream,
+ image_pull_image_response **response)
{
int ret = -1;
im_pull_request *im_req = NULL;
@@ -988,7 +990,7 @@ static int image_pull_cb(const image_pull_image_request *request, image_pull_ima
// current only oci image support pull
im_req->type = util_strdup_s(IMAGE_TYPE_OCI);
- ret = im_pull_image(im_req, &im_rsp);
+ ret = im_pull_image(im_req, stream, &im_rsp);
if (ret != 0) {
cc = ISULAD_ERR_EXEC;
goto out;
@@ -1203,4 +1205,4 @@ void image_callback_init(service_image_callback_t *cb)
#ifdef ENABLE_IMAGE_SEARCH
cb->search = image_search_cb;
#endif
-}
\ No newline at end of file
+}
diff --git a/src/daemon/modules/api/image_api.h b/src/daemon/modules/api/image_api.h
index 2f2c00a2..bbe89ad7 100644
--- a/src/daemon/modules/api/image_api.h
+++ b/src/daemon/modules/api/image_api.h
@@ -32,6 +32,7 @@
#ifdef ENABLE_IMAGE_SEARCH
#include "isula_libutils/imagetool_search_result.h"
#endif
+#include "stream_wrapper.h"
#ifdef __cplusplus
extern "C" {
@@ -150,6 +151,8 @@ typedef struct {
char *server_address;
char *identity_token;
char *registry_token;
+
+ bool is_progress_visible;
} im_pull_request;
typedef struct {
@@ -304,7 +307,7 @@ void free_im_load_request(im_load_request *ptr);
void free_im_load_response(im_load_response *ptr);
-int im_pull_image(const im_pull_request *request, im_pull_response **response);
+int im_pull_image(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response **response);
void free_im_pull_request(im_pull_request *req);
diff --git a/src/daemon/modules/image/image.c b/src/daemon/modules/image/image.c
index a14f2ac3..8d7e2c1a 100644
--- a/src/daemon/modules/image/image.c
+++ b/src/daemon/modules/image/image.c
@@ -86,7 +86,7 @@ struct bim_ops {
int (*load_image)(const im_load_request *request);
/* pull image */
- int (*pull_image)(const im_pull_request *request, im_pull_response *response);
+ int (*pull_image)(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response);
/* login */
int (*login)(const im_login_request *request);
@@ -999,7 +999,7 @@ static bool check_im_pull_args(const im_pull_request *req, im_pull_response * co
return true;
}
-int im_pull_image(const im_pull_request *request, im_pull_response **response)
+int im_pull_image(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response **response)
{
int ret = -1;
struct bim *bim = NULL;
@@ -1029,7 +1029,7 @@ int im_pull_image(const im_pull_request *request, im_pull_response **response)
}
EVENT("Event: {Object: %s, Type: Pulling}", request->image);
- ret = bim->ops->pull_image(request, tmp_res);
+ ret = bim->ops->pull_image(request, stream, tmp_res);
if (ret != 0) {
ERROR("Pull image %s failed", request->image);
ret = -1;
@@ -1044,6 +1044,7 @@ out:
}
DAEMON_CLEAR_ERRMSG();
*response = tmp_res;
+
return ret;
}
@@ -2395,4 +2396,4 @@ out:
}
return ret;
}
-#endif
\ No newline at end of file
+#endif
diff --git a/src/daemon/modules/image/oci/oci_image.c b/src/daemon/modules/image/oci/oci_image.c
index f712a446..471510e7 100644
--- a/src/daemon/modules/image/oci/oci_image.c
+++ b/src/daemon/modules/image/oci/oci_image.c
@@ -359,7 +359,7 @@ void oci_exit(void)
free_oci_image_data();
}
-int oci_pull_rf(const im_pull_request *request, im_pull_response *response)
+int oci_pull_rf(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response)
{
int ret = 0;
if (request == NULL || request->image == NULL || response == NULL) {
@@ -381,7 +381,7 @@ int oci_pull_rf(const im_pull_request *request, im_pull_response *response)
}
#endif
- ret = oci_do_pull_image(request, response);
+ ret = oci_do_pull_image(request, stream, response);
#ifdef ENABLE_REMOTE_LAYER_STORE
if (g_enable_remote) {
diff --git a/src/daemon/modules/image/oci/oci_image.h b/src/daemon/modules/image/oci/oci_image.h
index 07f10c8d..c7304897 100644
--- a/src/daemon/modules/image/oci/oci_image.h
+++ b/src/daemon/modules/image/oci/oci_image.h
@@ -43,7 +43,7 @@ struct oci_image_module_data *get_oci_image_data(void);
int oci_init(const isulad_daemon_configs *args);
void oci_exit(void);
-int oci_pull_rf(const im_pull_request *request, im_pull_response *response);
+int oci_pull_rf(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response);
int oci_rmi(const im_rmi_request *request);
int oci_get_filesystem_info(im_fs_info_response **response);
int oci_load_image(const im_load_request *request);
diff --git a/src/daemon/modules/image/oci/oci_pull.c b/src/daemon/modules/image/oci/oci_pull.c
index e7ff77df..2706af91 100644
--- a/src/daemon/modules/image/oci/oci_pull.c
+++ b/src/daemon/modules/image/oci/oci_pull.c
@@ -14,20 +14,25 @@
*******************************************************************************/
#include "oci_pull.h"
+#include <isula_libutils/image_progress.h>
+#include <isula_libutils/log.h>
+#include <pthread.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
+#include <unistd.h>
-#include "isula_libutils/log.h"
-#include "utils.h"
-#include "utils_images.h"
-#include "registry.h"
#include "err_msg.h"
+#include "map.h"
+#include "oci_image.h"
+#include "progress.h"
+#include "registry.h"
#include "storage.h"
+#include "utils.h"
#include "utils_array.h"
#include "utils_base64.h"
+#include "utils_images.h"
#include "utils_string.h"
-#include "oci_image.h"
static int decode_auth(const char *auth, char **username, char **password)
{
@@ -85,7 +90,7 @@ static void update_option_insecure_registry(registry_pull_options *options, char
}
}
-static int pull_image(const im_pull_request *request, char **name)
+static int pull_image(const im_pull_request *request, progress_status_map *progress_status_store, char **name)
{
int ret = -1;
registry_pull_options *options = NULL;
@@ -112,6 +117,7 @@ static int pull_image(const im_pull_request *request, char **name)
options->auth.username = util_strdup_s(request->username);
options->auth.password = util_strdup_s(request->password);
}
+ options->progress_status_store = progress_status_store;
oci_image_data = get_oci_image_data();
options->skip_tls_verify = oci_image_data->insecure_skip_verify_enforce;
@@ -174,21 +180,131 @@ out:
return ret;
}
-int oci_do_pull_image(const im_pull_request *request, im_pull_response *response)
+typedef struct status_arg {
+ progress_status_map *status_store;
+ bool should_terminal;
+ imagetool_image_summary *image;
+ char *image_name;
+ stream_func_wrapper *stream;
+} status_arg;
+
+void *get_progress_status(void *arg)
+{
+ status_arg *status = (status_arg *)arg;
+ const int delay = 100; // Sleep for 100 milliseconds
+ bool write_ok = false;
+
+ if (status == NULL || status->status_store == NULL || status->stream == NULL) {
+ ERROR("Get progress status condition error");
+ return NULL;
+ }
+
+ for (;;) {
+ int i = 0;
+
+ usleep(delay * 1000); // Sleep for 100 milliseconds
+
+ if (status->should_terminal && status->image == NULL) {
+ break;
+ }
+
+ image_progress *progresses;
+ size_t progress_size = progress_status_map_size(status->status_store);
+
+ progresses = util_common_calloc_s(sizeof(image_progress));
+ if (progresses == NULL) {
+ ERROR("Out of memory. Skip progress show.");
+ break;
+ }
+
+ progresses->progresses = util_smart_calloc_s(sizeof(image_progress_progresses_element *), progress_size);
+ if (progresses->progresses == NULL) {
+ ERROR("Out of memory. Skip progress show.");
+ goto roundend;
+ }
+ if (status->image != NULL) {
+ progresses->image = util_strdup_s(status->image_name);
+ status->image = NULL;
+ }
+
+ if (!progress_status_map_lock(status->status_store)) {
+ ERROR("Cannot itorate progress status map for locking failed");
+ goto roundend;
+ }
+ map_itor *itor = map_itor_new(status->status_store->map);
+ for (i = 0; map_itor_valid(itor) && i < progress_size; map_itor_next(itor), i++) {
+ void *id = map_itor_key(itor);
+ const progress *value = (progress *)map_itor_value(itor);
+ const int ID_LEN = 12; // The last 12 charactos of image digest.
+
+ progresses->progresses[i] = util_common_calloc_s(sizeof(image_progress_progresses_element));
+ if (progresses->progresses[i] == NULL) {
+ WARN("Out of memory. Skip progress show.");
+ map_itor_free(itor);
+ progress_status_map_unlock(status->status_store);
+ goto roundend;
+ }
+ progresses->progresses[i]->id = util_strdup_s((char *)id + strlen((char *)id) - ID_LEN);
+ progresses->progresses[i]->total = value->dltotal;
+ progresses->progresses[i]->current = value->dlnow;
+ progresses->progresses_len++;
+ }
+ map_itor_free(itor);
+ progress_status_map_unlock(status->status_store);
+
+ /* send to client */
+ write_ok = status->stream->write_func(status->stream->writer, progresses);
+ if (write_ok) {
+ goto roundend;
+ }
+ if (status->stream->is_cancelled(status->stream->context)) {
+ ERROR("pull stream is cancelled");
+ goto roundend;
+ }
+ ERROR("Send progress data to client failed");
+roundend:
+ free_image_progress(progresses);
+ }
+ return NULL;
+}
+
+int oci_do_pull_image(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response)
{
int ret = 0;
imagetool_image_summary *image = NULL;
imagetool_image_summary *image2 = NULL;
char *dest_image_name = NULL;
+ progress_status_map *progress_status_store = NULL;
if (request == NULL || request->image == NULL || response == NULL) {
ERROR("Invalid NULL param");
return -1;
}
- ret = pull_image(request, &dest_image_name);
+ pthread_t tid = 0;
+ status_arg arg = {0};
+ if (request->is_progress_visible && stream != NULL) {
+ progress_status_store = progress_status_map_new();
+ if (progress_status_store == NULL) {
+ ERROR("Out of memory and will not show the pull progress");
+ isulad_set_error_message("Failed to pull image %s with error: out of memory", request->image);
+ ret = -1;
+ goto out;
+ }
+ arg.should_terminal = false;
+ arg.status_store = progress_status_store;
+ arg.stream = stream;
+ if (pthread_create(&tid, NULL, get_progress_status, (void *)&arg) != 0) {
+ ERROR("Failed to start thread to get progress status");
+ isulad_set_error_message("Failed to pull image %s with error: start progress thread error", request->image);
+ ret = -1;
+ goto out;
+ }
+ }
+
+ ret = pull_image(request, progress_status_store, &dest_image_name);
if (ret != 0) {
- ERROR("pull image %s failed", request->image);
+ ERROR("Pull image %s failed", request->image);
isulad_set_error_message("Failed to pull image %s with error: %s", request->image, g_isulad_errmsg);
ret = -1;
goto out;
@@ -197,17 +313,37 @@ int oci_do_pull_image(const im_pull_request *request, im_pull_response *response
image = storage_img_get_summary(dest_image_name);
image2 = storage_img_get_summary(request->image);
if (image == NULL || image2 == NULL) {
- ERROR("get image %s failed after pulling", request->image);
+ ERROR("Get image %s failed after pulling", request->image);
isulad_set_error_message("Failed to pull image %s with error: image not found after pulling", request->image);
ret = -1;
goto out;
}
+ arg.image = image;
+ arg.image_name = dest_image_name;
+ if (!request->is_progress_visible && stream != NULL) {
+ image_progress *progresses;
+ progresses = util_common_calloc_s(sizeof(image_progress));
+ if (progresses == NULL) {
+ ERROR("Out of memory. Skip progress show.");
+ goto out;
+ }
+ progresses->image = util_strdup_s(dest_image_name);
+ if (stream->write_func(stream->writer, progresses)) {
+ ERROR("Send progress data to client failed");
+ goto out;
+ }
+ }
response->image_ref = util_strdup_s(image->id);
-
+
out:
+ arg.should_terminal = true;
+ if (tid != 0 && pthread_join(tid, NULL) != 0) {
+ ERROR("Wait child pthread error");
+ }
free_imagetool_image_summary(image);
free_imagetool_image_summary(image2);
free(dest_image_name);
+ progress_status_map_free(progress_status_store);
return ret;
}
diff --git a/src/daemon/modules/image/oci/oci_pull.h b/src/daemon/modules/image/oci/oci_pull.h
index 1b2eca33..79404cfe 100644
--- a/src/daemon/modules/image/oci/oci_pull.h
+++ b/src/daemon/modules/image/oci/oci_pull.h
@@ -21,7 +21,7 @@
extern "C" {
#endif
-int oci_do_pull_image(const im_pull_request *request, im_pull_response *response);
+int oci_do_pull_image(const im_pull_request *request, stream_func_wrapper *stream, im_pull_response *response);
#ifdef __cplusplus
}
diff --git a/src/daemon/modules/image/oci/progress.c b/src/daemon/modules/image/oci/progress.c
new file mode 100644
index 00000000..110f22c0
--- /dev/null
+++ b/src/daemon/modules/image/oci/progress.c
@@ -0,0 +1,124 @@
+/******************************************************************************
+ * Copyright (c) China Unicom Technologies Co., Ltd. 2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: Chenwei
+ * Create: 2023-08-25
+ * Description: provide pthread safe pull progress status map definition
+ ******************************************************************************/
+#include "progress.h"
+#include <isula_libutils/log.h>
+#include <stdlib.h>
+
+#include "utils.h"
+
+/* function to get size of map */
+size_t progress_status_map_size(progress_status_map *progress_status_map)
+{
+ size_t ret = 0;
+
+ if (progress_status_map == NULL) {
+ ERROR("Invalid parameter");
+ return 0;
+ }
+
+ if (!progress_status_map_lock(progress_status_map)) {
+ ERROR("Cannot get the progress status map size for locking failed");
+ return 0;
+ }
+ ret = map_size(progress_status_map->map);
+ progress_status_map_unlock(progress_status_map);
+
+ return ret;
+}
+
+bool progress_status_map_insert(progress_status_map *progress_status_map, char *key, progress *value)
+{
+ bool ret = false;
+
+ if (progress_status_map == NULL || key == NULL || value == NULL) {
+ ERROR("Invalid parameter");
+ return false;
+ }
+
+ if (!progress_status_map_lock(progress_status_map)) {
+ ERROR("Cannot replace the progress status map item for locking failed");
+ return false;
+ }
+ ret = map_insert(progress_status_map->map, key, value);
+ progress_status_map_unlock(progress_status_map);
+
+ return ret;
+}
+
+// malloc a new map by type
+progress_status_map *progress_status_map_new()
+{
+ progress_status_map *progress_status_map = NULL;
+ progress_status_map = util_common_calloc_s(sizeof(struct progress_status_map));
+ if (progress_status_map == NULL) {
+ ERROR("Out of memory");
+ return NULL;
+ }
+ progress_status_map->map = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
+ if (progress_status_map->map == NULL) {
+ free(progress_status_map);
+ ERROR("Out of memory");
+ return NULL;
+ }
+ if (pthread_mutex_init(&(progress_status_map->mutex), NULL) != 0) {
+ map_free(progress_status_map->map);
+ free(progress_status_map);
+ ERROR("New map failed for mutex init");
+ return NULL;
+ }
+ return progress_status_map;
+}
+
+/* map free */
+void progress_status_map_free(progress_status_map *progress_status_map)
+{
+ if (progress_status_map == NULL) {
+ return;
+ }
+
+ pthread_mutex_destroy(&(progress_status_map->mutex));
+ map_free(progress_status_map->map);
+ free(progress_status_map);
+}
+
+bool progress_status_map_lock(progress_status_map *progress_status_map)
+{
+ int ret = 0;
+
+ if (progress_status_map == NULL) {
+ return false;
+ }
+
+ ret = pthread_mutex_lock(&(progress_status_map->mutex));
+ if (ret != 0) {
+ ERROR("Lock progress status map failed: %s", strerror(ret));
+ return false;
+ }
+ return true;
+}
+
+void progress_status_map_unlock(progress_status_map *progress_status_map)
+{
+ int ret = 0;
+
+ if (progress_status_map == NULL) {
+ return;
+ }
+
+ ret = pthread_mutex_unlock(&(progress_status_map->mutex));
+ if (ret != 0) {
+ ERROR("Unlock progress status map failed: %s", strerror(ret));
+ }
+}
diff --git a/src/daemon/modules/image/oci/progress.h b/src/daemon/modules/image/oci/progress.h
new file mode 100644
index 00000000..496a32f3
--- /dev/null
+++ b/src/daemon/modules/image/oci/progress.h
@@ -0,0 +1,52 @@
+/******************************************************************************
+ * Copyright (c) China Unicom Technologies Co., Ltd. 2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: Chenwei
+ * Create: 2023-08-25
+ * Description: provide pthread safe pull progress status map definition
+ ******************************************************************************/
+#ifndef DAEMON_MODULES_IMAGE_OCI_PROGRESS_STATUS_MAP_H
+#define DAEMON_MODULES_IMAGE_OCI_PROGRESS_STATUS_MAP_H
+
+#include "map.h"
+#include <pthread.h>
+#include <stdint.h>
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+typedef struct progress_status_map {
+ struct _map_t *map;
+ pthread_mutex_t mutex;
+} progress_status_map;
+
+typedef struct progress {
+ int64_t dlnow;
+ int64_t dltotal;
+} progress;
+
+bool progress_status_map_insert(progress_status_map *progress_status_map, char *key, progress *value);
+
+progress_status_map *progress_status_map_new();
+
+size_t progress_status_map_size(progress_status_map *progress_status_map);
+
+void progress_status_map_free(progress_status_map *map);
+
+bool progress_status_map_lock(progress_status_map *progress_status_map);
+
+void progress_status_map_unlock(progress_status_map *progress_status_map);
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif // DAEMON_MODULES_IMAGE_OCI_PROGRESS_STATUS_MAP_H
diff --git a/src/daemon/modules/image/oci/registry/http_request.c b/src/daemon/modules/image/oci/registry/http_request.c
index a514aaef..748c9a9b 100644
--- a/src/daemon/modules/image/oci/registry/http_request.c
+++ b/src/daemon/modules/image/oci/registry/http_request.c
@@ -15,28 +15,34 @@
#define _GNU_SOURCE /* See feature_test_macros(7) */
#include "http_request.h"
-#include <stdio.h>
-#include <string.h>
+#include <curl/curl.h>
#include <isula_libutils/json_common.h>
+#include <isula_libutils/log.h>
+#include <isula_libutils/registry_token.h>
+#include <pthread.h>
#include <stdbool.h>
+#include <stdio.h>
#include <stdlib.h>
+#include <string.h>
#include <strings.h>
#include <time.h>
-#include <curl/curl.h>
-#include <pthread.h>
-#include "isula_libutils/log.h"
#include "buffer.h"
+#include "certs.h"
+#include "err_msg.h"
#include "http.h"
#include "utils.h"
#include "utils_images.h"
-#include "certs.h"
-#include "isula_libutils/registry_token.h"
-#include "err_msg.h"
+#include "progress.h"
#include "utils_array.h"
#include "utils_base64.h"
#include "utils_string.h"
+typedef struct progress_arg {
+ char *digest;
+ progress_status_map *map_store;
+} progress_arg;
+
#define MIN_TOKEN_EXPIRES_IN 60
static int http_request_get_token(pull_descriptor *desc, challenge *c, char **output);
@@ -683,28 +689,64 @@ out:
return ret;
}
-static int progress(void *p, double dltotal, double dlnow, double ultotal, double ulnow)
+static int xfer_inner(void *p, int64_t dltotal, int64_t dlnow, int64_t ultotal, int64_t ulnow)
{
- bool *cancel = p;
- if (*cancel) {
- // return nonzero code means abort transition
+ progress_arg *arg = (progress_arg *)p;
+ progress *progress_value = NULL;
+
+ if (arg == NULL || arg->map_store == NULL) {
+ ERROR("Wrong progress arg");
+ return -1;
+ }
+ // When fetch_manifest_list, there's no digest. It's not a layer pulling progress and skip it.
+ if (arg->digest == NULL) {
+ return 0;
+ }
+
+ if (!progress_status_map_lock(arg->map_store)) {
+ ERROR("Cannot update progress status map for locking failed");
return -1;
}
+
+ // If the item exists, only replace the value.
+ progress_value = map_search(arg->map_store->map, arg->digest);
+ if (progress_value != NULL) {
+ progress_value->dlnow = dlnow;
+ progress_value->dltotal = dltotal;
+ progress_status_map_unlock(arg->map_store);
+
+ return 0;
+ }
+ progress_status_map_unlock(arg->map_store);
+
+ progress_value = util_common_calloc_s(sizeof(progress));
+ if (progress_value == NULL) {
+ ERROR("Out of memory");
+ return -1;
+ }
+
+ progress_value->dlnow = dlnow;
+ progress_value->dltotal = dltotal;
+
+ progress_status_map_insert(arg->map_store, arg->digest, progress_value);
+
return 0;
}
+#if (LIBCURL_VERSION_NUM >= 0x072000)
static int xfer(void *p, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
{
- bool *cancel = p;
- if (*cancel) {
- // return nonzero code means abort transition
- return -1;
- }
- return 0;
+ return xfer_inner(p, (int64_t)dltotal, (int64_t)dlnow, (int64_t)ultotal, (int64_t)ulnow);
+}
+#else
+static int get_progress(void *p, double dltotal, double dlnow, double ultotal, double ulnow)
+{
+ return xfer_inner(p, (int64_t)dltotal, (int64_t)dlnow, (int64_t)ultotal, (int64_t)ulnow);
}
+#endif
int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file,
- resp_data_type type, CURLcode *errcode)
+ resp_data_type type, CURLcode *errcode, char *digest)
{
int ret = 0;
struct http_get_options *options = NULL;
@@ -730,11 +772,24 @@ int http_request_file(pull_descriptor *desc, const char *url, const char **custo
}
options->outputtype = HTTP_REQUEST_FILE;
options->output = file;
- options->show_progress = 1;
- options->progressinfo = &desc->cancel;
- options->progress_info_op = progress;
- options->xferinfo = &desc->cancel;
- options->xferinfo_op = xfer;
+ progress_arg *arg = util_common_calloc_s(sizeof(progress_arg));
+ if (arg == NULL) {
+ ERROR("Out of memory");
+ goto out;
+ }
+ options->show_progress = 0;
+ if (desc->progress_status_store != NULL) {
+ arg->digest = digest;
+ arg->map_store = desc->progress_status_store;
+#if (LIBCURL_VERSION_NUM >= 0x072000)
+ options->xferinfo = arg;
+ options->xferinfo_op = xfer;
+#else
+ options->progressinfo = arg;
+ options->progress_info_op = get_progress;
+#endif
+ options->show_progress = 1;
+ }
options->timeout = true;
ret = setup_common_options(desc, options, url, custom_headers);
@@ -755,6 +810,7 @@ int http_request_file(pull_descriptor *desc, const char *url, const char **custo
out:
*errcode = options->errcode;
free_http_get_options(options);
+ free(arg);
options = NULL;
return ret;
diff --git a/src/daemon/modules/image/oci/registry/http_request.h b/src/daemon/modules/image/oci/registry/http_request.h
index 71df37d7..ed3f7e98 100644
--- a/src/daemon/modules/image/oci/registry/http_request.h
+++ b/src/daemon/modules/image/oci/registry/http_request.h
@@ -32,7 +32,7 @@ typedef enum {
int http_request_buf(pull_descriptor *desc, const char *url, const char **custom_headers, char **output,
resp_data_type type);
int http_request_file(pull_descriptor *desc, const char *url, const char **custom_headers, char *file,
- resp_data_type type, CURLcode *errcode);
+ resp_data_type type, CURLcode *errcode, char *digest);
#ifdef __cplusplus
}
diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c
index 4124281d..875f2df2 100644
--- a/src/daemon/modules/image/oci/registry/registry.c
+++ b/src/daemon/modules/image/oci/registry/registry.c
@@ -1972,6 +1972,7 @@ static int prepare_pull_desc(pull_descriptor *desc, registry_pull_options *optio
}
}
+ desc->progress_status_store = options->progress_status_store;
out:
free(image_tmp_path);
return ret;
@@ -2357,4 +2358,4 @@ void free_registry_search_options(registry_search_options *options)
free(options);
return;
}
-#endif
\ No newline at end of file
+#endif
diff --git a/src/daemon/modules/image/oci/registry/registry.h b/src/daemon/modules/image/oci/registry/registry.h
index cafb11c6..bb2af348 100644
--- a/src/daemon/modules/image/oci/registry/registry.h
+++ b/src/daemon/modules/image/oci/registry/registry.h
@@ -16,6 +16,7 @@
#define DAEMON_MODULES_IMAGE_OCI_REGISTRY_REGISTRY_H
#include <stdbool.h>
+#include "progress.h"
#ifdef ENABLE_IMAGE_SEARCH
#include <isula_libutils/imagetool_search_result.h>
@@ -36,6 +37,7 @@ typedef struct {
registry_auth auth;
bool skip_tls_verify;
bool insecure_registry;
+ progress_status_map *progress_status_store; // Don't free it. It's freed at oci_pull.c.
} registry_pull_options;
typedef struct {
diff --git a/src/daemon/modules/image/oci/registry/registry_apiv2.c b/src/daemon/modules/image/oci/registry/registry_apiv2.c
index db4d311e..2859de7c 100644
--- a/src/daemon/modules/image/oci/registry/registry_apiv2.c
+++ b/src/daemon/modules/image/oci/registry/registry_apiv2.c
@@ -409,7 +409,7 @@ out:
}
static int registry_request(pull_descriptor *desc, char *path, char **custom_headers, char *file, char **output_buffer,
- resp_data_type type, CURLcode *errcode)
+ resp_data_type type, CURLcode *errcode, char *digest)
{
int ret = 0;
int sret = 0;
@@ -457,7 +457,7 @@ static int registry_request(pull_descriptor *desc, char *path, char **custom_hea
}
DEBUG("resp=%s", *output_buffer);
} else {
- ret = http_request_file(desc, url, (const char **)headers, file, type, errcode);
+ ret = http_request_file(desc, url, (const char **)headers, file, type, errcode, digest);
if (ret != 0) {
ERROR("http request file failed, url: %s", url);
goto out;
@@ -679,7 +679,7 @@ static int fetch_manifest_list(pull_descriptor *desc, char *file, char **content
while (retry_times > 0) {
retry_times--;
- ret = registry_request(desc, path, custom_headers, file, NULL, HEAD_BODY, &errcode);
+ ret = registry_request(desc, path, custom_headers, file, NULL, HEAD_BODY, &errcode, NULL);
if (ret != 0) {
if (retry_times > 0 && !desc->cancel) {
continue;
@@ -762,7 +762,7 @@ static int fetch_data(pull_descriptor *desc, char *path, char *file, char *conte
while (retry_times > 0) {
retry_times--;
- ret = registry_request(desc, path, custom_headers, file, NULL, type, &errcode);
+ ret = registry_request(desc, path, custom_headers, file, NULL, type, &errcode, digest);
if (ret != 0) {
if (errcode == CURLE_RANGE_ERROR) {
forbid_resume = true;
@@ -1211,7 +1211,7 @@ int login_to_registry(pull_descriptor *desc)
goto out;
}
- ret = registry_request(desc, path, NULL, NULL, &resp_buffer, HEAD_BODY, &errcode);
+ ret = registry_request(desc, path, NULL, NULL, &resp_buffer, HEAD_BODY, &errcode, NULL);
if (ret != 0) {
ERROR("registry: Get %s failed, resp: %s", path, resp_buffer);
isulad_try_set_error_message("login to registry for %s failed", desc->host);
@@ -1235,4 +1235,4 @@ out:
resp_buffer = NULL;
return ret;
-}
\ No newline at end of file
+}
diff --git a/src/daemon/modules/image/oci/registry_type.h b/src/daemon/modules/image/oci/registry_type.h
index f232f227..8ddfcfea 100644
--- a/src/daemon/modules/image/oci/registry_type.h
+++ b/src/daemon/modules/image/oci/registry_type.h
@@ -20,6 +20,7 @@
#include <time.h>
#include <stdbool.h>
+#include "progress.h"
#include "utils_timestamp.h"
// 8 is enough for challenge, usually only one challenge is provided.
@@ -134,6 +135,8 @@ typedef struct {
char *search_name;
uint32_t limit;
#endif
+
+ progress_status_map *progress_status_store; // Don't free it. It's freed at other place.
} pull_descriptor;
void free_challenge(challenge *c);
diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt
index 6933caf5..42814fd6 100644
--- a/src/utils/CMakeLists.txt
+++ b/src/utils/CMakeLists.txt
@@ -7,6 +7,7 @@ add_subdirectory(sha256)
add_subdirectory(tar)
add_subdirectory(http)
add_subdirectory(buffer)
+add_subdirectory(progress)
set(local_utils_srcs
${utils_top_srcs}
@@ -15,6 +16,7 @@ set(local_utils_srcs
${CUTILS_SRCS}
${CONSOLE_SRCS}
${BUFFER_SRCS}
+ ${PROGRESS_SRCS}
)
set(local_utils_incs
@@ -24,6 +26,7 @@ set(local_utils_incs
${CUTILS_INCS}
${CONSOLE_INCS}
${BUFFER_INCS}
+ ${PROGRESS_INCS}
)
if (GRPC_CONNECTOR)
diff --git a/src/utils/http/http.h b/src/utils/http/http.h
index 02d56ba8..585afdf1 100644
--- a/src/utils/http/http.h
+++ b/src/utils/http/http.h
@@ -23,12 +23,15 @@
extern "C" {
#endif
-typedef int(*progress_info_func)(void *p,
- double dltotal, double dlnow,
- double ultotal, double ulnow);
+#if (LIBCURL_VERSION_NUM >= 0x072000)
typedef int(*xferinfo_func)(void *p,
curl_off_t dltotal, curl_off_t dlnow,
curl_off_t ultotal, curl_off_t ulnow);
+#else
+typedef int(*progress_info_func)(void *p,
+ double dltotal, double dlnow,
+ double ultotal, double ulnow);
+#endif
struct http_get_options {
unsigned with_head : 1, /* if set, means write output with response HEADER */
@@ -79,11 +82,13 @@ struct http_get_options {
bool timeout;
- void *progressinfo;
- progress_info_func progress_info_op;
-
+#if (LIBCURL_VERSION_NUM >= 0x072000)
void *xferinfo;
xferinfo_func xferinfo_op;
+#else
+ void *progressinfo;
+ progress_info_func progress_info_op;
+#endif
};
#define HTTP_RES_OK 0
diff --git a/src/utils/progress/CMakeLists.txt b/src/utils/progress/CMakeLists.txt
new file mode 100644
index 00000000..d06cca33
--- /dev/null
+++ b/src/utils/progress/CMakeLists.txt
@@ -0,0 +1,13 @@
+# get current directory sources files
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} local_progress_srcs)
+
+set(PROGRESS_SRCS
+ ${local_progress_srcs}
+ PARENT_SCOPE
+ )
+
+set(PROGRESS_INCS
+ ${CMAKE_CURRENT_SOURCE_DIR}
+ PARENT_SCOPE
+ )
+
diff --git a/src/utils/progress/show.c b/src/utils/progress/show.c
new file mode 100644
index 00000000..fbefe344
--- /dev/null
+++ b/src/utils/progress/show.c
@@ -0,0 +1,64 @@
+/******************************************************************************
+ * Copyright (c) China Unicom Technologies Co., Ltd. 2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: Chenwei
+ * Create: 2023-08-25
+ * Description: print progress
+ ******************************************************************************/
+
+#include "show.h"
+#include <sys/ioctl.h>
+#include <stdio.h>
+#include <term.h>
+#include <unistd.h>
+
+void move_to_row(int row)
+{
+ printf("\033[%d;1H", row);
+ fflush(stdout);
+}
+
+void move_cursor_up(int rows)
+{
+ printf("\033[%dA", rows); // ANSI escape code to move cursor up 'rows' rows
+}
+
+void clear_row(int row)
+{
+ move_to_row(row);
+ printf("\033[2K");
+ fflush(stdout);
+}
+
+void clear_lines_below()
+{
+ printf("\x1b[J"); // ANSI escape code to clear from cursor to end of screen
+ fflush(stdout);
+}
+
+int get_current_row()
+{
+ struct termios term;
+ if (tcgetattr(STDOUT_FILENO, &term) == -1) {
+ perror("tcgetattr");
+ return -1;
+ }
+ return term.c_cc[VERASE];
+}
+
+int get_terminal_width()
+{
+ struct winsize ws;
+ if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) == -1) {
+ perror("ioctl");
+ return -1; // Error
+ }
+ return ws.ws_col;
+}
diff --git a/src/utils/progress/show.h b/src/utils/progress/show.h
new file mode 100644
index 00000000..c1f71d86
--- /dev/null
+++ b/src/utils/progress/show.h
@@ -0,0 +1,34 @@
+/******************************************************************************
+ * Copyright (c) China Unicom Technologies Co., Ltd. 2023. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ * http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: Chenwei
+ * Create: 2023-08-25
+ * Description: print progress
+ ******************************************************************************/
+
+#ifndef UTILS_SHOW_H
+#define UTILS_SHOW_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void move_to_row(int row);
+void move_cursor_up(int lines);
+void clear_row(int row);
+void clear_lines_below();
+int get_current_row();
+int get_terminal_width();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/test/cutils/CMakeLists.txt b/test/cutils/CMakeLists.txt
index 10a10db9..9e681cc9 100644
--- a/test/cutils/CMakeLists.txt
+++ b/test/cutils/CMakeLists.txt
@@ -34,3 +34,4 @@ add_subdirectory(utils_utils)
add_subdirectory(utils_verify)
add_subdirectory(utils_network)
add_subdirectory(utils_transform)
+add_subdirectory(map)
diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt
index f9ba056e..77a7907e 100644
--- a/test/image/oci/registry/CMakeLists.txt
+++ b/test/image/oci/registry/CMakeLists.txt
@@ -18,6 +18,7 @@ add_executable(${EXE}
${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/map/rb_tree.c
${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/utils_timestamp.c
${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/utils_images.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/progress.c
${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/err_msg.c
${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/http/parser.c
${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/buffer/buffer.c
diff --git a/test/image/oci/registry/registry_ut.cc b/test/image/oci/registry/registry_ut.cc
index f4f8a763..3cb3e371 100644
--- a/test/image/oci/registry/registry_ut.cc
+++ b/test/image/oci/registry/registry_ut.cc
@@ -214,21 +214,7 @@ int invokeHttpRequestV2(const char *url, struct http_get_options *options, long
} else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/blobs/sha256:c7c37e47")) {
file = data_path + "config";
if (count == COUNT_TEST_CANCEL) {
- bool *cancel = (bool *)options->progressinfo;
- while (!(*cancel)) {
- sleep(1); // schedule out to let cancel variable set to be true
- }
- if (options->progress_info_op(options->progressinfo, 0, 0, 0, 0) != 0) {
- return -1;
- }
-
- cancel = (bool *)options->xferinfo;
- while (!(*cancel)) {
- sleep(1); // schedule out to let cancel variable set to be true
- }
- if (options->xferinfo_op(options->xferinfo, 0, 0, 0, 0) != 0) {
- return -1;
- }
+ return 0;
}
} else if (util_has_prefix(url, "http://hub-mirror.c.163.com/v2/library/busybox/blobs/sha256:91f30d77")) {
if (retry) {
--
2.42.0
Reference in New Issue Copy Permalink