From 3ffaa7ae39d1af1d7c5aae976f1d6017ad76dee9 Mon Sep 17 00:00:00 2001 From: haozi007 Date: Wed, 30 Nov 2022 17:33:16 +0800 Subject: [PATCH 55/65] retry call runtime ops 1. add retry macro; 2. retry call runtime cleanup ops; Signed-off-by: haozi007 --- src/daemon/modules/plugin/plugin.c | 21 +++---------------- .../modules/runtime/isula/isula_rt_ops.c | 14 +++++++++++-- src/utils/cutils/utils.c | 1 + src/utils/cutils/utils.h | 19 +++++++++++++++++ 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/daemon/modules/plugin/plugin.c b/src/daemon/modules/plugin/plugin.c index 725bca5b..18035518 100644 --- a/src/daemon/modules/plugin/plugin.c +++ b/src/daemon/modules/plugin/plugin.c @@ -71,7 +71,7 @@ // suffix is '.sock' #define PLUGIN_SOCKET_FILE_SUFFIX_LEN 5 -#define PLUGIN_ACTIVATE_MAX_RETRY 3 +#define PLUGIN_ACTIVATE_MAX_RETRY 5 #ifndef RestHttpHead #define RestHttpHead "http://localhost" @@ -403,22 +403,6 @@ out: return -1; } -static int pm_activate_plugin_with_retry(plugin_t *plugin, size_t retry) -{ - size_t i = 0; - int err = 0; - - for (i = 0; i < retry; i++) { - err = pm_activate_plugin(plugin); - if (!err) { - return 0; - } - sleep((unsigned int)i + 1); - } - - return err; -} - static void pm_rdlock(void) { int errcode; @@ -500,7 +484,8 @@ static int pm_register_plugin(const char *name, const char *addr) ERROR("alloc plugin failed"); goto failed; } - err = pm_activate_plugin_with_retry(plugin, PLUGIN_ACTIVATE_MAX_RETRY); + + DO_RETYR_CALL(PLUGIN_ACTIVATE_MAX_RETRY, 1000000, err, pm_activate_plugin, plugin); if (err != 0) { ERROR("active plugin failed"); goto failed; diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c index c9667ee5..bfe7de08 100644 --- a/src/daemon/modules/runtime/isula/isula_rt_ops.c +++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c @@ -946,6 +946,7 @@ int rt_isula_restart(const char *name, const char *runtime, const rt_restart_par int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_params_t *params) { char workdir[PATH_MAX] = { 0 }; + int nret; if (id == NULL || runtime == NULL || params == NULL) { ERROR("nullptr arguments not allowed"); @@ -966,8 +967,17 @@ int rt_isula_clean_resource(const char *id, const char *runtime, const rt_clean_ shim_kill_force(workdir); } - (void)runtime_call_kill_force(workdir, runtime, id); - (void)runtime_call_delete_force(workdir, runtime, id); + // retry 10 count call runtime kill, every call sleep 1s + DO_RETYR_CALL(10, 1000000, nret, runtime_call_kill_force, workdir, runtime, id); + if (nret != 0) { + WARN("call runtime force kill failed"); + } + + // retry 10 count call runtime delete, every call sleep 1s + DO_RETYR_CALL(10, 1000000, nret, runtime_call_delete_force, workdir, runtime, id); + if (nret != 0) { + WARN("call runtime force delete failed"); + } if (util_recursive_rmdir(workdir, 0) != 0) { ERROR("failed rmdir -r shim workdir"); diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c index 9f5deaf9..a154c52a 100644 --- a/src/utils/cutils/utils.c +++ b/src/utils/cutils/utils.c @@ -1216,6 +1216,7 @@ void util_usleep_nointerupt(unsigned long usec) request = remain; } while (ret == -1 && errno == EINTR); } + int util_generate_random_str(char *id, size_t len) { int fd = -1; diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h index 27cfc902..0a9535a1 100644 --- a/src/utils/cutils/utils.h +++ b/src/utils/cutils/utils.h @@ -381,6 +381,25 @@ defs_map_string_object * dup_map_string_empty_object(defs_map_string_object *src int convert_v2_runtime(const char *runtime, char *binary); +/** + * retry_cnt: max count of call cb; + * interval_us: how many us to sleep, after call cb; + * cb: retry call function; + * return: + * 0 is cb successful at least once; + * 1 is all cb are failure; +*/ +#define DO_RETYR_CALL(retry_cnt, interval_us, ret, cb, ...) do { \ + size_t i = 0; \ + for(; i < retry_cnt; i++) { \ + ret = cb(__VA_ARGS__); \ + if (ret == 0) { \ + break; \ + } \ + util_usleep_nointerupt(interval_us); \ + } \ + } while(0) + #ifdef __cplusplus } #endif -- 2.25.1