From 52aa8985248200041b2a093634a8c36cb4bb8414 Mon Sep 17 00:00:00 2001 From: haozi007 Date: Tue, 14 Apr 2020 18:33:53 +0800 Subject: [PATCH 25/49] support oci hooks Signed-off-by: haozi007 --- src/lxc/conf.c | 542 +++++++++++++++++++++++++++++++++++++++++++++++++ src/lxc/conf.h | 17 ++ src/lxc/lxccontainer.c | 110 ++++++++++ src/lxc/lxccontainer.h | 28 +++ src/lxc/start.c | 277 +++++++++++++++++++++++++ src/lxc/start.h | 8 + src/lxc/sync.h | 4 + src/lxc/utils.c | 38 ++++ src/lxc/utils.h | 2 + 9 files changed, 1026 insertions(+) diff --git a/src/lxc/conf.c b/src/lxc/conf.c index e0a6f98..71fd6f9 100644 --- a/src/lxc/conf.c +++ b/src/lxc/conf.c @@ -33,6 +33,11 @@ #include #include +#ifdef HAVE_ISULAD +#include +#include "sync.h" +#endif + #include "af_unix.h" #include "caps.h" #include "cgroup.h" @@ -121,7 +126,14 @@ char *lxchook_names[NUM_LXC_HOOKS] = { "post-stop", "clone", "destroy", +#ifdef HAVE_ISULAD + "start-host", + "oci-prestart", + "oci-poststart", + "oci-poststop" +#else "start-host" +#endif }; struct mount_opt { @@ -3947,6 +3959,530 @@ static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs) } return 0; } + +struct oci_hook_conf { + defs_hook *ocihook; + + int errfd; + int which; +}; + +struct wait_conf { + pid_t pid; + unsigned long long startat; + int timeout; + int errfd; + int which; +}; + +static char* generate_json_str(const char *name, const char *lxcpath, const char *rootfs) +{ + char *cpid = NULL; + char *inmsg = NULL; + int rc = 0, ret = 0; + size_t size; + + if (!name || !lxcpath || !rootfs) { + ERROR("Invalid arguments"); + return NULL; + } + cpid = getenv("LXC_PID"); + if (!cpid) { + ERROR("Get container %s pid failed: %s", name, strerror(errno)); + cpid = "-1"; + } + + if ((strlen(name) + strlen(cpid) + strlen(rootfs) + strlen(lxcpath) + strlen(name)) > + SIZE_MAX - (strlen("{\"ociVersion\":\"\",\"id\":\"\",\"pid\":,\"root\":\"\",\"bundle\":\"\"}") - 1 - 1)) { + ERROR("Out of memory"); + ret = -1; + goto out_free; + } + + // {"ociVersion":"","id":"xxx","pid":777,"root":"xxx","bundle":"xxx"} + size = strlen("{\"ociVersion\":\"\",\"id\":\"\",\"pid\":,\"root\":\"\",\"bundle\":\"\"}") + + strlen(name) + strlen(cpid) + strlen(rootfs) + strlen(lxcpath) + 1 + strlen(name) + 1; + inmsg = malloc(size); + if (inmsg == NULL) { + ERROR("Out of memory"); + ret = -1; + goto out_free; + } + rc = snprintf(inmsg, size, + "{\"ociVersion\":\"\",\"id\":\"%s\",\"pid\":%s,\"root\":\"%s\",\"bundle\":\"%s/%s\"}", + name, cpid, rootfs, lxcpath, name); + if (rc < 0 || rc >= size) { + ERROR("Create json string failed"); + ret = -1; + } + +out_free: + if (ret) { + free(inmsg); + inmsg = NULL; + } + return inmsg; +} + +static char **merge_ocihook_env(char **oldenvs, size_t env_len, size_t *merge_env_len) +{ + char **result = NULL; + size_t result_len = env_len; + size_t i, j; + char *tmpenv = NULL; + char *lxc_envs[] = {"LD_LIBRARY_PATH", "PATH", "LXC_CGNS_AWARE", "LXC_PID", "LXC_ROOTFS_MOUNT", + "LXC_CONFIG_FILE", "LXC_CGROUP_PATH", "LXC_ROOTFS_PATH", "LXC_NAME" + }; + char *lxcenv_buf = NULL; + + if (result_len > SIZE_MAX - (sizeof(lxc_envs) / sizeof(char *)) - 1) + return NULL; + result_len += (sizeof(lxc_envs) / sizeof(char *)) + 1; + result = malloc(sizeof(char *) * result_len); + if (result == NULL) + return NULL; + memset(result, 0, sizeof(char *) * result_len); + + for(i = 0; i < env_len; i++) { + if (oldenvs[i]) + result[i] = safe_strdup(oldenvs[i]); + } + + for(j = 0; j < (sizeof(lxc_envs) / sizeof(char *)); j++) { + size_t env_buf_len = 0; + tmpenv = getenv(lxc_envs[j]); + if (tmpenv && i < (result_len - 1)) { + if (strlen(tmpenv) > (SIZE_MAX - 1 - 1 - strlen(lxc_envs[j]))) { + lxc_free_array((void **)result, free); + return NULL; + } + env_buf_len = ((strlen(tmpenv) + 1) + strlen(lxc_envs[j])) + 1; + lxcenv_buf = malloc(env_buf_len); + if (lxcenv_buf == NULL) { + lxc_free_array((void **)result, free); + return NULL; + } + if (snprintf(lxcenv_buf, env_buf_len, "%s=%s", lxc_envs[j], tmpenv) < 0) { + free(lxcenv_buf); + continue; + } + result[i++] = lxcenv_buf; + lxcenv_buf = NULL; + } + } + + *merge_env_len = i; + return result; +} + +static struct lxc_popen_FILE *lxc_popen_ocihook(const char *commandpath, char **args, int args_len, + char **envs, int env_len, const char *instr) +{ + int ret; + struct lxc_popen_FILE *fp = NULL; + int pipe_fds[2] = {-1, -1}; + int pipe_msg[2] = {-1, -1}; + pid_t child_pid; + + ret = pipe2(pipe_fds, O_CLOEXEC | O_NONBLOCK); + if (ret < 0) + return NULL; + + ret = pipe2(pipe_msg, O_CLOEXEC | O_NONBLOCK); + if (ret < 0) { + ERROR("Pipe msg failure"); + close(pipe_fds[0]); + close(pipe_fds[1]); + return NULL; + } + + child_pid = fork(); + if (child_pid < 0) + goto on_error; + + if (child_pid == 0) { + close(pipe_msg[1]); + if (pipe_msg[0] != STDIN_FILENO) + dup2(pipe_msg[0], STDIN_FILENO); + else { + if (fcntl(pipe_msg[0], F_SETFD, 0) != 0) { + fprintf(stderr, "Failed to remove FD_CLOEXEC from fd."); + exit(127); + } + } + close(pipe_msg[0]); + + close(pipe_fds[0]); + + /* duplicate stdout */ + if (pipe_fds[1] != STDOUT_FILENO) + ret = dup2(pipe_fds[1], STDOUT_FILENO); + else + ret = fcntl(pipe_fds[1], F_SETFD, 0); + if (ret < 0) { + close(pipe_fds[1]); + _exit(EXIT_FAILURE); + } + + /* duplicate stderr */ + if (pipe_fds[1] != STDERR_FILENO) + ret = dup2(pipe_fds[1], STDERR_FILENO); + else + ret = fcntl(pipe_fds[1], F_SETFD, 0); + close(pipe_fds[1]); + if (ret < 0) + _exit(EXIT_FAILURE); + + if (lxc_check_inherited(NULL, true, NULL, 0) != 0) { + fprintf(stderr, "check inherited fd failed"); + exit(127); + } + + /* + * Unblock signals. + * This is the main/only reason + * why we do our lousy popen() emulation. + */ + { + sigset_t mask; + sigfillset(&mask); + sigprocmask(SIG_UNBLOCK, &mask, NULL); + } + + if (env_len > 0) + execvpe(commandpath, args, envs); + else + execvp(commandpath, args); + fprintf(stderr, "fork/exec %s: %s", commandpath, strerror(errno)); + exit(127); + } + + /* parent */ + + close(pipe_fds[1]); + pipe_fds[1] = -1; + + close(pipe_msg[0]); + pipe_msg[0]= -1; + if (instr) { + size_t len = strlen(instr); + if (lxc_write_nointr(pipe_msg[1], instr, len) != len) { + WARN("Write instr: %s failed", instr); + } + } + close(pipe_msg[1]); + pipe_msg[1]= -1; + + fp = calloc(1, sizeof(*fp)); + if (!fp) { + ERROR("Failed to allocate memory"); + goto on_error; + } + + fp->child_pid = child_pid; + fp->pipe = pipe_fds[0]; + + return fp; + +on_error: + + if (pipe_fds[0] >= 0) + close(pipe_fds[0]); + + if (pipe_fds[1] >= 0) + close(pipe_fds[1]); + + if (pipe_msg[0] >= 0) + close(pipe_msg[0]); + + if (pipe_msg[1] >= 0) + close(pipe_msg[1]); + + if (fp) + free(fp); + + return NULL; +} + +void* wait_ocihook_timeout(void *arg) +{ + bool alive = false; + struct wait_conf *conf = (struct wait_conf *)arg; + + if (!conf || conf->timeout < 1) + goto out; + + sleep(conf->timeout); + + alive = lxc_process_alive(conf->pid, conf->startat); + + if (alive) { + ERROR("%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\"", + __FILE__, __LINE__, lxchook_names[conf->which], + (double)conf->timeout); + + lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\".", + __FILE__, __LINE__, lxchook_names[conf->which], + (double)conf->timeout); + + if (kill(conf->pid, SIGKILL) && errno != ESRCH) { + ERROR("Send kill signal failed"); + goto out; + } + } + +out: + free(conf); + return ((void *)0); +} + +static int run_ocihook_buffer(struct oci_hook_conf *oconf, const char *inmsg) +{ + struct lxc_popen_FILE *f; + char output[LXC_LOG_BUFFER_SIZE] = {0}; + int ret; + pthread_t ptid; + int err; + struct wait_conf *conf = NULL; + pthread_attr_t attr; + char *buffer = oconf->ocihook->path; + char *err_args_msg = NULL; + char *err_envs_msg = NULL; + char **hookenvs = NULL; + size_t hookenvs_len = 0; + + hookenvs = merge_ocihook_env(oconf->ocihook->env, oconf->ocihook->env_len, &hookenvs_len); + if (!hookenvs) { + ERROR("Out of memory."); + return -1; + } + + f = lxc_popen_ocihook(buffer, oconf->ocihook->args, oconf->ocihook->args_len, hookenvs, hookenvs_len, inmsg); + lxc_free_array((void **)hookenvs, free); + if (!f) { + SYSERROR("Failed to popen() %s.", buffer); + return -1; + } + + conf = malloc(sizeof(struct wait_conf)); + if (conf == NULL) { + SYSERROR("Failed to malloc."); + goto on_error; + } + + memset(conf, 0x00, sizeof(struct wait_conf)); + + conf->pid = f->child_pid; + conf->startat = lxc_get_process_startat(conf->pid); + + INFO("hook_conf timeout %d", oconf->ocihook->timeout); + if(oconf->ocihook->timeout > 0) + conf->timeout = oconf->ocihook->timeout; + else { + conf->timeout = 30; + INFO("Set hook timeout 30s"); + } + conf->errfd = oconf->errfd; + conf->which = oconf->which; + + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + err = pthread_create(&ptid, &attr, wait_ocihook_timeout, conf); + if (err != 0) { + ERROR("Create wait timeout thread failed"); + free(conf); + goto on_error; + } + + ret = lxc_wait_for_pid_status(f->child_pid); + + lxc_read_nointr(f->pipe, output, sizeof(output) - 1); + close(f->pipe); + free(f); + + if (ret == -1) { + SYSERROR("Script exited with error."); + goto print_hook; + } else if (WIFEXITED(ret) && WEXITSTATUS(ret) != 0) { + ERROR("Script exited with status %d. output: %s", WEXITSTATUS(ret), output); + lxc_write_error_message(oconf->errfd, "%s:%d: running %s hook caused \"error running hook: exit status %d, output: %s\".", + __FILE__, __LINE__, + (oconf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[oconf->which], + WEXITSTATUS(ret), output); + + goto print_hook; + } else if (WIFSIGNALED(ret)) { + ERROR("Script terminated by signal %d.", WTERMSIG(ret)); + lxc_write_error_message(oconf->errfd, "%s:%d: running %s hook caused \"error running hook: Script terminated by signal %d\".", + __FILE__, __LINE__, + (oconf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[oconf->which], + WTERMSIG(ret)); + + goto print_hook; + } + + return 0; + +on_error: + if (f) { + if (f->pipe >= 0) + close(f->pipe); + free(f); + } + +print_hook: + if (oconf->ocihook->args) + err_args_msg = lxc_string_join(" ", (const char **)oconf->ocihook->args, false); + if (oconf->ocihook->env) + err_envs_msg = lxc_string_join(" ", (const char **)oconf->ocihook->env, false); + ERROR("Hook script command: \"%s\", args: \"%s\", envs: \"%s\", timeout: %d.", + buffer, err_args_msg ? err_args_msg : "", + err_envs_msg ? err_envs_msg : "", oconf->ocihook->timeout); + + free(err_args_msg); + free(err_envs_msg); + return -1; +} + +static int run_ocihook_script_argv(const char *name, const char *section, + struct oci_hook_conf *oconf, + const char *lxcpath, const char *rootfs) +{ + int ret; + const char *script = oconf->ocihook->path; + char *inmsg = NULL; + + INFO("Executing script \"%s\" for container \"%s\", config section \"%s\".", + script, name, section); + + inmsg = generate_json_str(name, lxcpath, rootfs); + if (!inmsg) { + return -1; + } + + ret = run_ocihook_buffer(oconf, inmsg); + free(inmsg); + inmsg = NULL; + return ret; +} + +static char *get_root_path(const char *path, const char *backend) +{ + char *ret = NULL; + char *tmp = NULL; + + if (!path) { + ret = safe_strdup("/"); + return ret; + } + if (!backend) { + goto default_out; + } + + if (strcmp(backend, "aufs") == 0 || + strcmp(backend, "overlayfs") == 0 || + strcmp(backend, "loop") == 0) { + tmp = strrchr(path, ':'); + if (tmp == NULL) { + ERROR("Invalid root path format"); + return NULL; + } + tmp++; + ret = safe_strdup(tmp); + return ret; + } + +default_out: + ret = safe_strdup(path); + return ret; +} + +static int do_run_oci_hooks(const char *name, const char *lxcpath, struct lxc_conf *lc, int which, int errfd) +{ + struct oci_hook_conf work_conf = {0}; + size_t i; + int ret = 0; + int nret = 0; + char *rootpath = NULL; + + if (!lc) { + return -1; + } + if (!lc->ocihooks) { + return 0; + } + + rootpath = get_root_path(lc->rootfs.path, lc->rootfs.bdev_type); + if (!rootpath) { + ERROR("Get container %s rootpath failed.", name); + return -1; + } + + work_conf.errfd = errfd; + work_conf.which = which; + switch (which) { + case OCI_HOOK_PRESTART: + for (i = 0; i < lc->ocihooks->prestart_len; i++) { + work_conf.ocihook = lc->ocihooks->prestart[i]; + ret = run_ocihook_script_argv(name, "lxc", &work_conf, lxcpath, rootpath); + if (ret != 0) + break; + } + break; + case OCI_HOOK_POSTSTART: + for (i = 0; i < lc->ocihooks->poststart_len; i++) { + work_conf.ocihook = lc->ocihooks->poststart[i]; + nret = run_ocihook_script_argv(name, "lxc", &work_conf, lxcpath, rootpath); + if (nret != 0) + WARN("running poststart hook %zu failed, ContainerId: %s", i, name); + } + break; + case OCI_HOOK_POSTSTOP: + for (i = 0; i < lc->ocihooks->poststop_len; i++) { + work_conf.ocihook = lc->ocihooks->poststop[i]; + nret = run_ocihook_script_argv(name, "lxc", &work_conf, lxcpath, rootpath); + if (nret != 0) + WARN("running poststart hook %zu failed, ContainerId: %s", i, name); + } + break; + default: + ret = -1; + } + if (rootpath) + free(rootpath); + return ret; +} + +int run_oci_hooks(const char *name, const char *hookname, struct lxc_conf *conf, const char *lxcpath) +{ + int which = -1; + + if (strcmp(hookname, "oci-prestart") == 0) { + which = OCI_HOOK_PRESTART; + if (!lxcpath) { + ERROR("oci hook require lxcpath"); + return -1; + } + return do_run_oci_hooks(name, lxcpath, conf, which, conf->errpipe[1]); + } else if (strcmp(hookname, "oci-poststart") == 0) { + which = OCI_HOOK_POSTSTART; + if (!lxcpath) { + ERROR("oci hook require lxcpath"); + return -1; + } + return do_run_oci_hooks(name, lxcpath, conf, which, conf->errpipe[1]); + } else if (strcmp(hookname, "oci-poststop") == 0) { + which = OCI_HOOK_POSTSTOP; + if (!lxcpath) { + ERROR("oci hook require lxcpath"); + return -1; + } + return do_run_oci_hooks(name, lxcpath, conf, which, conf->errpipe[1]); + } else + return -1; + + return 0; +} #endif int lxc_setup(struct lxc_handler *handler) @@ -4083,6 +4619,12 @@ int lxc_setup(struct lxc_handler *handler) if (ret < 0) return log_error(-1, "Failed to \"/proc\" LSMs"); +#ifdef HAVE_ISULAD + /* Ask father to run oci prestart hooks and wait for him to finish. */ + if (lxc_sync_barrier_parent(handler, LXC_SYNC_OCI_PRESTART_HOOK)) { + return log_error(-1, "Failed to sync parent to start host hook"); + } +#endif ret = lxc_setup_rootfs_switch_root(&lxc_conf->rootfs); if (ret < 0) return log_error(-1, "Failed to pivot root into rootfs"); diff --git a/src/lxc/conf.h b/src/lxc/conf.h index 22c554d..61c3383 100644 --- a/src/lxc/conf.h +++ b/src/lxc/conf.h @@ -23,6 +23,10 @@ #include "start.h" #include "terminal.h" +#ifdef HAVE_ISULAD +#include "oci_runtime_hooks.h" +#endif + #if HAVE_SYS_RESOURCE_H #include #endif @@ -212,6 +216,11 @@ enum lxchooks { LXCHOOK_CLONE, LXCHOOK_DESTROY, LXCHOOK_START_HOST, +#ifdef HAVE_ISULAD + OCI_HOOK_PRESTART, + OCI_HOOK_POSTSTART, + OCI_HOOK_POSTSTOP, +#endif NUM_LXC_HOOKS }; @@ -433,6 +442,11 @@ struct lxc_conf { } shmount; #ifdef HAVE_ISULAD + /* + * isulad: support oci hook + * */ + oci_runtime_spec_hooks *ocihooks; + /* isulad add: init args used to repalce init_cmd*/ char **init_argv; size_t init_argc; @@ -447,6 +461,8 @@ struct lxc_conf { char *errmsg; /* record error messages */ + int errpipe[2];//pipdfd for get error message of child or grandchild process. + char *systemd; //systemd value #endif @@ -535,5 +551,6 @@ int lxc_clear_init_args(struct lxc_conf *lxc_conf); int lxc_clear_populate_devices(struct lxc_conf *c); int lxc_clear_rootfs_masked_paths(struct lxc_conf *c); int lxc_clear_rootfs_ro_paths(struct lxc_conf *c); +int run_oci_hooks(const char *name, const char *hookname, struct lxc_conf *conf, const char *lxcpath); #endif #endif /* __LXC_CONF_H */ diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c index 821cfa1..9b3ab75 100644 --- a/src/lxc/lxccontainer.c +++ b/src/lxc/lxccontainer.c @@ -288,6 +288,8 @@ static void lxc_container_free(struct lxc_container *c) #ifdef HAVE_ISULAD free(c->exit_fifo); c->exit_fifo = NULL; + free(c->ocihookfile); + c->ocihookfile = NULL; #endif free(c); @@ -632,6 +634,66 @@ static bool load_config_locked(struct lxc_container *c, const char *fname) return true; } +#ifdef HAVE_ISULAD +static bool load_ocihooks_locked(struct lxc_container *c) +{ + parser_error err = NULL; + oci_runtime_spec_hooks *hooks = NULL; + + if (!c->lxc_conf) + c->lxc_conf = lxc_conf_init(); + + if (!c->lxc_conf) + return false; + + hooks = oci_runtime_spec_hooks_parse_file(c->ocihookfile, NULL, &err); + if (!hooks) { + fprintf(stderr, "parse oci hooks config failed: %s\n", err); + free(err); + return true; + } + c->lxc_conf->ocihooks = hooks; + + if (err) + free(err); + return true; +} + +/* + * isulad: set oci hook file path + * */ +static bool set_oci_hook_config_filename(struct lxc_container *c) +{ +#define OCI_HOOK_JSON_FILE_NAME "ocihooks.json" + char *newpath = NULL; + int len, ret; + + if (!c->config_path) + return false; + + /* $lxc_path + "/" + c->name + "/" + "config" + '\0' */ + if (strlen(c->config_path) + strlen(c->name) > SIZE_MAX - strlen(OCI_HOOK_JSON_FILE_NAME) - 3) + return false; + len = strlen(c->config_path) + strlen(c->name) + strlen(OCI_HOOK_JSON_FILE_NAME) + 3; + + newpath = malloc(len); + if (newpath == NULL) + return false; + + ret = snprintf(newpath, len, "%s/%s/%s", c->config_path, c->name, OCI_HOOK_JSON_FILE_NAME); + if (ret < 0 || ret >= len) { + fprintf(stderr, "Error printing out config file name\n"); + free(newpath); + return false; + } + + free(c->ocihookfile); + c->ocihookfile = newpath; + + return true; +} +#endif + static bool do_lxcapi_load_config(struct lxc_container *c, const char *alt_file) { int lret; @@ -665,6 +727,11 @@ static bool do_lxcapi_load_config(struct lxc_container *c, const char *alt_file) ret = load_config_locked(c, fname); +#ifdef HAVE_ISULAD + if (ret && file_exists(c->ocihookfile)) + ret = load_ocihooks_locked(c); +#endif + if (need_disklock) container_disk_unlock(c); else @@ -5492,6 +5559,40 @@ static bool do_lxcapi_set_exec_terminal_winch(struct lxc_container *c, const cha } WRAP_API_3(bool, lxcapi_set_exec_terminal_winch, const char *, unsigned int, unsigned int) + +/* isulad add clean resources */ +static bool do_lxcapi_clean_container_resource(struct lxc_container *c, pid_t pid) +{ + int ret; + + if (!c) + return false; + + ret = do_lxcapi_clean_resource(c->name, c->config_path, c->lxc_conf, pid); + if (ret) + ERROR("Failed to clean container %s resource", c->name); + return ret == 0; + +} + +WRAP_API_1(bool, lxcapi_clean_container_resource, pid_t) + +/* isulad get coantainer pids */ +static bool do_lxcapi_get_container_pids(struct lxc_container *c, pid_t **pids,size_t *pids_len) +{ + int ret; + + if (!c) + return false; + + ret = do_lxcapi_get_pids(c->name, c->config_path, c->lxc_conf, pids,pids_len); + if (ret) + ERROR("Failed to get container %s pids", c->name); + return ret == 0; + +} + +WRAP_API_2(bool, lxcapi_get_container_pids, pid_t **,size_t *) #endif struct lxc_container *lxc_container_new(const char *name, const char *configpath) @@ -5547,6 +5648,13 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath goto err; } +#ifdef HAVE_ISULAD + if (!set_oci_hook_config_filename(c)) { + fprintf(stderr, "Error allocating oci hooks file pathname\n"); + goto err; + } +#endif + if (file_exists(c->configfile) && !lxcapi_load_config(c, NULL)) { fprintf(stderr, "Failed to load config for %s\n", name); goto err; @@ -5643,6 +5751,8 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath c->set_exec_terminal_winch = lxcapi_set_exec_terminal_winch; c->want_disable_pty = lxcapi_want_disable_pty; c->want_open_stdin = lxcapi_want_open_stdin; + c->clean_container_resource = lxcapi_clean_container_resource; + c->get_container_pids = lxcapi_get_container_pids; #endif return c; diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h index de2ee46..f1621f9 100644 --- a/src/lxc/lxccontainer.h +++ b/src/lxc/lxccontainer.h @@ -118,6 +118,13 @@ struct lxc_container { /*! Whether container wishes to keep stdin active */ bool open_stdin; + + /*! + * \private + * isulad: support oci hook from json file + * full path of json file + * */ + char *ocihookfile; #endif /*! @@ -935,6 +942,27 @@ struct lxc_container { * \return \c true on success, else \c false. */ bool (*want_open_stdin)(struct lxc_container *c, bool state); + + /*! isulad add + * \brief An API call to clean resources of container + * + * \param c Container. + * \param pid Value of container process. + * + * \return \c true on success, else \c false. + */ + bool (*clean_container_resource) (struct lxc_container *c, pid_t pid); + + /*! isulad add + * \brief An API call to get container pids + * + * \param c Container. + * \param pids Value of container pids. + * \param pids_len Value of container pids len. + * \param pid Value of container pid. + * \return \c true on success, else \c false. + */ + bool (*get_container_pids)(struct lxc_container *c,pid_t **pids,size_t *pids_len); #endif }; diff --git a/src/lxc/start.c b/src/lxc/start.c index f6a96b4..4f45776 100644 --- a/src/lxc/start.c +++ b/src/lxc/start.c @@ -2239,6 +2239,20 @@ static int lxc_spawn(struct lxc_handler *handler) ERROR("Failed to run lxc.hook.start-host"); goto out_delete_net; } +#ifdef HAVE_ISULAD + /* isulad: Run oci prestart hook at here */ + ret = run_oci_hooks(name, "oci-prestart", conf, lxcpath); + if (ret < 0) { + ERROR("Failed to run oci prestart hooks"); + goto out_delete_net; + } + + /* Tell the child to continue its initialization. We'll get + * LXC_SYNC_POST_OCI_PRESTART_HOOK when it is ready for us to run oci prestart hooks. + */ + if (lxc_sync_barrier_child(handler, LXC_SYNC_POST_OCI_PRESTART_HOOK)) + goto out_delete_net; +#endif /* Tell the child to complete its initialization and wait for it to exec * or return an error. (The child will never return @@ -2282,6 +2296,15 @@ static int lxc_spawn(struct lxc_handler *handler) if (ret < 0) goto out_abort; +#ifdef HAVE_ISULAD + /* isulad: Run oci prestart hook at here */ + ret = run_oci_hooks(name, "oci-poststart", conf, lxcpath); + if (ret < 0) { + ERROR("Failed to run oci poststart hooks"); + goto out_abort; + } +#endif + ret = lxc_set_state(name, handler, RUNNING); if (ret < 0) { ERROR("Failed to set state to \"%s\"", lxc_state2str(RUNNING)); @@ -2592,3 +2615,257 @@ static bool do_destroy_container(struct lxc_handler *handler) return storage_destroy(handler->conf); } + +#ifdef HAVE_ISULAD +/*isulad: set env for clean resources */ +static int clean_resource_set_env(struct lxc_handler *handler) +{ + const char *name = handler->name; + struct lxc_conf *conf = handler->conf; + char bufstr[PATH_MAX + 1]; + int i = 0; + int j = 0; + int len = 2; //set "LXC_PID" and "LXC_CGNS_AWARE" + + if (conf == NULL || conf->ocihooks == NULL || conf->ocihooks->poststop_len == 0) { + return 0; + } + + if (name) { + len++; + } + if (conf->rcfile) { + len++; + } + if (conf->rootfs.mount) { + len++; + } + if (conf->rootfs.path) { + len++; + } + if (conf->console.path) { + len++; + } + if (conf->console.log_path) { + len++; + } + if (handler->cgroup_ops->container_cgroup) { + len++; + } + + for (; i < conf->ocihooks->poststop_len; i++) { + size_t cap = conf->ocihooks->poststop[i]->env_len; + size_t newcap = cap + len + 1; + if (lxc_grow_array((void ***)&(conf->ocihooks->poststop[i]->env), &cap, newcap, 1) != 0) { + return -1; + } + j = conf->ocihooks->poststop[i]->env_len; + /* Start of environment variable setup for hooks. */ + if (name) { + snprintf(bufstr, PATH_MAX + 1, "LXC_NAME=%s", name); + conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr); + } + if (conf->rcfile) { + snprintf(bufstr, PATH_MAX + 1, "LXC_CONFIG_FILE=%s", conf->rcfile); + conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr); + } + if (conf->rootfs.mount) { + snprintf(bufstr, PATH_MAX + 1, "LXC_ROOTFS_MOUNT=%s", conf->rootfs.mount); + conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr); + } + if (conf->rootfs.path) { + snprintf(bufstr, PATH_MAX + 1, "LXC_ROOTFS_PATH=%s", conf->rootfs.path); + conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr); + } + if (conf->console.path) { + snprintf(bufstr, PATH_MAX + 1, "LXC_CONSOLE=%s", conf->console.path); + conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr); + } + if (conf->console.log_path) { + snprintf(bufstr, PATH_MAX + 1, "LXC_CONSOLE_LOGPATH=%s", conf->console.log_path); + conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr); + } + conf->ocihooks->poststop[i]->env[j++] = safe_strdup("LXC_CGNS_AWARE=1"); + + snprintf(bufstr, PATH_MAX + 1, "LXC_PID=%d", handler->pid); + conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr); + if (handler->cgroup_ops->container_cgroup) { + snprintf(bufstr, PATH_MAX + 1, "LXC_CGROUP_PATH=%s", handler->cgroup_ops->container_cgroup); + conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr); + } + conf->ocihooks->poststop[i]->env_len = j; + /* End of environment variable setup for hooks. */ + } + return 0; +} + +/*isulad: init handler for clean */ +static struct lxc_handler *lxc_init_clean_handler(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid) +{ + int i; + struct lxc_handler *handler; + + handler = malloc(sizeof(*handler)); + if (handler == NULL) + return NULL; + + memset(handler, 0, sizeof(*handler)); + + /* Note that am_guest_unpriv() checks the effective uid. We + * probably don't care if we are real root only if we are running + * as root so this should be fine. + */ + handler->am_root = !am_guest_unpriv(); + handler->data_sock[0] = handler->data_sock[1] = -1; + handler->conf = conf; + handler->lxcpath = lxcpath; + handler->pinfd = -1; + handler->sigfd = -EBADF; + handler->init_died = false; + handler->pid = pid; + handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1; + if (handler->conf->reboot == REBOOT_NONE) + lxc_list_init(&handler->conf->state_clients); + + for (i = 0; i < LXC_NS_MAX; i++) + handler->nsfd[i] = -1; + + handler->name = name; + handler->exit_code = -1; /* isulad: record exit code of container */ + + handler->cgroup_ops = cgroup_init(conf); + if (!handler->cgroup_ops) { + ERROR("Failed to initialize cgroup driver"); + goto on_error; + } + + INFO("Container \"%s\" 's clean handler is initialized.", name); + + return handler; + +on_error: + lxc_free_handler(handler); + + return NULL; +} + +/*isulad: init handler for clean */ +static struct lxc_handler *lxc_init_pids_handler(char *name, char *lxcpath, struct lxc_conf *conf) +{ + int i; + struct lxc_handler *handler; + + handler = malloc(sizeof(*handler)); + if (handler == NULL) + return NULL; + + memset(handler, 0, sizeof(*handler)); + + /* Note that am_guest_unpriv() checks the effective uid. We + * probably don't care if we are real root only if we are running + * as root so this should be fine. + */ + handler->am_root = !am_guest_unpriv(); + handler->data_sock[0] = handler->data_sock[1] = -1; + handler->conf = conf; + handler->lxcpath = lxcpath; + handler->pinfd = -1; + handler->sigfd = -EBADF; + handler->init_died = false; + handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1; + if (handler->conf->reboot == REBOOT_NONE) + lxc_list_init(&handler->conf->state_clients); + + for (i = 0; i < LXC_NS_MAX; i++) + handler->nsfd[i] = -1; + + handler->name = name; + handler->exit_code = -1; /* isulad: record exit code of container */ + + handler->cgroup_ops = cgroup_init(conf); + if (!handler->cgroup_ops) { + ERROR("Failed to initialize cgroup driver"); + goto on_error; + } + + INFO("Container \"%s\" 's clean handler is initialized.", name); + + return handler; + +on_error: + lxc_free_handler(handler); + + return NULL; +} + +/*isulad: do_lxcapi_clean_resource */ +int do_lxcapi_clean_resource(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid) +{ + int ret = 0; + struct lxc_handler *handler = NULL; + int retry_count = 0; + int max_retry = 10; + + handler = lxc_init_clean_handler(name, lxcpath, conf, pid); + if (!handler) { + ERROR("Failed to init container %s clean handler", name); + ret = -1; + goto out; + } + + if (clean_resource_set_env(handler) != 0) { + ERROR("Failed to set env for poststop hooks"); + ret = -1; + goto out; + } + + if (run_oci_hooks(handler->name, "oci-poststop", handler->conf, handler->lxcpath)) { + ERROR("Failed to run lxc.hook.post-stop for container \"%s\".", handler->name); + ret = -1; + } + +retry: + if (!handler->cgroup_ops->payload_destroy(handler->cgroup_ops, handler)) { + TRACE("Trying to kill all subprocess"); + signal_all_processes(handler); + TRACE("Finished kill all subprocess"); + if (retry_count < max_retry) { + usleep(100 * 1000); /* 100 millisecond */ + retry_count++; + goto retry; + } + SYSERROR("Failed to destroy cgroup path for container: \"%s\"", handler->name); + ret = -1; + } + +out: + lxc_free_handler(handler); + return ret; +} + +/*isulad: do_lxcapi_get_pids */ +int do_lxcapi_get_pids(char *name, char *lxcpath, struct lxc_conf *conf, pid_t **pids,size_t *pids_len) +{ + int ret = 0; + struct lxc_handler *handler = NULL; + struct cgroup_ops *cg_ops = NULL; + + handler = lxc_init_pids_handler(name, lxcpath, conf); + if (!handler) { + ERROR("Failed to init container %s clean handler", name); + ret = -1; + goto out; + } + + cg_ops = handler->cgroup_ops; + ret = get_all_pids(cg_ops, pids, pids_len); + if (ret < 0) { + WARN("failed to get all pids"); + } + +out: + lxc_free_handler(handler); + return ret; +} + +#endif diff --git a/src/lxc/start.h b/src/lxc/start.h index 5ea5fe2..4fc3ff7 100644 --- a/src/lxc/start.h +++ b/src/lxc/start.h @@ -175,4 +175,12 @@ extern int __lxc_start(struct lxc_handler *, struct lxc_operations *, void *, extern int resolve_clone_flags(struct lxc_handler *handler); +#ifdef HAVE_ISULAD +/*isulad: do_lxcapi_clean_resource */ +extern int do_lxcapi_clean_resource(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid); + +/*isulad: do_lxcapi_get_pids */ +extern int do_lxcapi_get_pids(char *name, char *lxcpath, struct lxc_conf *conf, pid_t **pids,size_t *pids_len); +#endif + #endif diff --git a/src/lxc/sync.h b/src/lxc/sync.h index ff7a1eb..56c1dfc 100644 --- a/src/lxc/sync.h +++ b/src/lxc/sync.h @@ -11,6 +11,10 @@ enum { LXC_SYNC_POST_CONFIGURE, LXC_SYNC_CGROUP, LXC_SYNC_CGROUP_UNSHARE, +#ifdef HAVE_ISULAD + LXC_SYNC_OCI_PRESTART_HOOK, + LXC_SYNC_POST_OCI_PRESTART_HOOK, +#endif LXC_SYNC_CGROUP_LIMITS, LXC_SYNC_READY_START, LXC_SYNC_RESTART, diff --git a/src/lxc/utils.c b/src/lxc/utils.c index 27078e2..39413ee 100644 --- a/src/lxc/utils.c +++ b/src/lxc/utils.c @@ -2122,4 +2122,42 @@ set_env: return 0; } +bool lxc_process_alive(pid_t pid, unsigned long long start_time) +{ + int sret = 0; + bool alive = true; + proc_t *pid_info = NULL; + char filename[PATH_MAX] = {0}; + char sbuf[1024] = {0}; /* bufs for stat */ + + sret = kill(pid, 0); + if (sret < 0 && errno == ESRCH) + return false; + + sret = snprintf(filename, sizeof(filename), "/proc/%d/stat", pid); + if (sret < 0 || sret >= sizeof(filename)) { + ERROR("Failed to sprintf filename"); + goto out; + } + + if ((lxc_file2str(filename, sbuf, sizeof(sbuf))) == -1) { + ERROR("Failed to read pidfile %s", filename); + alive = false; + goto out; + } + + pid_info = lxc_stat2proc(sbuf); + if (!pid_info) { + ERROR("Failed to get proc stat info"); + alive = false; + goto out; + } + + if (start_time != pid_info->start_time) + alive = false; +out: + free(pid_info); + return alive; +} + #endif diff --git a/src/lxc/utils.h b/src/lxc/utils.h index 36c458e..a213ba7 100644 --- a/src/lxc/utils.h +++ b/src/lxc/utils.h @@ -322,6 +322,8 @@ extern int lxc_file2str(const char *filename, char ret[], int cap); extern int unsigned long long lxc_get_process_startat(pid_t pid); // set env home in container extern int lxc_setup_env_home(uid_t uid); + +extern bool lxc_process_alive(pid_t pid, unsigned long long start_time); #endif #endif /* __LXC_UTILS_H */ -- 1.8.3.1