From e54379b65d19ab25dbe69cf16d3180ae923bf27b Mon Sep 17 00:00:00 2001 From: tanyifeng Date: Thu, 21 Mar 2019 17:21:44 +0800 Subject: [PATCH 070/138] lxc: signal all process for shared container when container init exited Signed-off-by: tanyifeng Signed-off-by: LiFeng --- src/lxc/cgroups/cgfsng.c | 2 +- src/lxc/start.c | 332 +++++++++++++++++++++++++---------------------- 2 files changed, 175 insertions(+), 159 deletions(-) diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c index ab7ca35..5ceb06b 100644 --- a/src/lxc/cgroups/cgfsng.c +++ b/src/lxc/cgroups/cgfsng.c @@ -1056,7 +1056,7 @@ static int cgroup_rmdir(struct hierarchy **hierarchies, ret = recursive_destroy(h->container_full_path); if (ret < 0) { - SYSERROR("Failed to destroy \"%s\"", h->container_full_path); + SYSWARN("Failed to destroy \"%s\"", h->container_full_path); return -1; } diff --git a/src/lxc/start.c b/src/lxc/start.c index 1c9eb0a..b14e46f 100644 --- a/src/lxc/start.c +++ b/src/lxc/start.c @@ -941,6 +941,162 @@ out_close_maincmd_fd: return -1; } +void trim_line(char *s) +{ + size_t len; + + len = strlen(s); + while ((len > 1) && (s[len - 1] == '\n')) + s[--len] = '\0'; +} + +static int _read_procs_file(const char *path, pid_t **pids, size_t *len) +{ + FILE *f; + char *line = NULL; + size_t sz = 0; + + f = fopen_cloexec(path, "r"); + if (!f) + return -1; + + while (getline(&line, &sz, f) != -1) { + pid_t pid; + trim_line(line); + pid = (pid_t)atoll(line); + *pids = realloc(*pids, sizeof(pid_t) * (*len + 1)); + (*pids)[*len] = pid; + (*len)++; + } + + free(line); + fclose(f); + return 0; +} + +static int _recursive_read_cgroup_procs(const char *dirpath, pid_t **pids, size_t *len) +{ + struct dirent *direntp; + DIR *dir; + int ret, failed = 0; + char pathname[PATH_MAX]; + + dir = opendir(dirpath); + if (!dir && errno != ENOENT) { + WARN("Failed to open \"%s\"", dirpath); + return 0; + } + + while ((direntp = readdir(dir))) { + struct stat mystat; + int rc; + + if (!strcmp(direntp->d_name, ".") || + !strcmp(direntp->d_name, "..")) + continue; + + rc = snprintf(pathname, PATH_MAX, "%s/%s", dirpath, direntp->d_name); + if (rc < 0 || rc >= PATH_MAX) { + failed = 1; + continue; + } + + if (strcmp(direntp->d_name, "cgroup.procs") == 0) { + if (_read_procs_file(pathname, pids, len)) { + failed = 1; + + } + continue; + } + + ret = lstat(pathname, &mystat); + if (ret) { + failed = 1; + continue; + } + + if (S_ISDIR(mystat.st_mode)) { + if (_recursive_read_cgroup_procs(pathname, pids, len) < 0) + failed = 1; + } + } + + ret = closedir(dir); + if (ret) { + WARN("Failed to close directory \"%s\"", dirpath); + failed = 1; + } + + return failed ? -1 : 0; +} + +int get_all_pids(struct cgroup_ops *cg_ops, pid_t **pids, size_t *len) +{ + char *devices_path = NULL; + int ret; + + devices_path = must_make_path("/sys/fs/cgroup", "devices", cg_ops->container_cgroup, NULL); + if (!file_exists(devices_path)) { + free(devices_path); + return 0; + } + + ret = _recursive_read_cgroup_procs(devices_path, pids, len); + free(devices_path); + return ret; +} + +static int set_cgroup_freezer(struct cgroup_ops *cg_ops, const char *value) +{ + char *fullpath; + int ret; + + fullpath = must_make_path("/sys/fs/cgroup", "freezer", cg_ops->container_cgroup, "freezer.state", NULL); + ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666); + free(fullpath); + return ret; +} + +/* isulad: kill all process in container cgroup path */ +static void signal_all_processes(struct lxc_handler *handler) +{ + int ret; + struct cgroup_ops *cg_ops = handler->cgroup_ops; + pid_t *pids = NULL; + size_t len = 0, i; + + ret = set_cgroup_freezer(cg_ops, "FROZEN"); + if (ret < 0 && errno != ENOENT) { + WARN("cgroup_set frozen failed"); + } + + ret = get_all_pids(cg_ops, &pids, &len); + if (ret < 0) { + WARN("failed to get all pids"); + } + + for (i = 0; i < len; i++) { + ret = kill(pids[i], SIGKILL); + if (ret < 0 && errno != ESRCH) { + WARN("Can not kill process (pid=%d) with SIGKILL for container %s", pids[i], handler->name); + } + } + + ret = set_cgroup_freezer(cg_ops, "THAWED"); + if (ret < 0 && errno != ENOENT) { + WARN("cgroup_set thawed failed"); + } + + for (i = 0; i < len; i++) { + ret = lxc_wait_for_pid_status(pids[i]); + if (ret < 0 && errno != ECHILD) { + WARN("Failed to wait pid %d for container %s: %s", pids[i], handler->name, strerror(errno)); + } + } + + free(pids); +} + void lxc_fini(const char *name, struct lxc_handler *handler) { int i, ret; @@ -949,6 +1105,8 @@ void lxc_fini(const char *name, struct lxc_handler *handler) char *namespaces[LXC_NS_MAX + 1]; size_t namespace_count = 0; struct cgroup_ops *cgroup_ops = handler->cgroup_ops; + int retry_count = 0; + int max_retry = 10; /* The STOPPING state is there for future cleanup code which can take * awhile. @@ -1013,7 +1171,21 @@ void lxc_fini(const char *name, struct lxc_handler *handler) while (namespace_count--) free(namespaces[namespace_count]); - cgroup_ops->destroy(cgroup_ops, handler); + // if we shared pid namespace with others, should kill all processes within container cgroup + if (handler->conf->ns_share[LXC_NS_PID] != NULL) { + TRACE("Trying to kill all subprocess"); + signal_all_processes(handler); + TRACE("Finished kill all subprocess"); + } +retry: + if (!cgroup_ops->destroy(cgroup_ops, handler)) { + if (retry_count < max_retry) { + usleep(100 * 1000); /* 100 millisecond */ + retry_count++; + goto retry; + } + SYSERROR("Failed to destroy cgroup path for container: \"%s\"", handler->name); + } if (handler->conf->reboot == REBOOT_NONE) { /* For all new state clients simply close the command socket. @@ -2494,162 +2666,6 @@ static void clean_resource_set_env(struct lxc_handler *handler) /* End of environment variable setup for hooks. */ } -void trim_line(char *s) -{ - size_t len; - - len = strlen(s); - while ((len > 1) && (s[len - 1] == '\n')) - s[--len] = '\0'; -} - -static int _read_procs_file(const char *path, pid_t **pids, size_t *len) -{ - FILE *f; - char *line = NULL; - size_t sz = 0; - - f = fopen_cloexec(path, "r"); - if (!f) - return -1; - - while (getline(&line, &sz, f) != -1) { - pid_t pid; - trim_line(line); - pid = (pid_t)atoll(line); - *pids = realloc(*pids, sizeof(pid_t) * (*len + 1)); - (*pids)[*len] = pid; - (*len)++; - } - - free(line); - fclose(f); - return 0; -} - -static int _recursive_read_cgroup_procs(const char *dirpath, pid_t **pids, size_t *len) -{ - struct dirent *direntp; - DIR *dir; - int ret, failed = 0; - char pathname[PATH_MAX]; - - dir = opendir(dirpath); - if (!dir && errno != ENOENT) { - WARN("Failed to open \"%s\"", dirpath); - return 0; - } - - while ((direntp = readdir(dir))) { - struct stat mystat; - int rc; - - if (!strcmp(direntp->d_name, ".") || - !strcmp(direntp->d_name, "..")) - continue; - - rc = snprintf(pathname, PATH_MAX, "%s/%s", dirpath, direntp->d_name); - if (rc < 0 || rc >= PATH_MAX) { - failed = 1; - continue; - } - - if (strcmp(direntp->d_name, "cgroup.procs") == 0) { - if (_read_procs_file(pathname, pids, len)) { - failed = 1; - - } - continue; - } - - ret = lstat(pathname, &mystat); - if (ret) { - failed = 1; - continue; - } - - if (S_ISDIR(mystat.st_mode)) { - if (_recursive_read_cgroup_procs(pathname, pids, len) < 0) - failed = 1; - } - } - - ret = closedir(dir); - if (ret) { - WARN("Failed to close directory \"%s\"", dirpath); - failed = 1; - } - - return failed ? -1 : 0; -} - -int get_all_pids(struct cgroup_ops *cg_ops, pid_t **pids, size_t *len) -{ - char *devices_path = NULL; - int ret; - - devices_path = must_make_path("/sys/fs/cgroup", "devices", cg_ops->container_cgroup, NULL); - if (!file_exists(devices_path)) { - free(devices_path); - return 0; - } - - ret = _recursive_read_cgroup_procs(devices_path, pids, len); - free(devices_path); - return ret; -} - -static int set_cgroup_freezer(struct cgroup_ops *cg_ops, const char *value) -{ - char *fullpath; - int ret; - - fullpath = must_make_path("/sys/fs/cgroup", "freezer", cg_ops->container_cgroup, "freezer.state", NULL); - ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666); - free(fullpath); - return ret; -} - -/* isulad: kill all process in container cgroup path */ -static void signal_all_processes(struct lxc_handler *handler) -{ - int ret; - struct cgroup_ops *cg_ops = handler->cgroup_ops; - pid_t *pids = NULL; - size_t len = 0, i; - - ret = set_cgroup_freezer(cg_ops, "FROZEN"); - if (ret < 0 && errno != ENOENT) { - WARN("cgroup_set frozen failed"); - } - - ret = get_all_pids(cg_ops, &pids, &len); - if (ret < 0) { - WARN("failed to get all pids"); - } - - for (i = 0; i < len; i++) { - ret = kill(pids[i], SIGKILL); - if (ret < 0 && errno != ESRCH) { - WARN("Can not kill process (pid=%d) with SIGKILL for container %s", pids[i], handler->name); - } - } - - ret = set_cgroup_freezer(cg_ops, "THAWED"); - if (ret < 0 && errno != ENOENT) { - WARN("cgroup_set thawed failed"); - } - - for (i = 0; i < len; i++) { - ret = lxc_wait_for_pid_status(pids[i]); - if (ret < 0 && errno != ECHILD) { - WARN("Failed to wait pid %d for container %s: %s", pids[i], handler->name, strerror(errno)); - } - } - - free(pids); -} - /*isulad: do_lxcapi_clean_resource */ int do_lxcapi_clean_resource(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid) { @@ -2687,7 +2703,7 @@ retry: retry_count++; goto retry; } - ERROR("Failed to destroy cgroup for container \"%s\".", handler->name); + SYSERROR("Failed to destroy cgroup path for container: \"%s\"", handler->name); ret = -1; } -- 1.8.3.1