From 0d54daf204fd2bc41c45c7c159af6436d66b272c Mon Sep 17 00:00:00 2001 From: wujing Date: Mon, 13 Apr 2020 05:48:03 -0400 Subject: [PATCH 07/49] check and save pid info file Signed-off-by: wujing --- src/lxc/conf.c | 1 + src/lxc/conf.h | 2 + src/lxc/lxccontainer.c | 29 +++++++++++- src/lxc/lxccontainer.h | 12 +++++ src/lxc/start.c | 98 ++++++++++++++++++++++++++++++++++++++ src/lxc/tools/arguments.h | 16 +++++++ src/lxc/tools/lxc_start.c | 28 +++++++++++ src/lxc/utils.c | 117 ++++++++++++++++++++++++++++++++++++++++++++++ src/lxc/utils.h | 69 +++++++++++++++++++++++++++ 9 files changed, 371 insertions(+), 1 deletion(-) diff --git a/src/lxc/conf.c b/src/lxc/conf.c index e806605..43437af 100644 --- a/src/lxc/conf.c +++ b/src/lxc/conf.c @@ -3961,6 +3961,7 @@ void lxc_conf_free(struct lxc_conf *conf) free(conf->shmount.path_host); free(conf->shmount.path_cont); #ifdef HAVE_ISULAD + free(conf->container_info_file); lxc_clear_init_args(conf); lxc_clear_populate_devices(conf); #endif diff --git a/src/lxc/conf.h b/src/lxc/conf.h index 7ed3cd0..23942ac 100644 --- a/src/lxc/conf.h +++ b/src/lxc/conf.h @@ -428,6 +428,8 @@ struct lxc_conf { /* populate devices*/ struct lxc_list populate_devs; mode_t umask; //umask value + + char *container_info_file; #endif }; diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c index a617172..33bb3ec 100644 --- a/src/lxc/lxccontainer.c +++ b/src/lxc/lxccontainer.c @@ -5293,6 +5293,31 @@ static int do_lxcapi_seccomp_notify_fd(struct lxc_container *c) WRAP_API(int, lxcapi_seccomp_notify_fd) +#ifdef HAVE_ISULAD +/* isulad add set info file path */ +static bool do_lxcapi_set_container_info_file(struct lxc_container *c, const char *info_file) +{ + struct lxc_conf *conf = NULL; + + if (!c || !c->lxc_conf || !info_file) + return false; + if (container_mem_lock(c)) { + ERROR("Error getting mem lock"); + return false; + } + + conf = c->lxc_conf; + if (conf->container_info_file) + free(conf->container_info_file); + conf->container_info_file = safe_strdup(info_file); + + container_mem_unlock(c); + return true; +} + +WRAP_API_1(bool, lxcapi_set_container_info_file, const char *) +#endif + struct lxc_container *lxc_container_new(const char *name, const char *configpath) { struct lxc_container *c; @@ -5434,7 +5459,9 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath c->mount = lxcapi_mount; c->umount = lxcapi_umount; c->seccomp_notify_fd = lxcapi_seccomp_notify_fd; - +#ifdef HAVE_ISULAD + c->set_container_info_file = lxcapi_set_container_info_file; +#endif return c; err: diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h index 4577de7..edfff32 100644 --- a/src/lxc/lxccontainer.h +++ b/src/lxc/lxccontainer.h @@ -865,6 +865,18 @@ struct lxc_container { * \return pidfd of init process of the container. */ int (*init_pidfd)(struct lxc_container *c); + +#ifdef HAVE_ISULAD + /*! isulad add + * \brief An API call to set the path of info file + * + * \param c Container. + * \param info_file Value of the path of info file. + * + * \return \c true on success, else \c false. + */ + bool (*set_container_info_file) (struct lxc_container *c, const char *info_file); +#endif }; /*! diff --git a/src/lxc/start.c b/src/lxc/start.c index 5dcf828..f5f9565 100644 --- a/src/lxc/start.c +++ b/src/lxc/start.c @@ -1537,6 +1537,94 @@ static inline int do_share_ns(void *arg) return 0; } +#ifdef HAVE_ISULAD +static int lxc_write_container_info(char *filename, pid_t pid, pid_t p_pid, + unsigned long long start_at, unsigned long long p_start_at) +{ + FILE *pid_fp = NULL; + int ret = 0; + + pid_fp = lxc_fopen(filename, "w"); + if (pid_fp == NULL) { + SYSERROR("Failed to create pidfile '%s'",filename); + ret = -1; + goto out; + } + + if (fprintf(pid_fp, "%d %llu %d %llu\n", pid, start_at, p_pid, p_start_at) < 0) { + SYSERROR("Failed to write '%s'", filename); + ret = -1; + goto out; + } +out: + if (pid_fp) + fclose(pid_fp); + pid_fp = NULL; + return ret; +} + +static int lxc_check_container_info(char *filename, pid_t pid, pid_t p_pid, + unsigned long long start_at, unsigned long long p_start_at) +{ + int ret = 0; + int num; + char sbuf[1024] = {0}; /* bufs for stat */ + int saved_pid; /* process id */ + int saved_ppid; /* pid of parent process */ + unsigned long long saved_start_time; /* start time of process -- seconds since 1-1-70 */ + unsigned long long saved_pstart_time; /* start time of parent process -- seconds since 1-1-70 */ + + if ((lxc_file2str(filename, sbuf, sizeof(sbuf))) == -1) { + SYSERROR("Failed to read pidfile %s", filename); + ret = -1; + goto out; + } + + num = sscanf(sbuf, "%d %Lu %d %Lu", &saved_pid, &saved_start_time, &saved_ppid, &saved_pstart_time); + if (num != 4) { + SYSERROR("Call sscanf error"); + ret = -1; + goto out; + } + + if (pid != saved_pid || p_pid != saved_ppid + || start_at != saved_start_time || p_start_at != saved_pstart_time) { + ERROR("Check container info failed"); + ret = -1; + goto out; + } + +out: + return ret; +} + +/* isuald: save pid/ppid info */ +static int lxc_save_container_info(char *filename, pid_t pid) +{ + int ret = 0; + pid_t p_pid = 0; + unsigned long long start_at = 0; + unsigned long long p_start_at = 0; + + start_at = lxc_get_process_startat(pid); + p_pid = getpid(); + p_start_at = lxc_get_process_startat(p_pid); + + ret = lxc_write_container_info(filename, pid, p_pid, start_at, p_start_at); + if (ret != 0) { + goto out; + } + + ret = lxc_check_container_info(filename, pid, p_pid, start_at, p_start_at); + if (ret != 0) { + goto out; + } + +out: + return ret; +} +#endif + /* lxc_spawn() performs crucial setup tasks and clone()s the new process which * exec()s the requested container binary. * Note that lxc_spawn() runs in the parent namespaces. Any operations performed @@ -1648,6 +1736,16 @@ static int lxc_spawn(struct lxc_handler *handler) } TRACE("Cloned child process %d", handler->pid); +#ifdef HAVE_ISULAD + /* isulad: save pid/ppid info into file*/ + if (handler->conf->container_info_file) { + if (lxc_save_container_info(handler->conf->container_info_file, handler->pid)) { + ERROR("Failed to save cloned container pid"); + goto out_delete_net; + } + } +#endif + /* Verify that we can actually make use of pidfds. */ if (!lxc_can_use_pidfd(handler->pidfd)) close_prot_errno_disarm(handler->pidfd); diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h index cb0ba74..91f4e9a 100644 --- a/src/lxc/tools/arguments.h +++ b/src/lxc/tools/arguments.h @@ -40,6 +40,9 @@ struct lxc_arguments { /* for lxc-start */ const char *share_ns[32]; /* size must be greater than LXC_NS_MAX */ +#ifdef HAVE_ISULAD + const char *container_info; /* isulad: file used to store pid and ppid info of container */ +#endif /* for lxc-console */ unsigned int ttynum; @@ -152,6 +155,19 @@ struct lxc_arguments { #define OPT_SHARE_UTS OPT_USAGE - 5 #define OPT_SHARE_PID OPT_USAGE - 6 +#ifdef HAVE_ISULAD +#define OPT_INPUT_FIFO OPT_USAGE - 7 +#define OPT_OUTPUT_FIFO OPT_USAGE - 8 +#define OPT_STDERR_FIFO OPT_USAGE - 9 +#define OPT_CONTAINER_INFO OPT_USAGE - 10 +#define OPT_EXIT_FIFO OPT_USAGE - 11 +#define OPT_START_TIMEOUT OPT_USAGE - 12 +#define OPT_DISABLE_PTY OPT_USAGE - 13 +#define OPT_OPEN_STDIN OPT_USAGE - 14 +#define OPT_ATTACH_TIMEOUT OPT_USAGE - 15 +#define OPT_ATTACH_SUFFIX OPT_USAGE - 16 +#endif + extern int lxc_arguments_parse(struct lxc_arguments *args, int argc, char *const argv[]); diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c index 459b867..83ee75a 100644 --- a/src/lxc/tools/lxc_start.c +++ b/src/lxc/tools/lxc_start.c @@ -48,6 +48,9 @@ static const struct option my_longopts[] = { {"share-ipc", required_argument, 0, OPT_SHARE_IPC}, {"share-uts", required_argument, 0, OPT_SHARE_UTS}, {"share-pid", required_argument, 0, OPT_SHARE_PID}, +#ifdef HAVE_ISULAD + {"container-pidfile", required_argument, 0, OPT_CONTAINER_INFO}, +#endif LXC_COMMON_OPTIONS }; @@ -118,6 +121,11 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg) case OPT_SHARE_PID: args->share_ns[LXC_NS_PID] = arg; break; +#ifdef HAVE_ISULAD + case OPT_CONTAINER_INFO: + args->container_info = arg; + break; +#endif } return 0; } @@ -163,6 +171,9 @@ int main(int argc, char *argv[]) "/sbin/init", NULL, }; +#ifdef HAVE_ISULAD + char *container_info_file = NULL; +#endif lxc_list_init(&defines); @@ -283,6 +294,20 @@ int main(int argc, char *argv[]) goto out; } +#ifdef HAVE_ISULAD + /* isulad: container info file used to store pid and ppid info of container*/ + if (my_args.container_info != NULL) { + if (ensure_path(&container_info_file, my_args.container_info) < 0) { + ERROR("Failed to ensure container's piddile '%s'", my_args.container_info); + goto out; + } + if (!c->set_container_info_file(c, container_info_file)) { + ERROR("Failed to set container's piddile '%s'", container_info_file); + goto out; + } + } +#endif + if (my_args.console) if (!c->set_config_item(c, "lxc.console.path", my_args.console)) goto out; @@ -320,5 +345,8 @@ int main(int argc, char *argv[]) out: lxc_container_put(c); +#ifdef HAVE_ISULAD + free(container_info_file); +#endif exit(err); } diff --git a/src/lxc/utils.c b/src/lxc/utils.c index 160b3db..ebcdae0 100644 --- a/src/lxc/utils.c +++ b/src/lxc/utils.c @@ -1931,4 +1931,121 @@ void lxc_write_error_message(int errfd, const char *format, ...) if (sret < 0) SYSERROR("Write errbuf failed"); } + +/* isulad: read file to buffer */ +int lxc_file2str(const char *filename, char ret[], int cap) +{ + int fd, num_read; + + if ((fd = lxc_open(filename, O_RDONLY | O_CLOEXEC, 0)) == -1) + return -1; + if ((num_read = read(fd, ret, cap - 1)) <= 0) + num_read = -1; + else + ret[num_read] = 0; + close(fd); + + return num_read; +} + +/* isuald: lxc_stat2proc() makes sure it can handle arbitrary executable file basenames + * for `cmd', i.e. those with embedded whitespace or embedded ')'s. + * Such names confuse %s (see scanf(3)), so the string is split and %39c + * is used instead. (except for embedded ')' "(%[^)]c)" would work. + */ +static proc_t *lxc_stat2proc(const char *S) +{ + int num; + proc_t *P = NULL; + char *tmp = NULL; + + if (!S) + return NULL; + + tmp = strrchr(S, ')'); /* split into "PID (cmd" and "" */ + if (!tmp) + return NULL; + *tmp = '\0'; /* replace trailing ')' with NUL */ + + P = malloc(sizeof(proc_t)); + if (P == NULL) + return NULL; + (void)memset(P, 0x00, sizeof(proc_t)); + + /* parse these two strings separately, skipping the leading "(". */ + num = sscanf(S, "%d (%15c", &P->pid, P->cmd); /* comm[16] in kernel */ + if (num != 2) { + ERROR("Call sscanf error: %s", errno ? strerror(errno) : ""); + free(P); + return NULL; + } + num = sscanf(tmp + 2, /* skip space after ')' too */ + "%c " + "%d %d %d %d %d " + "%lu %lu %lu %lu %lu " + "%Lu %Lu %Lu %Lu " /* utime stime cutime cstime */ + "%ld %ld %ld %ld " + "%Lu " /* start_time */ + "%lu " + "%ld " + "%lu %lu %lu %lu %lu %lu " + "%*s %*s %*s %*s " /* discard, no RT signals & Linux 2.1 used hex */ + "%lu %lu %lu " + "%d %d " + "%lu %lu", + &P->state, + &P->ppid, &P->pgrp, &P->session, &P->tty, &P->tpgid, + &P->flags, &P->min_flt, &P->cmin_flt, &P->maj_flt, &P->cmaj_flt, + &P->utime, &P->stime, &P->cutime, &P->cstime, + &P->priority, &P->nice, &P->timeout, &P->it_real_value, + &P->start_time, + &P->vsize, + &P->rss, + &P->rss_rlim, &P->start_code, &P->end_code, &P->start_stack, &P->kstk_esp, + &P->kstk_eip, + &P->wchan, &P->nswap, &P->cnswap, + &P->exit_signal, &P->processor, /* 2.2.1 ends with "exit_signal" */ + &P->rtprio, &P->sched /* both added to 2.5.18 */ + ); + if (num != 35) { + ERROR("Call sscanf error: %s", errno ? strerror(errno) : ""); + free(P); + return NULL; + } + if (P->tty == 0) + P->tty = -1; /* the old notty val, update elsewhere bef. moving to 0 */ + return P; +} + +/* isulad: get starttime of process pid */ +unsigned long long lxc_get_process_startat(pid_t pid) +{ + int sret = 0; + unsigned long long startat = 0; + proc_t *pid_info = NULL; + char filename[PATH_MAX] = {0}; + char sbuf[1024] = {0}; /* bufs for stat */ + + sret = snprintf(filename, sizeof(filename), "/proc/%d/stat", pid); + if (sret < 0 || sret >= sizeof(filename)) { + ERROR("Failed to sprintf filename"); + goto out; + } + + if ((lxc_file2str(filename, sbuf, sizeof(sbuf))) == -1) { + SYSERROR("Failed to read pidfile %s", filename); + goto out; + } + + pid_info = lxc_stat2proc(sbuf); + if (!pid_info) { + ERROR("Failed to get proc stat info"); + goto out; + } + + startat = pid_info->start_time; +out: + free(pid_info); + return startat; +} #endif diff --git a/src/lxc/utils.h b/src/lxc/utils.h index 3c30565..11d6548 100644 --- a/src/lxc/utils.h +++ b/src/lxc/utils.h @@ -44,6 +44,73 @@ extern char *get_rundir(void); #endif #endif +#ifdef HAVE_ISULAD +/* isulad: + ld cutime, cstime, priority, nice, timeout, it_real_value, rss, + c state, + d ppid, pgrp, session, tty, tpgid, + s signal, blocked, sigignore, sigcatch, + lu flags, min_flt, cmin_flt, maj_flt, cmaj_flt, utime, stime, + lu rss_rlim, start_code, end_code, start_stack, kstk_esp, kstk_eip, + lu start_time, vsize, wchan, nswap, cnswap, +*/ + +/* Basic data structure which holds all information we can get about a process. + * (unless otherwise specified, fields are read from /proc/#/stat) + * + * Most of it comes from task_struct in linux/sched.h + */ +typedef struct proc_t { + // 1st 16 bytes + int pid; /* process id */ + int ppid; /* pid of parent process */ + + char state; /* single-char code for process state (S=sleeping) */ + + unsigned long long + utime, /* user-mode CPU time accumulated by process */ + stime, /* kernel-mode CPU time accumulated by process */ + // and so on... + cutime, /* cumulative utime of process and reaped children */ + cstime, /* cumulative stime of process and reaped children */ + start_time; /* start time of process -- seconds since 1-1-70 */ + + long + priority, /* kernel scheduling priority */ + timeout, /* ? */ + nice, /* standard unix nice level of process */ + rss, /* resident set size from /proc/#/stat (pages) */ + it_real_value; /* ? */ + unsigned long + rtprio, /* real-time priority */ + sched, /* scheduling class */ + vsize, /* number of pages of virtual memory ... */ + rss_rlim, /* resident set size limit? */ + flags, /* kernel flags for the process */ + min_flt, /* number of minor page faults since process start */ + maj_flt, /* number of major page faults since process start */ + cmin_flt, /* cumulative min_flt of process and child processes */ + cmaj_flt, /* cumulative maj_flt of process and child processes */ + nswap, /* ? */ + cnswap, /* cumulative nswap ? */ + start_code, /* address of beginning of code segment */ + end_code, /* address of end of code segment */ + start_stack, /* address of the bottom of stack for the process */ + kstk_esp, /* kernel stack pointer */ + kstk_eip, /* kernel instruction pointer */ + wchan; /* address of kernel wait channel proc is sleeping in */ + + char cmd[16]; /* basename of executable file in call to exec(2) */ + int + pgrp, /* process group id */ + session, /* session id */ + tty, /* full device number of controlling terminal */ + tpgid, /* terminal process group id */ + exit_signal, /* might not be SIGCHLD */ + processor; /* current (or most recent?) CPU */ +} proc_t; +#endif + static inline int lxc_set_cloexec(int fd) { return fcntl(fd, F_SETFD, FD_CLOEXEC); @@ -246,6 +313,8 @@ extern int fix_stdio_permissions(uid_t uid); #ifdef HAVE_ISULAD extern void lxc_write_error_message(int errfd, const char *format, ...); +extern int lxc_file2str(const char *filename, char ret[], int cap); +extern int unsigned long long lxc_get_process_startat(pid_t pid); #endif #endif /* __LXC_UTILS_H */ -- 1.8.3.1