lxc/0025-support-oci-hooks.patch

From 52aa8985248200041b2a093634a8c36cb4bb8414 Mon Sep 17 00:00:00 2001
From: haozi007 <liuhao27@huawei.com>
Date: Tue, 14 Apr 2020 18:33:53 +0800
Subject: [PATCH 25/49] support oci hooks

Signed-off-by: haozi007 <liuhao27@huawei.com>
---
 src/lxc/conf.c         | 542 +++++++++++++++++++++++++++++++++++++++++++++++++
 src/lxc/conf.h         |  17 ++
 src/lxc/lxccontainer.c | 110 ++++++++++
 src/lxc/lxccontainer.h |  28 +++
 src/lxc/start.c        | 277 +++++++++++++++++++++++++
 src/lxc/start.h        |   8 +
 src/lxc/sync.h         |   4 +
 src/lxc/utils.c        |  38 ++++
 src/lxc/utils.h        |   2 +
 9 files changed, 1026 insertions(+)

diff --git a/src/lxc/conf.c b/src/lxc/conf.c
index e0a6f98..71fd6f9 100644
--- a/src/lxc/conf.c
+++ b/src/lxc/conf.c
@@ -33,6 +33,11 @@
 #include <time.h>
 #include <unistd.h>
 
+#ifdef HAVE_ISULAD
+#include <pthread.h>
+#include "sync.h"
+#endif
+
 #include "af_unix.h"
 #include "caps.h"
 #include "cgroup.h"
@@ -121,7 +126,14 @@ char *lxchook_names[NUM_LXC_HOOKS] = {
 	"post-stop",
 	"clone",
 	"destroy",
+#ifdef HAVE_ISULAD
+	"start-host",
+	"oci-prestart",
+	"oci-poststart",
+	"oci-poststop"
+#else
 	"start-host"
+#endif
 };
 
 struct mount_opt {
@@ -3947,6 +3959,530 @@ static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs)
 	}
 	return 0;
 }
+
+struct oci_hook_conf {
+	defs_hook *ocihook;
+
+	int errfd;
+	int which;
+};
+
+struct wait_conf {
+	pid_t pid;
+	unsigned long long startat;
+	int timeout;
+	int errfd;
+	int which;
+};
+
+static char* generate_json_str(const char *name, const char *lxcpath, const char *rootfs)
+{
+	char *cpid = NULL;
+	char *inmsg = NULL;
+	int rc = 0, ret = 0;
+	size_t size;
+
+	if (!name || !lxcpath || !rootfs) {
+		ERROR("Invalid arguments");
+		return NULL;
+	}
+	cpid = getenv("LXC_PID");
+	if (!cpid) {
+		ERROR("Get container %s pid failed: %s", name, strerror(errno));
+		cpid = "-1";
+	}
+
+	if ((strlen(name) + strlen(cpid) + strlen(rootfs) + strlen(lxcpath) + strlen(name)) >
+	       SIZE_MAX - (strlen("{\"ociVersion\":\"\",\"id\":\"\",\"pid\":,\"root\":\"\",\"bundle\":\"\"}") - 1 - 1)) {
+		ERROR("Out of memory");
+		ret = -1;
+		goto out_free;
+	}
+
+	// {"ociVersion":"","id":"xxx","pid":777,"root":"xxx","bundle":"xxx"}
+	size = strlen("{\"ociVersion\":\"\",\"id\":\"\",\"pid\":,\"root\":\"\",\"bundle\":\"\"}") +
+	       strlen(name) + strlen(cpid) + strlen(rootfs) + strlen(lxcpath) + 1 + strlen(name) + 1;
+	inmsg = malloc(size);
+	if (inmsg == NULL) {
+		ERROR("Out of memory");
+		ret = -1;
+		goto out_free;
+	}
+	rc = snprintf(inmsg, size,
+	              "{\"ociVersion\":\"\",\"id\":\"%s\",\"pid\":%s,\"root\":\"%s\",\"bundle\":\"%s/%s\"}",
+	              name, cpid, rootfs, lxcpath, name);
+	if (rc < 0 || rc >= size) {
+		ERROR("Create json string failed");
+		ret = -1;
+	}
+
+out_free:
+	if (ret) {
+		free(inmsg);
+		inmsg = NULL;
+	}
+	return inmsg;
+}
+
+static char **merge_ocihook_env(char **oldenvs, size_t env_len, size_t *merge_env_len)
+{
+	char **result  = NULL;
+	size_t result_len = env_len;
+	size_t i, j;
+	char *tmpenv = NULL;
+	char *lxc_envs[] = {"LD_LIBRARY_PATH", "PATH", "LXC_CGNS_AWARE", "LXC_PID", "LXC_ROOTFS_MOUNT",
+	                    "LXC_CONFIG_FILE", "LXC_CGROUP_PATH", "LXC_ROOTFS_PATH", "LXC_NAME"
+	                   };
+	char *lxcenv_buf = NULL;
+
+	if (result_len > SIZE_MAX - (sizeof(lxc_envs) / sizeof(char *)) - 1)
+		return NULL;
+	result_len += (sizeof(lxc_envs) / sizeof(char *)) + 1;
+	result = malloc(sizeof(char *) * result_len);
+	if (result == NULL)
+		return NULL;
+	memset(result, 0, sizeof(char *) * result_len);
+
+	for(i = 0; i < env_len; i++) {
+		if (oldenvs[i])
+			result[i] = safe_strdup(oldenvs[i]);
+	}
+
+	for(j = 0; j < (sizeof(lxc_envs) / sizeof(char *)); j++) {
+		size_t env_buf_len = 0;
+		tmpenv = getenv(lxc_envs[j]);
+		if (tmpenv && i < (result_len - 1)) {
+			if (strlen(tmpenv) > (SIZE_MAX - 1 - 1 - strlen(lxc_envs[j]))) {
+				lxc_free_array((void **)result, free);
+				return NULL;
+			}
+			env_buf_len = ((strlen(tmpenv) + 1) + strlen(lxc_envs[j])) + 1;
+			lxcenv_buf = malloc(env_buf_len);
+			if (lxcenv_buf == NULL) {
+				lxc_free_array((void **)result, free);
+				return NULL;
+			}
+			if (snprintf(lxcenv_buf, env_buf_len, "%s=%s", lxc_envs[j], tmpenv) < 0) {
+				free(lxcenv_buf);
+				continue;
+			}
+			result[i++] = lxcenv_buf;
+			lxcenv_buf = NULL;
+		}
+	}
+
+	*merge_env_len = i;
+	return result;
+}
+
+static struct lxc_popen_FILE *lxc_popen_ocihook(const char *commandpath, char **args, int args_len,
+                char **envs, int env_len, const char *instr)
+{
+	int ret;
+	struct lxc_popen_FILE *fp = NULL;
+	int pipe_fds[2] = {-1, -1};
+	int pipe_msg[2] = {-1, -1};
+	pid_t child_pid;
+
+	ret = pipe2(pipe_fds, O_CLOEXEC | O_NONBLOCK);
+	if (ret < 0)
+		return NULL;
+
+	ret = pipe2(pipe_msg, O_CLOEXEC | O_NONBLOCK);
+	if (ret < 0) {
+		ERROR("Pipe msg failure");
+		close(pipe_fds[0]);
+		close(pipe_fds[1]);
+		return NULL;
+	}
+
+	child_pid = fork();
+	if (child_pid < 0)
+		goto on_error;
+
+	if (child_pid == 0) {
+		close(pipe_msg[1]);
+		if (pipe_msg[0] != STDIN_FILENO)
+			dup2(pipe_msg[0], STDIN_FILENO);
+		else {
+			if (fcntl(pipe_msg[0], F_SETFD, 0) != 0) {
+				fprintf(stderr, "Failed to remove FD_CLOEXEC from fd.");
+				exit(127);
+			}
+		}
+		close(pipe_msg[0]);
+
+		close(pipe_fds[0]);
+
+		/* duplicate stdout */
+		if (pipe_fds[1] != STDOUT_FILENO)
+			ret = dup2(pipe_fds[1], STDOUT_FILENO);
+		else
+			ret = fcntl(pipe_fds[1], F_SETFD, 0);
+		if (ret < 0) {
+			close(pipe_fds[1]);
+			_exit(EXIT_FAILURE);
+		}
+
+		/* duplicate stderr */
+		if (pipe_fds[1] != STDERR_FILENO)
+			ret = dup2(pipe_fds[1], STDERR_FILENO);
+		else
+			ret = fcntl(pipe_fds[1], F_SETFD, 0);
+		close(pipe_fds[1]);
+		if (ret < 0)
+			_exit(EXIT_FAILURE);
+
+		if (lxc_check_inherited(NULL, true, NULL, 0) != 0) {
+			fprintf(stderr, "check inherited fd failed");
+			exit(127);
+		}
+
+		/*
+		 * Unblock signals.
+		 * This is the main/only reason
+		 * why we do our lousy popen() emulation.
+		 */
+		{
+			sigset_t mask;
+			sigfillset(&mask);
+			sigprocmask(SIG_UNBLOCK, &mask, NULL);
+		}
+
+		if (env_len > 0)
+			execvpe(commandpath, args, envs);
+		else
+			execvp(commandpath, args);
+		fprintf(stderr, "fork/exec %s: %s", commandpath, strerror(errno));
+		exit(127);
+	}
+
+	/* parent */
+
+	close(pipe_fds[1]);
+	pipe_fds[1] = -1;
+
+	close(pipe_msg[0]);
+	pipe_msg[0]= -1;
+	if (instr) {
+		size_t len = strlen(instr);
+		if (lxc_write_nointr(pipe_msg[1], instr, len) != len) {
+			WARN("Write instr: %s failed", instr);
+		}
+	}
+	close(pipe_msg[1]);
+	pipe_msg[1]= -1;
+
+	fp = calloc(1, sizeof(*fp));
+	if (!fp) {
+		ERROR("Failed to allocate memory");
+		goto on_error;
+	}
+
+	fp->child_pid = child_pid;
+	fp->pipe = pipe_fds[0];
+
+	return fp;
+
+on_error:
+
+	if (pipe_fds[0] >= 0)
+		close(pipe_fds[0]);
+
+	if (pipe_fds[1] >= 0)
+		close(pipe_fds[1]);
+
+	if (pipe_msg[0] >= 0)
+		close(pipe_msg[0]);
+
+	if (pipe_msg[1] >= 0)
+		close(pipe_msg[1]);
+
+	if (fp)
+		free(fp);
+
+	return NULL;
+}
+
+void* wait_ocihook_timeout(void *arg)
+{
+	bool alive = false;
+	struct wait_conf *conf = (struct wait_conf *)arg;
+
+	if (!conf || conf->timeout < 1)
+		goto out;
+
+	sleep(conf->timeout);
+
+	alive = lxc_process_alive(conf->pid, conf->startat);
+
+	if (alive) {
+		ERROR("%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\"",
+		      __FILE__, __LINE__, lxchook_names[conf->which],
+		      (double)conf->timeout);
+
+		lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\".",
+		                        __FILE__, __LINE__, lxchook_names[conf->which],
+		                        (double)conf->timeout);
+
+		if (kill(conf->pid, SIGKILL) && errno != ESRCH) {
+			ERROR("Send kill signal failed");
+			goto out;
+		}
+	}
+
+out:
+	free(conf);
+	return ((void *)0);
+}
+
+static int run_ocihook_buffer(struct oci_hook_conf *oconf, const char *inmsg)
+{
+	struct lxc_popen_FILE *f;
+	char output[LXC_LOG_BUFFER_SIZE] = {0};
+	int ret;
+	pthread_t ptid;
+	int err;
+	struct wait_conf *conf = NULL;
+	pthread_attr_t attr;
+	char *buffer = oconf->ocihook->path;
+	char *err_args_msg = NULL;
+	char *err_envs_msg = NULL;
+	char **hookenvs = NULL;
+	size_t hookenvs_len = 0;
+
+	hookenvs = merge_ocihook_env(oconf->ocihook->env, oconf->ocihook->env_len, &hookenvs_len);
+	if (!hookenvs) {
+		ERROR("Out of memory.");
+		return -1;
+	}
+
+	f = lxc_popen_ocihook(buffer, oconf->ocihook->args, oconf->ocihook->args_len, hookenvs, hookenvs_len, inmsg);
+	lxc_free_array((void **)hookenvs, free);
+	if (!f) {
+		SYSERROR("Failed to popen() %s.", buffer);
+		return -1;
+	}
+
+	conf = malloc(sizeof(struct wait_conf));
+	if (conf == NULL) {
+		SYSERROR("Failed to malloc.");
+		goto on_error;
+	}
+
+	memset(conf, 0x00, sizeof(struct wait_conf));
+
+	conf->pid = f->child_pid;
+	conf->startat = lxc_get_process_startat(conf->pid);
+
+	INFO("hook_conf timeout %d", oconf->ocihook->timeout);
+	if(oconf->ocihook->timeout > 0)
+		conf->timeout = oconf->ocihook->timeout;
+	else {
+		conf->timeout = 30;
+		INFO("Set hook timeout 30s");
+	}
+	conf->errfd = oconf->errfd;
+	conf->which = oconf->which;
+
+	pthread_attr_init(&attr);
+	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+	err = pthread_create(&ptid, &attr, wait_ocihook_timeout, conf);
+	if (err != 0) {
+		ERROR("Create wait timeout thread failed");
+		free(conf);
+		goto on_error;
+	}
+
+	ret = lxc_wait_for_pid_status(f->child_pid);
+
+	lxc_read_nointr(f->pipe, output, sizeof(output) - 1);
+	close(f->pipe);
+	free(f);
+
+	if (ret == -1) {
+		SYSERROR("Script exited with error.");
+		goto print_hook;
+	} else if (WIFEXITED(ret) && WEXITSTATUS(ret) != 0) {
+		ERROR("Script exited with status %d. output: %s", WEXITSTATUS(ret), output);
+		lxc_write_error_message(oconf->errfd, "%s:%d: running %s hook caused \"error running hook: exit status %d, output: %s\".",
+		                        __FILE__, __LINE__,
+		                        (oconf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[oconf->which],
+		                        WEXITSTATUS(ret), output);
+
+		goto print_hook;
+	} else if (WIFSIGNALED(ret)) {
+		ERROR("Script terminated by signal %d.", WTERMSIG(ret));
+		lxc_write_error_message(oconf->errfd, "%s:%d: running %s hook caused \"error running hook: Script terminated by signal %d\".",
+		                        __FILE__, __LINE__,
+		                        (oconf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[oconf->which],
+		                        WTERMSIG(ret));
+
+		goto print_hook;
+	}
+
+	return 0;
+
+on_error:
+	if (f) {
+		if (f->pipe >= 0)
+			close(f->pipe);
+		free(f);
+	}
+
+print_hook:
+	if (oconf->ocihook->args)
+		err_args_msg = lxc_string_join(" ", (const char **)oconf->ocihook->args, false);
+	if (oconf->ocihook->env)
+		err_envs_msg = lxc_string_join(" ", (const char **)oconf->ocihook->env, false);
+	ERROR("Hook script command: \"%s\", args: \"%s\", envs: \"%s\", timeout: %d.",
+	      buffer, err_args_msg ? err_args_msg : "",
+	      err_envs_msg ? err_envs_msg : "", oconf->ocihook->timeout);
+
+	free(err_args_msg);
+	free(err_envs_msg);
+	return -1;
+}
+
+static int run_ocihook_script_argv(const char *name, const char *section,
+                                   struct oci_hook_conf *oconf,
+                                   const char *lxcpath, const char *rootfs)
+{
+	int ret;
+	const char *script = oconf->ocihook->path;
+	char *inmsg = NULL;
+
+	INFO("Executing script \"%s\" for container \"%s\", config section \"%s\".",
+	     script, name, section);
+
+	inmsg = generate_json_str(name, lxcpath, rootfs);
+	if (!inmsg) {
+		return -1;
+	}
+
+	ret = run_ocihook_buffer(oconf, inmsg);
+	free(inmsg);
+	inmsg = NULL;
+	return ret;
+}
+
+static char *get_root_path(const char *path, const char *backend)
+{
+	char *ret = NULL;
+	char *tmp = NULL;
+
+	if (!path) {
+		ret = safe_strdup("/");
+		return ret;
+	}
+	if (!backend) {
+		goto default_out;
+	}
+
+	if (strcmp(backend, "aufs") == 0 ||
+	                strcmp(backend, "overlayfs") == 0 ||
+	                strcmp(backend, "loop") == 0) {
+		tmp = strrchr(path, ':');
+		if (tmp == NULL) {
+			ERROR("Invalid root path format");
+			return NULL;
+		}
+		tmp++;
+		ret = safe_strdup(tmp);
+		return ret;
+	}
+
+default_out:
+	ret = safe_strdup(path);
+	return ret;
+}
+
+static int do_run_oci_hooks(const char *name, const char *lxcpath, struct lxc_conf *lc, int which, int errfd)
+{
+	struct oci_hook_conf work_conf = {0};
+	size_t i;
+	int ret = 0;
+	int nret = 0;
+	char *rootpath = NULL;
+
+	if (!lc) {
+		return -1;
+	}
+	if (!lc->ocihooks) {
+		return 0;
+	}
+
+	rootpath = get_root_path(lc->rootfs.path, lc->rootfs.bdev_type);
+	if (!rootpath) {
+		ERROR("Get container %s rootpath failed.", name);
+		return -1;
+	}
+
+	work_conf.errfd = errfd;
+	work_conf.which = which;
+	switch (which) {
+	case OCI_HOOK_PRESTART:
+		for (i = 0; i < lc->ocihooks->prestart_len; i++) {
+			work_conf.ocihook = lc->ocihooks->prestart[i];
+			ret = run_ocihook_script_argv(name, "lxc", &work_conf, lxcpath, rootpath);
+			if (ret != 0)
+				break;
+		}
+		break;
+	case OCI_HOOK_POSTSTART:
+		for (i = 0; i < lc->ocihooks->poststart_len; i++) {
+			work_conf.ocihook = lc->ocihooks->poststart[i];
+			nret = run_ocihook_script_argv(name, "lxc", &work_conf, lxcpath, rootpath);
+			if (nret != 0)
+				WARN("running poststart hook %zu failed, ContainerId: %s", i, name);
+		}
+		break;
+	case OCI_HOOK_POSTSTOP:
+		for (i = 0; i < lc->ocihooks->poststop_len; i++) {
+			work_conf.ocihook = lc->ocihooks->poststop[i];
+			nret = run_ocihook_script_argv(name, "lxc", &work_conf, lxcpath, rootpath);
+			if (nret != 0)
+				WARN("running poststart hook %zu failed, ContainerId: %s", i, name);
+		}
+		break;
+	default:
+		ret = -1;
+	}
+	if (rootpath)
+		free(rootpath);
+	return ret;
+}
+
+int run_oci_hooks(const char *name, const char *hookname, struct lxc_conf *conf, const char *lxcpath)
+{
+	int which = -1;
+
+	if (strcmp(hookname, "oci-prestart") == 0) {
+		which = OCI_HOOK_PRESTART;
+		if (!lxcpath) {
+			ERROR("oci hook require lxcpath");
+			return -1;
+		}
+		return do_run_oci_hooks(name, lxcpath, conf, which, conf->errpipe[1]);
+	} else if (strcmp(hookname, "oci-poststart") == 0) {
+		which = OCI_HOOK_POSTSTART;
+		if (!lxcpath) {
+			ERROR("oci hook require lxcpath");
+			return -1;
+		}
+		return do_run_oci_hooks(name, lxcpath, conf, which, conf->errpipe[1]);
+	} else if (strcmp(hookname, "oci-poststop") == 0) {
+		which = OCI_HOOK_POSTSTOP;
+		if (!lxcpath) {
+			ERROR("oci hook require lxcpath");
+			return -1;
+		}
+		return do_run_oci_hooks(name, lxcpath, conf, which, conf->errpipe[1]);
+	} else
+		return -1;
+
+	return 0;
+}
 #endif
 
 int lxc_setup(struct lxc_handler *handler)
@@ -4083,6 +4619,12 @@ int lxc_setup(struct lxc_handler *handler)
 	if (ret < 0)
 		return log_error(-1, "Failed to \"/proc\" LSMs");
 
+#ifdef HAVE_ISULAD
+	/* Ask father to run oci prestart hooks and wait for him to finish. */
+	if (lxc_sync_barrier_parent(handler, LXC_SYNC_OCI_PRESTART_HOOK)) {
+		return log_error(-1, "Failed to sync parent to start host hook");
+	}
+#endif
 	ret = lxc_setup_rootfs_switch_root(&lxc_conf->rootfs);
 	if (ret < 0)
 		return log_error(-1, "Failed to pivot root into rootfs");
diff --git a/src/lxc/conf.h b/src/lxc/conf.h
index 22c554d..61c3383 100644
--- a/src/lxc/conf.h
+++ b/src/lxc/conf.h
@@ -23,6 +23,10 @@
 #include "start.h"
 #include "terminal.h"
 
+#ifdef HAVE_ISULAD
+#include "oci_runtime_hooks.h"
+#endif
+
 #if HAVE_SYS_RESOURCE_H
 #include <sys/resource.h>
 #endif
@@ -212,6 +216,11 @@ enum lxchooks {
 	LXCHOOK_CLONE,
 	LXCHOOK_DESTROY,
 	LXCHOOK_START_HOST,
+#ifdef HAVE_ISULAD
+	OCI_HOOK_PRESTART,
+	OCI_HOOK_POSTSTART,
+	OCI_HOOK_POSTSTOP,
+#endif
 	NUM_LXC_HOOKS
 };
 
@@ -433,6 +442,11 @@ struct lxc_conf {
 	} shmount;
 
 #ifdef HAVE_ISULAD
+	/*
+	 * isulad: support oci hook
+	 * */
+	oci_runtime_spec_hooks *ocihooks;
+
 	/* isulad add: init args used to repalce init_cmd*/
 	char **init_argv;
 	size_t init_argc;
@@ -447,6 +461,8 @@ struct lxc_conf {
 
 	char *errmsg; /* record error messages */
 
+	int errpipe[2];//pipdfd for get error message of child or grandchild process.
+
 	char *systemd; //systemd value
 #endif
 
@@ -535,5 +551,6 @@ int lxc_clear_init_args(struct lxc_conf *lxc_conf);
 int lxc_clear_populate_devices(struct lxc_conf *c);
 int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
 int lxc_clear_rootfs_ro_paths(struct lxc_conf *c);
+int run_oci_hooks(const char *name, const char *hookname, struct lxc_conf *conf, const char *lxcpath);
 #endif
 #endif /* __LXC_CONF_H */
diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
index 821cfa1..9b3ab75 100644
--- a/src/lxc/lxccontainer.c
+++ b/src/lxc/lxccontainer.c
@@ -288,6 +288,8 @@ static void lxc_container_free(struct lxc_container *c)
 #ifdef HAVE_ISULAD
 	free(c->exit_fifo);
 	c->exit_fifo = NULL;
+	free(c->ocihookfile);
+	c->ocihookfile = NULL;
 #endif
 
 	free(c);
@@ -632,6 +634,66 @@ static bool load_config_locked(struct lxc_container *c, const char *fname)
 	return true;
 }
 
+#ifdef HAVE_ISULAD
+static bool load_ocihooks_locked(struct lxc_container *c)
+{
+	parser_error err = NULL;
+	oci_runtime_spec_hooks *hooks = NULL;
+
+	if (!c->lxc_conf)
+		c->lxc_conf = lxc_conf_init();
+
+	if (!c->lxc_conf)
+		return false;
+
+	hooks = oci_runtime_spec_hooks_parse_file(c->ocihookfile, NULL, &err);
+	if (!hooks) {
+		fprintf(stderr, "parse oci hooks config failed: %s\n", err);
+		free(err);
+		return true;
+	}
+	c->lxc_conf->ocihooks = hooks;
+
+	if (err)
+		free(err);
+	return true;
+}
+
+/*
+ * isulad: set oci hook file path
+ * */
+static bool set_oci_hook_config_filename(struct lxc_container *c)
+{
+#define OCI_HOOK_JSON_FILE_NAME "ocihooks.json"
+	char *newpath = NULL;
+	int len, ret;
+
+	if (!c->config_path)
+		return false;
+
+	/* $lxc_path + "/" + c->name + "/" + "config" + '\0' */
+	if (strlen(c->config_path) + strlen(c->name) > SIZE_MAX - strlen(OCI_HOOK_JSON_FILE_NAME) - 3)
+		return false;
+	len = strlen(c->config_path) + strlen(c->name) + strlen(OCI_HOOK_JSON_FILE_NAME) + 3;
+
+	newpath = malloc(len);
+	if (newpath == NULL)
+		return false;
+
+	ret = snprintf(newpath, len, "%s/%s/%s", c->config_path, c->name, OCI_HOOK_JSON_FILE_NAME);
+	if (ret < 0 || ret >= len) {
+		fprintf(stderr, "Error printing out config file name\n");
+		free(newpath);
+		return false;
+	}
+
+	free(c->ocihookfile);
+	c->ocihookfile = newpath;
+
+	return true;
+}
+#endif
+
 static bool do_lxcapi_load_config(struct lxc_container *c, const char *alt_file)
 {
 	int lret;
@@ -665,6 +727,11 @@ static bool do_lxcapi_load_config(struct lxc_container *c, const char *alt_file)
 
 	ret = load_config_locked(c, fname);
 
+#ifdef HAVE_ISULAD
+	if (ret && file_exists(c->ocihookfile))
+		ret = load_ocihooks_locked(c);
+#endif
+
 	if (need_disklock)
 		container_disk_unlock(c);
 	else
@@ -5492,6 +5559,40 @@ static bool do_lxcapi_set_exec_terminal_winch(struct lxc_container *c, const cha
 }
 
 WRAP_API_3(bool, lxcapi_set_exec_terminal_winch, const char *, unsigned int, unsigned int)
+
+/* isulad add clean resources */
+static bool do_lxcapi_clean_container_resource(struct lxc_container *c, pid_t pid)
+{
+	int ret;
+
+	if (!c)
+		return false;
+
+	ret = do_lxcapi_clean_resource(c->name, c->config_path, c->lxc_conf, pid);
+	if (ret)
+		ERROR("Failed to clean container %s resource", c->name);
+	return ret == 0;
+
+}
+
+WRAP_API_1(bool, lxcapi_clean_container_resource, pid_t)
+
+/* isulad get coantainer pids */
+static bool do_lxcapi_get_container_pids(struct lxc_container *c, pid_t **pids,size_t *pids_len)
+{
+	int ret;
+
+	if (!c)
+		return false;
+
+	ret = do_lxcapi_get_pids(c->name, c->config_path, c->lxc_conf, pids,pids_len);
+	if (ret)
+		ERROR("Failed to get container %s pids", c->name);
+	return ret == 0;
+
+}
+
+WRAP_API_2(bool, lxcapi_get_container_pids, pid_t **,size_t *)
 #endif
 
 struct lxc_container *lxc_container_new(const char *name, const char *configpath)
@@ -5547,6 +5648,13 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
 		goto err;
 	}
 
+#ifdef HAVE_ISULAD
+	if (!set_oci_hook_config_filename(c)) {
+		fprintf(stderr, "Error allocating oci hooks file pathname\n");
+		goto err;
+	}
+#endif
+
 	if (file_exists(c->configfile) && !lxcapi_load_config(c, NULL)) {
 		fprintf(stderr, "Failed to load config for %s\n", name);
 		goto err;
@@ -5643,6 +5751,8 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
 	c->set_exec_terminal_winch = lxcapi_set_exec_terminal_winch;
 	c->want_disable_pty = lxcapi_want_disable_pty;
 	c->want_open_stdin = lxcapi_want_open_stdin;
+	c->clean_container_resource = lxcapi_clean_container_resource;
+	c->get_container_pids = lxcapi_get_container_pids;
 #endif
 	return c;
 
diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
index de2ee46..f1621f9 100644
--- a/src/lxc/lxccontainer.h
+++ b/src/lxc/lxccontainer.h
@@ -118,6 +118,13 @@ struct lxc_container {
 
 	/*! Whether container wishes to keep stdin active */
 	bool open_stdin;
+
+	/*!
+	 * \private
+	 * isulad: support oci hook from json file
+	 * full path of json file
+	 * */
+	char *ocihookfile;
 #endif
 
 	/*!
@@ -935,6 +942,27 @@ struct lxc_container {
 	 * \return \c true on success, else \c false.
 	 */
 	bool (*want_open_stdin)(struct lxc_container *c, bool state);
+
+	/*! isulad add
+	 * \brief An API call to clean resources of container
+	 *
+	 * \param c Container.
+	 * \param pid Value of container process.
+	 *
+	 * \return \c true on success, else \c false.
+	 */
+	bool (*clean_container_resource) (struct lxc_container *c, pid_t pid);
+
+	/*! isulad add
+	 * \brief An API call to get container pids
+	 *
+	 * \param c Container.
+	 * \param pids Value of container pids.
+	 * \param pids_len Value of container pids len.
+	 * \param pid Value of container pid.
+	 * \return \c true on success, else \c false.
+	 */
+	bool (*get_container_pids)(struct lxc_container *c,pid_t **pids,size_t *pids_len);
 #endif
 };
 
diff --git a/src/lxc/start.c b/src/lxc/start.c
index f6a96b4..4f45776 100644
--- a/src/lxc/start.c
+++ b/src/lxc/start.c
@@ -2239,6 +2239,20 @@ static int lxc_spawn(struct lxc_handler *handler)
 		ERROR("Failed to run lxc.hook.start-host");
 		goto out_delete_net;
 	}
+#ifdef HAVE_ISULAD
+	/* isulad: Run oci prestart hook at here */
+	ret = run_oci_hooks(name, "oci-prestart", conf, lxcpath);
+	if (ret < 0) {
+		ERROR("Failed to run oci prestart hooks");
+		goto out_delete_net;
+	}
+
+       /* Tell the child to continue its initialization. We'll get
+        * LXC_SYNC_POST_OCI_PRESTART_HOOK when it is ready for us to run oci prestart hooks.
+        */
+       if (lxc_sync_barrier_child(handler, LXC_SYNC_POST_OCI_PRESTART_HOOK))
+               goto out_delete_net;
+#endif
 
 	/* Tell the child to complete its initialization and wait for it to exec
 	 * or return an error. (The child will never return
@@ -2282,6 +2296,15 @@ static int lxc_spawn(struct lxc_handler *handler)
 	if (ret < 0)
 		goto out_abort;
 
+#ifdef HAVE_ISULAD
+	/* isulad: Run oci prestart hook at here */
+	ret = run_oci_hooks(name, "oci-poststart", conf, lxcpath);
+	if (ret < 0) {
+		ERROR("Failed to run oci poststart hooks");
+		goto out_abort;
+	}
+#endif
+
 	ret = lxc_set_state(name, handler, RUNNING);
 	if (ret < 0) {
 		ERROR("Failed to set state to \"%s\"", lxc_state2str(RUNNING));
@@ -2592,3 +2615,257 @@ static bool do_destroy_container(struct lxc_handler *handler)
 
 	return storage_destroy(handler->conf);
 }
+
+#ifdef HAVE_ISULAD
+/*isulad: set env for clean resources */
+static int clean_resource_set_env(struct lxc_handler *handler)
+{
+	const char *name = handler->name;
+	struct lxc_conf *conf = handler->conf;
+	char bufstr[PATH_MAX + 1];
+	int i = 0;
+	int j = 0;
+	int len = 2; //set "LXC_PID" and "LXC_CGNS_AWARE"
+
+	if (conf == NULL || conf->ocihooks == NULL || conf->ocihooks->poststop_len == 0) {
+		return 0;
+	}
+
+	if (name) {
+		len++;
+	}
+	if (conf->rcfile) {
+		len++;
+	}
+	if (conf->rootfs.mount) {
+		len++;
+	}
+	if (conf->rootfs.path) {
+		len++;
+	}
+	if (conf->console.path) {
+		len++;
+	}
+	if (conf->console.log_path) {
+		len++;
+	}
+	if (handler->cgroup_ops->container_cgroup) {
+		len++;
+	}
+
+	for (; i < conf->ocihooks->poststop_len; i++) {
+		size_t cap = conf->ocihooks->poststop[i]->env_len;
+		size_t newcap = cap + len + 1;
+		if (lxc_grow_array((void ***)&(conf->ocihooks->poststop[i]->env), &cap, newcap, 1) != 0) {
+			return -1;
+		}
+		j = conf->ocihooks->poststop[i]->env_len;
+		/* Start of environment variable setup for hooks. */
+		if (name) {
+			snprintf(bufstr, PATH_MAX + 1, "LXC_NAME=%s", name);
+			conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr);
+		}
+		if (conf->rcfile) {
+			snprintf(bufstr, PATH_MAX + 1, "LXC_CONFIG_FILE=%s", conf->rcfile);
+			conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr);
+		}
+		if (conf->rootfs.mount) {
+			snprintf(bufstr, PATH_MAX + 1, "LXC_ROOTFS_MOUNT=%s", conf->rootfs.mount);
+			conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr);
+		}
+		if (conf->rootfs.path) {
+			snprintf(bufstr, PATH_MAX + 1, "LXC_ROOTFS_PATH=%s", conf->rootfs.path);
+			conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr);
+		}
+		if (conf->console.path) {
+			snprintf(bufstr, PATH_MAX + 1, "LXC_CONSOLE=%s", conf->console.path);
+			conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr);
+		}
+		if (conf->console.log_path) {
+			snprintf(bufstr, PATH_MAX + 1, "LXC_CONSOLE_LOGPATH=%s", conf->console.log_path);
+			conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr);
+		}
+		conf->ocihooks->poststop[i]->env[j++] = safe_strdup("LXC_CGNS_AWARE=1");
+
+		snprintf(bufstr, PATH_MAX + 1, "LXC_PID=%d", handler->pid);
+		conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr);
+		if (handler->cgroup_ops->container_cgroup) {
+			snprintf(bufstr, PATH_MAX + 1, "LXC_CGROUP_PATH=%s", handler->cgroup_ops->container_cgroup);
+			conf->ocihooks->poststop[i]->env[j++] = safe_strdup(bufstr);
+		}
+		conf->ocihooks->poststop[i]->env_len = j;
+		/* End of environment variable setup for hooks. */
+	}
+	return 0;
+}
+
+/*isulad: init handler for clean */
+static struct lxc_handler *lxc_init_clean_handler(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid)
+{
+	int i;
+	struct lxc_handler *handler;
+
+	handler = malloc(sizeof(*handler));
+	if (handler == NULL)
+		return NULL;
+
+	memset(handler, 0, sizeof(*handler));
+
+	/* Note that am_guest_unpriv() checks the effective uid. We
+	 * probably don't care if we are real root only if we are running
+	 * as root so this should be fine.
+	 */
+	handler->am_root = !am_guest_unpriv();
+	handler->data_sock[0] = handler->data_sock[1] = -1;
+	handler->conf = conf;
+	handler->lxcpath = lxcpath;
+	handler->pinfd = -1;
+	handler->sigfd = -EBADF;
+	handler->init_died = false;
+	handler->pid = pid;
+	handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1;
+	if (handler->conf->reboot == REBOOT_NONE)
+		lxc_list_init(&handler->conf->state_clients);
+
+	for (i = 0; i < LXC_NS_MAX; i++)
+		handler->nsfd[i] = -1;
+
+	handler->name = name;
+	handler->exit_code = -1; /* isulad: record exit code of container */
+
+	handler->cgroup_ops = cgroup_init(conf);
+	if (!handler->cgroup_ops) {
+		ERROR("Failed to initialize cgroup driver");
+		goto on_error;
+	}
+
+	INFO("Container \"%s\" 's clean handler is initialized.", name);
+
+	return handler;
+
+on_error:
+	lxc_free_handler(handler);
+
+	return NULL;
+}
+
+/*isulad: init handler for clean */
+static struct lxc_handler *lxc_init_pids_handler(char *name, char *lxcpath, struct lxc_conf *conf)
+{
+	int i;
+	struct lxc_handler *handler;
+
+	handler = malloc(sizeof(*handler));
+	if (handler == NULL)
+		return NULL;
+
+	memset(handler, 0, sizeof(*handler));
+
+	/* Note that am_guest_unpriv() checks the effective uid. We
+	 * probably don't care if we are real root only if we are running
+	 * as root so this should be fine.
+	 */
+	handler->am_root = !am_guest_unpriv();
+	handler->data_sock[0] = handler->data_sock[1] = -1;
+	handler->conf = conf;
+	handler->lxcpath = lxcpath;
+	handler->pinfd = -1;
+	handler->sigfd = -EBADF;
+	handler->init_died = false;
+	handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1;
+	if (handler->conf->reboot == REBOOT_NONE)
+		lxc_list_init(&handler->conf->state_clients);
+
+	for (i = 0; i < LXC_NS_MAX; i++)
+		handler->nsfd[i] = -1;
+
+	handler->name = name;
+	handler->exit_code = -1; /* isulad: record exit code of container */
+
+	handler->cgroup_ops = cgroup_init(conf);
+	if (!handler->cgroup_ops) {
+		ERROR("Failed to initialize cgroup driver");
+		goto on_error;
+	}
+
+	INFO("Container \"%s\" 's clean handler is initialized.", name);
+
+	return handler;
+
+on_error:
+	lxc_free_handler(handler);
+
+	return NULL;
+}
+
+/*isulad: do_lxcapi_clean_resource */
+int do_lxcapi_clean_resource(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid)
+{
+	int ret = 0;
+	struct lxc_handler *handler = NULL;
+	int retry_count = 0;
+	int max_retry = 10;
+
+	handler = lxc_init_clean_handler(name, lxcpath, conf, pid);
+	if (!handler) {
+		ERROR("Failed to init container %s clean handler", name);
+		ret = -1;
+		goto out;
+	}
+
+	if (clean_resource_set_env(handler) != 0) {
+		ERROR("Failed to set env for poststop hooks");
+		ret = -1;
+		goto out;
+	}
+
+	if (run_oci_hooks(handler->name, "oci-poststop", handler->conf, handler->lxcpath)) {
+		ERROR("Failed to run lxc.hook.post-stop for container \"%s\".", handler->name);
+		ret = -1;
+	}
+
+retry:
+	if (!handler->cgroup_ops->payload_destroy(handler->cgroup_ops, handler)) {
+		TRACE("Trying to kill all subprocess");
+		signal_all_processes(handler);
+		TRACE("Finished kill all subprocess");
+		if (retry_count < max_retry) {
+			usleep(100 * 1000); /* 100 millisecond */
+			retry_count++;
+			goto retry;
+		}
+		SYSERROR("Failed to destroy cgroup path for container: \"%s\"", handler->name);
+		ret = -1;
+	}
+
+out:
+	lxc_free_handler(handler);
+	return ret;
+}
+
+/*isulad: do_lxcapi_get_pids */
+int do_lxcapi_get_pids(char *name, char *lxcpath, struct lxc_conf *conf, pid_t **pids,size_t *pids_len)
+{
+	int ret = 0;
+	struct lxc_handler *handler = NULL;
+	struct cgroup_ops *cg_ops = NULL;
+
+	handler = lxc_init_pids_handler(name, lxcpath, conf);
+	if (!handler) {
+		ERROR("Failed to init container %s clean handler", name);
+		ret = -1;
+		goto out;
+	}
+
+	cg_ops = handler->cgroup_ops;
+	ret = get_all_pids(cg_ops, pids, pids_len);
+	if (ret < 0) {
+		WARN("failed to get all pids");
+	}
+
+out:
+	lxc_free_handler(handler);
+	return ret;
+}
+
+#endif
diff --git a/src/lxc/start.h b/src/lxc/start.h
index 5ea5fe2..4fc3ff7 100644
--- a/src/lxc/start.h
+++ b/src/lxc/start.h
@@ -175,4 +175,12 @@ extern int __lxc_start(struct lxc_handler *, struct lxc_operations *, void *,
 
 extern int resolve_clone_flags(struct lxc_handler *handler);
 
+#ifdef HAVE_ISULAD
+/*isulad: do_lxcapi_clean_resource */
+extern int do_lxcapi_clean_resource(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid);
+
+/*isulad: do_lxcapi_get_pids */
+extern int do_lxcapi_get_pids(char *name, char *lxcpath, struct lxc_conf *conf, pid_t **pids,size_t *pids_len);
+#endif
+
 #endif
diff --git a/src/lxc/sync.h b/src/lxc/sync.h
index ff7a1eb..56c1dfc 100644
--- a/src/lxc/sync.h
+++ b/src/lxc/sync.h
@@ -11,6 +11,10 @@ enum {
 	LXC_SYNC_POST_CONFIGURE,
 	LXC_SYNC_CGROUP,
 	LXC_SYNC_CGROUP_UNSHARE,
+#ifdef HAVE_ISULAD
+	LXC_SYNC_OCI_PRESTART_HOOK,
+	LXC_SYNC_POST_OCI_PRESTART_HOOK,
+#endif
 	LXC_SYNC_CGROUP_LIMITS,
 	LXC_SYNC_READY_START,
 	LXC_SYNC_RESTART,
diff --git a/src/lxc/utils.c b/src/lxc/utils.c
index 27078e2..39413ee 100644
--- a/src/lxc/utils.c
+++ b/src/lxc/utils.c
@@ -2122,4 +2122,42 @@ set_env:
 	return 0;
 }
 
+bool lxc_process_alive(pid_t pid, unsigned long long start_time)
+{
+	int sret = 0;
+	bool alive = true;
+	proc_t *pid_info = NULL;
+	char filename[PATH_MAX] = {0};
+	char sbuf[1024] = {0}; /* bufs for stat */
+
+	sret = kill(pid, 0);
+	if (sret < 0 && errno == ESRCH)
+		return false;
+
+	sret = snprintf(filename, sizeof(filename), "/proc/%d/stat", pid);
+	if (sret < 0 || sret >= sizeof(filename)) {
+		ERROR("Failed to sprintf filename");
+		goto out;
+	}
+
+	if ((lxc_file2str(filename, sbuf, sizeof(sbuf))) == -1) {
+		ERROR("Failed to read pidfile %s", filename);
+		alive = false;
+		goto out;
+	}
+
+	pid_info = lxc_stat2proc(sbuf);
+	if (!pid_info) {
+		ERROR("Failed to get proc stat info");
+		alive = false;
+		goto out;
+	}
+
+	if (start_time != pid_info->start_time)
+		alive = false;
+out:
+	free(pid_info);
+	return alive;
+}
+
 #endif
diff --git a/src/lxc/utils.h b/src/lxc/utils.h
index 36c458e..a213ba7 100644
--- a/src/lxc/utils.h
+++ b/src/lxc/utils.h
@@ -322,6 +322,8 @@ extern int lxc_file2str(const char *filename, char ret[], int cap);
 extern int unsigned long long lxc_get_process_startat(pid_t pid);
 // set env home in container
 extern int lxc_setup_env_home(uid_t uid);
+
+extern bool lxc_process_alive(pid_t pid, unsigned long long start_time);
 #endif
 
 #endif /* __LXC_UTILS_H */
-- 
1.8.3.1