!18 lxc: update lxc to 4.0.1

Merge pull request !18 from lifeng_isula/4.0.1

0001-iSulad-add-HAVE_ISULAD-macro.patch

@@ -0,0 +1,50 @@
+From 49f7dc89e5ae690a0b81570a81321b1593aeb994 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Sat, 11 Apr 2020 15:43:38 +0800
+Subject: [PATCH 01/49] iSulad: add HAVE_ISULAD macro
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ configure.ac        | 11 +++++++++++
+ src/lxc/Makefile.am |  3 +++
+ 2 files changed, 14 insertions(+)
+
+diff --git a/configure.ac b/configure.ac
+index 90a4bd4..5f386d9 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -791,6 +791,17 @@ else
+ 	AC_MSG_RESULT([no])
+ fi
+ 
++AC_MSG_CHECKING([Whether adapt to iSulad])
++AC_ARG_ENABLE([isulad],
++	[AC_HELP_STRING([--enable-isulad], [enable adapt to iSulad [default=yes]])],
++	[adapt_isulad=$enableval], [adapt_isulad=yes])
++AM_CONDITIONAL([HAVE_ISULAD], [test "x$adapt_isulad" = "xyes"])
++if test "x$adapt_isulad" = "xyes"; then
++	AC_DEFINE([HAVE_ISULAD], 1, [adapt to iSulad])
++	AC_MSG_RESULT([yes])
++else
++	AC_MSG_RESULT([no])
++fi
+ # Files requiring some variable expansion
+ AC_CONFIG_FILES([
+ 	Makefile
+diff --git a/src/lxc/Makefile.am b/src/lxc/Makefile.am
+index c374c2d..e7fc844 100644
+--- a/src/lxc/Makefile.am
++++ b/src/lxc/Makefile.am
+@@ -212,6 +212,9 @@ AM_CFLAGS = -DLXCROOTFSMOUNT=\"$(LXCROOTFSMOUNT)\" \
+ 	    -I $(top_srcdir)/src/lxc/storage \
+ 	    -I $(top_srcdir)/src/lxc/cgroups
+ 
++if HAVE_ISULAD
++AM_CFLAGS += -DHAVE_ISULAD
++endif
+ if ENABLE_APPARMOR
+ AM_CFLAGS += -DHAVE_APPARMOR
+ endif
+-- 
+1.8.3.1
+

0002-confile-add-lxc.isulad.init.args-config-interface.patch

@@ -1,31 +1,43 @@
-From dd7c0b3cc5f4ce91f81dcb4f02bc8c4aaa023024 Mon Sep 17 00:00:00 2001
+From 549a0a959b84a483d9f733cf7a157900f4c889c4 Mon Sep 17 00:00:00 2001
 From: LiFeng <lifeng68@huawei.com>
-Date: Thu, 10 Jan 2019 06:54:37 -0500
+Date: Sat, 11 Apr 2020 16:16:15 +0800
-Subject: [PATCH 001/140] confile: add lxc.isulad.init.args config interface
+Subject: [PATCH 02/49] confile: add lxc.isulad.init.args config interface
 
 lxc.isulad.init.args config interface is used to specify the args for
 the container.
 
 Signed-off-by: LiFeng <lifeng68@huawei.com>
 ---
- src/lxc/conf.c         | 13 +++++++++++
+ src/lxc/conf.c         | 17 ++++++++++++++
- src/lxc/conf.h         |  8 +++++++
+ src/lxc/conf.h         | 11 ++++++++-
- src/lxc/confile.c      | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++
+ src/lxc/confile.c      | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++
- src/lxc/lxccontainer.c | 30 +++++++++++++++++++++++++
+ src/lxc/lxccontainer.c | 33 +++++++++++++++++++++++++++
- 4 files changed, 112 insertions(+)
+ 4 files changed, 122 insertions(+), 1 deletion(-)
 
 diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index d95bc4c..f20d629 100644
+index 2f6be9f..62a6979 100644
 --- a/src/lxc/conf.c
 +++ b/src/lxc/conf.c
-@@ -4014,6 +4014,18 @@ void lxc_clear_includes(struct lxc_conf *conf)
+@@ -3835,6 +3835,9 @@ void lxc_conf_free(struct lxc_conf *conf)
- 	}
+ 	free(conf->cgroup_meta.controllers);
+ 	free(conf->shmount.path_host);
+ 	free(conf->shmount.path_cont);
++#ifdef HAVE_ISULAD
++	lxc_clear_init_args(conf);
++#endif
+ 	free(conf);
  }
  
+@@ -4645,3 +4648,17 @@ struct lxc_list *sort_cgroup_settings(struct lxc_list *cgroup_settings)
+ 
+ 	return result;
+ }
++
++#ifdef HAVE_ISULAD
 +/*isulad clear init args*/
 +int lxc_clear_init_args(struct lxc_conf *lxc_conf)
 +{
-+	int i;
++	size_t i;
 +
 +	for (i = 0; i < lxc_conf->init_argc; i++)
 +		free(lxc_conf->init_argv[i]);
@@ -33,72 +45,65 @@ index d95bc4c..f20d629 100644
 +
 +	return 0;
 +}
-+
++#endif
- void lxc_conf_free(struct lxc_conf *conf)
- {
- 	if (!conf)
-@@ -4057,6 +4069,7 @@ void lxc_conf_free(struct lxc_conf *conf)
- 	lxc_clear_limits(conf, "lxc.prlimit");
- 	lxc_clear_sysctls(conf, "lxc.sysctl");
- 	lxc_clear_procs(conf, "lxc.proc");
-+        lxc_clear_init_args(conf);
- 	free(conf->cgroup_meta.dir);
- 	free(conf->cgroup_meta.controllers);
- 	free(conf);
 diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index 41f67cf..95c3027 100644
+index 64885c3..8a198e4 100644
 --- a/src/lxc/conf.h
 +++ b/src/lxc/conf.h
-@@ -376,6 +376,10 @@ struct lxc_conf {
+@@ -398,6 +398,13 @@ struct lxc_conf {
- 
+ 		/* Absolute path (in the container) to the shared mount point */
- 	/* procs */
+ 		char *path_cont;
- 	struct lxc_list procs;
+ 	} shmount;
 +
-+    /* isulad add: init args used to repalce init_cmd*/
++#ifdef HAVE_ISULAD
++	/* isulad add: init args used to repalce init_cmd*/
 +	char **init_argv;
 +	size_t init_argc;
++#endif
++
  };
  
  extern int write_id_mapping(enum idtype idtype, pid_t pid, const char *buf,
-@@ -442,4 +446,8 @@ extern int lxc_clear_sysctls(struct lxc_conf *c, const char *key);
+@@ -470,5 +477,7 @@ extern int lxc_clear_namespace(struct lxc_conf *c);
- extern int setup_proc_filesystem(struct lxc_list *procs, pid_t pid);
+ extern int userns_exec_minimal(const struct lxc_conf *conf,
- extern int lxc_clear_procs(struct lxc_conf *c, const char *key);
+ 			       int (*fn_parent)(void *), void *fn_parent_data,
- 
+ 			       int (*fn_child)(void *), void *fn_child_data);
-+/* isulad add begin */
+-
++#ifdef HAVE_ISULAD
 +int lxc_clear_init_args(struct lxc_conf *lxc_conf);
-+/* isulad add end */
++#endif
-+
  #endif /* __LXC_CONF_H */
 diff --git a/src/lxc/confile.c b/src/lxc/confile.c
-index 05c6823..7297b35 100644
+index 0ca577f..e535beb 100644
 --- a/src/lxc/confile.c
 +++ b/src/lxc/confile.c
-@@ -150,6 +150,10 @@ lxc_config_define(tty_dir);
+@@ -147,6 +147,9 @@ lxc_config_define(tty_dir);
  lxc_config_define(uts_name);
  lxc_config_define(sysctl);
  lxc_config_define(proc);
-+/*isulad add begin*/
++#ifdef HAVE_ISULAD
 +lxc_config_define(init_args);
-+/*isulad add end*/
++#endif
-+
  
- static struct lxc_config_t config_jump_table[] = {
+ /*
- 	{ "lxc.arch",                      set_config_personality,                 get_config_personality,                 clr_config_personality,               },
+  * Important Note:
-@@ -234,6 +238,10 @@ static struct lxc_config_t config_jump_table[] = {
+@@ -259,6 +262,10 @@ static struct lxc_config_t config_jump_table[] = {
  	{ "lxc.uts.name",                  set_config_uts_name,                    get_config_uts_name,                    clr_config_uts_name,                  },
  	{ "lxc.sysctl",                    set_config_sysctl,                      get_config_sysctl,                      clr_config_sysctl,                    },
  	{ "lxc.proc",                      set_config_proc,                        get_config_proc,                        clr_config_proc,                      },
++#ifdef HAVE_ISULAD
++	{ "lxc.isulad.init.args",          set_config_init_args,                   get_config_init_args,                   clr_config_init_args,                 },
 +
-+        /*isulad add begin*/
++#endif
-+        { "lxc.isulad.init.args",          set_config_init_args,                   get_config_init_args,                   clr_config_init_args,                 },
-+        /*isulad add end*/
  };
  
  static const size_t config_jump_table_size = sizeof(config_jump_table) / sizeof(struct lxc_config_t);
-@@ -2184,6 +2192,33 @@ static int set_config_namespace_share(const char *key, const char *value,
+@@ -6094,3 +6101,58 @@ int lxc_list_net(struct lxc_conf *c, const char *key, char *retv, int inlen)
- 	return set_config_string_item(&lxc_conf->ns_share[ns_idx], value);
- }
  
+ 	return fulllen;
+ }
++
++#ifdef HAVE_ISULAD
 +/* isulad: set config for init args */
 +static int set_config_init_args(const char *key, const char *value,
 +				 struct lxc_conf *lxc_conf, void *data)
@@ -126,13 +131,6 @@ index 05c6823..7297b35 100644
 +	return 0;
 +}
 +
- struct parse_line_conf {
- 	struct lxc_conf *conf;
- 	bool from_include;
-@@ -3716,6 +3751,25 @@ static int get_config_namespace_share(const char *key, char *retv, int inlen,
- 	return fulllen;
- }
- 
 +/* isulad: get config init args */
 +static int get_config_init_args(const char *key, char *retv, int inlen,
 +				  struct lxc_conf *c, void *data)
@@ -152,31 +150,22 @@ index 05c6823..7297b35 100644
 +	return fulllen;
 +}
 +
- /* Callbacks to clear config items. */
- static inline int clr_config_personality(const char *key, struct lxc_conf *c,
- 					 void *data)
-@@ -4520,6 +4574,13 @@ static int clr_config_net_ipv6_address(const char *key,
- 	return 0;
- }
- 
 +/* isulad: clr config init args*/
 +static inline int clr_config_init_args(const char *key, struct lxc_conf *c,
 +				   void *data)
 +{
 +	return lxc_clear_init_args(c);
 +}
-+
++#endif
- static int get_config_net_nic(const char *key, char *retv, int inlen,
- 			      struct lxc_conf *c, void *data)
- {
 diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index ad70886..b4cacce 100644
+index 487d838..f4462fd 100644
 --- a/src/lxc/lxccontainer.c
 +++ b/src/lxc/lxccontainer.c
-@@ -847,6 +847,31 @@ static bool wait_on_daemonized_start(struct lxc_handler *handler, int pid)
+@@ -857,6 +857,33 @@ static bool wait_on_daemonized_start(struct lxc_handler *handler, int pid)
  	return true;
  }
  
++#ifdef HAVE_ISULAD
 +/* isulad: use init argv as init cmd */
 +static char **use_init_args(char **init_argv, size_t init_args)
 +{
@@ -201,18 +190,20 @@ index ad70886..b4cacce 100644
 +	}
 +	return argv;
 +}
++#endif
 +
  static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const argv[])
  {
  	int ret;
-@@ -903,6 +928,11 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+@@ -914,6 +941,12 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
  			argv = init_cmd = split_init_cmd(conf->init_cmd);
  	}
  
-+    /* isulad: use init argv as init cmd */
++#ifdef HAVE_ISULAD
 +	if (!argv) {
 +		argv = init_cmd = use_init_args(conf->init_argv, conf->init_argc);
 +	}
++#endif
 +
  	/* ... otherwise use default_args. */
  	if (!argv) {

0002-namespace-add-support-share-namespace-by-path.patch

@@ -1,35 +0,0 @@
-From bcc0965a02a571c39713056536d63f3378dcba3a Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Thu, 10 Jan 2019 08:42:19 -0500
-Subject: [PATCH 002/140] namespace: add support share namespace by path
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/confile_utils.c | 11 +++++++++++
- 1 file changed, 11 insertions(+)
-
-diff --git a/src/lxc/confile_utils.c b/src/lxc/confile_utils.c
-index 7280463..9049ce8 100644
---- a/src/lxc/confile_utils.c
-+++ b/src/lxc/confile_utils.c
-@@ -789,6 +789,17 @@ int lxc_inherit_namespace(const char *lxcname_or_pid, const char *lxcpath,
- 	int fd, pid;
- 	char *dup, *lastslash;
- 
-+	/* isulad: add support share namespace by path.
-+	 * e.g. "lxc.namespace.share.net = /proc/PID/ns/net or /var/run/netns/net"
-+	 */
-+	if (file_exists(lxcname_or_pid) && !dir_exists(lxcname_or_pid)) {
-+		fd = open(lxcname_or_pid, O_RDONLY | O_CLOEXEC);
-+		if (fd < 0)
-+			return -EINVAL;
-+
-+		return fd;
-+	}
-+
- 	lastslash = strrchr(lxcname_or_pid, '/');
- 	if (lastslash) {
- 		dup = strdup(lxcname_or_pid);
--- 
-1.8.3.1
-

0004-confile-add-support-umask.patch

@@ -0,0 +1,132 @@
+From e1bf4afdac0f4e1c19ad24c7c9fb915ce72906ed Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Sat, 11 Apr 2020 17:24:47 +0800
+Subject: [PATCH 04/49] confile: add support umask
+
+lxc.isulad.umask=normal make the container umask to 0022
+lxc.isulad.umask=secure make the container umask to 0027 (default)
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/attach.c  |  5 +++++
+ src/lxc/conf.c    |  6 ++++++
+ src/lxc/conf.h    |  1 +
+ src/lxc/confile.c | 38 ++++++++++++++++++++++++++++++++++++++
+ 4 files changed, 50 insertions(+)
+
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index 406b8ec..56d62ed 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -659,6 +659,11 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 			 (options->attach_flags & LXC_ATTACH_LSM) &&
+ 			 init_ctx->lsm_label;
+ 
++#ifdef HAVE_ISULAD
++		/*isulad: set system umask */
++	umask(init_ctx->container->lxc_conf->umask);
++#endif
++
+ 	/* A description of the purpose of this functionality is provided in the
+ 	 * lxc-attach(1) manual page. We have to remount here and not in the
+ 	 * parent process, otherwise /proc may not properly reflect the new pid
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index e9c0a37..e3fce51 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -2567,6 +2567,7 @@ struct lxc_conf *lxc_conf_init(void)
+ #ifdef HAVE_ISULAD
+ 	/* isulad add begin */
+ 	lxc_list_init(&new->populate_devs);
++	new->umask = 0027; /*default umask 0027*/
+ #endif
+ 
+ 	return new;
+@@ -3522,6 +3523,11 @@ int lxc_setup(struct lxc_handler *handler)
+ 	if (ret < 0)
+ 		return -1;
+ 
++#ifdef HAVE_ISULAD
++	/*isulad: set system umask */
++	umask(lxc_conf->umask);
++#endif
++
+ 	ret = setup_personality(lxc_conf->personality);
+ 	if (ret < 0)
+ 		return log_error(-1, "Failed to set personality");
+diff --git a/src/lxc/conf.h b/src/lxc/conf.h
+index 452458c..7ed3cd0 100644
+--- a/src/lxc/conf.h
++++ b/src/lxc/conf.h
+@@ -427,6 +427,7 @@ struct lxc_conf {
+ 
+ 	/* populate devices*/
+ 	struct lxc_list populate_devs;
++	mode_t umask;  //umask value
+ #endif
+ 
+ };
+diff --git a/src/lxc/confile.c b/src/lxc/confile.c
+index f0772f9..2df269a 100644
+--- a/src/lxc/confile.c
++++ b/src/lxc/confile.c
+@@ -150,6 +150,7 @@ lxc_config_define(proc);
+ #ifdef HAVE_ISULAD
+ lxc_config_define(init_args);
+ lxc_config_define(populate_device);
++lxc_config_define(umask);
+ #endif
+ 
+ /*
+@@ -266,6 +267,7 @@ static struct lxc_config_t config_jump_table[] = {
+ #ifdef HAVE_ISULAD
+ 	{ "lxc.isulad.init.args",          set_config_init_args,                   get_config_init_args,                   clr_config_init_args,                 },
+ 	{ "lxc.isulad.populate.device",    set_config_populate_device,             get_config_populate_device,             clr_config_populate_device,           },
++	{ "lxc.isulad.umask",              set_config_umask,                       get_config_umask,                       clr_config_umask,                     },
+ #endif
+ };
+ 
+@@ -6273,4 +6275,40 @@ static inline int clr_config_populate_device(const char *key, struct lxc_conf *c
+ 	return lxc_clear_populate_devices(c);
+ }
+ 
++/* isulad: set config for umask */
++static int set_config_umask(const char *key, const char *value,
++			      struct lxc_conf *lxc_conf, void *data)
++{
++	if (lxc_config_value_empty(value)) {
++		ERROR("Empty umask");
++		return -1;
++	}
++
++	if (strcmp(value, "normal") == 0) {
++		lxc_conf->umask = 0022;
++		return 0;
++	} else if (strcmp(value, "secure") == 0) {
++		lxc_conf->umask = 0027;
++		return 0;
++	} else {
++		ERROR("Invalid native umask: %s", value);
++		return -1;
++	}
++}
++
++/* isulad add: get umask value*/
++static int get_config_umask(const char *key, char *retv, int inlen,
++			      struct lxc_conf *c, void *data)
++{
++	return lxc_get_conf_size_t(c, retv, inlen, c->umask);
++}
++
++/* isulad add: clear umask value */
++static inline int clr_config_umask(const char *key, struct lxc_conf *c,
++				     void *data)
++{
++	c->umask = 0027;
++	return 0;
++}
++
+ #endif
+-- 
+1.8.3.1
+

0004-support-isulad-fifo-log.patch

@@ -1,96 +0,0 @@
-From f83ae83808419f6742265b8bafc3441fdca65cbb Mon Sep 17 00:00:00 2001
-From: liuhao <liuhao27@huawei.com>
-Date: Fri, 11 Jan 2019 16:11:34 +0800
-Subject: [PATCH 004/140] support isulad fifo log
-
-support isulad fifo log in lxc3.0
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/log.c | 42 ++++++++++++++++++++++++++++++++++++++++--
- 1 file changed, 40 insertions(+), 2 deletions(-)
-
-diff --git a/src/lxc/log.c b/src/lxc/log.c
-index 1e0cc6a..4e74459 100644
---- a/src/lxc/log.c
-+++ b/src/lxc/log.c
-@@ -68,6 +68,7 @@ static int syslog_enable = 0;
- int lxc_quiet_specified;
- int lxc_log_use_global_fd;
- static int lxc_loglevel_specified;
-+static bool isulad_use_log_fifo_flag = false;
- 
- static char log_prefix[LXC_LOG_PREFIX_SIZE] = "lxc";
- static char *log_fname = NULL;
-@@ -138,6 +139,37 @@ static char *lxc_log_get_va_msg(struct lxc_log_event *event)
- 	return msg;
- }
- 
-+static const char *isulad_use_log_fifo(const char *file)
-+{
-+#define ISULAD_FIFO_PREFIX "fifo:"
-+
-+	if (strncmp(file, ISULAD_FIFO_PREFIX, strlen(ISULAD_FIFO_PREFIX)) == 0) {
-+		isulad_use_log_fifo_flag = true;
-+		return (file + strlen(ISULAD_FIFO_PREFIX));
-+	}
-+	return file;
-+}
-+
-+static int isulad_open_fifo(const char *file_path)
-+{
-+#define LOG_FIFO_SIZE (1024 * 1024)
-+	int fd = -1;
-+
-+	fd = lxc_unpriv(open(file_path, O_RDWR | O_NONBLOCK | O_CLOEXEC, 0640));
-+	if (fd == -1) {
-+		fprintf(stderr, "Open fifo %s failed: %s\n", file_path, strerror(errno));
-+		return -1;
-+	}
-+
-+	if (fcntl(fd, F_SETPIPE_SZ, LOG_FIFO_SIZE) == -1) {
-+		printf("Set fifo buffer size failed: %s", strerror(errno));
-+		close(fd);
-+		return -1;
-+	}
-+
-+	return fd;
-+}
-+
- /*---------------------------------------------------------------------------*/
- static int log_append_syslog(const struct lxc_log_appender *appender,
- 			     struct lxc_log_event *event)
-@@ -609,7 +641,11 @@ static int __lxc_log_set_file(const char *fname, int create_dirs)
- 		return -1;
- 	}
- 
--	lxc_log_fd = log_open(fname);
-+	if (isulad_use_log_fifo_flag) {
-+		lxc_log_fd = isulad_open_fifo(fname);
-+	} else {
-+		lxc_log_fd = log_open(fname);
-+	}
- 	if (lxc_log_fd == -1)
- 		return -1;
- 
-@@ -642,6 +678,7 @@ int lxc_log_init(struct lxc_log *log)
- {
- 	int ret;
- 	int lxc_priority = LXC_LOG_LEVEL_ERROR;
-+	const char *tmp_log_fname;
- 
- 	if (!log)
- 		return -1;
-@@ -673,7 +710,8 @@ int lxc_log_init(struct lxc_log *log)
- 		if (strcmp(log->file, "none") == 0)
- 			return 0;
- 
--		ret = __lxc_log_set_file(log->file, 1);
-+		tmp_log_fname = isulad_use_log_fifo(log->file);
-+		ret = __lxc_log_set_file(tmp_log_fname, 1);
- 		if (ret < 0) {
- 			ERROR("Failed to enable logfile");
- 			return -1;
--- 
-1.8.3.1
-

0005-auto-mount-cgroup-sys-and-proc.patch

@@ -1,80 +0,0 @@
-From 7cb675179d84838d751312fb675c442325413270 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Thu, 10 Jan 2019 20:40:19 +0800
-Subject: [PATCH 005/140] auto mount cgroup sys and proc
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/cgroups/cgfsng.c |  8 ++++++--
- src/lxc/conf.c           | 15 ++++++++++++---
- 2 files changed, 18 insertions(+), 5 deletions(-)
-
-diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index acc6c30..aff2b5e 100644
---- a/src/lxc/cgroups/cgfsng.c
-+++ b/src/lxc/cgroups/cgfsng.c
-@@ -1648,6 +1648,10 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
- 
- 	/* Mount tmpfs */
- 	tmpfspath = must_make_path(root, "/sys/fs/cgroup", NULL);
-+	if (mkdir_p(tmpfspath, 0755) < 0) {
-+		ERROR("Failed to create directory: %s", tmpfspath);
-+		goto on_error;
-+	}
- 	ret = safe_mount(NULL, tmpfspath, "tmpfs",
- 			 MS_NOSUID | MS_NODEV | MS_NOEXEC | MS_RELATIME,
- 			 "size=10240k,mode=755", root);
-@@ -1700,8 +1704,8 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
- 			continue;
- 		}
- 
--		path2 = must_make_path(controllerpath, h->container_base_path,
--				       ops->container_cgroup, NULL);
-+		// Ignore ops->container_cgroup so we will not see directory lxc after /sys/fs/cgroup/xxx in container
-+		path2 = must_make_path(controllerpath, h->container_base_path, NULL);
- 		ret = mkdir_p(path2, 0755);
- 		if (ret < 0) {
- 			free(controllerpath);
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 20b7aba..18753d1 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -670,8 +670,8 @@ static int lxc_mount_auto_mounts(struct lxc_conf *conf, int flags, struct lxc_ha
- 		{ LXC_AUTO_PROC_MASK, LXC_AUTO_PROC_MIXED, "%r/proc/sysrq-trigger",                          "%r/proc/sysrq-trigger",      NULL,    MS_BIND,                                         NULL },
- 		{ LXC_AUTO_PROC_MASK, LXC_AUTO_PROC_MIXED, NULL,                                             "%r/proc/sysrq-trigger",      NULL,    MS_REMOUNT|MS_BIND|MS_RDONLY,                    NULL },
- 		{ LXC_AUTO_PROC_MASK, LXC_AUTO_PROC_RW,    "proc",                                           "%r/proc",                    "proc",  MS_NODEV|MS_NOEXEC|MS_NOSUID,                    NULL },
--		{ LXC_AUTO_SYS_MASK,  LXC_AUTO_SYS_RW,     "sysfs",                                          "%r/sys",                     "sysfs", 0,                                               NULL },
--		{ LXC_AUTO_SYS_MASK,  LXC_AUTO_SYS_RO,     "sysfs",                                          "%r/sys",                     "sysfs", MS_RDONLY,                                       NULL },
-+		{ LXC_AUTO_SYS_MASK,  LXC_AUTO_SYS_RW,     "sysfs",                                          "%r/sys",                     "sysfs", MS_NODEV|MS_NOEXEC|MS_NOSUID,                                               NULL },
-+		{ LXC_AUTO_SYS_MASK,  LXC_AUTO_SYS_RO,     "sysfs",                                          "%r/sys",                     "sysfs", MS_RDONLY|MS_NODEV|MS_NOEXEC|MS_NOSUID,                                       NULL },
- 		{ LXC_AUTO_SYS_MASK,  LXC_AUTO_SYS_MIXED,  "sysfs",                                          "%r/sys",                     "sysfs", MS_NODEV|MS_NOEXEC|MS_NOSUID,                    NULL },
- 		{ LXC_AUTO_SYS_MASK,  LXC_AUTO_SYS_MIXED,  NULL,                                             "%r/sys",                     NULL,    MS_REMOUNT|MS_BIND|MS_RDONLY,                    NULL },
- 		{ LXC_AUTO_SYS_MASK,  LXC_AUTO_SYS_MIXED,  "sysfs",                                          "%r/sys/devices/virtual/net", "sysfs", 0,                                               NULL },
-@@ -710,6 +710,15 @@ static int lxc_mount_auto_mounts(struct lxc_conf *conf, int flags, struct lxc_ha
- 			return -1;
- 		}
- 
-+		if (mkdir_p(destination, 0755) < 0) {
-+			SYSERROR("Failed to create mount target '%s'", destination);
-+			saved_errno = errno;
-+			free(source);
-+			free(destination);
-+			errno = saved_errno;
-+			return -1;
-+		}
-+
- 		mflags = add_required_remount_flags(source, destination,
- 						    default_mounts[i].flags);
- 		r = safe_mount(source, destination, default_mounts[i].fstype,
-@@ -717,7 +726,7 @@ static int lxc_mount_auto_mounts(struct lxc_conf *conf, int flags, struct lxc_ha
- 			       conf->rootfs.path ? conf->rootfs.mount : NULL);
- 		saved_errno = errno;
- 		if (r < 0 && errno == ENOENT) {
--			INFO("Mount source or target for \"%s\" on \"%s\" does "
-+			INFO("Mount source for \"%s\" on \"%s\" does "
- 			     "not exist. Skipping", source, destination);
- 			r = 0;
- 		} else if (r < 0) {
--- 
-1.8.3.1
-

0006-conf.c-fix-bug-when-set-no-ro-mount-mount-propagatio.patch

@@ -1,39 +0,0 @@
-From 80fcf03d746825fcf809f7b415782a410d9d87d3 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Fri, 11 Jan 2019 16:55:01 +0800
-Subject: [PATCH 006/140] conf.c: fix bug when set no ro mount, mount
- propagation will be skipped
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c | 7 +++----
- 1 file changed, 3 insertions(+), 4 deletions(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 18753d1..37a5ff7 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -2087,6 +2087,9 @@ static int mount_entry(const char *fsname, const char *target,
- 		}
- 	}
- 
-+#ifdef HAVE_STATVFS
-+	skipremount:
-+#endif
- 	if (pflags) {
- 		ret = mount(NULL, target, NULL, pflags, NULL);
- 		if (ret < 0) {
-@@ -2103,10 +2106,6 @@ static int mount_entry(const char *fsname, const char *target,
- 		DEBUG("Changed mount propagation for \"%s\"", target);
- 	}
- 
--
--#ifdef HAVE_STATVFS
--skipremount:
--#endif
- 	DEBUG("Mounted \"%s\" on \"%s\" with filesystem type \"%s\"",
- 	      srcpath ? srcpath : "(null)", target, fstype);
- 
--- 
-1.8.3.1
-

0006-modify-container-exit-code-and-stop-signal.patch

@@ -0,0 +1,112 @@
+From ef7c687828efd488369ff860523aa15a42c72587 Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Mon, 13 Apr 2020 04:58:46 -0400
+Subject: [PATCH 06/49] modify container exit code and stop signal
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/lxccontainer.c | 10 ++++++++--
+ src/lxc/start.c        | 28 ++++++++++++++++++++++++++++
+ 2 files changed, 36 insertions(+), 2 deletions(-)
+
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index f4462fd..a617172 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -2101,7 +2101,12 @@ WRAP_API_1(bool, lxcapi_reboot2, int)
+ static bool do_lxcapi_shutdown(struct lxc_container *c, int timeout)
+ {
+ 	__do_close int pidfd = -EBADF, state_client_fd = -EBADF;
++#ifdef HAVE_ISULAD
++    // isulad: keep default signal the same as docker
++	int haltsignal = SIGTERM;
++#else
+ 	int haltsignal = SIGPWR;
++#endif
+ 	pid_t pid = -1;
+ 	lxc_state_t states[MAX_STATE] = {0};
+ 	int killret, ret;
+@@ -2117,12 +2122,13 @@ static bool do_lxcapi_shutdown(struct lxc_container *c, int timeout)
+ 	if (pid <= 0)
+ 		return true;
+ 
+-	/* Detect whether we should send SIGRTMIN + 3 (e.g. systemd). */
+ 	if (c->lxc_conf && c->lxc_conf->haltsignal)
+ 		haltsignal = c->lxc_conf->haltsignal;
++#ifndef HAVE_ISULAD
++	/* Detect whether we should send SIGRTMIN + 3 (e.g. systemd). */
+ 	else if (task_blocks_signal(pid, (SIGRTMIN + 3)))
+ 		haltsignal = (SIGRTMIN + 3);
+-
++#endif
+ 
+ 	/*
+ 	 * Add a new state client before sending the shutdown signal so
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index a25bd04..5dcf828 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -1886,10 +1886,18 @@ out_sync_fini:
+ 	return -1;
+ }
+ 
++#ifdef HAVE_ISULAD
++// isulad: send '128 + signal' if container is killed by signal.
++#define EXIT_SIGNAL_OFFSET 128
++#endif
++
+ int __lxc_start(struct lxc_handler *handler, struct lxc_operations *ops,
+ 		void *data, const char *lxcpath, bool daemonize, int *error_num)
+ {
+ 	int ret, status;
++#ifdef HAVE_ISULAD
++    int exit_code;
++#endif
+ 	const char *name = handler->name;
+ 	struct lxc_conf *conf = handler->conf;
+ 	struct cgroup_ops *cgroup_ops;
+@@ -1976,6 +1984,21 @@ int __lxc_start(struct lxc_handler *handler, struct lxc_operations *ops,
+ 	 * reboot. This should mean it was an lxc-execute which simply exited.
+ 	 * In any case, treat it as a 'halt'.
+ 	 */
++#ifdef HAVE_ISULAD
++	// isulad: recored log for container init exit
++	if (WIFSIGNALED(status)) {
++		int signal = WTERMSIG(status);
++		signal = WTERMSIG(status);
++		exit_code = EXIT_SIGNAL_OFFSET + signal;
++		ERROR("Container \"%s\" init exited with signal %d", name, signal);
++	} else if (WIFEXITED(status)) {
++		exit_code = WEXITSTATUS(status);
++		ERROR("Container \"%s\" init exited with status %d", name, exit_code);
++	} else {
++		exit_code = -1;
++		ERROR("Container \"%s\" init exited with unknown status", name);
++	}
++#else
+ 	if (WIFSIGNALED(status)) {
+ 		switch(WTERMSIG(status)) {
+ 		case SIGINT: /* halt */
+@@ -1993,6 +2016,7 @@ int __lxc_start(struct lxc_handler *handler, struct lxc_operations *ops,
+ 			break;
+ 		}
+ 	}
++#endif
+ 
+ 	ret = lxc_restore_phys_nics_to_netns(handler);
+ 	if (ret < 0)
+@@ -2000,7 +2024,11 @@ int __lxc_start(struct lxc_handler *handler, struct lxc_operations *ops,
+ 
+ 	close_prot_errno_disarm(handler->pinfd);
+ 
++#ifdef HAVE_ISULAD
++	lxc_monitor_send_exit_code(name, exit_code, handler->lxcpath);
++#else
+ 	lxc_monitor_send_exit_code(name, status, handler->lxcpath);
++#endif
+ 	lxc_error_set_and_log(handler->pid, status);
+ 	if (error_num)
+ 		*error_num = handler->exit_status;
+-- 
+1.8.3.1
+

0007-check-and-save-pid-info-file.patch

@@ -0,0 +1,545 @@
+From 0d54daf204fd2bc41c45c7c159af6436d66b272c Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Mon, 13 Apr 2020 05:48:03 -0400
+Subject: [PATCH 07/49] check and save pid info file
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/conf.c            |   1 +
+ src/lxc/conf.h            |   2 +
+ src/lxc/lxccontainer.c    |  29 +++++++++++-
+ src/lxc/lxccontainer.h    |  12 +++++
+ src/lxc/start.c           |  98 ++++++++++++++++++++++++++++++++++++++
+ src/lxc/tools/arguments.h |  16 +++++++
+ src/lxc/tools/lxc_start.c |  28 +++++++++++
+ src/lxc/utils.c           | 117 ++++++++++++++++++++++++++++++++++++++++++++++
+ src/lxc/utils.h           |  69 +++++++++++++++++++++++++++
+ 9 files changed, 371 insertions(+), 1 deletion(-)
+
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index e806605..43437af 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -3961,6 +3961,7 @@ void lxc_conf_free(struct lxc_conf *conf)
+ 	free(conf->shmount.path_host);
+ 	free(conf->shmount.path_cont);
+ #ifdef HAVE_ISULAD
++	free(conf->container_info_file);
+ 	lxc_clear_init_args(conf);
+ 	lxc_clear_populate_devices(conf);
+ #endif
+diff --git a/src/lxc/conf.h b/src/lxc/conf.h
+index 7ed3cd0..23942ac 100644
+--- a/src/lxc/conf.h
++++ b/src/lxc/conf.h
+@@ -428,6 +428,8 @@ struct lxc_conf {
+ 	/* populate devices*/
+ 	struct lxc_list populate_devs;
+ 	mode_t umask;  //umask value
++
++	char *container_info_file;
+ #endif
+ 
+ };
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index a617172..33bb3ec 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -5293,6 +5293,31 @@ static int do_lxcapi_seccomp_notify_fd(struct lxc_container *c)
+ 
+ WRAP_API(int, lxcapi_seccomp_notify_fd)
+ 
++#ifdef HAVE_ISULAD
++/* isulad add set info file path */
++static bool do_lxcapi_set_container_info_file(struct lxc_container *c, const char *info_file)
++{
++	struct lxc_conf *conf = NULL;
++
++	if (!c || !c->lxc_conf || !info_file)
++		return false;
++	if (container_mem_lock(c)) {
++		ERROR("Error getting mem lock");
++		return false;
++	}
++
++	conf = c->lxc_conf;
++	if (conf->container_info_file)
++		free(conf->container_info_file);
++	conf->container_info_file = safe_strdup(info_file);
++
++	container_mem_unlock(c);
++	return true;
++}
++
++WRAP_API_1(bool, lxcapi_set_container_info_file, const char *)
++#endif
++
+ struct lxc_container *lxc_container_new(const char *name, const char *configpath)
+ {
+ 	struct lxc_container *c;
+@@ -5434,7 +5459,9 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
+ 	c->mount = lxcapi_mount;
+ 	c->umount = lxcapi_umount;
+ 	c->seccomp_notify_fd = lxcapi_seccomp_notify_fd;
+-
++#ifdef HAVE_ISULAD
++	c->set_container_info_file = lxcapi_set_container_info_file;
++#endif
+ 	return c;
+ 
+ err:
+diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
+index 4577de7..edfff32 100644
+--- a/src/lxc/lxccontainer.h
++++ b/src/lxc/lxccontainer.h
+@@ -865,6 +865,18 @@ struct lxc_container {
+ 	 * \return pidfd of init process of the container.
+ 	 */
+ 	int (*init_pidfd)(struct lxc_container *c);
++
++#ifdef HAVE_ISULAD
++	/*! isulad add
++	 * \brief An API call to set the path of info file
++	 *
++	 * \param c Container.
++	 * \param info_file Value of the path of info file.
++	 *
++	 * \return \c true on success, else \c false.
++	 */
++	bool (*set_container_info_file) (struct lxc_container *c, const char *info_file);
++#endif
+ };
+ 
+ /*!
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 5dcf828..f5f9565 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -1537,6 +1537,94 @@ static inline int do_share_ns(void *arg)
+ 	return 0;
+ }
+ 
++#ifdef HAVE_ISULAD
++static int lxc_write_container_info(char *filename, pid_t pid, pid_t p_pid,
++		unsigned long long start_at, unsigned long long p_start_at)
++{
++	FILE *pid_fp = NULL;
++	int ret = 0;
++
++	pid_fp = lxc_fopen(filename, "w");
++	if (pid_fp == NULL) {
++		SYSERROR("Failed to create pidfile '%s'",filename);
++		ret = -1;
++		goto out;
++	}
++
++	if (fprintf(pid_fp, "%d %llu %d %llu\n", pid, start_at, p_pid, p_start_at) < 0) {
++		SYSERROR("Failed to write '%s'", filename);
++		ret = -1;
++		goto out;
++	}
++out:
++	if (pid_fp)
++		fclose(pid_fp);
++	pid_fp = NULL;
++	return ret;
++}
++
++static int lxc_check_container_info(char *filename, pid_t pid, pid_t p_pid,
++		unsigned long long start_at, unsigned long long p_start_at)
++{
++	int ret = 0;
++	int num;
++	char sbuf[1024] = {0}; /* bufs for stat */
++	int saved_pid;		/* process id */
++	int saved_ppid;		/* pid of parent process */
++	unsigned long long saved_start_time;	/* start time of process -- seconds since 1-1-70 */
++	unsigned long long saved_pstart_time;	/* start time of parent process -- seconds since 1-1-70 */
++
++	if ((lxc_file2str(filename, sbuf, sizeof(sbuf))) == -1) {
++		SYSERROR("Failed to read pidfile %s", filename);
++		ret = -1;
++		goto out;
++	}
++
++	num = sscanf(sbuf, "%d %Lu %d %Lu", &saved_pid, &saved_start_time, &saved_ppid, &saved_pstart_time);
++	if (num != 4) {
++		SYSERROR("Call sscanf error");
++		ret = -1;
++		goto out;
++	}
++
++	if (pid != saved_pid || p_pid != saved_ppid
++	                || start_at != saved_start_time || p_start_at != saved_pstart_time) {
++		ERROR("Check container info failed");
++		ret = -1;
++		goto out;
++	}
++
++out:
++	return ret;
++}
++
++/* isuald: save pid/ppid info */
++static int lxc_save_container_info(char *filename, pid_t pid)
++{
++	int ret = 0;
++	pid_t p_pid = 0;
++	unsigned long long start_at = 0;
++	unsigned long long p_start_at = 0;
++
++	start_at = lxc_get_process_startat(pid);
++	p_pid = getpid();
++	p_start_at = lxc_get_process_startat(p_pid);
++
++	ret = lxc_write_container_info(filename, pid, p_pid, start_at, p_start_at);
++	if (ret != 0) {
++		goto out;
++	}
++
++	ret = lxc_check_container_info(filename, pid, p_pid, start_at, p_start_at);
++	if (ret != 0) {
++		goto out;
++	}
++
++out:
++	return ret;
++}
++#endif
++
+ /* lxc_spawn() performs crucial setup tasks and clone()s the new process which
+  * exec()s the requested container binary.
+  * Note that lxc_spawn() runs in the parent namespaces. Any operations performed
+@@ -1648,6 +1736,16 @@ static int lxc_spawn(struct lxc_handler *handler)
+ 	}
+ 	TRACE("Cloned child process %d", handler->pid);
+ 
++#ifdef HAVE_ISULAD
++    /* isulad: save pid/ppid info into file*/
++    if (handler->conf->container_info_file) {
++        if (lxc_save_container_info(handler->conf->container_info_file, handler->pid)) {
++            ERROR("Failed to save cloned container pid");
++            goto out_delete_net;
++        }
++    }
++#endif
++
+ 	/* Verify that we can actually make use of pidfds. */
+ 	if (!lxc_can_use_pidfd(handler->pidfd))
+ 		close_prot_errno_disarm(handler->pidfd);
+diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
+index cb0ba74..91f4e9a 100644
+--- a/src/lxc/tools/arguments.h
++++ b/src/lxc/tools/arguments.h
+@@ -40,6 +40,9 @@ struct lxc_arguments {
+ 
+ 	/* for lxc-start */
+ 	const char *share_ns[32]; /* size must be greater than LXC_NS_MAX */
++#ifdef HAVE_ISULAD
++	const char *container_info; /* isulad: file used to store pid and ppid info of container */
++#endif
+ 
+ 	/* for lxc-console */
+ 	unsigned int ttynum;
+@@ -152,6 +155,19 @@ struct lxc_arguments {
+ #define OPT_SHARE_UTS OPT_USAGE - 5
+ #define OPT_SHARE_PID OPT_USAGE - 6
+ 
++#ifdef HAVE_ISULAD
++#define OPT_INPUT_FIFO OPT_USAGE - 7
++#define OPT_OUTPUT_FIFO OPT_USAGE - 8
++#define OPT_STDERR_FIFO OPT_USAGE - 9
++#define OPT_CONTAINER_INFO OPT_USAGE - 10
++#define OPT_EXIT_FIFO OPT_USAGE - 11
++#define OPT_START_TIMEOUT OPT_USAGE - 12
++#define OPT_DISABLE_PTY OPT_USAGE - 13
++#define OPT_OPEN_STDIN OPT_USAGE - 14
++#define OPT_ATTACH_TIMEOUT OPT_USAGE - 15
++#define OPT_ATTACH_SUFFIX OPT_USAGE - 16
++#endif
++
+ extern int lxc_arguments_parse(struct lxc_arguments *args, int argc,
+ 			       char *const argv[]);
+ 
+diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
+index 459b867..83ee75a 100644
+--- a/src/lxc/tools/lxc_start.c
++++ b/src/lxc/tools/lxc_start.c
+@@ -48,6 +48,9 @@ static const struct option my_longopts[] = {
+ 	{"share-ipc", required_argument, 0, OPT_SHARE_IPC},
+ 	{"share-uts", required_argument, 0, OPT_SHARE_UTS},
+ 	{"share-pid", required_argument, 0, OPT_SHARE_PID},
++#ifdef HAVE_ISULAD
++	{"container-pidfile", required_argument, 0, OPT_CONTAINER_INFO},
++#endif
+ 	LXC_COMMON_OPTIONS
+ };
+ 
+@@ -118,6 +121,11 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 	case OPT_SHARE_PID:
+ 		args->share_ns[LXC_NS_PID] = arg;
+ 		break;
++#ifdef HAVE_ISULAD
++	case OPT_CONTAINER_INFO:
++		args->container_info = arg;
++		break;
++#endif
+ 	}
+ 	return 0;
+ }
+@@ -163,6 +171,9 @@ int main(int argc, char *argv[])
+ 		"/sbin/init",
+ 		NULL,
+ 	};
++#ifdef HAVE_ISULAD
++	char *container_info_file = NULL;
++#endif
+ 
+ 	lxc_list_init(&defines);
+ 
+@@ -283,6 +294,20 @@ int main(int argc, char *argv[])
+ 			goto out;
+ 		}
+ 
++#ifdef HAVE_ISULAD
++	/* isulad: container info file used to store pid and ppid info of container*/
++	if (my_args.container_info != NULL) {
++		if (ensure_path(&container_info_file, my_args.container_info) < 0) {
++			ERROR("Failed to ensure container's piddile '%s'", my_args.container_info);
++			goto out;
++		}
++		if (!c->set_container_info_file(c, container_info_file)) {
++			ERROR("Failed to set container's piddile '%s'", container_info_file);
++			goto out;
++		}
++	}
++#endif
++
+ 	if (my_args.console)
+ 		if (!c->set_config_item(c, "lxc.console.path", my_args.console))
+ 			goto out;
+@@ -320,5 +345,8 @@ int main(int argc, char *argv[])
+ 
+ out:
+ 	lxc_container_put(c);
++#ifdef HAVE_ISULAD
++	free(container_info_file);
++#endif
+ 	exit(err);
+ }
+diff --git a/src/lxc/utils.c b/src/lxc/utils.c
+index 160b3db..ebcdae0 100644
+--- a/src/lxc/utils.c
++++ b/src/lxc/utils.c
+@@ -1931,4 +1931,121 @@ void lxc_write_error_message(int errfd, const char *format, ...)
+ 	if (sret < 0)
+ 		SYSERROR("Write errbuf failed");
+ }
++
++/* isulad: read file to buffer */
++int lxc_file2str(const char *filename, char ret[], int cap)
++{
++    int fd, num_read;
++
++    if ((fd = lxc_open(filename, O_RDONLY | O_CLOEXEC, 0)) == -1)
++        return -1;
++    if ((num_read = read(fd, ret, cap - 1)) <= 0)
++        num_read = -1;
++    else
++        ret[num_read] = 0;
++    close(fd);
++
++    return num_read;
++}
++
++/* isuald: lxc_stat2proc() makes sure it can handle arbitrary executable file basenames
++ * for `cmd', i.e. those with embedded whitespace or embedded ')'s.
++ * Such names confuse %s (see scanf(3)), so the string is split and %39c
++ * is used instead. (except for embedded ')' "(%[^)]c)" would work.
++ */
++static proc_t *lxc_stat2proc(const char *S)
++{
++    int num;
++    proc_t *P = NULL;
++    char *tmp = NULL;
++
++    if (!S)
++        return NULL;
++
++    tmp = strrchr(S, ')');  /* split into "PID (cmd" and "<rest>" */
++    if (!tmp)
++        return NULL;
++    *tmp = '\0';            /* replace trailing ')' with NUL */
++
++    P = malloc(sizeof(proc_t));
++    if (P == NULL)
++        return NULL;
++    (void)memset(P, 0x00, sizeof(proc_t));
++
++    /* parse these two strings separately, skipping the leading "(". */
++    num = sscanf(S, "%d (%15c", &P->pid, P->cmd);   /* comm[16] in kernel */
++    if (num != 2) {
++        ERROR("Call sscanf error: %s", errno ? strerror(errno) : "");
++        free(P);
++        return NULL;
++    }
++    num = sscanf(tmp + 2,           /* skip space after ')' too */
++                 "%c "
++                 "%d %d %d %d %d "
++                 "%lu %lu %lu %lu %lu "
++                 "%Lu %Lu %Lu %Lu "  /* utime stime cutime cstime */
++                 "%ld %ld %ld %ld "
++                 "%Lu "  /* start_time */
++                 "%lu "
++                 "%ld "
++                 "%lu %lu %lu %lu %lu %lu "
++                 "%*s %*s %*s %*s " /* discard, no RT signals & Linux 2.1 used hex */
++                 "%lu %lu %lu "
++                 "%d %d "
++                 "%lu %lu",
++                 &P->state,
++                 &P->ppid, &P->pgrp, &P->session, &P->tty, &P->tpgid,
++                 &P->flags, &P->min_flt, &P->cmin_flt, &P->maj_flt, &P->cmaj_flt,
++                 &P->utime, &P->stime, &P->cutime, &P->cstime,
++                 &P->priority, &P->nice, &P->timeout, &P->it_real_value,
++                 &P->start_time,
++                 &P->vsize,
++                 &P->rss,
++                 &P->rss_rlim, &P->start_code, &P->end_code, &P->start_stack, &P->kstk_esp,
++                 &P->kstk_eip,
++                 &P->wchan, &P->nswap, &P->cnswap,
++                 &P->exit_signal, &P->processor,  /* 2.2.1 ends with "exit_signal" */
++                 &P->rtprio, &P->sched  /* both added to 2.5.18 */
++                );
++    if (num != 35) {
++        ERROR("Call sscanf error: %s", errno ? strerror(errno) : "");
++        free(P);
++        return NULL;
++    }
++    if (P->tty == 0)
++        P->tty = -1;  /* the old notty val, update elsewhere bef. moving to 0 */
++    return P;
++}
++
++/* isulad: get starttime of process pid */
++unsigned long long lxc_get_process_startat(pid_t pid)
++{
++    int sret = 0;
++    unsigned long long startat = 0;
++    proc_t *pid_info = NULL;
++    char filename[PATH_MAX] = {0};
++    char sbuf[1024] = {0}; /* bufs for stat */
++
++    sret = snprintf(filename, sizeof(filename), "/proc/%d/stat", pid);
++    if (sret < 0 || sret >= sizeof(filename)) {
++        ERROR("Failed to sprintf filename");
++        goto out;
++    }
++
++    if ((lxc_file2str(filename, sbuf, sizeof(sbuf))) == -1) {
++        SYSERROR("Failed to read pidfile %s", filename);
++        goto out;
++    }
++
++    pid_info = lxc_stat2proc(sbuf);
++    if (!pid_info) {
++        ERROR("Failed to get proc stat info");
++        goto out;
++    }
++
++    startat = pid_info->start_time;
++out:
++    free(pid_info);
++    return startat;
++}
+ #endif
+diff --git a/src/lxc/utils.h b/src/lxc/utils.h
+index 3c30565..11d6548 100644
+--- a/src/lxc/utils.h
++++ b/src/lxc/utils.h
+@@ -44,6 +44,73 @@ extern char *get_rundir(void);
+ #endif
+ #endif
+ 
++#ifdef HAVE_ISULAD
++/* isulad:
++ ld cutime, cstime, priority, nice, timeout, it_real_value, rss,
++ c  state,
++ d  ppid, pgrp, session, tty, tpgid,
++ s  signal, blocked, sigignore, sigcatch,
++ lu flags, min_flt, cmin_flt, maj_flt, cmaj_flt, utime, stime,
++ lu rss_rlim, start_code, end_code, start_stack, kstk_esp, kstk_eip,
++ lu start_time, vsize, wchan, nswap, cnswap,
++*/
++
++/* Basic data structure which holds all information we can get about a process.
++ * (unless otherwise specified, fields are read from /proc/#/stat)
++ *
++ * Most of it comes from task_struct in linux/sched.h
++ */
++typedef struct proc_t {
++    // 1st 16 bytes
++    int pid;        /* process id */
++    int ppid;       /* pid of parent process */
++
++    char state;     /* single-char code for process state (S=sleeping) */
++
++    unsigned long long
++    utime,      /* user-mode CPU time accumulated by process */
++    stime,      /* kernel-mode CPU time accumulated by process */
++    // and so on...
++    cutime,     /* cumulative utime of process and reaped children */
++    cstime,     /* cumulative stime of process and reaped children */
++    start_time; /* start time of process -- seconds since 1-1-70 */
++
++    long
++    priority,   /* kernel scheduling priority */
++    timeout,    /* ? */
++    nice,       /* standard unix nice level of process */
++    rss,        /* resident set size from /proc/#/stat (pages) */
++    it_real_value;  /* ? */
++    unsigned long
++    rtprio,     /* real-time priority */
++    sched,      /* scheduling class */
++    vsize,      /* number of pages of virtual memory ... */
++    rss_rlim,   /* resident set size limit? */
++    flags,      /* kernel flags for the process */
++    min_flt,    /* number of minor page faults since process start */
++    maj_flt,    /* number of major page faults since process start */
++    cmin_flt,   /* cumulative min_flt of process and child processes */
++    cmaj_flt,   /* cumulative maj_flt of process and child processes */
++    nswap,      /* ? */
++    cnswap,     /* cumulative nswap ? */
++    start_code, /* address of beginning of code segment */
++    end_code,   /* address of end of code segment */
++    start_stack,    /* address of the bottom of stack for the process */
++    kstk_esp,   /* kernel stack pointer */
++    kstk_eip,   /* kernel instruction pointer */
++    wchan;      /* address of kernel wait channel proc is sleeping in */
++
++    char cmd[16];   /* basename of executable file in call to exec(2) */
++    int
++    pgrp,       /* process group id */
++    session,    /* session id */
++    tty,        /* full device number of controlling terminal */
++    tpgid,      /* terminal process group id */
++    exit_signal,    /* might not be SIGCHLD */
++    processor;      /* current (or most recent?) CPU */
++} proc_t;
++#endif
++
+ static inline int lxc_set_cloexec(int fd)
+ {
+ 	return fcntl(fd, F_SETFD, FD_CLOEXEC);
+@@ -246,6 +313,8 @@ extern int fix_stdio_permissions(uid_t uid);
+ 
+ #ifdef HAVE_ISULAD
+ extern void lxc_write_error_message(int errfd, const char *format, ...);
++extern int lxc_file2str(const char *filename, char ret[], int cap);
++extern int unsigned long long lxc_get_process_startat(pid_t pid);
+ #endif
+ 
+ #endif /* __LXC_UTILS_H */
+-- 
+1.8.3.1
+

0007-use-isulad-log-format.patch

@@ -1,74 +0,0 @@
-From 371dc5d5822cd629b86313c6b6cb39f02e7164e8 Mon Sep 17 00:00:00 2001
-From: liuhao <liuhao27@huawei.com>
-Date: Fri, 11 Jan 2019 17:00:48 +0800
-Subject: [PATCH 007/140] use isulad log format
-
-use isulad log format
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/log.c | 19 ++++++++++++++-----
- 1 file changed, 14 insertions(+), 5 deletions(-)
-
-diff --git a/src/lxc/log.c b/src/lxc/log.c
-index 4e74459..91fb7ef 100644
---- a/src/lxc/log.c
-+++ b/src/lxc/log.c
-@@ -139,6 +139,7 @@ static char *lxc_log_get_va_msg(struct lxc_log_event *event)
- 	return msg;
- }
- 
-+/* use fifo to save log */
- static const char *isulad_use_log_fifo(const char *file)
- {
- #define ISULAD_FIFO_PREFIX "fifo:"
-@@ -150,6 +151,7 @@ static const char *isulad_use_log_fifo(const char *file)
- 	return file;
- }
- 
-+/* open isulad fifo */
- static int isulad_open_fifo(const char *file_path)
- {
- #define LOG_FIFO_SIZE (1024 * 1024)
-@@ -349,6 +351,8 @@ static int log_append_logfile(const struct lxc_log_appender *appender,
- 	ssize_t ret;
- 	int fd_to_use = -1;
- 	const char *log_container_name;
-+	const char *isulad_prefix;
-+	size_t isulad_len = 0;
- 
- #ifndef NO_LXC_CONF
- 	if (current_config)
-@@ -367,11 +371,14 @@ static int log_append_logfile(const struct lxc_log_appender *appender,
- 	if (lxc_unix_epoch_to_utc(date_time, LXC_LOG_TIME_SIZE, &event->timestamp) < 0)
- 		return -1;
- 
-+	/* use isulad log format */
-+	if (log_container_name && strlen(log_container_name) > 15) {
-+		isulad_len = strlen(log_container_name) - 15;
-+	}
-+	isulad_prefix = log_container_name ? (log_container_name + isulad_len) : log_prefix;
- 	n = snprintf(buffer, sizeof(buffer),
--		     "%s%s%s %s %-8s %s - %s:%s:%d - ",
--		     log_prefix,
--		     log_container_name ? " " : "",
--		     log_container_name ? log_container_name : "",
-+		     "%15s %s %-8s %s - %s:%s:%d - ",
-+		     isulad_prefix,
- 		     date_time,
- 		     lxc_log_priority_to_string(event->priority),
- 		     event->category,
-@@ -752,7 +759,9 @@ int lxc_log_init(struct lxc_log *log)
- 
- 	if (lxc_log_fd != -1) {
- 		lxc_log_category_lxc.appender = &log_appender_logfile;
--		lxc_log_category_lxc.appender->next = &log_appender_stderr;
-+		if (!lxc_quiet_specified)
-+			if (!log->quiet)
-+				lxc_log_category_lxc.appender->next = &log_appender_stderr;
- 	}
- 
- 	return ret;
--- 
-1.8.3.1
-

0008-isulad-modify-exit-code-and-stop-signal.patch

@@ -1,106 +0,0 @@
-From 98dbcb7dacbf18393295a9dcf054a65f9033ea5d Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Fri, 11 Jan 2019 17:44:53 +0800
-Subject: [PATCH 008/140] isulad: modify exit code and stop signal
-
-1. modify default stop signal and disable reboot by signal.
-2. send '128 + signal' if container is killed by signal.
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/lxccontainer.c |  6 ++----
- src/lxc/start.c        | 33 ++++++++++++++++-----------------
- 2 files changed, 18 insertions(+), 21 deletions(-)
-
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index b4cacce..1d7f5be 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -2069,7 +2069,8 @@ static bool do_lxcapi_shutdown(struct lxc_container *c, int timeout)
- {
- 	int killret, ret;
- 	pid_t pid;
--	int haltsignal = SIGPWR, state_client_fd = -EBADF;
-+	// isulad: keep default signal the same as docker
-+	int haltsignal = SIGTERM, state_client_fd = -EBADF;
- 	lxc_state_t states[MAX_STATE] = {0};
- 
- 	if (!c)
-@@ -2082,11 +2083,8 @@ static bool do_lxcapi_shutdown(struct lxc_container *c, int timeout)
- 	if (pid <= 0)
- 		return true;
- 
--	/* Detect whether we should send SIGRTMIN + 3 (e.g. systemd). */
- 	if (c->lxc_conf && c->lxc_conf->haltsignal)
- 		haltsignal = c->lxc_conf->haltsignal;
--	else if (task_blocks_signal(pid, (SIGRTMIN + 3)))
--		haltsignal = (SIGRTMIN + 3);
- 
- 	/* Add a new state client before sending the shutdown signal so that we
- 	 * don't miss a state.
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 72e2de2..d64bdac 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -1893,11 +1893,14 @@ out_abort:
- 	return -1;
- }
- 
-+// isulad: send '128 + signal' if container is killed by signal.
-+#define ExitSignalOffset 128
-+
- int __lxc_start(const char *name, struct lxc_handler *handler,
- 		struct lxc_operations* ops, void *data, const char *lxcpath,
- 		bool daemonize, int *error_num)
- {
--	int ret, status;
-+	int ret, status, exit_code;
- 	struct lxc_conf *conf = handler->conf;
- 
- 	ret = lxc_init(name, handler);
-@@ -1966,22 +1969,18 @@ int __lxc_start(const char *name, struct lxc_handler *handler,
- 	 * reboot. This should mean it was an lxc-execute which simply exited.
- 	 * In any case, treat it as a 'halt'.
- 	 */
-+	// isulad: recored log for container init exit
- 	if (WIFSIGNALED(status)) {
--		switch(WTERMSIG(status)) {
--		case SIGINT: /* halt */
--			DEBUG("Container \"%s\" is halting", name);
--			break;
--		case SIGHUP: /* reboot */
--			DEBUG("Container \"%s\" is rebooting", name);
--			handler->conf->reboot = REBOOT_REQ;
--			break;
--		case SIGSYS: /* seccomp */
--			DEBUG("Container \"%s\" violated its seccomp policy", name);
--			break;
--		default:
--			DEBUG("Unknown exit status for container \"%s\" init %d", name, WTERMSIG(status));
--			break;
--		}
-+		int signal = WTERMSIG(status);
-+		signal = WTERMSIG(status);
-+		exit_code = ExitSignalOffset + signal;
-+		ERROR("Container \"%s\" init exited with signal %d", name, signal);
-+	} else if (WIFEXITED(status)) {
-+		exit_code = WEXITSTATUS(status);
-+		ERROR("Container \"%s\" init exited with status %d", name, exit_code);
-+	} else {
-+		exit_code = -1;
-+		ERROR("Container \"%s\" init exited with unknown status", name);
- 	}
- 
- 	ret = lxc_restore_phys_nics_to_netns(handler);
-@@ -1994,7 +1993,7 @@ int __lxc_start(const char *name, struct lxc_handler *handler,
- 		handler->pinfd = -1;
- 	}
- 
--	lxc_monitor_send_exit_code(name, status, handler->lxcpath);
-+	lxc_monitor_send_exit_code(name, exit_code, handler->lxcpath);
- 	lxc_error_set_and_log(handler->pid, status);
- 	if (error_num)
- 		*error_num = handler->exit_status;
--- 
-1.8.3.1
-

0008-support-block-device-as-rootfs.patch

@@ -1,26 +1,27 @@
-From c1b8d03c8911964c8323e4878ecfeca8162a7e08 Mon Sep 17 00:00:00 2001
+From 6792a7f76d9084734d94e815b462ed2977fe107e Mon Sep 17 00:00:00 2001
 From: tanyifeng <tanyifeng1@huawei.com>
 Date: Tue, 15 Jan 2019 16:00:30 +0800
-Subject: [PATCH 030/140] support block device as rootfs
+Subject: [PATCH 08/49] support block device as rootfs
 
 Signed-off-by: LiFeng <lifeng68@huawei.com>
+Signed-off-by: WangFengTu <wangfengtu@huawei.com>
 ---
  src/lxc/Makefile.am             |  1 +
- src/lxc/conf.c                  | 10 ++---
+ src/lxc/conf.c                  | 36 +++++++++++++++++
  src/lxc/storage/block.c         | 86 +++++++++++++++++++++++++++++++++++++++++
  src/lxc/storage/block.h         | 41 ++++++++++++++++++++
- src/lxc/storage/dir.c           | 10 +----
+ src/lxc/storage/dir.c           | 14 ++++++-
- src/lxc/storage/storage.c       | 18 +++++++++
+ src/lxc/storage/storage.c       | 21 ++++++++++
- src/lxc/storage/storage_utils.c |  2 +-
+ src/lxc/storage/storage_utils.c |  4 ++
- 7 files changed, 153 insertions(+), 15 deletions(-)
+ 7 files changed, 202 insertions(+), 1 deletion(-)
  create mode 100644 src/lxc/storage/block.c
  create mode 100644 src/lxc/storage/block.h
 
 diff --git a/src/lxc/Makefile.am b/src/lxc/Makefile.am
-index 5678b8d..260a7eb 100644
+index 21441c0..d8c2492 100644
 --- a/src/lxc/Makefile.am
 +++ b/src/lxc/Makefile.am
-@@ -130,6 +130,7 @@ liblxc_la_SOURCES = af_unix.c af_unix.h \
+@@ -139,6 +139,7 @@ liblxc_la_SOURCES = af_unix.c af_unix.h \
  		    start.c start.h \
  		    storage/btrfs.c storage/btrfs.h \
  		    storage/dir.c storage/dir.h \
@@ -29,36 +30,59 @@ index 5678b8d..260a7eb 100644
  		    storage/lvm.c storage/lvm.h \
  		    storage/nbd.c storage/nbd.h \
 diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 439353b..88763ee 100644
+index 43437af..35488e0 100644
 --- a/src/lxc/conf.c
 +++ b/src/lxc/conf.c
-@@ -3865,13 +3865,10 @@ static int setup_populate_devs(const struct lxc_rootfs *rootfs, struct lxc_list
+@@ -3383,6 +3383,36 @@ reset_umask:
- // isulad: setup rootfs mountopts
+ 	INFO("Populated devices into container /dev");
- static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs)
+ 	return ret;
- {
+ }
--	unsigned long mntflags, pflags;
++
++// isulad: setup rootfs mountopts
++static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs)
++{
 +	unsigned long mflags, mntflags, pflags;
- 	char *mntdata;
++	char *mntdata;
- 
++
--	// only remount / when container shares rootfs with host.
--	if(!rootfs || !rootfs->path || strcmp(rootfs->path, "/"))
--		return 0;
--	if (!rootfs->options)
 +	if(!rootfs || !rootfs->options)
- 		return 0;
++		return 0;
- 
++
- 	if (parse_mntopts(rootfs->options, &mntflags, &pflags, &mntdata) < 0) {
++	if (parse_propagationopts(rootfs->options, &pflags) < 0) {
-@@ -3881,8 +3878,9 @@ static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs)
++		return -1;
- 	free(mntdata);
++	}
- 
++
- 	if (mntflags & MS_RDONLY) {
++	if (parse_mntopts(rootfs->options, &mntflags, &mntdata) < 0) {
++		free(mntdata);
++		return -1;
++	}
++	free(mntdata);
++
++	if (mntflags & MS_RDONLY) {
 +		mflags = add_required_remount_flags("/", NULL, MS_BIND | MS_REC | mntflags | pflags | MS_REMOUNT);
- 		DEBUG("remounting / as readonly");
++		DEBUG("remounting /");
--		if (mount("/", "/", NULL, MS_BIND |MS_REMOUNT| MS_RDONLY, 0)) {
++		if (mount("/", "/", NULL, mflags, 0) < 0) {
-+		if (mount("/", "/", NULL, MS_BIND | MS_REMOUNT | MS_RDONLY, 0) < 0) {
++			SYSERROR("Failed to remount /");
- 			SYSERROR("Failed to make / readonly.");
++			return -1;
- 			return -1;
++		}
- 		}
++	}
++	return 0;
++}
+ #endif
+ 
+ int lxc_setup(struct lxc_handler *handler)
+@@ -3531,6 +3561,12 @@ int lxc_setup(struct lxc_handler *handler)
+ 	if (ret < 0)
+ 		return log_error(-1, "Failed to setup new devpts instance");
+ 
++#ifdef HAVE_ISULAD
++        if (setup_rootfs_mountopts(&lxc_conf->rootfs)) {
++               return log_error(-1, "failed to set rootfs for '%s'", name);
++        }
++#endif
++
+ 	ret = lxc_create_ttys(handler);
+ 	if (ret < 0)
+ 		return -1;
 diff --git a/src/lxc/storage/block.c b/src/lxc/storage/block.c
 new file mode 100644
 index 0000000..eb75e70
@@ -199,38 +223,52 @@ index 0000000..2fa7565
 +
 +#endif /* __LXC_BLK_H */
 diff --git a/src/lxc/storage/dir.c b/src/lxc/storage/dir.c
-index deeecec..2b548d0 100644
+index 18a10a4..b3dbbd0 100644
 --- a/src/lxc/storage/dir.c
 +++ b/src/lxc/storage/dir.c
-@@ -150,7 +150,7 @@ bool dir_detect(const char *path)
+@@ -127,7 +127,11 @@ bool dir_detect(const char *path)
  int dir_mount(struct lxc_storage *bdev)
  {
- 	int ret;
+ 	__do_free char *mntdata = NULL;
--	unsigned long mflags = 0, mntflags = 0, pflags = 0;
++#ifdef HAVE_ISULAD
 +	unsigned long mntflags = 0, pflags = 0;
- 	char *mntdata;
++#else
+ 	unsigned long mflags = 0, mntflags = 0, pflags = 0;
++#endif
+ 	int ret;
  	const char *src;
  
-@@ -169,13 +169,7 @@ int dir_mount(struct lxc_storage *bdev)
+@@ -147,6 +151,13 @@ int dir_mount(struct lxc_storage *bdev)
  
  	src = lxc_storage_get_path(bdev->src, bdev->type);
  
--	ret = mount(src, bdev->dest, "bind", MS_BIND | MS_REC | mntflags | pflags, mntdata);
++#ifdef HAVE_ISULAD
--	if ((0 == ret) && (mntflags & MS_RDONLY)) {
--		DEBUG("Remounting \"%s\" on \"%s\" readonly",
--		      src ? src : "(none)", bdev->dest ? bdev->dest : "(none)");
--		mflags = add_required_remount_flags(src, bdev->dest, MS_BIND | MS_REC | mntflags | pflags | MS_REMOUNT);
--		ret = mount(src, bdev->dest, "bind", mflags, mntdata);
--	}
 +	ret = mount(src, bdev->dest, "bind", MS_BIND | MS_REC | (mntflags & ~MS_RDONLY) | pflags, mntdata);
++	if (ret < 0) {
++		return log_error_errno(-errno, errno, "Failed to mount \"%s\" on \"%s\"", src, bdev->dest);
++	}
++	TRACE("Mounted \"%s\" on \"%s\"", src, bdev->dest);
++#else
+ 	ret = mount(src, bdev->dest, "bind", MS_BIND | MS_REC | mntflags | pflags, mntdata);
+ 	if (ret < 0)
+ 		return log_error_errno(-errno, errno, "Failed to mount \"%s\" on \"%s\"", src, bdev->dest);
+@@ -161,9 +172,10 @@ int dir_mount(struct lxc_storage *bdev)
+ 			DEBUG("Remounted \"%s\" on \"%s\" read-only with options \"%s\", mount flags \"%lu\", and propagation flags \"%lu\"",
+ 			      src ? src : "(none)", bdev->dest ? bdev->dest : "(none)", mntdata, mflags, pflags);
+ 	}
+-
+ 	TRACE("Mounted \"%s\" on \"%s\" with options \"%s\", mount flags \"%lu\", and propagation flags \"%lu\"",
+ 	      src ? src : "(none)", bdev->dest ? bdev->dest : "(none)", mntdata, mflags, pflags);
++#endif
++
+ 	return 0;
+ }
  
- 	if (ret < 0) {
- 		SYSERROR("Failed to mount \"%s\" on \"%s\"", src, bdev->dest);
 diff --git a/src/lxc/storage/storage.c b/src/lxc/storage/storage.c
-index c4f4c2e..18f754a 100644
+index 3f1b713..876311a 100644
 --- a/src/lxc/storage/storage.c
 +++ b/src/lxc/storage/storage.c
-@@ -61,6 +61,7 @@
+@@ -41,6 +41,7 @@
  #include "storage_utils.h"
  #include "utils.h"
  #include "zfs.h"
@@ -238,10 +276,11 @@ index c4f4c2e..18f754a 100644
  
  #ifndef HAVE_STRLCPY
  #include "include/strlcpy.h"
-@@ -114,6 +115,21 @@ static const struct lxc_storage_ops loop_ops = {
+@@ -94,6 +95,22 @@ static const struct lxc_storage_ops loop_ops = {
      .can_backup = true,
  };
  
++#ifdef HAVE_ISULAD
 +/* block */
 +static const struct lxc_storage_ops blk_ops = {
 +    .detect = &blk_detect,
@@ -255,30 +294,35 @@ index c4f4c2e..18f754a 100644
 +    .can_snapshot = false,
 +    .can_backup = true,
 +};
-+
++#endif
 +
  /* lvm */
  static const struct lxc_storage_ops lvm_ops = {
      .detect = &lvm_detect,
-@@ -199,6 +215,8 @@ static const struct lxc_storage_type bdevs[] = {
+@@ -179,6 +196,10 @@ static const struct lxc_storage_type bdevs[] = {
  	{ .name = "overlayfs", .ops = &ovl_ops,   },
  	{ .name = "loop",      .ops = &loop_ops,  },
  	{ .name = "nbd",       .ops = &nbd_ops,   },
++#ifdef HAVE_ISULAD
 +	//isulad: block device
 +	{ .name = "blk",       .ops = &blk_ops,   }
++#endif
  };
  
  static const size_t numbdevs = sizeof(bdevs) / sizeof(struct lxc_storage_type);
 diff --git a/src/lxc/storage/storage_utils.c b/src/lxc/storage/storage_utils.c
-index 46e08a3..b4dcb57 100644
+index a3ee353..bfbb782 100644
 --- a/src/lxc/storage/storage_utils.c
 +++ b/src/lxc/storage/storage_utils.c
-@@ -416,7 +416,7 @@ int find_fstype_cb(char *buffer, void *data)
+@@ -335,7 +335,11 @@ int find_fstype_cb(char *buffer, void *data)
  		return 0;
  	}
  
--	if (mount(cbarg->rootfs, cbarg->target, fstype, mntflags, mntdata)) {
++#ifdef HAVE_ISULAD
 +	if (mount(cbarg->rootfs, cbarg->target, fstype, (mntflags & ~MS_RDONLY), mntdata)) {
++#else
+ 	if (mount(cbarg->rootfs, cbarg->target, fstype, mntflags, mntdata)) {
++#endif
  		SYSDEBUG("Failed to mount");
  		free(mntdata);
  		return 0;

0009-lxc_start-add-default-terminal-fifos.patch

@@ -1,538 +0,0 @@
-From 3adac5cde53927f37a666fa8bda4ff06489ec694 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Fri, 11 Jan 2019 21:52:11 -0500
-Subject: [PATCH 009/140] lxc_start: add default terminal fifos
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c            |   4 +
- src/lxc/lxccontainer.c    |  30 +++++++
- src/lxc/lxccontainer.h    |  10 +++
- src/lxc/terminal.c        | 194 +++++++++++++++++++++++++++++++++++++++++++++-
- src/lxc/terminal.h        |  16 ++++
- src/lxc/tools/arguments.h |   5 ++
- src/lxc/tools/lxc_start.c |  11 +++
- src/lxc/utils.c           |  23 ++++++
- src/lxc/utils.h           |   4 +
- 9 files changed, 294 insertions(+), 3 deletions(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 37a5ff7..7b7f95b 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -2714,6 +2714,10 @@ struct lxc_conf *lxc_conf_init(void)
- 	new->console.slave = -1;
- 	new->console.name[0] = '\0';
- 	memset(&new->console.ringbuf, 0, sizeof(struct lxc_ringbuf));
-+	/* isulad init console fifos */
-+	new->console.init_fifo[0] = NULL;
-+	new->console.init_fifo[1] = NULL;
-+	lxc_list_init(&new->console.fifos);
- 	new->maincmd_fd = -1;
- 	new->nbd_idx = -1;
- 	new->rootfs.mount = strdup(default_rootfs_mount);
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index 1d7f5be..318c71e 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -4961,6 +4961,33 @@ out:
- 	return ret;
- }
- 
-+/* isulad add set console fifos*/
-+static bool do_lxcapi_set_terminal_default_fifos(struct lxc_container *c, const char *in, const char *out)
-+{
-+	struct lxc_conf *conf;
-+
-+	if (!c || !c->lxc_conf || !in || !out)
-+		return false;
-+	if (container_mem_lock(c)) {
-+		ERROR("Error getting mem lock");
-+		return false;
-+	}
-+
-+	conf = c->lxc_conf;
-+	if (conf->console.init_fifo[0])
-+		free(conf->console.init_fifo[0]);
-+	conf->console.init_fifo[0] = strdup(in);
-+
-+	if (conf->console.init_fifo[1])
-+		free(conf->console.init_fifo[1]);
-+	conf->console.init_fifo[1] = strdup(out);
-+
-+	container_mem_unlock(c);
-+	return true;
-+}
-+
-+WRAP_API_2(bool, lxcapi_set_terminal_default_fifos, const char *, const char *)
-+
- struct lxc_container *lxc_container_new(const char *name, const char *configpath)
- {
- 	struct lxc_container *c;
-@@ -5084,6 +5111,9 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
- 	c->migrate = lxcapi_migrate;
- 	c->console_log = lxcapi_console_log;
- 
-+	/* isulad add begin */
-+	c->set_terminal_init_fifos = lxcapi_set_terminal_default_fifos;
-+	/* isulad add end */
- 	return c;
- 
- err:
-diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
-index 9e06215..486531e 100644
---- a/src/lxc/lxccontainer.h
-+++ b/src/lxc/lxccontainer.h
-@@ -847,6 +847,16 @@ struct lxc_container {
- 	 * \return \c true if the container was rebooted successfully, else \c false.
- 	 */
- 	bool (*reboot2)(struct lxc_container *c, int timeout);
-+
-+	/*! isulad add
-+	 * \brief An API call to change the path of the console default fifos
-+	 *
-+	 * \param c Container.
-+	 * \param path Value of the console path.
-+	 *
-+	 * \return \c true on success, else \c false.
-+	 */
-+	bool (*set_terminal_init_fifos)(struct lxc_container *c, const char *in, const char *out);
- };
- 
- /*!
-diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
-index 4060e7f..c507712 100644
---- a/src/lxc/terminal.c
-+++ b/src/lxc/terminal.c
-@@ -364,6 +364,20 @@ static int lxc_terminal_write_log_file(struct lxc_terminal *terminal, char *buf,
- 	return bytes_read;
- }
- 
-+/* isulad: forward data to all fifos */
-+static void lxc_forward_data_to_fifo(struct lxc_list *list, char *buf, int r)
-+{
-+	struct lxc_list *it,*next;
-+	struct lxc_fifos_fd *elem = NULL;
-+
-+	lxc_list_for_each_safe(it, list, next) {
-+		elem = it->elem;
-+		lxc_write_nointr(elem->out_fd, buf, r);
-+	}
-+
-+	return;
-+}
-+
- int lxc_terminal_io_cb(int fd, uint32_t events, void *data,
- 		       struct lxc_epoll_descr *descr)
- {
-@@ -384,7 +398,13 @@ int lxc_terminal_io_cb(int fd, uint32_t events, void *data,
- 				terminal->tty_state = NULL;
- 			}
- 			terminal->peer = -EBADF;
--		} else {
-+			close(fd);
-+			return LXC_MAINLOOP_CONTINUE; /* isulad: do not close mainloop when peer close*/
-+		} else if (lxc_terminal_is_fifo(fd, &terminal->fifos)) {
-+			/* isulad: delete fifos when the client close */
-+			lxc_terminal_delete_fifo(fd, &terminal->fifos);
-+			return LXC_MAINLOOP_CONTINUE;
-+		}  else {
- 			ERROR("Handler received unexpected file descriptor");
- 		}
- 		close(fd);
-@@ -392,7 +412,7 @@ int lxc_terminal_io_cb(int fd, uint32_t events, void *data,
- 		return LXC_MAINLOOP_CLOSE;
- 	}
- 
--	if (fd == terminal->peer)
-+	if (fd == terminal->peer || lxc_terminal_is_fifo(fd, &terminal->fifos))
- 		w = lxc_write_nointr(terminal->master, buf, r);
- 
- 	w_rbuf = w_log = 0;
-@@ -401,6 +421,9 @@ int lxc_terminal_io_cb(int fd, uint32_t events, void *data,
- 		if (terminal->peer >= 0)
- 			w = lxc_write_nointr(terminal->peer, buf, r);
- 
-+		/* isulad: forward data to fifos */
-+		lxc_forward_data_to_fifo(&terminal->fifos, buf, r);
-+
- 		/* write to terminal ringbuffer */
- 		if (terminal->buffer_size > 0)
- 			w_rbuf = lxc_ringbuf_write(&terminal->ringbuf, buf, r);
-@@ -450,6 +473,27 @@ static int lxc_terminal_mainloop_add_peer(struct lxc_terminal *terminal)
- 	return 0;
- }
- 
-+/* isulad add fifo to mainloop */
-+static int lxc_console_mainloop_add_fifo(struct lxc_terminal *terminal)
-+{
-+	int ret = 0;
-+	struct lxc_list *it,*next;
-+	struct lxc_fifos_fd *elem = NULL;
-+
-+	lxc_list_for_each_safe(it, &terminal->fifos, next) {
-+		elem = it->elem;
-+		if (elem->in_fd >= 0) {
-+			ret = lxc_mainloop_add_handler(terminal->descr, elem->in_fd,
-+						     lxc_terminal_io_cb, terminal);
-+			if (ret) {
-+				ERROR("console fifo %s not added to mainloop", elem->in_fifo);
-+				return -1;
-+			}
-+		}
-+	}
-+	return ret;
-+}
-+
- int lxc_terminal_mainloop_add(struct lxc_epoll_descr *descr,
- 			      struct lxc_terminal *terminal)
- {
-@@ -473,7 +517,20 @@ int lxc_terminal_mainloop_add(struct lxc_epoll_descr *descr,
- 	 */
- 	terminal->descr = descr;
- 
--	return lxc_terminal_mainloop_add_peer(terminal);
-+	ret = lxc_terminal_mainloop_add_peer(terminal);
-+	if (ret < 0) {
-+		ERROR("Failed to add handler for terminal peer to mainloop");
-+		return -1;
-+	}
-+
-+	/* isulad add fifo to mainloop */
-+	ret = lxc_console_mainloop_add_fifo(terminal);
-+	if (ret < 0) {
-+		ERROR("Failed to add handler for terminal fifos to mainloop");
-+		return -1;
-+	}
-+
-+	return 0;
- }
- 
- int lxc_setup_tios(int fd, struct termios *oldtios)
-@@ -812,6 +869,9 @@ void lxc_terminal_delete(struct lxc_terminal *terminal)
- 	if (terminal->log_fd >= 0)
- 		close(terminal->log_fd);
- 	terminal->log_fd = -1;
-+
-+	/* isulad: delete all fifos */
-+	lxc_terminal_delete_fifo(-1, &terminal->fifos);
- }
- 
- /**
-@@ -880,6 +940,77 @@ int lxc_terminal_create_log_file(struct lxc_terminal *terminal)
- 	return 0;
- }
- 
-+/* isulad: open terminal fifos */
-+static int terminal_fifo_open(const char *fifo_path, int flags)
-+{
-+	int fd = -1;
-+
-+	fd = open(fifo_path, flags);
-+	if (fd < 0) {
-+		WARN("Failed to open fifo %s to send message: %s.", fifo_path,
-+		     strerror(errno));
-+		return -1;
-+	}
-+
-+	return fd;
-+}
-+
-+/* isulad: set terminal fifos */
-+static int lxc_terminal_set_fifo(struct lxc_terminal *console, const char *in, const char *out)
-+{
-+	int fifofd_in = -1, fifofd_out = -1;
-+	struct lxc_fifos_fd *fifo_elem = NULL;
-+
-+	if (!in || !out)
-+		return -1;
-+
-+	if (!fifo_exists(in) || !fifo_exists(out)) {
-+		ERROR("File %s or %s does not refer to a FIFO", in, out);
-+		return -1;
-+	}
-+
-+	fifofd_in = terminal_fifo_open(in, O_RDONLY | O_NONBLOCK | O_CLOEXEC);
-+	if (fifofd_in < 0) {
-+		ERROR("Failed to open FIFO: %s", in);
-+		return -1;
-+	}
-+
-+	fifofd_out = terminal_fifo_open(out, O_WRONLY | O_NONBLOCK | O_CLOEXEC);
-+	if (fifofd_out < 0) {
-+		ERROR("Failed to open FIFO: %s", out);
-+		close(fifofd_in);
-+		return -1;
-+	}
-+
-+	fifo_elem = malloc(sizeof(*fifo_elem));
-+	if (!fifo_elem) {
-+		close(fifofd_in);
-+		close(fifofd_out);
-+		return -1;
-+	}
-+	memset(fifo_elem, 0, sizeof(*fifo_elem));
-+
-+	fifo_elem->in_fifo = strdup(in);
-+	fifo_elem->out_fifo = strdup(out);
-+	fifo_elem->in_fd = fifofd_in;
-+	fifo_elem->out_fd = fifofd_out;
-+	lxc_list_add_elem(&fifo_elem->node, fifo_elem);
-+	lxc_list_add_tail(&console->fifos, &fifo_elem->node);
-+
-+	return fifofd_in;
-+}
-+
-+/* isulad: add default fifos */
-+static int lxc_terminal_fifo_default(struct lxc_terminal *terminal)
-+{
-+	if (!terminal->init_fifo[0] || !terminal->init_fifo[1]) {
-+		ERROR("Invalid default terminal fifos");
-+		return -1;
-+	}
-+
-+	return lxc_terminal_set_fifo(terminal, terminal->init_fifo[0], terminal->init_fifo[1]);
-+}
-+
- int lxc_terminal_create(struct lxc_terminal *terminal)
- {
- 	int ret;
-@@ -902,6 +1033,13 @@ int lxc_terminal_create(struct lxc_terminal *terminal)
- 		goto err;
- 	}
- 
-+	/* isulad: make master NONBLOCK */
-+	ret = fd_nonblock(terminal->master);
-+	if (ret < 0) {
-+		SYSERROR("Failed to set O_NONBLOCK flag on terminal master");
-+		goto err;
-+	}
-+
- 	ret = fd_cloexec(terminal->slave, true);
- 	if (ret < 0) {
- 		SYSERROR("Failed to set FD_CLOEXEC flag on terminal slave");
-@@ -914,6 +1052,13 @@ int lxc_terminal_create(struct lxc_terminal *terminal)
- 		goto err;
- 	}
- 
-+	/* isulad: open fifos */
-+	ret = lxc_terminal_fifo_default(terminal);
-+	if (ret < 0) {
-+		ERROR("Failed to allocate fifo terminal");
-+		goto err;
-+	}
-+
- 	return 0;
- 
- err:
-@@ -1198,12 +1343,55 @@ void lxc_terminal_init(struct lxc_terminal *terminal)
- 	lxc_terminal_info_init(&terminal->proxy);
- }
- 
-+/* isulad: judge the fd whether is fifo */
-+static bool lxc_terminal_is_fifo(int fd, struct lxc_list *list)
-+{
-+	struct lxc_list *it,*next;
-+	struct lxc_fifos_fd *elem = NULL;
-+
-+	lxc_list_for_each_safe(it, list, next) {
-+		elem = it->elem;
-+		if (elem->in_fd == fd)
-+			return true;
-+	}
-+
-+	return false;
-+}
-+
-+/* isulad: if fd == -1, means delete all the fifos*/
-+int lxc_terminal_delete_fifo(int fd, struct lxc_list *list)
-+{
-+	struct lxc_list *it,*next;
-+	struct lxc_fifos_fd *elem = NULL;
-+
-+	lxc_list_for_each_safe(it, list, next) {
-+		elem = it->elem;
-+		if (elem->in_fd == fd || -1 == fd) {
-+			INFO("Delete fifo fd %d", fd);
-+			lxc_list_del(it);
-+			if (elem->in_fifo)
-+				free(elem->in_fifo);
-+			if (elem->out_fifo)
-+				free(elem->out_fifo);
-+			close(elem->in_fd);
-+			close(elem->out_fd);
-+			free(elem);
-+		}
-+	}
-+
-+	return 0;
-+}
-+
- void lxc_terminal_conf_free(struct lxc_terminal *terminal)
- {
- 	free(terminal->log_path);
- 	free(terminal->path);
- 	if (terminal->buffer_size > 0 && terminal->ringbuf.addr)
- 		lxc_ringbuf_release(&terminal->ringbuf);
-+	/*isulad: free console fifos */
-+	free(terminal->init_fifo[0]);
-+	free(terminal->init_fifo[1]);
-+	lxc_terminal_delete_fifo(-1, &terminal->fifos);
- }
- 
- int lxc_terminal_map_ids(struct lxc_conf *c, struct lxc_terminal *terminal)
-diff --git a/src/lxc/terminal.h b/src/lxc/terminal.h
-index bfd271f..d25da65 100644
---- a/src/lxc/terminal.h
-+++ b/src/lxc/terminal.h
-@@ -115,6 +115,17 @@ struct lxc_terminal {
- 		/* the in-memory ringbuffer */
- 		struct lxc_ringbuf ringbuf;
- 	};
-+	char *init_fifo[2]; /* isulad: default fifos for the start */
-+	struct lxc_list fifos; /* isulad: fifos used to forward teminal */
-+};
-+
-+/* isulad: fifo struct */
-+struct lxc_fifos_fd {
-+	char *in_fifo;
-+	char *out_fifo;
-+	int in_fd;
-+	int out_fd;
-+	struct lxc_list node;
- };
- 
- /**
-@@ -295,4 +306,9 @@ extern void lxc_terminal_init(struct lxc_terminal *terminal);
- extern int lxc_terminal_map_ids(struct lxc_conf *c,
- 				struct lxc_terminal *terminal);
- 
-+/* isulad: judge the fd whether is fifo*/
-+static bool lxc_terminal_is_fifo(int fd, struct lxc_list *list);
-+/* isulad: if fd == -1, means delete all the fifos*/
-+int lxc_terminal_delete_fifo(int fd, struct lxc_list *list);
-+
- #endif /* __LXC_TERMINAL_H */
-diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
-index 810050a..b7af2b5 100644
---- a/src/lxc/tools/arguments.h
-+++ b/src/lxc/tools/arguments.h
-@@ -62,6 +62,7 @@ struct lxc_arguments {
- 
- 	/* for lxc-start */
- 	const char *share_ns[32]; /* size must be greater than LXC_NS_MAX */
-+	const char *terminal_fifos[2]; /* isulad add, fifos used to redirct stdin/out/err */
- 
- 	/* for lxc-console */
- 	unsigned int ttynum;
-@@ -172,6 +173,10 @@ struct lxc_arguments {
- #define OPT_SHARE_IPC OPT_USAGE - 4
- #define OPT_SHARE_UTS OPT_USAGE - 5
- #define OPT_SHARE_PID OPT_USAGE - 6
-+/* isulad add begin */
-+#define OPT_INPUT_FIFO OPT_USAGE - 7
-+#define OPT_OUTPUT_FIFO OPT_USAGE - 8
-+/* isulad add end*/
- 
- extern int lxc_arguments_parse(struct lxc_arguments *args, int argc,
- 			       char *const argv[]);
-diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
-index 4553cb5..8f03f11 100644
---- a/src/lxc/tools/lxc_start.c
-+++ b/src/lxc/tools/lxc_start.c
-@@ -69,6 +69,8 @@ static const struct option my_longopts[] = {
- 	{"share-ipc", required_argument, 0, OPT_SHARE_IPC},
- 	{"share-uts", required_argument, 0, OPT_SHARE_UTS},
- 	{"share-pid", required_argument, 0, OPT_SHARE_PID},
-+	{"in-fifo", required_argument, 0, OPT_INPUT_FIFO},
-+	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
- 	LXC_COMMON_OPTIONS
- };
- 
-@@ -140,6 +142,12 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
- 	case OPT_SHARE_PID:
- 		args->share_ns[LXC_NS_PID] = arg;
- 		break;
-+	case OPT_INPUT_FIFO:
-+		args->terminal_fifos[0] = arg;
-+		break;
-+	case OPT_OUTPUT_FIFO:
-+		args->terminal_fifos[1] = arg;
-+		break;
- 	}
- 	return 0;
- }
-@@ -322,6 +330,9 @@ int main(int argc, char *argv[])
- 	if (my_args.close_all_fds)
- 		c->want_close_all_fds(c, true);
- 
-+	if (my_args.terminal_fifos[0] && my_args.terminal_fifos[1])
-+		c->set_terminal_init_fifos(c, my_args.terminal_fifos[0], my_args.terminal_fifos[1]);
-+
- 	if (args == default_args)
- 		err = c->start(c, 0, NULL) ? EXIT_SUCCESS : EXIT_FAILURE;
- 	else
-diff --git a/src/lxc/utils.c b/src/lxc/utils.c
-index 6e9165a..67c3b3e 100644
---- a/src/lxc/utils.c
-+++ b/src/lxc/utils.c
-@@ -611,6 +611,19 @@ bool dir_exists(const char *path)
- 	return S_ISDIR(sb.st_mode);
- }
- 
-+bool fifo_exists(const char *path)
-+{
-+	struct stat sb;
-+	int ret;
-+
-+	ret = stat(path, &sb);
-+	if (ret < 0)
-+		// could be something other than eexist, just say no
-+		return false;
-+	return S_ISFIFO(sb.st_mode);
-+}
-+
-+
- /* Note we don't use SHA-1 here as we don't want to depend on HAVE_GNUTLS.
-  * FNV has good anti collision properties and we're not worried
-  * about pre-image resistance or one-way-ness, we're just trying to make
-@@ -1715,6 +1728,16 @@ int fd_cloexec(int fd, bool cloexec)
- 	return 0;
- }
- 
-+/* isulad: fd_nonblock */
-+int fd_nonblock(int fd)
-+{
-+	long flags;
-+
-+        flags = fcntl(fd, F_GETFL);
-+
-+	return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
-+}
-+
- int recursive_destroy(char *dirname)
- {
- 	int ret;
-diff --git a/src/lxc/utils.h b/src/lxc/utils.h
-index 94196d0..2d38178 100644
---- a/src/lxc/utils.h
-+++ b/src/lxc/utils.h
-@@ -147,6 +147,8 @@ extern gid_t get_ns_gid(gid_t orig);
- 
- extern bool dir_exists(const char *path);
- 
-+extern bool fifo_exists(const char *path);
-+
- #define FNV1A_64_INIT ((uint64_t)0xcbf29ce484222325ULL)
- extern uint64_t fnv_64a_buf(void *buf, size_t len, uint64_t hval);
- 
-@@ -242,4 +244,6 @@ extern int fd_cloexec(int fd, bool cloexec);
- extern int recursive_destroy(char *dirname);
- extern int lxc_setup_keyring(void);
- 
-+extern int fd_nonblock(int fd);
-+
- #endif /* __LXC_UTILS_H */
--- 
-1.8.3.1
-

0009-support-mount-squashfs-in-mount-entry.patch

@@ -1,33 +1,35 @@
-From 572633fe6deaea6b8e4d423225aab452c6f0b7c0 Mon Sep 17 00:00:00 2001
+From 6124835dde5abfeeb8ac796813f2f18803b96117 Mon Sep 17 00:00:00 2001
 From: tanyifeng <tanyifeng1@huawei.com>
 Date: Tue, 15 Jan 2019 19:54:13 +0800
-Subject: [PATCH 033/140] support mount squashfs in mount entry
+Subject: [PATCH 09/49] support mount squashfs in mount entry
 
 Signed-off-by: LiFeng <lifeng68@huawei.com>
+Signed-off-by: WangFengTu <wangfengtu@huawei.com>
 ---
- src/lxc/conf.c                  | 88 +++++++++++++++++++++++++++++++++++++++--
+ src/lxc/conf.c                  | 95 +++++++++++++++++++++++++++++++++++++++--
- src/lxc/storage/loop.c          | 36 ++++++++++++++---
+ src/lxc/storage/loop.c          | 36 +++++++++++++---
- src/lxc/storage/storage_utils.c | 36 ++++++++++++++++-
+ src/lxc/storage/storage_utils.c | 36 +++++++++++++++-
- src/lxc/utils.c                 | 33 ++++++++++++++--
+ src/lxc/utils.c                 | 35 ++++++++++++++-
  src/lxc/utils.h                 |  1 +
- 5 files changed, 181 insertions(+), 13 deletions(-)
+ 5 files changed, 191 insertions(+), 12 deletions(-)
 
 diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 54b967b..fea0f59 100644
+index 35488e0..1f779b9 100644
 --- a/src/lxc/conf.c
 +++ b/src/lxc/conf.c
-@@ -78,6 +78,7 @@
+@@ -58,6 +58,7 @@
  #include "storage/overlay.h"
  #include "syscall_wrappers.h"
  #include "terminal.h"
 +#include "loop.h"
- #include "path.h"
  #include "utils.h"
+ #include "uuid.h"
  
-@@ -2444,6 +2445,82 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
+@@ -2013,6 +2014,84 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
  	return 0;
  }
  
++#ifdef HAVE_ISULAD
 +static int mount_entry_with_loop_dev(const char *src, const char *dest, const char *fstype,
 +		char *mnt_opts, const char *rootfs)
 +{
@@ -103,48 +105,54 @@ index 54b967b..fea0f59 100644
 +
 +	return 0;
 +}
++#endif
 +
  /* rootfs, lxc_name, and lxc_path can be NULL when the container is created
   * without a rootfs. */
  static inline int mount_entry_on_generic(struct mntent *mntent,
-@@ -2502,8 +2579,14 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
+@@ -2026,6 +2105,7 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
- 		return -1;
+ 	char *rootfs_path = NULL;
- 	}
+ 	int ret;
+ 	bool dev, optional, relative;
++        const char *dest = path;
  
--	ret = mount_entry(mntent->mnt_fsname, dest, mntent->mnt_type, mntflags,
+ 	optional = hasmntopt(mntent, "optional") != NULL;
+ 	dev = hasmntopt(mntent, "dev") != NULL;
+@@ -2052,9 +2132,18 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
+ 	if (ret < 0)
+ 		return ret;
+ 
+-	ret = mount_entry(mntent->mnt_fsname, path, mntent->mnt_type, mntflags,
 -			  pflags, mntdata, optional, dev, relative, rootfs_path);
+-
++#ifdef HAVE_ISULAD
 +	// isulad: support squashfs
 +	if (strcmp(mntent->mnt_type, "squashfs") == 0) {
 +		ret = mount_entry_with_loop_dev(mntent->mnt_fsname, dest, mntent->mnt_type,
 +						mntent->mnt_opts, rootfs_path);
-+	} else {
++        } else {
-+		ret = mount_entry(mntent->mnt_fsname, dest, mntent->mnt_type, mntflags,
++#endif
-+				  pflags, mntdata, optional, dev, relative, rootfs_path);
++	        ret = mount_entry(mntent->mnt_fsname, dest, mntent->mnt_type, mntflags,
-+	}
++			          pflags, mntdata, optional, dev, relative, rootfs_path);
- 
++#ifdef HAVE_ISULAD
- 	free(mntdata);
++        }
- 	free(rpath);
++#endif
-@@ -3897,7 +3980,6 @@ static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs)
+ 	return ret;
- 	return 0;
  }
  
--
- int lxc_setup(struct lxc_handler *handler)
- {
- 	int ret;
 diff --git a/src/lxc/storage/loop.c b/src/lxc/storage/loop.c
-index 35cb13e..760def8 100644
+index eebc1b6..345be50 100644
 --- a/src/lxc/storage/loop.c
 +++ b/src/lxc/storage/loop.c
-@@ -41,6 +41,7 @@
+@@ -21,6 +21,7 @@
- #include "loop.h"
+ #include "memory_utils.h"
  #include "storage.h"
  #include "storage_utils.h"
 +#include "lxclock.h"
  #include "utils.h"
  
  lxc_log_define(loop, lxc);
-@@ -236,9 +237,11 @@ bool loop_detect(const char *path)
+@@ -216,9 +217,11 @@ bool loop_detect(const char *path)
  
  int loop_mount(struct lxc_storage *bdev)
  {
@@ -157,7 +165,7 @@ index 35cb13e..760def8 100644
  
  	if (strcmp(bdev->type, "loop"))
  		return -22;
-@@ -246,13 +249,29 @@ int loop_mount(struct lxc_storage *bdev)
+@@ -226,13 +229,29 @@ int loop_mount(struct lxc_storage *bdev)
  	if (!bdev->src || !bdev->dest)
  		return -22;
  
@@ -188,7 +196,7 @@ index 35cb13e..760def8 100644
  	}
  	DEBUG("Prepared loop device \"%s\"", loname);
  
-@@ -261,14 +280,21 @@ int loop_mount(struct lxc_storage *bdev)
+@@ -241,14 +260,21 @@ int loop_mount(struct lxc_storage *bdev)
  		ERROR("Failed to mount rootfs \"%s\" on \"%s\" via loop device \"%s\"",
  		      bdev->src, bdev->dest, loname);
  		close(loopfd);
@@ -214,10 +222,10 @@ index 35cb13e..760def8 100644
  
  int loop_umount(struct lxc_storage *bdev)
 diff --git a/src/lxc/storage/storage_utils.c b/src/lxc/storage/storage_utils.c
-index b4dcb57..0a87778 100644
+index bfbb782..07eee22 100644
 --- a/src/lxc/storage/storage_utils.c
 +++ b/src/lxc/storage/storage_utils.c
-@@ -339,10 +339,14 @@ int is_blktype(struct lxc_storage *b)
+@@ -259,10 +259,14 @@ int is_blktype(struct lxc_storage *b)
  	return 0;
  }
  
@@ -232,7 +240,7 @@ index b4dcb57..0a87778 100644
  	int ret;
  	struct cbarg {
  		const char *rootfs;
-@@ -371,15 +375,30 @@ int mount_unknown_fs(const char *rootfs, const char *target,
+@@ -291,15 +295,30 @@ int mount_unknown_fs(const char *rootfs, const char *target,
  		ret = lxc_file_for_each_line(fsfile[i], find_fstype_cb, &cbarg);
  		if (ret < 0) {
  			ERROR("Failed to parse \"%s\"", fsfile[i]);
@@ -265,17 +273,17 @@ index b4dcb57..0a87778 100644
  	return -1;
  }
  
-@@ -399,6 +418,8 @@ int find_fstype_cb(char *buffer, void *data)
+@@ -318,6 +337,8 @@ int find_fstype_cb(char *buffer, void *data)
- 	unsigned long mntflags, pflags;
+ 	unsigned long mntflags = 0;
- 	char *mntdata;
+ 	char *mntdata = NULL;
  	char *fstype;
 +	char mount_err[BUFSIZ] = {0};
 +	int ret;
  
  	/* we don't try 'nodev' entries */
  	if (strstr(buffer, "nodev"))
-@@ -419,6 +440,17 @@ int find_fstype_cb(char *buffer, void *data)
+@@ -342,6 +363,17 @@ int find_fstype_cb(char *buffer, void *data)
- 	if (mount(cbarg->rootfs, cbarg->target, fstype, (mntflags & ~MS_RDONLY), mntdata)) {
+ #endif
  		SYSDEBUG("Failed to mount");
  		free(mntdata);
 +		// isulad: recored error
@@ -293,10 +301,18 @@ index b4dcb57..0a87778 100644
  	}
  
 diff --git a/src/lxc/utils.c b/src/lxc/utils.c
-index d1a22f7..120a13d 100644
+index ebcdae0..90113e0 100644
 --- a/src/lxc/utils.c
 +++ b/src/lxc/utils.c
-@@ -1053,7 +1053,7 @@ static int open_if_safe(int dirfd, const char *nextpath)
+@@ -27,6 +27,7 @@
+ #include <sys/types.h>
+ #include <sys/wait.h>
+ #include <unistd.h>
++#include <sys/sysmacros.h>
+ 
+ #include "config.h"
+ #include "log.h"
+@@ -1008,7 +1009,7 @@ static int open_if_safe(int dirfd, const char *nextpath)
   *
   * Return an open fd for the path, or <0 on error.
   */
@@ -305,7 +321,7 @@ index d1a22f7..120a13d 100644
  {
  	int curlen = 0, dirfd, fulllen, i;
  	char *dup;
-@@ -1473,6 +1473,9 @@ static int lxc_get_unused_loop_dev(char *name_loop)
+@@ -1425,6 +1426,9 @@ static int lxc_get_unused_loop_dev(char *name_loop)
  {
  	int loop_nr, ret;
  	int fd_ctl = -1, fd_tmp = -1;
@@ -315,15 +331,15 @@ index d1a22f7..120a13d 100644
  
  	fd_ctl = open("/dev/loop-control", O_RDWR | O_CLOEXEC);
  	if (fd_ctl < 0) {
-@@ -1489,10 +1492,34 @@ static int lxc_get_unused_loop_dev(char *name_loop)
+@@ -1441,9 +1445,35 @@ static int lxc_get_unused_loop_dev(char *name_loop)
  	ret = snprintf(name_loop, LO_NAME_SIZE, "/dev/loop%d", loop_nr);
  	if (ret < 0 || ret >= LO_NAME_SIZE)
  		goto on_error;
 -
 +retry:
  	fd_tmp = open(name_loop, O_RDWR | O_CLOEXEC);
--	if (fd_tmp < 0)
+ 	if (fd_tmp < 0) {
-+	if (fd_tmp < 0) {
++#if HAVE_ISULAD
 +		/* Success of LOOP_CTL_GET_FREE doesn't mean /dev/loop$i is ready,
 +		* we try to make node by ourself to avoid wait. */
 +		if (try_mknod) {
@@ -346,17 +362,25 @@ index d1a22f7..120a13d 100644
 +			usleep(5000); /* 5 millisecond */
 +			goto retry;
 +		}
- 		SYSERROR("Failed to open loop \"%s\"", name_loop);
++		SYSERROR("Failed to open loop \"%s\"", name_loop);
 +		goto on_error;
-+	}
++#else
+ 		/* on Android loop devices are moved under /dev/block, give it a shot */
+ 		ret = snprintf(name_loop, LO_NAME_SIZE, "/dev/block/loop%d", loop_nr);
+                 if (ret < 0 || ret >= LO_NAME_SIZE)
+@@ -1452,6 +1482,7 @@ static int lxc_get_unused_loop_dev(char *name_loop)
+ 		fd_tmp = open(name_loop, O_RDWR | O_CLOEXEC);
+ 		if (fd_tmp < 0)
+ 			SYSERROR("Failed to open loop \"%s\"", name_loop);
++#endif
+ 	}
  
  on_error:
- 	close(fd_ctl);
 diff --git a/src/lxc/utils.h b/src/lxc/utils.h
-index abc88ca..4313942 100644
+index 11d6548..fbb0d55 100644
 --- a/src/lxc/utils.h
 +++ b/src/lxc/utils.h
-@@ -224,6 +224,7 @@ extern bool cgns_supported(void);
+@@ -215,6 +215,7 @@ extern bool cgns_supported(void);
  extern char *choose_init(const char *rootfs);
  extern bool switch_to_ns(pid_t pid, const char *ns);
  extern char *get_template_path(const char *t);

0010-Save-pid-ppid-info-into-file-for-isulad.patch

@@ -1,453 +0,0 @@
-From 523b705fb269a9463b9412aa3a814a9c71a743cc Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Fri, 11 Jan 2019 22:53:56 -0500
-Subject: [PATCH 010/140] Save pid/ppid info into file for isulad
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c            |   1 +
- src/lxc/conf.h            |   2 +
- src/lxc/lxccontainer.c    |  24 ++++++++++
- src/lxc/lxccontainer.h    |  10 ++++
- src/lxc/start.c           |  41 +++++++++++++++++
- src/lxc/tools/arguments.h |   2 +
- src/lxc/tools/lxc_start.c |  20 ++++++++
- src/lxc/utils.c           | 114 ++++++++++++++++++++++++++++++++++++++++++++++
- src/lxc/utils.h           |  66 +++++++++++++++++++++++++++
- 9 files changed, 280 insertions(+)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 7b7f95b..0b4b63b 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -4194,6 +4194,7 @@ void lxc_conf_free(struct lxc_conf *conf)
- 	/* isulad add begin */
- 	lxc_clear_init_args(conf);
- 	lxc_clear_populate_devices(conf);
-+	free(conf->container_info_file);
- 	/* isulad add end */
- 	free(conf);
- }
-diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index cced868..e0954f9 100644
---- a/src/lxc/conf.h
-+++ b/src/lxc/conf.h
-@@ -403,6 +403,8 @@ struct lxc_conf {
- 	size_t init_argc;
- 	/* populate devices*/
- 	struct lxc_list populate_devs;
-+
-+	char *container_info_file;
- 	/* isulad add end */
- };
- 
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index 318c71e..5679b9b 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -4988,6 +4988,29 @@ static bool do_lxcapi_set_terminal_default_fifos(struct lxc_container *c, const
- 
- WRAP_API_2(bool, lxcapi_set_terminal_default_fifos, const char *, const char *)
- 
-+/* isulad add set info file path */
-+static bool do_lxcapi_set_container_info_file(struct lxc_container *c, const char *info_file)
-+{
-+	struct lxc_conf *conf;
-+
-+	if (!c || !c->lxc_conf || !info_file)
-+		return false;
-+	if (container_mem_lock(c)) {
-+		ERROR("Error getting mem lock");
-+		return false;
-+	}
-+
-+	conf = c->lxc_conf;
-+	if (conf->container_info_file)
-+		free(conf->container_info_file);
-+	conf->container_info_file = strdup(info_file);
-+
-+	container_mem_unlock(c);
-+	return true;
-+}
-+
-+WRAP_API_1(bool, lxcapi_set_container_info_file, const char *)
-+
- struct lxc_container *lxc_container_new(const char *name, const char *configpath)
- {
- 	struct lxc_container *c;
-@@ -5113,6 +5136,7 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
- 
- 	/* isulad add begin */
- 	c->set_terminal_init_fifos = lxcapi_set_terminal_default_fifos;
-+	c->set_container_info_file = lxcapi_set_container_info_file;
- 	/* isulad add end */
- 	return c;
- 
-diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
-index 486531e..3c845fe 100644
---- a/src/lxc/lxccontainer.h
-+++ b/src/lxc/lxccontainer.h
-@@ -857,6 +857,16 @@ struct lxc_container {
- 	 * \return \c true on success, else \c false.
- 	 */
- 	bool (*set_terminal_init_fifos)(struct lxc_container *c, const char *in, const char *out);
-+
-+	/*! isulad add
-+	 * \brief An API call to set the path of info file
-+	 *
-+	 * \param c Container.
-+	 * \param info_file Value of the path of info file.
-+	 *
-+	 * \return \c true on success, else \c false.
-+	 */
-+	bool (*set_container_info_file) (struct lxc_container *c, const char *info_file);
- };
- 
- /*!
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index d64bdac..9d71dd7 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -1553,6 +1553,39 @@ static inline int do_share_ns(void *arg)
- 	return 0;
- }
- 
-+/* isuald: save pid/ppid info */
-+static int lxc_save_container_info(char *filename, pid_t pid)
-+{
-+	FILE *pid_fp = NULL;
-+	int ret = 0;
-+	pid_t p_pid = 0;
-+	unsigned long long start_at = 0;
-+	unsigned long long p_start_at = 0;
-+
-+	pid_fp = fopen(filename, "w");
-+	if (pid_fp == NULL) {
-+		SYSERROR("Failed to create pidfile '%s'",filename);
-+		ret = -1;
-+		goto out;
-+	}
-+
-+	start_at = lxc_get_process_startat(pid);
-+
-+	p_pid = getpid();
-+	p_start_at = lxc_get_process_startat(p_pid);
-+
-+	if (fprintf(pid_fp, "%d %llu %d %llu\n", pid, start_at, p_pid, p_start_at) < 0) {
-+		SYSERROR("Failed to write '%s'", filename);
-+		ret = -1;
-+		goto out;
-+	}
-+out:
-+	if (pid_fp)
-+		fclose(pid_fp);
-+	pid_fp = NULL;
-+	return ret;
-+}
-+
- /* lxc_spawn() performs crucial setup tasks and clone()s the new process which
-  * exec()s the requested container binary.
-  * Note that lxc_spawn() runs in the parent namespaces. Any operations performed
-@@ -1683,6 +1716,14 @@ static int lxc_spawn(struct lxc_handler *handler)
- 	}
- 	TRACE("Cloned child process %d", handler->pid);
- 
-+	/* isulad: save pid/ppid info into file*/
-+	if (handler->conf->container_info_file) {
-+		if (lxc_save_container_info(handler->conf->container_info_file, handler->pid)) {
-+			ERROR("Failed to save cloned container pid");
-+			goto out_delete_net;
-+		}
-+	}
-+
- 	for (i = 0; i < LXC_NS_MAX; i++)
- 		if (handler->ns_on_clone_flags & ns_info[i].clone_flag)
- 			INFO("Cloned %s", ns_info[i].flag_name);
-diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
-index b7af2b5..b6df23f 100644
---- a/src/lxc/tools/arguments.h
-+++ b/src/lxc/tools/arguments.h
-@@ -63,6 +63,7 @@ struct lxc_arguments {
- 	/* for lxc-start */
- 	const char *share_ns[32]; /* size must be greater than LXC_NS_MAX */
- 	const char *terminal_fifos[2]; /* isulad add, fifos used to redirct stdin/out/err */
-+	const char *container_info; /* isulad: file used to store pid and ppid info of container */
- 
- 	/* for lxc-console */
- 	unsigned int ttynum;
-@@ -176,6 +177,7 @@ struct lxc_arguments {
- /* isulad add begin */
- #define OPT_INPUT_FIFO OPT_USAGE - 7
- #define OPT_OUTPUT_FIFO OPT_USAGE - 8
-+#define OPT_CONTAINER_INFO OPT_USAGE - 9
- /* isulad add end*/
- 
- extern int lxc_arguments_parse(struct lxc_arguments *args, int argc,
-diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
-index 8f03f11..2f94d67 100644
---- a/src/lxc/tools/lxc_start.c
-+++ b/src/lxc/tools/lxc_start.c
-@@ -69,8 +69,11 @@ static const struct option my_longopts[] = {
- 	{"share-ipc", required_argument, 0, OPT_SHARE_IPC},
- 	{"share-uts", required_argument, 0, OPT_SHARE_UTS},
- 	{"share-pid", required_argument, 0, OPT_SHARE_PID},
-+	/* isulad add begin */
- 	{"in-fifo", required_argument, 0, OPT_INPUT_FIFO},
- 	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
-+	{"container-pidfile", required_argument, 0, OPT_CONTAINER_INFO},
-+	/* isulad add end */
- 	LXC_COMMON_OPTIONS
- };
- 
-@@ -148,6 +151,9 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
- 	case OPT_OUTPUT_FIFO:
- 		args->terminal_fifos[1] = arg;
- 		break;
-+	case OPT_CONTAINER_INFO:
-+		args->container_info = arg;
-+		break;
- 	}
- 	return 0;
- }
-@@ -189,6 +195,7 @@ int main(int argc, char *argv[])
- 	struct lxc_log log;
- 	int err = EXIT_FAILURE;
- 	char *rcfile = NULL;
-+	char *container_info_file = NULL; /* isulad: info file*/
- 	char *const default_args[] = {
- 		"/sbin/init",
- 		NULL,
-@@ -313,6 +320,18 @@ int main(int argc, char *argv[])
- 			goto out;
- 		}
- 
-+	/* isulad: container info file used to store pid and ppid info of container*/
-+	if (my_args.container_info != NULL) {
-+		if (ensure_path(&container_info_file, my_args.container_info) < 0) {
-+			ERROR("Failed to ensure container's piddile '%s'", my_args.container_info);
-+			goto out;
-+		}
-+		if (!c->set_container_info_file(c, container_info_file)) {
-+			ERROR("Failed to set container's piddile '%s'", container_info_file);
-+			goto out;
-+		}
-+	}
-+
- 	if (my_args.console)
- 		if (!c->set_config_item(c, "lxc.console.path", my_args.console))
- 			goto out;
-@@ -353,5 +372,6 @@ int main(int argc, char *argv[])
- 
- out:
- 	lxc_container_put(c);
-+	free(container_info_file);
- 	exit(err);
- }
-diff --git a/src/lxc/utils.c b/src/lxc/utils.c
-index 67c3b3e..4728284 100644
---- a/src/lxc/utils.c
-+++ b/src/lxc/utils.c
-@@ -1828,3 +1828,117 @@ int lxc_setup_keyring(void)
- 
- 	return ret;
- }
-+
-+/* isulad: read file to buffer */
-+static int lxc_file2str(const char *filename, char ret[], int cap)
-+{
-+	int fd, num_read;
-+
-+	if ((fd = open(filename, O_RDONLY | O_CLOEXEC)) == -1)
-+		return -1;/*lint !e960*/
-+	if ((num_read = read(fd, ret, cap - 1)) <= 0)
-+		num_read = -1;/*lint !e960*/
-+	else
-+		ret[num_read] = 0;/*lint !e613*//*lint !e960*/
-+	close(fd);
-+
-+	return num_read;
-+}
-+
-+/* isuald: lxc_stat2proc() makes sure it can handle arbitrary executable file basenames
-+ * for `cmd', i.e. those with embedded whitespace or embedded ')'s.
-+ * Such names confuse %s (see scanf(3)), so the string is split and %39c
-+ * is used instead. (except for embedded ')' "(%[^)]c)" would work.
-+ */
-+static proc_t *lxc_stat2proc(char *S)
-+{
-+	int num;
-+	proc_t *P = NULL;
-+	char *tmp = NULL;
-+
-+	if (!S)
-+		return NULL;/*lint !e960*/
-+
-+	tmp = strrchr(S, ')');	/* split into "PID (cmd" and "<rest>" *//*lint !e586*/
-+	if (!tmp)
-+		return NULL;/*lint !e960*/
-+	*tmp = '\0';			/* replace trailing ')' with NUL */
-+
-+	P = malloc(sizeof(proc_t));
-+	if (!P)
-+		return NULL;/*lint !e960*/
-+	memset(P, 0x00, sizeof(proc_t));
-+
-+	/* parse these two strings separately, skipping the leading "(". */
-+	num = sscanf(S, "%d (%15c", &P->pid, P->cmd);   /* comm[16] in kernel */
-+	if (num < 0 && errno) {
-+		ERROR("Call sscanf error: %s", strerror(errno));
-+		free(P);
-+		return NULL;
-+	}
-+	num = sscanf(tmp + 2,			/* skip space after ')' too */
-+		     "%c "
-+		     "%d %d %d %d %d "
-+		     "%lu %lu %lu %lu %lu "
-+		     "%Lu %Lu %Lu %Lu "  /* utime stime cutime cstime *//*lint !e566*/
-+		     "%ld %ld %ld %ld "
-+		     "%Lu "  /* start_time *//*lint !e566*/
-+		     "%lu "
-+		     "%ld "
-+		     "%lu %lu %lu %lu %lu %lu "
-+		     "%*s %*s %*s %*s " /* discard, no RT signals & Linux 2.1 used hex */
-+		     "%lu %lu %lu "
-+		     "%d %d "
-+		     "%lu %lu",
-+		     &P->state,
-+		     &P->ppid, &P->pgrp, &P->session, &P->tty, &P->tpgid,
-+		     &P->flags, &P->min_flt, &P->cmin_flt, &P->maj_flt, &P->cmaj_flt,
-+		     &P->utime, &P->stime, &P->cutime, &P->cstime,/*lint !e561*/
-+		     &P->priority, &P->nice, &P->timeout, &P->it_real_value,
-+		     &P->start_time,/*lint !e561*/
-+		     &P->vsize,
-+		     &P->rss,
-+		     &P->rss_rlim, &P->start_code, &P->end_code, &P->start_stack, &P->kstk_esp,
-+		     &P->kstk_eip,
-+		     &P->wchan, &P->nswap, &P->cnswap,
-+		     &P->exit_signal, &P->processor,  /* 2.2.1 ends with "exit_signal" */
-+		     &P->rtprio, &P->sched  /* both added to 2.5.18 */
-+		    );
-+
-+	if (P->tty == 0)
-+		P->tty = -1;  /* the old notty val, update elsewhere bef. moving to 0 *//*lint !e960*/
-+	return P;
-+}
-+
-+/* isulad: get starttime of process pid */
-+unsigned long long lxc_get_process_startat(pid_t pid)
-+{
-+	int sret = 0;
-+	unsigned long long startat = 0;
-+	proc_t *pid_info = NULL;
-+	char filename[PATH_MAX] = {0};
-+	char sbuf[1024] = {0}; /* bufs for stat */
-+
-+	sret = snprintf(filename, sizeof(filename), "/proc/%d/stat", pid);
-+	if (sret < 0 || sret >= sizeof(filename)) {/*lint !e574*/
-+		ERROR("Failed to sprintf filename");
-+		goto out;
-+	}
-+
-+	if ((lxc_file2str(filename, sbuf, sizeof(sbuf))) == -1) {
-+		SYSERROR("Failed to read pidfile %s", filename);
-+		goto out;
-+	}
-+
-+	pid_info = lxc_stat2proc(sbuf);
-+	if (!pid_info) {/*lint !e574*/
-+		ERROR("Failed to get proc stat info");
-+		goto out;
-+	}
-+
-+	startat = pid_info->start_time;
-+out:
-+	free(pid_info);
-+	return startat;
-+}
-+
-diff --git a/src/lxc/utils.h b/src/lxc/utils.h
-index 2d38178..8e4ed89 100644
---- a/src/lxc/utils.h
-+++ b/src/lxc/utils.h
-@@ -56,6 +56,71 @@ extern char *get_rundir(void);
- #endif
- #endif
- 
-+/* isuald:
-+ ld	cutime, cstime, priority, nice, timeout, it_real_value, rss,
-+ c	state,
-+ d	ppid, pgrp, session, tty, tpgid,
-+ s	signal, blocked, sigignore, sigcatch,
-+ lu	flags, min_flt, cmin_flt, maj_flt, cmaj_flt, utime, stime,
-+ lu	rss_rlim, start_code, end_code, start_stack, kstk_esp, kstk_eip,
-+ lu	start_time, vsize, wchan, nswap, cnswap,
-+*/
-+
-+/* Basic data structure which holds all information we can get about a process.
-+ * (unless otherwise specified, fields are read from /proc/#/stat)
-+ *
-+ * Most of it comes from task_struct in linux/sched.h
-+ */
-+typedef struct proc_t {
-+	// 1st 16 bytes
-+	int pid;		/* process id */
-+	int	ppid;		/* pid of parent process */
-+
-+	char state;		/* single-char code for process state (S=sleeping) */
-+
-+	unsigned long long
-+	utime,		/* user-mode CPU time accumulated by process */
-+	stime,		/* kernel-mode CPU time accumulated by process */
-+	// and so on...
-+	cutime,		/* cumulative utime of process and reaped children */
-+	cstime,		/* cumulative stime of process and reaped children */
-+	start_time;	/* start time of process -- seconds since 1-1-70 */
-+
-+	long
-+	priority,	/* kernel scheduling priority */
-+	timeout,	/* ? */
-+	nice,		/* standard unix nice level of process */
-+	rss,		/* resident set size from /proc/#/stat (pages) */
-+	it_real_value;	/* ? */
-+	unsigned long
-+	rtprio,	    /* real-time priority */
-+	sched,	    /* scheduling class */
-+	vsize,		/* number of pages of virtual memory ... */
-+	rss_rlim,	/* resident set size limit? */
-+	flags,		/* kernel flags for the process */
-+	min_flt,	/* number of minor page faults since process start */
-+	maj_flt,	/* number of major page faults since process start */
-+	cmin_flt,	/* cumulative min_flt of process and child processes */
-+	cmaj_flt,	/* cumulative maj_flt of process and child processes */
-+	nswap,		/* ? */
-+	cnswap,		/* cumulative nswap ? */
-+	start_code,	/* address of beginning of code segment */
-+	end_code,	/* address of end of code segment */
-+	start_stack,	/* address of the bottom of stack for the process */
-+	kstk_esp,	/* kernel stack pointer */
-+	kstk_eip,	/* kernel instruction pointer */
-+	wchan;		/* address of kernel wait channel proc is sleeping in */
-+
-+	char cmd[16];	/* basename of executable file in call to exec(2) */
-+	int
-+	pgrp,		/* process group id */
-+	session,	/* session id */
-+	tty,		/* full device number of controlling terminal */
-+	tpgid,		/* terminal process group id */
-+	exit_signal,    /* might not be SIGCHLD */
-+	processor;	    /* current (or most recent?) CPU */
-+} proc_t;
-+
- static inline int lxc_set_cloexec(int fd)
- {
- 	return fcntl(fd, F_SETFD, FD_CLOEXEC);
-@@ -245,5 +310,6 @@ extern int recursive_destroy(char *dirname);
- extern int lxc_setup_keyring(void);
- 
- extern int fd_nonblock(int fd);
-+extern int unsigned long long lxc_get_process_startat(pid_t pid);
- 
- #endif /* __LXC_UTILS_H */
--- 
-1.8.3.1
-

0011-Add-exit-FIFO-to-monitor-state-of-lxc-monitor.patch

@@ -1,261 +0,0 @@
-From 70b6cb3c2a07f49af1f7dd251eac39e20c539f7e Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Sat, 12 Jan 2019 02:07:15 -0500
-Subject: [PATCH 011/140] Add exit FIFO to monitor state of [lxc monitor]
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c            |  3 +++
- src/lxc/conf.h            |  2 ++
- src/lxc/confile.c         |  8 ++++----
- src/lxc/lxccontainer.c    | 20 +++++++++++++++++++-
- src/lxc/lxccontainer.h    |  6 ++++++
- src/lxc/start.c           | 10 ++++++++++
- src/lxc/start.h           |  2 ++
- src/lxc/terminal.c        |  4 ++--
- src/lxc/tools/arguments.h |  2 ++
- src/lxc/tools/lxc_start.c |  9 +++++++++
- 10 files changed, 59 insertions(+), 7 deletions(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 0b4b63b..bc45e44 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -2759,6 +2759,7 @@ struct lxc_conf *lxc_conf_init(void)
- 
- 	/* isulad add begin */
- 	lxc_list_init(&new->populate_devs);
-+	new->exit_fd = -1;
- 	/* isulad add end */
- 
- 	return new;
-@@ -4195,6 +4196,8 @@ void lxc_conf_free(struct lxc_conf *conf)
- 	lxc_clear_init_args(conf);
- 	lxc_clear_populate_devices(conf);
- 	free(conf->container_info_file);
-+	if (conf->exit_fd != -1)
-+		close(conf->exit_fd);
- 	/* isulad add end */
- 	free(conf);
- }
-diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index e0954f9..2d939cd 100644
---- a/src/lxc/conf.h
-+++ b/src/lxc/conf.h
-@@ -405,6 +405,8 @@ struct lxc_conf {
- 	struct lxc_list populate_devs;
- 
- 	char *container_info_file;
-+
-+	int exit_fd; /* exit fifo fd*/
- 	/* isulad add end */
- };
- 
-diff --git a/src/lxc/confile.c b/src/lxc/confile.c
-index e3212d3..cbef2e2 100644
---- a/src/lxc/confile.c
-+++ b/src/lxc/confile.c
-@@ -240,10 +240,10 @@ static struct lxc_config_t config_jump_table[] = {
- 	{ "lxc.sysctl",                    set_config_sysctl,                      get_config_sysctl,                      clr_config_sysctl,                    },
- 	{ "lxc.proc",                      set_config_proc,                        get_config_proc,                        clr_config_proc,                      },
- 
--        /*isulad add begin*/
--        { "lxc.isulad.init.args",          set_config_init_args,                   get_config_init_args,                   clr_config_init_args,                 },
--        { "lxc.isulad.populate.device",    set_config_populate_device,             get_config_populate_device,             clr_config_populate_device,           },
--        /*isulad add end*/
-+	/*isulad add begin*/
-+	{ "lxc.isulad.init.args",          set_config_init_args,                   get_config_init_args,                   clr_config_init_args,                 },
-+	{ "lxc.isulad.populate.device",    set_config_populate_device,             get_config_populate_device,             clr_config_populate_device,           },
-+	/*isulad add end*/
- };
- 
- static const size_t config_jump_table_size = sizeof(config_jump_table) / sizeof(struct lxc_config_t);
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index 5679b9b..8029f33 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -295,6 +295,10 @@ static void lxc_container_free(struct lxc_container *c)
- 	free(c->config_path);
- 	c->config_path = NULL;
- 
-+	/* isulad: free exit fifo */
-+	free(c->exit_fifo);
-+	c->exit_fifo = NULL;
-+
- 	free(c);
- }
- 
-@@ -882,7 +886,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 		NULL,
- 	};
- 	char **init_cmd = NULL;
--	int keepfds[3] = {-1, -1, -1};
-+	int keepfds[4] = {-1, -1, -1, -1};
- 
- 	/* container does exist */
- 	if (!c)
-@@ -1077,6 +1081,16 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 		}
- 	}
- 
-+	/* isulad: open exit fifo */
-+	if (c->exit_fifo) {
-+		conf->exit_fd = open(c->exit_fifo, O_WRONLY | O_NONBLOCK | O_CLOEXEC);
-+		if (conf->exit_fd < 0) {
-+			ERROR("Failed to open exit fifo %s: %s.", c->exit_fifo, strerror(errno));
-+			ret = 1;
-+			goto on_error;
-+		}
-+	}
-+
- 	conf->reboot = REBOOT_NONE;
- 
- 	/* Unshare the mount namespace if requested */
-@@ -1111,6 +1125,10 @@ reboot:
- 	keepfds[0] = handler->conf->maincmd_fd;
- 	keepfds[1] = handler->state_socket_pair[0];
- 	keepfds[2] = handler->state_socket_pair[1];
-+	/* isulad: keep exit fifo fd */
-+	if (conf->exit_fd >= 0) {
-+		keepfds[3] = conf->exit_fd;
-+	}
- 	ret = lxc_check_inherited(conf, c->daemonize, keepfds,
- 				  sizeof(keepfds) / sizeof(keepfds[0]));
- 	if (ret < 0) {
-diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
-index 3c845fe..503038a 100644
---- a/src/lxc/lxccontainer.h
-+++ b/src/lxc/lxccontainer.h
-@@ -81,6 +81,12 @@ struct lxc_container {
- 	 */
- 	char *pidfile;
- 
-+	/*! isulad:
-+	 * \private
-+	 * exit FIFO File to open used monitor the state of lxc monitor process.
-+	 */
-+	char *exit_fifo;
-+
- 	/*!
- 	 * \private
- 	 * Container semaphore lock.
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 9d71dd7..9365d11 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -730,6 +730,7 @@ struct lxc_handler *lxc_init_handler(const char *name, struct lxc_conf *conf,
- 		handler->nsfd[i] = -1;
- 
- 	handler->name = name;
-+	handler->exit_code = -1; /* isulad: record exit code of container */
- 
- 	if (daemonize && handler->conf->reboot == REBOOT_NONE) {
- 		/* Create socketpair() to synchronize on daemonized startup.
-@@ -1005,6 +1006,14 @@ void lxc_fini(const char *name, struct lxc_handler *handler)
- 		 */
- 		lxc_monitor_send_state(name, STOPPED, handler->lxcpath);
- 
-+
-+		/* isuald: write exit code to exit fifo */
-+		if (handler->conf->exit_fd >= 0) {
-+			ret = write(handler->conf->exit_fd, &handler->exit_code, sizeof(int));
-+			if (ret != sizeof(int))
-+				SYSERROR("Failed to write to exit code to exit fifo.");
-+		}
-+
- 		/* The command socket is closed so no one can acces the command
- 		 * socket anymore so there's no need to lock it.
- 		 */
-@@ -2038,6 +2047,7 @@ int __lxc_start(const char *name, struct lxc_handler *handler,
- 	lxc_error_set_and_log(handler->pid, status);
- 	if (error_num)
- 		*error_num = handler->exit_status;
-+	handler->exit_code = exit_code; /* isuald: record exit code*/
- 
- out_fini:
- 	lxc_delete_network(handler);
-diff --git a/src/lxc/start.h b/src/lxc/start.h
-index df987dc..f59bf54 100644
---- a/src/lxc/start.h
-+++ b/src/lxc/start.h
-@@ -133,6 +133,8 @@ struct lxc_handler {
- 	int exit_status;
- 
- 	struct cgroup_ops *cgroup_ops;
-+
-+	int exit_code;/* isulad: record the exit code of container */
- };
- 
- struct execute_args {
-diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
-index c507712..508e2e6 100644
---- a/src/lxc/terminal.c
-+++ b/src/lxc/terminal.c
-@@ -1004,8 +1004,8 @@ static int lxc_terminal_set_fifo(struct lxc_terminal *console, const char *in, c
- static int lxc_terminal_fifo_default(struct lxc_terminal *terminal)
- {
- 	if (!terminal->init_fifo[0] || !terminal->init_fifo[1]) {
--		ERROR("Invalid default terminal fifos");
--		return -1;
-+		DEBUG("Invalid default terminal fifos");
-+		return 0;
- 	}
- 
- 	return lxc_terminal_set_fifo(terminal, terminal->init_fifo[0], terminal->init_fifo[1]);
-diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
-index b6df23f..61f4a0a 100644
---- a/src/lxc/tools/arguments.h
-+++ b/src/lxc/tools/arguments.h
-@@ -64,6 +64,7 @@ struct lxc_arguments {
- 	const char *share_ns[32]; /* size must be greater than LXC_NS_MAX */
- 	const char *terminal_fifos[2]; /* isulad add, fifos used to redirct stdin/out/err */
- 	const char *container_info; /* isulad: file used to store pid and ppid info of container */
-+	const char *exit_monitor_fifo; /* isulad: fifo used to monitor state of monitor process */
- 
- 	/* for lxc-console */
- 	unsigned int ttynum;
-@@ -178,6 +179,7 @@ struct lxc_arguments {
- #define OPT_INPUT_FIFO OPT_USAGE - 7
- #define OPT_OUTPUT_FIFO OPT_USAGE - 8
- #define OPT_CONTAINER_INFO OPT_USAGE - 9
-+#define OPT_EXIT_FIFO OPT_USAGE - 10
- /* isulad add end*/
- 
- extern int lxc_arguments_parse(struct lxc_arguments *args, int argc,
-diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
-index 2f94d67..60c7d70 100644
---- a/src/lxc/tools/lxc_start.c
-+++ b/src/lxc/tools/lxc_start.c
-@@ -73,6 +73,7 @@ static const struct option my_longopts[] = {
- 	{"in-fifo", required_argument, 0, OPT_INPUT_FIFO},
- 	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
- 	{"container-pidfile", required_argument, 0, OPT_CONTAINER_INFO},
-+	{"exit-fifo", required_argument, 0, OPT_EXIT_FIFO},
- 	/* isulad add end */
- 	LXC_COMMON_OPTIONS
- };
-@@ -154,6 +155,9 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
- 	case OPT_CONTAINER_INFO:
- 		args->container_info = arg;
- 		break;
-+	case OPT_EXIT_FIFO:
-+		args->exit_monitor_fifo = arg;
-+		break;
- 	}
- 	return 0;
- }
-@@ -332,6 +336,11 @@ int main(int argc, char *argv[])
- 		}
- 	}
- 
-+	/* isulad: fifo used to monitor state of monitor process */
-+	if (my_args.exit_monitor_fifo != NULL) {
-+		c->exit_fifo = strdup(my_args.exit_monitor_fifo);
-+	}
-+
- 	if (my_args.console)
- 		if (!c->set_config_item(c, "lxc.console.path", my_args.console))
- 			goto out;
--- 
-1.8.3.1
-

0011-add-exit-fifo-to-monitor-state-of-lxc-monitor.patch

@@ -0,0 +1,271 @@
+From 5aee93396a979771f59d8accc4f4c168fd31b584 Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Mon, 13 Apr 2020 06:32:37 -0400
+Subject: [PATCH 11/49] add exit fifo to monitor state of lxc monitor
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/conf.c            |  4 ++++
+ src/lxc/conf.h            |  2 ++
+ src/lxc/lxccontainer.c    | 30 ++++++++++++++++++++++++++++++
+ src/lxc/lxccontainer.h    |  8 ++++++++
+ src/lxc/start.c           | 20 ++++++++++++++++++++
+ src/lxc/start.h           |  5 +++++
+ src/lxc/tools/arguments.h |  1 +
+ src/lxc/tools/lxc_start.c | 14 ++++++++++++++
+ 8 files changed, 84 insertions(+)
+
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index 1487b73..a904348 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -2668,6 +2668,7 @@ struct lxc_conf *lxc_conf_init(void)
+ #ifdef HAVE_ISULAD
+ 	/* isulad add begin */
+ 	lxc_list_init(&new->populate_devs);
++	new->exit_fd = -1;
+ 	new->umask = 0027; /*default umask 0027*/
+ 	new->console.init_fifo[0] = NULL;
+ 	new->console.init_fifo[1] = NULL;
+@@ -4097,6 +4098,9 @@ void lxc_conf_free(struct lxc_conf *conf)
+ 	free(conf->shmount.path_cont);
+ #ifdef HAVE_ISULAD
+ 	free(conf->container_info_file);
++	if (conf->exit_fd != -1) {
++		close(conf->exit_fd);
++	}
+ 	lxc_clear_init_args(conf);
+ 	lxc_clear_populate_devices(conf);
+ #endif
+diff --git a/src/lxc/conf.h b/src/lxc/conf.h
+index 23942ac..c5b70e1 100644
+--- a/src/lxc/conf.h
++++ b/src/lxc/conf.h
+@@ -430,6 +430,8 @@ struct lxc_conf {
+ 	mode_t umask;  //umask value
+ 
+ 	char *container_info_file;
++
++	int exit_fd; /* exit fifo fd*/
+ #endif
+ 
+ };
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index 891fc62..64cde99 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -281,6 +281,11 @@ static void lxc_container_free(struct lxc_container *c)
+ 	free(c->config_path);
+ 	c->config_path = NULL;
+ 
++#ifdef HAVE_ISULAD
++	free(c->exit_fifo);
++	c->exit_fifo = NULL;
++#endif
++
+ 	free(c);
+ }
+ 
+@@ -894,7 +899,11 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+ 		NULL,
+ 	};
+ 	char **init_cmd = NULL;
++#ifdef HAVE_ISULAD
++	int keepfds[] = {-1, -1, -1, -1, -1};
++#else
+ 	int keepfds[3] = {-1, -1, -1};
++#endif
+ 
+ 	/* container does exist */
+ 	if (!c)
+@@ -1091,6 +1100,19 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+ 		}
+ 	}
+ 
++#ifdef HAVE_ISULAD
++	/* isulad: open exit fifo */
++	if (c->exit_fifo) {
++		conf->exit_fd = lxc_open(c->exit_fifo, O_WRONLY | O_NONBLOCK | O_CLOEXEC, 0);
++		if (conf->exit_fd < 0) {
++			ERROR("Failed to open exit fifo %s: %s.", c->exit_fifo, strerror(errno));
++			lxc_free_handler(handler);
++			ret = 1;
++			goto on_error;
++		}
++	}
++#endif
++
+ 	conf->reboot = REBOOT_NONE;
+ 
+ 	/* Unshare the mount namespace if requested */
+@@ -1125,6 +1147,14 @@ reboot:
+ 	keepfds[0] = handler->conf->maincmd_fd;
+ 	keepfds[1] = handler->state_socket_pair[0];
+ 	keepfds[2] = handler->state_socket_pair[1];
++
++#ifdef HAVE_ISULAD
++	/* keep exit fifo fd */
++	if (conf->exit_fd >= 0) {
++		keepfds[3] = conf->exit_fd;
++	}
++#endif
++
+ 	ret = lxc_check_inherited(conf, c->daemonize, keepfds,
+ 				  sizeof(keepfds) / sizeof(keepfds[0]));
+ 	if (ret < 0) {
+diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
+index 4a9ba13..fa9bd5e 100644
+--- a/src/lxc/lxccontainer.h
++++ b/src/lxc/lxccontainer.h
+@@ -107,6 +107,14 @@ struct lxc_container {
+ 	/*! Full path to configuration file */
+ 	char *config_path;
+ 
++#ifdef HAVE_ISULAD
++	/*! isulad:
++	 * \private
++	 * exit FIFO File to open used monitor the state of lxc monitor process.
++	 */
++	char *exit_fifo;
++#endif
++
+ 	/*!
+ 	 * \brief Determine if \c /var/lib/lxc/$name/config exists.
+ 	 *
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index f5f9565..70e8282 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -681,6 +681,11 @@ struct lxc_handler *lxc_init_handler(const char *name, struct lxc_conf *conf,
+ 		handler->nsfd[i] = -EBADF;
+ 
+ 	handler->name = name;
++
++#ifdef HAVE_ISULAD
++	handler->exit_code = -1; /* isulad: record exit code of container */
++#endif
++
+ 	if (daemonize)
+ 		handler->transient_pid = lxc_raw_getpid();
+ 	else
+@@ -953,6 +958,16 @@ void lxc_end(struct lxc_handler *handler)
+ 		 */
+ 		lxc_monitor_send_state(name, STOPPED, handler->lxcpath);
+ 
++#ifdef HAVE_ISULAD
++		/* isuald: write exit code to exit fifo */
++		if (handler->conf->exit_fd >= 0) {
++			ret = write(handler->conf->exit_fd, &handler->exit_code, sizeof(int));
++			if (ret != sizeof(int)) {
++				SYSERROR("Failed to write to exit code to exit fifo.");
++			}
++		}
++#endif
++
+ 		/* The command socket is closed so no one can acces the command
+ 		 * socket anymore so there's no need to lock it.
+ 		 */
+@@ -2127,10 +2142,15 @@ int __lxc_start(struct lxc_handler *handler, struct lxc_operations *ops,
+ #else
+ 	lxc_monitor_send_exit_code(name, status, handler->lxcpath);
+ #endif
++
+ 	lxc_error_set_and_log(handler->pid, status);
+ 	if (error_num)
+ 		*error_num = handler->exit_status;
+ 
++#ifdef HAVE_ISULAD
++	handler->exit_code = exit_code; /* record exit code */
++#endif
++
+ /* These are not the droids you are looking for. */
+ __private_goto1:
+ 	lxc_delete_network(handler);
+diff --git a/src/lxc/start.h b/src/lxc/start.h
+index 7e2371c..1368d0e 100644
+--- a/src/lxc/start.h
++++ b/src/lxc/start.h
+@@ -122,6 +122,11 @@ struct lxc_handler {
+ 	int exit_status;
+ 
+ 	struct cgroup_ops *cgroup_ops;
++
++#ifdef HAVE_ISULAD
++	int exit_code;/* isulad: record the exit code of container */
++#endif
++
+ };
+ 
+ struct execute_args {
+diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
+index 214949b..ea5f938 100644
+--- a/src/lxc/tools/arguments.h
++++ b/src/lxc/tools/arguments.h
+@@ -43,6 +43,7 @@ struct lxc_arguments {
+ #ifdef HAVE_ISULAD
+ 	const char *container_info; /* isulad: file used to store pid and ppid info of container */
+ 	char *terminal_fifos[3]; /* isulad add, fifos used to redirct stdin/out/err */
++	const char *exit_monitor_fifo; /* isulad: fifo used to monitor state of monitor process */
+ #endif
+ 
+ 	/* for lxc-console */
+diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
+index 4c4c820..8041f02 100644
+--- a/src/lxc/tools/lxc_start.c
++++ b/src/lxc/tools/lxc_start.c
+@@ -28,6 +28,10 @@
+ #include "confile.h"
+ #include "log.h"
+ 
++#ifdef HAVE_ISULAD
++#include "isulad_utils.h"
++#endif
++
+ lxc_log_define(lxc_start, lxc);
+ 
+ static int my_parser(struct lxc_arguments *args, int c, char *arg);
+@@ -50,6 +54,7 @@ static const struct option my_longopts[] = {
+ 	{"share-pid", required_argument, 0, OPT_SHARE_PID},
+ #ifdef HAVE_ISULAD
+ 	{"container-pidfile", required_argument, 0, OPT_CONTAINER_INFO},
++	{"exit-fifo", required_argument, 0, OPT_EXIT_FIFO},
+ #endif
+ 	LXC_COMMON_OPTIONS
+ };
+@@ -121,6 +126,7 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 	case OPT_SHARE_PID:
+ 		args->share_ns[LXC_NS_PID] = arg;
+ 		break;
++
+ #ifdef HAVE_ISULAD
+ 	case OPT_CONTAINER_INFO:
+ 		args->container_info = arg;
+@@ -133,8 +139,11 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 		break;
+ 	case OPT_STDERR_FIFO:
+ 		args->terminal_fifos[2] = arg;
++	case OPT_EXIT_FIFO:
++		args->exit_monitor_fifo = arg;
+ 		break;
+ #endif
++
+ 	}
+ 	return 0;
+ }
+@@ -315,8 +324,13 @@ int main(int argc, char *argv[])
+ 			goto out;
+ 		}
+ 	}
++
+ 	if (my_args.terminal_fifos[0] || my_args.terminal_fifos[1] || my_args.terminal_fifos[2]) {
+ 		c->set_terminal_init_fifos(c, my_args.terminal_fifos[0], my_args.terminal_fifos[1], my_args.terminal_fifos[2]);
++
++	/* isulad: fifo used to monitor state of monitor process */
++	if (my_args.exit_monitor_fifo != NULL) {
++		c->exit_fifo = safe_strdup(my_args.exit_monitor_fifo);
+ 	}
+ #endif
+ 
+-- 
+1.8.3.1
+

0012-Adapt-to-isulad-log.patch

@@ -0,0 +1,142 @@
+From 74612254660138c0fe96290a6f1ae3c8e46295b8 Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Mon, 13 Apr 2020 16:44:17 +0800
+Subject: [PATCH 12/49] Adapt to isulad log
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ src/lxc/log.c             | 58 ++++++++++++++++++++++++++++++++++++++++++++++-
+ src/lxc/tools/lxc_start.c |  2 ++
+ 2 files changed, 59 insertions(+), 1 deletion(-)
+
+diff --git a/src/lxc/log.c b/src/lxc/log.c
+index 30d6773..9794582 100644
+--- a/src/lxc/log.c
++++ b/src/lxc/log.c
+@@ -55,6 +55,38 @@ static char *log_vmname = NULL;
+ 
+ lxc_log_define(log, lxc);
+ 
++#ifdef HAVE_ISULAD
++static inline const char *isulad_get_fifo_path(const char *file)
++{
++#define ISULAD_FIFO_PREFIX "fifo:"
++
++        if (strncmp(file, ISULAD_FIFO_PREFIX, strlen(ISULAD_FIFO_PREFIX)) == 0) {
++                return (file + strlen(ISULAD_FIFO_PREFIX));
++        }
++        return NULL;
++}
++
++static int isulad_open_fifo(const char *file_path)
++{
++#define LOG_FIFO_SIZE (1024 * 1024)
++        int fd;
++
++        fd = lxc_unpriv(open(file_path, O_RDWR | O_NONBLOCK | O_CLOEXEC, 0640));
++        if (fd == -1) {
++                fprintf(stderr, "Open fifo %s failed: %s\n", file_path, strerror(errno));
++                return -1;
++        }
++
++        if (fcntl(fd, F_SETPIPE_SZ, LOG_FIFO_SIZE) == -1) {
++                printf("Set fifo buffer size failed: %s", strerror(errno));
++                close(fd);
++                return -1;
++        }
++
++        return fd;
++}
++#endif
++
+ static int lxc_log_priority_to_syslog(int priority)
+ {
+ 	switch (priority) {
+@@ -321,6 +353,12 @@ static int log_append_logfile(const struct lxc_log_appender *appender,
+ #endif
+ 
+ 	log_container_name = lxc_log_get_container_name();
++#ifdef HAVE_ISULAD
++        /* use isulad log format */
++        if (log_container_name != NULL && strlen(log_container_name) > 15) {
++                log_container_name = log_container_name + (strlen(log_container_name) - 15);
++        }
++#endif
+ 
+ 	if (fd_to_use < 0)
+ 		fd_to_use = lxc_log_fd;
+@@ -333,9 +371,13 @@ static int log_append_logfile(const struct lxc_log_appender *appender,
+ 		return ret;
+ 
+ 	n = snprintf(buffer, sizeof(buffer),
++#if HAVE_ISULAD
++		     "%15s %s %-8s %s - %s:%s:%d -",
++#else
+ 		     "%s%s%s %s %-8s %s - %s:%s:%d - ",
+ 		     log_prefix,
+ 		     log_container_name ? " " : "",
++#endif
+ 		     log_container_name ? log_container_name : "",
+ 		     date_time,
+ 		     lxc_log_priority_to_string(event->priority),
+@@ -590,6 +632,13 @@ static int __lxc_log_set_file(const char *fname, int create_dirs)
+ 		return ret_errno(EINVAL);
+ 	}
+ 
++#ifdef HAVE_ISULAD
++        fname = isulad_get_fifo_path(fname);
++        if (fname == NULL) {
++                return ret_errno(EINVAL);
++        }
++#endif
++
+ #if USE_CONFIGPATH_LOGS
+ 	/* We don't build_dir for the default if the default is i.e.
+ 	 * /var/lib/lxc/$container/$container.log.
+@@ -599,7 +648,11 @@ static int __lxc_log_set_file(const char *fname, int create_dirs)
+ 	if (build_dir(fname))
+ 		return log_error_errno(-errno, errno, "Failed to create dir for log file \"%s\"", fname);
+ 
++#if HAVE_ISULAD
++        lxc_log_fd = isulad_open_fifo(fname);
++#else
+ 	lxc_log_fd = log_open(fname);
++#endif
+ 	if (lxc_log_fd < 0)
+ 		return lxc_log_fd;
+ 
+@@ -695,7 +748,10 @@ int lxc_log_init(struct lxc_log *log)
+ 
+ 	if (lxc_log_fd >= 0) {
+ 		lxc_log_category_lxc.appender = &log_appender_logfile;
+-		lxc_log_category_lxc.appender->next = &log_appender_stderr;
++#ifdef HAVE_ISULAD
++                if (!lxc_quiet_specified && !log->quiet)
++#endif
++                lxc_log_category_lxc.appender->next = &log_appender_stderr;
+ 	}
+ 
+ 	return ret;
+diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
+index 8041f02..11ff15b 100644
+--- a/src/lxc/tools/lxc_start.c
++++ b/src/lxc/tools/lxc_start.c
+@@ -139,6 +139,7 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 		break;
+ 	case OPT_STDERR_FIFO:
+ 		args->terminal_fifos[2] = arg;
++                break;
+ 	case OPT_EXIT_FIFO:
+ 		args->exit_monitor_fifo = arg;
+ 		break;
+@@ -327,6 +328,7 @@ int main(int argc, char *argv[])
+ 
+ 	if (my_args.terminal_fifos[0] || my_args.terminal_fifos[1] || my_args.terminal_fifos[2]) {
+ 		c->set_terminal_init_fifos(c, my_args.terminal_fifos[0], my_args.terminal_fifos[1], my_args.terminal_fifos[2]);
++        }
+ 
+ 	/* isulad: fifo used to monitor state of monitor process */
+ 	if (my_args.exit_monitor_fifo != NULL) {
+-- 
+1.8.3.1
+

0012-Init-fifos-in-lxc_attach_terminal.patch

@@ -1,28 +0,0 @@
-From 0c5c03e94eb4beb4d55275bef52b14f1eef09d66 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Sat, 12 Jan 2019 03:23:53 -0500
-Subject: [PATCH 012/140] Init fifos in lxc_attach_terminal
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/terminal.c | 4 ++++
- 1 file changed, 4 insertions(+)
-
-diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
-index 508e2e6..410f643 100644
---- a/src/lxc/terminal.c
-+++ b/src/lxc/terminal.c
-@@ -1341,6 +1341,10 @@ void lxc_terminal_init(struct lxc_terminal *terminal)
- 	terminal->peer = -EBADF;
- 	terminal->log_fd = -EBADF;
- 	lxc_terminal_info_init(&terminal->proxy);
-+	/* isulad init console fifos */
-+	terminal->init_fifo[0] = NULL;
-+	terminal->init_fifo[1] = NULL;
-+	lxc_list_init(&terminal->fifos);
- }
- 
- /* isulad: judge the fd whether is fifo */
--- 
-1.8.3.1
-

0013-isulad-set-env-home-in-container.patch

@@ -1,139 +0,0 @@
-From 3e630813a53666d5cdb8db81addcb86e9fe3c341 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Sat, 12 Jan 2019 14:42:27 +0800
-Subject: [PATCH 013/140] isulad: set env home in container
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/attach.c         |  5 +++++
- src/lxc/cgroups/cgfsng.c |  5 +++--
- src/lxc/conf.c           |  2 +-
- src/lxc/start.c          |  4 ++++
- src/lxc/utils.c          | 29 +++++++++++++++++++++++++++++
- src/lxc/utils.h          |  3 +++
- 6 files changed, 45 insertions(+), 3 deletions(-)
-
-diff --git a/src/lxc/attach.c b/src/lxc/attach.c
-index e7ba705..2bbf1eb 100644
---- a/src/lxc/attach.c
-+++ b/src/lxc/attach.c
-@@ -876,6 +876,11 @@ static int attach_child_main(struct attach_clone_payload *payload)
- 	else
- 		new_gid = ns_root_gid;
- 
-+	// isulad: set env home in container
-+	if (lxc_setup_env_home(new_uid) < 0) {
-+		goto on_error;
-+	}
-+
- 	if ((init_ctx->container && init_ctx->container->lxc_conf &&
- 	     init_ctx->container->lxc_conf->no_new_privs) ||
- 	    (options->attach_flags & LXC_ATTACH_NO_NEW_PRIVS)) {
-diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index aff2b5e..3e702b3 100644
---- a/src/lxc/cgroups/cgfsng.c
-+++ b/src/lxc/cgroups/cgfsng.c
-@@ -1704,8 +1704,9 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
- 			continue;
- 		}
- 
--		// Ignore ops->container_cgroup so we will not see directory lxc after /sys/fs/cgroup/xxx in container
--		path2 = must_make_path(controllerpath, h->container_base_path, NULL);
-+		// isulad: ignore ops->container_cgroup so we will not see directory lxc after /sys/fs/cgroup/xxx in container,
-+		// isulad: ignore h->container_base_path so we will not see subgroup of /sys/fs/cgroup/xxx/subgroup in container
-+		path2 = must_make_path(controllerpath, NULL);
- 		ret = mkdir_p(path2, 0755);
- 		if (ret < 0) {
- 			free(controllerpath);
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index bc45e44..5065e69 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -3680,7 +3680,7 @@ int lxc_setup(struct lxc_handler *handler)
- 		return -1;
- 	}
- 
--	/*isulad: move mount entrues here, before we do lxc_fill_autodev and populate devices  */
-+	/*isulad: move mount entries here, before we do lxc_fill_autodev and populate devices  */
- 	if (!lxc_list_empty(&lxc_conf->mount_list)) {
- 		ret = setup_mount_entries(lxc_conf, &lxc_conf->rootfs,
- 					  &lxc_conf->mount_list, name, lxcpath);
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 9365d11..b13326c 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -1398,6 +1398,10 @@ static int do_start(void *data)
- 	new_uid = handler->conf->init_uid;
- 	new_gid = handler->conf->init_gid;
- 
-+	// isulad: set env home in container
-+	if (lxc_setup_env_home(new_uid) < 0)
-+		goto out_warn_father;
-+
- 	/* Avoid unnecessary syscalls. */
- 	if (new_uid == nsuid)
- 		new_uid = LXC_INVALID_UID;
-diff --git a/src/lxc/utils.c b/src/lxc/utils.c
-index 4728284..74e74a1 100644
---- a/src/lxc/utils.c
-+++ b/src/lxc/utils.c
-@@ -45,6 +45,7 @@
- #include <sys/types.h>
- #include <sys/wait.h>
- #include <unistd.h>
-+#include <pwd.h>
- 
- #include "config.h"
- #include "log.h"
-@@ -1829,6 +1830,34 @@ int lxc_setup_keyring(void)
- 	return ret;
- }
- 
-+// isulad: set env home in container
-+int lxc_setup_env_home(uid_t uid)
-+{
-+#define __DEFAULT_HOMEDIR__ "/"
-+	int ret = 0;
-+	char *homedir;
-+	struct passwd pwd, *result = NULL;
-+	char buf[BUFSIZ];
-+
-+	ret = getpwuid_r(uid, &pwd, buf, BUFSIZ, &result);
-+	if (ret || !result || !result->pw_dir) {
-+		WARN("User invalid, can not find user '%u'", uid);
-+		homedir = __DEFAULT_HOMEDIR__;
-+	} else {
-+		homedir = result->pw_dir;
-+	}
-+
-+	// if we didn't configure HOME, set it based on uid
-+	if (setenv("HOME", homedir, 0) < 0) {
-+		SYSERROR("Unable to set env 'HOME'");
-+		return -1;
-+	}
-+
-+	NOTICE("Setted env 'HOME' to %s", homedir);
-+	return 0;
-+}
-+
-+
- /* isulad: read file to buffer */
- static int lxc_file2str(const char *filename, char ret[], int cap)
- {
-diff --git a/src/lxc/utils.h b/src/lxc/utils.h
-index 8e4ed89..364bf67 100644
---- a/src/lxc/utils.h
-+++ b/src/lxc/utils.h
-@@ -309,6 +309,9 @@ extern int fd_cloexec(int fd, bool cloexec);
- extern int recursive_destroy(char *dirname);
- extern int lxc_setup_keyring(void);
- 
-+// isulad: set env home in container
-+extern int lxc_setup_env_home(uid_t uid);
-+
- extern int fd_nonblock(int fd);
- extern int unsigned long long lxc_get_process_startat(pid_t pid);
- 
--- 
-1.8.3.1
-

0013-set-env-in-container.patch

@@ -0,0 +1,143 @@
+From 01d666e795a2cce1d4968202a38c73e673c42e88 Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Mon, 13 Apr 2020 07:04:20 -0400
+Subject: [PATCH 13/49] set env in container
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/attach.c          |  7 +++++++
+ src/lxc/start.c           |  7 +++++++
+ src/lxc/tools/lxc_start.c |  4 ++--
+ src/lxc/utils.c           | 39 +++++++++++++++++++++++++++++++++++++++
+ src/lxc/utils.h           |  2 ++
+ 5 files changed, 57 insertions(+), 2 deletions(-)
+
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index 78b4700..801dc27 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -779,6 +779,13 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 	else
+ 		new_gid = ns_root_gid;
+ 
++#ifdef HAVE_ISULAD
++	// isulad: set env home in container
++	if (lxc_setup_env_home(new_uid) < 0) {
++		goto on_error;
++	}
++#endif
++
+ 	if ((init_ctx->container && init_ctx->container->lxc_conf &&
+ 	     init_ctx->container->lxc_conf->no_new_privs) ||
+ 	    (options->attach_flags & LXC_ATTACH_NO_NEW_PRIVS)) {
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 70e8282..17766bc 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -1381,6 +1381,13 @@ static int do_start(void *data)
+ 	if (new_gid == nsgid)
+ 		new_gid = LXC_INVALID_GID;
+ 
++#ifdef HAVE_ISULAD
++	// isulad: set env home in container
++	if (lxc_setup_env_home(new_uid) < 0) {
++		goto out_warn_father;
++	}
++#endif
++
+ 	/* Make sure that the processes STDIO is correctly owned by the user that we are switching to */
+ 	ret = fix_stdio_permissions(new_uid);
+ 	if (ret)
+diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
+index 11ff15b..76802df6 100644
+--- a/src/lxc/tools/lxc_start.c
++++ b/src/lxc/tools/lxc_start.c
+@@ -139,7 +139,7 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 		break;
+ 	case OPT_STDERR_FIFO:
+ 		args->terminal_fifos[2] = arg;
+-                break;
++		break;
+ 	case OPT_EXIT_FIFO:
+ 		args->exit_monitor_fifo = arg;
+ 		break;
+@@ -328,7 +328,7 @@ int main(int argc, char *argv[])
+ 
+ 	if (my_args.terminal_fifos[0] || my_args.terminal_fifos[1] || my_args.terminal_fifos[2]) {
+ 		c->set_terminal_init_fifos(c, my_args.terminal_fifos[0], my_args.terminal_fifos[1], my_args.terminal_fifos[2]);
+-        }
++	}
+ 
+ 	/* isulad: fifo used to monitor state of monitor process */
+ 	if (my_args.exit_monitor_fifo != NULL) {
+diff --git a/src/lxc/utils.c b/src/lxc/utils.c
+index 90113e0..5b04fa4 100644
+--- a/src/lxc/utils.c
++++ b/src/lxc/utils.c
+@@ -28,6 +28,7 @@
+ #include <sys/wait.h>
+ #include <unistd.h>
+ #include <sys/sysmacros.h>
++#include <pwd.h>
+ 
+ #include "config.h"
+ #include "log.h"
+@@ -2079,4 +2080,42 @@ out:
+     free(pid_info);
+     return startat;
+ }
++
++// isulad: set env home in container
++int lxc_setup_env_home(uid_t uid)
++{
++#define __PASSWD_FILE__ "/etc/passwd"
++	char *homedir = "/"; // default home dir is /
++	FILE *stream = NULL;
++	struct passwd pw, *pwbufp = NULL;
++	char buf[BUFSIZ];
++
++	stream = fopen_cloexec(__PASSWD_FILE__, "r");
++	if (stream == NULL) {
++		SYSWARN("Failed to open %s", __PASSWD_FILE__);
++		goto set_env;
++	}
++
++	while (fgetpwent_r(stream, &pw, buf, sizeof(buf), &pwbufp) == 0 && pwbufp != NULL) {
++		if (pwbufp->pw_uid == uid) {
++			homedir = pwbufp->pw_dir;
++			goto set_env;
++		}
++	}
++	WARN("User invalid, can not find user '%u'", uid);
++
++set_env:
++	if (stream)
++		fclose(stream);
++
++	// if we didn't configure HOME, set it based on uid
++	if (setenv("HOME", homedir, 0) < 0) {
++		SYSERROR("Unable to set env 'HOME'");
++		return -1;
++	}
++
++	NOTICE("Setted env 'HOME' to %s", homedir);
++	return 0;
++}
++
+ #endif
+diff --git a/src/lxc/utils.h b/src/lxc/utils.h
+index fbb0d55..677f632 100644
+--- a/src/lxc/utils.h
++++ b/src/lxc/utils.h
+@@ -316,6 +316,8 @@ extern int fix_stdio_permissions(uid_t uid);
+ extern void lxc_write_error_message(int errfd, const char *format, ...);
+ extern int lxc_file2str(const char *filename, char ret[], int cap);
+ extern int unsigned long long lxc_get_process_startat(pid_t pid);
++// set env home in container
++extern int lxc_setup_env_home(uid_t uid);
+ #endif
+ 
+ #endif /* __LXC_UTILS_H */
+-- 
+1.8.3.1
+

0014-exec-refact-attach-progress.patch

@@ -0,0 +1,452 @@
+From 61b64be9ac4e5d46e9363bb605c7b2e14d0cd2a2 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Mon, 13 Apr 2020 20:41:03 +0800
+Subject: [PATCH 14/49] exec: refact attach progress
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/attach.c           |   2 +-
+ src/lxc/attach_options.h   |  23 +++-
+ src/lxc/conf.h             |   2 +
+ src/lxc/terminal.c         |  27 ++++-
+ src/lxc/tools/arguments.h  |   1 +
+ src/lxc/tools/lxc_attach.c | 273 +++++++++++++++++++++++++++++++++++++++++++++
+ 6 files changed, 324 insertions(+), 4 deletions(-)
+
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index 801dc27..e66ca1c 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -1420,7 +1420,7 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 	}
+ 
+ 	if (pid == 0) {
+-		if (options->attach_flags & LXC_ATTACH_TERMINAL) {
++		if (options->attach_flags & LXC_ATTACH_TERMINAL && terminal.tty_state) {
+ 			ret = pthread_sigmask(SIG_SETMASK,
+ 					      &terminal.tty_state->oldmask, NULL);
+ 			if (ret < 0) {
+diff --git a/src/lxc/attach_options.h b/src/lxc/attach_options.h
+index 3a02ee5..7b0ea5e 100644
+--- a/src/lxc/attach_options.h
++++ b/src/lxc/attach_options.h
+@@ -116,10 +116,12 @@ typedef struct lxc_attach_options_t {
+ #ifdef HAVE_ISULAD
+ 	char *init_fifo[3]; /* isulad: default fifos for the start */
+ 	int64_t timeout;/* isulad: Seconds for waiting on a container to attach/exec before it is killed*/
++	const char *suffix;
+ #endif
+ 
+ } lxc_attach_options_t;
+ 
++#ifdef HAVE_ISULAD
+ /*! Default attach options to use */
+ #define LXC_ATTACH_OPTIONS_DEFAULT                                             \
+ 	{                                                                      \
+@@ -136,8 +138,27 @@ typedef struct lxc_attach_options_t {
+ 		/* .stdout_fd = */      1,                                     \
+ 		/* .stderr_fd = */      2,                                     \
+ 		/* .log_fd    = */      -EBADF,                                \
++		/* .init_fifo = */  	{NULL, NULL, NULL},                    \
+ 	}
+-
++#else
++/*! Default attach options to use */
++#define LXC_ATTACH_OPTIONS_DEFAULT                                             \
++	{                                                                      \
++		/* .attach_flags = */   LXC_ATTACH_DEFAULT,                    \
++		/* .namespaces = */     -1,                                    \
++		/* .personality = */    -1,                                    \
++		/* .initial_cwd = */    NULL,                                  \
++		/* .uid = */            (uid_t)-1,                             \
++		/* .gid = */            (gid_t)-1,                             \
++		/* .env_policy = */     LXC_ATTACH_KEEP_ENV,                   \
++		/* .extra_env_vars = */ NULL,                                  \
++		/* .extra_keep_env = */ NULL,                                  \
++		/* .stdin_fd = */       0,                                     \
++		/* .stdout_fd = */      1,                                     \
++		/* .stderr_fd = */      2,                                     \
++		/* .log_fd    = */      -EBADF,                                \
++	}
++#endif
+ /*!
+  * Representation of a command to run in a container.
+  */
+diff --git a/src/lxc/conf.h b/src/lxc/conf.h
+index c5b70e1..52460a3 100644
+--- a/src/lxc/conf.h
++++ b/src/lxc/conf.h
+@@ -432,6 +432,8 @@ struct lxc_conf {
+ 	char *container_info_file;
+ 
+ 	int exit_fd; /* exit fifo fd*/
++
++	char *errmsg; /* record error messages */
+ #endif
+ 
+ };
+diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
+index c8cd83f..775743d 100644
+--- a/src/lxc/terminal.c
++++ b/src/lxc/terminal.c
+@@ -1174,8 +1174,25 @@ static int lxc_terminal_peer_default(struct lxc_terminal *terminal)
+ 
+ 	if (terminal->path)
+ 		path = terminal->path;
+-	else
+-		path = "/dev/tty";
++
++#ifdef HAVE_ISULAD
++	/* isulad: if no console was given, try current controlling terminal, there
++	 * won't be one if we were started as a daemon (-d)
++	 */
++	if (!path && !access("/dev/tty", F_OK)) {
++		int fd;
++		fd = open("/dev/tty", O_RDWR);
++		if (fd >= 0) {
++			close(fd);
++			path = "/dev/tty";
++		}
++	}
++
++	if (!path) {
++		DEBUG("Not have a controlling terminal");
++		return 0;
++	}
++#endif
+ 
+ 	terminal->peer = lxc_unpriv(open(path, O_RDWR | O_CLOEXEC));
+ 	if (terminal->peer < 0) {
+@@ -1884,9 +1901,15 @@ int lxc_terminal_prepare_login(int fd)
+ 	if (ret < 0)
+ 		return -1;
+ 
++#ifdef HAVE_ISULAD
++	ret = set_stdfds(fd);
++	if (ret < 0)
++		return -1;
++#else
+ 	ret = lxc_terminal_set_stdfds(fd);
+ 	if (ret < 0)
+ 		return -1;
++#endif
+ 
+ 	if (fd > STDERR_FILENO)
+ 		close(fd);
+diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
+index ea5f938..e0866d6 100644
+--- a/src/lxc/tools/arguments.h
++++ b/src/lxc/tools/arguments.h
+@@ -44,6 +44,7 @@ struct lxc_arguments {
+ 	const char *container_info; /* isulad: file used to store pid and ppid info of container */
+ 	char *terminal_fifos[3]; /* isulad add, fifos used to redirct stdin/out/err */
+ 	const char *exit_monitor_fifo; /* isulad: fifo used to monitor state of monitor process */
++	const char *suffix; /* isulad add, suffix used for connect with parent of execed process*/
+ #endif
+ 
+ 	/* for lxc-console */
+diff --git a/src/lxc/tools/lxc_attach.c b/src/lxc/tools/lxc_attach.c
+index a8f493a..47ac2f2 100644
+--- a/src/lxc/tools/lxc_attach.c
++++ b/src/lxc/tools/lxc_attach.c
+@@ -74,6 +74,12 @@ static const struct option my_longopts[] = {
+ 	{"rcfile", required_argument, 0, 'f'},
+ 	{"uid", required_argument, 0, 'u'},
+ 	{"gid", required_argument, 0, 'g'},
++#ifdef HAVE_ISULAD
++	{"in-fifo", required_argument, 0, OPT_INPUT_FIFO}, /* isulad add terminal fifos*/
++	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
++	{"err-fifo", required_argument, 0, OPT_STDERR_FIFO},
++	{"suffix", required_argument, 0, OPT_ATTACH_SUFFIX},
++#endif
+ 	LXC_COMMON_OPTIONS
+ };
+ 
+@@ -201,6 +207,20 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 		if (lxc_safe_uint(arg, &args->gid) < 0)
+ 			return -1;
+ 		break;
++#ifdef HAVE_ISULAD
++	case OPT_INPUT_FIFO:
++		args->terminal_fifos[0] = arg;
++		break;
++	case OPT_OUTPUT_FIFO:
++		args->terminal_fifos[1] = arg;
++		break;
++	case OPT_STDERR_FIFO:
++		args->terminal_fifos[2] = arg;
++		break;
++	case OPT_ATTACH_SUFFIX:
++		args->suffix = arg;
++		break;
++#endif
+ 	}
+ 
+ 	return 0;
+@@ -264,6 +284,258 @@ static int lxc_attach_create_log_file(const char *log_file)
+ 	return fd;
+ }
+ 
++#ifdef HAVE_ISULAD
++// isulad: send '128 + signal' if container is killed by signal.
++#define ExitSignalOffset 128
++
++/*isulad: attach with terminal*/
++static int do_attach_foreground(struct lxc_container *c, lxc_attach_command_t *command,
++				lxc_attach_options_t *attach_options,
++				char **errmsg)
++{
++	int ret = 0;
++	pid_t pid;
++	int wexit = -1;
++	int signal;
++
++	if (command->program)
++		ret = c->attach(c, lxc_attach_run_command, command, attach_options, &pid);
++	else
++		ret = c->attach(c, lxc_attach_run_shell, NULL, attach_options, &pid);
++	if (ret < 0)
++		goto out;
++
++	ret = lxc_wait_for_pid_status(pid);
++	if (ret < 0)
++		goto out;
++
++	if (WIFEXITED(ret))
++		wexit = WEXITSTATUS(ret);
++	else
++		wexit = -1;
++
++	if (WIFSIGNALED(ret)) {
++		signal = WTERMSIG(ret);
++		wexit = ExitSignalOffset + signal;
++	}
++out:
++	if (c->lxc_conf->errmsg)
++		*errmsg = safe_strdup(c->lxc_conf->errmsg);
++	return wexit;
++}
++
++static void close_msg_pipe(int *errpipe)
++{
++	if (errpipe[0] >= 0) {
++		close(errpipe[0]);
++		errpipe[0] = -1;
++	}
++	if (errpipe[1] >= 0) {
++		close(errpipe[1]);
++		errpipe[1] = -1;
++	}
++}
++
++/*isulad: attach without terminal in background */
++static int do_attach_background(struct lxc_container *c, lxc_attach_command_t *command,
++				lxc_attach_options_t *attach_options,
++				char **errmsg)
++{
++	int ret = 0;
++	int msgpipe[2];
++	pid_t pid = 0;
++	ssize_t size_read;
++	char msgbuf[BUFSIZ + 1] = {0};
++
++	//pipdfd for get error message of child or grandchild process.
++	if (pipe2(msgpipe, O_CLOEXEC) != 0) {
++		SYSERROR("Failed to init msgpipe");
++		return -1;
++	}
++
++	pid = fork();
++	if (pid < 0) {
++		close_msg_pipe(msgpipe);
++		return -1;
++	}
++
++	if (pid != 0) {
++		close(msgpipe[1]);
++		msgpipe[1] = -1;
++		size_read = read(msgpipe[0], msgbuf, BUFSIZ);
++		if (size_read > 0) {
++			*errmsg = safe_strdup(msgbuf);
++			ret = -1;
++		}
++
++		close(msgpipe[0]);
++		msgpipe[0] = -1;
++
++		return ret;
++	}
++
++	/* second fork to be reparented by init */
++	pid = fork();
++	if (pid < 0) {
++		SYSERROR("Error doing dual-fork");
++		close_msg_pipe(msgpipe);
++		exit(1);
++	}
++	if (pid != 0) {
++		close_msg_pipe(msgpipe);
++		exit(0);
++	}
++
++	close(msgpipe[0]);
++	msgpipe[0] = -1;
++
++	if (null_stdfds() < 0) {
++		ERROR("failed to close fds");
++		exit(1);
++	}
++	setsid();
++
++	if (command->program)
++		ret = c->attach(c, lxc_attach_run_command, command, attach_options, &pid);
++	else
++		ret = c->attach(c, lxc_attach_run_shell, NULL, attach_options, &pid);
++	if (ret < 0) {
++		if (c->lxc_conf->errmsg)
++			lxc_write_error_message(msgpipe[1], "%s", c->lxc_conf->errmsg);
++		else
++			lxc_write_error_message(msgpipe[1], "Failed to attach container");
++		close(msgpipe[1]);
++		msgpipe[1] = -1;
++		ret = -1;
++		goto out;
++	}
++
++	close(msgpipe[1]);
++	msgpipe[1] = -1;
++
++	ret = wait_for_pid(pid);
++out:
++	lxc_container_put(c);
++	if (ret)
++		exit(EXIT_FAILURE);
++	else
++		exit(0);
++}
++
++int main(int argc, char *argv[])
++{
++	int ret = -1;
++	int wexit = 0;
++	struct lxc_log log;
++	char *errmsg = NULL;
++	pid_t pid;
++	lxc_attach_options_t attach_options = LXC_ATTACH_OPTIONS_DEFAULT;
++	lxc_attach_command_t command = (lxc_attach_command_t){.program = NULL};
++
++	if (lxc_caps_init())
++		exit(EXIT_FAILURE);
++
++	if (lxc_arguments_parse(&my_args, argc, argv))
++		exit(EXIT_FAILURE);
++
++	log.name = my_args.name;
++	log.file = my_args.log_file;
++	log.level = my_args.log_priority;
++	log.prefix = my_args.progname;
++	log.quiet = my_args.quiet;
++	log.lxcpath = my_args.lxcpath[0];
++
++	if (lxc_log_init(&log))
++		exit(EXIT_FAILURE);
++
++	if (geteuid())
++		if (access(my_args.lxcpath[0], O_RDONLY) < 0) {
++			ERROR("You lack access to %s", my_args.lxcpath[0]);
++			exit(EXIT_FAILURE);
++		}
++
++	struct lxc_container *c = lxc_container_new(my_args.name, my_args.lxcpath[0]);
++	if (!c)
++		exit(EXIT_FAILURE);
++
++	if (my_args.rcfile) {
++		c->clear_config(c);
++		if (!c->load_config(c, my_args.rcfile)) {
++			ERROR("Failed to load rcfile");
++			lxc_container_put(c);
++			exit(EXIT_FAILURE);
++		}
++
++		c->configfile = strdup(my_args.rcfile);
++		if (!c->configfile) {
++			ERROR("Out of memory setting new config filename");
++			lxc_container_put(c);
++			exit(EXIT_FAILURE);
++		}
++	}
++
++	if (!c->may_control(c)) {
++		ERROR("Insufficent privileges to control %s", c->name);
++		lxc_container_put(c);
++		exit(EXIT_FAILURE);
++	}
++
++	if (remount_sys_proc)
++		attach_options.attach_flags |= LXC_ATTACH_REMOUNT_PROC_SYS;
++
++	if (elevated_privileges)
++		attach_options.attach_flags &= ~(elevated_privileges);
++
++	if (my_args.terminal_fifos[0] || my_args.terminal_fifos[1] || my_args.terminal_fifos[2]) {
++		attach_options.init_fifo[0] = my_args.terminal_fifos[0];
++		attach_options.init_fifo[1] = my_args.terminal_fifos[1];
++		attach_options.init_fifo[2] = my_args.terminal_fifos[2];
++		attach_options.attach_flags |= LXC_ATTACH_TERMINAL;
++	} else if (stdfd_is_pty()) {
++		attach_options.attach_flags |= LXC_ATTACH_TERMINAL;
++	}
++
++	attach_options.namespaces = namespace_flags;
++	attach_options.personality = new_personality;
++	attach_options.env_policy = env_policy;
++	attach_options.extra_env_vars = extra_env;
++	attach_options.extra_keep_env = extra_keep;
++
++	if (my_args.argc > 0) {
++		command.program = my_args.argv[0];
++		command.argv = (char**)my_args.argv;
++	}
++
++	if (my_args.console_log) {
++		attach_options.log_fd = lxc_attach_create_log_file(my_args.console_log);
++		if (attach_options.log_fd < 0) {
++			ERROR("Failed to create log file for %s", c->name);
++			lxc_container_put(c);
++			exit(EXIT_FAILURE);
++		}
++	}
++
++	attach_options.suffix = my_args.suffix;
++
++	/* isulad: add do attach background */
++	if (attach_options.attach_flags & LXC_ATTACH_TERMINAL)
++		wexit = do_attach_foreground(c, &command, &attach_options, &errmsg);
++	else
++		wexit = do_attach_background(c, &command, &attach_options, &errmsg);
++
++	if (errmsg) {
++		fprintf(stderr, "%s:%s:%s:%d starting container process caused \"%s\"", c->name,
++		__FILE__, __func__, __LINE__, errmsg);
++		free(errmsg);
++	}
++
++	lxc_container_put(c);
++	if (wexit >= 0)
++		exit(wexit);
++
++	exit(EXIT_FAILURE);
++}
++#else
+ int main(int argc, char *argv[])
+ {
+ 	int ret = -1;
+@@ -377,3 +649,4 @@ out:
+ 
+ 	exit(EXIT_FAILURE);
+ }
++#endif
+-- 
+1.8.3.1
+

0014-support-rotate-for-container-log-file.patch

@@ -1,93 +0,0 @@
-From 42efaa5362d755d9f9bf028c283ca24bc6a03cbb Mon Sep 17 00:00:00 2001
-From: liuhao <liuhao27@huawei.com>
-Date: Sat, 12 Jan 2019 15:29:56 +0800
-Subject: [PATCH 014/140] support rotate for container log file
-
-support rotate for container log file
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/confile.c  |  3 +++
- src/lxc/terminal.c | 40 ++++++++++++++++++++++++++++++++++++++++
- 2 files changed, 43 insertions(+)
-
-diff --git a/src/lxc/confile.c b/src/lxc/confile.c
-index cbef2e2..e782211 100644
---- a/src/lxc/confile.c
-+++ b/src/lxc/confile.c
-@@ -1802,11 +1802,14 @@ static int set_config_console_rotate(const char *key, const char *value,
- 	if (lxc_safe_uint(value, &lxc_conf->console.log_rotate) < 0)
- 		return -1;
- 
-+	/*
-+	 * isulad: support rotate muti-files
- 	if (lxc_conf->console.log_rotate > 1) {
- 		ERROR("The \"lxc.console.rotate\" config key can only be set "
- 		      "to 0 or 1");
- 		return -1;
- 	}
-+	*/
- 
- 	return 0;
- }
-diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
-index 410f643..3bb3a52 100644
---- a/src/lxc/terminal.c
-+++ b/src/lxc/terminal.c
-@@ -229,6 +229,39 @@ static int lxc_terminal_truncate_log_file(struct lxc_terminal *terminal)
- 	return lxc_unpriv(ftruncate(terminal->log_fd, 0));
- }
- 
-+/*
-+ * isuald: support mult-logfiles
-+ * */
-+static int lxc_terminal_rename_old_log_file(struct lxc_terminal *terminal)
-+{
-+	int ret;
-+	size_t i;
-+	char tmp[PATH_MAX] = {0};
-+	char *rename_fname = NULL;
-+
-+	for (i = terminal->log_rotate - 1; i > 1; i--) {
-+		ret = sprintf(tmp, "%s.%d", terminal->log_path, i);
-+		if (ret < 0)
-+			return -EFBIG;
-+		if (rename_fname)
-+			free(rename_fname);
-+		rename_fname = strdup(tmp);
-+		ret = sprintf(tmp, "%s.%d", terminal->log_path, (i - 1));
-+		if (ret < 0) {
-+			free(rename_fname);
-+			return -EFBIG;
-+		}
-+		ret = lxc_unpriv(rename(tmp, rename_fname));
-+		if (ret < 0 && errno != ENOENT)
-+			return ret;
-+	}
-+
-+	if (rename_fname)
-+		free(rename_fname);
-+
-+	return 0;
-+}
-+
- static int lxc_terminal_rotate_log_file(struct lxc_terminal *terminal)
- {
- 	int ret;
-@@ -242,6 +275,13 @@ static int lxc_terminal_rotate_log_file(struct lxc_terminal *terminal)
- 	if (terminal->log_fd < 0)
- 		return -EBADF;
- 
-+	/* isuald: rotate old log file first */
-+	ret = lxc_terminal_rename_old_log_file(terminal);
-+	if(ret != 0) {
-+		ERROR("Rename old log file failed");
-+		return ret;
-+	}
-+
- 	len = strlen(terminal->log_path) + sizeof(".1");
- 	tmp = alloca(len);
- 
--- 
-1.8.3.1
-

0015-add-masked-paths-and-readonly-paths.patch

@@ -1,24 +1,24 @@
-From 6abc6c02684ec9d48033969399352050789da2d6 Mon Sep 17 00:00:00 2001
+From c5ea37649d728630df34e1af22908b8e8124f772 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
+From: wujing <wujing50@huawei.com>
-Date: Sat, 12 Jan 2019 15:55:52 +0800
+Date: Mon, 13 Apr 2020 09:11:21 -0400
-Subject: [PATCH 016/140] add masked paths and ro paths
+Subject: [PATCH 15/49] add masked paths and readonly paths
 
-Signed-off-by: LiFeng <lifeng68@huawei.com>
+Signed-off-by: wujing <wujing50@huawei.com>
 ---
- src/lxc/conf.c    | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ src/lxc/conf.c    | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
- src/lxc/conf.h    |   8 ++++
+ src/lxc/conf.h    |  12 +++++
- src/lxc/confile.c | 113 ++++++++++++++++++++++++++++++++++++++++++++-
+ src/lxc/confile.c | 106 +++++++++++++++++++++++++++++++++++++++++
- 3 files changed, 255 insertions(+), 1 deletion(-)
+ 3 files changed, 256 insertions(+), 1 deletion(-)
 
 diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 5065e69..537f956 100644
+index a904348..fce241b 100644
 --- a/src/lxc/conf.c
 +++ b/src/lxc/conf.c
-@@ -1343,6 +1343,95 @@ static int lxc_mount_rootfs(struct lxc_conf *conf)
+@@ -1275,6 +1275,96 @@ static int lxc_mount_rootfs(struct lxc_conf *conf)
  	return 0;
  }
  
-+
++#ifdef HAVE_ISULAD
 +// maskPath masks the top of the specified path inside a container to avoid
 +// security issues from processes reading information from non-namespace aware
 +// mounts ( proc/kcore ).
@@ -49,7 +49,7 @@ index 5065e69..537f956 100644
 +// remount_readonly will bind over the top of an existing path and ensure that it is read-only.
 +static bool remount_readonly(const char *path)
 +{
-+	int ret, savederrno, i;
++	int ret, i;
 +
 +	if (!path)
 +		return true;
@@ -63,7 +63,7 @@ index 5065e69..537f956 100644
 +				if (ret < 0)
 +					goto on_error;
 +				ret = mount(path, path, "", MS_BIND | MS_REMOUNT | MS_RDONLY | MS_REC | \
-+					    MS_NOEXEC | MS_NOSUID | MS_NODEV, "");
++				            MS_NOEXEC | MS_NOSUID | MS_NODEV, "");
 +				if (ret < 0)
 +					goto on_error;
 +			} else if (errno == EBUSY) {
@@ -88,8 +88,8 @@ index 5065e69..537f956 100644
 +	struct lxc_list *it;
 +
 +	lxc_list_for_each(it, maskedpaths) {
-+	       if (!mask_path((char *)it->elem))
++		if (!mask_path((char *)it->elem))
-+	               return -1;
++			return -1;
 +	}
 +
 +	return 0;
@@ -100,56 +100,69 @@ index 5065e69..537f956 100644
 +	struct lxc_list *it;
 +
 +	lxc_list_for_each(it, ropaths) {
-+	       if (!remount_readonly((char *)it->elem))
++		if (!remount_readonly((char *)it->elem))
-+	               return -1;
++			return -1;
 +	}
 +
 +	return 0;
 +}
++#endif
 +
  int lxc_chroot(const struct lxc_rootfs *rootfs)
  {
- 	int i, ret;
+ 	__do_free char *nroot = NULL;
-@@ -2759,6 +2848,8 @@ struct lxc_conf *lxc_conf_init(void)
+@@ -2666,8 +2756,9 @@ struct lxc_conf *lxc_conf_init(void)
+ 	seccomp_conf_init(new);
  
- 	/* isulad add begin */
+ #ifdef HAVE_ISULAD
+-	/* isulad add begin */
  	lxc_list_init(&new->populate_devs);
 +	lxc_list_init(&new->rootfs.maskedpaths);
 +	lxc_list_init(&new->rootfs.ropaths);
  	new->exit_fd = -1;
- 	/* isulad add end */
+ 	new->umask = 0027; /*default umask 0027*/
+ 	new->console.init_fifo[0] = NULL;
+@@ -3690,6 +3781,22 @@ int lxc_setup(struct lxc_handler *handler)
+ 			return log_error(-1, "Failed to setup sysctl parameters");
+ 	}
  
-@@ -3759,6 +3850,22 @@ int lxc_setup(struct lxc_handler *handler)
++#ifdef HAVE_ISULAD
- 	if (ret < 0)
++	// isulad: setup rootfs masked paths
- 		return -1;
- 
-+	//isulad: setup rootfs masked paths
 +	if (!lxc_list_empty(&lxc_conf->rootfs.maskedpaths)) {
 +		if (setup_rootfs_maskedpaths(&lxc_conf->rootfs.maskedpaths)) {
-+			ERROR("failed to setup maskedpaths");
++			return log_error(-1, "failed to setup maskedpaths");
-+			return -1;
 +		}
 +	}
 +
 +	// isulad: setup rootfs ro paths
 +	if (!lxc_list_empty(&lxc_conf->rootfs.ropaths)) {
 +		if (setup_rootfs_ropaths(&lxc_conf->rootfs.ropaths)) {
-+			ERROR("failed to setup readonlypaths");
++			return log_error(-1, "failed to setup readonlypaths");
-+			return -1;
 +		}
 +	}
++#endif
 +
- 	ret = setup_personality(lxc_conf->personality);
+ 	if (!lxc_list_empty(&lxc_conf->keepcaps)) {
- 	if (ret < 0) {
+ 		if (!lxc_list_empty(&lxc_conf->caps))
- 		ERROR("Failed to set personality");
+ 			return log_error(-1, "Container requests lxc.cap.drop and lxc.cap.keep: either use lxc.cap.drop or lxc.cap.keep, not both");
-@@ -4147,6 +4254,32 @@ int lxc_clear_populate_devices(struct lxc_conf *c)
+@@ -4103,6 +4210,8 @@ void lxc_conf_free(struct lxc_conf *conf)
+ 	}
+ 	lxc_clear_init_args(conf);
+ 	lxc_clear_populate_devices(conf);
++	lxc_clear_rootfs_masked_paths(conf);
++	lxc_clear_rootfs_ro_paths(conf);
+ #endif
+ 	free(conf);
+ }
+@@ -4945,4 +5054,32 @@ int lxc_clear_populate_devices(struct lxc_conf *c)
  	return 0;
  }
  
 +/*isulad: clear rootfs masked paths*/
 +int lxc_clear_rootfs_masked_paths(struct lxc_conf *c)
 +{
-+	struct lxc_list *it,*next;
++	struct lxc_list *it = NULL;
++	struct lxc_list *next = NULL;
 +
 +	lxc_list_for_each_safe(it, &c->rootfs.maskedpaths, next) {
 +		lxc_list_del(it);
@@ -162,7 +175,8 @@ index 5065e69..537f956 100644
 +/*isulad: clear rootfs ro paths*/
 +int lxc_clear_rootfs_ro_paths(struct lxc_conf *c)
 +{
-+	struct lxc_list *it,*next;
++	struct lxc_list *it = NULL;
++	struct lxc_list *next = NULL;
 +
 +	lxc_list_for_each_safe(it, &c->rootfs.ropaths, next) {
 +		lxc_list_del(it);
@@ -172,91 +186,72 @@ index 5065e69..537f956 100644
 +	return 0;
 +}
 +
- void lxc_conf_free(struct lxc_conf *conf)
+ #endif
- {
- 	if (!conf)
-@@ -4195,6 +4328,8 @@ void lxc_conf_free(struct lxc_conf *conf)
- 	/* isulad add begin */
- 	lxc_clear_init_args(conf);
- 	lxc_clear_populate_devices(conf);
-+	lxc_clear_rootfs_masked_paths(conf);
-+	lxc_clear_rootfs_ro_paths(conf);
- 	free(conf->container_info_file);
- 	if (conf->exit_fd != -1)
- 		close(conf->exit_fd);
 diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index 2d939cd..7927812 100644
+index 52460a3..482fe0d 100644
 --- a/src/lxc/conf.h
 +++ b/src/lxc/conf.h
-@@ -160,6 +160,8 @@ struct lxc_tty_info {
+@@ -143,6 +143,8 @@ struct lxc_tty_info {
-  * @options    : mount options
   * @mountflags : the portion of @options that are flags
   * @data       : the portion of @options that are not flags
+  * @managed    : whether it is managed by LXC
 + * @maskedpaths: A list of paths to be msked over inside the container
 + * @ropaths    : A list of paths to be remounted with readonly inside the container
   */
  struct lxc_rootfs {
  	char *path;
-@@ -168,6 +170,10 @@ struct lxc_rootfs {
+@@ -152,6 +154,14 @@ struct lxc_rootfs {
- 	char *options;
  	unsigned long mountflags;
  	char *data;
-+	/* isulad: maskedpaths */
+ 	bool managed;
-+	struct lxc_list maskedpaths;
++
-+	/* isulad: ropaths */
++#ifdef HAVE_ISULAD
-+	struct lxc_list ropaths;
++    /* isulad: maskedpaths */
++    struct lxc_list maskedpaths;
++    /* isulad: ropaths */
++    struct lxc_list ropaths;
++#endif
++
  };
  
  /*
-@@ -477,6 +483,8 @@ extern int lxc_clear_procs(struct lxc_conf *c, const char *key);
+@@ -511,5 +521,7 @@ extern int userns_exec_minimal(const struct lxc_conf *conf,
- /* isulad add begin */
+ #ifdef HAVE_ISULAD
  int lxc_clear_init_args(struct lxc_conf *lxc_conf);
  int lxc_clear_populate_devices(struct lxc_conf *c);
 +int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
 +int lxc_clear_rootfs_ro_paths(struct lxc_conf *c);
- 
+ #endif
- /* isulad add end */
+ #endif /* __LXC_CONF_H */
- 
 diff --git a/src/lxc/confile.c b/src/lxc/confile.c
-index e782211..e199965 100644
+index 2df269a..bf0fdf0 100644
 --- a/src/lxc/confile.c
 +++ b/src/lxc/confile.c
-@@ -139,6 +139,8 @@ lxc_config_define(pty_max);
+@@ -151,6 +151,8 @@ lxc_config_define(proc);
- lxc_config_define(rootfs_mount);
+ lxc_config_define(init_args);
- lxc_config_define(rootfs_options);
+ lxc_config_define(populate_device);
- lxc_config_define(rootfs_path);
+ lxc_config_define(umask);
 +lxc_config_define(rootfs_masked_paths);
 +lxc_config_define(rootfs_ro_paths);
- lxc_config_define(seccomp_profile);
+ #endif
- lxc_config_define(selinux_context);
+ 
- lxc_config_define(signal_halt);
+ /*
-@@ -243,6 +245,8 @@ static struct lxc_config_t config_jump_table[] = {
+@@ -268,6 +270,8 @@ static struct lxc_config_t config_jump_table[] = {
- 	/*isulad add begin*/
  	{ "lxc.isulad.init.args",          set_config_init_args,                   get_config_init_args,                   clr_config_init_args,                 },
  	{ "lxc.isulad.populate.device",    set_config_populate_device,             get_config_populate_device,             clr_config_populate_device,           },
+ 	{ "lxc.isulad.umask",              set_config_umask,                       get_config_umask,                       clr_config_umask,                     },
 +	{ "lxc.isulad.rootfs.maskedpaths", set_config_rootfs_masked_paths,         get_config_rootfs_masked_paths,         clr_config_rootfs_masked_paths,       },
 +	{ "lxc.isulad.rootfs.ropaths",     set_config_rootfs_ro_paths,             get_config_rootfs_ro_paths,             clr_config_rootfs_ro_paths,           },
- 	/*isulad add end*/
+ #endif
  };
  
-@@ -2224,7 +2228,7 @@ static int set_config_init_args(const char *key, const char *value,
+@@ -6311,4 +6315,106 @@ static inline int clr_config_umask(const char *key, struct lxc_conf *c,
  	return 0;
  }
  
--/* isulad: set config for init args */
-+/* isulad: set config for populate device */
- static int set_config_populate_device(const char *key, const char *value,
- 				 struct lxc_conf *lxc_conf, void *data)
- {
-@@ -2308,6 +2312,62 @@ on_error:
-                 free(dev_elem);
-         }
-         return -1;
-+}
-+
 +/* isulad: set config for rootfs masked paths */
 +static int set_config_rootfs_masked_paths(const char *key, const char *value,
-+				 struct lxc_conf *lxc_conf, void *data)
++                struct lxc_conf *lxc_conf, void *data)
 +{
 +	struct lxc_list *list_item = NULL;
 +
@@ -264,13 +259,10 @@ index e782211..e199965 100644
 +		return lxc_clear_rootfs_masked_paths(lxc_conf);
 +
 +	list_item = malloc(sizeof(*list_item));
-+	if (!list_item)
++	if (list_item == NULL)
 +		goto on_error;
 +
-+	list_item->elem = strdup(value);
++	list_item->elem = safe_strdup(value);
-+
-+	if (!list_item->elem)
-+		goto on_error;
 +
 +	lxc_list_add_tail(&lxc_conf->rootfs.maskedpaths, list_item);
 +
@@ -282,45 +274,12 @@ index e782211..e199965 100644
 +	return -1;
 +}
 +
-+/* isulad: set config for rootfs ro paths */
-+static int set_config_rootfs_ro_paths(const char *key, const char *value,
-+				 struct lxc_conf *lxc_conf, void *data)
-+{
-+	struct lxc_list *list_item = NULL;
-+
-+	if (lxc_config_value_empty(value))
-+		return lxc_clear_rootfs_ro_paths(lxc_conf);
-+
-+	list_item = malloc(sizeof(*list_item));
-+	if (!list_item)
-+		goto on_error;
-+
-+	list_item->elem = strdup(value);
-+
-+	if (!list_item->elem)
-+		goto on_error;
-+
-+	lxc_list_add_tail(&lxc_conf->rootfs.ropaths, list_item);
-+
-+	return 0;
-+
-+on_error:
-+	free(list_item);
-+
-+	return -1;
- 
- }
- 
-@@ -3889,6 +3949,43 @@ static int get_config_populate_device(const char *key, char *retv, int inlen,
- 	return fulllen;
- }
- 
 +// isulad: get config rootfs masked paths
 +static int get_config_rootfs_masked_paths(const char *key, char *retv, int inlen,
-+				  struct lxc_conf *c, void *data)
++                struct lxc_conf *c, void *data)
 +{
 +	int len, fulllen = 0;
-+	struct lxc_list *it;
++	struct lxc_list *it = NULL;
 +
 +	if (!retv)
 +		inlen = 0;
@@ -334,12 +293,37 @@ index e782211..e199965 100644
 +	return fulllen;
 +}
 +
++/* isulad: set config for rootfs ro paths */
++static int set_config_rootfs_ro_paths(const char *key, const char *value,
++                                      struct lxc_conf *lxc_conf, void *data)
++{
++	struct lxc_list *list_item = NULL;
++
++	if (lxc_config_value_empty(value))
++		return lxc_clear_rootfs_ro_paths(lxc_conf);
++
++	list_item = malloc(sizeof(*list_item));
++	if (list_item == NULL)
++		goto on_error;
++
++	list_item->elem = safe_strdup(value);
++
++	lxc_list_add_tail(&lxc_conf->rootfs.ropaths, list_item);
++
++	return 0;
++
++on_error:
++	free(list_item);
++
++	return -1;
++}
++
 +// isulad: get config rootfs ro paths
 +static int get_config_rootfs_ro_paths(const char *key, char *retv, int inlen,
-+				  struct lxc_conf *c, void *data)
++                                      struct lxc_conf *c, void *data)
 +{
 +	int len, fulllen = 0;
-+	struct lxc_list *it;
++	struct lxc_list *it = NULL;
 +
 +	if (!retv)
 +		inlen = 0;
@@ -352,30 +336,22 @@ index e782211..e199965 100644
 +
 +	return fulllen;
 +}
- 
++
- /* Callbacks to clear config items. */
- static inline int clr_config_personality(const char *key, struct lxc_conf *c,
-@@ -4708,6 +4805,20 @@ static inline int clr_config_populate_device(const char *key, struct lxc_conf *c
- 	return lxc_clear_populate_devices(c);
- }
- 
 +/* isulad: clr config rootfs masked paths */
 +static inline int clr_config_rootfs_masked_paths(const char *key, struct lxc_conf *c,
-+				   void *data)
++                void *data)
 +{
 +	return lxc_clear_rootfs_masked_paths(c);
 +}
 +
 +/* isulad: clr config rootfs ro paths */
 +static inline int clr_config_rootfs_ro_paths(const char *key, struct lxc_conf *c,
-+				   void *data)
++                void *data)
 +{
 +	return lxc_clear_rootfs_ro_paths(c);
 +}
 +
- static int get_config_net_nic(const char *key, char *retv, int inlen,
+ #endif
- 			      struct lxc_conf *c, void *data)
- {
 -- 
 1.8.3.1
 

0015-fix-high-gcc-compile-bug.patch

@@ -1,41 +0,0 @@
-From 5b07aee14b1c49e11cdf42fb2b9c8887751cedb7 Mon Sep 17 00:00:00 2001
-From: liuhao <liuhao27@huawei.com>
-Date: Sat, 12 Jan 2019 16:28:41 +0800
-Subject: [PATCH 015/140] fix high gcc compile bug
-
-fix high gcc compile bug
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/terminal.c | 6 +++---
- 1 file changed, 3 insertions(+), 3 deletions(-)
-
-diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
-index 3bb3a52..7aa4730 100644
---- a/src/lxc/terminal.c
-+++ b/src/lxc/terminal.c
-@@ -235,18 +235,18 @@ static int lxc_terminal_truncate_log_file(struct lxc_terminal *terminal)
- static int lxc_terminal_rename_old_log_file(struct lxc_terminal *terminal)
- {
- 	int ret;
--	size_t i;
-+	unsigned int i;
- 	char tmp[PATH_MAX] = {0};
- 	char *rename_fname = NULL;
- 
- 	for (i = terminal->log_rotate - 1; i > 1; i--) {
--		ret = sprintf(tmp, "%s.%d", terminal->log_path, i);
-+		ret = sprintf(tmp, "%s.%u", terminal->log_path, i);
- 		if (ret < 0)
- 			return -EFBIG;
- 		if (rename_fname)
- 			free(rename_fname);
- 		rename_fname = strdup(tmp);
--		ret = sprintf(tmp, "%s.%d", terminal->log_path, (i - 1));
-+		ret = sprintf(tmp, "%s.%u", terminal->log_path, (i - 1));
- 		if (ret < 0) {
- 			free(rename_fname);
- 			return -EFBIG;
--- 
-1.8.3.1
-

0016-start-separate-i-and-t.patch

@@ -0,0 +1,369 @@
+From 204008167f00ddda3f5f2b0121bcf29b3d55c689 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Mon, 13 Apr 2020 21:17:15 +0800
+Subject: [PATCH 16/49] start: separate -i and -t
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/lxccontainer.c    |  44 +++++++++++++++++-
+ src/lxc/lxccontainer.h    |  27 +++++++++++
+ src/lxc/start.c           | 111 +++++++++++++++++++++++++++++++++++++++++++++-
+ src/lxc/start.h           |   4 ++
+ src/lxc/tools/arguments.h |   2 +
+ src/lxc/tools/lxc_start.c |  14 ++++++
+ 6 files changed, 198 insertions(+), 4 deletions(-)
+
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index 64cde99..e27b63b 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -1163,12 +1163,18 @@ reboot:
+ 		goto on_error;
+ 	}
+ 
+-	if (useinit)
++	if (useinit) {
+ 		ret = lxc_execute(c->name, argv, 1, handler, c->config_path,
+ 				  c->daemonize, &c->error_num);
+-	else
++	}
++	else {
++#ifdef HAVE_ISULAD
++		handler->disable_pty = c->disable_pty;
++		handler->open_stdin = c->open_stdin;
++#endif
+ 		ret = lxc_start(argv, handler, c->config_path, c->daemonize,
+ 				&c->error_num);
++	}
+ 
+ 	if (conf->reboot == REBOOT_REQ) {
+ 		INFO("Container requested reboot");
+@@ -5381,6 +5387,40 @@ static bool do_lxcapi_set_container_info_file(struct lxc_container *c, const cha
+ }
+ 
+ WRAP_API_1(bool, lxcapi_set_container_info_file, const char *)
++
++static bool do_lxcapi_want_disable_pty(struct lxc_container *c, bool state)
++{
++	if (!c || !c->lxc_conf)
++		return false;
++
++	if (container_mem_lock(c))
++		return false;
++
++	c->disable_pty = state;
++
++	container_mem_unlock(c);
++
++	return true;
++}
++
++WRAP_API_1(bool, lxcapi_want_disable_pty, bool)
++
++static bool do_lxcapi_want_open_stdin(struct lxc_container *c, bool state)
++{
++	if (!c || !c->lxc_conf)
++		return false;
++
++	if (container_mem_lock(c))
++		return false;
++
++	c->open_stdin = state;
++
++	container_mem_unlock(c);
++
++	return true;
++}
++
++WRAP_API_1(bool, lxcapi_want_open_stdin, bool)
+ #endif
+ 
+ struct lxc_container *lxc_container_new(const char *name, const char *configpath)
+diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
+index fa9bd5e..de2ee46 100644
+--- a/src/lxc/lxccontainer.h
++++ b/src/lxc/lxccontainer.h
+@@ -113,6 +113,11 @@ struct lxc_container {
+ 	 * exit FIFO File to open used monitor the state of lxc monitor process.
+ 	 */
+ 	char *exit_fifo;
++	/*! Whether container wishes to create pty or pipes for console log */
++	bool disable_pty;
++
++	/*! Whether container wishes to keep stdin active */
++	bool open_stdin;
+ #endif
+ 
+ 	/*!
+@@ -908,6 +913,28 @@ struct lxc_container {
+ 	bool (*set_terminal_winch)(struct lxc_container *c, unsigned int height, unsigned int width);
+ 
+ 	bool (*set_exec_terminal_winch)(struct lxc_container *c, const char *suffix, unsigned int height, unsigned int width);
++
++	/*!
++	 * \brief Change whether the container wants to create pty or pipes
++	 * from the console log.
++	 *
++	 * \param c Container.
++	 * \param state Value for the disable pty bit (0 or 1).
++	 *
++	 * \return \c true on success, else \c false.
++	 */
++	bool (*want_disable_pty)(struct lxc_container *c, bool state);
++
++	/*!
++	 * \brief Change whether the container wants to keep stdin active
++	 * for parent process of container
++	 *
++	 * \param c Container.
++	 * \param state Value for the open_stdin bit (0 or 1).
++	 *
++	 * \return \c true on success, else \c false.
++	 */
++	bool (*want_open_stdin)(struct lxc_container *c, bool state);
+ #endif
+ };
+ 
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 17766bc..145b015 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -580,6 +580,16 @@ int lxc_poll(const char *name, struct lxc_handler *handler)
+ 
+ 	TRACE("Mainloop is ready");
+ 
++#ifdef HAVE_ISULAD
++	// iSulad: close stdin pipe if we do not want open_stdin with container stdin
++	if (!handler->conf->console.open_stdin) {
++		if (handler->conf->console.pipes[0][1] > 0) {
++			close(handler->conf->console.pipes[0][1]);
++			handler->conf->console.pipes[0][1] = -1;
++		}
++	}
++#endif
++
+ 	ret = lxc_mainloop(&descr, -1);
+ 	close_prot_errno_disarm(descr.epfd);
+ 	if (ret < 0 || !handler->init_died)
+@@ -733,6 +743,10 @@ int lxc_init(const char *name, struct lxc_handler *handler)
+ 	int ret;
+ 	const char *loglevel;
+ 	struct lxc_conf *conf = handler->conf;
++#ifdef HAVE_ISULAD
++	conf->console.disable_pty = handler->disable_pty;
++	conf->console.open_stdin = handler->open_stdin;
++#endif
+ 
+ 	handler->monitor_pid = lxc_raw_getpid();
+ 	status_fd = open("/proc/self/status", O_RDONLY | O_CLOEXEC);
+@@ -1178,6 +1192,25 @@ static int do_start(void *data)
+ 	 * means that migration won't work, but at least we won't spew output
+ 	 * where it isn't wanted.
+ 	 */
++#ifdef HAVE_ISULAD
++	if (!handler->disable_pty && handler->daemonize && !handler->conf->autodev) {
++		char path[PATH_MAX];
++
++		ret = snprintf(path, sizeof(path), "%s/dev/null",
++			       handler->conf->rootfs.mount);
++		if (ret < 0 || ret >= sizeof(path))
++			goto out_warn_father;
++
++		ret = access(path, F_OK);
++		if (ret != 0) {
++			devnull_fd = open_devnull();
++
++			if (devnull_fd < 0)
++				goto out_warn_father;
++			WARN("Using /dev/null from the host for container init's standard file descriptors. Migration will not work");
++		}
++	}
++#else
+ 	if (handler->daemonize && !handler->conf->autodev) {
+ 		char path[PATH_MAX];
+ 
+@@ -1195,6 +1228,7 @@ static int do_start(void *data)
+ 			WARN("Using /dev/null from the host for container init's standard file descriptors. Migration will not work");
+ 		}
+ 	}
++#endif
+ 
+ 	/* Ask father to setup cgroups and wait for him to finish. */
+ 	ret = lxc_sync_barrier_parent(handler, LXC_SYNC_CGROUP);
+@@ -1266,12 +1300,70 @@ static int do_start(void *data)
+ 		DEBUG("Set PR_SET_NO_NEW_PRIVS to block execve() gainable privileges");
+ 	}
+ 
++#ifdef HAVE_ISULAD
++	/* isulad: dup2 pipe[0][0] to container stdin, pipe[1][1] to container stdout, pipe[2][1] to container stderr */
++	if (handler->disable_pty) {
++		if (handler->conf->console.pipes[0][1] >= 0) {
++			close(handler->conf->console.pipes[0][1]);
++			handler->conf->console.pipes[0][1] = -1;
++		}
++
++		if (handler->conf->console.pipes[0][0] >= 0) {
++			ret = dup2(handler->conf->console.pipes[0][0], STDIN_FILENO);
++			if (ret < 0)
++				goto out_warn_father;
++		}
++
++		if (handler->conf->console.pipes[1][0] >= 0) {
++			close(handler->conf->console.pipes[1][0]);
++			handler->conf->console.pipes[1][0] = -1;
++		}
++
++		if (handler->conf->console.pipes[1][1] >= 0) {
++			ret = dup2(handler->conf->console.pipes[1][1], STDOUT_FILENO);
++			if (ret < 0)
++				goto out_warn_father;
++		}
++		if (handler->conf->console.pipes[2][0] >= 0) {
++			close(handler->conf->console.pipes[2][0]);
++			handler->conf->console.pipes[2][0] = -1;
++		}
++
++		if (handler->conf->console.pipes[2][1] >= 0) {
++			ret = dup2(handler->conf->console.pipes[2][1], STDERR_FILENO);
++			if (ret < 0)
++				goto out_warn_father;
++		}
++	}
++#endif
++
+ 	/* Some init's such as busybox will set sane tty settings on stdin,
+ 	 * stdout, stderr which it thinks is the console. We already set them
+ 	 * the way we wanted on the real terminal, and we want init to do its
+ 	 * setup on its console ie. the pty allocated in lxc_terminal_setup() so
+ 	 * make sure that that pty is stdin,stdout,stderr.
+ 	 */
++	setsid();
++#ifdef HAVE_ISULAD
++	if (!handler->disable_pty && handler->conf->console.slave >= 0) {
++		/* isulad:make the given terminal as controlling terminal to avoid warning
++		* sh: cannot set terminal process group (-1): Inappropriate ioctl for device
++		* sh: no job control in this shell */
++		if (ioctl(handler->conf->console.slave, TIOCSCTTY, NULL) < 0) {
++			ERROR("Faild to make the given terminal the controlling terminal of the calling process");
++			goto out_warn_father;
++		}
++		if (handler->daemonize || !handler->conf->is_execute)
++			ret = set_stdfds(handler->conf->console.slave);
++		else
++			ret = lxc_terminal_set_stdfds(handler->conf->console.slave);
++		if (ret < 0) {
++			ERROR("Failed to redirect std{in,out,err} to pty file "
++			      "descriptor %d", handler->conf->console.slave);
++			goto out_warn_father;
++		}
++	}
++#else
+ 	 if (handler->conf->console.slave >= 0) {
+ 		 if (handler->daemonize || !handler->conf->is_execute)
+ 			 ret = set_stdfds(handler->conf->console.slave);
+@@ -1283,6 +1375,7 @@ static int do_start(void *data)
+ 			goto out_warn_father;
+ 		 }
+ 	 }
++#endif
+ 
+ 	/* If we mounted a temporary proc, then unmount it now. */
+ 	tmp_proc_unmount(handler->conf);
+@@ -1306,6 +1399,21 @@ static int do_start(void *data)
+ 
+ 	close_prot_errno_disarm(handler->sigfd);
+ 
++	#ifdef HAVE_ISULAD
++	if (!handler->disable_pty && handler->conf->console.slave < 0 && handler->daemonize) {
++		if (devnull_fd < 0) {
++			devnull_fd = open_devnull();
++			if (devnull_fd < 0)
++				goto out_warn_father;
++		}
++
++		ret = set_stdfds(devnull_fd);
++		if (ret < 0) {
++			ERROR("Failed to redirect std{in,out,err} to \"/dev/null\"");
++			goto out_warn_father;
++		}
++	}
++	#else
+ 	if (handler->conf->console.slave < 0 && handler->daemonize) {
+ 		if (devnull_fd < 0) {
+ 			devnull_fd = open_devnull();
+@@ -1319,11 +1427,10 @@ static int do_start(void *data)
+ 			goto out_warn_father;
+ 		}
+ 	}
++	#endif
+ 
+ 	close_prot_errno_disarm(devnull_fd);
+ 
+-	setsid();
+-
+ 	if (handler->conf->init_cwd) {
+ 		ret = chdir(handler->conf->init_cwd);
+ 		if (ret < 0) {
+diff --git a/src/lxc/start.h b/src/lxc/start.h
+index 1368d0e..5ea5fe2 100644
+--- a/src/lxc/start.h
++++ b/src/lxc/start.h
+@@ -125,6 +125,10 @@ struct lxc_handler {
+ 
+ #ifdef HAVE_ISULAD
+ 	int exit_code;/* isulad: record the exit code of container */
++	/* Indicates whether should we using pipes or pty dup to std{in,out,err} for console log. */
++	bool disable_pty;
++	/* Indicates whether should we keep stdin active. */
++	bool open_stdin;
+ #endif
+ 
+ };
+diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
+index e0866d6..a6d9967 100644
+--- a/src/lxc/tools/arguments.h
++++ b/src/lxc/tools/arguments.h
+@@ -45,6 +45,8 @@ struct lxc_arguments {
+ 	char *terminal_fifos[3]; /* isulad add, fifos used to redirct stdin/out/err */
+ 	const char *exit_monitor_fifo; /* isulad: fifo used to monitor state of monitor process */
+ 	const char *suffix; /* isulad add, suffix used for connect with parent of execed process*/
++	int disable_pty;
++	int open_stdin;
+ #endif
+ 
+ 	/* for lxc-console */
+diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
+index 76802df6..321c847 100644
+--- a/src/lxc/tools/lxc_start.c
++++ b/src/lxc/tools/lxc_start.c
+@@ -143,6 +143,12 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 	case OPT_EXIT_FIFO:
+ 		args->exit_monitor_fifo = arg;
+ 		break;
++	case OPT_DISABLE_PTY:
++		args->disable_pty = 1;
++		break;
++	case OPT_OPEN_STDIN:
++		args->open_stdin = 1;
++		break;
+ #endif
+ 
+ 	}
+@@ -334,6 +340,14 @@ int main(int argc, char *argv[])
+ 	if (my_args.exit_monitor_fifo != NULL) {
+ 		c->exit_fifo = safe_strdup(my_args.exit_monitor_fifo);
+ 	}
++
++	if (my_args.disable_pty) {
++		c->want_disable_pty(c, true);
++	}
++
++	if (my_args.open_stdin) {
++		c->want_open_stdin(c, true);
++	}
+ #endif
+ 
+ 	if (my_args.console)
+-- 
+1.8.3.1
+

0017-attach-add_terminal_fifos-Add-terminal-fifos-dynamic.patch

@@ -0,0 +1,289 @@
+From 29c1823334219c03b1ef4d6b4965529b73ff071b Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Mon, 13 Apr 2020 21:39:35 +0800
+Subject: [PATCH 17/49] attach: add_terminal_fifos: Add terminal fifos
+ dynamically
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/commands.c     | 94 +++++++++++++++++++++++++++++++++++++++++++++-----
+ src/lxc/commands.h     |  8 +++++
+ src/lxc/lxccontainer.c | 24 +++++++++++++
+ src/lxc/terminal.c     | 57 ++++++++++++++++++++++++++++++
+ src/lxc/terminal.h     |  4 +++
+ 5 files changed, 178 insertions(+), 9 deletions(-)
+
+diff --git a/src/lxc/commands.c b/src/lxc/commands.c
+index 991bca2..0ffc5c7 100644
+--- a/src/lxc/commands.c
++++ b/src/lxc/commands.c
+@@ -75,15 +75,18 @@ static const char *lxc_cmd_str(lxc_cmd_t cmd)
+ 		[LXC_CMD_GET_CONFIG_ITEM]     		= "get_config_item",
+ 		[LXC_CMD_GET_NAME]            		= "get_name",
+ 		[LXC_CMD_GET_LXCPATH]         		= "get_lxcpath",
+-		[LXC_CMD_ADD_STATE_CLIENT]		= "add_state_client",
+-		[LXC_CMD_CONSOLE_LOG]			= "console_log",
++		[LXC_CMD_ADD_STATE_CLIENT]			= "add_state_client",
++		[LXC_CMD_CONSOLE_LOG]				= "console_log",
+ 		[LXC_CMD_SERVE_STATE_CLIENTS]		= "serve_state_clients",
+ 		[LXC_CMD_SECCOMP_NOTIFY_ADD_LISTENER]	= "seccomp_notify_add_listener",
+ 		[LXC_CMD_ADD_BPF_DEVICE_CGROUP]		= "add_bpf_device_cgroup",
+-		[LXC_CMD_FREEZE]			= "freeze",
+-		[LXC_CMD_UNFREEZE]			= "unfreeze",
+-		[LXC_CMD_GET_CGROUP2_FD]		= "get_cgroup2_fd",
++		[LXC_CMD_FREEZE]					= "freeze",
++		[LXC_CMD_UNFREEZE]					= "unfreeze",
++		[LXC_CMD_GET_CGROUP2_FD]			= "get_cgroup2_fd",
+ 		[LXC_CMD_GET_INIT_PIDFD]        	= "get_init_pidfd",
++#ifdef HAVE_ISULAD
++		[LXC_CMD_SET_TERMINAL_FIFOS]  		= "set_terminal_fifos",
++#endif
+ 	};
+ 
+ 	if (cmd >= LXC_CMD_MAX)
+@@ -1388,6 +1391,76 @@ static int lxc_cmd_get_cgroup2_fd_callback(int fd, struct lxc_cmd_req *req,
+ 	return 0;
+ }
+ 
++#ifdef HAVE_ISULAD
++/*
++ * isulad: lxc_cmd_set_terminal_fifos: Set the fifos used for the container as terminal input/output
++ *
++ * @hashed_sock_name: hashed socket name
++ *
++ * Returns 0 when success, else when fail.
++ */
++int lxc_cmd_set_terminal_fifos(const char *name, const char *lxcpath, const char *in_fifo,
++				const char *out_fifo, const char *err_fifo)
++{
++	int ret = 0, stopped = 0;
++	int len = 0;
++	char *tmp = NULL;
++	const char *split = "&&&&", *none_fifo_name = "none";
++	const char *cmd_in_fifo = in_fifo ? in_fifo : none_fifo_name;
++	const char *cmd_out_fifo = out_fifo ? out_fifo : none_fifo_name;
++	const char *cmd_err_fifo = err_fifo ? err_fifo : none_fifo_name;
++
++	if (len + strlen(cmd_in_fifo) + strlen(split) + strlen(cmd_out_fifo) +
++	     strlen(split) + strlen(cmd_err_fifo) == SIZE_MAX)
++		return -1;
++	len += strlen(cmd_in_fifo) + strlen(split) + strlen(cmd_out_fifo) + strlen(split) + strlen(cmd_err_fifo) + 1;
++	tmp = malloc(len);
++	if (tmp == NULL)
++		return -1;
++	ret = snprintf(tmp, len, "%s%s%s%s%s", cmd_in_fifo, split, cmd_out_fifo, split, cmd_err_fifo);
++	if (ret < 0 || ret >= len) {
++		ERROR("Failed to snprintf in fifo of command");
++		free(tmp);
++		return -1;
++	}
++
++	struct lxc_cmd_rr cmd = {
++		.req = {
++			.cmd = LXC_CMD_SET_TERMINAL_FIFOS,
++			.datalen = strlen(tmp)+1,
++			.data = tmp,
++		},
++	};
++
++	ret = lxc_cmd(name, &cmd, &stopped, lxcpath, NULL);
++	if (ret < 0) {
++		ERROR("Failed to send command to container");
++		free(tmp);
++		return -1;
++	}
++
++	if (cmd.rsp.ret != 0) {
++		ERROR("Command response error:%d", cmd.rsp.ret);
++		free(tmp);
++		return -1;
++	}
++
++	free(tmp);
++	return 0;
++}
++
++static int lxc_cmd_set_terminal_fifos_callback(int fd, struct lxc_cmd_req *req,
++					struct lxc_handler *handler, struct lxc_epoll_descr *descr)
++{
++	struct lxc_cmd_rsp rsp;
++	memset(&rsp, 0, sizeof(rsp));
++
++	rsp.ret = lxc_terminal_add_fifos(handler->conf, req->data);;
++
++	return lxc_cmd_rsp_send(fd, &rsp);
++}
++#endif
++
+ static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
+ 			   struct lxc_handler *handler,
+ 			   struct lxc_epoll_descr *descr)
+@@ -1410,11 +1483,14 @@ static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
+ 		[LXC_CMD_CONSOLE_LOG]                 	= lxc_cmd_console_log_callback,
+ 		[LXC_CMD_SERVE_STATE_CLIENTS]         	= lxc_cmd_serve_state_clients_callback,
+ 		[LXC_CMD_SECCOMP_NOTIFY_ADD_LISTENER] 	= lxc_cmd_seccomp_notify_add_listener_callback,
+-		[LXC_CMD_ADD_BPF_DEVICE_CGROUP]		= lxc_cmd_add_bpf_device_cgroup_callback,
+-		[LXC_CMD_FREEZE]			= lxc_cmd_freeze_callback,
+-		[LXC_CMD_UNFREEZE]			= lxc_cmd_unfreeze_callback,
+-		[LXC_CMD_GET_CGROUP2_FD]		= lxc_cmd_get_cgroup2_fd_callback,
++		[LXC_CMD_ADD_BPF_DEVICE_CGROUP]			= lxc_cmd_add_bpf_device_cgroup_callback,
++		[LXC_CMD_FREEZE]						= lxc_cmd_freeze_callback,
++		[LXC_CMD_UNFREEZE]						= lxc_cmd_unfreeze_callback,
++		[LXC_CMD_GET_CGROUP2_FD]				= lxc_cmd_get_cgroup2_fd_callback,
+ 		[LXC_CMD_GET_INIT_PIDFD]                = lxc_cmd_get_init_pidfd_callback,
++#ifdef HAVE_ISULAD
++		[LXC_CMD_SET_TERMINAL_FIFOS]  			= lxc_cmd_set_terminal_fifos_callback,
++#endif
+ 	};
+ 
+ 	if (req->cmd >= LXC_CMD_MAX)
+diff --git a/src/lxc/commands.h b/src/lxc/commands.h
+index 9e52484..95815e6 100644
+--- a/src/lxc/commands.h
++++ b/src/lxc/commands.h
+@@ -38,6 +38,9 @@ typedef enum {
+ 	LXC_CMD_UNFREEZE,
+ 	LXC_CMD_GET_CGROUP2_FD,
+ 	LXC_CMD_GET_INIT_PIDFD,
++#ifdef HAVE_ISULAD
++	LXC_CMD_SET_TERMINAL_FIFOS,
++#endif
+ 	LXC_CMD_MAX,
+ } lxc_cmd_t;
+ 
+@@ -130,4 +133,9 @@ extern int lxc_cmd_freeze(const char *name, const char *lxcpath, int timeout);
+ extern int lxc_cmd_unfreeze(const char *name, const char *lxcpath, int timeout);
+ extern int lxc_cmd_get_cgroup2_fd(const char *name, const char *lxcpath);
+ 
++#ifdef HAVE_ISULAD
++extern int lxc_cmd_set_terminal_fifos(const char *name, const char *lxcpath,
++				const char *in_fifo, const char *out_fifo, const char *err_fifo);
++#endif
++
+ #endif /* __commands_h */
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index e27b63b..d0e6e2b 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -5421,6 +5421,29 @@ static bool do_lxcapi_want_open_stdin(struct lxc_container *c, bool state)
+ }
+ 
+ WRAP_API_1(bool, lxcapi_want_open_stdin, bool)
++
++/* isulad add clean resources */
++static bool do_lxcapi_add_terminal_fifo(struct lxc_container *c, const char *in_fifo, const char *out_fifo, const char *err_fifo)
++{
++	bool ret = true;
++
++	if (!c || !c->lxc_conf)
++		return false;
++	if (container_mem_lock(c)) {
++		ERROR("Error getting mem lock");
++		return false;
++	}
++
++	if (lxc_cmd_set_terminal_fifos(c->name, c->config_path, in_fifo, out_fifo, err_fifo)) {
++		ERROR("Error set console fifos");
++		ret = false;
++	}
++
++	container_mem_unlock(c);
++	return ret;
++}
++
++WRAP_API_3(bool, lxcapi_add_terminal_fifo, const char *, const char *, const char *)
+ #endif
+ 
+ struct lxc_container *lxc_container_new(const char *name, const char *configpath)
+@@ -5567,6 +5590,7 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
+ #ifdef HAVE_ISULAD
+ 	c->set_container_info_file = lxcapi_set_container_info_file;
+ 	c->set_terminal_init_fifos = lxcapi_set_terminal_default_fifos;
++	c->add_terminal_fifos = lxcapi_add_terminal_fifo;
+ #endif
+ 	return c;
+ 
+diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
+index 775743d..4d7c2cd 100644
+--- a/src/lxc/terminal.c
++++ b/src/lxc/terminal.c
+@@ -1607,6 +1607,63 @@ err:
+ 	lxc_terminal_delete(terminal);
+ 	return -ENODEV;
+ }
++
++/* isulad: add fifos dynamic*/
++int lxc_terminal_add_fifos(struct lxc_conf *conf, const char *fifonames)
++{
++	int ret = 0;
++	struct lxc_terminal *terminal = &conf->console;
++	int fifofd_in = -1;
++	char *tmp = NULL, *saveptr = NULL, *in = NULL, *out = NULL, *err = NULL;
++	const char *none_fifo_name = "none";
++
++	tmp = safe_strdup(fifonames);
++
++	in = strtok_r(tmp, "&&&&", &saveptr);
++	if (!in) {
++		ret = -1;
++		goto free_out;
++	}
++	if (strcmp(in, none_fifo_name) == 0)
++		in = NULL;
++
++	out = strtok_r(NULL, "&&&&",  &saveptr);
++	if (!out) {
++		ret = -1;
++		goto free_out;
++	}
++	if (strcmp(out, none_fifo_name) == 0)
++		out = NULL;
++
++	err = strtok_r(NULL, "&&&&",  &saveptr);
++	if (!err) {
++		ret = -1;
++		goto free_out;
++	}
++	if (strcmp(err, none_fifo_name) == 0)
++		err = NULL;
++
++	ret = lxc_terminal_set_fifo(terminal, in, out, err, &fifofd_in);
++	if (ret < 0) {
++		ERROR("Faild to set fifos to console config");
++		ret = -1;
++		goto free_out;
++	}
++
++	if (lxc_mainloop_add_handler(terminal->descr, fifofd_in,
++	                             lxc_terminal_io_cb, terminal)) {
++		ERROR("console fifo not added to mainloop");
++		lxc_terminal_delete_fifo(fifofd_in, &terminal->fifos);
++		ret = -1;
++		goto free_out;
++	}
++
++free_out:
++	if (tmp)
++		free(tmp);
++	return ret;
++}
++
+ #else
+ int lxc_terminal_create(struct lxc_terminal *terminal)
+ {
+diff --git a/src/lxc/terminal.h b/src/lxc/terminal.h
+index dfc03c6..b4160b3 100644
+--- a/src/lxc/terminal.h
++++ b/src/lxc/terminal.h
+@@ -274,4 +274,8 @@ extern void lxc_terminal_init(struct lxc_terminal *terminal);
+ extern int lxc_terminal_map_ids(struct lxc_conf *c,
+ 				struct lxc_terminal *terminal);
+ 
++#ifdef HAVE_ISULAD
++int lxc_terminal_add_fifos(struct lxc_conf *conf, const char *fifonames);
++#endif
++
+ #endif /* __LXC_TERMINAL_H */
+-- 
+1.8.3.1
+

0017-isulad-check-cgroup-cpu.shares-after-setted.patch

@@ -1,99 +0,0 @@
-From 068331c965f3011f03e31b67d9791b2f81fd69b0 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Mon, 14 Jan 2019 11:03:03 +0800
-Subject: [PATCH 017/140] isulad: check cgroup cpu.shares after setted
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/cgroups/cgfsng.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++
- 1 file changed, 61 insertions(+)
-
-diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index 3e702b3..ab5732b 100644
---- a/src/lxc/cgroups/cgfsng.c
-+++ b/src/lxc/cgroups/cgfsng.c
-@@ -2204,6 +2204,42 @@ static int cg_legacy_set_data(struct cgroup_ops *ops, const char *filename,
- 	return ret;
- }
- 
-+/* Called from setup_limits - here we have the container's cgroup_data because
-+ * we created the cgroups.
-+ */
-+static int cg_legacy_get_data(struct cgroup_ops *ops, const char *filename,
-+			      char *value, size_t len)
-+{
-+	char *fullpath, *p;
-+	struct hierarchy *h;
-+	int ret = 0;
-+	char *controller = NULL;
-+
-+	len = strlen(filename);
-+	controller = alloca(len + 1);
-+	(void)strlcpy(controller, filename, len + 1);
-+
-+	p = strchr(controller, '.');
-+	if (p)
-+		*p = '\0';
-+
-+
-+	h = get_hierarchy(ops, controller);
-+	if (!h) {
-+		ERROR("Failed to setup limits for the \"%s\" controller. "
-+		      "The controller seems to be unused by \"cgfsng\" cgroup "
-+		      "driver or not enabled on the cgroup hierarchy",
-+		      controller);
-+		errno = ENOENT;
-+		return -ENOENT;
-+	}
-+
-+	fullpath = must_make_path(h->container_full_path, filename, NULL);
-+	ret = lxc_read_from_file(fullpath, value, len);
-+	free(fullpath);
-+	return ret;
-+}
-+
- static bool __cg_legacy_setup_limits(struct cgroup_ops *ops,
- 				     struct lxc_list *cgroup_settings,
- 				     bool do_devices)
-@@ -2211,6 +2247,8 @@ static bool __cg_legacy_setup_limits(struct cgroup_ops *ops,
- 	struct lxc_list *iterator, *next, *sorted_cgroup_settings;
- 	struct lxc_cgroup *cg;
- 	bool ret = false;
-+	char value[21];
-+	long long int readvalue, setvalue;
- 
- 	if (lxc_list_empty(cgroup_settings))
- 		return true;
-@@ -2236,6 +2274,29 @@ static bool __cg_legacy_setup_limits(struct cgroup_ops *ops,
- 			DEBUG("Set controller \"%s\" set to \"%s\"",
- 			      cg->subsystem, cg->value);
- 		}
-+		// isulad: check cpu shares
-+		if (strcmp(cg->subsystem, "cpu.shares") == 0) {
-+			if (cg_legacy_get_data(ops, cg->subsystem, value, sizeof(value)) < 0) {
-+				SYSERROR("Error get %s", cg->subsystem);
-+				goto out;
-+			}
-+			trim(value);
-+			if (lxc_safe_long_long(cg->value, &setvalue) != 0) {
-+				SYSERROR("Invalid value %s", cg->value);
-+				goto out;
-+			}
-+			if (lxc_safe_long_long(value, &readvalue) != 0) {
-+				SYSERROR("Invalid value %s", value);
-+				goto out;
-+			}
-+			if (setvalue > readvalue) {
-+				ERROR("The maximum allowed cpu-shares is %s", value);
-+				goto out;
-+			} else if (setvalue < readvalue) {
-+				ERROR("The minimum allowed cpu-shares is %s", value);
-+				goto out;
-+			}
-+		}
- 	}
- 
- 	ret = true;
--- 
-1.8.3.1
-

0018-lxc-attach-add-support-terminal-fifos.patch

@@ -1,410 +0,0 @@
-From 0c11550c088cca12b6900623240799dc911da458 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Mon, 14 Jan 2019 02:18:26 -0500
-Subject: [PATCH 018/140] lxc-attach: add support terminal fifos
-
-1. support terminal fifos to redirect terminal
-2. support lxc-attach run in background
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/attach.c           |  18 ++++-
- src/lxc/attach_options.h   |   3 +
- src/lxc/terminal.c         |  27 +++++--
- src/lxc/tools/arguments.h  |   2 +-
- src/lxc/tools/lxc_attach.c | 181 +++++++++++++++++++++++++++++++++++++++++----
- 5 files changed, 204 insertions(+), 27 deletions(-)
-
-diff --git a/src/lxc/attach.c b/src/lxc/attach.c
-index 2bbf1eb..1886bde 100644
---- a/src/lxc/attach.c
-+++ b/src/lxc/attach.c
-@@ -988,12 +988,23 @@ on_error:
- }
- 
- static int lxc_attach_terminal(struct lxc_conf *conf,
--			       struct lxc_terminal *terminal)
-+			       struct lxc_terminal *terminal, lxc_attach_options_t *options)
- {
- 	int ret;
- 
- 	lxc_terminal_init(terminal);
- 
-+	/* isulad: if we pass fifo in option, use them as init fifos */
-+	if (options->init_fifo[0] && options->init_fifo[1]) {
-+		if (terminal->init_fifo[0])
-+			free(terminal->init_fifo[0]);
-+		terminal->init_fifo[0] = strdup(options->init_fifo[0]);
-+
-+		if (terminal->init_fifo[1])
-+			free(terminal->init_fifo[1]);
-+		terminal->init_fifo[1] = strdup(options->init_fifo[1]);
-+	}
-+
- 	ret = lxc_terminal_create(terminal);
- 	if (ret < 0) {
- 		ERROR("Failed to create terminal");
-@@ -1203,7 +1214,7 @@ int lxc_attach(const char *name, const char *lxcpath,
- 	}
- 
- 	if (options->attach_flags & LXC_ATTACH_TERMINAL) {
--		ret = lxc_attach_terminal(conf, &terminal);
-+		ret = lxc_attach_terminal(conf, &terminal, options);
- 		if (ret < 0) {
- 			ERROR("Failed to setup new terminal");
- 			free(cwd);
-@@ -1489,7 +1500,7 @@ int lxc_attach(const char *name, const char *lxcpath,
- 	}
- 
- 	if (pid == 0) {
--		if (options->attach_flags & LXC_ATTACH_TERMINAL) {
-+		if (options->attach_flags & LXC_ATTACH_TERMINAL && terminal.tty_state) {
- 			ret = pthread_sigmask(SIG_SETMASK,
- 					      &terminal.tty_state->oldmask, NULL);
- 			if (ret < 0) {
-@@ -1497,7 +1508,6 @@ int lxc_attach(const char *name, const char *lxcpath,
- 				_exit(EXIT_FAILURE);
- 			}
- 		}
--
- 		ret = attach_child_main(&payload);
- 		if (ret < 0)
- 			ERROR("Failed to exec");
-diff --git a/src/lxc/attach_options.h b/src/lxc/attach_options.h
-index 193fd7e..081618c 100644
---- a/src/lxc/attach_options.h
-+++ b/src/lxc/attach_options.h
-@@ -135,6 +135,8 @@ typedef struct lxc_attach_options_t {
- 
- 	/*! File descriptor to log output. */
- 	int log_fd;
-+
-+	char *init_fifo[2]; /* isulad: default fifos for the start */
- } lxc_attach_options_t;
- 
- /*! Default attach options to use */
-@@ -153,6 +155,7 @@ typedef struct lxc_attach_options_t {
- 		/* .stdout_fd = */      1,                                     \
- 		/* .stderr_fd = */      2,                                     \
- 		/* .log_fd    = */      -EBADF,                                \
-+		/* .init_fifo = */	{NULL, NULL},                          \
- 	}
- 
- /*!
-diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
-index 7aa4730..ee3aef2 100644
---- a/src/lxc/terminal.c
-+++ b/src/lxc/terminal.c
-@@ -514,7 +514,7 @@ static int lxc_terminal_mainloop_add_peer(struct lxc_terminal *terminal)
- }
- 
- /* isulad add fifo to mainloop */
--static int lxc_console_mainloop_add_fifo(struct lxc_terminal *terminal)
-+static int lxc_terminal_mainloop_add_fifo(struct lxc_terminal *terminal)
- {
- 	int ret = 0;
- 	struct lxc_list *it,*next;
-@@ -564,7 +564,7 @@ int lxc_terminal_mainloop_add(struct lxc_epoll_descr *descr,
- 	}
- 
- 	/* isulad add fifo to mainloop */
--	ret = lxc_console_mainloop_add_fifo(terminal);
-+	ret = lxc_terminal_mainloop_add_fifo(terminal);
- 	if (ret < 0) {
- 		ERROR("Failed to add handler for terminal fifos to mainloop");
- 		return -1;
-@@ -789,13 +789,28 @@ void lxc_terminal_free(struct lxc_conf *conf, int fd)
- static int lxc_terminal_peer_default(struct lxc_terminal *terminal)
- {
- 	struct lxc_terminal_state *ts;
--	const char *path;
-+	const char *path = NULL;
- 	int ret = 0;
- 
- 	if (terminal->path)
- 		path = terminal->path;
--	else
--		path = "/dev/tty";
-+
-+	/* isulad: if no console was given, try current controlling terminal, there
-+	 * won't be one if we were started as a daemon (-d)
-+	 */
-+	if (!path && !access("/dev/tty", F_OK)) {
-+		int fd;
-+		fd = open("/dev/tty", O_RDWR);
-+		if (fd >= 0) {
-+			close(fd);
-+			path = "/dev/tty";
-+		}
-+	}
-+
-+	if (!path) {
-+		DEBUG("Not have a controlling terminal");
-+		return 0;
-+	}
- 
- 	terminal->peer = lxc_unpriv(open(path, O_RDWR | O_CLOEXEC));
- 	if (terminal->peer < 0) {
-@@ -1355,7 +1370,7 @@ int lxc_terminal_prepare_login(int fd)
- 	if (ret < 0)
- 		return -1;
- 
--	ret = lxc_terminal_set_stdfds(fd);
-+	ret = set_stdfds(fd);
- 	if (ret < 0)
- 		return -1;
- 
-diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
-index 61f4a0a..047e9f1 100644
---- a/src/lxc/tools/arguments.h
-+++ b/src/lxc/tools/arguments.h
-@@ -62,7 +62,7 @@ struct lxc_arguments {
- 
- 	/* for lxc-start */
- 	const char *share_ns[32]; /* size must be greater than LXC_NS_MAX */
--	const char *terminal_fifos[2]; /* isulad add, fifos used to redirct stdin/out/err */
-+	char *terminal_fifos[2]; /* isulad add, fifos used to redirct stdin/out/err */
- 	const char *container_info; /* isulad: file used to store pid and ppid info of container */
- 	const char *exit_monitor_fifo; /* isulad: fifo used to monitor state of monitor process */
- 
-diff --git a/src/lxc/tools/lxc_attach.c b/src/lxc/tools/lxc_attach.c
-index 8c8e7d3..6d0ffe5 100644
---- a/src/lxc/tools/lxc_attach.c
-+++ b/src/lxc/tools/lxc_attach.c
-@@ -75,6 +75,8 @@ static const struct option my_longopts[] = {
- 	{"set-var", required_argument, 0, 'v'},
- 	{"pty-log", required_argument, 0, 'L'},
- 	{"rcfile", required_argument, 0, 'f'},
-+	{"in-fifo", required_argument, 0, OPT_INPUT_FIFO}, /* isulad add terminal fifos*/
-+	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
- 	LXC_COMMON_OPTIONS
- };
- 
-@@ -133,6 +135,9 @@ Options :\n\
- 	.log_file     = "none",
- };
- 
-+// isulad: send '128 + signal' if container is killed by signal.
-+#define ExitSignalOffset 128
-+
- static int my_parser(struct lxc_arguments *args, int c, char *arg)
- {
- 	int ret;
-@@ -190,6 +195,12 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
- 	case 'f':
- 		args->rcfile = arg;
- 		break;
-+	case OPT_INPUT_FIFO:
-+		args->terminal_fifos[0] = arg;
-+		break;
-+	case OPT_OUTPUT_FIFO:
-+		args->terminal_fifos[1] = arg;
-+		break;
- 	}
- 
- 	return 0;
-@@ -253,10 +264,143 @@ static int lxc_attach_create_log_file(const char *log_file)
- 	return fd;
- }
- 
-+/*isulad: attach with terminal*/
-+static int do_attach_foreground(struct lxc_container *c, lxc_attach_command_t *command,
-+				lxc_attach_options_t *attach_options,
-+				char **errmsg)
-+{
-+	int ret = 0;
-+	pid_t pid;
-+	int wexit = -1;
-+	int signal;
-+
-+	if (command->program)
-+		ret = c->attach(c, lxc_attach_run_command, command, attach_options, &pid);
-+	else
-+		ret = c->attach(c, lxc_attach_run_shell, NULL, attach_options, &pid);
-+	if (ret < 0)
-+		goto out;
-+
-+	ret = lxc_wait_for_pid_status(pid);
-+	if (ret < 0)
-+		goto out;
-+
-+	if (WIFEXITED(ret))
-+		wexit = WEXITSTATUS(ret);
-+	else
-+		wexit = -1;
-+
-+	if (WIFSIGNALED(ret)) {
-+		signal = WTERMSIG(ret);
-+		wexit = ExitSignalOffset + signal;
-+	}
-+out:
-+	//if (c->lxc_conf->errmsg)
-+	//	*errmsg = strdup(c->lxc_conf->errmsg);
-+	return wexit;
-+}
-+
-+static void close_msg_pipe(int *errpipe)
-+{
-+	if (errpipe[0] >= 0) {
-+		close(errpipe[0]);
-+		errpipe[0] = -1;
-+	}
-+	if (errpipe[1] >= 0) {
-+		close(errpipe[1]);
-+		errpipe[1] = -1;
-+	}
-+}
-+
-+/*isulad: attach without terminal in background */
-+static int do_attach_background(struct lxc_container *c, lxc_attach_command_t *command,
-+				lxc_attach_options_t *attach_options,
-+				char **errmsg)
-+{
-+	int ret = 0;
-+	int msgpipe[2];
-+	pid_t pid = 0;
-+	ssize_t size_read;
-+	char msgbuf[BUFSIZ + 1] = {0};
-+
-+	//pipdfd for get error message of child or grandchild process.
-+	if (pipe2(msgpipe, O_CLOEXEC) != 0) {
-+		SYSERROR("Failed to init msgpipe");
-+		return -1;
-+	}
-+
-+	pid = fork();
-+	if (pid < 0) {
-+		close_msg_pipe(msgpipe);
-+		return -1;
-+	}
-+
-+	if (pid != 0) {
-+		close(msgpipe[1]);
-+		msgpipe[1] = -1;
-+		size_read = read(msgpipe[0], msgbuf, BUFSIZ);
-+		if (size_read > 0) {
-+			*errmsg = strdup(msgbuf);
-+			ret = -1;
-+		}
-+
-+		close(msgpipe[0]);
-+		msgpipe[0] = -1;
-+
-+		return ret;
-+	}
-+
-+	/* second fork to be reparented by init */
-+	pid = fork();
-+	if (pid < 0) {
-+		SYSERROR("Error doing dual-fork");
-+		close_msg_pipe(msgpipe);
-+		exit(1);
-+	}
-+	if (pid != 0) {
-+		close_msg_pipe(msgpipe);
-+		exit(0);
-+	}
-+
-+	close(msgpipe[0]);
-+	msgpipe[0] = -1;
-+
-+	if (null_stdfds() < 0) {
-+		ERROR("failed to close fds");
-+		exit(1);
-+	}
-+	setsid();
-+
-+	if (command->program)
-+		ret = c->attach(c, lxc_attach_run_command, command, attach_options, &pid);
-+	else
-+		ret = c->attach(c, lxc_attach_run_shell, NULL, attach_options, &pid);
-+	if (ret < 0) {
-+		//if (c->lxc_conf->errmsg)
-+		//	lxc_write_error_message(msgpipe[1], "%s", c->lxc_conf->errmsg);
-+		close(msgpipe[1]);
-+		msgpipe[1] = -1;
-+		ret = -1;
-+		goto out;
-+	}
-+
-+	close(msgpipe[1]);
-+	msgpipe[1] = -1;
-+
-+	ret = wait_for_pid(pid);
-+out:
-+	lxc_container_put(c);
-+	if (ret)
-+		exit(EXIT_FAILURE);
-+	else
-+		exit(0);
-+}
-+
- int main(int argc, char *argv[])
- {
- 	int ret = -1;
- 	int wexit = 0;
-+	char *errmsg = NULL;
- 	struct lxc_log log;
- 	pid_t pid;
- 	lxc_attach_options_t attach_options = LXC_ATTACH_OPTIONS_DEFAULT;
-@@ -316,8 +460,13 @@ int main(int argc, char *argv[])
- 	if (elevated_privileges)
- 		attach_options.attach_flags &= ~(elevated_privileges);
- 
--	if (stdfd_is_pty())
-+	if (my_args.terminal_fifos[0] && my_args.terminal_fifos[1]) {
-+		attach_options.init_fifo[0] = my_args.terminal_fifos[0];
-+		attach_options.init_fifo[1] = my_args.terminal_fifos[1];
-+		attach_options.attach_flags |= LXC_ATTACH_TERMINAL;
-+	} else if (stdfd_is_pty()) {
- 		attach_options.attach_flags |= LXC_ATTACH_TERMINAL;
-+	}
- 
- 	attach_options.namespaces = namespace_flags;
- 	attach_options.personality = new_personality;
-@@ -332,27 +481,27 @@ int main(int argc, char *argv[])
- 
- 	if (my_args.console_log) {
- 		attach_options.log_fd = lxc_attach_create_log_file(my_args.console_log);
--		if (attach_options.log_fd < 0)
--			goto out;
-+		if (attach_options.log_fd < 0) {
-+			ERROR("Failed to create log file for %s", c->name);
-+			lxc_container_put(c);
-+			exit(EXIT_FAILURE);
-+		}
- 	}
- 
--	if (command.program)
--		ret = c->attach(c, lxc_attach_run_command, &command, &attach_options, &pid);
-+	/* isulad: add do attach background */
-+	if (attach_options.attach_flags & LXC_ATTACH_TERMINAL)
-+		wexit = do_attach_foreground(c, &command, &attach_options, &errmsg);
- 	else
--		ret = c->attach(c, lxc_attach_run_shell, NULL, &attach_options, &pid);
--	if (ret < 0)
--		goto out;
-+		wexit = do_attach_background(c, &command, &attach_options, &errmsg);
- 
--	ret = lxc_wait_for_pid_status(pid);
--	if (ret < 0)
--		goto out;
--
--	if (WIFEXITED(ret))
--		wexit = WEXITSTATUS(ret);
-+	if (errmsg) {
-+		fprintf(stderr, "%s:%s:%s:%d starting container process caused \"%s\"", c->name,
-+		__FILE__, __func__, __LINE__, errmsg);
-+		free(errmsg);
-+	}
- 
--out:
- 	lxc_container_put(c);
--	if (ret >= 0)
-+	if (wexit >= 0)
- 		exit(wexit);
- 
- 	exit(EXIT_FAILURE);
--- 
-1.8.3.1
-

0018-pty-setup-pty-after-setup-rootfs-mount-options.patch

@@ -0,0 +1,157 @@
+From 8d2f80168d89abc317affee358120dcf25b8af19 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Tue, 14 Apr 2020 12:50:55 +0800
+Subject: [PATCH 18/49] pty: setup pty after setup rootfs mount options
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/conf.c     | 16 +++++++++----
+ src/lxc/start.c    |  2 +-
+ src/lxc/terminal.c | 67 ++++++++++++++++++++++++++++++++++++------------------
+ 3 files changed, 57 insertions(+), 28 deletions(-)
+
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index fce241b..2e93227 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -3748,14 +3748,20 @@ int lxc_setup(struct lxc_handler *handler)
+ 	if (lxc_conf->autodev > 0)
+ 		(void)lxc_setup_boot_id();
+ 
++#ifdef HAVE_ISULAD
++	if (setup_rootfs_mountopts(&lxc_conf->rootfs)) {
++			return log_error(-1, "failed to set rootfs for '%s'", name);
++	}
++	if (lxc_conf->rootfs.path) {
++		ret = lxc_setup_devpts(lxc_conf);
++		if (ret < 0) {
++			return log_error(-1, "Failed to setup new devpts instance for '%s'", name);
++		}
++	}
++#else
+ 	ret = lxc_setup_devpts(lxc_conf);
+ 	if (ret < 0)
+ 		return log_error(-1, "Failed to setup new devpts instance");
+-
+-#ifdef HAVE_ISULAD
+-        if (setup_rootfs_mountopts(&lxc_conf->rootfs)) {
+-               return log_error(-1, "failed to set rootfs for '%s'", name);
+-        }
+ #endif
+ 
+ 	ret = lxc_create_ttys(handler);
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 145b015..800f884 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -596,7 +596,7 @@ int lxc_poll(const char *name, struct lxc_handler *handler)
+ 		goto out_mainloop_console;
+ 
+ 	if (has_console)
+-		ret = lxc_mainloop(&descr_console, 0);
++		ret = lxc_mainloop(&descr_console, 100);
+ 
+ out_mainloop_console:
+ 	if (has_console) {
+diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
+index 4d7c2cd..1f46d49 100644
+--- a/src/lxc/terminal.c
++++ b/src/lxc/terminal.c
+@@ -186,6 +186,42 @@ static int lxc_terminal_truncate_log_file(struct lxc_terminal *terminal)
+ 	return lxc_unpriv(ftruncate(terminal->log_fd, 0));
+ }
+ 
++#ifdef HAVE_ISULAD
++/*
++ * isulad: support mult-logfiles
++ * */
++static int lxc_terminal_rename_old_log_file(struct lxc_terminal *terminal)
++{
++	int ret;
++	unsigned int i;
++	char tmp[PATH_MAX] = {0};
++	char *rename_fname = NULL;
++
++	for (i = terminal->log_rotate - 1; i > 1; i--) {
++		ret = snprintf(tmp, PATH_MAX, "%s.%u", terminal->log_path, i);
++		if (ret < 0 || ret >= PATH_MAX) {
++			free(rename_fname);
++			return -EFBIG;
++		}
++		free(rename_fname);
++		rename_fname = safe_strdup(tmp);
++		ret = snprintf(tmp, PATH_MAX, "%s.%u", terminal->log_path, (i - 1));
++		if (ret < 0 || ret >= PATH_MAX) {
++			free(rename_fname);
++			return -EFBIG;
++		}
++		ret = lxc_unpriv(rename(tmp, rename_fname));
++		if (ret < 0 && errno != ENOENT) {
++			free(rename_fname);
++			return ret;
++		}
++	}
++
++	free(rename_fname);
++	return 0;
++}
++#endif
++
+ static int lxc_terminal_rotate_log_file(struct lxc_terminal *terminal)
+ {
+ 	__do_free char *tmp = NULL;
+@@ -199,6 +235,15 @@ static int lxc_terminal_rotate_log_file(struct lxc_terminal *terminal)
+ 	if (terminal->log_fd < 0)
+ 		return -EBADF;
+ 
++#ifdef HAVE_ISULAD
++	/* isuald: rotate old log file first */
++	ret = lxc_terminal_rename_old_log_file(terminal);
++	if(ret != 0) {
++		ERROR("Rename old log file failed");
++		return ret;
++	}
++#endif
++
+ 	len = strlen(terminal->log_path) + sizeof(".1");
+ 	tmp = must_realloc(NULL, len);
+ 
+@@ -1512,21 +1557,6 @@ static int lxc_terminal_fifo_default(struct lxc_terminal *terminal)
+ 	return 0;
+ }
+ 
+-static int use_unix_newline(int master_fd)
+-{
+-	struct termios oldtios;
+-	int ret;
+-
+-	ret = tcgetattr(master_fd, &oldtios);
+-	if (ret < 0)
+-		return -1;
+-	oldtios.c_oflag &= ~ONLCR;
+-	ret = tcsetattr(master_fd, TCSAFLUSH, &oldtios);
+-	if (ret < 0)
+-		return -1;
+-	return 0;
+-}
+-
+ int lxc_terminal_create(struct lxc_terminal *terminal)
+ {
+ 	int ret;
+@@ -1544,13 +1574,6 @@ int lxc_terminal_create(struct lxc_terminal *terminal)
+ 			goto err;
+ 		}
+ 
+-		/* isulad: clear ONLCR flag */
+-		ret = use_unix_newline(terminal->master);
+-		if (ret < 0) {
+-			SYSERROR("Failed to clear ONLCR flag on terminal master");
+-			goto err;
+-		}
+-
+ 		ret = fd_cloexec(terminal->master, true);
+ 		if (ret < 0) {
+ 			SYSERROR("Failed to set FD_CLOEXEC flag on terminal master");
+-- 
+1.8.3.1
+

0019-remount-cgroup-readonly-and-make-soft-link-of-subcgr.patch

@@ -1,87 +0,0 @@
-From 16616f224d5577594548b2ce1ee50f51a449e20d Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Mon, 14 Jan 2019 13:51:01 +0800
-Subject: [PATCH 019/140] remount cgroup readonly and make soft link of
- subcgroup
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/cgroups/cgfsng.c | 44 ++++++++++++++++++++++++++++++++++++++++++++
- 1 file changed, 44 insertions(+)
-
-diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index ab5732b..705985f 100644
---- a/src/lxc/cgroups/cgfsng.c
-+++ b/src/lxc/cgroups/cgfsng.c
-@@ -1621,6 +1621,7 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
- 	int i, ret;
- 	char *tmpfspath = NULL;
- 	bool has_cgns = false, retval = false, wants_force_mount = false;
-+	char **merged = NULL;
- 
- 	if ((type & LXC_AUTO_CGROUP_MASK) == 0)
- 		return true;
-@@ -1667,6 +1668,14 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
- 			continue;
- 		controller++;
- 
-+		// isulad: symlink subcgroup
-+		if (strchr(controller, ',') != NULL) {
-+			int pret;
-+			pret = lxc_append_string(&merged, controller);
-+			if (pret < 0)
-+				goto on_error;
-+		}
-+
- 		controllerpath = must_make_path(tmpfspath, controller, NULL);
- 		if (dir_exists(controllerpath)) {
- 			free(controllerpath);
-@@ -1721,10 +1730,45 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
- 		if (ret < 0)
- 			goto on_error;
- 	}
-+
-+	// isulad: symlink subcgroup
-+	if (merged) {
-+		char **mc;
-+		for (mc = merged; *mc; mc++) {
-+			char *token;
-+			char *merge = must_copy_string(*mc);
-+			lxc_iterate_parts(token, merge, ",") {
-+				int mret;
-+				char *link;
-+				link = must_make_path(tmpfspath, token, NULL);
-+				mret = symlink(*mc, link);
-+				if (mret < 0 && errno != EEXIST) {
-+					SYSERROR("Failed to create link %s for target %s", link, merge);
-+					free(merge);
-+					free(link);
-+					goto on_error;
-+				}
-+				free(link);
-+			}
-+			free(merge);
-+		}
-+	}
-+
-+
-+	// isulad: remount /sys/fs/cgroup to readonly
-+	if (type == LXC_AUTO_CGROUP_FULL_RO || type == LXC_AUTO_CGROUP_RO) {
-+		ret = mount(tmpfspath, tmpfspath, "bind",
-+			    MS_NOSUID|MS_NODEV|MS_NOEXEC|MS_RELATIME|MS_RDONLY|MS_BIND|MS_REMOUNT, NULL);
-+		if (ret < 0) {
-+			SYSERROR("Failed to remount /sys/fs/cgroup.");
-+			goto on_error;
-+		}
-+	}
- 	retval = true;
- 
- on_error:
- 	free(tmpfspath);
-+	lxc_free_array((void **)merged, free);
- 	return retval;
- }
- 
--- 
-1.8.3.1
-

0019-resize-implement-resize-function-in-exec-start.patch

@@ -1,88 +1,178 @@
-From d51a587ee6a9ba82e52f5f1a61191e257865041d Mon Sep 17 00:00:00 2001
+From f690df5983011dd3b61f0e139c3e939f3fecf37a Mon Sep 17 00:00:00 2001
 From: LiFeng <lifeng68@huawei.com>
-Date: Fri, 29 Nov 2019 22:12:10 -0500
+Date: Tue, 14 Apr 2020 16:04:15 +0800
-Subject: [PATCH 138/140] resize: implement resize function in exec/start
+Subject: [PATCH 19/49] resize: implement resize function in exec/start
 
 Signed-off-by: LiFeng <lifeng68@huawei.com>
 ---
- src/lxc/Makefile.am        |   1 +
+ src/lxc/Makefile.am     |  10 +-
- src/lxc/attach.c           |  45 ++++-
+ src/lxc/af_unix.c       |  33 +++-
- src/lxc/attach.h           |   2 +-
+ src/lxc/af_unix.h       |   5 +-
- src/lxc/commands.c         |  44 +++++
+ src/lxc/attach.c        |  51 +++++-
- src/lxc/commands.h         |   9 +
+ src/lxc/commands.c      |  50 ++++++
- src/lxc/exec_commands.c    | 416 +++++++++++++++++++++++++++++++++++++++++++++
+ src/lxc/commands.h      |   2 +
- src/lxc/exec_commands.h    |  73 ++++++++
+ src/lxc/exec_commands.c | 416 ++++++++++++++++++++++++++++++++++++++++++++++++
- src/lxc/lxccontainer.c     |  64 ++++++-
+ src/lxc/exec_commands.h |  73 +++++++++
- src/lxc/lxccontainer.h     |  15 +-
+ src/lxc/lxccontainer.c  |  50 ++++++
- src/lxc/terminal.c         |  26 +++
+ src/lxc/terminal.c      |  27 ++++
- src/lxc/terminal.h         |   1 +
+ src/lxc/terminal.h      |   1 +
- src/lxc/tools/arguments.h  |   3 +-
+ 11 files changed, 710 insertions(+), 8 deletions(-)
- src/lxc/tools/lxc_attach.c |  20 ++-
- src/lxc/tools/lxc_copy.c   |   2 +-
- src/lxc/tools/lxc_ls.c     |   2 +-
- 15 files changed, 698 insertions(+), 25 deletions(-)
  create mode 100644 src/lxc/exec_commands.c
  create mode 100644 src/lxc/exec_commands.h
 
 diff --git a/src/lxc/Makefile.am b/src/lxc/Makefile.am
-index 27240cc..c21eb85 100644
+index 23935e5..c288c51 100644
 --- a/src/lxc/Makefile.am
 +++ b/src/lxc/Makefile.am
-@@ -101,6 +101,7 @@ liblxc_la_SOURCES = af_unix.c af_unix.h \
+@@ -52,14 +52,15 @@ noinst_HEADERS = api_extensions.h \
- 		    cgroups/cgroup_utils.c cgroups/cgroup_utils.h \
+ 		 utils.h \
- 		    compiler.h \
+ 		 uuid.h
- 		    commands.c commands.h \
-+		    exec_commands.c exec_commands.h \
- 		    commands_utils.c commands_utils.h \
- 		    conf.c conf.h \
- 		    confile.c confile.h \
-diff --git a/src/lxc/attach.c b/src/lxc/attach.c
-index 1f14eb4..03a7646 100644
---- a/src/lxc/attach.c
-+++ b/src/lxc/attach.c
-@@ -64,6 +64,7 @@
- #include "syscall_wrappers.h"
- #include "terminal.h"
- #include "utils.h"
-+#include "exec_commands.h"
  
- #if HAVE_SYS_PERSONALITY_H
+-#if HAVE_ISULAD
- #include <sys/personality.h>
++if HAVE_ISULAD
-@@ -1208,7 +1209,7 @@ out:
+ noinst_HEADERS += isulad_utils.h path.h \
- 	return ret;
+ 				  json/json_common.h json/defs.h \
+ 				  json/oci_runtime_hooks.h \
+ 				  json/logger_json_file.h \
+ 				  json/oci_runtime_spec.h \
+-				  json/read-file.h
+-#endif
++				  json/read-file.h \
++				  exec_commands.h
++endif
+ 
+ if IS_BIONIC
+ noinst_HEADERS += ../include/fexecve.h \
+@@ -172,7 +173,8 @@ liblxc_la_SOURCES += isulad_utils.c isulad_utils.h \
+ 			json/oci_runtime_hooks.c json/oci_runtime_hooks.h \
+ 			json/logger_json_file.c json/logger_json_file.h \
+ 			json/oci_runtime_spec.c json/oci_runtime_spec.h \
+-			json/read-file.c json/read-file.h
++			json/read-file.c json/read-file.h \
++			exec_commands.c exec_commands.h
+ endif
+ 
+ if IS_BIONIC
+diff --git a/src/lxc/af_unix.c b/src/lxc/af_unix.c
+index e172088..9f268be 100644
+--- a/src/lxc/af_unix.c
++++ b/src/lxc/af_unix.c
+@@ -168,7 +168,7 @@ int lxc_unix_send_fds(int fd, int *sendfds, int num_sendfds, void *data,
  }
  
--int lxc_attach(const char *name, const char *lxcpath,
+ static int lxc_abstract_unix_recv_fds_iov(int fd, int *recvfds, int num_recvfds,
-+int lxc_attach(const char *name, const char *lxcpath, const char *suffix,
+-					  struct iovec *iov, size_t iovlen)
- 	       lxc_attach_exec_t exec_function, void *exec_payload,
++					  struct iovec *iov, size_t iovlen, unsigned int timeout)
- 	       lxc_attach_options_t *options, pid_t *attached_process, char **err_msg)
  {
-@@ -1221,6 +1222,10 @@ int lxc_attach(const char *name, const char *lxcpath,
+ 	__do_free char *cmsgbuf = NULL;
- 	struct lxc_terminal terminal;
+ 	int ret;
+@@ -188,6 +188,22 @@ static int lxc_abstract_unix_recv_fds_iov(int fd, int *recvfds, int num_recvfds,
+ 	msg.msg_iov = iov;
+ 	msg.msg_iovlen = iovlen;
+ 
++#ifdef HAVE_ISULAD
++	struct timeval out;
++	if (timeout > 0) {
++		memset(&out, 0, sizeof(out));
++		out.tv_sec = timeout / 1000000;
++		out.tv_usec = timeout % 1000000;
++		ret = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO,
++		                 (const void *)&out, sizeof(out));
++		if (ret < 0) {
++			ERROR("Failed to set %u timeout on containter "
++			      "state socket", timeout);
++			return ret;
++		}
++	}
++#endif
++
+ 	do {
+ 		ret = recvmsg(fd, &msg, 0);
+ 	} while (ret < 0 && errno == EINTR);
+@@ -220,8 +236,21 @@ int lxc_abstract_unix_recv_fds(int fd, int *recvfds, int num_recvfds,
+ 		.iov_base = data ? data : buf,
+ 		.iov_len = data ? size : sizeof(buf),
+ 	};
+-	return lxc_abstract_unix_recv_fds_iov(fd, recvfds, num_recvfds, &iov, 1);
++	return lxc_abstract_unix_recv_fds_iov(fd, recvfds, num_recvfds, &iov, 1, 0);
++}
++
++#ifdef HAVE_ISULAD
++int lxc_abstract_unix_recv_fds_timeout(int fd, int *recvfds, int num_recvfds,
++                                       void *data, size_t size, unsigned int timeout)
++{
++	char buf[1] = {0};
++	struct iovec iov = {
++		.iov_base = data ? data : buf,
++		.iov_len = data ? size : sizeof(buf),
++	};
++	return lxc_abstract_unix_recv_fds_iov(fd, recvfds, num_recvfds, &iov, 1, timeout);
+ }
++#endif
+ 
+ int lxc_abstract_unix_send_credential(int fd, void *data, size_t size)
+ {
+diff --git a/src/lxc/af_unix.h b/src/lxc/af_unix.h
+index 2531b0b..6943a61 100644
+--- a/src/lxc/af_unix.h
++++ b/src/lxc/af_unix.h
+@@ -27,5 +27,8 @@ extern int lxc_unix_sockaddr(struct sockaddr_un *ret, const char *path);
+ extern int lxc_unix_connect(struct sockaddr_un *addr);
+ extern int lxc_unix_connect_type(struct sockaddr_un *addr, int type);
+ extern int lxc_socket_set_timeout(int fd, int rcv_timeout, int snd_timeout);
+-
++#ifdef HAVE_ISULAD
++int lxc_abstract_unix_recv_fds_timeout(int fd, int *recvfds, int num_recvfds,
++                                       void *data, size_t size, unsigned int timeout);
++#endif
+ #endif /* __LXC_AF_UNIX_H */
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index e66ca1c..33946bb 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -45,6 +45,10 @@
+ #include "terminal.h"
+ #include "utils.h"
+ 
++#ifdef HAVE_ISULAD
++#include "exec_commands.h"
++#endif
++
+ #if HAVE_SYS_PERSONALITY_H
+ #include <sys/personality.h>
+ #endif
+@@ -998,6 +1002,13 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
  	struct lxc_conf *conf;
+ 	char *name, *lxcpath;
  	struct attach_clone_payload payload = {0};
++#ifdef HAVE_ISULAD
 +	struct lxc_exec_command_handler exec_command;
++	const char *suffix = options->suffix;
 +
 +	exec_command.maincmd_fd = -1;
 +	exec_command.terminal = &terminal;
++#endif
  
  	ret = access("/proc/self/ns", X_OK);
- 	if (ret) {
+ 	if (ret)
-@@ -1354,6 +1359,10 @@ int lxc_attach(const char *name, const char *lxcpath,
+@@ -1129,6 +1140,12 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
  		}
  
  		terminal.log_fd = options->log_fd;
++#ifdef HAVE_ISULAD
 +		if (suffix != NULL) {
 +			exec_command.maincmd_fd = lxc_exec_cmd_init(name, lxcpath, suffix);
 +			exec_command.terminal = &terminal;
 +		}
++#endif
  	} else {
  		lxc_terminal_init(&terminal);
  	}
-@@ -1394,15 +1403,30 @@ int lxc_attach(const char *name, const char *lxcpath,
+@@ -1169,6 +1186,15 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
  	ret = socketpair(PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0, ipc_sockets);
  	if (ret < 0) {
  		SYSERROR("Could not set up required IPC mechanism for attaching");
++#ifdef HAVE_ISULAD
 +		if (options->attach_flags & LXC_ATTACH_TERMINAL) {
 +			lxc_terminal_delete(&terminal);
 +			lxc_terminal_conf_free(&terminal);
@@ -90,31 +180,15 @@ index 1f14eb4..03a7646 100644
 +				close(exec_command.maincmd_fd);
 +			}
 +		}
++#endif
  		free(cwd);
  		lxc_proc_put_context_info(init_ctx);
  		return -1;
- 	}
+@@ -1184,6 +1210,17 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
- 
--
- 	/* isulad: pipdfd for get error message of child or grandchild process. */
- 	if (pipe2(conf->errpipe, O_CLOEXEC) != 0) {
- 		SYSERROR("Failed to init errpipe");
-+		if (options->attach_flags & LXC_ATTACH_TERMINAL) {
-+			lxc_terminal_delete(&terminal);
-+			lxc_terminal_conf_free(&terminal);
-+			if (exec_command.maincmd_fd != -1) {
-+				close(exec_command.maincmd_fd);
-+			}
-+		}
-+		close(ipc_sockets[0]);
-+		close(ipc_sockets[1]);
- 		free(cwd);
- 		lxc_proc_put_context_info(init_ctx);
- 		return -1;
-@@ -1418,6 +1442,15 @@ int lxc_attach(const char *name, const char *lxcpath,
  	pid = fork();
  	if (pid < 0) {
  		SYSERROR("Failed to create first subprocess");
++#ifdef HAVE_ISULAD
 +		if (options->attach_flags & LXC_ATTACH_TERMINAL) {
 +			lxc_terminal_delete(&terminal);
 +			lxc_terminal_conf_free(&terminal);
@@ -124,67 +198,67 @@ index 1f14eb4..03a7646 100644
 +		}
 +		close(ipc_sockets[0]);
 +		close(ipc_sockets[1]);
++#endif
  		free(cwd);
  		lxc_proc_put_context_info(init_ctx);
  		return -1;
-@@ -1474,6 +1507,8 @@ int lxc_attach(const char *name, const char *lxcpath,
+@@ -1239,7 +1276,9 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 			ret = lxc_attach_terminal_mainloop_init(&terminal, &descr);
  			if (ret < 0)
  				goto on_error;
- 
+-
++#ifdef HAVE_ISULAD
 +			(void)lxc_exec_cmd_mainloop_add(&descr, &exec_command);
-+
++#endif
  			TRACE("Initialized terminal mainloop");
  		}
  
-@@ -1597,6 +1632,9 @@ int lxc_attach(const char *name, const char *lxcpath,
+@@ -1352,6 +1391,11 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
  		if (options->attach_flags & LXC_ATTACH_TERMINAL) {
  			lxc_terminal_delete(&terminal);
  			lxc_terminal_conf_free(&terminal);
++#ifdef HAVE_ISULAD
 +			if (exec_command.maincmd_fd != -1) {
 +				close(exec_command.maincmd_fd);
 +			}
++#endif
  		}
  
  		lxc_proc_put_context_info(init_ctx);
-@@ -1615,6 +1653,9 @@ int lxc_attach(const char *name, const char *lxcpath,
+@@ -1365,6 +1409,11 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
  		lxc_attach_terminal_close_master(&terminal);
  		lxc_attach_terminal_close_peer(&terminal);
  		lxc_attach_terminal_close_log(&terminal);
++#ifdef HAVE_ISULAD
 +		if (exec_command.maincmd_fd != -1) {
 +			close(exec_command.maincmd_fd);
 +		}
++#endif
  	}
  
  	/* Wait for the parent to have setup cgroups. */
-diff --git a/src/lxc/attach.h b/src/lxc/attach.h
-index e62b98b..8626a8e 100644
---- a/src/lxc/attach.h
-+++ b/src/lxc/attach.h
-@@ -41,7 +41,7 @@ struct lxc_proc_context_info {
- 	int ns_fd[LXC_NS_MAX];
- };
- 
--extern int lxc_attach(const char *name, const char *lxcpath,
-+extern int lxc_attach(const char *name, const char *lxcpath, const char *suffix,
- 	       lxc_attach_exec_t exec_function, void *exec_payload,
- 	       lxc_attach_options_t *options, pid_t *attached_process, char **err_msg);
- 
 diff --git a/src/lxc/commands.c b/src/lxc/commands.c
-index b70564f..8fd929d 100644
+index 0ffc5c7..184a219 100644
 --- a/src/lxc/commands.c
 +++ b/src/lxc/commands.c
-@@ -97,6 +97,7 @@ static const char *lxc_cmd_str(lxc_cmd_t cmd)
+@@ -86,6 +86,7 @@ static const char *lxc_cmd_str(lxc_cmd_t cmd)
- 		[LXC_CMD_CONSOLE_LOG]         = "console_log",
+ 		[LXC_CMD_GET_INIT_PIDFD]        	= "get_init_pidfd",
- 		[LXC_CMD_SERVE_STATE_CLIENTS] = "serve_state_clients",
+ #ifdef HAVE_ISULAD
- 		[LXC_CMD_SET_TERMINAL_FIFOS]  = "set_terminal_fifos",
+ 		[LXC_CMD_SET_TERMINAL_FIFOS]  		= "set_terminal_fifos",
-+		[LXC_CMD_SET_TERMINAL_WINCH]  = "set_terminal_winch",
++		[LXC_CMD_SET_TERMINAL_WINCH]  		= "set_terminal_winch",
+ #endif
  	};
  
- 	if (cmd >= LXC_CMD_MAX)
+@@ -1459,6 +1460,54 @@ static int lxc_cmd_set_terminal_fifos_callback(int fd, struct lxc_cmd_req *req,
-@@ -1126,6 +1127,48 @@ static int lxc_cmd_set_terminal_fifos_callback(int fd, struct lxc_cmd_req *req,
  
+ 	return lxc_cmd_rsp_send(fd, &rsp);
  }
- 
++
++struct lxc_cmd_set_terminal_winch_request {
++	unsigned int height;
++	unsigned int width;
++};
++
 +int lxc_cmd_set_terminal_winch(const char *name, const char *lxcpath, unsigned int height, unsigned int width)
 +{
 +	int ret = 0, stopped = 0;
@@ -215,7 +289,7 @@ index b70564f..8fd929d 100644
 +}
 +
 +static int lxc_cmd_set_terminal_winch_callback(int fd, struct lxc_cmd_req *req,
-+					struct lxc_handler *handler)
++					struct lxc_handler *handler, struct lxc_epoll_descr *descr)
 +{
 +	struct lxc_cmd_rsp rsp;
 +	struct lxc_cmd_set_terminal_winch_request *data = (struct lxc_cmd_set_terminal_winch_request *)(req->data);
@@ -227,48 +301,36 @@ index b70564f..8fd929d 100644
 +
 +}
 +
+ #endif
+ 
  static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
- 			   struct lxc_handler *handler)
+@@ -1490,6 +1539,7 @@ static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
- {
+ 		[LXC_CMD_GET_INIT_PIDFD]                = lxc_cmd_get_init_pidfd_callback,
-@@ -1146,6 +1189,7 @@ static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
+ #ifdef HAVE_ISULAD
- 		[LXC_CMD_CONSOLE_LOG]         = lxc_cmd_console_log_callback,
+ 		[LXC_CMD_SET_TERMINAL_FIFOS]  			= lxc_cmd_set_terminal_fifos_callback,
- 		[LXC_CMD_SERVE_STATE_CLIENTS] = lxc_cmd_serve_state_clients_callback,
++		[LXC_CMD_SET_TERMINAL_WINCH]  			= lxc_cmd_set_terminal_winch_callback,
- 		[LXC_CMD_SET_TERMINAL_FIFOS]  = lxc_cmd_set_terminal_fifos_callback,
+ #endif
-+		[LXC_CMD_SET_TERMINAL_WINCH]  = lxc_cmd_set_terminal_winch_callback,
  	};
  
- 	if (req->cmd >= LXC_CMD_MAX) {
 diff --git a/src/lxc/commands.h b/src/lxc/commands.h
-index 6b64849..c8cc8cd 100644
+index 95815e6..aa8289d 100644
 --- a/src/lxc/commands.h
 +++ b/src/lxc/commands.h
-@@ -47,6 +47,7 @@ typedef enum {
+@@ -40,6 +40,7 @@ typedef enum {
- 	LXC_CMD_CONSOLE_LOG,
+ 	LXC_CMD_GET_INIT_PIDFD,
- 	LXC_CMD_SERVE_STATE_CLIENTS,
+ #ifdef HAVE_ISULAD
  	LXC_CMD_SET_TERMINAL_FIFOS,
 +	LXC_CMD_SET_TERMINAL_WINCH,
+ #endif
  	LXC_CMD_MAX,
  } lxc_cmd_t;
- 
+@@ -136,6 +137,7 @@ extern int lxc_cmd_get_cgroup2_fd(const char *name, const char *lxcpath);
-@@ -80,6 +81,11 @@ struct lxc_cmd_console_log {
+ #ifdef HAVE_ISULAD
- 
- };
- 
-+struct lxc_cmd_set_terminal_winch_request {
-+	unsigned int height;
-+	unsigned int width;
-+};
-+
- extern int lxc_cmd_terminal_winch(const char *name, const char *lxcpath);
- extern int lxc_cmd_console(const char *name, int *ttynum, int *fd,
- 			   const char *lxcpath);
-@@ -129,4 +135,7 @@ extern int lxc_cmd_console_log(const char *name, const char *lxcpath,
  extern int lxc_cmd_set_terminal_fifos(const char *name, const char *lxcpath,
  				const char *in_fifo, const char *out_fifo, const char *err_fifo);
- 
 +extern int lxc_cmd_set_terminal_winch(const char *name, const char *lxcpath, unsigned int height, unsigned int width);
-+
+ #endif
-+
+ 
  #endif /* __commands_h */
 diff --git a/src/lxc/exec_commands.c b/src/lxc/exec_commands.c
 new file mode 100644
@@ -772,87 +834,25 @@ index 0000000..2581ee9
 +
 +#endif /* __exec_commands_h */
 diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index 9f9cbfc..7ef57f0 100644
+index d0e6e2b..75c1bbc 100644
 --- a/src/lxc/lxccontainer.c
 +++ b/src/lxc/lxccontainer.c
-@@ -47,6 +47,7 @@
+@@ -62,6 +62,10 @@
- #include "cgroup.h"
+ #include "utils.h"
- #include "commands.h"
+ #include "version.h"
- #include "commands_utils.h"
+ 
++#ifdef HAVE_ISULAD
 +#include "exec_commands.h"
- #include "conf.h"
++#endif
- #include "config.h"
++
- #include "confile.h"
+ #if HAVE_OPENSSL
-@@ -4214,7 +4215,7 @@ static bool do_lxcapi_rename(struct lxc_container *c, const char *newname)
+ #include <openssl/evp.h>
- 
+ #endif
- WRAP_API_1(bool, lxcapi_rename, const char *)
+@@ -5444,6 +5448,50 @@ static bool do_lxcapi_add_terminal_fifo(struct lxc_container *c, const char *in_
- 
--static int lxcapi_attach(struct lxc_container *c, lxc_attach_exec_t exec_function, void *exec_payload, lxc_attach_options_t *options, pid_t *attached_process)
-+static int lxcapi_attach(struct lxc_container *c, const char *suffix, lxc_attach_exec_t exec_function, void *exec_payload, lxc_attach_options_t *options, pid_t *attached_process)
- {
- 	int ret;
- 
-@@ -4223,12 +4224,12 @@ static int lxcapi_attach(struct lxc_container *c, lxc_attach_exec_t exec_functio
- 
- 	current_config = c->lxc_conf;
- 
--	ret = lxc_attach(c->name, c->config_path, exec_function, exec_payload, options, attached_process, &c->lxc_conf->errmsg);
-+	ret = lxc_attach(c->name, c->config_path, suffix, exec_function, exec_payload, options, attached_process, &c->lxc_conf->errmsg);
- 	current_config = NULL;
- 	return ret;
  }
  
--static int do_lxcapi_attach_run_wait(struct lxc_container *c, lxc_attach_options_t *options, const char *program, const char * const argv[])
+ WRAP_API_3(bool, lxcapi_add_terminal_fifo, const char *, const char *, const char *)
-+static int do_lxcapi_attach_run_wait(struct lxc_container *c, const char *suffix, lxc_attach_options_t *options, const char *program, const char * const argv[])
++
- {
- 	lxc_attach_command_t command;
- 	pid_t pid;
-@@ -4240,7 +4241,7 @@ static int do_lxcapi_attach_run_wait(struct lxc_container *c, lxc_attach_options
- 	command.program = (char*)program;
- 	command.argv = (char**)argv;
- 
--	r = lxc_attach(c->name, c->config_path, lxc_attach_run_command, &command, options, &pid, NULL);
-+	r = lxc_attach(c->name, c->config_path, suffix, lxc_attach_run_command, &command, options, &pid, NULL);
- 	if (r < 0) {
- 		ERROR("ups");
- 		return r;
-@@ -4249,12 +4250,12 @@ static int do_lxcapi_attach_run_wait(struct lxc_container *c, lxc_attach_options
- 	return lxc_wait_for_pid_status(pid);
- }
- 
--static int lxcapi_attach_run_wait(struct lxc_container *c, lxc_attach_options_t *options, const char *program, const char * const argv[])
-+static int lxcapi_attach_run_wait(struct lxc_container *c, const char *suffix, lxc_attach_options_t *options, const char *program, const char * const argv[])
- {
- 	int ret;
- 
- 	current_config = c ? c->lxc_conf : NULL;
--	ret = do_lxcapi_attach_run_wait(c, options, program, argv);
-+	ret = do_lxcapi_attach_run_wait(c, suffix, options, program, argv);
- 	current_config = NULL;
- 
- 	return ret;
-@@ -5109,7 +5110,7 @@ static bool do_lxcapi_restore(struct lxc_container *c, char *directory, bool ver
- 
- WRAP_API_2(bool, lxcapi_restore, char *, bool)
- 
--static int lxcapi_attach_run_waitl(struct lxc_container *c, lxc_attach_options_t *options, const char *program, const char *arg, ...)
-+static int lxcapi_attach_run_waitl(struct lxc_container *c, const char *suffix, lxc_attach_options_t *options, const char *program, const char *arg, ...)
- {
- 	va_list ap;
- 	const char **argv;
-@@ -5131,7 +5132,7 @@ static int lxcapi_attach_run_waitl(struct lxc_container *c, lxc_attach_options_t
- 	}
- 	argv[0] = arg;
- 
--	ret = do_lxcapi_attach_run_wait(c, options, program, (const char * const *)argv);
-+	ret = do_lxcapi_attach_run_wait(c, suffix, options, program, (const char * const *)argv);
- 	free((void*)argv);
- 
- out:
-@@ -5230,6 +5231,51 @@ static bool do_lxcapi_clean_container_resource(struct lxc_container *c, pid_t pi
- 
- WRAP_API_1(bool, lxcapi_clean_container_resource, pid_t)
- 
 +static bool do_lxcapi_set_terminal_winch(struct lxc_container *c, unsigned int height, unsigned int width)
 +{
 +	bool ret = true;
@@ -896,75 +896,27 @@ index 9f9cbfc..7ef57f0 100644
 +}
 +
 +WRAP_API_3(bool, lxcapi_set_exec_terminal_winch, const char *, unsigned int, unsigned int)
-+
+ #endif
-+
+ 
- /* isulad get coantainer pids */
+ struct lxc_container *lxc_container_new(const char *name, const char *configpath)
- static bool do_lxcapi_get_container_pids(struct lxc_container *c, pid_t **pids,size_t *pids_len)
+@@ -5591,6 +5639,8 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
- {
+ 	c->set_container_info_file = lxcapi_set_container_info_file;
-@@ -5484,6 +5530,8 @@ static struct lxc_container *do_lxc_container_new(const char *name, const char *
+ 	c->set_terminal_init_fifos = lxcapi_set_terminal_default_fifos;
- 	c->clean_container_resource = lxcapi_clean_container_resource;
  	c->add_terminal_fifos = lxcapi_add_terminal_fifo;
- 	c->get_container_pids = lxcapi_get_container_pids;
 +	c->set_terminal_winch = lxcapi_set_terminal_winch;
 +	c->set_exec_terminal_winch = lxcapi_set_exec_terminal_winch;
- 	/* isulad add end */
+ #endif
  	return c;
  
-diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
-index 608f815..976acf4 100644
---- a/src/lxc/lxccontainer.h
-+++ b/src/lxc/lxccontainer.h
-@@ -674,7 +674,7 @@ struct lxc_container {
- 	 *
- 	 * \return \c 0 on success, \c -1 on error.
- 	 */
--	int (*attach)(struct lxc_container *c, lxc_attach_exec_t exec_function,
-+	int (*attach)(struct lxc_container *c, const char *suffix, lxc_attach_exec_t exec_function,
- 			void *exec_payload, lxc_attach_options_t *options, pid_t *attached_process);
- 
- 	/*!
-@@ -688,7 +688,7 @@ struct lxc_container {
- 	 * \return \c waitpid(2) status of exited process that ran \p
- 	 * program, or \c -1 on error.
- 	 */
--	int (*attach_run_wait)(struct lxc_container *c, lxc_attach_options_t *options, const char *program, const char * const argv[]);
-+	int (*attach_run_wait)(struct lxc_container *c, const char *suffix, lxc_attach_options_t *options, const char *program, const char * const argv[]);
- 
- 	/*!
- 	 * \brief Run a program inside a container and wait for it to exit (list variant).
-@@ -701,7 +701,7 @@ struct lxc_container {
- 	 * \return \c waitpid(2) status of exited process that ran \p
- 	 * program, or \c -1 on error.
- 	 */
--	int (*attach_run_waitl)(struct lxc_container *c, lxc_attach_options_t *options, const char *program, const char *arg, ...);
-+	int (*attach_run_waitl)(struct lxc_container *c, const char *suffix, lxc_attach_options_t *options, const char *program, const char *arg, ...);
- 
- 	/*!
- 	 * \brief Create a container snapshot.
-@@ -958,6 +958,15 @@ struct lxc_container {
- 	 * \return \c true on success, else \c false.
- 	 */
- 	bool (*get_container_pids)(struct lxc_container *c,pid_t **pids,size_t *pids_len);
-+	/*! isulad add
-+	 * \brief An API call to set terminal winch
-+	 *
-+	 * \param c Container.
-+	 * \return \c true on success, else \c false.
-+	 */
-+	bool (*set_terminal_winch)(struct lxc_container *c, unsigned int height, unsigned int width);
-+
-+	bool (*set_exec_terminal_winch)(struct lxc_container *c, const char *suffix, unsigned int height, unsigned int width);
- };
- 
- /*!
 diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
-index bccc23d..535f374 100644
+index 1f46d49..39a6718 100644
 --- a/src/lxc/terminal.c
 +++ b/src/lxc/terminal.c
-@@ -69,6 +69,32 @@ __attribute__((constructor)) void lxc_terminal_init_global(void)
+@@ -187,6 +187,33 @@ static int lxc_terminal_truncate_log_file(struct lxc_terminal *terminal)
- 	lxc_list_init(&lxc_ttys);
  }
  
+ #ifdef HAVE_ISULAD
++
 +int lxc_set_terminal_winsz(struct lxc_terminal *terminal, unsigned int height, unsigned int width)
 +{
 +	int ret = 0;
@@ -991,143 +943,21 @@ index bccc23d..535f374 100644
 +	return ret;
 +}
 +
- void lxc_terminal_winsz(int srcfd, int dstfd)
+ /*
- {
+  * isulad: support mult-logfiles
- 	int ret;
+  * */
 diff --git a/src/lxc/terminal.h b/src/lxc/terminal.h
-index 9bb341f..4b5c70e 100644
+index b4160b3..f49142d 100644
 --- a/src/lxc/terminal.h
 +++ b/src/lxc/terminal.h
-@@ -313,5 +313,6 @@ extern int lxc_terminal_map_ids(struct lxc_conf *c,
+@@ -276,6 +276,7 @@ extern int lxc_terminal_map_ids(struct lxc_conf *c,
- /* isulad: if fd == -1, means delete all the fifos*/
+ 
- int lxc_terminal_delete_fifo(int fd, struct lxc_list *list);
+ #ifdef HAVE_ISULAD
  int lxc_terminal_add_fifos(struct lxc_conf *conf, const char *fifonames);
 +int lxc_set_terminal_winsz(struct lxc_terminal *terminal, unsigned int height, unsigned int width);
+ #endif
  
  #endif /* __LXC_TERMINAL_H */
-diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
-index de02aeb..cb7f776 100644
---- a/src/lxc/tools/arguments.h
-+++ b/src/lxc/tools/arguments.h
-@@ -71,6 +71,7 @@ struct lxc_arguments {
- 
- 	/* for lxc-attach */
- 	int64_t attach_timeout;
-+	const char *suffix; /* isulad add, suffix used for connect with parent of execed process*/
- 
- 	/* for lxc-console */
- 	unsigned int ttynum;
-@@ -191,7 +192,7 @@ struct lxc_arguments {
- #define OPT_DISABLE_PTY OPT_USAGE - 13
- #define OPT_OPEN_STDIN OPT_USAGE - 14
- #define OPT_ATTACH_TIMEOUT OPT_USAGE - 15
--
-+#define OPT_ATTACH_SUFFIX OPT_USAGE - 16
- /* isulad add end*/
- 
- extern int lxc_arguments_parse(struct lxc_arguments *args, int argc,
-diff --git a/src/lxc/tools/lxc_attach.c b/src/lxc/tools/lxc_attach.c
-index 0d40155..56684e7 100644
---- a/src/lxc/tools/lxc_attach.c
-+++ b/src/lxc/tools/lxc_attach.c
-@@ -82,6 +82,7 @@ static const struct option my_longopts[] = {
- 	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
- 	{"err-fifo", required_argument, 0, OPT_STDERR_FIFO},
- 	{"timeout", required_argument, 0, OPT_ATTACH_TIMEOUT},
-+	{"suffix", required_argument, 0, OPT_ATTACH_SUFFIX},
- 	LXC_COMMON_OPTIONS
- };
- 
-@@ -287,6 +288,9 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
- 		}
- 		args->attach_timeout = (unsigned int)atoll(arg);
- 		break;
-+	case OPT_ATTACH_SUFFIX:
-+		args->suffix = arg;
-+		break;
- 	}
- 
- 	return 0;
-@@ -351,7 +355,7 @@ static int lxc_attach_create_log_file(const char *log_file)
- }
- 
- /*isulad: attach with terminal*/
--static int do_attach_foreground(struct lxc_container *c, lxc_attach_command_t *command,
-+static int do_attach_foreground(struct lxc_container *c, const char *suffix, lxc_attach_command_t *command,
- 				lxc_attach_options_t *attach_options,
- 				char **errmsg)
- {
-@@ -361,9 +365,9 @@ static int do_attach_foreground(struct lxc_container *c, lxc_attach_command_t *c
- 	int signal;
- 
- 	if (command->program)
--		ret = c->attach(c, lxc_attach_run_command, command, attach_options, &pid);
-+		ret = c->attach(c, suffix, lxc_attach_run_command, command, attach_options, &pid);
- 	else
--		ret = c->attach(c, lxc_attach_run_shell, NULL, attach_options, &pid);
-+		ret = c->attach(c, suffix, lxc_attach_run_shell, NULL, attach_options, &pid);
- 	if (ret < 0)
- 		goto out;
- 
-@@ -399,7 +403,7 @@ static void close_msg_pipe(int *errpipe)
- }
- 
- /*isulad: attach without terminal in background */
--static int do_attach_background(struct lxc_container *c, lxc_attach_command_t *command,
-+static int do_attach_background(struct lxc_container *c, const char *suffix, lxc_attach_command_t *command,
- 				lxc_attach_options_t *attach_options,
- 				char **errmsg)
- {
-@@ -458,9 +462,9 @@ static int do_attach_background(struct lxc_container *c, lxc_attach_command_t *c
- 	setsid();
- 
- 	if (command->program)
--		ret = c->attach(c, lxc_attach_run_command, command, attach_options, &pid);
-+		ret = c->attach(c, suffix, lxc_attach_run_command, command, attach_options, &pid);
- 	else
--		ret = c->attach(c, lxc_attach_run_shell, NULL, attach_options, &pid);
-+		ret = c->attach(c, suffix, lxc_attach_run_shell, NULL, attach_options, &pid);
- 	if (ret < 0) {
- 		if (c->lxc_conf->errmsg)
- 			lxc_write_error_message(msgpipe[1], "%s", c->lxc_conf->errmsg);
-@@ -580,9 +584,9 @@ int main(int argc, char *argv[])
- 
- 	/* isulad: add do attach background */
- 	if (attach_options.attach_flags & LXC_ATTACH_TERMINAL)
--		wexit = do_attach_foreground(c, &command, &attach_options, &errmsg);
-+		wexit = do_attach_foreground(c, my_args.suffix, &command, &attach_options, &errmsg);
- 	else
--		wexit = do_attach_background(c, &command, &attach_options, &errmsg);
-+		wexit = do_attach_background(c, my_args.suffix, &command, &attach_options, &errmsg);
- 
- 	if (errmsg) {
- 		fprintf(stderr, "%s:%s:%s:%d starting container process caused \"%s\"", c->name,
-diff --git a/src/lxc/tools/lxc_copy.c b/src/lxc/tools/lxc_copy.c
-index 954f1dd..b4e2b8f 100644
---- a/src/lxc/tools/lxc_copy.c
-+++ b/src/lxc/tools/lxc_copy.c
-@@ -467,7 +467,7 @@ static int do_clone_ephemeral(struct lxc_container *c,
- 		goto destroy_and_put;
- 
- 	if (arg->daemonize && arg->argc) {
--		ret = clone->attach_run_wait(clone, &attach_options, arg->argv[0], (const char *const *)arg->argv);
-+		ret = clone->attach_run_wait(clone, NULL, &attach_options, arg->argv[0], (const char *const *)arg->argv);
- 		if (ret < 0)
- 			goto destroy_and_put;
- 		clone->shutdown(clone, -1);
-diff --git a/src/lxc/tools/lxc_ls.c b/src/lxc/tools/lxc_ls.c
-index e261c7b..9fd1bf3 100644
---- a/src/lxc/tools/lxc_ls.c
-+++ b/src/lxc/tools/lxc_ls.c
-@@ -544,7 +544,7 @@ static int ls_get(struct ls **m, size_t *size, const struct lxc_arguments *args,
- 
- 			/* fork(): Attach to the namespace of the container and
- 			 * run ls_get() in it which is called in ls_get_wrapper(). */
--			check = c->attach(c, ls_get_wrapper, &wargs, &aopt, &out);
-+			check = c->attach(c, NULL, ls_get_wrapper, &wargs, &aopt, &out);
- 			/* close the socket */
- 			close(wargs.pipefd[1]);
- 
 -- 
 1.8.3.1
 

0020-confile-decode-escape-charactors-in-config.patch

@@ -1,22 +1,22 @@
-From a6fd611c354ba62320661ad4eef4dd822423fcb6 Mon Sep 17 00:00:00 2001
+From 200cc2a1e95c0c0f17cf14f16d8ceb60ca3628c0 Mon Sep 17 00:00:00 2001
-From: gaohuatao <gaohuatao@huawei.com>
+From: LiFeng <lifeng68@huawei.com>
-Date: Wed, 1 Apr 2020 06:53:27 -0400
+Date: Tue, 14 Apr 2020 16:16:14 +0800
-Subject: [PATCH 141/142] lxc: decode some escape charactors of lxc config file
+Subject: [PATCH 20/49] confile: decode escape charactors in config
- string
 
-Signed-off-by: gaohuatao <gaohuatao@huawei.com>
+Signed-off-by: LiFeng <lifeng68@huawei.com>
 ---
- src/lxc/confile.c | 55 +++++++++++++++++++++++++++++++++++++++++++++--
+ src/lxc/confile.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
- 1 file changed, 53 insertions(+), 2 deletions(-)
+ 1 file changed, 62 insertions(+)
 
 diff --git a/src/lxc/confile.c b/src/lxc/confile.c
-index 3eaae4a9..747ccec7 100644
+index bf0fdf0..a28c5da 100644
 --- a/src/lxc/confile.c
 +++ b/src/lxc/confile.c
-@@ -2443,6 +2443,52 @@ struct parse_line_conf {
+@@ -2732,6 +2732,54 @@ struct parse_line_conf {
  	bool from_include;
  };
  
++#ifdef HAVE_ISULAD
 +// escape_string_decode compress some escape characters
 +static char *escape_string_decode(const char *src)
 +{
@@ -24,7 +24,7 @@ index 3eaae4a9..747ccec7 100644
 +	size_t dst_end = 0;
 +	size_t len = 0;
 +	char *dst = NULL;
-+	
++
 +	if (src == NULL) {
 +		return NULL;
 +	}
@@ -33,13 +33,13 @@ index 3eaae4a9..747ccec7 100644
 +	if (len == 0) {
 +		return NULL;
 +	}
-+	
++
 +	dst = calloc(1, len + 1);
 +	if (dst == NULL) {
 +		ERROR("Out of memory");
 +		return NULL;
 +	}
-+	
++
 +	while(src_end < len) {
 +		if (src[src_end] == '\\') {
 +			switch (src[++src_end])
@@ -54,44 +54,51 @@ index 3eaae4a9..747ccec7 100644
 +				default: dst[dst_end++] = '\\'; dst[dst_end] = src[src_end]; break;
 +			}
 +		} else {
-+			dst[dst_end] = src[src_end];	
++			dst[dst_end] = src[src_end];
 +		}
 +		dst_end++;
 +		src_end++;
 +	}
-+	
++
 +	return dst;
 +}
++#endif
 +
  static int parse_line(char *buffer, void *data)
  {
  	char *dot, *key, *line, *linep, *value;
-@@ -2451,6 +2497,7 @@ static int parse_line(char *buffer, void *data)
+@@ -2740,6 +2788,9 @@ static int parse_line(char *buffer, void *data)
  	int ret = 0;
  	char *dup = buffer;
  	struct parse_line_conf *plc = data;
++#ifdef HAVE_ISULAD
 +	char *value_decode = NULL;
++#endif
  
  	/* If there are newlines in the config file we should keep them. */
  	empty_line = lxc_is_line_empty(dup);
-@@ -2517,11 +2564,15 @@ static int parse_line(char *buffer, void *data)
+@@ -2806,10 +2857,21 @@ static int parse_line(char *buffer, void *data)
  		goto on_error;
  	}
  
--	ret = config->set(key, value, plc->conf, NULL);
++#ifdef HAVE_ISULAD
 +	value_decode = escape_string_decode(value);
 +	if (value_decode == NULL) {
 +		ERROR("Value %s decode failed", value);
 +	}
 +	ret = config->set(key, value_decode ? value_decode: value, plc->conf, NULL);
++#else
+ 	ret = config->set(key, value, plc->conf, NULL);
++#endif
  
  on_error:
  	free(linep);
--
++#ifdef HAVE_ISULAD
 +	free(value_decode);
++#endif
+ 
  	return ret;
  }
- 
 -- 
-2.19.1
+1.8.3.1
 

0020-fix-log-error-when-symlink-subcgroup.patch

@@ -1,44 +0,0 @@
-From a4a3b0c8f6a33676c62729ab7fc1e7d6f54536b8 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Mon, 14 Jan 2019 15:33:12 +0800
-Subject: [PATCH 020/140] fix log error when symlink subcgroup
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/cgroups/cgfsng.c | 10 +++++-----
- 1 file changed, 5 insertions(+), 5 deletions(-)
-
-diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index 705985f..7f2a200 100644
---- a/src/lxc/cgroups/cgfsng.c
-+++ b/src/lxc/cgroups/cgfsng.c
-@@ -1736,21 +1736,21 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
- 		char **mc;
- 		for (mc = merged; *mc; mc++) {
- 			char *token;
--			char *merge = must_copy_string(*mc);
--			lxc_iterate_parts(token, merge, ",") {
-+			char *copy = must_copy_string(*mc);
-+			lxc_iterate_parts(token, copy, ",") {
- 				int mret;
- 				char *link;
- 				link = must_make_path(tmpfspath, token, NULL);
- 				mret = symlink(*mc, link);
- 				if (mret < 0 && errno != EEXIST) {
--					SYSERROR("Failed to create link %s for target %s", link, merge);
--					free(merge);
-+					SYSERROR("Failed to create link %s for target %s", link, *mc);
-+					free(copy);
- 					free(link);
- 					goto on_error;
- 				}
- 				free(link);
- 			}
--			free(merge);
-+			free(copy);
- 		}
- 	}
- 
--- 
-1.8.3.1
-

0021-cgroup-add-retry-for-destory-cgroups.patch

@@ -0,0 +1,387 @@
+From 4814d02fd3d364b599707b3cb298a8cc945033f9 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Tue, 14 Apr 2020 17:07:24 +0800
+Subject: [PATCH 21/49] cgroup: add retry for destory cgroups
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/cgroups/cgfsng.c | 105 +++++++++++++++++++++++++
+ src/lxc/cgroups/cgroup.h |   4 +
+ src/lxc/start.c          | 195 ++++++++++++++++++++++++++++++++++++++++++++++-
+ 3 files changed, 303 insertions(+), 1 deletion(-)
+
+diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
+index 881dd39..00270ab 100644
+--- a/src/lxc/cgroups/cgfsng.c
++++ b/src/lxc/cgroups/cgfsng.c
+@@ -1000,6 +1000,106 @@ static int cgroup_tree_remove_wrapper(void *data)
+ 	return cgroup_tree_remove(arg->hierarchies, arg->container_cgroup);
+ }
+ 
++#ifdef HAVE_ISULAD
++
++static int isulad_cgroup_tree_remove(struct hierarchy **hierarchies,
++			const char *container_cgroup)
++{
++	if (!container_cgroup || !hierarchies)
++		return 0;
++
++	for (int i = 0; hierarchies[i]; i++) {
++		struct hierarchy *h = hierarchies[i];
++		int ret;
++
++		if (!h->container_full_path)
++			continue;
++
++		ret = lxc_rm_rf(h->container_full_path);
++		if (ret < 0) {
++			SYSERROR("Failed to destroy \"%s\"", h->container_full_path);
++			return -1;
++		}
++
++		free_disarm(h->container_full_path);
++	}
++
++	return 0;
++}
++
++static int isulad_cgroup_tree_remove_wrapper(void *data)
++{
++	struct generic_userns_exec_data *arg = data;
++	uid_t nsuid = (arg->conf->root_nsuid_map != NULL) ? 0 : arg->conf->init_uid;
++	gid_t nsgid = (arg->conf->root_nsgid_map != NULL) ? 0 : arg->conf->init_gid;
++	int ret;
++
++	if (!lxc_setgroups(0, NULL) && errno != EPERM)
++		return log_error_errno(-1, errno, "Failed to setgroups(0, NULL)");
++
++	ret = setresgid(nsgid, nsgid, nsgid);
++	if (ret < 0)
++		return log_error_errno(-1, errno, "Failed to setresgid(%d, %d, %d)",
++				       (int)nsgid, (int)nsgid, (int)nsgid);
++
++	ret = setresuid(nsuid, nsuid, nsuid);
++	if (ret < 0)
++		return log_error_errno(-1, errno, "Failed to setresuid(%d, %d, %d)",
++				       (int)nsuid, (int)nsuid, (int)nsuid);
++
++	return isulad_cgroup_tree_remove(arg->hierarchies, arg->container_cgroup);
++}
++
++__cgfsng_ops static bool isulad_cgfsng_payload_destroy(struct cgroup_ops *ops,
++						struct lxc_handler *handler)
++{
++	int ret;
++
++	if (!ops) {
++		ERROR("Called with uninitialized cgroup operations");
++		return false;
++	}
++
++	if (!ops->hierarchies) {
++		return false;
++	}
++
++	if (!handler) {
++		ERROR("Called with uninitialized handler");
++		return false;
++	}
++
++	if (!handler->conf) {
++		ERROR("Called with uninitialized conf");
++		return false;
++	}
++
++#ifdef HAVE_STRUCT_BPF_CGROUP_DEV_CTX
++	ret = bpf_program_cgroup_detach(handler->conf->cgroup2_devices);
++	if (ret < 0)
++		WARN("Failed to detach bpf program from cgroup");
++#endif
++
++	if (handler->conf && !lxc_list_empty(&handler->conf->id_map)) {
++		struct generic_userns_exec_data wrap = {
++			.conf			= handler->conf,
++			.container_cgroup	= ops->container_cgroup,
++			.hierarchies		= ops->hierarchies,
++			.origuid		= 0,
++		};
++		ret = userns_exec_1(handler->conf, isulad_cgroup_tree_remove_wrapper,
++				    &wrap, "cgroup_tree_remove_wrapper");
++	} else {
++		ret = isulad_cgroup_tree_remove(ops->hierarchies, ops->container_cgroup);
++	}
++	if (ret < 0) {
++		SYSWARN("Failed to destroy cgroups");
++		return false;
++	}
++
++	return true;
++}
++#else
+ __cgfsng_ops static void cgfsng_payload_destroy(struct cgroup_ops *ops,
+ 						struct lxc_handler *handler)
+ {
+@@ -1044,6 +1144,7 @@ __cgfsng_ops static void cgfsng_payload_destroy(struct cgroup_ops *ops,
+ 	if (ret < 0)
+ 		SYSWARN("Failed to destroy cgroups");
+ }
++#endif
+ 
+ #ifdef HAVE_ISULAD
+ __cgfsng_ops static void cgfsng_monitor_destroy(struct cgroup_ops *ops,
+@@ -4107,7 +4208,11 @@ struct cgroup_ops *cgfsng_ops_init(struct lxc_conf *conf)
+ 		return NULL;
+ 
+ 	cgfsng_ops->data_init = cgfsng_data_init;
++#ifdef HAVE_ISULAD
++	cgfsng_ops->payload_destroy = isulad_cgfsng_payload_destroy;
++#else
+ 	cgfsng_ops->payload_destroy = cgfsng_payload_destroy;
++#endif
+ 	cgfsng_ops->monitor_destroy = cgfsng_monitor_destroy;
+ 	cgfsng_ops->monitor_create = cgfsng_monitor_create;
+ 	cgfsng_ops->monitor_enter = cgfsng_monitor_enter;
+diff --git a/src/lxc/cgroups/cgroup.h b/src/lxc/cgroups/cgroup.h
+index dcdc76b..a9048c4 100644
+--- a/src/lxc/cgroups/cgroup.h
++++ b/src/lxc/cgroups/cgroup.h
+@@ -144,7 +144,11 @@ struct cgroup_ops {
+ 	cgroup_layout_t cgroup_layout;
+ 
+ 	int (*data_init)(struct cgroup_ops *ops, struct lxc_conf *conf);
++#ifdef HAVE_ISULAD
++	bool (*payload_destroy)(struct cgroup_ops *ops, struct lxc_handler *handler);
++#else
+ 	void (*payload_destroy)(struct cgroup_ops *ops, struct lxc_handler *handler);
++#endif
+ 	void (*monitor_destroy)(struct cgroup_ops *ops, struct lxc_handler *handler);
+ 	bool (*monitor_create)(struct cgroup_ops *ops, struct lxc_handler *handler);
+ 	bool (*monitor_enter)(struct cgroup_ops *ops, struct lxc_handler *handler);
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 800f884..0942c31 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -879,6 +879,170 @@ out_restore_sigmask:
+ 	return -1;
+ }
+ 
++#ifdef HAVE_ISULAD
++void trim_line(char *s)
++{
++	size_t len;
++
++	len = strlen(s);
++	while ((len > 1) && (s[len - 1] == '\n'))
++		s[--len] = '\0';
++}
++
++static int _read_procs_file(const char *path, pid_t **pids, size_t *len)
++{
++	FILE *f;
++	char *line = NULL;
++	size_t sz = 0;
++	pid_t *tmp_pids = NULL;
++
++	f = fopen_cloexec(path, "r");
++	if (!f)
++		return -1;
++
++	while (getline(&line, &sz, f) != -1) {
++		pid_t pid;
++		trim_line(line);
++		pid = (pid_t)atoll(line);
++		if (lxc_mem_realloc((void **)&tmp_pids, sizeof(pid_t) * (*len + 1), *pids, sizeof(pid_t) * (*len)) != 0) {
++			free(*pids);
++			*pids = NULL;
++			ERROR("out of memory");
++			free(line);
++			fclose(f);
++			return -1;
++		}
++		*pids = tmp_pids;
++
++		(*pids)[*len] = pid;
++		(*len)++;
++	}
++
++	free(line);
++	fclose(f);
++	return 0;
++}
++
++static int _recursive_read_cgroup_procs(const char *dirpath, pid_t **pids, size_t *len)
++{
++	struct dirent *direntp = NULL;
++	DIR *dir = NULL;
++	int ret, failed = 0;
++	char pathname[PATH_MAX];
++
++	dir = opendir(dirpath);
++	if (dir == NULL) {
++		WARN("Failed to open \"%s\"", dirpath);
++		return 0;
++	}
++
++	while ((direntp = readdir(dir))) {
++		struct stat mystat;
++		int rc;
++
++		if (!strcmp(direntp->d_name, ".") ||
++		                !strcmp(direntp->d_name, ".."))
++			continue;
++
++		rc = snprintf(pathname, PATH_MAX, "%s/%s", dirpath, direntp->d_name);
++		if (rc < 0 || rc >= PATH_MAX) {
++			failed = 1;
++			continue;
++		}
++
++		if (strcmp(direntp->d_name, "cgroup.procs") == 0) {
++			if (_read_procs_file(pathname, pids, len)) {
++				failed = 1;
++
++			}
++			continue;
++		}
++
++		ret = lstat(pathname, &mystat);
++		if (ret) {
++			failed = 1;
++			continue;
++		}
++
++		if (S_ISDIR(mystat.st_mode)) {
++			if (_recursive_read_cgroup_procs(pathname, pids, len) < 0)
++				failed = 1;
++		}
++	}
++
++	ret = closedir(dir);
++	if (ret) {
++		WARN("Failed to close directory \"%s\"", dirpath);
++		failed = 1;
++	}
++
++	return failed ? -1 : 0;
++}
++
++int get_all_pids(struct cgroup_ops *cg_ops, pid_t **pids, size_t *len)
++{
++	const char *devices_path = NULL;
++
++	devices_path = cg_ops->get_cgroup_full_path(cg_ops, "devices");
++	if (!file_exists(devices_path)) {
++		return 0;
++	}
++
++	return _recursive_read_cgroup_procs(devices_path, pids, len);
++}
++
++static int set_cgroup_freezer(struct cgroup_ops *cg_ops, const char *value)
++{
++	char *fullpath;
++	int ret;
++
++	fullpath = must_make_path(cg_ops->get_cgroup_full_path(cg_ops, "freezer"), "freezer.state", NULL);
++	ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666);
++	free(fullpath);
++	return ret;
++}
++
++/* isulad: kill all process in container cgroup path */
++static void signal_all_processes(struct lxc_handler *handler)
++{
++	int ret;
++	struct cgroup_ops *cg_ops = handler->cgroup_ops;
++	pid_t *pids = NULL;
++	size_t len = 0, i;
++
++	ret = set_cgroup_freezer(cg_ops, "FROZEN");
++	if (ret < 0 && errno != ENOENT) {
++		WARN("cgroup_set frozen failed");
++	}
++
++	ret = get_all_pids(cg_ops, &pids, &len);
++	if (ret < 0) {
++		WARN("failed to get all pids");
++	}
++
++	for (i = 0; i < len; i++) {
++		ret = kill(pids[i], SIGKILL);
++		if (ret < 0 && errno != ESRCH) {
++			WARN("Can not kill process (pid=%d) with SIGKILL for container %s", pids[i], handler->name);
++		}
++	}
++
++	ret = set_cgroup_freezer(cg_ops, "THAWED");
++	if (ret < 0 && errno != ENOENT) {
++		WARN("cgroup_set thawed failed");
++	}
++
++	for (i = 0; i < len; i++) {
++		ret = lxc_wait_for_pid_status(pids[i]);
++		if (ret < 0 && errno != ECHILD) {
++			WARN("Failed to wait pid %d for container %s: %s", pids[i], handler->name, strerror(errno));
++		}
++	}
++
++	free(pids);
++}
++#endif
++
+ void lxc_end(struct lxc_handler *handler)
+ {
+ 	int ret;
+@@ -952,11 +1116,37 @@ void lxc_end(struct lxc_handler *handler)
+ 
+ 	lsm_process_cleanup(handler->conf, handler->lxcpath);
+ 
++#ifdef HAVE_ISULAD
++	// close maincmd fd before destroy cgroup for isulad
++	if (handler->conf->reboot == REBOOT_NONE) {
++		/* For all new state clients simply close the command socket.
++		 * This will inform all state clients that the container is
++		 * STOPPED and also prevents a race between a open()/close() on
++		 * the command socket causing a new process to get ECONNREFUSED
++		 * because we haven't yet closed the command socket.
++		 */
++		close_prot_errno_disarm(handler->conf->maincmd_fd);
++		TRACE("Closed command socket");
++	}
++	int retry_count = 0;
++	int max_retry = 10;
++retry:
++	if (cgroup_ops != NULL && !cgroup_ops->payload_destroy(cgroup_ops, handler)) {
++	        TRACE("Trying to kill all subprocess");
++	        signal_all_processes(handler);
++	        TRACE("Finished kill all subprocess");
++	        if (retry_count < max_retry) {
++				usleep(100 * 1000); /* 100 millisecond */
++				retry_count++;
++				goto retry;
++		}
++		SYSERROR("Failed to destroy cgroup path for container: \"%s\"", handler->name);
++	}
++#else
+ 	if (cgroup_ops) {
+ 		cgroup_ops->payload_destroy(cgroup_ops, handler);
+ 		cgroup_ops->monitor_destroy(cgroup_ops, handler);
+ 	}
+-
+ 	if (handler->conf->reboot == REBOOT_NONE) {
+ 		/* For all new state clients simply close the command socket.
+ 		 * This will inform all state clients that the container is
+@@ -966,7 +1156,10 @@ void lxc_end(struct lxc_handler *handler)
+ 		 */
+ 		close_prot_errno_disarm(handler->conf->maincmd_fd);
+ 		TRACE("Closed command socket");
++	}
++#endif
+ 
++	if (handler->conf->reboot == REBOOT_NONE) {
+ 		/* This function will try to connect to the legacy lxc-monitord
+ 		 * state server and only exists for backwards compatibility.
+ 		 */
+-- 
+1.8.3.1
+

0021-lxc-attch-add-error-message.patch

@@ -1,374 +0,0 @@
-From c26c254376f73d9b43dd1747746a2bc265a85bb3 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Mon, 14 Jan 2019 04:29:40 -0500
-Subject: [PATCH 021/140] lxc-attch: add error message
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/attach.c           | 49 +++++++++++++++++++++++++++++++++++++++++-----
- src/lxc/attach.h           |  4 ++--
- src/lxc/attach_options.h   |  6 +++---
- src/lxc/conf.c             | 27 +++++++++++++++++++++----
- src/lxc/conf.h             |  3 +++
- src/lxc/lxccontainer.c     |  4 ++--
- src/lxc/tools/lxc_attach.c |  8 ++++----
- src/lxc/tools/lxc_ls.c     |  4 ++--
- src/lxc/utils.c            | 21 ++++++++++++++++++++
- src/lxc/utils.h            |  1 +
- 10 files changed, 105 insertions(+), 22 deletions(-)
-
-diff --git a/src/lxc/attach.c b/src/lxc/attach.c
-index 1886bde..570b9d0 100644
---- a/src/lxc/attach.c
-+++ b/src/lxc/attach.c
-@@ -757,10 +757,15 @@ static int attach_child_main(struct attach_clone_payload *payload)
- 	gid_t ns_root_gid = 0;
- 	lxc_attach_options_t* options = payload->options;
- 	struct lxc_proc_context_info* init_ctx = payload->init_ctx;
-+	int msg_fd = -1;
- 	bool needs_lsm = (options->namespaces & CLONE_NEWNS) &&
- 			 (options->attach_flags & LXC_ATTACH_LSM) &&
- 			 init_ctx->lsm_label;
- 
-+	/*isulad: record errpipe fd*/
-+	msg_fd = init_ctx->container->lxc_conf->errpipe[1];
-+	init_ctx->container->lxc_conf->errpipe[1] = -1;
-+
- 	/* A description of the purpose of this functionality is provided in the
- 	 * lxc-attach(1) manual page. We have to remount here and not in the
- 	 * parent process, otherwise /proc may not properly reflect the new pid
-@@ -980,7 +985,7 @@ static int attach_child_main(struct attach_clone_payload *payload)
- 		goto on_error;
- 
- 	/* We're done, so we can now do whatever the user intended us to do. */
--	_exit(payload->exec_function(payload->exec_payload));
-+	_exit(payload->exec_function(payload->exec_payload, msg_fd));
- 
- on_error:
- 	lxc_put_attach_clone_payload(payload);
-@@ -1085,7 +1090,7 @@ static inline void lxc_attach_terminal_close_log(struct lxc_terminal *terminal)
- 
- int lxc_attach(const char *name, const char *lxcpath,
- 	       lxc_attach_exec_t exec_function, void *exec_payload,
--	       lxc_attach_options_t *options, pid_t *attached_process)
-+	       lxc_attach_options_t *options, pid_t *attached_process, char **err_msg)
- {
- 	int i, ret, status;
- 	int ipc_sockets[2];
-@@ -1268,6 +1273,15 @@ int lxc_attach(const char *name, const char *lxcpath,
- 		return -1;
- 	}
- 
-+
-+	/* isulad: pipdfd for get error message of child or grandchild process. */
-+	if (pipe2(conf->errpipe, O_CLOEXEC) != 0) {
-+		SYSERROR("Failed to init errpipe");
-+		free(cwd);
-+		lxc_proc_put_context_info(init_ctx);
-+		return -1;
-+	}
-+
- 	/* Create intermediate subprocess, two reasons:
- 	 *       1. We can't setns() in the child itself, since we want to make
- 	 *          sure we are properly attached to the pidns.
-@@ -1291,6 +1305,11 @@ int lxc_attach(const char *name, const char *lxcpath,
- 		/* close unneeded file descriptors */
- 		close(ipc_sockets[1]);
- 		free(cwd);
-+
-+		/* isulad: close errpipe */
-+		close(conf->errpipe[1]);
-+		conf->errpipe[1] = -1;
-+
- 		lxc_proc_close_ns_fd(init_ctx);
- 		if (options->attach_flags & LXC_ATTACH_TERMINAL)
- 			lxc_attach_terminal_close_slave(&terminal);
-@@ -1399,6 +1418,19 @@ int lxc_attach(const char *name, const char *lxcpath,
- 
- 		*attached_process = attached_pid;
- 
-+		/* isulad: read error msg from pipe */
-+		ssize_t size_read;
-+		char errbuf[BUFSIZ + 1] = {0};
-+
-+		size_read = read(conf->errpipe[0], errbuf, BUFSIZ);
-+		if (size_read > 0) {
-+			if (err_msg)
-+				*err_msg = strdup(errbuf);
-+			if (!(*err_msg))
-+				ERROR("Out of memory");
-+			goto close_mainloop;
-+		}
-+
- 		/* Now shut down communication with child, we're done. */
- 		shutdown(ipc_sockets[0], SHUT_RDWR);
- 		close(ipc_sockets[0]);
-@@ -1439,7 +1471,11 @@ int lxc_attach(const char *name, const char *lxcpath,
- 
- 	/* close unneeded file descriptors */
- 	close(ipc_sockets[0]);
--	ipc_sockets[0] = -EBADF;
-+	ipc_sockets[0] = -EBADF;\
-+
-+	/* isulad: close errpipe */
-+	close(conf->errpipe[0]);
-+	conf->errpipe[0] = -1;
- 
- 	if (options->attach_flags & LXC_ATTACH_TERMINAL) {
- 		lxc_attach_terminal_close_master(&terminal);
-@@ -1539,7 +1575,7 @@ int lxc_attach(const char *name, const char *lxcpath,
- 	_exit(0);
- }
- 
--int lxc_attach_run_command(void *payload)
-+int lxc_attach_run_command(void *payload, int msg_fd)
- {
- 	int ret = -1;
- 	lxc_attach_command_t *cmd = payload;
-@@ -1556,11 +1592,14 @@ int lxc_attach_run_command(void *payload)
- 		}
- 	}
- 
-+	/* isulad: write errorm messages */
-+	lxc_write_error_message(msg_fd, "exec: \"%s\": %s", cmd->program, strerror(errno));
-+
- 	SYSERROR("Failed to exec \"%s\"", cmd->program);
- 	return ret;
- }
- 
--int lxc_attach_run_shell(void* payload)
-+int lxc_attach_run_shell(void* payload, int msg_fd)
- {
- 	uid_t uid;
- 	struct passwd pwent;
-diff --git a/src/lxc/attach.h b/src/lxc/attach.h
-index 4bf9578..e62b98b 100644
---- a/src/lxc/attach.h
-+++ b/src/lxc/attach.h
-@@ -42,7 +42,7 @@ struct lxc_proc_context_info {
- };
- 
- extern int lxc_attach(const char *name, const char *lxcpath,
--		      lxc_attach_exec_t exec_function, void *exec_payload,
--		      lxc_attach_options_t *options, pid_t *attached_process);
-+	       lxc_attach_exec_t exec_function, void *exec_payload,
-+	       lxc_attach_options_t *options, pid_t *attached_process, char **err_msg);
- 
- #endif /* __LXC_ATTACH_H */
-diff --git a/src/lxc/attach_options.h b/src/lxc/attach_options.h
-index 081618c..7b0a8cb 100644
---- a/src/lxc/attach_options.h
-+++ b/src/lxc/attach_options.h
-@@ -71,7 +71,7 @@ enum {
-  *
-  * \return Function should return \c 0 on success, and any other value to denote failure.
-  */
--typedef int (*lxc_attach_exec_t)(void* payload);
-+typedef int (*lxc_attach_exec_t)(void* payload, int msg_fd);
- 
- /*!
-  * LXC attach options for \ref lxc_container \c attach().
-@@ -173,7 +173,7 @@ typedef struct lxc_attach_command_t {
-  *
-  * \return \c -1 on error, exit code of lxc_attach_command_t program on success.
-  */
--extern int lxc_attach_run_command(void* payload);
-+extern int lxc_attach_run_command(void* payload, int msg_fd);
- 
- /*!
-  * \brief Run a shell command in the container.
-@@ -182,7 +182,7 @@ extern int lxc_attach_run_command(void* payload);
-  *
-  * \return Exit code of shell.
-  */
--extern int lxc_attach_run_shell(void* payload);
-+extern int lxc_attach_run_shell(void* payload, int msg_fd);
- 
- #ifdef  __cplusplus
- }
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 537f956..8d8230f 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -2803,10 +2803,6 @@ struct lxc_conf *lxc_conf_init(void)
- 	new->console.slave = -1;
- 	new->console.name[0] = '\0';
- 	memset(&new->console.ringbuf, 0, sizeof(struct lxc_ringbuf));
--	/* isulad init console fifos */
--	new->console.init_fifo[0] = NULL;
--	new->console.init_fifo[1] = NULL;
--	lxc_list_init(&new->console.fifos);
- 	new->maincmd_fd = -1;
- 	new->nbd_idx = -1;
- 	new->rootfs.mount = strdup(default_rootfs_mount);
-@@ -2851,6 +2847,14 @@ struct lxc_conf *lxc_conf_init(void)
- 	lxc_list_init(&new->rootfs.maskedpaths);
- 	lxc_list_init(&new->rootfs.ropaths);
- 	new->exit_fd = -1;
-+	/* isulad init console fifos */
-+	new->console.init_fifo[0] = NULL;
-+	new->console.init_fifo[1] = NULL;
-+	lxc_list_init(&new->console.fifos);
-+
-+	new->errmsg = NULL;
-+	new->errpipe[0] = -1;
-+	new->errpipe[1] = -1;
- 	/* isulad add end */
- 
- 	return new;
-@@ -4280,6 +4284,19 @@ int lxc_clear_rootfs_ro_paths(struct lxc_conf *c)
- 	return 0;
- }
- 
-+/*isulad: close error pipe */
-+void lxc_close_error_pipe(int *errpipe)
-+{
-+	if (errpipe[0] >= 0) {
-+		close(errpipe[0]);
-+		errpipe[0] = -1;
-+	}
-+	if (errpipe[1] >= 0) {
-+		close(errpipe[1]);
-+		errpipe[1] = -1;
-+	}
-+}
-+
- void lxc_conf_free(struct lxc_conf *conf)
- {
- 	if (!conf)
-@@ -4333,6 +4350,8 @@ void lxc_conf_free(struct lxc_conf *conf)
- 	free(conf->container_info_file);
- 	if (conf->exit_fd != -1)
- 		close(conf->exit_fd);
-+	free(conf->errmsg);
-+	lxc_close_error_pipe(conf->errpipe);
- 	/* isulad add end */
- 	free(conf);
- }
-diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index 7927812..db474e1 100644
---- a/src/lxc/conf.h
-+++ b/src/lxc/conf.h
-@@ -413,6 +413,9 @@ struct lxc_conf {
- 	char *container_info_file;
- 
- 	int exit_fd; /* exit fifo fd*/
-+
-+	char *errmsg; /* record error messages */
-+	int errpipe[2];//pipdfd for get error message of child or grandchild process.
- 	/* isulad add end */
- };
- 
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index 8029f33..31f4819 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -4063,7 +4063,7 @@ static int lxcapi_attach(struct lxc_container *c, lxc_attach_exec_t exec_functio
- 
- 	current_config = c->lxc_conf;
- 
--	ret = lxc_attach(c->name, c->config_path, exec_function, exec_payload, options, attached_process);
-+	ret = lxc_attach(c->name, c->config_path, exec_function, exec_payload, options, attached_process, &c->lxc_conf->errmsg);
- 	current_config = NULL;
- 	return ret;
- }
-@@ -4080,7 +4080,7 @@ static int do_lxcapi_attach_run_wait(struct lxc_container *c, lxc_attach_options
- 	command.program = (char*)program;
- 	command.argv = (char**)argv;
- 
--	r = lxc_attach(c->name, c->config_path, lxc_attach_run_command, &command, options, &pid);
-+	r = lxc_attach(c->name, c->config_path, lxc_attach_run_command, &command, options, &pid, NULL);
- 	if (r < 0) {
- 		ERROR("ups");
- 		return r;
-diff --git a/src/lxc/tools/lxc_attach.c b/src/lxc/tools/lxc_attach.c
-index 6d0ffe5..a590fd1 100644
---- a/src/lxc/tools/lxc_attach.c
-+++ b/src/lxc/tools/lxc_attach.c
-@@ -295,8 +295,8 @@ static int do_attach_foreground(struct lxc_container *c, lxc_attach_command_t *c
- 		wexit = ExitSignalOffset + signal;
- 	}
- out:
--	//if (c->lxc_conf->errmsg)
--	//	*errmsg = strdup(c->lxc_conf->errmsg);
-+	if (c->lxc_conf->errmsg)
-+		*errmsg = strdup(c->lxc_conf->errmsg);
- 	return wexit;
- }
- 
-@@ -376,8 +376,8 @@ static int do_attach_background(struct lxc_container *c, lxc_attach_command_t *c
- 	else
- 		ret = c->attach(c, lxc_attach_run_shell, NULL, attach_options, &pid);
- 	if (ret < 0) {
--		//if (c->lxc_conf->errmsg)
--		//	lxc_write_error_message(msgpipe[1], "%s", c->lxc_conf->errmsg);
-+		if (c->lxc_conf->errmsg)
-+			lxc_write_error_message(msgpipe[1], "%s", c->lxc_conf->errmsg);
- 		close(msgpipe[1]);
- 		msgpipe[1] = -1;
- 		ret = -1;
-diff --git a/src/lxc/tools/lxc_ls.c b/src/lxc/tools/lxc_ls.c
-index cb3eb1e..e261c7b 100644
---- a/src/lxc/tools/lxc_ls.c
-+++ b/src/lxc/tools/lxc_ls.c
-@@ -122,7 +122,7 @@ struct wrapargs {
- /*
-  * Takes struct wrapargs as argument.
-  */
--static int ls_get_wrapper(void *wrap);
-+static int ls_get_wrapper(void *wrap, int msgfd);
- 
- /*
-  * To calculate swap usage we should not simply check memory.usage_in_bytes and
-@@ -1023,7 +1023,7 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
- 	return 0;
- }
- 
--static int ls_get_wrapper(void *wrap)
-+static int ls_get_wrapper(void *wrap, int msgfd)
- {
- 	int ret = -1;
- 	size_t len = 0;
-diff --git a/src/lxc/utils.c b/src/lxc/utils.c
-index 74e74a1..8ec9f46 100644
---- a/src/lxc/utils.c
-+++ b/src/lxc/utils.c
-@@ -1971,3 +1971,24 @@ out:
- 	return startat;
- }
- 
-+/* isulad: write error message */
-+void lxc_write_error_message(int errfd, const char *format, ...)
-+{
-+	int ret;
-+	char errbuf[BUFSIZ + 1] = {0};
-+	ssize_t sret;
-+	va_list argp;
-+
-+	if (errfd <= 0)
-+		return;
-+
-+	va_start(argp, format);
-+	ret = vsnprintf(errbuf, BUFSIZ, format, argp);
-+	va_end(argp);
-+	if (ret < 0)
-+		SYSERROR("Failed to call vsnprintf");
-+	sret = write(errfd, errbuf, strlen(errbuf));
-+	if (sret < 0)
-+		SYSERROR("Write errbuf failed");
-+}
-+
-diff --git a/src/lxc/utils.h b/src/lxc/utils.h
-index 364bf67..3d56fd9 100644
---- a/src/lxc/utils.h
-+++ b/src/lxc/utils.h
-@@ -314,5 +314,6 @@ extern int lxc_setup_env_home(uid_t uid);
- 
- extern int fd_nonblock(int fd);
- extern int unsigned long long lxc_get_process_startat(pid_t pid);
-+extern void lxc_write_error_message(int errfd, const char *format, ...);
- 
- #endif /* __LXC_UTILS_H */
--- 
-1.8.3.1
-

0022-support-rootfs-mount-propagation.patch

@@ -1,507 +0,0 @@
-From 26046eb2a18d2a7668dc213c86806efeca7d87a8 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Mon, 14 Jan 2019 17:02:02 +0800
-Subject: [PATCH 022/140] support rootfs mount propagation
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c                  | 230 ++++++++++++++++++++++++++--------------
- src/lxc/conf.h                  |   2 +-
- src/lxc/confile.c               |   8 +-
- src/lxc/criu.c                  |   4 +-
- src/lxc/storage/btrfs.c         |   4 +-
- src/lxc/storage/dir.c           |   9 +-
- src/lxc/storage/overlay.c       |   4 +-
- src/lxc/storage/storage_utils.c |   4 +-
- src/lxc/storage/zfs.c           |   4 +-
- 9 files changed, 165 insertions(+), 104 deletions(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 8d8230f..55d1e45 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -1296,11 +1296,102 @@ static int lxc_fill_autodev(const struct lxc_rootfs *rootfs)
- 	return 0;
- }
- 
-+static void null_endofword(char *word)
-+{
-+	while (*word && *word != ' ' && *word != '\t')
-+		word++;
-+	*word = '\0';
-+}
-+
-+/* skip @nfields spaces in @src */
-+static char *get_field(char *src, int nfields)
-+{
-+	int i;
-+	char *p = src;
-+
-+	for (i = 0; i < nfields; i++) {
-+		while (*p && *p != ' ' && *p != '\t')
-+			p++;
-+
-+		if (!*p)
-+			break;
-+
-+		p++;
-+	}
-+
-+	return p;
-+}
-+
-+static int rootfs_parent_mount_private(char *rootfs)
-+{
-+	/* walk /proc/self/mountinfo and change parent of rootfs to private */
-+	FILE *f = fopen("/proc/self/mountinfo", "r");
-+	char *line = NULL;
-+	char *parent = NULL, *options = NULL;
-+	size_t len = 0;
-+	int ret = 0;
-+
-+	if (!f) {
-+		SYSERROR("Failed to open /proc/self/mountinfo to make parent of rootfs to private");
-+		return -1;
-+	}
-+
-+	while (getline(&line, &len, f) != -1) {
-+		char *target, *opts, *tmptarget;
-+		target = get_field(line, 4);
-+		if (!target)
-+			continue;
-+		tmptarget = strdup(target);
-+		if (!tmptarget)
-+			continue;
-+		null_endofword(tmptarget);
-+		if (!strstr(rootfs, tmptarget)) {
-+			free(tmptarget);
-+			continue;
-+		}
-+		if (!parent || strlen(tmptarget) > strlen(parent)) {
-+			free(parent);
-+			parent = tmptarget;
-+		} else {
-+			free(tmptarget);
-+			continue;
-+		}
-+		opts = get_field(target, 2);
-+		if (!opts)
-+			continue;
-+		null_endofword(opts);
-+		free(options);
-+		options = strdup(opts);
-+		if (!options)
-+			continue;
-+	}
-+
-+	if (!parent || !options) {
-+		ERROR("Could not find parent mount of %s", rootfs);
-+		ret = -1;
-+	} else {
-+		if (strstr(options, "shared")) {
-+			if (mount(NULL, parent, NULL, MS_PRIVATE, NULL)) {
-+				SYSERROR("Failed to make %s private", parent);
-+				ret = -1;
-+			}
-+			DEBUG("Mounted parent %s of rootfs %s to private", parent, rootfs);
-+		}
-+	}
-+	free(parent);
-+	free(options);
-+	fclose(f);
-+	free(line);
-+	return ret;
-+}
-+
- static int lxc_mount_rootfs(struct lxc_conf *conf)
- {
- 	int ret;
- 	struct lxc_storage *bdev;
- 	const struct lxc_rootfs *rootfs = &conf->rootfs;
-+	unsigned long flags, mntflags, pflags;
-+	char *mntdata;
- 
- 	if (!rootfs->path) {
- 		ret = mount("", "/", NULL, MS_SLAVE | MS_REC, 0);
-@@ -1319,6 +1410,44 @@ static int lxc_mount_rootfs(struct lxc_conf *conf)
- 		return -1;
- 	}
- 
-+	// isulad-start: support mount propagations of rootfs
-+	//Get rootfs mnt propagation options, such as slave or shared
-+	if (parse_mntopts(conf->rootfs.options, &mntflags, &pflags, &mntdata) < 0) {
-+		free(mntdata);
-+		return -1;
-+	}
-+	free(mntdata);
-+
-+	flags = MS_SLAVE | MS_REC;
-+	if (pflags)
-+		flags = pflags;
-+
-+	/* Mount propagation inside container can not greater than host.
-+	* So we must change propagation of root according to flags, default is rslave.
-+	* That means shared propagation inside container is disabled by default.
-+	*/
-+	ret = mount("", "/", NULL, flags, NULL);
-+	if (ret < 0) {
-+		SYSERROR("Failed to make / to propagation flags %lu.", flags);
-+		return -1;
-+	}
-+
-+	/* Make parent mount private to make sure following bind mount does
-+	* not propagate in other namespaces. Also it will help with kernel
-+	* check pass in pivot_root. (IS_SHARED(new_mnt->mnt_parent))
-+	*/
-+	ret = rootfs_parent_mount_private(conf->rootfs.mount);
-+	if (ret != 0) {
-+		ERROR("Failed to make parent of rootfs %s to private.", conf->rootfs.mount);
-+		return -1;
-+	}
-+
-+	ret = mount(conf->rootfs.mount, conf->rootfs.mount, "bind", MS_BIND | MS_REC, NULL);
-+	if (ret < 0) {
-+		SYSERROR("Failed to mount rootfs %s", conf->rootfs.mount);
-+		return -1;
-+	}// isulad-end: support mount propagations of rootfs
-+
- 	bdev = storage_init(conf);
- 	if (!bdev) {
- 		ERROR("Failed to mount rootfs \"%s\" onto \"%s\" with options \"%s\"",
-@@ -1960,7 +2089,7 @@ static int lxc_setup_console(const struct lxc_rootfs *rootfs,
- 	return lxc_setup_ttydir_console(rootfs, console, ttydir);
- }
- 
--static void parse_mntopt(char *opt, unsigned long *flags, char **data, size_t size)
-+static void parse_mntopt(char *opt, unsigned long *mflags, unsigned long *pflags, char **data, size_t size)
- {
- 	struct mount_opt *mo;
- 
-@@ -1970,26 +2099,40 @@ static void parse_mntopt(char *opt, unsigned long *flags, char **data, size_t si
- 	for (mo = &mount_opt[0]; mo->name != NULL; mo++) {
- 		if (strncmp(opt, mo->name, strlen(mo->name)) == 0) {
- 			if (mo->clear)
--				*flags &= ~mo->flag;
-+				*mflags &= ~mo->flag;
- 			else
--				*flags |= mo->flag;
-+				*mflags |= mo->flag;
- 			return;
- 		}
- 	}
- 
-+	/* If opt is found in propagation_opt, set or clear flags. */
-+	for (mo = &propagation_opt[0]; mo->name != NULL; mo++) {
-+		if (strncmp(opt, mo->name, strlen(mo->name)) != 0)
-+			continue;
-+
-+		if (mo->clear)
-+			*pflags &= ~mo->flag;
-+		else
-+			*pflags |= mo->flag;
-+
-+		return;
-+	}
-+
- 	if (strlen(*data))
- 		(void)strlcat(*data, ",", size);
- 
- 	(void)strlcat(*data, opt, size);
- }
- 
--int parse_mntopts(const char *mntopts, unsigned long *mntflags, char **mntdata)
-+int parse_mntopts(const char *mntopts, unsigned long *mntflags, unsigned long *pflags, char **mntdata)
- {
- 	char *data, *p, *s;
- 	size_t size;
- 
- 	*mntdata = NULL;
- 	*mntflags = 0L;
-+	*pflags = 0L;
- 
- 	if (!mntopts)
- 		return 0;
-@@ -2007,7 +2150,7 @@ int parse_mntopts(const char *mntopts, unsigned long *mntflags, char **mntdata)
- 	*data = 0;
- 
- 	lxc_iterate_parts(p, s, ",")
--		parse_mntopt(p, mntflags, &data, size);
-+		parse_mntopt(p, mntflags, pflags, &data, size);
- 
- 	if (*data)
- 		*mntdata = data;
-@@ -2018,71 +2161,6 @@ int parse_mntopts(const char *mntopts, unsigned long *mntflags, char **mntdata)
- 	return 0;
- }
- 
--static void parse_propagationopt(char *opt, unsigned long *flags)
--{
--	struct mount_opt *mo;
--
--	/* If opt is found in propagation_opt, set or clear flags. */
--	for (mo = &propagation_opt[0]; mo->name != NULL; mo++) {
--		if (strncmp(opt, mo->name, strlen(mo->name)) != 0)
--			continue;
--
--		if (mo->clear)
--			*flags &= ~mo->flag;
--		else
--			*flags |= mo->flag;
--
--		return;
--	}
--}
--
--int parse_propagationopts(const char *mntopts, unsigned long *pflags)
--{
--	char *p, *s;
--
--	if (!mntopts)
--		return 0;
--
--	s = strdup(mntopts);
--	if (!s) {
--		SYSERROR("Failed to allocate memory");
--		return -ENOMEM;
--	}
--
--	*pflags = 0L;
--	lxc_iterate_parts(p, s, ",")
--		parse_propagationopt(p, pflags);
--	free(s);
--
--	return 0;
--}
--
--static void null_endofword(char *word)
--{
--	while (*word && *word != ' ' && *word != '\t')
--		word++;
--	*word = '\0';
--}
--
--/* skip @nfields spaces in @src */
--static char *get_field(char *src, int nfields)
--{
--	int i;
--	char *p = src;
--
--	for (i = 0; i < nfields; i++) {
--		while (*p && *p != ' ' && *p != '\t')
--			p++;
--
--		if (!*p)
--			break;
--
--		p++;
--	}
--
--	return p;
--}
--
- static int mount_entry(const char *fsname, const char *target,
- 		       const char *fstype, unsigned long mountflags,
- 		       unsigned long pflags, const char *data, bool optional,
-@@ -2289,10 +2367,9 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
- 					 const char *lxc_path)
- {
- 	int ret;
--	unsigned long mntflags;
-+	unsigned long mntflags, pflags;
- 	char *mntdata;
- 	bool dev, optional, relative;
--	unsigned long pflags = 0;
- 	char *rootfs_path = NULL;
- 
- 	optional = hasmntopt(mntent, "optional") != NULL;
-@@ -2312,11 +2389,7 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
- 	}
- 	cull_mntent_opt(mntent);
- 
--	ret = parse_propagationopts(mntent->mnt_opts, &pflags);
--	if (ret < 0)
--		return -1;
--
--	ret = parse_mntopts(mntent->mnt_opts, &mntflags, &mntdata);
-+	ret = parse_mntopts(mntent->mnt_opts, &mntflags, &pflags, &mntdata);
- 	if (ret < 0)
- 		return -1;
- 
-@@ -3544,7 +3617,8 @@ int lxc_setup_rootfs_prepare_root(struct lxc_conf *conf, const char *name,
- 		return 0;
- 	}
- 
--	remount_all_slave();
-+	if (!conf->rootfs.options)
-+		remount_all_slave();
- 
- 	ret = run_lxc_hooks(name, "pre-mount", conf, NULL);
- 	if (ret < 0) {
-diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index db474e1..7393dbf 100644
---- a/src/lxc/conf.h
-+++ b/src/lxc/conf.h
-@@ -463,7 +463,7 @@ extern int userns_exec_1(struct lxc_conf *conf, int (*fn)(void *), void *data,
- extern int userns_exec_full(struct lxc_conf *conf, int (*fn)(void *),
- 			    void *data, const char *fn_name);
- extern int parse_mntopts(const char *mntopts, unsigned long *mntflags,
--			 char **mntdata);
-+			 unsigned long *pflags, char **mntdata);
- extern int parse_propagationopts(const char *mntopts, unsigned long *pflags);
- extern void tmp_proc_unmount(struct lxc_conf *lxc_conf);
- extern void remount_all_slave(void);
-diff --git a/src/lxc/confile.c b/src/lxc/confile.c
-index e199965..db63b55 100644
---- a/src/lxc/confile.c
-+++ b/src/lxc/confile.c
-@@ -2065,16 +2065,10 @@ static int set_config_rootfs_options(const char *key, const char *value,
- 	char *mdata = NULL, *opts = NULL;
- 	struct lxc_rootfs *rootfs = &lxc_conf->rootfs;
- 
--	ret = parse_mntopts(value, &mflags, &mdata);
-+	ret = parse_mntopts(value, &mflags, &pflags, &mdata);
- 	if (ret < 0)
- 		return -EINVAL;
- 
--	ret = parse_propagationopts(value, &pflags);
--	if (ret < 0) {
--		free(mdata);
--		return -EINVAL;
--	}
--
- 	ret = set_config_string_item(&opts, value);
- 	if (ret < 0) {
- 		free(mdata);
-diff --git a/src/lxc/criu.c b/src/lxc/criu.c
-index 31c1940..bb97859 100644
---- a/src/lxc/criu.c
-+++ b/src/lxc/criu.c
-@@ -389,9 +389,9 @@ static void exec_criu(struct cgroup_ops *cgroup_ops, struct lxc_conf *conf,
- 	while (getmntent_r(mnts, &mntent, buf, sizeof(buf))) {
- 		char *fmt, *key, *val, *mntdata;
- 		char arg[2 * PATH_MAX + 2];
--		unsigned long flags;
-+		unsigned long flags, pflags;
- 
--		if (parse_mntopts(mntent.mnt_opts, &flags, &mntdata) < 0)
-+		if (parse_mntopts(mntent.mnt_opts, &flags, &pflags, &mntdata) < 0)
- 			goto err;
- 
- 		free(mntdata);
-diff --git a/src/lxc/storage/btrfs.c b/src/lxc/storage/btrfs.c
-index bbfce61..a02c215 100644
---- a/src/lxc/storage/btrfs.c
-+++ b/src/lxc/storage/btrfs.c
-@@ -212,7 +212,7 @@ bool btrfs_detect(const char *path)
- 
- int btrfs_mount(struct lxc_storage *bdev)
- {
--	unsigned long mntflags;
-+	unsigned long mntflags, pflags;
- 	char *mntdata;
- 	const char *src;
- 	int ret;
-@@ -223,7 +223,7 @@ int btrfs_mount(struct lxc_storage *bdev)
- 	if (!bdev->src || !bdev->dest)
- 		return -22;
- 
--	if (parse_mntopts(bdev->mntopts, &mntflags, &mntdata) < 0) {
-+	if (parse_mntopts(bdev->mntopts, &mntflags, &pflags, &mntdata) < 0) {
- 		free(mntdata);
- 		return -22;
- 	}
-diff --git a/src/lxc/storage/dir.c b/src/lxc/storage/dir.c
-index 79b6469..c7b5ee2 100644
---- a/src/lxc/storage/dir.c
-+++ b/src/lxc/storage/dir.c
-@@ -170,20 +170,13 @@ int dir_mount(struct lxc_storage *bdev)
- 	if (!bdev->src || !bdev->dest)
- 		return -22;
- 
--	ret = parse_mntopts(bdev->mntopts, &mntflags, &mntdata);
-+	ret = parse_mntopts(bdev->mntopts, &mntflags, &pflags, &mntdata);
- 	if (ret < 0) {
- 		ERROR("Failed to parse mount options \"%s\"", bdev->mntopts);
- 		free(mntdata);
- 		return -EINVAL;
- 	}
- 
--	ret = parse_propagationopts(bdev->mntopts, &pflags);
--	if (ret < 0) {
--		ERROR("Failed to parse propagation options \"%s\"", bdev->mntopts);
--		free(mntdata);
--		return -EINVAL;
--	}
--
- 	src = lxc_storage_get_path(bdev->src, bdev->type);
- 
- 	ret = mount(src, bdev->dest, "bind", MS_BIND | MS_REC | mntflags | pflags, mntdata);
-diff --git a/src/lxc/storage/overlay.c b/src/lxc/storage/overlay.c
-index 01546b1..90408a3 100644
---- a/src/lxc/storage/overlay.c
-+++ b/src/lxc/storage/overlay.c
-@@ -495,7 +495,7 @@ int ovl_mount(struct lxc_storage *bdev)
- 	char *options_work, *work, *lastslash;
- 	int lastslashidx;
- 	size_t len, len2;
--	unsigned long mntflags;
-+	unsigned long mntflags, pflags;
- 	char *mntdata;
- 	int ret, ret2;
- 
-@@ -575,7 +575,7 @@ int ovl_mount(struct lxc_storage *bdev)
- 	memcpy(work + lastslashidx, "olwork", STRLITERALLEN("olwork"));
- 	work[lastslashidx + STRLITERALLEN("olwork")] = '\0';
- 
--	ret = parse_mntopts(bdev->mntopts, &mntflags, &mntdata);
-+	ret = parse_mntopts(bdev->mntopts, &mntflags, &pflags, &mntdata);
- 	if (ret < 0) {
- 		ERROR("Failed to parse mount options");
- 		free(mntdata);
-diff --git a/src/lxc/storage/storage_utils.c b/src/lxc/storage/storage_utils.c
-index fa4e727..46e08a3 100644
---- a/src/lxc/storage/storage_utils.c
-+++ b/src/lxc/storage/storage_utils.c
-@@ -396,7 +396,7 @@ int find_fstype_cb(char *buffer, void *data)
- 		const char *options;
- 	} *cbarg = data;
- 
--	unsigned long mntflags;
-+	unsigned long mntflags, pflags;
- 	char *mntdata;
- 	char *fstype;
- 
-@@ -411,7 +411,7 @@ int find_fstype_cb(char *buffer, void *data)
- 	DEBUG("Trying to mount \"%s\"->\"%s\" with FSType \"%s\"", cbarg->rootfs,
- 	      cbarg->target, fstype);
- 
--	if (parse_mntopts(cbarg->options, &mntflags, &mntdata) < 0) {
-+	if (parse_mntopts(cbarg->options, &mntflags, &pflags, &mntdata) < 0) {
- 		free(mntdata);
- 		return 0;
- 	}
-diff --git a/src/lxc/storage/zfs.c b/src/lxc/storage/zfs.c
-index ba104da..752b0c5 100644
---- a/src/lxc/storage/zfs.c
-+++ b/src/lxc/storage/zfs.c
-@@ -184,7 +184,7 @@ int zfs_mount(struct lxc_storage *bdev)
- 	size_t oldlen, newlen, totallen;
- 	char *mntdata, *tmp;
- 	const char *src;
--	unsigned long mntflags;
-+	unsigned long mntflags, pflags;
- 	char cmd_output[PATH_MAX] = {0};
- 
- 	if (strcmp(bdev->type, "zfs"))
-@@ -193,7 +193,7 @@ int zfs_mount(struct lxc_storage *bdev)
- 	if (!bdev->src || !bdev->dest)
- 		return -22;
- 
--	ret = parse_mntopts(bdev->mntopts, &mntflags, &mntdata);
-+	ret = parse_mntopts(bdev->mntopts, &mntflags, &pflags, &mntdata);
- 	if (ret < 0) {
- 		ERROR("Failed to parse mount options");
- 		free(mntdata);
--- 
-1.8.3.1
-

0022-support-terminal-log.patch

@@ -0,0 +1,313 @@
+From cd19f650d43fdae95e7e72bebe207f4ddc9deb85 Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Mon, 13 Apr 2020 19:18:26 +0800
+Subject: [PATCH 22/49] support terminal log
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ src/lxc/commands.c        | 14 ++++++------
+ src/lxc/conf.c            |  6 ++++++
+ src/lxc/confile.c         |  3 +++
+ src/lxc/log.c             | 54 +++++++++++++++++++++++------------------------
+ src/lxc/lxccontainer.c    |  2 ++
+ src/lxc/start.c           | 48 ++++++++++++++++++++---------------------
+ src/lxc/terminal.c        |  1 +
+ src/lxc/tools/lxc_start.c |  6 ++++++
+ 8 files changed, 76 insertions(+), 58 deletions(-)
+
+diff --git a/src/lxc/commands.c b/src/lxc/commands.c
+index 184a219..b21c12b 100644
+--- a/src/lxc/commands.c
++++ b/src/lxc/commands.c
+@@ -80,9 +80,9 @@ static const char *lxc_cmd_str(lxc_cmd_t cmd)
+ 		[LXC_CMD_SERVE_STATE_CLIENTS]		= "serve_state_clients",
+ 		[LXC_CMD_SECCOMP_NOTIFY_ADD_LISTENER]	= "seccomp_notify_add_listener",
+ 		[LXC_CMD_ADD_BPF_DEVICE_CGROUP]		= "add_bpf_device_cgroup",
+-		[LXC_CMD_FREEZE]					= "freeze",
+-		[LXC_CMD_UNFREEZE]					= "unfreeze",
+-		[LXC_CMD_GET_CGROUP2_FD]			= "get_cgroup2_fd",
++		[LXC_CMD_FREEZE]			= "freeze",
++		[LXC_CMD_UNFREEZE]			= "unfreeze",
++		[LXC_CMD_GET_CGROUP2_FD]		= "get_cgroup2_fd",
+ 		[LXC_CMD_GET_INIT_PIDFD]        	= "get_init_pidfd",
+ #ifdef HAVE_ISULAD
+ 		[LXC_CMD_SET_TERMINAL_FIFOS]  		= "set_terminal_fifos",
+@@ -1532,10 +1532,10 @@ static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
+ 		[LXC_CMD_CONSOLE_LOG]                 	= lxc_cmd_console_log_callback,
+ 		[LXC_CMD_SERVE_STATE_CLIENTS]         	= lxc_cmd_serve_state_clients_callback,
+ 		[LXC_CMD_SECCOMP_NOTIFY_ADD_LISTENER] 	= lxc_cmd_seccomp_notify_add_listener_callback,
+-		[LXC_CMD_ADD_BPF_DEVICE_CGROUP]			= lxc_cmd_add_bpf_device_cgroup_callback,
+-		[LXC_CMD_FREEZE]						= lxc_cmd_freeze_callback,
+-		[LXC_CMD_UNFREEZE]						= lxc_cmd_unfreeze_callback,
+-		[LXC_CMD_GET_CGROUP2_FD]				= lxc_cmd_get_cgroup2_fd_callback,
++		[LXC_CMD_ADD_BPF_DEVICE_CGROUP]		= lxc_cmd_add_bpf_device_cgroup_callback,
++		[LXC_CMD_FREEZE]			= lxc_cmd_freeze_callback,
++		[LXC_CMD_UNFREEZE]			= lxc_cmd_unfreeze_callback,
++		[LXC_CMD_GET_CGROUP2_FD]		= lxc_cmd_get_cgroup2_fd_callback,
+ 		[LXC_CMD_GET_INIT_PIDFD]                = lxc_cmd_get_init_pidfd_callback,
+ #ifdef HAVE_ISULAD
+ 		[LXC_CMD_SET_TERMINAL_FIFOS]  			= lxc_cmd_set_terminal_fifos_callback,
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index 2e93227..4088363 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -1767,6 +1767,9 @@ static int lxc_setup_ttydir_console(const struct lxc_rootfs *rootfs,
+ 	if (ret < 0 && errno != EEXIST)
+ 		return log_error_errno(-errno, errno, "Failed to create console");
+ 
++#ifdef HAVE_ISULAD
++	if (console->slave > 0) {
++#endif
+ 	ret = fchmod(console->slave, S_IXUSR | S_IXGRP);
+ 	if (ret < 0)
+ 		return log_error_errno(-errno, errno, "Failed to set mode \"0%o\" to \"%s\"", S_IXUSR | S_IXGRP, console->name);
+@@ -1776,6 +1779,9 @@ static int lxc_setup_ttydir_console(const struct lxc_rootfs *rootfs,
+ 	if (ret < 0)
+ 		return log_error_errno(-1, errno, "Failed to mount \"%s\" on \"%s\"", console->name, lxcpath);
+ 	DEBUG("Mounted \"%s\" onto \"%s\"", console->name, lxcpath);
++#ifdef HAVE_ISULAD
++	}
++#endif
+ 
+ 	/* bind mount '/dev/<ttydir>/console'  to '/dev/console'  */
+ 	ret = safe_mount(lxcpath, path, "none", MS_BIND, 0, rootfs_path);
+diff --git a/src/lxc/confile.c b/src/lxc/confile.c
+index a28c5da..0fcebd4 100644
+--- a/src/lxc/confile.c
++++ b/src/lxc/confile.c
+@@ -2301,11 +2301,14 @@ static int set_config_console_rotate(const char *key, const char *value,
+ 	if (lxc_safe_uint(value, &lxc_conf->console.log_rotate) < 0)
+ 		return -1;
+ 
++#ifndef HAVE_ISULAD
++	/* isulad: support rotate muti-files */
+ 	if (lxc_conf->console.log_rotate > 1) {
+ 		ERROR("The \"lxc.console.rotate\" config key can only be set "
+ 		      "to 0 or 1");
+ 		return -1;
+ 	}
++#endif
+ 
+ 	return 0;
+ }
+diff --git a/src/lxc/log.c b/src/lxc/log.c
+index 9794582..79caa2c 100644
+--- a/src/lxc/log.c
++++ b/src/lxc/log.c
+@@ -60,30 +60,30 @@ static inline const char *isulad_get_fifo_path(const char *file)
+ {
+ #define ISULAD_FIFO_PREFIX "fifo:"
+ 
+-        if (strncmp(file, ISULAD_FIFO_PREFIX, strlen(ISULAD_FIFO_PREFIX)) == 0) {
+-                return (file + strlen(ISULAD_FIFO_PREFIX));
+-        }
+-        return NULL;
++	if (strncmp(file, ISULAD_FIFO_PREFIX, strlen(ISULAD_FIFO_PREFIX)) == 0) {
++		return (file + strlen(ISULAD_FIFO_PREFIX));
++	}
++	return NULL;
+ }
+ 
+ static int isulad_open_fifo(const char *file_path)
+ {
+ #define LOG_FIFO_SIZE (1024 * 1024)
+-        int fd;
++	int fd;
+ 
+-        fd = lxc_unpriv(open(file_path, O_RDWR | O_NONBLOCK | O_CLOEXEC, 0640));
+-        if (fd == -1) {
+-                fprintf(stderr, "Open fifo %s failed: %s\n", file_path, strerror(errno));
+-                return -1;
+-        }
++	fd = lxc_unpriv(open(file_path, O_RDWR | O_NONBLOCK | O_CLOEXEC, 0640));
++	if (fd == -1) {
++		fprintf(stderr, "Open fifo %s failed: %s\n", file_path, strerror(errno));
++		return -1;
++	}
+ 
+-        if (fcntl(fd, F_SETPIPE_SZ, LOG_FIFO_SIZE) == -1) {
+-                printf("Set fifo buffer size failed: %s", strerror(errno));
+-                close(fd);
+-                return -1;
+-        }
++	if (fcntl(fd, F_SETPIPE_SZ, LOG_FIFO_SIZE) == -1) {
++		printf("Set fifo buffer size failed: %s", strerror(errno));
++		close(fd);
++		return -1;
++	}
+ 
+-        return fd;
++	return fd;
+ }
+ #endif
+ 
+@@ -354,10 +354,10 @@ static int log_append_logfile(const struct lxc_log_appender *appender,
+ 
+ 	log_container_name = lxc_log_get_container_name();
+ #ifdef HAVE_ISULAD
+-        /* use isulad log format */
+-        if (log_container_name != NULL && strlen(log_container_name) > 15) {
+-                log_container_name = log_container_name + (strlen(log_container_name) - 15);
+-        }
++	/* use isulad log format */
++	if (log_container_name != NULL && strlen(log_container_name) > 15) {
++		log_container_name = log_container_name + (strlen(log_container_name) - 15);
++	}
+ #endif
+ 
+ 	if (fd_to_use < 0)
+@@ -633,10 +633,10 @@ static int __lxc_log_set_file(const char *fname, int create_dirs)
+ 	}
+ 
+ #ifdef HAVE_ISULAD
+-        fname = isulad_get_fifo_path(fname);
+-        if (fname == NULL) {
+-                return ret_errno(EINVAL);
+-        }
++	fname = isulad_get_fifo_path(fname);
++	if (fname == NULL) {
++		return ret_errno(EINVAL);
++	}
+ #endif
+ 
+ #if USE_CONFIGPATH_LOGS
+@@ -649,7 +649,7 @@ static int __lxc_log_set_file(const char *fname, int create_dirs)
+ 		return log_error_errno(-errno, errno, "Failed to create dir for log file \"%s\"", fname);
+ 
+ #if HAVE_ISULAD
+-        lxc_log_fd = isulad_open_fifo(fname);
++	lxc_log_fd = isulad_open_fifo(fname);
+ #else
+ 	lxc_log_fd = log_open(fname);
+ #endif
+@@ -749,9 +749,9 @@ int lxc_log_init(struct lxc_log *log)
+ 	if (lxc_log_fd >= 0) {
+ 		lxc_log_category_lxc.appender = &log_appender_logfile;
+ #ifdef HAVE_ISULAD
+-                if (!lxc_quiet_specified && !log->quiet)
++		if (!lxc_quiet_specified && !log->quiet)
+ #endif
+-                lxc_log_category_lxc.appender->next = &log_appender_stderr;
++		lxc_log_category_lxc.appender->next = &log_appender_stderr;
+ 	}
+ 
+ 	return ret;
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index 75c1bbc..821cfa1 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -5641,6 +5641,8 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
+ 	c->add_terminal_fifos = lxcapi_add_terminal_fifo;
+ 	c->set_terminal_winch = lxcapi_set_terminal_winch;
+ 	c->set_exec_terminal_winch = lxcapi_set_exec_terminal_winch;
++	c->want_disable_pty = lxcapi_want_disable_pty;
++	c->want_open_stdin = lxcapi_want_open_stdin;
+ #endif
+ 	return c;
+ 
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 0942c31..f6a96b4 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -1387,24 +1387,9 @@ static int do_start(void *data)
+ 	 */
+ #ifdef HAVE_ISULAD
+ 	if (!handler->disable_pty && handler->daemonize && !handler->conf->autodev) {
+-		char path[PATH_MAX];
+-
+-		ret = snprintf(path, sizeof(path), "%s/dev/null",
+-			       handler->conf->rootfs.mount);
+-		if (ret < 0 || ret >= sizeof(path))
+-			goto out_warn_father;
+-
+-		ret = access(path, F_OK);
+-		if (ret != 0) {
+-			devnull_fd = open_devnull();
+-
+-			if (devnull_fd < 0)
+-				goto out_warn_father;
+-			WARN("Using /dev/null from the host for container init's standard file descriptors. Migration will not work");
+-		}
+-	}
+ #else
+ 	if (handler->daemonize && !handler->conf->autodev) {
++#endif
+ 		char path[PATH_MAX];
+ 
+ 		ret = snprintf(path, sizeof(path), "%s/dev/null",
+@@ -1421,7 +1406,6 @@ static int do_start(void *data)
+ 			WARN("Using /dev/null from the host for container init's standard file descriptors. Migration will not work");
+ 		}
+ 	}
+-#endif
+ 
+ 	/* Ask father to setup cgroups and wait for him to finish. */
+ 	ret = lxc_sync_barrier_parent(handler, LXC_SYNC_CGROUP);
+@@ -2059,13 +2043,29 @@ static int lxc_spawn(struct lxc_handler *handler)
+ 	TRACE("Cloned child process %d", handler->pid);
+ 
+ #ifdef HAVE_ISULAD
+-    /* isulad: save pid/ppid info into file*/
+-    if (handler->conf->container_info_file) {
+-        if (lxc_save_container_info(handler->conf->container_info_file, handler->pid)) {
+-            ERROR("Failed to save cloned container pid");
+-            goto out_delete_net;
+-        }
+-    }
++	/* isulad: close pipe after clone */
++	if (handler->conf->console.pipes[0][0] >= 0) {
++		close(handler->conf->console.pipes[0][0]);
++		handler->conf->console.pipes[0][0] = -1;
++	}
++
++	if (handler->conf->console.pipes[1][1] >= 0) {
++		close(handler->conf->console.pipes[1][1]);
++		handler->conf->console.pipes[1][1] = -1;
++	}
++
++	if (handler->conf->console.pipes[2][1] >= 0) {
++		close(handler->conf->console.pipes[2][1]);
++		handler->conf->console.pipes[2][1] = -1;
++	}
++
++	/* isulad: save pid/ppid info into file*/
++	if (handler->conf->container_info_file) {
++		if (lxc_save_container_info(handler->conf->container_info_file, handler->pid)) {
++			ERROR("Failed to save cloned container pid");
++			goto out_delete_net;
++		}
++	}
+ #endif
+ 
+ 	/* Verify that we can actually make use of pidfds. */
+diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
+index 39a6718..14686fc 100644
+--- a/src/lxc/terminal.c
++++ b/src/lxc/terminal.c
+@@ -2090,3 +2090,4 @@ int lxc_terminal_map_ids(struct lxc_conf *c, struct lxc_terminal *terminal)
+ 
+ 	return 0;
+ }
++
+diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
+index 321c847..72a4494 100644
+--- a/src/lxc/tools/lxc_start.c
++++ b/src/lxc/tools/lxc_start.c
+@@ -53,8 +53,14 @@ static const struct option my_longopts[] = {
+ 	{"share-uts", required_argument, 0, OPT_SHARE_UTS},
+ 	{"share-pid", required_argument, 0, OPT_SHARE_PID},
+ #ifdef HAVE_ISULAD
++	{"in-fifo", required_argument, 0, OPT_INPUT_FIFO},
++	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
++	{"err-fifo", required_argument, 0, OPT_STDERR_FIFO},
+ 	{"container-pidfile", required_argument, 0, OPT_CONTAINER_INFO},
+ 	{"exit-fifo", required_argument, 0, OPT_EXIT_FIFO},
++	{"start-timeout", required_argument, 0, OPT_START_TIMEOUT},
++	{"disable-pty", no_argument, 0, OPT_DISABLE_PTY},
++	{"open-stdin", no_argument, 0, OPT_OPEN_STDIN},
+ #endif
+ 	LXC_COMMON_OPTIONS
+ };
+-- 
+1.8.3.1
+

0023-Supporting-rootfs-mount-propagation.patch

@@ -0,0 +1,933 @@
+From 0f756bece17253aadfe72e8f2eafe8a61d969f87 Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Tue, 14 Apr 2020 04:53:05 -0400
+Subject: [PATCH 23/49] Supporting rootfs mount propagation
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/conf.c                  | 429 +++++++++++++++++++++++++++++++++++-----
+ src/lxc/conf.h                  |  24 ++-
+ src/lxc/confile.c               |  19 ++
+ src/lxc/criu.c                  |   7 +
+ src/lxc/storage/btrfs.c         |  11 ++
+ src/lxc/storage/dir.c           |  38 +++-
+ src/lxc/storage/overlay.c       |   8 +
+ src/lxc/storage/storage_utils.c |  13 +-
+ src/lxc/storage/zfs.c           |   9 +
+ src/lxc/utils.h                 |   4 +
+ 10 files changed, 498 insertions(+), 64 deletions(-)
+
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index 4088363..e0a6f98 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -61,6 +61,8 @@
+ #include "loop.h"
+ #include "utils.h"
+ #include "uuid.h"
++#include "path.h"
++#include "utils.h"
+ 
+ #ifdef MAJOR_IN_MKDEV
+ #include <sys/mkdev.h>
+@@ -1236,12 +1238,106 @@ static int lxc_fill_autodev(const struct lxc_rootfs *rootfs)
+ 	return 0;
+ }
+ 
++static void null_endofword(char *word)
++{
++	while (*word && *word != ' ' && *word != '\t')
++		word++;
++	*word = '\0';
++}
++
++/* skip @nfields spaces in @src */
++static char *get_field(char *src, int nfields)
++{
++	int i;
++	char *p = src;
++
++	for (i = 0; i < nfields; i++) {
++		while (*p && *p != ' ' && *p != '\t')
++			p++;
++
++		if (!*p)
++			break;
++
++		p++;
++	}
++
++	return p;
++}
++
++#ifdef HAVE_ISULAD
++static int rootfs_parent_mount_private(char *rootfs)
++{
++	/* walk /proc/self/mountinfo and change parent of rootfs to private */
++	FILE *f = fopen("/proc/self/mountinfo", "r");
++	char *line = NULL;
++	char *parent = NULL, *options = NULL;
++	size_t len = 0;
++	int ret = 0;
++
++	if (!f) {
++		SYSERROR("Failed to open /proc/self/mountinfo to make parent of rootfs to private");
++		return -1;
++	}
++
++	while (getline(&line, &len, f) != -1) {
++		char *target = NULL;
++		char *opts = NULL;
++		char *tmptarget = NULL;
++		target = get_field(line, 4);
++		if (!target)
++			continue;
++		tmptarget = safe_strdup(target);
++		null_endofword(tmptarget);
++		if (!strstr(rootfs, tmptarget)) {
++			free(tmptarget);
++			continue;
++		}
++		if (!parent || strlen(tmptarget) > strlen(parent)) {
++			free(parent);
++			parent = tmptarget;
++		} else {
++			free(tmptarget);
++			continue;
++		}
++		opts = get_field(target, 2);
++		if (!opts)
++			continue;
++		null_endofword(opts);
++		free(options);
++		options = safe_strdup(opts);
++	}
++
++	if (!parent || !options) {
++		ERROR("Could not find parent mount of %s", rootfs);
++		ret = -1;
++	} else {
++		if (strstr(options, "shared")) {
++			if (mount(NULL, parent, NULL, MS_PRIVATE, NULL)) {
++				SYSERROR("Failed to make %s private", parent);
++				ret = -1;
++			}
++			DEBUG("Mounted parent %s of rootfs %s to private", parent, rootfs);
++		}
++	}
++	free(parent);
++	free(options);
++	fclose(f);
++	free(line);
++	return ret;
++}
++#endif
++
+ static int lxc_mount_rootfs(struct lxc_conf *conf)
+ {
+ 	int ret;
+ 	struct lxc_storage *bdev;
+ 	const struct lxc_rootfs *rootfs = &conf->rootfs;
+ 
++#ifdef HAVE_ISULAD
++	unsigned long flags, mntflags, pflags;
++	char *mntdata = NULL;
++#endif
++
+ 	if (!rootfs->path) {
+ 		ret = mount("", "/", NULL, MS_SLAVE | MS_REC, 0);
+ 		if (ret < 0)
+@@ -1255,6 +1351,44 @@ static int lxc_mount_rootfs(struct lxc_conf *conf)
+ 		return log_error_errno(-1, errno, "Failed to access to \"%s\". Check it is present",
+ 				       rootfs->mount);
+ 
++#ifdef HAVE_ISULAD
++	// Support mount propagations of rootfs
++	// Get rootfs mnt propagation options, such as slave or shared
++	if (parse_mntopts(conf->rootfs.options, &mntflags, &pflags, &mntdata) < 0) {
++		free(mntdata);
++		return -1;
++	}
++	free(mntdata);
++
++	flags = MS_SLAVE | MS_REC;
++	if (pflags)
++		flags = pflags;
++
++	/* Mount propagation inside container can not greater than host.
++	* So we must change propagation of root according to flags, default is rslave.
++	* That means shared propagation inside container is disabled by default.
++	*/
++	ret = mount("", "/", NULL, flags, NULL);
++	if (ret < 0) {
++		return log_error_errno(-1, errno, "Failed to make / to propagation flags %lu.", flags);
++	}
++
++	/* Make parent mount private to make sure following bind mount does
++	* not propagate in other namespaces. Also it will help with kernel
++	* check pass in pivot_root. (IS_SHARED(new_mnt->mnt_parent))
++	*/
++	ret = rootfs_parent_mount_private(conf->rootfs.mount);
++	if (ret != 0) {
++		return log_error(-1, "Failed to make parent of rootfs %s to private.", conf->rootfs.mount);
++	}
++
++	ret = mount(conf->rootfs.mount, conf->rootfs.mount, "bind", MS_BIND | MS_REC, NULL);
++	if (ret < 0) {
++		SYSERROR("Failed to mount rootfs %s", conf->rootfs.mount);
++		return -1;
++	}
++#endif
++
+ 	bdev = storage_init(conf);
+ 	if (!bdev)
+ 		return log_error(-1, "Failed to mount rootfs \"%s\" onto \"%s\" with options \"%s\"",
+@@ -1802,7 +1936,43 @@ static int lxc_setup_console(const struct lxc_rootfs *rootfs,
+ 
+ 	return lxc_setup_ttydir_console(rootfs, console, ttydir);
+ }
++#ifdef HAVE_ISULAD
++static void parse_mntopt(char *opt, unsigned long *mflags, unsigned long *pflags, char **data, size_t size)
++{
++	struct mount_opt *mo;
++
++	/* If opt is found in mount_opt, set or clear flags.
++	 * Otherwise append it to data. */
++
++	for (mo = &mount_opt[0]; mo->name != NULL; mo++) {
++		if (strncmp(opt, mo->name, strlen(mo->name)) == 0) {
++			if (mo->clear)
++				*mflags &= ~mo->flag;
++			else
++				*mflags |= mo->flag;
++			return;
++		}
++	}
++
++	/* If opt is found in propagation_opt, set or clear flags. */
++	for (mo = &propagation_opt[0]; mo->name != NULL; mo++) {
++		if (strncmp(opt, mo->name, strlen(mo->name)) != 0)
++			continue;
++
++		if (mo->clear)
++			*pflags &= ~mo->flag;
++		else
++			*pflags |= mo->flag;
++
++		return;
++	}
+ 
++	if (strlen(*data))
++		(void)strlcat(*data, ",", size);
++
++	(void)strlcat(*data, opt, size);
++}
++#else
+ static int parse_mntopt(char *opt, unsigned long *flags, char **data, size_t size)
+ {
+ 	ssize_t ret;
+@@ -1839,7 +2009,43 @@ static int parse_mntopt(char *opt, unsigned long *flags, char **data, size_t siz
+ 
+ 	return 0;
+ }
++#endif
++
++#ifdef HAVE_ISULAD
++int parse_mntopts(const char *mntopts, unsigned long *mntflags, unsigned long *pflags, char **mntdata)
++{
++	char *data, *p, *s;
++	size_t size;
++
++	*mntdata = NULL;
++	*mntflags = 0L;
++	*pflags = 0L;
++
++	if (!mntopts)
++		return 0;
++
++	s = safe_strdup(mntopts);
+ 
++	size = strlen(s) + 1;
++	data = malloc(size);
++	if (!data) {
++		free(s);
++		return -1;
++	}
++	*data = 0;
++
++	lxc_iterate_parts(p, s, ",")
++	parse_mntopt(p, mntflags, pflags, &data, size);
++
++	if (*data)
++		*mntdata = data;
++	else
++		free(data);
++	free(s);
++
++	return 0;
++}
++#else
+ int parse_mntopts(const char *mntopts, unsigned long *mntflags, char **mntdata)
+ {
+ 	__do_free char *mntopts_new = NULL, *mntopts_dup = NULL;
+@@ -1870,6 +2076,7 @@ int parse_mntopts(const char *mntopts, unsigned long *mntflags, char **mntdata)
+ 
+ 	return 0;
+ }
++#endif
+ 
+ static void parse_propagationopt(char *opt, unsigned long *flags)
+ {
+@@ -1908,32 +2115,6 @@ int parse_propagationopts(const char *mntopts, unsigned long *pflags)
+ 	return 0;
+ }
+ 
+-static void null_endofword(char *word)
+-{
+-	while (*word && *word != ' ' && *word != '\t')
+-		word++;
+-	*word = '\0';
+-}
+-
+-/* skip @nfields spaces in @src */
+-static char *get_field(char *src, int nfields)
+-{
+-	int i;
+-	char *p = src;
+-
+-	for (i = 0; i < nfields; i++) {
+-		while (*p && *p != ' ' && *p != '\t')
+-			p++;
+-
+-		if (!*p)
+-			break;
+-
+-		p++;
+-	}
+-
+-	return p;
+-}
+-
+ static int mount_entry(const char *fsname, const char *target,
+ 		       const char *fstype, unsigned long mountflags,
+ 		       unsigned long pflags, const char *data, bool optional,
+@@ -2186,6 +2367,82 @@ retry:
+ 
+ 	return 0;
+ }
++
++/* isulad: checkMountDestination checks to ensure that the mount destination is not over the top of /proc.
++ * dest is required to be an abs path and have any symlinks resolved before calling this function. */
++static int check_mount_destination(const char *rootfs, const char *dest)
++{
++	const char *invalid_destinations[] = {
++		"/proc",
++		NULL
++	};
++	// White list, it should be sub directories of invalid destinations
++	const char *valid_destinations[] = {
++		// These entries can be bind mounted by files emulated by fuse,
++		// so commands like top, free displays stats in container.
++		"/proc/cpuinfo",
++		"/proc/diskstats",
++		"/proc/meminfo",
++		"/proc/stat",
++		"/proc/swaps",
++		"/proc/uptime",
++		"/proc/net/dev",
++		NULL
++	};
++	const char **valid = NULL;
++	const char **invalid = NULL;
++
++	for(valid = valid_destinations; *valid != NULL; valid++) {
++		char *fullpath = NULL;
++		char *relpath = NULL;
++		const char *parts[3] = {
++			rootfs,
++			*valid,
++			NULL
++		};
++		fullpath = lxc_string_join("/", parts, false);
++		if (!fullpath) {
++			ERROR("Out of memory");
++			return -1;
++		}
++		relpath = path_relative(fullpath, dest);
++		free(fullpath);
++		if (!relpath)
++			return -1;
++		if (!strcmp(relpath, ".")) {
++			free(relpath);
++			return 0;
++		}
++		free(relpath);
++	}
++
++	for(invalid = invalid_destinations; *invalid != NULL; invalid++) {
++		char *fullpath = NULL;
++		char *relpath = NULL;
++		const char *parts[3] = {
++			rootfs,
++			*invalid,
++			NULL
++		};
++		fullpath = lxc_string_join("/", parts, false);
++		if (!fullpath) {
++			ERROR("Out of memory");
++			return -1;
++		}
++		relpath = path_relative(fullpath, dest);
++		free(fullpath);
++		if (!relpath)
++			return -1;
++		if (!strcmp(relpath, ".") || strncmp(relpath, "..", 2)) {
++			ERROR("%s cannot be mounted because it is located inside %s", dest, *invalid);
++			free(relpath);
++			return -1;
++		}
++		free(relpath);
++	}
++
++	return 0;
++}
+ #endif
+ 
+ /* rootfs, lxc_name, and lxc_path can be NULL when the container is created
+@@ -2201,18 +2458,51 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
+ 	char *rootfs_path = NULL;
+ 	int ret;
+ 	bool dev, optional, relative;
+-        const char *dest = path;
++	const char *dest = path;
++
++#ifdef HAVE_ISULAD
++	char *rpath = NULL;
++#endif
+ 
+ 	optional = hasmntopt(mntent, "optional") != NULL;
+ 	dev = hasmntopt(mntent, "dev") != NULL;
+ 	relative = hasmntopt(mntent, "relative") != NULL;
+ 
++#ifdef HAVE_ISULAD
++	// isulad: ensure that the destination of the bind mount is resolved of symlinks at mount time because
++	// any previous mounts can invalidate the next mount's destination.
++	// this can happen when a user specifies mounts within other mounts to cause breakouts or other
++	// evil stuff to try to escape the container's rootfs.
++	if (rootfs_path) {
++		rpath = follow_symlink_in_scope(path, rootfs_path);
++		if (!rpath) {
++			ERROR("Failed to get real path of '%s' in scope '%s'.", path, rootfs_path);
++			lxc_write_error_message(rootfs->errfd, "%s:%d: failed to get real path of '%s' in scope '%s'.",
++			                        __FILE__, __LINE__, path, rootfs_path);
++			return -1;
++		}
++		dest = rpath;
++
++		ret = check_mount_destination(rootfs_path, dest);
++		if (ret) {
++			ERROR("Mount destination is invalid: '%s'", dest);
++			lxc_write_error_message(rootfs->errfd, "%s:%d: mount destination is invalid: '%s'.",
++			                        __FILE__, __LINE__, dest);
++			free(rpath);
++			return -1;
++		}
++	}
++#else
+ 	if (rootfs && rootfs->path)
+ 		rootfs_path = rootfs->mount;
++#endif
+ 
+ 	ret = mount_entry_create_dir_file(mntent, path, rootfs, lxc_name,
+ 					  lxc_path);
+ 	if (ret < 0) {
++#ifdef HAVE_ISULAD
++		free(rpath);
++#endif
+ 		if (optional)
+ 			return 0;
+ 
+@@ -2220,6 +2510,29 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
+ 	}
+ 	cull_mntent_opt(mntent);
+ 
++#ifdef HAVE_ISULAD
++	ret = parse_mntopts(mntent->mnt_opts, &mntflags, &pflags, &mntdata);
++	if (ret < 0) {
++		free(rpath);
++		return -1;
++	}
++
++	// support squashfs
++	if (strcmp(mntent->mnt_type, "squashfs") == 0) {
++		ret = mount_entry_with_loop_dev(mntent->mnt_fsname, dest, mntent->mnt_type,
++						mntent->mnt_opts, rootfs_path);
++	} else {
++		ret = mount_entry(mntent->mnt_fsname, dest, mntent->mnt_type, mntflags,
++					pflags, mntdata, optional, dev, relative, rootfs_path);
++	}
++
++	if (ret < 0) {
++		lxc_write_error_message(rootfs->errfd, "%s:%d: failed to mount %s as type %s.",
++		                        __FILE__, __LINE__, mntent->mnt_fsname, mntent->mnt_type);
++	}
++
++	free(rpath);
++#else
+ 	ret = parse_propagationopts(mntent->mnt_opts, &pflags);
+ 	if (ret < 0)
+ 		return -1;
+@@ -2228,18 +2541,10 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
+ 	if (ret < 0)
+ 		return ret;
+ 
+-#ifdef HAVE_ISULAD
+-	// isulad: support squashfs
+-	if (strcmp(mntent->mnt_type, "squashfs") == 0) {
+-		ret = mount_entry_with_loop_dev(mntent->mnt_fsname, dest, mntent->mnt_type,
+-						mntent->mnt_opts, rootfs_path);
+-        } else {
+-#endif
+-	        ret = mount_entry(mntent->mnt_fsname, dest, mntent->mnt_type, mntflags,
+-			          pflags, mntdata, optional, dev, relative, rootfs_path);
+-#ifdef HAVE_ISULAD
+-        }
++	ret = mount_entry(mntent->mnt_fsname, dest, mntent->mnt_type, mntflags,
++			pflags, mntdata, optional, dev, relative, rootfs_path);
+ #endif
++
+ 	return ret;
+ }
+ 
+@@ -2329,6 +2634,28 @@ static int mount_file_entries(const struct lxc_conf *conf,
+ 	while (getmntent_r(file, &mntent, buf, sizeof(buf))) {
+ 		int ret;
+ 
++#ifdef HAVE_ISULAD
++		//isulad, system contaienr, skip "proc/sys/xxx" path
++		if (conf->systemd != NULL && strcmp(conf->systemd, "true") == 0) {
++			if (strstr(mntent.mnt_dir, "proc/sys") != NULL) {
++				continue;
++			}
++		}
++
++		/* Note: Workaround for volume file path with space*/
++		mntent.mnt_fsname = lxc_string_replace(SPACE_MAGIC_STR, " ", mntent.mnt_fsname);
++		if(!mntent.mnt_fsname) {
++			SYSERROR("memory allocation error");
++			return -1;
++		}
++		mntent.mnt_dir = lxc_string_replace(SPACE_MAGIC_STR, " ", mntent.mnt_dir);
++		if(!mntent.mnt_dir) {
++			SYSERROR("memory allocation error");
++			free(mntent.mnt_fsname);
++			return -1;
++		}
++#endif
++
+ 		if (!rootfs->path)
+ 			ret = mount_entry_on_systemfs(&mntent);
+ 		else if (mntent.mnt_dir[0] != '/')
+@@ -2337,6 +2664,14 @@ static int mount_file_entries(const struct lxc_conf *conf,
+ 		else
+ 			ret = mount_entry_on_absolute_rootfs(&mntent, rootfs,
+ 							     lxc_name, lxc_path);
++
++#ifdef HAVE_ISULAD
++		free(mntent.mnt_fsname);
++		mntent.mnt_fsname = NULL;
++		free(mntent.mnt_dir);
++		mntent.mnt_dir = NULL;
++#endif
++
+ 		if (ret < 0)
+ 			return -1;
+ 	}
+@@ -3391,7 +3726,13 @@ int lxc_setup_rootfs_prepare_root(struct lxc_conf *conf, const char *name,
+ 		return log_trace(0, "Bind mounted container / onto itself");
+ 	}
+ 
++#ifdef HAVE_ISULAD
++	if (!conf->rootfs.options) {
++		remount_all_slave();
++	}
++#else
+ 	remount_all_slave();
++#endif
+ 
+ 	ret = run_lxc_hooks(name, "pre-mount", conf, NULL);
+ 	if (ret < 0)
+@@ -3585,16 +3926,12 @@ reset_umask:
+ static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs)
+ {
+ 	unsigned long mflags, mntflags, pflags;
+-	char *mntdata;
++	char *mntdata = NULL;
+ 
+ 	if(!rootfs || !rootfs->options)
+ 		return 0;
+ 
+-	if (parse_propagationopts(rootfs->options, &pflags) < 0) {
+-		return -1;
+-	}
+-
+-	if (parse_mntopts(rootfs->options, &mntflags, &mntdata) < 0) {
++	if (parse_mntopts(rootfs->options, &mntflags, &pflags, &mntdata) < 0) {
+ 		free(mntdata);
+ 		return -1;
+ 	}
+@@ -3602,9 +3939,9 @@ static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs)
+ 
+ 	if (mntflags & MS_RDONLY) {
+ 		mflags = add_required_remount_flags("/", NULL, MS_BIND | MS_REC | mntflags | pflags | MS_REMOUNT);
+-		DEBUG("remounting /");
++		DEBUG("remounting / as readonly");
+ 		if (mount("/", "/", NULL, mflags, 0) < 0) {
+-			SYSERROR("Failed to remount /");
++			SYSERROR("Failed to make / readonly.");
+ 			return -1;
+ 		}
+ 	}
+diff --git a/src/lxc/conf.h b/src/lxc/conf.h
+index 482fe0d..22c554d 100644
+--- a/src/lxc/conf.h
++++ b/src/lxc/conf.h
+@@ -156,10 +156,12 @@ struct lxc_rootfs {
+ 	bool managed;
+ 
+ #ifdef HAVE_ISULAD
+-    /* isulad: maskedpaths */
+-    struct lxc_list maskedpaths;
+-    /* isulad: ropaths */
+-    struct lxc_list ropaths;
++	/* isulad: maskedpaths */
++	struct lxc_list maskedpaths;
++	/* isulad: ropaths */
++	struct lxc_list ropaths;
++	/* isulad: errfd */
++	int errfd;
+ #endif
+ 
+ };
+@@ -444,6 +446,8 @@ struct lxc_conf {
+ 	int exit_fd; /* exit fifo fd*/
+ 
+ 	char *errmsg; /* record error messages */
++
++	char *systemd; //systemd value
+ #endif
+ 
+ };
+@@ -492,8 +496,6 @@ extern int userns_exec_1(const struct lxc_conf *conf, int (*fn)(void *),
+ 			 void *data, const char *fn_name);
+ extern int userns_exec_full(struct lxc_conf *conf, int (*fn)(void *),
+ 			    void *data, const char *fn_name);
+-extern int parse_mntopts(const char *mntopts, unsigned long *mntflags,
+-			 char **mntdata);
+ extern int parse_propagationopts(const char *mntopts, unsigned long *pflags);
+ extern void tmp_proc_unmount(struct lxc_conf *lxc_conf);
+ extern void remount_all_slave(void);
+@@ -519,6 +521,16 @@ extern int userns_exec_minimal(const struct lxc_conf *conf,
+ 			       int (*fn_parent)(void *), void *fn_parent_data,
+ 			       int (*fn_child)(void *), void *fn_child_data);
+ #ifdef HAVE_ISULAD
++// isulad modify
++extern int parse_mntopts(const char *mntopts, unsigned long *mntflags,
++			unsigned long *pflags, char **mntdata);
++#else
++extern int parse_mntopts(const char *mntopts, unsigned long *mntflags,
++			char **mntdata);
++#endif
++
++#ifdef HAVE_ISULAD
++// isulad add
+ int lxc_clear_init_args(struct lxc_conf *lxc_conf);
+ int lxc_clear_populate_devices(struct lxc_conf *c);
+ int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
+diff --git a/src/lxc/confile.c b/src/lxc/confile.c
+index 0fcebd4..9ba3c7c 100644
+--- a/src/lxc/confile.c
++++ b/src/lxc/confile.c
+@@ -1358,6 +1358,10 @@ static int set_config_environment(const char *key, const char *value,
+ {
+ 	struct lxc_list *list_item = NULL;
+ 
++#ifdef HAVE_ISULAD
++	char *replaced = NULL;
++#endif
++
+ 	if (lxc_config_value_empty(value))
+ 		return lxc_clear_environment(lxc_conf);
+ 
+@@ -1378,7 +1382,16 @@ static int set_config_environment(const char *key, const char *value,
+ 		env_var[1] = env_val;
+ 		list_item->elem = lxc_string_join("=", env_var, false);
+ 	} else {
++#ifdef HAVE_ISULAD
++		/* isulad: recover space replaced by SPACE_MAGIC_STR */
++		replaced = lxc_string_replace(SPACE_MAGIC_STR, " ", value);
++		if(!replaced)
++			goto on_error;
++
++		list_item->elem = replaced;
++#else
+ 		list_item->elem = strdup(value);
++#endif
+ 	}
+ 
+ 	if (!list_item->elem)
+@@ -2594,6 +2607,11 @@ static int set_config_rootfs_options(const char *key, const char *value,
+ 	int ret;
+ 	struct lxc_rootfs *rootfs = &lxc_conf->rootfs;
+ 
++#ifdef HAVE_ISULAD
++	ret = parse_mntopts(value, &mflags, &pflags, &mdata);
++	if (ret < 0)
++		return -EINVAL;
++#else
+ 	ret = parse_mntopts(value, &mflags, &mdata);
+ 	if (ret < 0)
+ 		return -EINVAL;
+@@ -2603,6 +2621,7 @@ static int set_config_rootfs_options(const char *key, const char *value,
+ 		free(mdata);
+ 		return -EINVAL;
+ 	}
++#endif
+ 
+ 	ret = set_config_string_item(&opts, value);
+ 	if (ret < 0) {
+diff --git a/src/lxc/criu.c b/src/lxc/criu.c
+index 1a909bb..14a8aae 100644
+--- a/src/lxc/criu.c
++++ b/src/lxc/criu.c
+@@ -371,8 +371,15 @@ static void exec_criu(struct cgroup_ops *cgroup_ops, struct lxc_conf *conf,
+ 		char *mntdata = NULL;
+ 		char arg[2 * PATH_MAX + 2];
+ 
++#ifdef HAVE_ISULAD
++		unsigned long pflags;
++
++		if (parse_mntopts(mntent.mnt_opts, &flags, &pflags, &mntdata) < 0)
++			goto err;
++#else
+ 		if (parse_mntopts(mntent.mnt_opts, &flags, &mntdata) < 0)
+ 			goto err;
++#endif
+ 
+ 		free(mntdata);
+ 
+diff --git a/src/lxc/storage/btrfs.c b/src/lxc/storage/btrfs.c
+index 92a4a6d..069a9dd 100644
+--- a/src/lxc/storage/btrfs.c
++++ b/src/lxc/storage/btrfs.c
+@@ -197,16 +197,27 @@ int btrfs_mount(struct lxc_storage *bdev)
+ 	const char *src;
+ 	int ret;
+ 
++#ifdef HAVE_ISULAD
++    unsigned long pflags = 0;
++#endif
++
+ 	if (strcmp(bdev->type, "btrfs"))
+ 		return -22;
+ 
+ 	if (!bdev->src || !bdev->dest)
+ 		return -22;
+ 
++#ifdef HAVE_ISULAD
++    if (parse_mntopts(bdev->mntopts, &mntflags, &pflags, &mntdata) < 0) {
++		free(mntdata);
++		return -22;
++	}
++#else
+ 	if (parse_mntopts(bdev->mntopts, &mntflags, &mntdata) < 0) {
+ 		free(mntdata);
+ 		return -22;
+ 	}
++#endif
+ 
+ 	src = lxc_storage_get_path(bdev->src, "btrfs");
+ 
+diff --git a/src/lxc/storage/dir.c b/src/lxc/storage/dir.c
+index b3dbbd0..1dc95f1 100644
+--- a/src/lxc/storage/dir.c
++++ b/src/lxc/storage/dir.c
+@@ -124,14 +124,39 @@ bool dir_detect(const char *path)
+ 	return false;
+ }
+ 
++#ifdef HAVE_ISULAD
+ int dir_mount(struct lxc_storage *bdev)
+ {
+ 	__do_free char *mntdata = NULL;
+-#ifdef HAVE_ISULAD
+ 	unsigned long mntflags = 0, pflags = 0;
++	int ret;
++	const char *src;
++
++	if (strcmp(bdev->type, "dir"))
++		return -22;
++
++	if (!bdev->src || !bdev->dest)
++		return -22;
++
++	ret = parse_mntopts(bdev->mntopts, &mntflags, &pflags, &mntdata);
++	if (ret < 0)
++		return log_error_errno(ret, errno, "Failed to parse mount options \"%s\"", bdev->mntopts);
++
++	src = lxc_storage_get_path(bdev->src, bdev->type);
++
++	ret = mount(src, bdev->dest, "bind", MS_BIND | MS_REC | (mntflags & ~MS_RDONLY) | pflags, mntdata);
++	if (ret < 0) {
++		return log_error_errno(-errno, errno, "Failed to mount \"%s\" on \"%s\"", src, bdev->dest);
++	}
++	TRACE("Mounted \"%s\" on \"%s\"", src, bdev->dest);
++
++	return 0;
++}
+ #else
++int dir_mount(struct lxc_storage *bdev)
++{
++	__do_free char *mntdata = NULL;
+ 	unsigned long mflags = 0, mntflags = 0, pflags = 0;
+-#endif
+ 	int ret;
+ 	const char *src;
+ 
+@@ -151,13 +176,6 @@ int dir_mount(struct lxc_storage *bdev)
+ 
+ 	src = lxc_storage_get_path(bdev->src, bdev->type);
+ 
+-#ifdef HAVE_ISULAD
+-	ret = mount(src, bdev->dest, "bind", MS_BIND | MS_REC | (mntflags & ~MS_RDONLY) | pflags, mntdata);
+-	if (ret < 0) {
+-		return log_error_errno(-errno, errno, "Failed to mount \"%s\" on \"%s\"", src, bdev->dest);
+-	}
+-	TRACE("Mounted \"%s\" on \"%s\"", src, bdev->dest);
+-#else
+ 	ret = mount(src, bdev->dest, "bind", MS_BIND | MS_REC | mntflags | pflags, mntdata);
+ 	if (ret < 0)
+ 		return log_error_errno(-errno, errno, "Failed to mount \"%s\" on \"%s\"", src, bdev->dest);
+@@ -174,10 +192,10 @@ int dir_mount(struct lxc_storage *bdev)
+ 	}
+ 	TRACE("Mounted \"%s\" on \"%s\" with options \"%s\", mount flags \"%lu\", and propagation flags \"%lu\"",
+ 	      src ? src : "(none)", bdev->dest ? bdev->dest : "(none)", mntdata, mflags, pflags);
+-#endif
+ 
+ 	return 0;
+ }
++#endif
+ 
+ int dir_umount(struct lxc_storage *bdev)
+ {
+diff --git a/src/lxc/storage/overlay.c b/src/lxc/storage/overlay.c
+index 770785c..75a81de 100644
+--- a/src/lxc/storage/overlay.c
++++ b/src/lxc/storage/overlay.c
+@@ -349,6 +349,9 @@ int ovl_mount(struct lxc_storage *bdev)
+ 	char *work, *lastslash;
+ 	size_t len, len2;
+ 	int ret, ret2;
++#ifdef HAVE_ISULAD
++	unsigned long pflags = 0;
++#endif
+ 
+ 	if (strcmp(bdev->type, "overlay") && strcmp(bdev->type, "overlayfs"))
+ 		return -22;
+@@ -414,7 +417,12 @@ int ovl_mount(struct lxc_storage *bdev)
+ 	work = must_make_path(upper, LXC_OVERLAY_WORK_DIR, NULL);
+ 	upper[lastslash - upper] = '/';
+ 
++#ifdef HAVE_ISULAD
++	ret = parse_mntopts(bdev->mntopts, &mntflags, &pflags, &mntdata);
++#else
+ 	ret = parse_mntopts(bdev->mntopts, &mntflags, &mntdata);
++#endif
++
+ 	if (ret < 0) {
+ 		ERROR("Failed to parse mount options");
+ 		free(mntdata);
+diff --git a/src/lxc/storage/storage_utils.c b/src/lxc/storage/storage_utils.c
+index 07eee22..6fec638 100644
+--- a/src/lxc/storage/storage_utils.c
++++ b/src/lxc/storage/storage_utils.c
+@@ -340,6 +340,10 @@ int find_fstype_cb(char *buffer, void *data)
+ 	char mount_err[BUFSIZ] = {0};
+ 	int ret;
+ 
++#ifdef HAVE_ISULAD
++	unsigned long pflags = 0;
++#endif
++
+ 	/* we don't try 'nodev' entries */
+ 	if (strstr(buffer, "nodev"))
+ 		return 0;
+@@ -351,14 +355,19 @@ int find_fstype_cb(char *buffer, void *data)
+ 	DEBUG("Trying to mount \"%s\"->\"%s\" with FSType \"%s\"", cbarg->rootfs,
+ 	      cbarg->target, fstype);
+ 
+-	if (parse_mntopts(cbarg->options, &mntflags, &mntdata) < 0) {
++#ifdef HAVE_ISULAD
++	if (parse_mntopts(cbarg->options, &mntflags, &pflags, &mntdata) < 0) {
+ 		free(mntdata);
+ 		return 0;
+ 	}
+ 
+-#ifdef HAVE_ISULAD
+ 	if (mount(cbarg->rootfs, cbarg->target, fstype, (mntflags & ~MS_RDONLY), mntdata)) {
+ #else
++	if (parse_mntopts(cbarg->options, &mntflags, &mntdata) < 0) {
++		free(mntdata);
++		return 0;
++	}
++
+ 	if (mount(cbarg->rootfs, cbarg->target, fstype, mntflags, mntdata)) {
+ #endif
+ 		SYSDEBUG("Failed to mount");
+diff --git a/src/lxc/storage/zfs.c b/src/lxc/storage/zfs.c
+index 4cc171f..025cf95 100644
+--- a/src/lxc/storage/zfs.c
++++ b/src/lxc/storage/zfs.c
+@@ -167,13 +167,22 @@ int zfs_mount(struct lxc_storage *bdev)
+ 	const char *src;
+ 	char cmd_output[PATH_MAX] = {0};
+ 
++#ifdef HAVE_ISULAD
++	unsigned long pflags = 0;
++#endif
++
+ 	if (strcmp(bdev->type, "zfs"))
+ 		return -22;
+ 
+ 	if (!bdev->src || !bdev->dest)
+ 		return -22;
+ 
++#ifdef HAVE_ISULAD
++	ret = parse_mntopts(bdev->mntopts, &mntflags, &pflags, &mntdata);
++#else
+ 	ret = parse_mntopts(bdev->mntopts, &mntflags, &mntdata);
++#endif
++
+ 	if (ret < 0) {
+ 		ERROR("Failed to parse mount options");
+ 		free(mntdata);
+diff --git a/src/lxc/utils.h b/src/lxc/utils.h
+index 677f632..36c458e 100644
+--- a/src/lxc/utils.h
++++ b/src/lxc/utils.h
+@@ -27,8 +27,12 @@
+ #include "memory_utils.h"
+ #include "raw_syscalls.h"
+ #include "string_utils.h"
++
+ #ifdef HAVE_ISULAD
+ #include "isulad_utils.h"
++
++/* isulad: replace space with SPACE_MAGIC_STR */
++#define SPACE_MAGIC_STR "[#)"
+ #endif
+ 
+ /* returns 1 on success, 0 if there were any failures */
+-- 
+1.8.3.1
+

0023-attach.c-change-uid-and-gid-from-lxc-container-confi.patch

@@ -1,31 +0,0 @@
-From ecd40a856d039f7bc67ac076041a07cb369eaa77 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Mon, 14 Jan 2019 17:09:57 +0800
-Subject: [PATCH 023/140] attach.c: change uid and gid from lxc container
- config
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/attach.c | 6 ++++++
- 1 file changed, 6 insertions(+)
-
-diff --git a/src/lxc/attach.c b/src/lxc/attach.c
-index 570b9d0..e6e4b0d 100644
---- a/src/lxc/attach.c
-+++ b/src/lxc/attach.c
-@@ -1146,6 +1146,12 @@ int lxc_attach(const char *name, const char *lxcpath,
- 	}
- 	conf = init_ctx->container->lxc_conf;
- 
-+	// isulad: always switch uid and gid for attach
-+	if (options->uid == -1)
-+		options->uid = init_ctx->container->lxc_conf->init_uid;
-+	if (options->gid == -1)
-+		options->gid = init_ctx->container->lxc_conf->init_gid;
-+
- 	if (!fetch_seccomp(init_ctx->container, options))
- 		WARN("Failed to get seccomp policy");
- 
--- 
-1.8.3.1
-

0024-isulad-support-symlink-in-mount-entry-and-not-permit.patch

@@ -1,821 +0,0 @@
-From cf669a5cfd241dc8ec9ce2571e73fc13d4266bce Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Mon, 14 Jan 2019 20:12:06 +0800
-Subject: [PATCH 024/140] isulad: support symlink in mount entry, and not
- permit mount to /proc
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/Makefile.am |   2 +
- src/lxc/conf.c      | 108 ++++++++++-
- src/lxc/path.c      | 546 ++++++++++++++++++++++++++++++++++++++++++++++++++++
- src/lxc/path.h      |  70 +++++++
- 4 files changed, 721 insertions(+), 5 deletions(-)
- create mode 100644 src/lxc/path.c
- create mode 100644 src/lxc/path.h
-
-diff --git a/src/lxc/Makefile.am b/src/lxc/Makefile.am
-index 08e2fab..f2928b7 100644
---- a/src/lxc/Makefile.am
-+++ b/src/lxc/Makefile.am
-@@ -12,6 +12,7 @@ noinst_HEADERS = attach.h \
- 		 confile_utils.h \
- 		 criu.h \
- 		 error.h \
-+		 path.h \
- 		 file_utils.h \
- 		 ../include/netns_ifaddrs.h \
- 		 initutils.h \
-@@ -95,6 +96,7 @@ liblxc_la_SOURCES = af_unix.c af_unix.h \
- 		    commands_utils.c commands_utils.h \
- 		    conf.c conf.h \
- 		    confile.c confile.h \
-+		    path.c path.h \
- 		    confile_utils.c confile_utils.h \
- 		    criu.c criu.h \
- 		    error.c error.h \
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 55d1e45..800573a 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -77,6 +77,7 @@
- #include "storage/overlay.h"
- #include "syscall_wrappers.h"
- #include "terminal.h"
-+#include "path.h"
- #include "utils.h"
- 
- #ifdef MAJOR_IN_MKDEV
-@@ -2309,6 +2310,79 @@ static void cull_mntent_opt(struct mntent *mntent)
- 	}
- }
- 
-+/* isulad: checkMountDestination checks to ensure that the mount destination is not over the top of /proc.
-+ * dest is required to be an abs path and have any symlinks resolved before calling this function. */
-+static int check_mount_destination(const char *rootfs, const char *dest)
-+{
-+	const char *invalid_destinations[] = {
-+		"/proc",
-+		NULL
-+	};
-+	// White list, it should be sub directories of invalid destinations
-+	const char *valid_destinations[] = {
-+		// These entries can be bind mounted by files emulated by fuse,
-+		// so commands like top, free displays stats in container.
-+		"/proc/cpuinfo",
-+		"/proc/diskstats",
-+		"/proc/meminfo",
-+		"/proc/stat",
-+		"/proc/swaps",
-+		"/proc/uptime",
-+		"/proc/net/dev",
-+		NULL
-+	};
-+	const char **valid, **invalid;
-+
-+	for(valid = valid_destinations; *valid != NULL; valid++) {
-+		char *fullpath, *relpath;
-+		const char *parts[3] = {
-+			rootfs,
-+			*valid,
-+			NULL
-+		};
-+		fullpath = lxc_string_join("/", parts, false);
-+		if (!fullpath) {
-+			ERROR("Out of memory");
-+			return -1;
-+		}
-+		relpath = path_relative(fullpath, dest);
-+		free(fullpath);
-+		if (!relpath)
-+			return -1;
-+		if (!strcmp(relpath, ".")) {
-+			free(relpath);
-+			return 0;
-+		}
-+		free(relpath);
-+	}
-+
-+	for(invalid = invalid_destinations; *invalid != NULL; invalid++) {
-+		char *fullpath, *relpath;
-+		const char *parts[3] = {
-+			rootfs,
-+			*invalid,
-+			NULL
-+		};
-+		fullpath = lxc_string_join("/", parts, false);
-+		if (!fullpath) {
-+			ERROR("Out of memory");
-+			return -1;
-+		}
-+		relpath = path_relative(fullpath, dest);
-+		free(fullpath);
-+		if (!relpath)
-+			return -1;
-+		if (!strcmp(relpath, ".") || strncmp(relpath, "..", 2)) {
-+			ERROR("%s cannot be mounted because it is located inside %s", dest, *invalid);
-+			free(relpath);
-+			return -1;
-+		}
-+		free(relpath);
-+	}
-+
-+	return 0;
-+}
-+
- static int mount_entry_create_dir_file(const struct mntent *mntent,
- 				       const char *path,
- 				       const struct lxc_rootfs *rootfs,
-@@ -2370,7 +2444,8 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
- 	unsigned long mntflags, pflags;
- 	char *mntdata;
- 	bool dev, optional, relative;
--	char *rootfs_path = NULL;
-+	char *rootfs_path = NULL, *rpath = NULL;
-+	const char *dest = path;
- 
- 	optional = hasmntopt(mntent, "optional") != NULL;
- 	dev = hasmntopt(mntent, "dev") != NULL;
-@@ -2379,9 +2454,29 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
- 	if (rootfs && rootfs->path)
- 		rootfs_path = rootfs->mount;
- 
--	ret = mount_entry_create_dir_file(mntent, path, rootfs, lxc_name,
--					  lxc_path);
-+	// isulad: ensure that the destination of the bind mount is resolved of symlinks at mount time because
-+	// any previous mounts can invalidate the next mount's destination.
-+	// this can happen when a user specifies mounts within other mounts to cause breakouts or other
-+	// evil stuff to try to escape the container's rootfs.
-+	if (rootfs_path) {
-+		rpath = follow_symlink_in_scope(path, rootfs_path);
-+		if (!rpath) {
-+			ERROR("Failed to get real path for '%s'", path);
-+			return -1;
-+		}
-+		dest = rpath;
-+
-+		ret = check_mount_destination(rootfs_path, dest);
-+		if (ret) {
-+			ERROR("Mount destination is invalid: '%s'", dest);
-+			free(rpath);
-+			return -1;
-+		}
-+	}
-+
-+	ret = mount_entry_create_dir_file(mntent, dest, rootfs, lxc_name, lxc_path);
- 	if (ret < 0) {
-+		free(rpath);
- 		if (optional)
- 			return 0;
- 
-@@ -2390,13 +2485,16 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
- 	cull_mntent_opt(mntent);
- 
- 	ret = parse_mntopts(mntent->mnt_opts, &mntflags, &pflags, &mntdata);
--	if (ret < 0)
-+	if (ret < 0) {
-+		free(rpath);
- 		return -1;
-+	}
- 
--	ret = mount_entry(mntent->mnt_fsname, path, mntent->mnt_type, mntflags,
-+	ret = mount_entry(mntent->mnt_fsname, dest, mntent->mnt_type, mntflags,
- 			  pflags, mntdata, optional, dev, relative, rootfs_path);
- 
- 	free(mntdata);
-+	free(rpath);
- 	return ret;
- }
- 
-diff --git a/src/lxc/path.c b/src/lxc/path.c
-new file mode 100644
-index 0000000..e917dcb
---- /dev/null
-+++ b/src/lxc/path.c
-@@ -0,0 +1,546 @@
-+#include <unistd.h>
-+#include <stdlib.h>
-+#include <limits.h>
-+#include <string.h>
-+#include <stdio.h>
-+#include <sys/types.h>
-+#include <sys/stat.h>
-+#include <sys/param.h>
-+#include <libgen.h>
-+
-+#include "path.h"
-+#include "log.h"
-+
-+lxc_log_define(lxc_path_ui, lxc);
-+
-+#define ISSLASH(C) ((C) == '/')
-+#define IS_ABSOLUTE_FILE_NAME(F) (ISSLASH ((F)[0]))
-+#define IS_RELATIVE_FILE_NAME(F) (! IS_ABSOLUTE_FILE_NAME (F))
-+
-+bool specify_current_dir(const char *path)
-+{
-+	char *basec = NULL, *bname = NULL;
-+	bool res = false;
-+
-+	basec = strdup(path);
-+	if (!basec) {
-+		ERROR("Out of memory");
-+		return false;
-+	}
-+
-+	bname = basename(basec);
-+	res = !strcmp(bname, ".");
-+	free(basec);
-+	return res;
-+}
-+
-+bool has_traling_path_separator(const char *path)
-+{
-+	return path && strlen(path) && (path[strlen(path) - 1] == '/');
-+}
-+
-+// PreserveTrailingDotOrSeparator returns the given cleaned path
-+// and appends a trailing `/.` or `/` if its corresponding  original
-+// path ends with a trailing `/.` or `/`. If the cleaned
-+// path already ends in a `.` path segment, then another is not added. If the
-+// clean path already ends in a path separator, then another is not added.
-+char *preserve_trailing_dot_or_separator(const char *cleanedpath,
-+					 const char *originalpath)
-+{
-+	char *respath = NULL;
-+	size_t len;
-+
-+	len = strlen(cleanedpath) + 3;
-+	respath = malloc(len);
-+	if (!respath) {
-+		ERROR("Out of memory");
-+		return NULL;
-+	}
-+	memset(respath, 0x00, len);
-+	strcat(respath, cleanedpath);
-+
-+	if (!specify_current_dir(cleanedpath) && specify_current_dir(originalpath)) {
-+		if (!has_traling_path_separator(respath))
-+			strcat(respath, "/");
-+		strcat(respath, ".");
-+	}
-+
-+	if (!has_traling_path_separator(respath) &&
-+	    has_traling_path_separator(originalpath))
-+		strcat(respath, "/");
-+
-+	return respath;
-+}
-+
-+
-+// Split splits path immediately following the final Separator,
-+// separating it into a directory and file name component.
-+// If there is no Separator in path, Split returns an empty dir
-+// and file set to path.
-+// The returned values have the property that path = dir+file.
-+bool filepath_split(const char *path, char **dir, char **base)
-+{
-+	ssize_t i;
-+	size_t len;
-+
-+	len = strlen(path);
-+	i = len - 1;
-+	while (i >= 0 && path[i] != '/')
-+		i--;
-+
-+	*dir = malloc(i + 2);
-+	if (!*dir) {
-+		ERROR("Out of memory");
-+		return false;
-+	}
-+	memcpy(*dir, path, i + 1);
-+	*(*dir + i + 1) = '\0';
-+
-+	*base = strdup(path + i + 1);
-+	if (!*base) {
-+		ERROR("Out of memory");
-+		free(*dir);
-+		*dir = NULL;
-+		return false;
-+	}
-+
-+	return true;
-+}
-+
-+/*
-+ * cleanpath is similar to realpath of glibc, but not expands symbolic links,
-+ * and not check the existence of components of the path.
-+ */
-+char *cleanpath(const char *path, char *resolved)
-+{
-+	char *rpath, *dest;
-+	const char *start, *end, *rpath_limit;
-+
-+	if (path == NULL || path[0] == '\0')
-+		return NULL;
-+
-+	if (resolved == NULL) {
-+		rpath = malloc(PATH_MAX);
-+		if (rpath == NULL) {
-+			ERROR("Out of memory");
-+			return NULL;
-+		}
-+	} else {
-+		rpath = resolved;
-+	}
-+	rpath_limit = rpath + PATH_MAX;
-+
-+	if (!IS_ABSOLUTE_FILE_NAME(path)) {
-+		if (!getcwd(rpath, PATH_MAX)) {
-+			ERROR("Failed to getcwd");
-+			rpath[0] = '\0';
-+			goto error;
-+		}
-+		dest = strchr(rpath, '\0');
-+		start = path;
-+	} else {
-+		dest = rpath;
-+		*dest++ = '/';
-+		start = path;
-+	}
-+
-+	for (end = start; *start; start = end) {
-+		/* Skip sequence of multiple path-separators.  */
-+		while (ISSLASH(*start))
-+			++start;
-+
-+		/* Find end of path component.  */
-+		for (end = start; *end && !ISSLASH(*end); ++end)
-+			/* Nothing.  */;
-+
-+		if (end - start == 0) {
-+			break;
-+		} else if (end - start == 1 && start[0] == '.') {
-+			/* nothing */;
-+		} else if (end - start == 2 && start[0] == '.' && start[1] == '.') {
-+			/* Back up to previous component, ignore if at root already.  */
-+			if (dest > rpath + 1)
-+				for (--dest; dest > rpath && !ISSLASH(dest[-1]); --dest)
-+					continue;
-+		} else {
-+			size_t new_size;
-+
-+			if (!ISSLASH(dest[-1]))
-+				*dest++ = '/';
-+
-+			if (dest + (end - start) >= rpath_limit) {
-+				long long dest_offset = dest - rpath;
-+				char *new_rpath;
-+
-+				if (resolved) {
-+					printf("Path is to long");
-+					if (dest > rpath + 1)
-+						dest--;
-+					*dest = '\0';
-+					goto error;
-+				}
-+
-+				new_size = rpath_limit - rpath;
-+				if (end - start + 1 > PATH_MAX)
-+					new_size += end - start + 1;
-+				else
-+					new_size += PATH_MAX;
-+				new_rpath = (char *) realloc(rpath, new_size);
-+				if (new_rpath == NULL) {
-+					ERROR("Out of memory");
-+					goto error;
-+				}
-+				rpath = new_rpath;
-+				rpath_limit = rpath + new_size;
-+
-+				dest = rpath + dest_offset;
-+			}
-+
-+			memcpy(dest, start, end - start);
-+			dest += end - start;
-+			*dest = '\0';
-+		}
-+	}
-+	if (dest > rpath + 1 && ISSLASH(dest[-1]))
-+		--dest;
-+	*dest = '\0';
-+
-+	return rpath;
-+
-+error:
-+	if (resolved == NULL)
-+		free(rpath);
-+	return NULL;
-+}
-+
-+// evalSymlinksInScope will evaluate symlinks in `path` within a scope `root` and return
-+// a result guaranteed to be contained within the scope `root`, at the time of the call.
-+// Symlinks in `root` are not evaluated and left as-is.
-+// Errors encountered while attempting to evaluate symlinks in path will be returned.
-+// Non-existing paths are valid and do not constitute an error.
-+// `path` has to contain `root` as a prefix, or else an error will be returned.
-+// Trying to break out from `root` does not constitute an error.
-+//
-+// Example:
-+//   If /foo/bar -> /outside,
-+//   FollowSymlinkInScope("/foo/bar", "/foo") == "/foo/outside" instead of "/oustide"
-+char *eval_symlinks_in_scope(const char *fullpath, const char *rootpath)
-+{
-+	char resroot[PATH_MAX] = {0}, *root = NULL;
-+	char *rpath, *dest, *prefix, *extra_buf = NULL;
-+	const char *start, *end, *rpath_limit;
-+	int num_links = 0;
-+	size_t prefix_len;
-+
-+	if (!fullpath || !rootpath)
-+		return NULL;
-+
-+	root = cleanpath(rootpath, resroot);
-+	if (!root) {
-+		ERROR("Failed to get cleaned path");
-+		return NULL;
-+	}
-+
-+	if (!strcmp(fullpath, root))
-+		return strdup(fullpath);
-+
-+	if (!strstr(fullpath, root)) {
-+		ERROR("Path '%s' is not in '%s'", fullpath, root);
-+		return NULL;
-+	}
-+
-+	rpath = malloc(PATH_MAX);
-+	if (rpath == NULL) {
-+		ERROR("Out of memory");
-+		goto error;
-+		return NULL;
-+	}
-+	rpath_limit = rpath + PATH_MAX;
-+
-+	prefix = root;
-+	prefix_len = strlen(prefix);
-+	if (!strcmp(prefix, "/"))
-+		prefix_len = 0;
-+
-+	dest = rpath;
-+	if (prefix_len) {
-+		memcpy(rpath, prefix, prefix_len);
-+		dest += prefix_len;
-+	}
-+	*dest++ = '/';
-+	start = fullpath + prefix_len;
-+
-+	for (end = start; *start; start = end) {
-+		struct stat st;
-+		int n;
-+
-+		/* Skip sequence of multiple path-separators.  */
-+		while (ISSLASH(*start))
-+			++start;
-+
-+		/* Find end of path component.  */
-+		for (end = start; *end && !ISSLASH(*end); ++end)
-+			/* Nothing.  */;
-+
-+		if (end - start == 0) {
-+			break;
-+		} else if (end - start == 1 && start[0] == '.') {
-+			/* nothing */;
-+		} else if (end - start == 2 && start[0] == '.' && start[1] == '.') {
-+			/* Back up to previous component, ignore if at root already.  */
-+			if (dest > rpath + prefix_len + 1)
-+				for (--dest; dest > rpath && !ISSLASH(dest[-1]); --dest)
-+					continue;
-+		} else {
-+			size_t new_size;
-+
-+			if (!ISSLASH(dest[-1]))
-+				*dest++ = '/';
-+
-+			if (dest + (end - start) >= rpath_limit) {
-+				long long dest_offset = dest - rpath;
-+				char *new_rpath;
-+
-+				new_size = rpath_limit - rpath;
-+				if (end - start + 1 > PATH_MAX)
-+					new_size += end - start + 1;
-+				else
-+					new_size += PATH_MAX;
-+				new_rpath = (char *) realloc(rpath, new_size);
-+				if (new_rpath == NULL) {
-+					ERROR("Out of memory");
-+					goto error;
-+				}
-+				rpath = new_rpath;
-+				rpath_limit = rpath + new_size;
-+
-+				dest = rpath + dest_offset;
-+			}
-+
-+			memcpy(dest, start, end - start);
-+			dest += end - start;
-+			*dest = '\0';
-+
-+			if (lstat(rpath, &st) < 0) {
-+				// if rpath does not exist, accept it
-+				continue;
-+			}
-+
-+			if (S_ISLNK(st.st_mode)) {
-+				char *buf;
-+				size_t len;
-+
-+				if (++num_links > MAXSYMLINKS) {
-+					ERROR("Too many links in '%s'", fullpath);
-+					goto error;
-+				}
-+
-+				buf = malloc(PATH_MAX);
-+				if (!buf) {
-+					ERROR("Out of memory");
-+					goto error;
-+				}
-+
-+				n = readlink(rpath, buf, PATH_MAX - 1);
-+				if (n < 0) {
-+					free(buf);
-+					goto error;
-+				}
-+				buf[n] = '\0';
-+
-+				if (!extra_buf) {
-+					extra_buf = malloc(PATH_MAX);
-+					if (!extra_buf) {
-+						ERROR("Out of memory");
-+						free(buf);
-+						goto error;
-+					}
-+				}
-+
-+				len = strlen(end);
-+				if ((long int)(n + len) >= PATH_MAX) {
-+					free(buf);
-+					ERROR("Path is too long");
-+					goto error;
-+				}
-+
-+				/* Careful here, end may be a pointer into extra_buf... */
-+				memmove(&extra_buf[n], end, len + 1);
-+				fullpath = end = memcpy(extra_buf, buf, n);
-+
-+				if (IS_ABSOLUTE_FILE_NAME(buf)) {
-+					if (prefix_len)
-+						memcpy(rpath, prefix, prefix_len);
-+					dest = rpath + prefix_len;
-+					*dest++ = '/'; /* It's an absolute symlink */
-+				} else {
-+					/* Back up to previous component, ignore if at root
-+					already: */
-+					if (dest > rpath + prefix_len + 1)
-+						for (--dest; dest > rpath && !ISSLASH(dest[-1]); --dest)
-+							continue;
-+				}
-+			}
-+		}
-+	}
-+	if (dest > rpath + prefix_len + 1 && ISSLASH(dest[-1]))
-+		--dest;
-+	*dest = '\0';
-+
-+	if (extra_buf)
-+		free(extra_buf);
-+
-+	return rpath;
-+
-+error:
-+	if (extra_buf)
-+		free(extra_buf);
-+	free(rpath);
-+	return NULL;
-+}
-+
-+// FollowSymlinkInScope is a wrapper around evalSymlinksInScope that returns an
-+// absolute path. This function handles paths in a platform-agnostic manner.
-+char *follow_symlink_in_scope(const char *fullpath, const char *rootpath)
-+{
-+	char resfull[PATH_MAX] = {0}, *full = NULL;
-+	char resroot[PATH_MAX] = {0}, *root = NULL;
-+
-+	full = cleanpath(fullpath, resfull);
-+	if (!full) {
-+		ERROR("Failed to get cleaned path");
-+		return NULL;
-+	}
-+
-+	root = cleanpath(rootpath, resroot);
-+	if (!root) {
-+		ERROR("Failed to get cleaned path");
-+		return NULL;
-+	}
-+
-+	return eval_symlinks_in_scope(full, root);
-+}
-+
-+// GetResourcePath evaluates `path` in the scope of the container's rootpath, with proper path
-+// sanitisation. Symlinks are all scoped to the rootpath of the container, as
-+// though the container's rootpath was `/`.
-+//
-+// The BaseFS of a container is the host-facing path which is bind-mounted as
-+// `/` inside the container. This method is essentially used to access a
-+// particular path inside the container as though you were a process in that
-+// container.
-+int get_resource_path(const char *rootpath, const char *path,
-+		      char **scopepath)
-+{
-+	char resolved[PATH_MAX] = {0}, *cleanedpath = NULL;
-+	char *fullpath = NULL;
-+	size_t len;
-+
-+	if (!rootpath || !path || !scopepath)
-+		return -1;
-+
-+	*scopepath = NULL;
-+
-+	cleanedpath = cleanpath(path, resolved);
-+	if (!cleanedpath) {
-+		ERROR("Failed to get cleaned path");
-+		return -1;
-+	}
-+
-+	len = strlen(rootpath) + strlen(cleanedpath) + 1;
-+	fullpath = malloc(len);
-+	if (!fullpath) {
-+		ERROR("Out of memory");
-+		return -1;
-+	}
-+	snprintf(fullpath, len, "%s%s", rootpath, cleanedpath);
-+
-+	*scopepath = follow_symlink_in_scope(fullpath, rootpath);
-+
-+	free(fullpath);
-+	return 0;
-+}
-+
-+// Rel returns a relative path that is lexically equivalent to targpath when
-+// joined to basepath with an intervening separator. That is,
-+// Join(basepath, Rel(basepath, targpath)) is equivalent to targpath itself.
-+// On success, the returned path will always be relative to basepath,
-+// even if basepath and targpath share no elements.
-+// An error is returned if targpath can't be made relative to basepath or if
-+// knowing the current working directory would be necessary to compute it.
-+// Rel calls Clean on the result.
-+char *path_relative(const char *basepath, const char *targpath)
-+{
-+	char resbase[PATH_MAX] = {0}, *base = NULL;
-+	char restarg[PATH_MAX] = {0}, *targ = NULL;
-+	size_t bl = 0, tl = 0, b0 = 0, bi = 0, t0 = 0, ti = 0;
-+
-+	base = cleanpath(basepath, resbase);
-+	if (!base) {
-+		ERROR("Failed to get cleaned path");
-+		return NULL;
-+	}
-+
-+	targ = cleanpath(targpath, restarg);
-+	if (!targ) {
-+		ERROR("Failed to get cleaned path");
-+		return NULL;
-+	}
-+
-+	if (strcmp(base, targ) == 0)
-+		return strdup(".");
-+
-+	bl = strlen(base);
-+	tl = strlen(targ);
-+	while(true) {
-+		while(bi < bl && !ISSLASH(base[bi]))
-+			bi++;
-+		while(ti < tl && !ISSLASH(targ[ti]))
-+			ti++;
-+		//not the same string
-+		if (((bi - b0) != (ti - t0)) || strncmp(base + b0, targ + t0, bi - b0))
-+			break;
-+		if (bi < bl)
-+			bi++;
-+		if (ti < tl)
-+			ti++;
-+		b0 = bi;
-+		t0 = ti;
-+	}
-+
-+	if (b0 != bl) {
-+		// Base elements left. Must go up before going down.
-+		int seps = 0, i;
-+		size_t ncopyed = 0, seps_size;
-+		char *buf;
-+
-+		for (bi = b0; bi < bl; bi++) {
-+			if (ISSLASH(base[bi]))
-+				seps++;
-+		}
-+		//strlen(..) + strlen(/..) + '\0'
-+		seps_size = 2 + seps * 3 + 1;
-+		if (t0 != tl)
-+			seps_size += 1 + tl - t0;
-+
-+		buf = calloc(seps_size, 1);
-+		if (!buf) {
-+			ERROR("Out of memory");
-+			return NULL;
-+		}
-+		buf[ncopyed++] = '.';
-+		buf[ncopyed++] = '.';
-+		for (i = 0; i < seps; i++) {
-+			buf[ncopyed++] = '/';
-+			buf[ncopyed++] = '.';
-+			buf[ncopyed++] = '.';
-+		}
-+		if (t0 != tl) {
-+			buf[ncopyed++] = '/';
-+			memcpy(buf + ncopyed, targ + t0, tl - t0 + 1);
-+		}
-+		return buf;
-+	}
-+
-+	return strdup(targ + t0);
-+}
-\ No newline at end of file
-diff --git a/src/lxc/path.h b/src/lxc/path.h
-new file mode 100644
-index 0000000..e3a04cc
---- /dev/null
-+++ b/src/lxc/path.h
-@@ -0,0 +1,70 @@
-+#ifndef __LCRD_PATH_H_
-+#define __LCRD_PATH_H_
-+
-+#include <stdbool.h>
-+
-+bool specify_current_dir(const char *path);
-+
-+bool has_traling_path_separator(const char *path);
-+
-+// PreserveTrailingDotOrSeparator returns the given cleaned path
-+// and appends a trailing `/.` or `/` if its corresponding  original
-+// path ends with a trailing `/.` or `/`. If the cleaned
-+// path already ends in a `.` path segment, then another is not added. If the
-+// clean path already ends in a path separator, then another is not added.
-+char *preserve_trailing_dot_or_separator(const char *cleanedpath,
-+					 const char *originalpath);
-+
-+
-+// Split splits path immediately following the final Separator,
-+// separating it into a directory and file name component.
-+// If there is no Separator in path, Split returns an empty dir
-+// and file set to path.
-+// The returned values have the property that path = dir+file.
-+bool filepath_split(const char *path, char **dir, char **base);
-+
-+/*
-+ * cleanpath is similar to realpath of glibc, but not expands symbolic links,
-+ * and not check the existence of components of the path.
-+ */
-+char *cleanpath(const char *path, char *resolved);
-+
-+// evalSymlinksInScope will evaluate symlinks in `path` within a scope `root` and return
-+// a result guaranteed to be contained within the scope `root`, at the time of the call.
-+// Symlinks in `root` are not evaluated and left as-is.
-+// Errors encountered while attempting to evaluate symlinks in path will be returned.
-+// Non-existing paths are valid and do not constitute an error.
-+// `path` has to contain `root` as a prefix, or else an error will be returned.
-+// Trying to break out from `root` does not constitute an error.
-+//
-+// Example:
-+//   If /foo/bar -> /outside,
-+//   FollowSymlinkInScope("/foo/bar", "/foo") == "/foo/outside" instead of "/oustide"
-+char *eval_symlinks_in_scope(const char *fullpath, const char *rootpath);
-+
-+// FollowSymlinkInScope is a wrapper around evalSymlinksInScope that returns an
-+// absolute path. This function handles paths in a platform-agnostic manner.
-+char *follow_symlink_in_scope(const char *fullpath, const char *rootpath);
-+
-+// GetResourcePath evaluates `path` in the scope of the container's rootpath, with proper path
-+// sanitisation. Symlinks are all scoped to the rootpath of the container, as
-+// though the container's rootpath was `/`.
-+//
-+// The BaseFS of a container is the host-facing path which is bind-mounted as
-+// `/` inside the container. This method is essentially used to access a
-+// particular path inside the container as though you were a process in that
-+// container.
-+int get_resource_path(const char *rootpath, const char *path,
-+		      char **scopepath);
-+
-+// Rel returns a relative path that is lexically equivalent to targpath when
-+// joined to basepath with an intervening separator. That is,
-+// Join(basepath, Rel(basepath, targpath)) is equivalent to targpath itself.
-+// On success, the returned path will always be relative to basepath,
-+// even if basepath and targpath share no elements.
-+// An error is returned if targpath can't be made relative to basepath or if
-+// knowing the current working directory would be necessary to compute it.
-+// Rel calls Clean on the result.
-+char *path_relative(const char *basepath, const char *targpath);
-+
-+#endif
--- 
-1.8.3.1
-

0024-start-do-not-check-ppid-when-set-death-signal.patch

@@ -0,0 +1,40 @@
+From 52b97324185142285f78143509244d88c6962826 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Tue, 14 Apr 2020 17:38:44 +0800
+Subject: [PATCH 24/49] start: do not check ppid when set death signal
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/utils.c | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/src/lxc/utils.c b/src/lxc/utils.c
+index 5b04fa4..27078e2 100644
+--- a/src/lxc/utils.c
++++ b/src/lxc/utils.c
+@@ -1734,11 +1734,14 @@ static int process_dead(/* takes */ int status_fd)
+ int lxc_set_death_signal(int signal, pid_t parent, int parent_status_fd)
+ {
+ 	int ret;
++#ifndef HAVE_ISULAD
+ 	pid_t ppid;
++#endif
+ 
+ 	ret = prctl(PR_SET_PDEATHSIG, prctl_arg(signal), prctl_arg(0),
+ 		    prctl_arg(0), prctl_arg(0));
+ 
++#ifndef HAVE_ISULAD
+ 	/* verify that we haven't been orphaned in the meantime */
+ 	ppid = (pid_t)syscall(SYS_getppid);
+ 	if (ppid == 0) { /* parent outside our pidns */
+@@ -1750,6 +1753,7 @@ int lxc_set_death_signal(int signal, pid_t parent, int parent_status_fd)
+ 	} else if (ppid != parent) {
+ 		return raise(SIGKILL);
+ 	}
++#endif
+ 
+ 	if (ret < 0)
+ 		return -1;
+-- 
+1.8.3.1
+

0026-Supporting-UID-GID-configuration.patch

@@ -0,0 +1,306 @@
+From 61bfa0ce515288897b93640507e48f09f0d78010 Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Tue, 14 Apr 2020 23:17:04 -0400
+Subject: [PATCH 26/49] Supporting UID GID configuration
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/attach.c           | 16 ++++++++
+ src/lxc/conf.c             | 19 ++++++++-
+ src/lxc/conf.h             |  4 ++
+ src/lxc/start.c            |  8 ++++
+ src/lxc/tools/lxc_attach.c | 99 +++++++++++++++++++++++++++++++++++++++++++---
+ 5 files changed, 139 insertions(+), 7 deletions(-)
+
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index 33946bb..c77b929 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -749,8 +749,10 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 			goto on_error;
+ 	}
+ 
++#ifndef HAVE_ISULAD
+ 	if (!lxc_setgroups(0, NULL) && errno != EPERM)
+ 		goto on_error;
++#endif
+ 
+ 	if (options->namespaces & CLONE_NEWUSER) {
+ 		/* Check whether nsuid 0 has a mapping. */
+@@ -892,6 +894,12 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 	if (new_gid == ns_root_gid)
+ 		new_gid = LXC_INVALID_GID;
+ 
++#ifdef HAVE_ISULAD
++	if (!lxc_setgroups(init_ctx->container->lxc_conf->init_groups_len,
++			init_ctx->container->lxc_conf->init_groups))
++		goto on_error;
++#endif
++
+ 	/* Make sure that the processes STDIO is correctly owned by the user that we are switching to */
+ 	ret = fix_stdio_permissions(new_uid);
+ 	if (ret)
+@@ -1058,6 +1066,14 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 	}
+ 	conf = init_ctx->container->lxc_conf;
+ 
++#ifdef HAVE_ISULAD
++	// always switch uid and gid for attach
++	if (options->uid == -1)
++		options->uid = init_ctx->container->lxc_conf->init_uid;
++	if (options->gid == -1)
++		options->gid = init_ctx->container->lxc_conf->init_gid;
++#endif
++
+ 	if (!fetch_seccomp(init_ctx->container, options))
+ 		WARN("Failed to get seccomp policy");
+ 
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index 71fd6f9..43ef067 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -5100,6 +5100,7 @@ void lxc_conf_free(struct lxc_conf *conf)
+ 		close(conf->exit_fd);
+ 	}
+ 	lxc_clear_init_args(conf);
++	lxc_clear_init_groups(conf);
+ 	lxc_clear_populate_devices(conf);
+ 	lxc_clear_rootfs_masked_paths(conf);
+ 	lxc_clear_rootfs_ro_paths(conf);
+@@ -5919,11 +5920,25 @@ struct lxc_list *sort_cgroup_settings(struct lxc_list *cgroup_settings)
+ /*isulad clear init args*/
+ int lxc_clear_init_args(struct lxc_conf *lxc_conf)
+ {
+-	size_t i;
++	int i;
+ 
+-	for (i = 0; i < lxc_conf->init_argc; i++)
++	for (i = 0; i < lxc_conf->init_argc; i++) {
+ 		free(lxc_conf->init_argv[i]);
++		lxc_conf->init_argv[i] = NULL;
++	}
+ 	free(lxc_conf->init_argv);
++	lxc_conf->init_argv = NULL;
++	lxc_conf->init_argc = 0;
++
++	return 0;
++}
++
++/*isulad clear init groups*/
++int lxc_clear_init_groups(struct lxc_conf *lxc_conf)
++{
++	free(lxc_conf->init_groups);
++	lxc_conf->init_groups = NULL;
++	lxc_conf->init_groups_len = 0;
+ 
+ 	return 0;
+ }
+diff --git a/src/lxc/conf.h b/src/lxc/conf.h
+index 61c3383..879e427 100644
+--- a/src/lxc/conf.h
++++ b/src/lxc/conf.h
+@@ -451,6 +451,9 @@ struct lxc_conf {
+ 	char **init_argv;
+ 	size_t init_argc;
+ 
++	gid_t *init_groups;
++	size_t init_groups_len;
++
+ 	/* populate devices*/
+ 	struct lxc_list populate_devs;
+ 	mode_t umask;  //umask value
+@@ -548,6 +551,7 @@ extern int parse_mntopts(const char *mntopts, unsigned long *mntflags,
+ #ifdef HAVE_ISULAD
+ // isulad add
+ int lxc_clear_init_args(struct lxc_conf *lxc_conf);
++int lxc_clear_init_groups(struct lxc_conf *lxc_conf);
+ int lxc_clear_populate_devices(struct lxc_conf *c);
+ int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
+ int lxc_clear_rootfs_ro_paths(struct lxc_conf *c);
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 4f45776..e2311ec 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -1685,8 +1685,16 @@ static int do_start(void *data)
+ 		#if HAVE_LIBCAP
+ 		if (lxc_proc_cap_is_set(CAP_SETGID, CAP_EFFECTIVE))
+ 		#endif
++		#ifdef HAVE_ISULAD
++			/* isulad: set groups for init process, and before we set uid and gid */
++			if (!lxc_setgroups(handler->conf->init_groups_len, handler->conf->init_groups)) {
++				ERROR("Can not set groups");
++				goto out_warn_father;
++				}
++		#else
+ 			if (!lxc_setgroups(0, NULL))
+ 				goto out_warn_father;
++		#endif
+ 
+ 	if (!lxc_switch_uid_gid(new_uid, new_gid))
+ 		goto out_warn_father;
+diff --git a/src/lxc/tools/lxc_attach.c b/src/lxc/tools/lxc_attach.c
+index 47ac2f2..b068c9a 100644
+--- a/src/lxc/tools/lxc_attach.c
++++ b/src/lxc/tools/lxc_attach.c
+@@ -72,9 +72,11 @@ static const struct option my_longopts[] = {
+ 	{"set-var", required_argument, 0, 'v'},
+ 	{"pty-log", required_argument, 0, 'L'},
+ 	{"rcfile", required_argument, 0, 'f'},
++#ifndef HAVE_ISULAD
+ 	{"uid", required_argument, 0, 'u'},
+ 	{"gid", required_argument, 0, 'g'},
+-#ifdef HAVE_ISULAD
++#else
++	{"user", required_argument, 0, 'u'},
+ 	{"in-fifo", required_argument, 0, OPT_INPUT_FIFO}, /* isulad add terminal fifos*/
+ 	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
+ 	{"err-fifo", required_argument, 0, OPT_STDERR_FIFO},
+@@ -130,9 +132,18 @@ Options :\n\
+                     multiple times.\n\
+   -f, --rcfile=FILE\n\
+                     Load configuration file FILE\n\
++"
++#ifndef HAVE_ISULAD
++"\
+   -u, --uid=UID     Execute COMMAND with UID inside the container\n\
+   -g, --gid=GID     Execute COMMAND with GID inside the container\n\
+-",
++"
++#else
++"\
++  --user        User ID (format: UID[:GID])\n\
++"
++#endif
++,
+ 	.options      = my_longopts,
+ 	.parser       = my_parser,
+ 	.checker      = NULL,
+@@ -142,6 +153,71 @@ Options :\n\
+ 	.gid          = LXC_INVALID_GID,
+ };
+ 
++#ifdef HAVE_ISULAD
++static int parse_user_id(const char *username, char **uid, char **gid, char **tmp_dup)
++{
++	char *tmp = NULL;
++	char *pdot = NULL;
++
++	if (uid == NULL || gid == NULL || tmp_dup == NULL) {
++		return -1;
++	}
++
++	if (username != NULL) {
++		tmp = strdup(username);
++		if (tmp == NULL) {
++			ERROR("Failed to duplicate user name");
++			return -1;
++		}
++
++		// for free tmp in caller
++		*tmp_dup = tmp;
++		pdot = strstr(tmp, ":");
++		if (pdot != NULL) {
++			*pdot = '\0';
++			if (pdot != tmp) {
++				// uid found
++				*uid = tmp;
++			}
++
++			if (*(pdot + 1) != '\0') {
++				// gid found
++				*gid = pdot + 1;
++			}
++		} else {
++			// No : found
++			if (*tmp != '\0') {
++				*uid = tmp;
++			}
++		}
++	}
++
++	return 0;
++}
++
++static int get_attach_uid_gid(const char *username, uid_t *user_id, gid_t *group_id)
++{
++	char *tmp = NULL;
++	char *uid = NULL;
++	char *gid = NULL;
++
++	// parse uid and gid by username
++	if (parse_user_id(username, &uid, &gid, &tmp) != 0) {
++		return -1;
++	}
++
++	if (uid != NULL) {
++		*user_id = (unsigned int)atoll(uid);
++	}
++	if (gid != NULL) {
++		*group_id = (unsigned int)atoll(gid);
++	}
++
++	free(tmp);
++	return 0;
++}
++#endif
++
+ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ {
+ 	int ret;
+@@ -199,6 +275,7 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 	case 'f':
+ 		args->rcfile = arg;
+ 		break;
++#ifndef HAVE_ISULAD
+ 	case 'u':
+ 		if (lxc_safe_uint(arg, &args->uid) < 0)
+ 			return -1;
+@@ -207,7 +284,13 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 		if (lxc_safe_uint(arg, &args->gid) < 0)
+ 			return -1;
+ 		break;
+-#ifdef HAVE_ISULAD
++#else
++	case 'u':
++		if (get_attach_uid_gid(arg, &args->uid, &args->gid) != 0) {
++			ERROR("Failed to get attach user U/GID");
++			return -1;
++		}
++		break;
+ 	case OPT_INPUT_FIFO:
+ 		args->terminal_fifos[0] = arg;
+ 		break;
+@@ -286,7 +369,7 @@ static int lxc_attach_create_log_file(const char *log_file)
+ 
+ #ifdef HAVE_ISULAD
+ // isulad: send '128 + signal' if container is killed by signal.
+-#define ExitSignalOffset 128
++#define EXIT_SIGNAL_OFFSET 128
+ 
+ /*isulad: attach with terminal*/
+ static int do_attach_foreground(struct lxc_container *c, lxc_attach_command_t *command,
+@@ -316,7 +399,7 @@ static int do_attach_foreground(struct lxc_container *c, lxc_attach_command_t *c
+ 
+ 	if (WIFSIGNALED(ret)) {
+ 		signal = WTERMSIG(ret);
+-		wexit = ExitSignalOffset + signal;
++		wexit = EXIT_SIGNAL_OFFSET + signal;
+ 	}
+ out:
+ 	if (c->lxc_conf->errmsg)
+@@ -515,6 +598,12 @@ int main(int argc, char *argv[])
+ 		}
+ 	}
+ 
++	if (my_args.uid != LXC_INVALID_UID)
++		attach_options.uid = my_args.uid;
++
++	if (my_args.gid != LXC_INVALID_GID)
++		attach_options.gid = my_args.gid;
++
+ 	attach_options.suffix = my_args.suffix;
+ 
+ 	/* isulad: add do attach background */
+-- 
+1.8.3.1
+

0026-remove-filelock-and-do-not-destroy-directory-when-de.patch

@@ -1,132 +0,0 @@
-From 238d4ac14b663aa5db904c55568c3710429d9bca Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Mon, 14 Jan 2019 21:13:00 +0800
-Subject: [PATCH 026/140] remove filelock and do not destroy directory when
- destroy container
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/lxccontainer.c | 14 ++++++++++++--
- src/lxc/lxclock.c      | 25 +++++++++++++++++++++++++
- src/lxc/lxclock.h      |  5 +++++
- src/lxc/storage/dir.c  | 12 +-----------
- 4 files changed, 43 insertions(+), 13 deletions(-)
-
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index 68134d8..81c0ec3 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -2996,8 +2996,14 @@ static bool container_destroy(struct lxc_container *c,
- 	bool bret = false;
- 	int ret = 0;
- 
--	if (!c || !do_lxcapi_is_defined(c))
-+	if (!c)
- 		return false;
-+	// isulad: if container is not defined, we need to remove disk lock file
-+	// which is created in lxc_container_new.
-+	if (!do_lxcapi_is_defined(c)) {
-+		container_disk_removelock(c);
-+		return false;
-+	}
- 
- 	conf = c->lxc_conf;
- 	if (container_disk_lock(c))
-@@ -3124,12 +3130,16 @@ out:
- 		free(path);
- 
- 	container_disk_unlock(c);
-+	if (bret) {
-+		if (container_disk_removelock(c))
-+			bret = false;
-+	}
- 	return bret;
- }
- 
- static bool do_lxcapi_destroy(struct lxc_container *c)
- {
--	if (!c || !lxcapi_is_defined(c))
-+	if (!c)
- 		return false;
- 
- 	if (has_snapshots(c)) {
-diff --git a/src/lxc/lxclock.c b/src/lxc/lxclock.c
-index e3d4654..8890968 100644
---- a/src/lxc/lxclock.c
-+++ b/src/lxc/lxclock.c
-@@ -198,6 +198,21 @@ on_error:
- 	return l;
- }
- 
-+static int lxc_removelock(struct lxc_lock *l)
-+{
-+	int ret = 0;
-+
-+	if (l->type == LXC_LOCK_FLOCK) {
-+		ret = unlink(l->u.f.fname);
-+		if (ret && errno != ENOENT) {
-+			SYSERROR("Error unlink %s", l->u.f.fname);
-+			return ret;
-+		}
-+	}
-+
-+	return ret;
-+}
-+
- int lxclock(struct lxc_lock *l, int timeout)
- {
- 	struct flock lk;
-@@ -386,3 +401,13 @@ void container_disk_unlock(struct lxc_container *c)
- 	lxcunlock(c->slock);
- 	lxcunlock(c->privlock);
- }
-+
-+int container_disk_removelock(struct lxc_container *c)
-+{
-+	int ret;
-+
-+	ret = lxc_removelock(c->slock);
-+	if (ret)
-+		return ret;
-+	return lxc_removelock(c->privlock);
-+}
-diff --git a/src/lxc/lxclock.h b/src/lxc/lxclock.h
-index 364a71b..e86bc34 100644
---- a/src/lxc/lxclock.h
-+++ b/src/lxc/lxclock.h
-@@ -174,4 +174,9 @@ extern int container_disk_lock(struct lxc_container *c);
-  */
- extern void container_disk_unlock(struct lxc_container *c);
- 
-+/*!
-+ * \brief isulad: remove the containers disk lock file.
-+ */
-+int container_disk_removelock(struct lxc_container *c);
-+
- #endif
-diff --git a/src/lxc/storage/dir.c b/src/lxc/storage/dir.c
-index c7b5ee2..deeecec 100644
---- a/src/lxc/storage/dir.c
-+++ b/src/lxc/storage/dir.c
-@@ -123,17 +123,7 @@ int dir_create(struct lxc_storage *bdev, const char *dest, const char *n,
- 
- int dir_destroy(struct lxc_storage *orig)
- {
--	int ret;
--	const char *src;
--
--	src = lxc_storage_get_path(orig->src, orig->src);
--
--	ret = lxc_rmdir_onedev(src, NULL);
--	if (ret < 0) {
--		ERROR("Failed to delete \"%s\"", src);
--		return -1;
--	}
--
-+	// isulad: do not destroy rootfs for directory, it should be managed by caller
- 	return 0;
- }
- 
--- 
-1.8.3.1
-

0027-Capabilites-security-feature-enhanced.patch

@@ -0,0 +1,240 @@
+From e71dabf21ddd2a093ebdfcc6f6c79200415d12b1 Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Tue, 14 Apr 2020 23:30:46 -0400
+Subject: [PATCH 27/49] Capabilites security feature enhanced
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/attach.c |  24 +++++++++++++
+ src/lxc/conf.c   | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ src/lxc/conf.h   |   1 +
+ src/lxc/start.c  |  20 +++++++++++
+ 4 files changed, 148 insertions(+)
+
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index c77b929..231fa5f 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -832,10 +832,12 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 			goto on_error;
+ 	}
+ 
++#ifndef HAVE_ISULAD
+ 	close(payload->ipc_socket);
+ 	payload->ipc_socket = -EBADF;
+ 	lxc_proc_put_context_info(init_ctx);
+ 	payload->init_ctx = NULL;
++#endif
+ 
+ 	/* The following is done after the communication socket is shut down.
+ 	 * That way, all errors that might (though unlikely) occur up until this
+@@ -895,6 +897,11 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 		new_gid = LXC_INVALID_GID;
+ 
+ #ifdef HAVE_ISULAD
++	if (prctl(PR_SET_KEEPCAPS, 1) < 0) {
++		SYSERROR("Failed to keep permitted capabilities");
++		goto on_error;
++	}
++
+ 	if (!lxc_setgroups(init_ctx->container->lxc_conf->init_groups_len,
+ 			init_ctx->container->lxc_conf->init_groups))
+ 		goto on_error;
+@@ -908,6 +915,23 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 	if (!lxc_switch_uid_gid(new_uid, new_gid))
+ 		goto on_error;
+ 
++#ifdef HAVE_ISULAD
++	if (prctl(PR_SET_KEEPCAPS, 0) < 0) {
++		SYSERROR("Failed to clear permitted capabilities");
++		goto on_error;
++	}
++
++	if (lxc_drop_caps(init_ctx->container->lxc_conf) != 0) {
++		ERROR("Failed to drop caps.");
++		goto on_error;
++	}
++
++	close(payload->ipc_socket);
++	payload->ipc_socket = -EBADF;
++	lxc_proc_put_context_info(init_ctx);
++	payload->init_ctx = NULL;
++#endif
++
+ 	/* We're done, so we can now do whatever the user intended us to do. */
+ 	_exit(payload->exec_function(payload->exec_payload));
+ 
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index 43ef067..325e0c2 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -2902,6 +2902,16 @@ static int dropcaps_except(struct lxc_list *caps)
+ 	lxc_list_for_each (iterator, caps) {
+ 		keep_entry = iterator->elem;
+ 
++#ifdef HAVE_ISULAD
++		/* Do not keep any cap*/
++		if (strcmp(keep_entry, "ISULAD_KEEP_NONE") == 0) {
++			DEBUG("Do not keep any capability");
++			for(i = 0; i < numcaps; i++) {
++				caplist[i] = 0;
++			}
++			break;
++		}
++#endif
+ 		capid = parse_cap(keep_entry);
+ 		if (capid == -2)
+ 			continue;
+@@ -4703,6 +4713,99 @@ int lxc_setup(struct lxc_handler *handler)
+ 	return 0;
+ }
+ 
++#ifdef HAVE_ISULAD
++/* isulad drop caps for container*/
++int lxc_drop_caps(struct lxc_conf *conf)
++{
++#define __DEF_CAP_TO_MASK(x) (1U << ((x) & 31))
++#if HAVE_LIBCAP
++	int ret = 0;
++	struct lxc_list *iterator = NULL;
++	char *keep_entry = NULL;
++	size_t i = 0;
++	int capid;
++	size_t numcaps = (size_t)lxc_caps_last_cap() + 1;
++	struct lxc_list *caps = NULL;
++	int *caplist = NULL;
++
++	if (lxc_list_empty(&conf->keepcaps))
++		return 0;
++
++	caps = &conf->keepcaps;
++
++	if (numcaps <= 0 || numcaps > 200)
++		return -1;
++
++	// caplist[i] is 1 if we keep capability i
++	caplist = malloc(numcaps * sizeof(int));
++	if (caplist == NULL) {
++		ERROR("Out of memory");
++		return -1;
++	}
++	(void)memset(caplist, 0, numcaps * sizeof(int));
++
++	lxc_list_for_each(iterator, caps) {
++
++		keep_entry = iterator->elem;
++		/* isulad: Do not keep any cap*/
++		if (strcmp(keep_entry, "ISULAD_KEEP_NONE") == 0) {
++			DEBUG("Do not keep any capability");
++			for(i = 0; i < numcaps; i++) {
++				caplist[i] = 0;
++			}
++			break;
++		}
++
++		capid = parse_cap(keep_entry);
++
++		if (capid == -2)
++			continue;
++
++		if (capid < 0) {
++			ERROR("unknown capability %s", keep_entry);
++			ret = -1;
++			goto out;
++		}
++
++		DEBUG("keep capability '%s' (%d)", keep_entry, capid);
++
++		caplist[capid] = 1;
++	}
++
++	struct __user_cap_header_struct cap_header_data;
++	struct __user_cap_data_struct cap_data_data[2];
++
++	cap_user_header_t cap_header = &cap_header_data;
++	cap_user_data_t cap_data = &cap_data_data[0];
++
++	memset(cap_header, 0,sizeof(struct __user_cap_header_struct));
++	memset(cap_data, 0, sizeof(struct __user_cap_data_struct) * 2);
++
++	cap_header->pid = 0;
++	cap_header->version = _LINUX_CAPABILITY_VERSION_3;
++
++	for (i = 0; i < numcaps; i++) {
++		if (caplist[i]) {
++			cap_data[CAP_TO_INDEX(i)].effective = cap_data[CAP_TO_INDEX(i)].effective | (i > 31 ? __DEF_CAP_TO_MASK(i % 32) : __DEF_CAP_TO_MASK(i));
++			cap_data[CAP_TO_INDEX(i)].permitted = cap_data[CAP_TO_INDEX(i)].permitted | (i > 31 ? __DEF_CAP_TO_MASK(i % 32) : __DEF_CAP_TO_MASK(i));
++			cap_data[CAP_TO_INDEX(i)].inheritable = cap_data[CAP_TO_INDEX(i)].inheritable | (i > 31 ? __DEF_CAP_TO_MASK(i % 32) : __DEF_CAP_TO_MASK(i));
++		}
++	}
++
++	if (capset(cap_header, cap_data)) {
++		SYSERROR("Failed to set capabilitys");
++		ret = -1;
++		goto out;
++	}
++
++#endif
++
++out:
++	free(caplist);
++	return ret;
++}
++#endif
++
+ int run_lxc_hooks(const char *name, char *hookname, struct lxc_conf *conf,
+ 		  char *argv[])
+ {
+diff --git a/src/lxc/conf.h b/src/lxc/conf.h
+index 879e427..7b6fd3b 100644
+--- a/src/lxc/conf.h
++++ b/src/lxc/conf.h
+@@ -555,6 +555,7 @@ int lxc_clear_init_groups(struct lxc_conf *lxc_conf);
+ int lxc_clear_populate_devices(struct lxc_conf *c);
+ int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
+ int lxc_clear_rootfs_ro_paths(struct lxc_conf *c);
++int lxc_drop_caps(struct lxc_conf *conf);
+ int run_oci_hooks(const char *name, const char *hookname, struct lxc_conf *conf, const char *lxcpath);
+ #endif
+ #endif /* __LXC_CONF_H */
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index e2311ec..bb2e74a 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -1652,6 +1652,13 @@ static int do_start(void *data)
+ 		}
+ 	}
+ 
++#ifdef HAVE_ISULAD
++	if (prctl(PR_SET_KEEPCAPS, 1) < 0) {
++		SYSERROR("Failed to keep permitted capabilities");
++		goto out_warn_father;
++	}
++#endif
++
+ 	/* The container has been setup. We can now switch to an unprivileged
+ 	 * uid/gid.
+ 	 */
+@@ -1705,6 +1712,19 @@ static int do_start(void *data)
+ 		goto out_warn_father;
+ 	}
+ 
++#ifdef HAVE_ISULAD
++	/* isulad: drop the cap of current process */
++	if (prctl(PR_SET_KEEPCAPS, 0) < 0) {
++		SYSERROR("Failed to clear permitted capabilities");
++		goto out_warn_father;
++	}
++
++	if (lxc_drop_caps(handler->conf)) {
++		SYSERROR("Failed to drop caps");
++		goto out_warn_father;
++	}
++#endif
++
+ 	if (handler->conf->monitor_signal_pdeath != SIGKILL) {
+ 		ret = lxc_set_death_signal(handler->conf->monitor_signal_pdeath,
+ 					   handler->monitor_pid, status_fd);
+-- 
+1.8.3.1
+

0027-fix-bug-of-memory-leak.patch

@@ -1,78 +0,0 @@
-From 9b7c539dec8af7c01abbcba4333b5f14e23c3b3b Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Mon, 14 Jan 2019 21:24:25 +0800
-Subject: [PATCH 027/140] fix bug of memory leak
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c         | 11 +++++++++++
- src/lxc/conf.h         |  1 +
- src/lxc/lxccontainer.c | 11 +++++++----
- 3 files changed, 19 insertions(+), 4 deletions(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 6a14de1..e076bf2 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -4822,6 +4822,16 @@ int lxc_clear_procs(struct lxc_conf *c, const char *key)
- 	return 0;
- }
- 
-+int lxc_clear_namespace(struct lxc_conf *c)
-+{
-+	int i;
-+	for (i = 0; i < LXC_NS_MAX; i++) {
-+		free(c->ns_share[i]);
-+		c->ns_share[i] = NULL;
-+	}
-+	return 0;
-+}
-+
- int lxc_clear_groups(struct lxc_conf *c)
- {
- 	struct lxc_list *it, *next;
-@@ -5036,6 +5046,7 @@ void lxc_conf_free(struct lxc_conf *conf)
- 	lxc_clear_limits(conf, "lxc.prlimit");
- 	lxc_clear_sysctls(conf, "lxc.sysctl");
- 	lxc_clear_procs(conf, "lxc.proc");
-+	lxc_clear_namespace(conf);
- 	free(conf->cgroup_meta.dir);
- 	free(conf->cgroup_meta.controllers);
- 	/* isulad add begin */
-diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index 2263e47..44feb98 100644
---- a/src/lxc/conf.h
-+++ b/src/lxc/conf.h
-@@ -491,6 +491,7 @@ extern int setup_sysctl_parameters(struct lxc_list *sysctls);
- extern int lxc_clear_sysctls(struct lxc_conf *c, const char *key);
- extern int setup_proc_filesystem(struct lxc_list *procs, pid_t pid);
- extern int lxc_clear_procs(struct lxc_conf *c, const char *key);
-+extern int lxc_clear_namespace(struct lxc_conf *c);
- 
- /* isulad add begin */
- int lxc_clear_init_args(struct lxc_conf *lxc_conf);
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index 81c0ec3..e6272fc 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -5418,10 +5418,13 @@ int list_active_containers(const char *lxcpath, char ***nret,
- 				continue;
- 		}
- 
--		if (array_contains(&ct_name, p, ct_name_cnt)) {
--			if (is_hashed)
--				free(p);
--			continue;
-+
-+		if (ct_name && ct_name_cnt) {
-+			if (array_contains(&ct_name, p, ct_name_cnt)) {
-+				if (is_hashed)
-+					free(p);
-+				continue;
-+			}
- 		}
- 
- 		if (!add_to_array(&ct_name, p, ct_name_cnt)) {
--- 
-1.8.3.1
-

0028-Supporting-workdir-configuration.patch

@@ -1,95 +1,106 @@
-From 193bc24b1974e5149cf751ae9c50383b9c1a3999 Mon Sep 17 00:00:00 2001
+From cbe77bd42528e92d9e3871a36133a2a11f5a3f21 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
+From: wujing <wujing50@huawei.com>
-Date: Tue, 15 Jan 2019 20:39:11 +0800
+Date: Wed, 15 Apr 2020 00:28:40 -0400
-Subject: [PATCH 034/140] some small bugfix
+Subject: [PATCH 28/49] Supporting workdir configuration
 
-1. support new container without load config to save time
+Signed-off-by: wujing <wujing50@huawei.com>
-2. try to create workdir if not exist
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
 ---
- src/lxc/attach.c       | 16 ++++++++++++++++
+ src/lxc/attach.c       | 18 ++++++++++++++++++
- src/lxc/lxc.h          |  5 +++++
+ src/lxc/lxc.h          |  7 +++++++
- src/lxc/lxccontainer.c | 21 +++++++++++++++++----
+ src/lxc/lxccontainer.c | 26 +++++++++++++++++++++++++-
- src/lxc/lxccontainer.h | 12 ++++++++++++
+ src/lxc/lxccontainer.h | 14 ++++++++++++++
- src/lxc/start.c        | 12 ++++++++----
+ src/lxc/start.c        | 10 ++++++++++
- 5 files changed, 58 insertions(+), 8 deletions(-)
+ 5 files changed, 74 insertions(+), 1 deletion(-)
 
 diff --git a/src/lxc/attach.c b/src/lxc/attach.c
-index e6e4b0d..8cbbf96 100644
+index 231fa5f..cb480ed 100644
 --- a/src/lxc/attach.c
 +++ b/src/lxc/attach.c
-@@ -806,6 +806,22 @@ static int attach_child_main(struct attach_clone_payload *payload)
+@@ -708,6 +708,24 @@ static int attach_child_main(struct attach_clone_payload *payload)
  		TRACE("Dropped capabilities");
  	}
  
++#ifdef HAVE_ISULAD
 +	/* isulad: set workdir */
-+	if (init_ctx && init_ctx->container && init_ctx->container->lxc_conf && init_ctx->container->lxc_conf->init_cwd) {
++	if (init_ctx->container->lxc_conf->init_cwd) {
 +		char *init_cwd;
 +		init_cwd = init_ctx->container->lxc_conf->init_cwd;
-+	        /* try to create workdir if not exist */
++		/* try to create workdir if not exist */
-+	        struct stat st;
++		struct stat st;
-+	        if (stat(init_cwd, &st) < 0 && mkdir_p(init_cwd, 0750) < 0) {
++		if (stat(init_cwd, &st) < 0 && mkdir_p(init_cwd, 0750) < 0) {
 +			SYSERROR("Try to create directory \"%s\" as workdir failed when attach", init_cwd);
 +			goto on_error;
-+	        }
++		}
-+	        if (chdir(init_cwd)) {
++		if (chdir(init_cwd)) {
 +			SYSERROR("Could not change directory to \"%s\" when attach", init_cwd);
 +			goto on_error;
-+	        }
++		}
 +	}
++#endif
 +
  	/* Always set the environment (specify (LXC_ATTACH_KEEP_ENV, NULL, NULL)
  	 * if you want this to be a no-op).
  	 */
 diff --git a/src/lxc/lxc.h b/src/lxc/lxc.h
-index 687b4b2..5df5080 100644
+index 630eff0..99fd422 100644
 --- a/src/lxc/lxc.h
 +++ b/src/lxc/lxc.h
-@@ -104,6 +104,11 @@ extern lxc_state_t lxc_state(const char *name, const char *lxcpath);
+@@ -83,6 +83,13 @@ extern lxc_state_t lxc_state(const char *name, const char *lxcpath);
+  */
  extern struct lxc_container *lxc_container_new(const char *name, const char *configpath);
  
- /*
++#ifdef HAVE_ISULAD
++/*
 + * Create a new container without loading config.
 + */
 +extern struct lxc_container *lxc_container_without_config_new(const char *name, const char *configpath);
++#endif
 +
-+/*
+ /*
   * Returns 1 on success, 0 on failure.
   */
- extern int lxc_container_get(struct lxc_container *c);
 diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index 38059fa..e99c41c 100644
+index 9b3ab75..ce2b2bf 100644
 --- a/src/lxc/lxccontainer.c
 +++ b/src/lxc/lxccontainer.c
-@@ -5133,7 +5133,7 @@ static bool do_lxcapi_clean_container_resource(struct lxc_container *c, pid_t pi
+@@ -5595,7 +5595,11 @@ static bool do_lxcapi_get_container_pids(struct lxc_container *c, pid_t **pids,s
+ WRAP_API_2(bool, lxcapi_get_container_pids, pid_t **,size_t *)
+ #endif
  
- WRAP_API_1(bool, lxcapi_clean_container_resource, pid_t)
++#ifdef HAVE_ISULAD
- 
--struct lxc_container *lxc_container_new(const char *name, const char *configpath)
 +static struct lxc_container *do_lxc_container_new(const char *name, const char *configpath, bool load_config)
++#else
+ struct lxc_container *lxc_container_new(const char *name, const char *configpath)
++#endif
  {
  	struct lxc_container *c;
  	size_t len;
-@@ -5190,9 +5190,11 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
+@@ -5653,12 +5657,19 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
+ 		fprintf(stderr, "Error allocating oci hooks file pathname\n");
  		goto err;
  	}
+-#endif
  
--	if (file_exists(c->configfile) && !lxcapi_load_config(c, NULL)) {
++	if (load_config && file_exists(c->configfile)) {
--		fprintf(stderr, "Failed to load config for %s\n", name);
++		if (!lxcapi_load_config(c, NULL)) {
--		goto err;
-+	if (load_config) {
-+		if (file_exists(c->configfile) && !lxcapi_load_config(c, NULL)) {
 +			fprintf(stderr, "Failed to load config for %s\n", name);
 +			goto err;
 +		}
++	}
++#else
+ 	if (file_exists(c->configfile) && !lxcapi_load_config(c, NULL)) {
+ 		fprintf(stderr, "Failed to load config for %s\n", name);
+ 		goto err;
  	}
++#endif
  
- 	if (ongoing_create(c) == 2) {
+ 	rc = ongoing_create(c);
-@@ -5274,6 +5276,17 @@ err:
+ 	switch (rc) {
+@@ -5761,6 +5772,19 @@ err:
  	return NULL;
  }
  
++#ifdef HAVE_ISULAD
 +// isulad: new container without load config to save time
 +struct lxc_container *lxc_container_without_config_new(const char *name, const char *configpath)
 +{
@@ -100,18 +111,21 @@ index 38059fa..e99c41c 100644
 +{
 +	return do_lxc_container_new(name, configpath, true);
 +}
++#endif
 +
  int lxc_get_wait_states(const char **states)
  {
  	int i;
 diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
-index 679ca42..a00e0ec 100644
+index f1621f9..e69be8f 100644
 --- a/src/lxc/lxccontainer.h
 +++ b/src/lxc/lxccontainer.h
-@@ -1040,6 +1040,18 @@ struct lxc_console_log {
+@@ -1097,6 +1097,20 @@ struct lxc_console_log {
+  */
  struct lxc_container *lxc_container_new(const char *name, const char *configpath);
  
- /*!
++#ifdef HAVE_ISULAD
++/*!
 + * \brief Create a new container without loading config.
 + *
 + * \param name Name to use for container.
@@ -122,34 +136,32 @@ index 679ca42..a00e0ec 100644
 + * \note This function can only used for listing container.
 + */
 +struct lxc_container *lxc_container_without_config_new(const char *name, const char *configpath);
++#endif
 +
-+/*!
+ /*!
   * \brief Add a reference to the specified container.
   *
-  * \param c Container.
 diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 08d753a..040909c 100644
+index bb2e74a..70ce1bd 100644
 --- a/src/lxc/start.c
 +++ b/src/lxc/start.c
-@@ -1364,10 +1364,14 @@ static int do_start(void *data)
+@@ -1609,6 +1609,16 @@ static int do_start(void *data)
- 	setsid();
+ 	close_prot_errno_disarm(devnull_fd);
  
  	if (handler->conf->init_cwd) {
--		ret = chdir(handler->conf->init_cwd);
++#ifdef HAVE_ISULAD
--		if (ret < 0) {
++		/* try to craete workdir if not exist */
--			SYSERROR("Could not change directory to \"%s\"",
--				 handler->conf->init_cwd);
-+		/* isulad: try to craete workdir if not exist */
 +		struct stat st;
 +		if (stat(handler->conf->init_cwd, &st) < 0 && mkdir_p(handler->conf->init_cwd, 0755) < 0) {
 +			SYSERROR("Try to create directory \"%s\" as workdir failed", handler->conf->init_cwd);
++			lxc_write_error_message(handler->conf->errpipe[1], "%s:%d: Failed to create workdir: %s.",
++			                        __FILE__, __LINE__, strerror(errno));
 +			goto out_warn_father;
 +		}
-+		if (chdir(handler->conf->init_cwd)) {
++#endif
-+			SYSERROR("Could not change directory to \"%s\"", handler->conf->init_cwd);
+ 		ret = chdir(handler->conf->init_cwd);
- 			goto out_warn_father;
+ 		if (ret < 0) {
- 		}
+ 			SYSERROR("Could not change directory to \"%s\"",
- 	}
 -- 
 1.8.3.1
 

0028-support-rootfs-for-container.patch

@@ -1,123 +0,0 @@
-From 0d83272984a667bcd530553ad4fb7ca0805ceb59 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Tue, 15 Jan 2019 09:45:44 +0800
-Subject: [PATCH 028/140] support rootfs / for container
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++--------
- 1 file changed, 53 insertions(+), 8 deletions(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index e076bf2..f429491 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -888,9 +888,6 @@ static int lxc_setup_ttys(struct lxc_conf *conf)
- 	char *ttydir = ttys->dir;
- 	char path[PATH_MAX], lxcpath[PATH_MAX];
- 
--	if (!conf->rootfs.path)
--		return 0;
--
- 	for (i = 0; i < ttys->max; i++) {
- 		struct lxc_terminal_info *tty = &ttys->tty[i];
- 
-@@ -1394,7 +1391,7 @@ static int lxc_mount_rootfs(struct lxc_conf *conf)
- {
- 	int ret;
- 	struct lxc_storage *bdev;
--	const struct lxc_rootfs *rootfs = &conf->rootfs;
-+	struct lxc_rootfs *rootfs = &conf->rootfs;
- 	unsigned long flags, mntflags, pflags;
- 	char *mntdata;
- 
-@@ -1405,6 +1402,17 @@ static int lxc_mount_rootfs(struct lxc_conf *conf)
- 			return -1;
- 		}
- 
-+		// isulad: bind mount / to rootfs.mount. then we can do pivot root even if we use / as root.
-+		if (!access(rootfs->mount, F_OK)) {
-+			rootfs->path = strdup("/");
-+			if (mount("/", rootfs->mount, NULL, MS_BIND, 0)) {
-+				SYSERROR("Failed to mount / to %s.", rootfs->mount);
-+				return -1;
-+			}
-+		} else {
-+			INFO("Use '/' as container rootfs, but no valid mountpoint provided. Something may go wrong.");
-+		}
-+
- 		return 0;
- 	}
- 
-@@ -3854,6 +3862,35 @@ static int setup_populate_devs(const struct lxc_rootfs *rootfs, struct lxc_list
- 	return 0;
- }
- 
-+// isulad: setup rootfs mountopts
-+static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs)
-+{
-+	unsigned long mntflags, pflags;
-+	char *mntdata;
-+
-+	// only remount / when container shares rootfs with host.
-+	if(!rootfs || !rootfs->path || strcmp(rootfs->path, "/"))
-+		return 0;
-+	if (!rootfs->options)
-+		return 0;
-+
-+	if (parse_mntopts(rootfs->options, &mntflags, &pflags, &mntdata) < 0) {
-+		free(mntdata);
-+		return -1;
-+	}
-+	free(mntdata);
-+
-+	if (mntflags & MS_RDONLY) {
-+		DEBUG("remounting / as readonly");
-+		if (mount("/", "/", NULL, MS_BIND |MS_REMOUNT| MS_RDONLY, 0)) {
-+			SYSERROR("Failed to make / readonly.");
-+			return -1;
-+		}
-+	}
-+	return 0;
-+}
-+
-+
- int lxc_setup(struct lxc_handler *handler)
- {
- 	int ret;
-@@ -4020,12 +4057,20 @@ int lxc_setup(struct lxc_handler *handler)
- 		return -1;
- 	}
- 
--	ret = lxc_setup_devpts(lxc_conf);
--	if (ret < 0) {
--		ERROR("Failed to setup new devpts instance");
-+	/* isulad: remount rootfs readonly if necessary */
-+	if (setup_rootfs_mountopts(&lxc_conf->rootfs)) {
-+		ERROR("failed to set rootfs for '%s'", name);
- 		return -1;
- 	}
- 
-+	if (lxc_conf->rootfs.path) {
-+		ret = lxc_setup_devpts(lxc_conf);
-+		if (ret < 0) {
-+			ERROR("Failed to setup new devpts instance");
-+			return -1;
-+		}
-+	}
-+
- 	ret = lxc_create_ttys(handler);
- 	if (ret < 0)
- 		return -1;
-@@ -4184,7 +4229,7 @@ static char **merge_ocihook_env(char **oldenvs, size_t env_len, size_t *merge_en
- 	return result;
- }
- 
--static struct lxc_popen_FILE *lxc_popen_ocihook(char *commandpath, char **args, int args_len, 
-+static struct lxc_popen_FILE *lxc_popen_ocihook(char *commandpath, char **args, int args_len,
- 				char **envs, int env_len, const char *instr)
- {
- 	int ret;
--- 
-1.8.3.1
-

0029-Supporting-additional-groups-configuration.patch

@@ -0,0 +1,103 @@
+From f25dd7358ea454e78c41094e79764ea9e09f28e7 Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Wed, 15 Apr 2020 03:57:20 -0400
+Subject: [PATCH 29/49] Supporting additional groups configuration
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/confile.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 65 insertions(+)
+
+diff --git a/src/lxc/confile.c b/src/lxc/confile.c
+index 9ba3c7c..55cba6d 100644
+--- a/src/lxc/confile.c
++++ b/src/lxc/confile.c
+@@ -149,6 +149,7 @@ lxc_config_define(sysctl);
+ lxc_config_define(proc);
+ #ifdef HAVE_ISULAD
+ lxc_config_define(init_args);
++lxc_config_define(init_groups);
+ lxc_config_define(populate_device);
+ lxc_config_define(umask);
+ lxc_config_define(rootfs_masked_paths);
+@@ -268,6 +269,7 @@ static struct lxc_config_t config_jump_table[] = {
+ 	{ "lxc.proc",                      set_config_proc,                        get_config_proc,                        clr_config_proc,                      },
+ #ifdef HAVE_ISULAD
+ 	{ "lxc.isulad.init.args",          set_config_init_args,                   get_config_init_args,                   clr_config_init_args,                 },
++	{ "lxc.isulad.init.groups",        set_config_init_groups,                 get_config_init_groups,                 clr_config_init_groups,               },
+ 	{ "lxc.isulad.populate.device",    set_config_populate_device,             get_config_populate_device,             clr_config_populate_device,           },
+ 	{ "lxc.isulad.umask",              set_config_umask,                       get_config_umask,                       clr_config_umask,                     },
+ 	{ "lxc.isulad.rootfs.maskedpaths", set_config_rootfs_masked_paths,         get_config_rootfs_masked_paths,         clr_config_rootfs_masked_paths,       },
+@@ -6247,6 +6249,69 @@ static inline int clr_config_init_args(const char *key, struct lxc_conf *c,
+ 	return lxc_clear_init_args(c);
+ }
+ 
++/* isulad: set config for init groups */
++static int set_config_init_groups(const char *key, const char *value,
++                                  struct lxc_conf *lxc_conf, void *data)
++{
++	char *groups = NULL;
++	char *token = NULL;
++	int ret = -1;
++
++	if (lxc_config_value_empty(value))
++		return lxc_clear_init_groups(lxc_conf);
++
++	groups = strdup(value);
++	if (!groups)
++		return -1;
++
++	/* In case several capability keep is specified in a single line
++	 * split these caps in a single element for the list.
++	 */
++	lxc_iterate_parts(token, groups, " \t") {
++		gid_t *tmp = NULL;
++		if (lxc_mem_realloc((void **)&tmp, (lxc_conf->init_groups_len + 1) * sizeof(gid_t), lxc_conf->init_groups,
++		                    (lxc_conf->init_groups_len) * sizeof(gid_t)) != 0) {
++			ERROR("Out of memory");
++			goto on_error;
++		}
++		lxc_conf->init_groups = tmp;
++		tmp[lxc_conf->init_groups_len] = atoll(token);
++		lxc_conf->init_groups_len++;
++	}
++
++	ret = 0;
++
++on_error:
++	free(groups);
++
++	return ret;
++}
++
++/* isulad: get config init groups */
++static int get_config_init_groups(const char *key, char *retv, int inlen,
++                                  struct lxc_conf *c, void *data)
++{
++	int i, len, fulllen = 0;
++
++	if (!retv)
++		inlen = 0;
++	else
++		memset(retv, 0, inlen);
++
++	for (i = 0; i < c->init_groups_len; i++) {
++		strprint(retv, inlen, "%u\n", c->init_groups[i]);
++	}
++
++	return fulllen;
++}
++
++/* isulad: clr config init args*/
++static inline int clr_config_init_groups(const char *key, struct lxc_conf *c,
++                void *data)
++{
++	return lxc_clear_init_groups(c);
++}
++
+ /* isulad: set config for populate device */
+ static int set_config_populate_device(const char *key, const char *value,
+                                       struct lxc_conf *lxc_conf, void *data)
+-- 
+1.8.3.1
+

0029-add-start-timeout-to-limit-start-time.patch

@@ -1,436 +0,0 @@
-From be744bf5f224a4a14790c8f6a37c5335590daaf6 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Mon, 14 Jan 2019 21:38:07 -0500
-Subject: [PATCH 029/140] add start timeout to limit start time
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c            |   6 +--
- src/lxc/execute.c         |   4 +-
- src/lxc/lxc.h             |   8 ++--
- src/lxc/lxccontainer.c    |  21 ++++++++-
- src/lxc/lxccontainer.h    |  16 +++++++
- src/lxc/start.c           | 106 +++++++++++++++++++++++++++++++++++++++++++---
- src/lxc/start.h           |   6 +--
- src/lxc/tools/arguments.h |   2 +
- src/lxc/tools/lxc_start.c |  26 ++++++++++++
- 9 files changed, 175 insertions(+), 20 deletions(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index f429491..439353b 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -4381,14 +4381,12 @@ void* wait_ocihook_timeout(void *arg)
- 
- 	if (alive) {
- 		ERROR("%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\"",
--			__FILE__, __LINE__,
--			(conf->which == LXCHOOK_START_HOST) ? "prestart" : lxchook_names[conf->which],
-+			__FILE__, __LINE__, lxchook_names[conf->which],
- 			(double)conf->timeout);
- 
- 		if (conf->errfd >= 0) {
- 			lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\"",
--				__FILE__, __LINE__,
--				(conf->which == LXCHOOK_START_HOST) ? "prestart" : lxchook_names[conf->which],
-+				__FILE__, __LINE__, lxchook_names[conf->which],
- 				(double)conf->timeout);
- 		}
- 
-diff --git a/src/lxc/execute.c b/src/lxc/execute.c
-index 45ca67e..d388e63 100644
---- a/src/lxc/execute.c
-+++ b/src/lxc/execute.c
-@@ -111,12 +111,12 @@ static struct lxc_operations execute_start_ops = {
- 
- int lxc_execute(const char *name, char *const argv[], int quiet,
- 		struct lxc_handler *handler, const char *lxcpath,
--		bool daemonize, int *error_num)
-+		bool daemonize, int *error_num, unsigned int start_timeout)
- {
- 	struct execute_args args = {.argv = argv, .quiet = quiet};
- 
- 	TRACE("Doing lxc_execute");
- 	handler->conf->is_execute = true;
- 	return __lxc_start(name, handler, &execute_start_ops, &args, lxcpath,
--			   daemonize, error_num);
-+			   daemonize, error_num, start_timeout);
- }
-diff --git a/src/lxc/lxc.h b/src/lxc/lxc.h
-index 22e3509..687b4b2 100644
---- a/src/lxc/lxc.h
-+++ b/src/lxc/lxc.h
-@@ -55,7 +55,7 @@ struct lxc_handler;
-  */
- extern int lxc_start(const char *name, char *const argv[],
- 		     struct lxc_handler *handler, const char *lxcpath,
--		     bool daemonize, int *error_num);
-+		     bool daemonize, int *error_num, unsigned int start_timeout);
- 
- /*
-  * Start the specified command inside an application container
-@@ -66,9 +66,9 @@ extern int lxc_start(const char *name, char *const argv[],
-  * @daemonize    : whether or not the container is daemonized
-  * Returns 0 on success, < 0 otherwise
-  */
--extern int lxc_execute(const char *name, char *const argv[], int quiet,
--		       struct lxc_handler *handler, const char *lxcpath,
--		       bool daemonize, int *error_num);
-+int lxc_execute(const char *name, char *const argv[], int quiet,
-+		struct lxc_handler *handler, const char *lxcpath,
-+		bool daemonize, int *error_num, unsigned int start_timeout);
- 
- /*
-  * Close the fd associated with the monitoring
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index e6272fc..beae459 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -1169,10 +1169,10 @@ reboot:
- 
- 	if (useinit)
- 		ret = lxc_execute(c->name, argv, 1, handler, c->config_path,
--				  c->daemonize, &c->error_num);
-+				  c->daemonize, &c->error_num, c->start_timeout);
- 	else
- 		ret = lxc_start(c->name, argv, handler, c->config_path,
--				c->daemonize, &c->error_num);
-+				c->daemonize, &c->error_num, c->start_timeout);
- 
- 	if (conf->reboot == REBOOT_REQ) {
- 		INFO("Container requested reboot");
-@@ -5100,6 +5100,22 @@ static bool do_lxcapi_set_container_info_file(struct lxc_container *c, const cha
- 
- WRAP_API_1(bool, lxcapi_set_container_info_file, const char *)
- 
-+/* isulad add start timeout */
-+static bool do_lxcapi_set_start_timeout(struct lxc_container *c,  unsigned int start_timeout)
-+{
-+	if (!c || !c->lxc_conf)
-+		return false;
-+	if (container_mem_lock(c)) {
-+		ERROR("Error getting mem lock");
-+		return false;
-+	}
-+	c->start_timeout = start_timeout;
-+	container_mem_unlock(c);
-+	return true;
-+}
-+
-+WRAP_API_1(bool, lxcapi_set_start_timeout, unsigned int)
-+
- struct lxc_container *lxc_container_new(const char *name, const char *configpath)
- {
- 	struct lxc_container *c;
-@@ -5231,6 +5247,7 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
- 	/* isulad add begin */
- 	c->set_terminal_init_fifos = lxcapi_set_terminal_default_fifos;
- 	c->set_container_info_file = lxcapi_set_container_info_file;
-+	c->set_start_timeout = lxcapi_set_start_timeout;
- 	/* isulad add end */
- 	return c;
- 
-diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
-index 5d23cc7..77de704 100644
---- a/src/lxc/lxccontainer.h
-+++ b/src/lxc/lxccontainer.h
-@@ -94,6 +94,12 @@ struct lxc_container {
- 	 */
- 	char *exit_fifo;
- 
-+	/*! isulad:
-+	 * \private
-+	 * start_timeout.
-+	 */
-+	unsigned int start_timeout;
-+
- 	/*!
- 	 * \private
- 	 * Container semaphore lock.
-@@ -880,6 +886,16 @@ struct lxc_container {
- 	 * \return \c true on success, else \c false.
- 	 */
- 	bool (*set_container_info_file) (struct lxc_container *c, const char *info_file);
-+
-+	/*! isulad add
-+	 * \brief An API call to set start timeout
-+	 *
-+	 * \param c Container.
-+	 * \param start_timeout Value of start timeout.
-+	 *
-+	 * \return \c true on success, else \c false.
-+	 */
-+	bool (*set_start_timeout)(struct lxc_container *c, unsigned int start_timeout);
- };
- 
- /*!
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 63f5af8..f7be9e4 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -93,7 +93,22 @@ extern void mod_all_rdeps(struct lxc_container *c, bool inc);
- static bool do_destroy_container(struct lxc_handler *handler);
- static int lxc_rmdir_onedev_wrapper(void *data);
- static void lxc_destroy_container_on_signal(struct lxc_handler *handler,
--					    const char *name);
-+				const char *name);
-+
-+/* isulad: start timeout thread */
-+typedef enum {
-+	START_INIT,
-+	START_TIMEOUT,
-+	START_MAX,
-+} start_timeout_t;
-+
-+static start_timeout_t global_timeout_state = START_INIT;
-+static sem_t global_timeout_sem;
-+
-+struct start_timeout_conf {
-+	unsigned int timeout;
-+	int errfd;
-+};
- 
- static void print_top_failing_dir(const char *path)
- {
-@@ -1897,6 +1912,12 @@ static int lxc_spawn(struct lxc_handler *handler)
- 		goto out_delete_net;
- 	}
- 
-+	if (START_TIMEOUT == global_timeout_state) {
-+		//lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
-+		ERROR("Starting the container \"%s\" timeout.", name);
-+		goto out_delete_net;
-+	}
-+
- 	/* Tell the child to complete its initialization and wait for it to exec
- 	 * or return an error. (The child will never return
- 	 * LXC_SYNC_READY_START+1. It will either close the sync pipe, causing
-@@ -1936,7 +1957,13 @@ static int lxc_spawn(struct lxc_handler *handler)
- 	ret = run_lxc_hooks(name, "oci-poststart", conf, oci_hook_args);
- 	if (ret < 0) {
- 		ERROR("Failed to run oci poststart hooks");
--		goto out_delete_net;
-+		goto out_abort;
-+	}
-+
-+	if (START_TIMEOUT == global_timeout_state) {
-+		//lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
-+		ERROR("Starting the container \"%s\" timeout.", name);
-+		goto out_abort;
- 	}
- 
- 	ret = lxc_set_state(name, handler, RUNNING);
-@@ -1964,12 +1991,71 @@ out_abort:
- 	return -1;
- }
- 
-+/* isulad: start timeout thread function */
-+static void* wait_start_timeout(void *arg)
-+{
-+	struct start_timeout_conf *conf = (struct start_timeout_conf *)arg;
-+
-+	sem_post(&global_timeout_sem);
-+
-+	if (!conf || conf->timeout < 1)
-+		goto out;
-+
-+	sleep(conf->timeout);
-+
-+	global_timeout_state = START_TIMEOUT;
-+
-+out:
-+	free(conf);
-+	return ((void *)0);
-+}
-+
-+/* isulad: create start timeout thread */
-+static int create_start_timeout_thread(struct lxc_conf *conf, unsigned int start_timeout)
-+{
-+	int ret = 0;
-+	pthread_t ptid;
-+	pthread_attr_t attr;
-+	struct start_timeout_conf *timeout_conf = NULL;
-+
-+	if (sem_init(&global_timeout_sem, 0, 0)) {
-+		ERROR("Failed to init start timeout semaphore");/*lint !e613*/
-+		ret = -1;
-+		return ret;
-+	}
-+
-+	timeout_conf = malloc(sizeof(struct start_timeout_conf));
-+	if (!timeout_conf) {
-+		ERROR("Failed to malloc start timeout conf");
-+		ret = -1;
-+		goto out;
-+	}
-+
-+	memset(timeout_conf, 0, sizeof(struct start_timeout_conf));
-+	timeout_conf->errfd = conf->errpipe[1];
-+	timeout_conf->timeout = start_timeout;
-+
-+	pthread_attr_init(&attr);
-+	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
-+	ret = pthread_create(&ptid, &attr, wait_start_timeout, timeout_conf);
-+	if (ret != 0) {
-+		ERROR("Create start wait timeout thread failed");
-+		free(timeout_conf);
-+		goto out;
-+	}
-+
-+	sem_wait(&global_timeout_sem);
-+out:
-+	sem_destroy(&global_timeout_sem);
-+	return ret;
-+}
-+
- // isulad: send '128 + signal' if container is killed by signal.
- #define ExitSignalOffset 128
- 
- int __lxc_start(const char *name, struct lxc_handler *handler,
- 		struct lxc_operations* ops, void *data, const char *lxcpath,
--		bool daemonize, int *error_num)
-+		bool daemonize, int *error_num, unsigned int start_timeout)
- {
- 	int ret, status, exit_code;
- 	struct lxc_conf *conf = handler->conf;
-@@ -1983,8 +2069,18 @@ int __lxc_start(const char *name, struct lxc_handler *handler,
- 	handler->data = data;
- 	handler->daemonize = daemonize;
- 
-+	/* isulad: add start timeout limit */
-+	if (start_timeout > 0) {
-+		ret = create_start_timeout_thread(conf, start_timeout);
-+		if (ret) {
-+			ERROR("Failed to create start timeout thread for container \"%s\".", name);
-+			goto out_fini_nonet;
-+		}
-+	}
-+
- 	if (!attach_block_device(handler->conf)) {
- 		ERROR("Failed to attach block device");
-+		ret = -1;
- 		goto out_fini_nonet;
- 	}
- 
-@@ -2114,14 +2210,14 @@ static struct lxc_operations start_ops = {
- };
- 
- int lxc_start(const char *name, char *const argv[], struct lxc_handler *handler,
--	      const char *lxcpath, bool daemonize, int *error_num)
-+	      const char *lxcpath, bool daemonize, int *error_num, unsigned int start_timeout)
- {
- 	struct start_args start_arg = {
- 		.argv = argv,
- 	};
- 
- 	TRACE("Doing lxc_start");
--	return __lxc_start(name, handler, &start_ops, &start_arg, lxcpath, daemonize, error_num);
-+	return __lxc_start(name, handler, &start_ops, &start_arg, lxcpath, daemonize, error_num, start_timeout);
- }
- 
- static void lxc_destroy_container_on_signal(struct lxc_handler *handler,
-diff --git a/src/lxc/start.h b/src/lxc/start.h
-index f59bf54..a96f2ae 100644
---- a/src/lxc/start.h
-+++ b/src/lxc/start.h
-@@ -174,9 +174,9 @@ extern void lxc_fini(const char *name, struct lxc_handler *handler);
-  */
- extern int lxc_check_inherited(struct lxc_conf *conf, bool closeall,
- 			       int *fds_to_ignore, size_t len_fds);
--extern int __lxc_start(const char *, struct lxc_handler *,
--		       struct lxc_operations *, void *, const char *, bool,
--		       int *);
-+extern int __lxc_start(const char *name, struct lxc_handler *handler,
-+		struct lxc_operations* ops, void *data, const char *lxcpath,
-+		bool daemonize, int *error_num, unsigned int start_timeout);
- 
- extern int resolve_clone_flags(struct lxc_handler *handler);
- 
-diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
-index 047e9f1..afab9f5 100644
---- a/src/lxc/tools/arguments.h
-+++ b/src/lxc/tools/arguments.h
-@@ -65,6 +65,7 @@ struct lxc_arguments {
- 	char *terminal_fifos[2]; /* isulad add, fifos used to redirct stdin/out/err */
- 	const char *container_info; /* isulad: file used to store pid and ppid info of container */
- 	const char *exit_monitor_fifo; /* isulad: fifo used to monitor state of monitor process */
-+	unsigned int start_timeout; /* isulad: Seconds for waiting on a container to start before it is killed*/
- 
- 	/* for lxc-console */
- 	unsigned int ttynum;
-@@ -180,6 +181,7 @@ struct lxc_arguments {
- #define OPT_OUTPUT_FIFO OPT_USAGE - 8
- #define OPT_CONTAINER_INFO OPT_USAGE - 9
- #define OPT_EXIT_FIFO OPT_USAGE - 10
-+#define OPT_START_TIMEOUT OPT_USAGE - 11
- /* isulad add end*/
- 
- extern int lxc_arguments_parse(struct lxc_arguments *args, int argc,
-diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
-index 60c7d70..f37f8a6 100644
---- a/src/lxc/tools/lxc_start.c
-+++ b/src/lxc/tools/lxc_start.c
-@@ -40,6 +40,7 @@
- #include <sys/types.h>
- #include <sys/utsname.h>
- #include <unistd.h>
-+#include <ctype.h>
- 
- #include <lxc/lxccontainer.h>
- 
-@@ -74,6 +75,7 @@ static const struct option my_longopts[] = {
- 	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
- 	{"container-pidfile", required_argument, 0, OPT_CONTAINER_INFO},
- 	{"exit-fifo", required_argument, 0, OPT_EXIT_FIFO},
-+	{"start-timeout", required_argument, 0, OPT_START_TIMEOUT},
- 	/* isulad add end */
- 	LXC_COMMON_OPTIONS
- };
-@@ -108,6 +110,18 @@ Options :\n\
- 	.pidfile      = NULL,
- };
- 
-+static bool is_non_negative_num(const char *s)
-+{
-+	if (!s || !strcmp(s, ""))
-+		return false;
-+	while(*s  != '\0') {
-+		if(!isdigit(*s))
-+			return false;
-+		++s;
-+	}
-+	return true;
-+}
-+
- static int my_parser(struct lxc_arguments *args, int c, char *arg)
- {
- 	switch (c) {
-@@ -158,6 +172,13 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
- 	case OPT_EXIT_FIFO:
- 		args->exit_monitor_fifo = arg;
- 		break;
-+	case OPT_START_TIMEOUT:
-+		if(!is_non_negative_num(arg)) {
-+			fprintf(stderr, "Error start timeout parameter:%s.\n", arg);
-+			return -1;
-+		}
-+		args->start_timeout = (unsigned int)atoi(arg);
-+		break;
- 	}
- 	return 0;
- }
-@@ -341,6 +362,11 @@ int main(int argc, char *argv[])
- 		c->exit_fifo = strdup(my_args.exit_monitor_fifo);
- 	}
- 
-+	/* isulad: add start timeout */
-+	if(my_args.start_timeout) {
-+		c->set_start_timeout(c, my_args.start_timeout);
-+	}
-+
- 	if (my_args.console)
- 		if (!c->set_config_item(c, "lxc.console.path", my_args.console))
- 			goto out;
--- 
-1.8.3.1
-

0030-set-negative-files.limit-value-to-max.patch

@@ -0,0 +1,71 @@
+From eef7e463b7d4bebc305d65c35288c9fab6a5486f Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Wed, 15 Apr 2020 05:45:49 -0400
+Subject: [PATCH 30/49] set negative files.limit value to max
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/cgroups/cgfsng.c  | 19 +++++++++++++++----
+ src/lxc/storage/storage.c |  8 +++++++-
+ 2 files changed, 22 insertions(+), 5 deletions(-)
+
+diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
+index 00270ab..b6aef12 100644
+--- a/src/lxc/cgroups/cgfsng.c
++++ b/src/lxc/cgroups/cgfsng.c
+@@ -3481,16 +3481,27 @@ __cgfsng_ops static bool cgfsng_setup_limits_legacy(struct cgroup_ops *ops,
+ 		cg = iterator->elem;
+ 
+ 		if (do_devices == !strncmp("devices", cg->subsystem, 7)) {
+-			if (isulad_cg_legacy_set_data(ops, cg->subsystem, cg->value)) {
++			const char *cgvalue = cg->value;
++			if (strcmp(cg->subsystem, "files.limit") == 0) {
++				if (lxc_safe_long_long(cgvalue, &setvalue) != 0) {
++					SYSERROR("Invalid integer value %s", cgvalue);
++					goto out;
++				}
++				if (setvalue <= 0) {
++					cgvalue = "max";
++				}
++			}
++			if (isulad_cg_legacy_set_data(ops, cg->subsystem, cgvalue)) {
+ 				if (do_devices && (errno == EACCES || errno == EPERM)) {
+-					SYSWARN("Failed to set \"%s\" to \"%s\"", cg->subsystem, cg->value);
++					SYSWARN("Failed to set \"%s\" to \"%s\"", cg->subsystem, cgvalue);
+ 					continue;
+ 				}
+-				SYSERROR("Failed to set \"%s\" to \"%s\"", cg->subsystem, cg->value);
++				SYSERROR("Failed to set \"%s\" to \"%s\"", cg->subsystem, cgvalue);
+ 				goto out;
+ 			}
+-			DEBUG("Set controller \"%s\" set to \"%s\"", cg->subsystem, cg->value);
++			DEBUG("Set controller \"%s\" set to \"%s\"", cg->subsystem, cgvalue);
+ 		}
++
+ 		// isulad: check cpu shares
+ 		if (strcmp(cg->subsystem, "cpu.shares") == 0) {
+ 			if (isulad_cg_legacy_get_data(ops, cg->subsystem, value, sizeof(value) - 1) < 0) {
+diff --git a/src/lxc/storage/storage.c b/src/lxc/storage/storage.c
+index 876311a..5291b24 100644
+--- a/src/lxc/storage/storage.c
++++ b/src/lxc/storage/storage.c
+@@ -591,9 +591,15 @@ bool storage_destroy(struct lxc_conf *conf)
+ 	int destroy_rv = 0;
+ 
+ 	r = storage_init(conf);
++#ifdef HAVE_ISULAD
++	if (r == NULL) {
++		WARN("%s 's storage init failed, the storage may be deleted already", conf->name);
++		return true;
++	}
++#else
+ 	if (!r)
+ 		return ret;
+-
++#endif
+ 	destroy_rv = r->ops->destroy(r);
+ 	if (destroy_rv == 0)
+ 		ret = true;
+-- 
+1.8.3.1
+

0031-clean-add-clean-resources-api.patch

@@ -1,488 +0,0 @@
-From 97a9081b91b573df9c381331fde2dc6ec7f257e6 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Tue, 15 Jan 2019 04:20:57 -0500
-Subject: [PATCH 031/140] clean: add clean resources api
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/cgroups/cgfsng.c | 124 +++++++++++++++++------------------------
- src/lxc/cgroups/cgroup.c |   2 +-
- src/lxc/cgroups/cgroup.h |   4 +-
- src/lxc/lxccontainer.c   |  18 ++++++
- src/lxc/lxccontainer.h   |  10 ++++
- src/lxc/start.c          | 141 +++++++++++++++++++++++++++++++++++++++++++++++
- src/lxc/start.h          |   4 ++
- 7 files changed, 228 insertions(+), 75 deletions(-)
-
-diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index 7f2a200..8b913a6 100644
---- a/src/lxc/cgroups/cgfsng.c
-+++ b/src/lxc/cgroups/cgfsng.c
-@@ -1050,12 +1050,15 @@ static int cgroup_rmdir(struct hierarchy **hierarchies,
- 		int ret;
- 		struct hierarchy *h = hierarchies[i];
- 
--		if (!h->container_full_path)
--			continue;
-+		if (!h->container_full_path) {
-+			h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, container_cgroup, NULL);
-+		}
- 
- 		ret = recursive_destroy(h->container_full_path);
--		if (ret < 0)
--			WARN("Failed to destroy \"%s\"", h->container_full_path);
-+		if (ret < 0) {
-+			ERROR("Failed to destroy \"%s\"", h->container_full_path);
-+			return -1;
-+		}
- 
- 		free(h->container_full_path);
- 		h->container_full_path = NULL;
-@@ -1102,7 +1105,8 @@ static int cgroup_rmdir_wrapper(void *data)
- 	return cgroup_rmdir(arg->hierarchies, arg->container_cgroup);
- }
- 
--__cgfsng_ops static void cgfsng_payload_destroy(struct cgroup_ops *ops,
-+/* isulad: fix return bool instead of void*/
-+__cgfsng_ops static bool cgfsng_payload_destroy(struct cgroup_ops *ops,
- 						struct lxc_handler *handler)
- {
- 	int ret;
-@@ -1113,6 +1117,8 @@ __cgfsng_ops static void cgfsng_payload_destroy(struct cgroup_ops *ops,
- 	wrap.hierarchies = ops->hierarchies;
- 	wrap.conf = handler->conf;
- 
-+	INFO("cgfsng_payload_destroy.%p, %s", ops->hierarchies, ops->container_cgroup);
-+
- 	if (handler->conf && !lxc_list_empty(&handler->conf->id_map))
- 		ret = userns_exec_1(handler->conf, cgroup_rmdir_wrapper, &wrap,
- 				    "cgroup_rmdir_wrapper");
-@@ -1120,8 +1126,10 @@ __cgfsng_ops static void cgfsng_payload_destroy(struct cgroup_ops *ops,
- 		ret = cgroup_rmdir(ops->hierarchies, ops->container_cgroup);
- 	if (ret < 0) {
- 		WARN("Failed to destroy cgroups");
--		return;
-+		return false;
- 	}
-+
-+	return true;
- }
- 
- static bool cg_unified_create_cgroup(struct hierarchy *h, char *cgname)
-@@ -1232,12 +1240,20 @@ static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname)
- {
- 	int ret;
- 
-+	h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, cgname, NULL);
-+
-+	if (file_exists(h->container_full_path)) { // it must not already exist
-+		ERROR("Cgroup path \"%s\" already exist.", h->container_full_path);
-+		//lxc_write_error_message(errfd, "%s:%d: Cgroup path \"%s\" already exist.",
-+		//	__FILE__, __LINE__, h->fullcgpath);
-+		return false;
-+	}
-+
- 	if (!cg_legacy_handle_cpuset_hierarchy(h, cgname)) {
- 		ERROR("Failed to handle legacy cpuset controller");
- 		return false;
- 	}
- 
--	h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, cgname, NULL);
- 	ret = mkdir_eexist_on_last(h->container_full_path, 0755);
- 	if (ret < 0) {
- 		ERROR("Failed to create cgroup \"%s\"", h->container_full_path);
-@@ -1259,83 +1275,26 @@ static void remove_path_for_hierarchy(struct hierarchy *h, char *cgname)
- 	h->container_full_path = NULL;
- }
- 
--/* Try to create the same cgroup in all hierarchies. Start with cgroup_pattern;
-- * next cgroup_pattern-1, -2, ..., -999.
-- */
-+/* isulad: create hierarchies path, if fail, return the error */
- __cgfsng_ops static bool cgfsng_payload_create(struct cgroup_ops *ops,
- 					       struct lxc_handler *handler)
- {
- 	int i;
--	size_t len;
--	char *container_cgroup, *offset, *tmp;
--	int idx = 0;
--	struct lxc_conf *conf = handler->conf;
--
--	if (ops->container_cgroup) {
--		WARN("cgfsng_create called a second time: %s", ops->container_cgroup);
--		return false;
--	}
-+	char *container_cgroup = ops->container_cgroup;
- 
--	if (!conf)
--		return false;
--
--	if (conf->cgroup_meta.dir)
--		tmp = lxc_string_join("/", (const char *[]){conf->cgroup_meta.dir, handler->name, NULL}, false);
--	else
--		tmp = lxc_string_replace("%n", handler->name, ops->cgroup_pattern);
--	if (!tmp) {
--		ERROR("Failed expanding cgroup name pattern");
-+	if (!container_cgroup) {
-+		ERROR("cgfsng_create container_cgroup is invalid");
- 		return false;
- 	}
- 
--	len = strlen(tmp) + 5; /* leave room for -NNN\0 */
--	container_cgroup = must_realloc(NULL, len);
--	(void)strlcpy(container_cgroup, tmp, len);
--	free(tmp);
--	offset = container_cgroup + len - 5;
--
--again:
--	if (idx == 1000) {
--		ERROR("Too many conflicting cgroup names");
--		goto out_free;
--	}
--
--	if (idx) {
--		int ret;
--
--		ret = snprintf(offset, 5, "-%d", idx);
--		if (ret < 0 || (size_t)ret >= 5) {
--			FILE *f = fopen("/dev/null", "w");
--			if (f) {
--				fprintf(f, "Workaround for GCC7 bug: "
--					   "https://gcc.gnu.org/bugzilla/"
--					   "show_bug.cgi?id=78969");
--				fclose(f);
--			}
--		}
--	}
--
- 	for (i = 0; ops->hierarchies[i]; i++) {
- 		if (!create_path_for_hierarchy(ops->hierarchies[i], container_cgroup)) {
--			int j;
--			ERROR("Failed to create cgroup \"%s\"", ops->hierarchies[i]->container_full_path);
--			free(ops->hierarchies[i]->container_full_path);
--			ops->hierarchies[i]->container_full_path = NULL;
--			for (j = 0; j < i; j++)
--				remove_path_for_hierarchy(ops->hierarchies[j], container_cgroup);
--			idx++;
--			goto again;
-+			SYSERROR("Failed to create %s", ops->hierarchies[i]->container_full_path);
-+			return false;
- 		}
- 	}
- 
--	ops->container_cgroup = container_cgroup;
--
- 	return true;
--
--out_free:
--	free(container_cgroup);
--
--	return false;
- }
- 
- __cgfsng_ops static bool cgfsng_payload_enter(struct cgroup_ops *ops, pid_t pid)
-@@ -2701,9 +2660,15 @@ static bool cg_init(struct cgroup_ops *ops)
- 	return cg_hybrid_init(ops);
- }
- 
--__cgfsng_ops static bool cgfsng_data_init(struct cgroup_ops *ops)
-+__cgfsng_ops static bool cgfsng_data_init(struct cgroup_ops *ops, struct lxc_handler *handler)
- {
- 	const char *cgroup_pattern;
-+	char *container_cgroup, *tmp;
-+	struct lxc_conf *conf = handler->conf;
-+	size_t len;
-+
-+	if (!conf)
-+		return false;
- 
- 	/* copy system-wide cgroup information */
- 	cgroup_pattern = lxc_global_config_value("lxc.cgroup.pattern");
-@@ -2714,6 +2679,22 @@ __cgfsng_ops static bool cgfsng_data_init(struct cgroup_ops *ops)
- 	}
- 	ops->cgroup_pattern = must_copy_string(cgroup_pattern);
- 
-+	/* isulad: init ops->container_cgroup here instead of in cgfsng_payload_create*/
-+	if (conf->cgroup_meta.dir)
-+		tmp = lxc_string_join("/", (const char *[]){conf->cgroup_meta.dir, handler->name, NULL}, false);
-+	else
-+		tmp = lxc_string_replace("%n", handler->name, ops->cgroup_pattern);
-+	if (!tmp) {
-+		ERROR("Failed expanding cgroup name pattern");
-+		return false;
-+	}
-+
-+	len = strlen(tmp) + 1;
-+	container_cgroup = must_realloc(NULL, len);
-+	(void)strlcpy(container_cgroup, tmp, len);
-+	free(tmp);
-+	ops->container_cgroup = container_cgroup;
-+
- 	return true;
- }
- 
-@@ -2735,7 +2716,6 @@ struct cgroup_ops *cgfsng_ops_init(void)
- 
- 	cgfsng_ops->data_init = cgfsng_data_init;
- 	cgfsng_ops->destroy = cgfsng_payload_destroy;
--	cgfsng_ops->destroy = cgfsng_payload_destroy;
- 	cgfsng_ops->payload_create = cgfsng_payload_create;
- 	cgfsng_ops->payload_enter = cgfsng_payload_enter;
- 	cgfsng_ops->escape = cgfsng_escape;
-diff --git a/src/lxc/cgroups/cgroup.c b/src/lxc/cgroups/cgroup.c
-index 04e0311..8e7aef9 100644
---- a/src/lxc/cgroups/cgroup.c
-+++ b/src/lxc/cgroups/cgroup.c
-@@ -50,7 +50,7 @@ struct cgroup_ops *cgroup_init(struct lxc_handler *handler)
- 		return NULL;
- 	}
- 
--	if (!cgroup_ops->data_init(cgroup_ops))
-+	if (!cgroup_ops->data_init(cgroup_ops, handler))
- 		return NULL;
- 
- 	TRACE("Initialized cgroup driver %s", cgroup_ops->driver);
-diff --git a/src/lxc/cgroups/cgroup.h b/src/lxc/cgroups/cgroup.h
-index ba4c153..fa4871e 100644
---- a/src/lxc/cgroups/cgroup.h
-+++ b/src/lxc/cgroups/cgroup.h
-@@ -123,8 +123,8 @@ struct cgroup_ops {
- 	 */
- 	cgroup_layout_t cgroup_layout;
- 
--	bool (*data_init)(struct cgroup_ops *ops);
--	void (*destroy)(struct cgroup_ops *ops, struct lxc_handler *handler);
-+	bool (*data_init)(struct cgroup_ops *ops, struct lxc_handler *handler);
-+	bool (*destroy)(struct cgroup_ops *ops, struct lxc_handler *handler);
- 	bool (*payload_create)(struct cgroup_ops *ops, struct lxc_handler *handler);
- 	bool (*payload_enter)(struct cgroup_ops *ops, pid_t pid);
- 	const char *(*get_cgroup)(struct cgroup_ops *ops, const char *controller);
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index beae459..38059fa 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -5116,6 +5116,23 @@ static bool do_lxcapi_set_start_timeout(struct lxc_container *c,  unsigned int s
- 
- WRAP_API_1(bool, lxcapi_set_start_timeout, unsigned int)
- 
-+/* isulad add clean resources */
-+static bool do_lxcapi_clean_container_resource(struct lxc_container *c, pid_t pid)
-+{
-+	int ret;
-+
-+	if (!c)
-+		return false;
-+
-+	ret = do_lxcapi_clean_resource(c->name, c->config_path, c->lxc_conf, pid);
-+	if (ret)
-+		ERROR("Failed to clean container %s resource", c->name);
-+	return ret == 0;
-+
-+}
-+
-+WRAP_API_1(bool, lxcapi_clean_container_resource, pid_t)
-+
- struct lxc_container *lxc_container_new(const char *name, const char *configpath)
- {
- 	struct lxc_container *c;
-@@ -5248,6 +5265,7 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
- 	c->set_terminal_init_fifos = lxcapi_set_terminal_default_fifos;
- 	c->set_container_info_file = lxcapi_set_container_info_file;
- 	c->set_start_timeout = lxcapi_set_start_timeout;
-+	c->clean_container_resource = lxcapi_clean_container_resource;
- 	/* isulad add end */
- 	return c;
- 
-diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
-index 77de704..679ca42 100644
---- a/src/lxc/lxccontainer.h
-+++ b/src/lxc/lxccontainer.h
-@@ -896,6 +896,16 @@ struct lxc_container {
- 	 * \return \c true on success, else \c false.
- 	 */
- 	bool (*set_start_timeout)(struct lxc_container *c, unsigned int start_timeout);
-+
-+	/*! isulad add
-+	 * \brief An API call to clean resources of container
-+	 *
-+	 * \param c Container.
-+	 * \param pid Value of container process.
-+	 *
-+	 * \return \c true on success, else \c false.
-+	 */
-+	bool (*clean_container_resource) (struct lxc_container *c, pid_t pid);
- };
- 
- /*!
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index f7be9e4..08d753a 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -1895,6 +1895,11 @@ static int lxc_spawn(struct lxc_handler *handler)
- 	if (ret < 0)
- 		SYSERROR("Failed to set environment variable: LXC_PID=%s", pidstr);
- 
-+	if (handler->cgroup_ops->container_cgroup) {
-+		if (setenv("LXC_CGROUP_PATH", handler->cgroup_ops->container_cgroup, 1))
-+			SYSERROR("Failed to set environment variable: LXC_CGROUP_PATH=%s.", handler->cgroup_ops->container_cgroup);
-+	}
-+
- 	/* Run any host-side start hooks */
- 	ret = run_lxc_hooks(name, "start-host", conf, NULL);
- 	if (ret < 0) {
-@@ -2289,3 +2294,139 @@ static bool do_destroy_container(struct lxc_handler *handler)
- 
- 	return storage_destroy(handler->conf);
- }
-+
-+/*isulad: init handler for clean */
-+static struct lxc_handler *lxc_init_clean_handler(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid)
-+{
-+	int i;
-+	struct lxc_handler *handler;
-+
-+	handler = malloc(sizeof(*handler));
-+	if (!handler)
-+		return NULL;
-+
-+	memset(handler, 0, sizeof(*handler));
-+
-+	/* Note that am_guest_unpriv() checks the effective uid. We
-+	 * probably don't care if we are real root only if we are running
-+	 * as root so this should be fine.
-+	 */
-+	handler->am_root = !am_guest_unpriv();
-+	handler->data_sock[0] = handler->data_sock[1] = -1;
-+	handler->conf = conf;
-+	handler->lxcpath = lxcpath;
-+	handler->pinfd = -1;
-+	handler->sigfd = -EBADF;
-+	handler->init_died = false;
-+	handler->pid = pid;
-+	handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1;
-+	if (handler->conf->reboot == REBOOT_NONE)
-+		lxc_list_init(&handler->conf->state_clients);
-+
-+	for (i = 0; i < LXC_NS_MAX; i++)
-+		handler->nsfd[i] = -1;
-+
-+	handler->name = name;
-+	handler->exit_code = -1; /* isulad: record exit code of container */
-+
-+	handler->cgroup_ops = cgroup_init(handler);
-+	if (!handler->cgroup_ops) {
-+		ERROR("Failed to initialize cgroup driver");
-+		goto on_error;
-+	}
-+
-+	INFO("Container \"%s\" 's clean handler is initialized.", name);
-+
-+	return handler;
-+
-+on_error:
-+	lxc_free_handler(handler);
-+
-+	return NULL;
-+}
-+
-+/*isulad: set env for clean resources */
-+static void clean_resource_set_env(struct lxc_handler *handler)
-+{
-+	const char *name = handler->name;
-+	struct lxc_conf *conf = handler->conf;
-+	char pidstr[20];
-+
-+	/* Start of environment variable setup for hooks. */
-+	if (name && setenv("LXC_NAME", name, 1))
-+		SYSERROR("Failed to set environment variable: LXC_NAME=%s.", name);
-+
-+	if (conf->rcfile && setenv("LXC_CONFIG_FILE", conf->rcfile, 1))
-+		SYSERROR("Failed to set environment variable: LXC_CONFIG_FILE=%s.", conf->rcfile);
-+
-+	if (conf->rootfs.mount && setenv("LXC_ROOTFS_MOUNT", conf->rootfs.mount, 1))
-+		SYSERROR("Failed to set environment variable: LXC_ROOTFS_MOUNT=%s.", conf->rootfs.mount);
-+
-+	if (conf->rootfs.path && setenv("LXC_ROOTFS_PATH", conf->rootfs.path, 1))
-+		SYSERROR("Failed to set environment variable: LXC_ROOTFS_PATH=%s.", conf->rootfs.path);
-+
-+	if (conf->console.path && setenv("LXC_CONSOLE", conf->console.path, 1))
-+		SYSERROR("Failed to set environment variable: LXC_CONSOLE=%s.", conf->console.path);
-+
-+	if (conf->console.log_path && setenv("LXC_CONSOLE_LOGPATH", conf->console.log_path, 1))
-+		SYSERROR("Failed to set environment variable: LXC_CONSOLE_LOGPATH=%s.", conf->console.log_path);
-+
-+	if (setenv("LXC_CGNS_AWARE", "1", 1))
-+		SYSERROR("Failed to set environment variable LXC_CGNS_AWARE=1.");
-+
-+
-+	snprintf(pidstr, 20, "%d", handler->pid);
-+	if (setenv("LXC_PID", pidstr, 1))
-+		SYSERROR("Failed to set environment variable: LXC_PID=%s.", pidstr);
-+
-+	if (handler->cgroup_ops->container_cgroup) {
-+		if (setenv("LXC_CGROUP_PATH", handler->cgroup_ops->container_cgroup, 1))
-+			SYSERROR("Failed to set environment variable: LXC_CGROUP_PATH=%s.", handler->cgroup_ops->container_cgroup);
-+	}
-+	/* End of environment variable setup for hooks. */
-+}
-+
-+/*isulad: do_lxcapi_clean_resource */
-+int do_lxcapi_clean_resource(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid)
-+{
-+	int ret = 0;
-+	struct lxc_handler *handler = NULL;
-+	int retry_count = 0;
-+	int max_retry = 10;
-+
-+	handler = lxc_init_clean_handler(name, lxcpath, conf, pid);
-+	if (!handler) {
-+		ERROR("Failed to init container %s clean handler", name);
-+		ret = -1;
-+		goto out;
-+	}
-+
-+	clean_resource_set_env(handler);
-+
-+	char* oci_hook_args[1];
-+	oci_hook_args[0] = alloca(strlen(handler->lxcpath) + 1);
-+	(void)strlcpy(oci_hook_args[0], handler->lxcpath, strlen(handler->lxcpath));
-+
-+	if (run_lxc_hooks(handler->name, "oci-poststop", handler->conf, oci_hook_args)) {
-+		ERROR("Failed to run lxc.hook.post-stop for container \"%s\".", handler->name);
-+		ret = -1;
-+	}
-+
-+retry:
-+	if (!handler->cgroup_ops->destroy(handler->cgroup_ops, handler)) {
-+		if (retry_count < max_retry) {
-+			usleep(100 * 1000); /* 100 millisecond */
-+			retry_count++;
-+			goto retry;
-+		}
-+		ERROR("Failed to destroy cgroup for container \"%s\".", handler->name);
-+		ret = -1;
-+	}
-+
-+
-+out_fini_handler:
-+	lxc_free_handler(handler);
-+out:
-+	return ret;
-+}
-+
-diff --git a/src/lxc/start.h b/src/lxc/start.h
-index a96f2ae..1d84325 100644
---- a/src/lxc/start.h
-+++ b/src/lxc/start.h
-@@ -180,4 +180,8 @@ extern int __lxc_start(const char *name, struct lxc_handler *handler,
- 
- extern int resolve_clone_flags(struct lxc_handler *handler);
- 
-+/*isulad: do_lxcapi_clean_resource */
-+extern int do_lxcapi_clean_resource(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid);
-+
-+
- #endif
--- 
-1.8.3.1
-

0031-head-file-remove-macro-HAVE_ISULAD-in-installed-head.patch

@@ -0,0 +1,115 @@
+From 5065f006a775e345fa80c1250442ff0f5c05383f Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Wed, 15 Apr 2020 17:46:54 +0800
+Subject: [PATCH 31/49] head file: remove macro HAVE_ISULAD in installed head
+ file
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/attach_options.h | 25 ++-----------------------
+ src/lxc/lxccontainer.h   |  6 ------
+ 2 files changed, 2 insertions(+), 29 deletions(-)
+
+diff --git a/src/lxc/attach_options.h b/src/lxc/attach_options.h
+index 7b0ea5e..5f01739 100644
+--- a/src/lxc/attach_options.h
++++ b/src/lxc/attach_options.h
+@@ -113,15 +113,12 @@ typedef struct lxc_attach_options_t {
+ 
+ 	/*! File descriptor to log output. */
+ 	int log_fd;
+-#ifdef HAVE_ISULAD
++
+ 	char *init_fifo[3]; /* isulad: default fifos for the start */
+ 	int64_t timeout;/* isulad: Seconds for waiting on a container to attach/exec before it is killed*/
+ 	const char *suffix;
+-#endif
+-
+ } lxc_attach_options_t;
+ 
+-#ifdef HAVE_ISULAD
+ /*! Default attach options to use */
+ #define LXC_ATTACH_OPTIONS_DEFAULT                                             \
+ 	{                                                                      \
+@@ -140,25 +137,7 @@ typedef struct lxc_attach_options_t {
+ 		/* .log_fd    = */      -EBADF,                                \
+ 		/* .init_fifo = */  	{NULL, NULL, NULL},                    \
+ 	}
+-#else
+-/*! Default attach options to use */
+-#define LXC_ATTACH_OPTIONS_DEFAULT                                             \
+-	{                                                                      \
+-		/* .attach_flags = */   LXC_ATTACH_DEFAULT,                    \
+-		/* .namespaces = */     -1,                                    \
+-		/* .personality = */    -1,                                    \
+-		/* .initial_cwd = */    NULL,                                  \
+-		/* .uid = */            (uid_t)-1,                             \
+-		/* .gid = */            (gid_t)-1,                             \
+-		/* .env_policy = */     LXC_ATTACH_KEEP_ENV,                   \
+-		/* .extra_env_vars = */ NULL,                                  \
+-		/* .extra_keep_env = */ NULL,                                  \
+-		/* .stdin_fd = */       0,                                     \
+-		/* .stdout_fd = */      1,                                     \
+-		/* .stderr_fd = */      2,                                     \
+-		/* .log_fd    = */      -EBADF,                                \
+-	}
+-#endif
++
+ /*!
+  * Representation of a command to run in a container.
+  */
+diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
+index e69be8f..6ede70c 100644
+--- a/src/lxc/lxccontainer.h
++++ b/src/lxc/lxccontainer.h
+@@ -107,7 +107,6 @@ struct lxc_container {
+ 	/*! Full path to configuration file */
+ 	char *config_path;
+ 
+-#ifdef HAVE_ISULAD
+ 	/*! isulad:
+ 	 * \private
+ 	 * exit FIFO File to open used monitor the state of lxc monitor process.
+@@ -125,7 +124,6 @@ struct lxc_container {
+ 	 * full path of json file
+ 	 * */
+ 	char *ocihookfile;
+-#endif
+ 
+ 	/*!
+ 	 * \brief Determine if \c /var/lib/lxc/$name/config exists.
+@@ -886,7 +884,6 @@ struct lxc_container {
+ 	 */
+ 	int (*init_pidfd)(struct lxc_container *c);
+ 
+-#ifdef HAVE_ISULAD
+ 	/*! isulad add
+ 	 * \brief An API call to set the path of info file
+ 	 *
+@@ -963,7 +960,6 @@ struct lxc_container {
+ 	 * \return \c true on success, else \c false.
+ 	 */
+ 	bool (*get_container_pids)(struct lxc_container *c,pid_t **pids,size_t *pids_len);
+-#endif
+ };
+ 
+ /*!
+@@ -1097,7 +1093,6 @@ struct lxc_console_log {
+  */
+ struct lxc_container *lxc_container_new(const char *name, const char *configpath);
+ 
+-#ifdef HAVE_ISULAD
+ /*!
+  * \brief Create a new container without loading config.
+  *
+@@ -1109,7 +1104,6 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
+  * \note This function can only used for listing container.
+  */
+ struct lxc_container *lxc_container_without_config_new(const char *name, const char *configpath);
+-#endif
+ 
+ /*!
+  * \brief Add a reference to the specified container.
+-- 
+1.8.3.1
+

0032-Drop-all-caps-when-cap.keep-ISULAD_KEEP_NONE.patch

@@ -1,33 +0,0 @@
-From 9a8aa1961330227da81a6081db7254441afb3031 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Tue, 15 Jan 2019 05:39:39 -0500
-Subject: [PATCH 032/140] Drop all caps when cap.keep=ISULAD_KEEP_NONE
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c | 9 +++++++++
- 1 file changed, 9 insertions(+)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 88763ee..54b967b 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -2805,6 +2805,15 @@ static int dropcaps_except(struct lxc_list *caps)
- 	lxc_list_for_each (iterator, caps) {
- 		keep_entry = iterator->elem;
- 
-+		/* isulad: Do not keep any cap*/
-+		if (strcmp(keep_entry, "ISULAD_KEEP_NONE") == 0) {
-+			DEBUG("Do not keep any capability");
-+			for(i = 0; i < numcaps; i++) {
-+				caplist[i] = 0;
-+			}
-+			break;
-+		}
-+
- 		capid = parse_cap(keep_entry);
- 		if (capid == -2)
- 			continue;
--- 
-1.8.3.1
-

0032-link-proc-mounts-to-etc-mtab.patch

@@ -1,20 +1,20 @@
-From 2bca52aab48cf7337df9dfb64d20f55ceac3a9ff Mon Sep 17 00:00:00 2001
+From e86ebe923f1ecc072d8a99949871b46fe8188ac9 Mon Sep 17 00:00:00 2001
 From: wujing <wujing50@huawei.com>
-Date: Tue, 10 Dec 2019 21:28:47 +0800
+Date: Wed, 15 Apr 2020 06:01:36 -0400
-Subject: [PATCH 128/140] link /proc/mounts to /etc/mtab
+Subject: [PATCH 32/49] link proc mounts to etc mtab
 
 Signed-off-by: wujing <wujing50@huawei.com>
 ---
- src/lxc/conf.c | 39 ++++++++++++++++++++++++++++++++++++++-
+ src/lxc/conf.c | 37 +++++++++++++++++++++++++++++++++++++
- 1 file changed, 38 insertions(+), 1 deletion(-)
+ 1 file changed, 37 insertions(+)
 
 diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 21ec340..b66e7bc 100644
+index 325e0c2..6856b1d 100644
 --- a/src/lxc/conf.c
 +++ b/src/lxc/conf.c
-@@ -4101,6 +4101,37 @@ static int setup_rootfs_mountopts(const struct lxc_rootfs *rootfs)
+@@ -3977,6 +3977,37 @@ struct oci_hook_conf {
- 	return 0;
+ 	int which;
- }
+ };
  
 +static int create_mtab_link()
 +{
@@ -47,31 +47,22 @@ index 21ec340..b66e7bc 100644
 +	return 0;
 +}
 +
- int lxc_setup(struct lxc_handler *handler)
+ struct wait_conf {
- {
+ 	pid_t pid;
- 	int ret;
+ 	unsigned long long startat;
-@@ -4331,7 +4362,7 @@ int lxc_setup(struct lxc_handler *handler)
+@@ -4696,6 +4727,12 @@ int lxc_setup(struct lxc_handler *handler)
+ 			return log_error(-1, "failed to setup readonlypaths");
  		}
  	}
- 
++
--	//isulad: system container, remount /proc/sys/xxx by mount_list
-+	// isulad: system container, remount /proc/sys/xxx by mount_list
- 	if (lxc_conf->systemd != NULL && strcmp(lxc_conf->systemd, "true") == 0) {
- 		if (!lxc_list_empty(&lxc_conf->mount_list)) {
- 			if (remount_proc_sys_mount_entries(&lxc_conf->mount_list)) {
-@@ -4341,6 +4372,12 @@ int lxc_setup(struct lxc_handler *handler)
- 		}
- 	}
- 
 +	// isulad: create link /etc/mtab for /proc/mounts
 +	if (create_mtab_link() != 0) {
 +		ERROR("failed to create link /etc/mtab for target /proc/mounts");
 +		goto on_error;
 +	}
-+
+ #endif
+ 
  	if (!lxc_list_empty(&lxc_conf->keepcaps)) {
- 		if (!lxc_list_empty(&lxc_conf->caps)) {
- 			ERROR("Container requests lxc.cap.drop and "
 -- 
 1.8.3.1
 

0033-build-add-secure-build-flags.patch

@@ -0,0 +1,423 @@
+From 2b69f168a6847885233e88be02a1143a4c55b59f Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Wed, 15 Apr 2020 18:58:24 +0800
+Subject: [PATCH 33/49] build: add secure build flags
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ configure.ac               |  14 +++++-
+ src/lxc/Makefile.am        |   8 +++-
+ src/lxc/cgroups/cgfsng.c   | 117 ++++++++++++++++++++++++---------------------
+ src/lxc/confile.c          |   1 -
+ src/lxc/terminal.c         |   6 ++-
+ src/lxc/tools/lxc_attach.c |   2 -
+ src/lxc/utils.c            |   7 +++
+ 7 files changed, 93 insertions(+), 62 deletions(-)
+
+diff --git a/configure.ac b/configure.ac
+index 56d0cb7..438d292 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -43,6 +43,7 @@ AM_INIT_AUTOMAKE([-Wall -Werror -Wno-portability subdir-objects])
+ AC_CANONICAL_HOST
+ AM_PROG_CC_C_O
+ AC_USE_SYSTEM_EXTENSIONS
++CFLAGS=`echo "${CFLAGS#\-g}"`
+ 
+ # Test if we have a new enough compiler.
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+@@ -190,6 +191,11 @@ AC_ARG_ENABLE([werror],
+ 	[AS_HELP_STRING([--disable-werror], [do not treat warnings as errors])],
+ 	[enable_werror=$enableval], [enable_werror=yes])
+ 
++AC_ARG_ENABLE([debug],
++	[AC_HELP_STRING([--enable-debug],
++	[set -g into cflags [default=no]])],
++	[], [enable_debug=no])
++
+ # Allow disabling rpath
+ AC_ARG_ENABLE([rpath],
+ 	[AS_HELP_STRING([--enable-rpath], [set rpath in executables [default=no]])],
+@@ -732,7 +738,6 @@ AX_CHECK_COMPILE_FLAG([-fno-strict-aliasing], [CFLAGS="$CFLAGS -fno-strict-alias
+ AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [CFLAGS="$CFLAGS -fstack-clash-protection"],,[-Werror])
+ AX_CHECK_LINK_FLAG([-fstack-protector-strong], [CFLAGS="$CFLAGS -fstack-protector-strong"],,[-Werror])
+ AX_CHECK_LINK_FLAG([--param=ssp-buffer-size=4], [CFLAGS="$CFLAGS --param=ssp-buffer-size=4"],,[-Werror])
+-AX_CHECK_COMPILE_FLAG([-g], [CFLAGS="$CFLAGS -g"],,[-Werror])
+ AX_CHECK_COMPILE_FLAG([--mcet -fcf-protection], [CFLAGS="$CFLAGS --mcet -fcf-protection"],,[-Werror])
+ AX_CHECK_COMPILE_FLAG([-Werror=implicit-function-declaration], [CFLAGS="$CFLAGS -Werror=implicit-function-declaration"],,[-Werror])
+ AX_CHECK_COMPILE_FLAG([-Wlogical-op], [CFLAGS="$CFLAGS -Wlogical-op"],,[-Werror])
+@@ -759,12 +764,17 @@ AX_CHECK_COMPILE_FLAG([-fexceptions], [CFLAGS="$CFLAGS -fexceptions"],,[-Werror]
+ 
+ AX_CHECK_LINK_FLAG([-z relro], [LDFLAGS="$LDFLAGS -z relro"],,[])
+ AX_CHECK_LINK_FLAG([-z now], [LDFLAGS="$LDFLAGS -z now"],,[])
++AX_CHECK_LINK_FLAG([-z noexecstack], [LDFLAGS="$LDFLAGS -z noexecstack"],,[])
+ 
+-CFLAGS="$CFLAGS -Wvla -std=gnu11"
++CFLAGS="$CFLAGS -Wvla -std=gnu11 -D_FORTIFY_SOURCE=2 -Wall -fPIC -fPIE -pie"
+ if test "x$enable_werror" = "xyes"; then
+ 	CFLAGS="$CFLAGS -Werror"
+ fi
+ 
++if test "x$enable_debug" = "xyes"; then
++    CFLAGS="$CFLAGS -g"
++fi
++
+ AC_ARG_ENABLE([thread-safety],
+ 	[AS_HELP_STRING([--enable-thread-safety], [enforce thread-safety otherwise fail the build [default=yes]])],
+ 	[enable_thread_safety=$enableval], [enable_thread_safety=yes])
+diff --git a/src/lxc/Makefile.am b/src/lxc/Makefile.am
+index c288c51..0e1ba8d 100644
+--- a/src/lxc/Makefile.am
++++ b/src/lxc/Makefile.am
+@@ -276,6 +276,10 @@ liblxc_la_CFLAGS += -fsanitize=address \
+ 		    -fno-omit-frame-pointer
+ endif
+ 
++if HAVE_ISULAD
++liblxc_la_CFLAGS += -D_FORTIFY_SOURCE=2 -Wall
++endif
++
+ if ENABLE_UBSAN
+ liblxc_la_CFLAGS += -fsanitize=undefined
+ endif
+@@ -286,7 +290,9 @@ liblxc_la_LDFLAGS = -pthread \
+ 		    -version-info @LXC_ABI_MAJOR@
+ 
+ if HAVE_ISULAD
+-liblxc_la_LDFLAGS += @YAJL_LIBS@
++liblxc_la_LDFLAGS += @YAJL_LIBS@ -Wl,-z,relro \
++		    -Wl,-z,now \
++		    -Wl,-z,noexecstack
+ endif
+ 
+ liblxc_la_LIBADD = $(CAP_LIBS) \
+diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
+index b6aef12..1047c08 100644
+--- a/src/lxc/cgroups/cgfsng.c
++++ b/src/lxc/cgroups/cgfsng.c
+@@ -214,6 +214,7 @@ static char *read_file(const char *fnam)
+ 	return move_ptr(buf);
+ }
+ 
++#ifndef HAVE_ISULAD
+ /* Taken over modified from the kernel sources. */
+ #define NBITS 32 /* bits in uint32_t */
+ #define DIV_ROUND_UP(n, d) (((n) + (d)-1) / (d))
+@@ -476,12 +477,14 @@ static bool copy_parent_file(const char *parent_cgroup,
+ 				       value, child_cgroup, file);
+ 	return true;
+ }
++#endif
+ 
+ static inline bool is_unified_hierarchy(const struct hierarchy *h)
+ {
+ 	return h->version == CGROUP2_SUPER_MAGIC;
+ }
+ 
++#ifndef HAVE_ISULAD
+ /*
+  * Initialize the cpuset hierarchy in first directory of @cgroup_leaf and set
+  * cgroup.clone_children so that children inherit settings. Since the
+@@ -561,6 +564,7 @@ static int cg_legacy_handle_cpuset_hierarchy(struct hierarchy *h,
+ 
+ 	return fret;
+ }
++#endif
+ 
+ /* Given two null-terminated lists of strings, return true if any string is in
+  * both.
+@@ -946,29 +950,6 @@ static void lxc_cgfsng_print_basecg_debuginfo(char *basecginfo, char **klist,
+ 		TRACE("named subsystem %d: %s", k, *it);
+ }
+ 
+-static int cgroup_tree_remove(struct hierarchy **hierarchies,
+-			const char *container_cgroup)
+-{
+-	if (!container_cgroup || !hierarchies)
+-		return 0;
+-
+-	for (int i = 0; hierarchies[i]; i++) {
+-		struct hierarchy *h = hierarchies[i];
+-		int ret;
+-
+-		if (!h->container_full_path)
+-			continue;
+-
+-		ret = lxc_rm_rf(h->container_full_path);
+-		if (ret < 0)
+-			WARN("Failed to destroy \"%s\"", h->container_full_path);
+-
+-		free_disarm(h->container_full_path);
+-	}
+-
+-	return 0;
+-}
+-
+ struct generic_userns_exec_data {
+ 	struct hierarchy **hierarchies;
+ 	const char *container_cgroup;
+@@ -977,29 +958,6 @@ struct generic_userns_exec_data {
+ 	char *path;
+ };
+ 
+-static int cgroup_tree_remove_wrapper(void *data)
+-{
+-	struct generic_userns_exec_data *arg = data;
+-	uid_t nsuid = (arg->conf->root_nsuid_map != NULL) ? 0 : arg->conf->init_uid;
+-	gid_t nsgid = (arg->conf->root_nsgid_map != NULL) ? 0 : arg->conf->init_gid;
+-	int ret;
+-
+-	if (!lxc_setgroups(0, NULL) && errno != EPERM)
+-		return log_error_errno(-1, errno, "Failed to setgroups(0, NULL)");
+-
+-	ret = setresgid(nsgid, nsgid, nsgid);
+-	if (ret < 0)
+-		return log_error_errno(-1, errno, "Failed to setresgid(%d, %d, %d)",
+-				       (int)nsgid, (int)nsgid, (int)nsgid);
+-
+-	ret = setresuid(nsuid, nsuid, nsuid);
+-	if (ret < 0)
+-		return log_error_errno(-1, errno, "Failed to setresuid(%d, %d, %d)",
+-				       (int)nsuid, (int)nsuid, (int)nsuid);
+-
+-	return cgroup_tree_remove(arg->hierarchies, arg->container_cgroup);
+-}
+-
+ #ifdef HAVE_ISULAD
+ 
+ static int isulad_cgroup_tree_remove(struct hierarchy **hierarchies,
+@@ -1100,6 +1058,52 @@ __cgfsng_ops static bool isulad_cgfsng_payload_destroy(struct cgroup_ops *ops,
+ 	return true;
+ }
+ #else
++static int cgroup_tree_remove(struct hierarchy **hierarchies,
++			const char *container_cgroup)
++{
++	if (!container_cgroup || !hierarchies)
++		return 0;
++
++	for (int i = 0; hierarchies[i]; i++) {
++		struct hierarchy *h = hierarchies[i];
++		int ret;
++
++		if (!h->container_full_path)
++			continue;
++
++		ret = lxc_rm_rf(h->container_full_path);
++		if (ret < 0)
++			WARN("Failed to destroy \"%s\"", h->container_full_path);
++
++		free_disarm(h->container_full_path);
++	}
++
++	return 0;
++}
++
++static int cgroup_tree_remove_wrapper(void *data)
++{
++	struct generic_userns_exec_data *arg = data;
++	uid_t nsuid = (arg->conf->root_nsuid_map != NULL) ? 0 : arg->conf->init_uid;
++	gid_t nsgid = (arg->conf->root_nsgid_map != NULL) ? 0 : arg->conf->init_gid;
++	int ret;
++
++	if (!lxc_setgroups(0, NULL) && errno != EPERM)
++		return log_error_errno(-1, errno, "Failed to setgroups(0, NULL)");
++
++	ret = setresgid(nsgid, nsgid, nsgid);
++	if (ret < 0)
++		return log_error_errno(-1, errno, "Failed to setresgid(%d, %d, %d)",
++				       (int)nsgid, (int)nsgid, (int)nsgid);
++
++	ret = setresuid(nsuid, nsuid, nsuid);
++	if (ret < 0)
++		return log_error_errno(-1, errno, "Failed to setresuid(%d, %d, %d)",
++				       (int)nsuid, (int)nsuid, (int)nsuid);
++
++	return cgroup_tree_remove(arg->hierarchies, arg->container_cgroup);
++}
++
+ __cgfsng_ops static void cgfsng_payload_destroy(struct cgroup_ops *ops,
+ 						struct lxc_handler *handler)
+ {
+@@ -1227,6 +1231,14 @@ try_lxc_rm_rf:
+ }
+ #endif
+ 
++#ifdef HAVE_ISULAD
++__cgfsng_ops static inline bool cgfsng_monitor_create(struct cgroup_ops *ops,
++						      struct lxc_handler *handler)
++{
++	return true;
++}
++#else
++
+ static int mkdir_eexist_on_last(const char *dir, mode_t mode)
+ {
+ 	const char *tmp = dir;
+@@ -1311,13 +1323,6 @@ static void cgroup_tree_leaf_remove(struct hierarchy *h, bool payload)
+ 		SYSWARN("Failed to rmdir(\"%s\") cgroup", full_path);
+ }
+ 
+-#ifdef HAVE_ISULAD
+-__cgfsng_ops static inline bool cgfsng_monitor_create(struct cgroup_ops *ops,
+-						      struct lxc_handler *handler)
+-{
+-	return true;
+-}
+-#else
+ __cgfsng_ops static inline bool cgfsng_monitor_create(struct cgroup_ops *ops,
+ 						      struct lxc_handler *handler)
+ {
+@@ -3038,6 +3043,7 @@ static int device_cgroup_parse_access(struct device_item *device, const char *va
+ 	return 0;
+ }
+ 
++#ifndef HAVE_ISULAD
+ static int device_cgroup_rule_parse(struct device_item *device, const char *key,
+ 				    const char *val)
+ {
+@@ -3124,6 +3130,7 @@ static int device_cgroup_rule_parse(struct device_item *device, const char *key,
+ 
+ 	return device_cgroup_parse_access(device, ++val);
+ }
++#endif
+ 
+ #ifdef HAVE_ISULAD
+ __cgfsng_ops static int cgfsng_set(struct cgroup_ops *ops,
+@@ -3307,6 +3314,7 @@ static int convert_devpath(const char *invalue, char *dest)
+ 	return 0;
+ }
+ 
++#ifndef HAVE_ISULAD
+ /* Called from setup_limits - here we have the container's cgroup_data because
+  * we created the cgroups.
+  */
+@@ -3339,6 +3347,7 @@ static int cg_legacy_set_data(struct cgroup_ops *ops, const char *filename,
+ 
+ 	return lxc_write_openat(h->container_full_path, filename, value, strlen(value));
+ }
++#endif
+ 
+ #ifdef HAVE_ISULAD
+ /* Called from setup_limits - here we have the container's cgroup_data because
+@@ -3923,7 +3932,6 @@ static int cg_hybrid_init(struct cgroup_ops *ops, bool relative, bool unprivileg
+ 		__do_free char *base_cgroup = NULL, *mountpoint = NULL;
+ 		__do_free_string_list char **controller_list = NULL;
+ 		int type;
+-		bool writeable;
+ 		struct hierarchy *new;
+ 
+ 		type = get_cgroup_version(line);
+@@ -3989,6 +3997,7 @@ static int cg_hybrid_init(struct cgroup_ops *ops, bool relative, bool unprivileg
+ 			base_cgroup[1] = '\0';
+ 		}
+ #else
++		bool writeable;
+ 		if (type == CGROUP2_SUPER_MAGIC)
+ 			writeable = test_writeable_v2(mountpoint, base_cgroup);
+ 		else
+diff --git a/src/lxc/confile.c b/src/lxc/confile.c
+index 55cba6d..f00afe9 100644
+--- a/src/lxc/confile.c
++++ b/src/lxc/confile.c
+@@ -6228,7 +6228,6 @@ static int get_config_init_args(const char *key, char *retv, int inlen,
+ 				  struct lxc_conf *c, void *data)
+ {
+ 	int i, len, fulllen = 0;
+-	struct lxc_list *it;
+ 
+ 	if (!retv)
+ 		inlen = 0;
+diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
+index 14686fc..c0a4d1a 100644
+--- a/src/lxc/terminal.c
++++ b/src/lxc/terminal.c
+@@ -287,6 +287,7 @@ static int lxc_terminal_rotate_log_file(struct lxc_terminal *terminal)
+ 	return lxc_terminal_create_log_file(terminal);
+ }
+ 
++#ifndef HAVE_ISULAD
+ static int lxc_terminal_write_log_file(struct lxc_terminal *terminal, char *buf,
+ 				       int bytes_read)
+ {
+@@ -392,6 +393,7 @@ static int lxc_terminal_write_log_file(struct lxc_terminal *terminal, char *buf,
+ 	bytes_read -= ret;
+ 	return bytes_read;
+ }
++#endif
+ 
+ #ifdef HAVE_ISULAD
+ /* get time buffer */
+@@ -1240,8 +1242,8 @@ void lxc_terminal_free(struct lxc_conf *conf, int fd)
+ 
+ static int lxc_terminal_peer_default(struct lxc_terminal *terminal)
+ {
+-	struct lxc_terminal_state *ts;
+-	const char *path;
++	struct lxc_terminal_state *ts = NULL;
++	const char *path = NULL;
+ 	int ret = 0;
+ 
+ 	if (terminal->path)
+diff --git a/src/lxc/tools/lxc_attach.c b/src/lxc/tools/lxc_attach.c
+index b068c9a..48e18bb 100644
+--- a/src/lxc/tools/lxc_attach.c
++++ b/src/lxc/tools/lxc_attach.c
+@@ -507,11 +507,9 @@ out:
+ 
+ int main(int argc, char *argv[])
+ {
+-	int ret = -1;
+ 	int wexit = 0;
+ 	struct lxc_log log;
+ 	char *errmsg = NULL;
+-	pid_t pid;
+ 	lxc_attach_options_t attach_options = LXC_ATTACH_OPTIONS_DEFAULT;
+ 	lxc_attach_command_t command = (lxc_attach_command_t){.program = NULL};
+ 
+diff --git a/src/lxc/utils.c b/src/lxc/utils.c
+index 39413ee..ba69995 100644
+--- a/src/lxc/utils.c
++++ b/src/lxc/utils.c
+@@ -1427,9 +1427,11 @@ static int lxc_get_unused_loop_dev(char *name_loop)
+ {
+ 	int loop_nr, ret;
+ 	int fd_ctl = -1, fd_tmp = -1;
++#if HAVE_ISULAD
+ 	// isulad: retry and try mknod
+ 	int max_retry = 200;
+ 	bool try_mknod = true;
++#endif
+ 
+ 	fd_ctl = open("/dev/loop-control", O_RDWR | O_CLOEXEC);
+ 	if (fd_ctl < 0) {
+@@ -1446,7 +1448,10 @@ static int lxc_get_unused_loop_dev(char *name_loop)
+ 	ret = snprintf(name_loop, LO_NAME_SIZE, "/dev/loop%d", loop_nr);
+ 	if (ret < 0 || ret >= LO_NAME_SIZE)
+ 		goto on_error;
++
++#if HAVE_ISULAD
+ retry:
++#endif
+ 	fd_tmp = open(name_loop, O_RDWR | O_CLOEXEC);
+ 	if (fd_tmp < 0) {
+ #if HAVE_ISULAD
+@@ -1693,6 +1698,7 @@ uint64_t lxc_find_next_power2(uint64_t n)
+ 	return n;
+ }
+ 
++#ifndef HAVE_ISULAD
+ static int process_dead(/* takes */ int status_fd)
+ {
+ 	__do_close int dupfd = -EBADF;
+@@ -1730,6 +1736,7 @@ static int process_dead(/* takes */ int status_fd)
+ 
+ 	return ret;
+ }
++#endif
+ 
+ int lxc_set_death_signal(int signal, pid_t parent, int parent_status_fd)
+ {
+-- 
+1.8.3.1
+

0034-support-timeout.patch

@@ -0,0 +1,726 @@
+From b13e288b8a46262e2c062ddfc152f10739b1691a Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Wed, 15 Apr 2020 17:35:53 +0800
+Subject: [PATCH 34/49] support timeout
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ src/lxc/attach.c           | 104 ++++++++++++++++++++++++++++++++++++++--
+ src/lxc/attach.h           |   6 +++
+ src/lxc/commands.c         |   5 ++
+ src/lxc/execute.c          |  12 +++++
+ src/lxc/lxc.h              |  13 ++++-
+ src/lxc/lxccontainer.c     |  41 ++++++++++++++--
+ src/lxc/lxccontainer.h     |  16 +++++++
+ src/lxc/start.c            | 115 +++++++++++++++++++++++++++++++++++++++++++--
+ src/lxc/start.h            |   6 +++
+ src/lxc/tools/arguments.h  |   2 +
+ src/lxc/tools/lxc_attach.c |  10 ++++
+ src/lxc/tools/lxc_start.c  |  14 ++++++
+ src/lxc/utils.c            |  11 +++++
+ src/lxc/utils.h            |   2 +
+ 14 files changed, 346 insertions(+), 11 deletions(-)
+
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index cb480ed..510c069 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -45,12 +45,27 @@
+ #include "terminal.h"
+ #include "utils.h"
+ 
++#if HAVE_SYS_PERSONALITY_H
++#include <sys/personality.h>
++#endif
++
+ #ifdef HAVE_ISULAD
+ #include "exec_commands.h"
+-#endif
+ 
+-#if HAVE_SYS_PERSONALITY_H
+-#include <sys/personality.h>
++typedef enum {
++	ATTACH_INIT,
++	ATTACH_TIMEOUT,
++	ATTACH_MAX,
++} attach_timeout_t;
++
++static volatile attach_timeout_t g_attach_timeout_state = ATTACH_INIT;
++
++struct attach_timeout_conf {
++	int64_t timeout;
++	unsigned long long start_time;
++	pid_t pid;
++};
++
+ #endif
+ 
+ lxc_log_define(attach, lxc);
+@@ -1038,9 +1053,67 @@ static inline void lxc_attach_terminal_close_log(struct lxc_terminal *terminal)
+ 	close_prot_errno_disarm(terminal->log_fd);
+ }
+ 
++#ifdef HAVE_ISULAD
++/* isulad: attach timeout thread function */
++static void* wait_attach_timeout(void *arg)
++{
++	struct attach_timeout_conf *conf = (struct attach_timeout_conf *)arg;
++
++	if (!conf || conf->timeout < 1)
++		goto out;
++	sleep(conf->timeout);
++	if (lxc_process_alive(conf->pid, conf->start_time)) {
++		g_attach_timeout_state = ATTACH_TIMEOUT;
++		if (kill(conf->pid, SIGKILL) < 0) {
++			ERROR("Failed to send signal %d to pid %d", SIGKILL, conf->pid);
++		}
++	}
++
++out:
++	free(conf);
++	return ((void *)0);
++}
++
++/* isulad: create attach timeout thread */
++static int create_attach_timeout_thread(int64_t attach_timeout, pid_t pid)
++{
++	int ret = 0;
++	pthread_t ptid;
++	pthread_attr_t attr;
++	struct attach_timeout_conf *timeout_conf = NULL;
++
++	timeout_conf = malloc(sizeof(struct attach_timeout_conf));
++	if (timeout_conf == NULL) {
++		ERROR("Failed to malloc attach timeout conf");
++		ret = -1;
++		goto out;
++	}
++
++	memset(timeout_conf, 0, sizeof(struct attach_timeout_conf));
++	timeout_conf->timeout = attach_timeout;
++	timeout_conf->pid = pid;
++	timeout_conf->start_time = lxc_get_process_startat(pid);
++
++	pthread_attr_init(&attr);
++	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
++	ret = pthread_create(&ptid, &attr, wait_attach_timeout, timeout_conf);
++	if (ret != 0) {
++		ERROR("Create attach wait timeout thread failed");
++		free(timeout_conf);
++		goto out;
++	}
++
++out:
++	return ret;
++}
++int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
++	       void *exec_payload, lxc_attach_options_t *options,
++	       pid_t *attached_process, char **err_msg)
++#else
+ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 	       void *exec_payload, lxc_attach_options_t *options,
+ 	       pid_t *attached_process)
++#endif
+ {
+ 	int i, ret, status;
+ 	int ipc_sockets[2];
+@@ -1417,6 +1490,26 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 
+ 		*attached_process = attached_pid;
+ 
++#ifdef HAVE_ISULAD
++		if (options->timeout > 0) {
++			ret = create_attach_timeout_thread(options->timeout, *attached_process);
++			if (ret) {
++				ERROR("Failed to create attach timeout thread for container.");
++				goto close_mainloop;
++			}
++		}
++		/* isulad: read error msg from pipe */
++		ssize_t size_read;
++		char errbuf[BUFSIZ + 1] = {0};
++
++		size_read = read(conf->errpipe[0], errbuf, BUFSIZ);
++		if (size_read > 0) {
++			if (err_msg)
++				*err_msg = safe_strdup(errbuf);
++			goto close_mainloop;
++		}
++#endif
++
+ 		/* Now shut down communication with child, we're done. */
+ 		shutdown(ipc_sockets[0], SHUT_RDWR);
+ 		close(ipc_sockets[0]);
+@@ -1433,6 +1526,11 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 			}
+ 		}
+ 
++#ifdef HAVE_ISULAD
++		if (g_attach_timeout_state == ATTACH_TIMEOUT && err_msg != NULL && *err_msg == NULL) {
++			*err_msg = safe_strdup("Attach exceeded timeout");
++		}
++#endif
+ 	close_mainloop:
+ 		if (options->attach_flags & LXC_ATTACH_TERMINAL)
+ 			lxc_mainloop_close(&descr);
+diff --git a/src/lxc/attach.h b/src/lxc/attach.h
+index ef5a6c1..8316344 100644
+--- a/src/lxc/attach.h
++++ b/src/lxc/attach.h
+@@ -20,9 +20,15 @@ struct lxc_proc_context_info {
+ 	int ns_fd[LXC_NS_MAX];
+ };
+ 
++#ifdef HAVE_ISULAD
++extern int lxc_attach(struct lxc_container *container,
++		      lxc_attach_exec_t exec_function, void *exec_payload,
++		      lxc_attach_options_t *options, pid_t *attached_process, char **err_msg);
++#else
+ extern int lxc_attach(struct lxc_container *container,
+ 		      lxc_attach_exec_t exec_function, void *exec_payload,
+ 		      lxc_attach_options_t *options, pid_t *attached_process);
++#endif
+ 
+ extern int lxc_attach_remount_sys_proc(void);
+ 
+diff --git a/src/lxc/commands.c b/src/lxc/commands.c
+index b21c12b..c32aef1 100644
+--- a/src/lxc/commands.c
++++ b/src/lxc/commands.c
+@@ -119,7 +119,12 @@ static int lxc_cmd_rsp_recv(int sock, struct lxc_cmd_rr *cmd)
+ 	int ret;
+ 	struct lxc_cmd_rsp *rsp = &cmd->rsp;
+ 
++#ifdef HAVE_ISULAD
++	/*isulad: add timeout 1s to avoid long block due to [lxc monitor] error*/
++	ret = lxc_abstract_unix_recv_fds_timeout(sock, &fd_rsp, 1, rsp, sizeof(*rsp), 1000 * 1000);
++#else
+ 	ret = lxc_abstract_unix_recv_fds(sock, &fd_rsp, 1, rsp, sizeof(*rsp));
++#endif
+ 	if (ret < 0)
+ 		return log_warn_errno(-1,
+ 				      errno, "Failed to receive response for command \"%s\"",
+diff --git a/src/lxc/execute.c b/src/lxc/execute.c
+index 7dd8358..59ff604 100644
+--- a/src/lxc/execute.c
++++ b/src/lxc/execute.c
+@@ -88,14 +88,26 @@ static struct lxc_operations execute_start_ops = {
+ 	.post_start = execute_post_start
+ };
+ 
++#ifdef HAVE_ISULAD
++int lxc_execute(const char *name, char *const argv[], int quiet,
++		struct lxc_handler *handler, const char *lxcpath,
++		bool daemonize, int *error_num, unsigned int start_timeout)
++#else
+ int lxc_execute(const char *name, char *const argv[], int quiet,
+ 		struct lxc_handler *handler, const char *lxcpath,
+ 		bool daemonize, int *error_num)
++#endif
+ {
++
+ 	struct execute_args args = {.argv = argv, .quiet = quiet};
+ 
+ 	TRACE("Doing lxc_execute");
+ 	handler->conf->is_execute = true;
++#ifdef HAVE_ISULAD
++	return __lxc_start(handler, &execute_start_ops, &args, lxcpath,
++			   daemonize, error_num, start_timeout);
++#else
+ 	return __lxc_start(handler, &execute_start_ops, &args, lxcpath,
+ 			   daemonize, error_num);
++#endif
+ }
+diff --git a/src/lxc/lxc.h b/src/lxc/lxc.h
+index 99fd422..ec2feaa 100644
+--- a/src/lxc/lxc.h
++++ b/src/lxc/lxc.h
+@@ -32,9 +32,14 @@ struct lxc_handler;
+  * @daemonize    : whether or not the container is daemonized
+  * Returns 0 on success, < 0 otherwise
+  */
++#ifdef HAVE_ISULAD
++extern int lxc_start(char *const argv[], struct lxc_handler *handler,
++		     const char *lxcpath, bool daemonize, int *error_num,
++		     unsigned int start_timeout);
++#else
+ extern int lxc_start(char *const argv[], struct lxc_handler *handler,
+ 		     const char *lxcpath, bool daemonize, int *error_num);
+-
++#endif
+ /*
+  * Start the specified command inside an application container
+  * @name         : the name of the container
+@@ -44,9 +49,15 @@ extern int lxc_start(char *const argv[], struct lxc_handler *handler,
+  * @daemonize    : whether or not the container is daemonized
+  * Returns 0 on success, < 0 otherwise
+  */
++#ifdef HAVE_ISULAD
++extern int lxc_execute(const char *name, char *const argv[], int quiet,
++		       struct lxc_handler *handler, const char *lxcpath,
++		       bool daemonize, int *error_num, unsigned int start_timeout);
++#else
+ extern int lxc_execute(const char *name, char *const argv[], int quiet,
+ 		       struct lxc_handler *handler, const char *lxcpath,
+ 		       bool daemonize, int *error_num);
++#endif
+ 
+ /*
+  * Close the fd associated with the monitoring
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index ce2b2bf..f622a63 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -1234,17 +1234,25 @@ reboot:
+ 		goto on_error;
+ 	}
+ 
++#ifdef HAVE_ISULAD
+ 	if (useinit) {
+ 		ret = lxc_execute(c->name, argv, 1, handler, c->config_path,
+-				  c->daemonize, &c->error_num);
++				  c->daemonize, &c->error_num, c->start_timeout);
+ 	}
+ 	else {
+-#ifdef HAVE_ISULAD
+ 		handler->disable_pty = c->disable_pty;
+ 		handler->open_stdin = c->open_stdin;
+-#endif
++		ret = lxc_start(argv, handler, c->config_path, c->daemonize,
++				&c->error_num, c->start_timeout);
++#else
++	if (useinit) {
++		ret = lxc_execute(c->name, argv, 1, handler, c->config_path,
++				  c->daemonize, &c->error_num);
++	}
++	else {
+ 		ret = lxc_start(argv, handler, c->config_path, c->daemonize,
+ 				&c->error_num);
++#endif
+ 	}
+ 
+ 	if (conf->reboot == REBOOT_REQ) {
+@@ -4200,8 +4208,13 @@ static int lxcapi_attach(struct lxc_container *c,
+ 
+ 	current_config = c->lxc_conf;
+ 
++#ifdef HAVE_ISULAD
++	ret = lxc_attach(c, exec_function, exec_payload, options,
++			 attached_process, &c->lxc_conf->errmsg);
++#else
+ 	ret = lxc_attach(c, exec_function, exec_payload, options,
+ 			 attached_process);
++#endif
+ 	current_config = NULL;
+ 	return ret;
+ }
+@@ -4221,7 +4234,11 @@ static int do_lxcapi_attach_run_wait(struct lxc_container *c,
+ 	command.program = (char *)program;
+ 	command.argv = (char **)argv;
+ 
++#ifdef HAVE_ISULAD
++	ret = lxc_attach(c, lxc_attach_run_command, &command, options, &pid, NULL);
++#else
+ 	ret = lxc_attach(c, lxc_attach_run_command, &command, options, &pid);
++#endif
+ 	if (ret < 0)
+ 		return ret;
+ 
+@@ -5593,6 +5610,23 @@ static bool do_lxcapi_get_container_pids(struct lxc_container *c, pid_t **pids,s
+ }
+ 
+ WRAP_API_2(bool, lxcapi_get_container_pids, pid_t **,size_t *)
++
++/* isulad add start timeout */
++static bool do_lxcapi_set_start_timeout(struct lxc_container *c,  unsigned int start_timeout)
++{
++	if (!c || !c->lxc_conf)
++		return false;
++	if (container_mem_lock(c)) {
++		ERROR("Error getting mem lock");
++		return false;
++	}
++	c->start_timeout = start_timeout;
++	container_mem_unlock(c);
++	return true;
++}
++
++WRAP_API_1(bool, lxcapi_set_start_timeout, unsigned int)
++
+ #endif
+ 
+ #ifdef HAVE_ISULAD
+@@ -5764,6 +5798,7 @@ struct lxc_container *lxc_container_new(const char *name, const char *configpath
+ 	c->want_open_stdin = lxcapi_want_open_stdin;
+ 	c->clean_container_resource = lxcapi_clean_container_resource;
+ 	c->get_container_pids = lxcapi_get_container_pids;
++	c->set_start_timeout = lxcapi_set_start_timeout;
+ #endif
+ 	return c;
+ 
+diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
+index 6ede70c..2951ac7 100644
+--- a/src/lxc/lxccontainer.h
++++ b/src/lxc/lxccontainer.h
+@@ -125,6 +125,12 @@ struct lxc_container {
+ 	 * */
+ 	char *ocihookfile;
+ 
++	/*! isulad:
++	 * \private
++	 * start_timeout.
++	 */
++	unsigned int start_timeout;
++
+ 	/*!
+ 	 * \brief Determine if \c /var/lib/lxc/$name/config exists.
+ 	 *
+@@ -960,6 +966,16 @@ struct lxc_container {
+ 	 * \return \c true on success, else \c false.
+ 	 */
+ 	bool (*get_container_pids)(struct lxc_container *c,pid_t **pids,size_t *pids_len);
++
++	/*! isulad add
++	 * \brief An API call to set start timeout
++	 *
++	 * \param c Container.
++	 * \param start_timeout Value of start timeout.
++	 *
++	 * \return \c true on success, else \c false.
++	 */
++	bool (*set_start_timeout)(struct lxc_container *c, unsigned int start_timeout);
+ };
+ 
+ /*!
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 70ce1bd..0bc1143 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -880,6 +880,21 @@ out_restore_sigmask:
+ }
+ 
+ #ifdef HAVE_ISULAD
++/* isulad: start timeout thread */
++typedef enum {
++	START_INIT,
++	START_TIMEOUT,
++	START_MAX,
++} start_timeout_t;
++
++static start_timeout_t global_timeout_state = START_INIT;
++static sem_t global_timeout_sem;
++
++struct start_timeout_conf {
++	unsigned int timeout;
++	int errfd;
++};
++
+ void trim_line(char *s)
+ {
+ 	size_t len;
+@@ -2285,6 +2300,12 @@ static int lxc_spawn(struct lxc_handler *handler)
+ 		goto out_delete_net;
+ 	}
+ 
++	if (START_TIMEOUT == global_timeout_state) {
++		lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
++		ERROR("Starting the container \"%s\" timeout.", name);
++		goto out_delete_net;
++	}
++
+        /* Tell the child to continue its initialization. We'll get
+         * LXC_SYNC_POST_OCI_PRESTART_HOOK when it is ready for us to run oci prestart hooks.
+         */
+@@ -2341,6 +2362,13 @@ static int lxc_spawn(struct lxc_handler *handler)
+ 		ERROR("Failed to run oci poststart hooks");
+ 		goto out_abort;
+ 	}
++
++	if (START_TIMEOUT == global_timeout_state) {
++		lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
++		ERROR("Starting the container \"%s\" timeout.", name);
++		goto out_abort;
++	}
++
+ #endif
+ 
+ 	ret = lxc_set_state(name, handler, RUNNING);
+@@ -2368,17 +2396,82 @@ out_sync_fini:
+ }
+ 
+ #ifdef HAVE_ISULAD
++/* isulad: start timeout thread function */
++static void* wait_start_timeout(void *arg)
++{
++	struct start_timeout_conf *conf = (struct start_timeout_conf *)arg;
++
++	sem_post(&global_timeout_sem);
++
++	if (!conf || conf->timeout < 1)
++		goto out;
++
++	sleep(conf->timeout);
++
++	global_timeout_state = START_TIMEOUT;
++
++out:
++	free(conf);
++	return ((void *)0);
++}
++
++/* isulad: create start timeout thread */
++static int create_start_timeout_thread(struct lxc_conf *conf, unsigned int start_timeout)
++{
++	int ret = 0;
++	pthread_t ptid;
++	pthread_attr_t attr;
++	struct start_timeout_conf *timeout_conf = NULL;
++
++	if (sem_init(&global_timeout_sem, 0, 0)) {
++		ERROR("Failed to init start timeout semaphore");/*lint !e613*/
++		ret = -1;
++		return ret;
++	}
++
++	timeout_conf = malloc(sizeof(struct start_timeout_conf));
++	if (timeout_conf == NULL) {
++		ERROR("Failed to malloc start timeout conf");
++		ret = -1;
++		goto out;
++	}
++
++	memset(timeout_conf, 0, sizeof(struct start_timeout_conf));
++	timeout_conf->errfd = conf->errpipe[1];
++	timeout_conf->timeout = start_timeout;
++
++	pthread_attr_init(&attr);
++	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
++	ret = pthread_create(&ptid, &attr, wait_start_timeout, timeout_conf);
++	if (ret != 0) {
++		ERROR("Create start wait timeout thread failed");
++		free(timeout_conf);
++		goto out;
++	}
++
++	sem_wait(&global_timeout_sem);
++out:
++	sem_destroy(&global_timeout_sem);
++	return ret;
++}
++
+ // isulad: send '128 + signal' if container is killed by signal.
+ #define EXIT_SIGNAL_OFFSET 128
+ #endif
+ 
++
++#ifdef HAVE_ISULAD
++int __lxc_start(struct lxc_handler *handler, struct lxc_operations *ops,
++		void *data, const char *lxcpath, bool daemonize, int *error_num,
++		unsigned int start_timeout)
++{
++	int exit_code;
++#else
+ int __lxc_start(struct lxc_handler *handler, struct lxc_operations *ops,
+ 		void *data, const char *lxcpath, bool daemonize, int *error_num)
+ {
+-	int ret, status;
+-#ifdef HAVE_ISULAD
+-    int exit_code;
+ #endif
++	int ret, status;
+ 	const char *name = handler->name;
+ 	struct lxc_conf *conf = handler->conf;
+ 	struct cgroup_ops *cgroup_ops;
+@@ -2393,6 +2486,16 @@ int __lxc_start(struct lxc_handler *handler, struct lxc_operations *ops,
+ 	handler->daemonize = daemonize;
+ 	cgroup_ops = handler->cgroup_ops;
+ 
++#ifdef HAVE_ISULAD
++	/* isulad: add start timeout limit */
++	if (start_timeout > 0) {
++		ret = create_start_timeout_thread(conf, start_timeout);
++		if (ret) {
++			ERROR("Failed to create start timeout thread for container \"%s\".", name);
++			goto out_abort;
++		}
++	}
++#endif
+ 	if (!attach_block_device(handler->conf)) {
+ 		ERROR("Failed to attach block device");
+ 		ret = -1;
+@@ -2574,14 +2677,18 @@ static struct lxc_operations start_ops = {
+ };
+ 
+ int lxc_start(char *const argv[], struct lxc_handler *handler,
+-	      const char *lxcpath, bool daemonize, int *error_num)
++	      const char *lxcpath, bool daemonize, int *error_num, unsigned int start_timeout)
+ {
+ 	struct start_args start_arg = {
+ 		.argv = argv,
+ 	};
+ 
+ 	TRACE("Doing lxc_start");
++#ifdef HAVE_ISULAD
++	return __lxc_start(handler, &start_ops, &start_arg, lxcpath, daemonize, error_num, start_timeout);
++#else
+ 	return __lxc_start(handler, &start_ops, &start_arg, lxcpath, daemonize, error_num);
++#endif
+ }
+ 
+ static void lxc_destroy_container_on_signal(struct lxc_handler *handler,
+diff --git a/src/lxc/start.h b/src/lxc/start.h
+index 4fc3ff7..cea37bc 100644
+--- a/src/lxc/start.h
++++ b/src/lxc/start.h
+@@ -170,8 +170,14 @@ extern void lxc_end(struct lxc_handler *handler);
+  */
+ extern int lxc_check_inherited(struct lxc_conf *conf, bool closeall,
+ 			       int *fds_to_ignore, size_t len_fds);
++#ifdef HAVE_ISULAD
++extern int __lxc_start(struct lxc_handler *handler,
++		struct lxc_operations* ops, void *data, const char *lxcpath,
++		bool daemonize, int *error_num, unsigned int start_timeout);
++#else
+ extern int __lxc_start(struct lxc_handler *, struct lxc_operations *, void *,
+ 		       const char *, bool, int *);
++#endif
+ 
+ extern int resolve_clone_flags(struct lxc_handler *handler);
+ 
+diff --git a/src/lxc/tools/arguments.h b/src/lxc/tools/arguments.h
+index a6d9967..41ea109 100644
+--- a/src/lxc/tools/arguments.h
++++ b/src/lxc/tools/arguments.h
+@@ -47,6 +47,8 @@ struct lxc_arguments {
+ 	const char *suffix; /* isulad add, suffix used for connect with parent of execed process*/
+ 	int disable_pty;
+ 	int open_stdin;
++	unsigned int start_timeout; /* isulad: Seconds for waiting on a container to start before it is killed*/
++	int64_t attach_timeout; /* for lxc-attach */
+ #endif
+ 
+ 	/* for lxc-console */
+diff --git a/src/lxc/tools/lxc_attach.c b/src/lxc/tools/lxc_attach.c
+index 48e18bb..a855a8d 100644
+--- a/src/lxc/tools/lxc_attach.c
++++ b/src/lxc/tools/lxc_attach.c
+@@ -81,6 +81,7 @@ static const struct option my_longopts[] = {
+ 	{"out-fifo", required_argument, 0, OPT_OUTPUT_FIFO},
+ 	{"err-fifo", required_argument, 0, OPT_STDERR_FIFO},
+ 	{"suffix", required_argument, 0, OPT_ATTACH_SUFFIX},
++	{"timeout", required_argument, 0, OPT_ATTACH_TIMEOUT},
+ #endif
+ 	LXC_COMMON_OPTIONS
+ };
+@@ -141,6 +142,7 @@ Options :\n\
+ #else
+ "\
+   --user        User ID (format: UID[:GID])\n\
++  --timeout   Timeout in seconds (default: 0)\n\
+ "
+ #endif
+ ,
+@@ -303,6 +305,13 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 	case OPT_ATTACH_SUFFIX:
+ 		args->suffix = arg;
+ 		break;
++	case OPT_ATTACH_TIMEOUT:
++		if(!is_non_negative_num(arg)) {
++			ERROR("Error attach timeout parameter:%s.\n", arg);
++			return -1;
++		}
++		args->attach_timeout = (unsigned int)atoll(arg);
++		break;
+ #endif
+ 	}
+ 
+@@ -581,6 +590,7 @@ int main(int argc, char *argv[])
+ 	attach_options.env_policy = env_policy;
+ 	attach_options.extra_env_vars = extra_env;
+ 	attach_options.extra_keep_env = extra_keep;
++	attach_options.timeout = my_args.attach_timeout;
+ 
+ 	if (my_args.argc > 0) {
+ 		command.program = my_args.argv[0];
+diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
+index 72a4494..b430a1e 100644
+--- a/src/lxc/tools/lxc_start.c
++++ b/src/lxc/tools/lxc_start.c
+@@ -29,6 +29,7 @@
+ #include "log.h"
+ 
+ #ifdef HAVE_ISULAD
++#include <ctype.h>
+ #include "isulad_utils.h"
+ #endif
+ 
+@@ -61,6 +62,7 @@ static const struct option my_longopts[] = {
+ 	{"start-timeout", required_argument, 0, OPT_START_TIMEOUT},
+ 	{"disable-pty", no_argument, 0, OPT_DISABLE_PTY},
+ 	{"open-stdin", no_argument, 0, OPT_OPEN_STDIN},
++	{"start-timeout", required_argument, 0, OPT_START_TIMEOUT},
+ #endif
+ 	LXC_COMMON_OPTIONS
+ };
+@@ -155,6 +157,13 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 	case OPT_OPEN_STDIN:
+ 		args->open_stdin = 1;
+ 		break;
++	case OPT_START_TIMEOUT:
++		if(!is_non_negative_num(arg)) {
++			fprintf(stderr, "Error start timeout parameter:%s.\n", arg);
++			return -1;
++		}
++		args->start_timeout = (unsigned int)atoi(arg);
++		break;
+ #endif
+ 
+ 	}
+@@ -354,6 +363,11 @@ int main(int argc, char *argv[])
+ 	if (my_args.open_stdin) {
+ 		c->want_open_stdin(c, true);
+ 	}
++
++	/* isulad: add start timeout */
++	if(my_args.start_timeout) {
++		c->set_start_timeout(c, my_args.start_timeout);
++	}
+ #endif
+ 
+ 	if (my_args.console)
+diff --git a/src/lxc/utils.c b/src/lxc/utils.c
+index ba69995..810b7fe 100644
+--- a/src/lxc/utils.c
++++ b/src/lxc/utils.c
+@@ -2167,4 +2167,15 @@ out:
+ 	return alive;
+ }
+ 
++bool is_non_negative_num(const char *s)
++{
++	if (!s || !strcmp(s, ""))
++		return false;
++	while(*s  != '\0') {
++		if(!isdigit(*s))
++			return false;
++		++s;
++	}
++	return true;
++}
+ #endif
+diff --git a/src/lxc/utils.h b/src/lxc/utils.h
+index a213ba7..39ef579 100644
+--- a/src/lxc/utils.h
++++ b/src/lxc/utils.h
+@@ -324,6 +324,8 @@ extern int unsigned long long lxc_get_process_startat(pid_t pid);
+ extern int lxc_setup_env_home(uid_t uid);
+ 
+ extern bool lxc_process_alive(pid_t pid, unsigned long long start_time);
++
++extern bool is_non_negative_num(const char *s);
+ #endif
+ 
+ #endif /* __LXC_UTILS_H */
+-- 
+1.8.3.1
+

0035-Seccomp-security-feature-enhanced.patch

@@ -0,0 +1,659 @@
+From 327e83ff13bec4bf1fa80ede9515d3f9531d7d1f Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Wed, 15 Apr 2020 06:37:43 -0400
+Subject: [PATCH 35/49] Seccomp security feature enhanced
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/conf.c    |   3 +-
+ src/lxc/seccomp.c | 548 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
+ 2 files changed, 548 insertions(+), 3 deletions(-)
+
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index 6856b1d..e8ee749 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -4730,8 +4730,7 @@ int lxc_setup(struct lxc_handler *handler)
+ 
+ 	// isulad: create link /etc/mtab for /proc/mounts
+ 	if (create_mtab_link() != 0) {
+-		ERROR("failed to create link /etc/mtab for target /proc/mounts");
+-		goto on_error;
++		return log_error(-1, "failed to create link /etc/mtab for target /proc/mounts");
+ 	}
+ #endif
+ 
+diff --git a/src/lxc/seccomp.c b/src/lxc/seccomp.c
+index 081d315..a75adb7 100644
+--- a/src/lxc/seccomp.c
++++ b/src/lxc/seccomp.c
+@@ -295,7 +295,11 @@ on_error:
+ #endif
+ 
+ #if HAVE_DECL_SECCOMP_SYSCALL_RESOLVE_NAME_ARCH
++#ifdef HAVE_ISULAD
++enum lxc_arch_t {
++#else
+ enum lxc_hostarch_t {
++#endif
+ 	lxc_seccomp_arch_all = 0,
+ 	lxc_seccomp_arch_native,
+ 	lxc_seccomp_arch_i386,
+@@ -351,8 +355,13 @@ int get_hostarch(void)
+ 	return lxc_seccomp_arch_unknown;
+ }
+ 
++#ifdef HAVE_ISULAD
++scmp_filter_ctx get_new_ctx(enum lxc_arch_t n_arch,
++			    uint32_t default_policy_action, uint32_t *architectures)
++#else
+ scmp_filter_ctx get_new_ctx(enum lxc_hostarch_t n_arch,
+ 			    uint32_t default_policy_action, bool *needs_merge)
++#endif
+ {
+ 	int ret;
+ 	uint32_t arch;
+@@ -475,10 +484,17 @@ scmp_filter_ctx get_new_ctx(enum lxc_hostarch_t n_arch,
+ 			return NULL;
+ 		}
+ 		TRACE("Removed native arch from main seccomp context");
+-
++#ifdef HAVE_ISULAD
++		*architectures = arch;
++#else
+ 		*needs_merge = true;
++#endif
+ 	} else {
++#ifdef HAVE_ISULAD
++		*architectures = SCMP_ARCH_NATIVE;
++#else
+ 		*needs_merge = false;
++#endif
+ 		TRACE("Arch %d already present in main seccomp context", (int)n_arch);
+ 	}
+ 
+@@ -510,7 +526,11 @@ bool do_resolve_add_rule(uint32_t arch, char *line, scmp_filter_ctx ctx,
+ 		if (ret < 0) {
+ 			errno = -ret;
+ 			SYSERROR("Failed loading rule to reject force umount");
++#ifdef HAVE_ISULAD
++			return true;
++#else
+ 			return false;
++#endif
+ 		}
+ 
+ 		INFO("Set seccomp rule to reject force umounts");
+@@ -519,14 +539,24 @@ bool do_resolve_add_rule(uint32_t arch, char *line, scmp_filter_ctx ctx,
+ 
+ 	nr = seccomp_syscall_resolve_name(line);
+ 	if (nr == __NR_SCMP_ERROR) {
++#ifdef HAVE_ISULAD
++		DEBUG("Failed to resolve syscall \"%s\"", line);
++		DEBUG("This syscall will NOT be handled by seccomp");
++#else
+ 		WARN("Failed to resolve syscall \"%s\"", line);
+ 		WARN("This syscall will NOT be handled by seccomp");
++#endif
+ 		return true;
+ 	}
+ 
+ 	if (nr < 0) {
++#ifdef HAVE_ISULAD
++		DEBUG("Got negative return value %d for syscall \"%s\"", nr, line);
++		DEBUG("This syscall will NOT be handled by seccomp");
++#else
+ 		WARN("Got negative return value %d for syscall \"%s\"", nr, line);
+ 		WARN("This syscall will NOT be handled by seccomp");
++#endif
+ 		return true;
+ 	}
+ 
+@@ -553,14 +583,42 @@ bool do_resolve_add_rule(uint32_t arch, char *line, scmp_filter_ctx ctx,
+ 					   rule->args_num, arg_cmp);
+ 	if (ret < 0) {
+ 		errno = -ret;
++#ifdef HAVE_ISULAD
++		DEBUG("Failed loading rule for %s (nr %d action %d (%s))",
++		      line, nr, rule->action, get_action_name(rule->action));
++#else
+ 		SYSERROR("Failed loading rule for %s (nr %d action %d (%s))",
+ 		         line, nr, rule->action, get_action_name(rule->action));
++#endif
+ 		return false;
+ 	}
+ 
+ 	return true;
+ }
+ 
++#ifdef HAVE_ISULAD
++#define SCMP_ARCH_INDEX_MAX 3
++
++struct scmp_ctx_info {
++	uint32_t architectures[SCMP_ARCH_INDEX_MAX];
++	enum lxc_arch_t lxc_arch[SCMP_ARCH_INDEX_MAX];
++	scmp_filter_ctx contexts[SCMP_ARCH_INDEX_MAX];
++	bool needs_merge[SCMP_ARCH_INDEX_MAX];
++};
++
++static int get_arch_index(enum lxc_arch_t arch, struct scmp_ctx_info *ctx)
++{
++	int i;
++
++	for (i = 0; i < SCMP_ARCH_INDEX_MAX; i++) {
++		if (ctx->lxc_arch[i] == arch)
++			return i;
++	}
++
++	return -1;
++}
++#endif
++
+ /*
+  * v2 consists of
+  * [x86]
+@@ -575,6 +633,493 @@ bool do_resolve_add_rule(uint32_t arch, char *line, scmp_filter_ctx ctx,
+  * write
+  * close
+  */
++#ifdef HAVE_ISULAD
++static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_conf *conf)
++{
++	int ret;
++	char *p;
++	enum lxc_arch_t cur_rule_arch, native_arch;
++	bool blacklist = false;
++	uint32_t default_policy_action = -1, default_rule_action = -1;
++	struct seccomp_v2_rule rule;
++	struct scmp_ctx_info ctx;
++
++	if (strncmp(line, "blacklist", 9) == 0)
++		blacklist = true;
++	else if (strncmp(line, "whitelist", 9) != 0) {
++		ERROR("Bad seccomp policy style \"%s\"", line);
++		return -1;
++	}
++
++	p = strchr(line, ' ');
++	if (p) {
++		default_policy_action = get_v2_default_action(p + 1);
++		if (default_policy_action == -2)
++			return -1;
++	}
++
++	/* for blacklist, allow any syscall which has no rule */
++	if (blacklist) {
++		if (default_policy_action == -1)
++			default_policy_action = SCMP_ACT_ALLOW;
++
++		if (default_rule_action == -1)
++			default_rule_action = SCMP_ACT_KILL;
++	} else {
++		if (default_policy_action == -1)
++			default_policy_action = SCMP_ACT_KILL;
++
++		if (default_rule_action == -1)
++			default_rule_action = SCMP_ACT_ALLOW;
++	}
++
++	memset(&ctx, 0, sizeof(ctx));
++	ctx.architectures[0] = SCMP_ARCH_NATIVE;
++	ctx.architectures[1] = SCMP_ARCH_NATIVE;
++	ctx.architectures[2] = SCMP_ARCH_NATIVE;
++	native_arch = get_hostarch();
++	cur_rule_arch = native_arch;
++	if (native_arch == lxc_seccomp_arch_amd64) {
++		cur_rule_arch = lxc_seccomp_arch_all;
++
++		ctx.lxc_arch[0] = lxc_seccomp_arch_i386;
++		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_i386,
++					      default_policy_action, &ctx.architectures[0]);
++		if (!ctx.contexts[0])
++			goto bad;
++
++		ctx.lxc_arch[1] = lxc_seccomp_arch_x32;
++		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_x32,
++					      default_policy_action, &ctx.architectures[1]);
++		if (!ctx.contexts[1])
++			goto bad;
++
++		ctx.lxc_arch[2] = lxc_seccomp_arch_amd64;
++		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_amd64,
++					      default_policy_action, &ctx.architectures[2]);
++		if (!ctx.contexts[2])
++			goto bad;
++#ifdef SCMP_ARCH_PPC
++	} else if (native_arch == lxc_seccomp_arch_ppc64) {
++		cur_rule_arch = lxc_seccomp_arch_all;
++
++		ctx.lxc_arch[0] = lxc_seccomp_arch_ppc;
++		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_ppc,
++					      default_policy_action, &ctx.architectures[0]);
++		if (!ctx.contexts[0])
++			goto bad;
++
++		ctx.lxc_arch[1] = lxc_seccomp_arch_ppc64;
++		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_ppc64,
++					      default_policy_action, &ctx.architectures[1]);
++		if (!ctx.contexts[1])
++			goto bad;
++#endif
++#ifdef SCMP_ARCH_ARM
++	} else if (native_arch == lxc_seccomp_arch_arm64) {
++		cur_rule_arch = lxc_seccomp_arch_all;
++
++		ctx.lxc_arch[0] = lxc_seccomp_arch_arm;
++		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_arm,
++					      default_policy_action, &ctx.architectures[0]);
++		if (!ctx.contexts[0])
++			goto bad;
++
++#ifdef SCMP_ARCH_AARCH64
++		ctx.lxc_arch[1] = lxc_seccomp_arch_arm64;
++		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_arm64,
++					      default_policy_action, &ctx.architectures[1]);
++		if (!ctx.contexts[1])
++			goto bad;
++#endif
++#endif
++#ifdef SCMP_ARCH_MIPS
++	} else if (native_arch == lxc_seccomp_arch_mips64) {
++		cur_rule_arch = lxc_seccomp_arch_all;
++
++		ctx.lxc_arch[0] = lxc_seccomp_arch_mips;
++		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_mips,
++					      default_policy_action, &ctx.architectures[0]);
++		if (!ctx.contexts[0])
++			goto bad;
++
++		ctx.lxc_arch[1] = lxc_seccomp_arch_mips64n32;
++		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_mips64n32,
++					      default_policy_action, &ctx.architectures[1]);
++		if (!ctx.contexts[1])
++			goto bad;
++
++		ctx.lxc_arch[2] = lxc_seccomp_arch_mips64;
++		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_mips64,
++					      default_policy_action, &ctx.architectures[2]);
++		if (!ctx.contexts[2])
++			goto bad;
++	} else if (native_arch == lxc_seccomp_arch_mipsel64) {
++		cur_rule_arch = lxc_seccomp_arch_all;
++		ctx.lxc_arch[0] = lxc_seccomp_arch_mipsel;
++		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_mipsel,
++					      default_policy_action, &ctx.architectures[0]);
++		if (!ctx.contexts[0])
++			goto bad;
++
++		ctx.lxc_arch[1] = lxc_seccomp_arch_mipsel64n32;
++		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_mipsel64n32,
++					      default_policy_action, &ctx.architectures[1]);
++		if (!ctx.contexts[1])
++			goto bad;
++
++		ctx.lxc_arch[2] = lxc_seccomp_arch_mipsel64;
++		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_mipsel64,
++					      default_policy_action, &ctx.architectures[2]);
++		if (!ctx.contexts[2])
++			goto bad;
++#endif
++	}
++
++	if (default_policy_action != SCMP_ACT_KILL) {
++		ret = seccomp_reset(conf->seccomp.seccomp_ctx, default_policy_action);
++		if (ret != 0) {
++			ERROR("Error re-initializing Seccomp");
++			return -1;
++		}
++
++		ret = seccomp_attr_set(conf->seccomp.seccomp_ctx, SCMP_FLTATR_CTL_NNP, 0);
++		if (ret < 0) {
++			errno = -ret;
++			SYSERROR("Failed to turn off no-new-privs");
++			return -1;
++		}
++
++#ifdef SCMP_FLTATR_ATL_TSKIP
++		ret = seccomp_attr_set(conf->seccomp.seccomp_ctx, SCMP_FLTATR_ATL_TSKIP, 1);
++		if (ret < 0) {
++			errno = -ret;
++			SYSWARN("Failed to turn on seccomp nop-skip, continuing");
++		}
++#endif
++	}
++
++	while (getline(&line, line_bufsz, f) != -1) {
++		if (line[0] == '#')
++			continue;
++
++		if (line[0] == '\0')
++			continue;
++
++		remove_trailing_newlines(line);
++
++		INFO("Processing \"%s\"", line);
++		if (line[0] == '[') {
++			/* Read the architecture for next set of rules. */
++			if (strcmp(line, "[x86]") == 0 ||
++			    strcmp(line, "[X86]") == 0) {
++				if (native_arch != lxc_seccomp_arch_i386 &&
++				    native_arch != lxc_seccomp_arch_amd64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_i386;
++			} else if (strcmp(line, "[x32]") == 0 ||
++				   strcmp(line, "[X32]") == 0) {
++				if (native_arch != lxc_seccomp_arch_amd64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_x32;
++			} else if (strcmp(line, "[X86_64]") == 0 ||
++				   strcmp(line, "[x86_64]") == 0) {
++				if (native_arch != lxc_seccomp_arch_amd64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_amd64;
++			} else if (strcmp(line, "[all]") == 0 ||
++				   strcmp(line, "[ALL]") == 0) {
++				cur_rule_arch = lxc_seccomp_arch_all;
++			}
++#ifdef SCMP_ARCH_ARM
++			else if (strcmp(line, "[arm]") == 0 ||
++				 strcmp(line, "[ARM]") == 0) {
++				if (native_arch != lxc_seccomp_arch_arm &&
++				    native_arch != lxc_seccomp_arch_arm64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_arm;
++			}
++#endif
++#ifdef SCMP_ARCH_AARCH64
++			else if (strcmp(line, "[arm64]") == 0 ||
++				 strcmp(line, "[ARM64]") == 0) {
++				if (native_arch != lxc_seccomp_arch_arm64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_arm64;
++			}
++#endif
++#ifdef SCMP_ARCH_PPC64LE
++			else if (strcmp(line, "[ppc64le]") == 0 ||
++				 strcmp(line, "[PPC64LE]") == 0) {
++				if (native_arch != lxc_seccomp_arch_ppc64le) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_ppc64le;
++			}
++#endif
++#ifdef SCMP_ARCH_PPC64
++			else if (strcmp(line, "[ppc64]") == 0 ||
++				 strcmp(line, "[PPC64]") == 0) {
++				if (native_arch != lxc_seccomp_arch_ppc64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_ppc64;
++			}
++#endif
++#ifdef SCMP_ARCH_PPC
++			else if (strcmp(line, "[ppc]") == 0 ||
++				 strcmp(line, "[PPC]") == 0) {
++				if (native_arch != lxc_seccomp_arch_ppc &&
++				    native_arch != lxc_seccomp_arch_ppc64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_ppc;
++			}
++#endif
++#ifdef SCMP_ARCH_MIPS
++			else if (strcmp(line, "[mips64]") == 0 ||
++				 strcmp(line, "[MIPS64]") == 0) {
++				if (native_arch != lxc_seccomp_arch_mips64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_mips64;
++			} else if (strcmp(line, "[mips64n32]") == 0 ||
++				   strcmp(line, "[MIPS64N32]") == 0) {
++				if (native_arch != lxc_seccomp_arch_mips64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_mips64n32;
++			} else if (strcmp(line, "[mips]") == 0 ||
++				   strcmp(line, "[MIPS]") == 0) {
++				if (native_arch != lxc_seccomp_arch_mips &&
++				    native_arch != lxc_seccomp_arch_mips64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_mips;
++			} else if (strcmp(line, "[mipsel64]") == 0 ||
++				   strcmp(line, "[MIPSEL64]") == 0) {
++				if (native_arch != lxc_seccomp_arch_mipsel64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_mipsel64;
++			} else if (strcmp(line, "[mipsel64n32]") == 0 ||
++				   strcmp(line, "[MIPSEL64N32]") == 0) {
++				if (native_arch != lxc_seccomp_arch_mipsel64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_mipsel64n32;
++			} else if (strcmp(line, "[mipsel]") == 0 ||
++				   strcmp(line, "[MIPSEL]") == 0) {
++				if (native_arch != lxc_seccomp_arch_mipsel &&
++				    native_arch != lxc_seccomp_arch_mipsel64) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_mipsel;
++			}
++#endif
++#ifdef SCMP_ARCH_S390X
++			else if (strcmp(line, "[s390x]") == 0 ||
++				 strcmp(line, "[S390X]") == 0) {
++				if (native_arch != lxc_seccomp_arch_s390x) {
++					cur_rule_arch = lxc_seccomp_arch_unknown;
++					continue;
++				}
++
++				cur_rule_arch = lxc_seccomp_arch_s390x;
++			}
++#endif
++			else {
++				goto bad_arch;
++			}
++
++			continue;
++		}
++
++		/* irrelevant arch - i.e. arm on i386 */
++		if (cur_rule_arch == lxc_seccomp_arch_unknown)
++			continue;
++
++		memset(&rule, 0, sizeof(rule));
++		/* read optional action which follows the syscall */
++		ret = parse_v2_rules(line, default_rule_action, &rule);
++		if (ret != 0) {
++			ERROR("Failed to interpret seccomp rule");
++			goto bad_rule;
++		}
++
++		if (cur_rule_arch == native_arch) {
++			/* add for native arch */
++			if (!do_resolve_add_rule(SCMP_ARCH_NATIVE, line,
++						 conf->seccomp.seccomp_ctx, &rule))
++				goto bad_rule;
++
++			INFO("Added native rule for arch %d for %s action %d(%s)",
++			     SCMP_ARCH_NATIVE, line, rule.action,
++			     get_action_name(rule.action));
++		} else if (cur_rule_arch != lxc_seccomp_arch_all) {
++			/* add for compat specified arch */
++			int arch_index = get_arch_index(cur_rule_arch, &ctx);
++			if (arch_index < 0)
++				goto bad_arch;
++
++			if (!do_resolve_add_rule(ctx.architectures[arch_index], line,
++						 ctx.contexts[arch_index], &rule))
++				goto bad_rule;
++
++			INFO("Added compat rule for arch %d for %s action %d(%s)",
++			     ctx.architectures[arch_index], line, rule.action,
++			     get_action_name(rule.action));
++			ctx.needs_merge[arch_index] = true;
++		} else {
++			/* add for all compat archs */
++			if (!do_resolve_add_rule(SCMP_ARCH_NATIVE, line,
++						 conf->seccomp.seccomp_ctx, &rule))
++				goto bad_rule;
++
++			INFO("Added native rule for arch %d for %s action %d(%s)",
++			     SCMP_ARCH_NATIVE, line, rule.action,
++			     get_action_name(rule.action));
++
++			if (ctx.architectures[0] != SCMP_ARCH_NATIVE) {
++				if (!do_resolve_add_rule(ctx.architectures[0], line,
++							 ctx.contexts[0], &rule))
++					goto bad_rule;
++
++				INFO("Added compat rule for arch %d for %s action %d(%s)",
++				     ctx.architectures[0], line, rule.action,
++				     get_action_name(rule.action));
++				ctx.needs_merge[0] = true;
++			}
++
++			if (ctx.architectures[1] != SCMP_ARCH_NATIVE) {
++				if (!do_resolve_add_rule(ctx.architectures[1], line,
++							 ctx.contexts[1], &rule))
++					goto bad_rule;
++
++				INFO("Added compat rule for arch %d for %s action %d(%s)",
++				     ctx.architectures[1], line, rule.action,
++				     get_action_name(rule.action));
++				ctx.needs_merge[1] = true;
++			}
++
++			if (ctx.architectures[2] != SCMP_ARCH_NATIVE) {
++				if (!do_resolve_add_rule(ctx.architectures[2], line,
++							ctx.contexts[2], &rule))
++					goto bad_rule;
++
++				INFO("Added native rule for arch %d for %s action %d(%s)",
++				     ctx.architectures[2], line, rule.action,
++				     get_action_name(rule.action));
++				ctx.needs_merge[2] = true;
++			}
++		}
++
++	}
++
++	INFO("Merging compat seccomp contexts into main context");
++	if (ctx.contexts[0]) {
++		if (ctx.needs_merge[0]) {
++			ret = seccomp_merge(conf->seccomp.seccomp_ctx, ctx.contexts[0]);
++			if (ret < 0) {
++				ERROR("%s - Failed to merge first compat seccomp "
++				      "context into main context", strerror(-ret));
++				goto bad;
++			}
++
++			TRACE("Merged first compat seccomp context into main context");
++		} else {
++			seccomp_release(ctx.contexts[0]);
++			ctx.contexts[0] = NULL;
++		}
++	}
++
++	if (ctx.contexts[1]) {
++		if (ctx.needs_merge[1]) {
++			ret = seccomp_merge(conf->seccomp.seccomp_ctx, ctx.contexts[1]);
++			if (ret < 0) {
++				ERROR("%s - Failed to merge second compat seccomp "
++				      "context into main context", strerror(-ret));
++				goto bad;
++			}
++
++			TRACE("Merged second compat seccomp context into main context");
++		} else {
++			seccomp_release(ctx.contexts[1]);
++			ctx.contexts[1] = NULL;
++		}
++	}
++
++	if (ctx.contexts[2]) {
++		if (ctx.needs_merge[2]) {
++			ret = seccomp_merge(conf->seccomp.seccomp_ctx, ctx.contexts[2]);
++			if (ret < 0) {
++				ERROR("%s - Failed to merge third compat seccomp "
++				      "context into main context", strerror(-ret));
++				goto bad;
++			}
++
++			TRACE("Merged third compat seccomp context into main context");
++		} else {
++			seccomp_release(ctx.contexts[2]);
++			ctx.contexts[2] = NULL;
++		}
++	}
++
++	free(line);
++	return 0;
++
++bad_arch:
++	ERROR("Unsupported architecture \"%s\"", line);
++
++bad_rule:
++bad:
++	if (ctx.contexts[0])
++		seccomp_release(ctx.contexts[0]);
++
++	if (ctx.contexts[1])
++		seccomp_release(ctx.contexts[1]);
++
++	if (ctx.contexts[2])
++		seccomp_release(ctx.contexts[2]);
++
++	free(line);
++
++	return -1;
++}
++#else
+ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_conf *conf)
+ {
+ 	int ret;
+@@ -1067,6 +1612,7 @@ bad:
+ 
+ 	return -1;
+ }
++#endif
+ #else /* HAVE_DECL_SECCOMP_SYSCALL_RESOLVE_NAME_ARCH */
+ static int parse_config_v2(FILE *f, char *line, struct lxc_conf *conf)
+ {
+-- 
+1.8.3.1
+

0035-lxc-fixup-builds-with-newer-glibc.patch

@@ -1,25 +0,0 @@
-From cf85b9009e0085ec861d9294e6e04f1baa2ba28d Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Wed, 16 Jan 2019 20:53:25 +0800
-Subject: [PATCH 035/140] lxc: fixup builds with newer glibc
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/utils.c | 1 +
- 1 file changed, 1 insertion(+)
-
-diff --git a/src/lxc/utils.c b/src/lxc/utils.c
-index 120a13d..c8fb993 100644
---- a/src/lxc/utils.c
-+++ b/src/lxc/utils.c
-@@ -44,6 +44,7 @@
- #include <sys/stat.h>
- #include <sys/types.h>
- #include <sys/wait.h>
-+#include <sys/sysmacros.h>
- #include <unistd.h>
- #include <pwd.h>
- 
--- 
-1.8.3.1
-

0036-Security-coding-modification.patch

@@ -0,0 +1,99 @@
+From ff35c1cd6118668e13f7ca83d7d704bb9363155a Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Wed, 15 Apr 2020 07:19:03 -0400
+Subject: [PATCH 36/49] Security coding modification
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/commands_utils.c |  2 +-
+ src/lxc/confile.c        | 22 ++++++++++++++++++++++
+ src/lxc/lxccontainer.c   |  8 ++++++++
+ 3 files changed, 31 insertions(+), 1 deletion(-)
+
+diff --git a/src/lxc/commands_utils.c b/src/lxc/commands_utils.c
+index 2f2670d..c5fc094 100644
+--- a/src/lxc/commands_utils.c
++++ b/src/lxc/commands_utils.c
+@@ -114,7 +114,7 @@ int lxc_make_abstract_socket_name(char *path, size_t pathlen,
+ 	}
+ 
+ 	ret = snprintf(offset, len, "%s/%s/%s", lxcpath, name, suffix);
+-	if (ret < 0)
++	if (ret < 0 || (size_t)ret >= len)
+ 		return log_error_errno(-1, errno, "Failed to create abstract socket name");
+ 
+ 	/*
+diff --git a/src/lxc/confile.c b/src/lxc/confile.c
+index f00afe9..8790494 100644
+--- a/src/lxc/confile.c
++++ b/src/lxc/confile.c
+@@ -4288,7 +4288,12 @@ static int get_config_prlimit(const char *key, char *retv, int inlen,
+ 
+ 	lxc_list_for_each(it, &c->limits) {
+ 		/* 2 colon separated 64 bit integers or the word 'unlimited' */
++#ifdef HAVE_ISULAD
++#define MAX_LIMIT_BUF_LEN ((INTTYPE_TO_STRLEN(uint64_t) * 2) + 2)
++		char buf[MAX_LIMIT_BUF_LEN] = { 0 };
++#else
+ 		char buf[INTTYPE_TO_STRLEN(uint64_t) * 2 + 2];
++#endif
+ 		int partlen;
+ 		struct lxc_limit *lim = it->elem;
+ 
+@@ -4296,17 +4301,34 @@ static int get_config_prlimit(const char *key, char *retv, int inlen,
+ 			memcpy(buf, "unlimited", STRLITERALLEN("unlimited") + 1);
+ 			partlen = STRLITERALLEN("unlimited");
+ 		} else {
++#ifdef HAVE_ISULAD
++			partlen = snprintf(buf, MAX_LIMIT_BUF_LEN, "%" PRIu64, (uint64_t)lim->limit.rlim_cur);
++			if (partlen < 0 || partlen >= MAX_LIMIT_BUF_LEN) {
++				return -1;
++			}
++#else
+ 			partlen = sprintf(buf, "%" PRIu64,
+ 					  (uint64_t)lim->limit.rlim_cur);
++#endif
+ 		}
+ 
+ 		if (lim->limit.rlim_cur != lim->limit.rlim_max) {
+ 			if (lim->limit.rlim_max == RLIM_INFINITY)
+ 				memcpy(buf + partlen, ":unlimited",
+ 				       STRLITERALLEN(":unlimited") + 1);
++#ifdef HAVE_ISULAD
++			else {
++				int nret = snprintf(buf + partlen, (MAX_LIMIT_BUF_LEN - partlen),
++								 ":%" PRIu64, (uint64_t)lim->limit.rlim_max);
++				if (nret < 0 || nret >= (MAX_LIMIT_BUF_LEN - partlen)) {
++					return -1;
++				}
++			}
++#else
+ 			else
+ 				sprintf(buf + partlen, ":%" PRIu64,
+ 					(uint64_t)lim->limit.rlim_max);
++#endif
+ 		}
+ 
+ 		if (get_all) {
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index f622a63..ab10ac6 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -3243,6 +3243,14 @@ static bool container_destroy(struct lxc_container *c,
+ 	if (ret < 0) {
+ 		ERROR("Failed to destroy directory \"%s\" for \"%s\"", path,
+ 		      c->name);
++#ifdef HAVE_ISULAD
++		char msg[BUFSIZ] = { 0 };
++		ret = snprintf(msg, BUFSIZ, "Failed to destroy directory \"%s\": %s", path, errno ? strerror(errno) : "error");
++		if (ret < 0 || ret >= BUFSIZ) {
++			ERROR("Sprintf failed");
++			goto out;
++		}
++#endif
+ 		goto out;
+ 	}
+ 	INFO("Destroyed directory \"%s\" for \"%s\"", path, c->name);
+-- 
+1.8.3.1
+

0036-drop_caps-add-drop-caps-of-current-process.patch

@@ -1,256 +0,0 @@
-From 35ef612b75dee5aa2bf313a8ceb24a7636319582 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Tue, 15 Jan 2019 22:55:06 -0500
-Subject: [PATCH 036/140] drop_caps: add drop caps of current process
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/attach.c         | 26 +++++++++++++---
- src/lxc/cgroups/cgfsng.c | 35 ++++++++++-----------
- src/lxc/conf.c           | 79 ++++++++++++++++++++++++++++++++++++++++++++++++
- src/lxc/conf.h           |  1 +
- src/lxc/start.c          | 16 ++++++++++
- 5 files changed, 135 insertions(+), 22 deletions(-)
-
-diff --git a/src/lxc/attach.c b/src/lxc/attach.c
-index 8cbbf96..3f60fe1 100644
---- a/src/lxc/attach.c
-+++ b/src/lxc/attach.c
-@@ -936,11 +936,6 @@ static int attach_child_main(struct attach_clone_payload *payload)
- 		TRACE("Loaded seccomp profile");
- 	}
- 
--	close(payload->ipc_socket);
--	payload->ipc_socket = -EBADF;
--	lxc_proc_put_context_info(init_ctx);
--	payload->init_ctx = NULL;
--
- 	/* The following is done after the communication socket is shut down.
- 	 * That way, all errors that might (though unlikely) occur up until this
- 	 * point will have their messages printed to the original stderr (if
-@@ -997,9 +992,30 @@ static int attach_child_main(struct attach_clone_payload *payload)
- 	if (new_gid == ns_root_gid)
- 		new_gid = LXC_INVALID_GID;
- 
-+	if (prctl(PR_SET_KEEPCAPS, 1) < 0) {
-+		SYSERROR("Failed to keep permitted capabilities");
-+		goto on_error;
-+	}
-+
- 	if (!lxc_switch_uid_gid(new_uid, new_gid))
- 		goto on_error;
- 
-+	if (prctl(PR_SET_KEEPCAPS, 0) < 0) {
-+		SYSERROR("Failed to clear permitted capabilities");
-+		goto on_error;
-+	}
-+
-+	if (init_ctx->container && init_ctx->container->lxc_conf &&
-+			lxc_drop_caps(init_ctx->container->lxc_conf) != 0) {
-+		ERROR("Failed to drop caps.");
-+		goto on_error;
-+	}
-+
-+	close(payload->ipc_socket);
-+	payload->ipc_socket = -EBADF;
-+	lxc_proc_put_context_info(init_ctx);
-+	payload->init_ctx = NULL;
-+
- 	/* We're done, so we can now do whatever the user intended us to do. */
- 	_exit(payload->exec_function(payload->exec_payload, msg_fd));
- 
-diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index 8b913a6..bc1481d 100644
---- a/src/lxc/cgroups/cgfsng.c
-+++ b/src/lxc/cgroups/cgfsng.c
-@@ -2664,11 +2664,11 @@ __cgfsng_ops static bool cgfsng_data_init(struct cgroup_ops *ops, struct lxc_han
- {
- 	const char *cgroup_pattern;
- 	char *container_cgroup, *tmp;
--	struct lxc_conf *conf = handler->conf;
-+	struct lxc_conf *conf = NULL;
- 	size_t len;
- 
--	if (!conf)
--		return false;
-+	if (handler)
-+		conf = handler->conf;
- 
- 	/* copy system-wide cgroup information */
- 	cgroup_pattern = lxc_global_config_value("lxc.cgroup.pattern");
-@@ -2680,21 +2680,22 @@ __cgfsng_ops static bool cgfsng_data_init(struct cgroup_ops *ops, struct lxc_han
- 	ops->cgroup_pattern = must_copy_string(cgroup_pattern);
- 
- 	/* isulad: init ops->container_cgroup here instead of in cgfsng_payload_create*/
--	if (conf->cgroup_meta.dir)
--		tmp = lxc_string_join("/", (const char *[]){conf->cgroup_meta.dir, handler->name, NULL}, false);
--	else
--		tmp = lxc_string_replace("%n", handler->name, ops->cgroup_pattern);
--	if (!tmp) {
--		ERROR("Failed expanding cgroup name pattern");
--		return false;
--	}
--
--	len = strlen(tmp) + 1;
--	container_cgroup = must_realloc(NULL, len);
--	(void)strlcpy(container_cgroup, tmp, len);
--	free(tmp);
--	ops->container_cgroup = container_cgroup;
-+	if (conf) {
-+		if (conf->cgroup_meta.dir)
-+			tmp = lxc_string_join("/", (const char *[]){conf->cgroup_meta.dir, handler->name, NULL}, false);
-+		else
-+			tmp = lxc_string_replace("%n", handler->name, ops->cgroup_pattern);
-+		if (!tmp) {
-+			ERROR("Failed expanding cgroup name pattern");
-+			return false;
-+		}
- 
-+		len = strlen(tmp) + 1;
-+		container_cgroup = must_realloc(NULL, len);
-+		(void)strlcpy(container_cgroup, tmp, len);
-+		free(tmp);
-+		ops->container_cgroup = container_cgroup;
-+	}
- 	return true;
- }
- 
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index fea0f59..6134ed3 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -4220,6 +4220,85 @@ int lxc_setup(struct lxc_handler *handler)
- 	return 0;
- }
- 
-+/* isulad drop caps for container*/
-+int lxc_drop_caps(struct lxc_conf *conf)
-+{
-+#define __DEF_CAP_TO_MASK(x) (1U << ((x) & 31))
-+#if HAVE_LIBCAP
-+	struct lxc_list *iterator;
-+	char *keep_entry;
-+	int i, capid;
-+	int numcaps = lxc_caps_last_cap() + 1;
-+	struct lxc_list *caps = NULL;
-+
-+	if (lxc_list_empty(&conf->keepcaps))
-+		return 0;
-+
-+	caps = &conf->keepcaps;
-+
-+	if (numcaps <= 0 || numcaps > 200)
-+		return -1;
-+
-+	// caplist[i] is 1 if we keep capability i
-+	int *caplist = alloca(numcaps * sizeof(int));
-+	memset(caplist, 0, numcaps * sizeof(int));
-+
-+	lxc_list_for_each(iterator, caps) {
-+
-+		keep_entry = iterator->elem;
-+		/* isulad: Do not keep any cap*/
-+		if (strcmp(keep_entry, "ISULAD_KEEP_NONE") == 0) {
-+			DEBUG("Do not keep any capability");
-+			for(i = 0; i < numcaps; i++) {
-+				caplist[i] = 0;
-+			}
-+			break;
-+		}
-+
-+		capid = parse_cap(keep_entry);
-+
-+		if (capid == -2)
-+			continue;
-+
-+	        if (capid < 0) {
-+			ERROR("unknown capability %s", keep_entry);
-+			return -1;
-+		}
-+
-+		DEBUG("keep capability '%s' (%d)", keep_entry, capid);
-+
-+		caplist[capid] = 1;
-+	}
-+
-+	struct __user_cap_header_struct cap_header_data;
-+	struct __user_cap_data_struct cap_data_data[2];
-+
-+	cap_user_header_t cap_header = &cap_header_data;
-+	cap_user_data_t cap_data = &cap_data_data[0];
-+
-+	memset(cap_header, 0 ,sizeof(struct __user_cap_header_struct));
-+	memset(cap_data, 0, sizeof(struct __user_cap_data_struct) * 2);
-+
-+	cap_header->pid = 0;
-+	cap_header->version = _LINUX_CAPABILITY_VERSION;
-+
-+	for (i = 0; i < numcaps; i++) {
-+		if (caplist[i]) {
-+			cap_data[CAP_TO_INDEX(i)].effective = cap_data[CAP_TO_INDEX(i)].effective | __DEF_CAP_TO_MASK(i);
-+			cap_data[CAP_TO_INDEX(i)].permitted = cap_data[CAP_TO_INDEX(i)].permitted | __DEF_CAP_TO_MASK(i);
-+			cap_data[CAP_TO_INDEX(i)].inheritable = cap_data[CAP_TO_INDEX(i)].inheritable | __DEF_CAP_TO_MASK(i);
-+		}
-+	}
-+
-+	if (capset(cap_header, cap_data)) {
-+			SYSERROR("Failed to set capabilitys");
-+			return -1;
-+	}
-+
-+#endif
-+	return 0;
-+}
-+
- struct oci_hook_conf {
- 	defs_hook *ocihook;
- 
-diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index 44feb98..b92c48e 100644
---- a/src/lxc/conf.h
-+++ b/src/lxc/conf.h
-@@ -498,6 +498,7 @@ int lxc_clear_init_args(struct lxc_conf *lxc_conf);
- int lxc_clear_populate_devices(struct lxc_conf *c);
- int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
- int lxc_clear_rootfs_ro_paths(struct lxc_conf *c);
-+int lxc_drop_caps(struct lxc_conf *conf);
- 
- /* isulad add end */
- 
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 040909c..357e81d 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -1411,6 +1411,11 @@ static int do_start(void *data)
- 		}
- 	}
- 
-+	if (prctl(PR_SET_KEEPCAPS, 1) < 0) {
-+		SYSERROR("Failed to keep permitted capabilities");
-+		goto out_warn_father;
-+	}
-+
- 	/* The container has been setup. We can now switch to an unprivileged
- 	 * uid/gid.
- 	 */
-@@ -1448,6 +1453,17 @@ static int do_start(void *data)
- 		goto out_warn_father;
- 	}
- 
-+	/* isulad: drop the cap of current process */
-+	if (prctl(PR_SET_KEEPCAPS, 0) < 0) {
-+		SYSERROR("Failed to clear permitted capabilities");
-+		goto out_warn_father;
-+	}
-+
-+	if (lxc_drop_caps(handler->conf)) {
-+		SYSERROR("Failed to drop caps");
-+		goto out_warn_father;
-+	}
-+
- 	/* After this call, we are in error because this ops should not return
- 	 * as it execs.
- 	 */
--- 
-1.8.3.1
-

0037-cgfsng-fix-build-error-device_cgroup_rule_parse.patch

@@ -0,0 +1,35 @@
+From 5f8d8bb11c42fbf6bca39438530db88729e50f5e Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Thu, 16 Apr 2020 11:12:28 +0800
+Subject: [PATCH 37/49] cgfsng: fix build error device_cgroup_rule_parse
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/cgroups/cgfsng.c | 4 +---
+ 1 file changed, 1 insertion(+), 3 deletions(-)
+
+diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
+index 1047c08..76576c5 100644
+--- a/src/lxc/cgroups/cgfsng.c
++++ b/src/lxc/cgroups/cgfsng.c
+@@ -3043,8 +3043,7 @@ static int device_cgroup_parse_access(struct device_item *device, const char *va
+ 	return 0;
+ }
+ 
+-#ifndef HAVE_ISULAD
+-static int device_cgroup_rule_parse(struct device_item *device, const char *key,
++int device_cgroup_rule_parse(struct device_item *device, const char *key,
+ 				    const char *val)
+ {
+ 	int count, ret;
+@@ -3130,7 +3129,6 @@ static int device_cgroup_rule_parse(struct device_item *device, const char *key,
+ 
+ 	return device_cgroup_parse_access(device, ++val);
+ }
+-#endif
+ 
+ #ifdef HAVE_ISULAD
+ __cgfsng_ops static int cgfsng_set(struct cgroup_ops *ops,
+-- 
+1.8.3.1
+

0037-restore-default-signal-handlers-and-set-umask-0027.patch

@@ -1,113 +0,0 @@
-From 3ac4fd8b31c3c9e650cbe02383f4d65cfa078b20 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Tue, 15 Jan 2019 23:14:14 -0500
-Subject: [PATCH 037/140] restore default signal handlers and set umask 0027
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/attach.c | 22 +++++++++++++++++++++-
- src/lxc/conf.c   |  3 +++
- src/lxc/start.c  | 18 +++++++++++++++++-
- 3 files changed, 41 insertions(+), 2 deletions(-)
-
-diff --git a/src/lxc/attach.c b/src/lxc/attach.c
-index 3f60fe1..4ccdd74 100644
---- a/src/lxc/attach.c
-+++ b/src/lxc/attach.c
-@@ -750,7 +750,7 @@ static void lxc_put_attach_clone_payload(struct attach_clone_payload *p)
- 
- static int attach_child_main(struct attach_clone_payload *payload)
- {
--	int fd, lsm_fd, ret;
-+	int fd, lsm_fd, ret, i;
- 	uid_t new_uid;
- 	gid_t new_gid;
- 	uid_t ns_root_uid = 0;
-@@ -761,11 +761,31 @@ static int attach_child_main(struct attach_clone_payload *payload)
- 	bool needs_lsm = (options->namespaces & CLONE_NEWNS) &&
- 			 (options->attach_flags & LXC_ATTACH_LSM) &&
- 			 init_ctx->lsm_label;
-+	sigset_t mask;
-+
- 
- 	/*isulad: record errpipe fd*/
- 	msg_fd = init_ctx->container->lxc_conf->errpipe[1];
- 	init_ctx->container->lxc_conf->errpipe[1] = -1;
- 
-+	/*isulad: set system umask 0027 for safe control */
-+	umask(0027);
-+
-+	/*isulad: restore default signal handlers and unblock all signals*/
-+	for (i = 1; i < NSIG; i++)
-+		signal(i, SIG_DFL);
-+
-+	ret = sigfillset(&mask);
-+	if (ret < 0) {
-+		SYSERROR("Failed to fill signal mask");
-+		goto on_error;;
-+	}
-+	ret = sigprocmask(SIG_UNBLOCK, &mask, NULL);
-+	if (ret < 0) {
-+		SYSERROR("Failed to set signal mask");
-+		goto on_error;
-+	}
-+
- 	/* A description of the purpose of this functionality is provided in the
- 	 * lxc-attach(1) manual page. We have to remount here and not in the
- 	 * parent process, otherwise /proc may not properly reflect the new pid
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 6134ed3..88cebfd 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -4180,6 +4180,9 @@ int lxc_setup(struct lxc_handler *handler)
- 		}
- 	}
- 
-+	/*isulad: set system umask 0027 for safe control*/
-+	umask(0027);
-+
- 	ret = setup_personality(lxc_conf->personality);
- 	if (ret < 0) {
- 		ERROR("Failed to set personality");
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 357e81d..708ab7f 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -1103,7 +1103,7 @@ void lxc_abort(const char *name, struct lxc_handler *handler)
- 
- static int do_start(void *data)
- {
--	int ret;
-+	int ret, i;
- 	char path[PATH_MAX];
- 	uid_t new_uid;
- 	gid_t new_gid;
-@@ -1112,9 +1112,25 @@ static int do_start(void *data)
- 	gid_t nsgid = 0;
- 	int devnull_fd = -1;
- 	struct lxc_handler *handler = data;
-+	sigset_t mask;
- 
- 	lxc_sync_fini_parent(handler);
- 
-+	/*isulad: restore default signal handlers and unblock all signals*/
-+	for (i = 1; i < NSIG; i++)
-+		signal(i, SIG_DFL);
-+
-+	ret = sigfillset(&mask);
-+	if (ret < 0) {
-+		SYSERROR("Failed to fill signal mask");
-+		goto out_warn_father;;
-+	}
-+	ret = sigprocmask(SIG_UNBLOCK, &mask, NULL);
-+	if (ret < 0) {
-+		SYSERROR("Failed to set signal mask");
-+		goto out_warn_father;
-+	}
-+
- 	/* This prctl must be before the synchro, so if the parent dies before
- 	 * we set the parent death signal, we will detect its death with the
- 	 * synchro right after, otherwise we have a window where the parent can
--- 
-1.8.3.1
-

0038-Ignore-errors-when-loading-rules-fail.patch

@@ -0,0 +1,31 @@
+From 06a4fd31155821e4425a94ed0ff0d2353d5d940d Mon Sep 17 00:00:00 2001
+From: wujing <wujing50@huawei.com>
+Date: Wed, 15 Apr 2020 23:28:36 -0400
+Subject: [PATCH 38/49] Ignore errors when loading rules fail
+
+Signed-off-by: wujing <wujing50@huawei.com>
+---
+ src/lxc/seccomp.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/src/lxc/seccomp.c b/src/lxc/seccomp.c
+index a75adb7..b7c4de3 100644
+--- a/src/lxc/seccomp.c
++++ b/src/lxc/seccomp.c
+@@ -586,11 +586,12 @@ bool do_resolve_add_rule(uint32_t arch, char *line, scmp_filter_ctx ctx,
+ #ifdef HAVE_ISULAD
+ 		DEBUG("Failed loading rule for %s (nr %d action %d (%s))",
+ 		      line, nr, rule->action, get_action_name(rule->action));
++		return true;
+ #else
+ 		SYSERROR("Failed loading rule for %s (nr %d action %d (%s))",
+ 		         line, nr, rule->action, get_action_name(rule->action));
+-#endif
+ 		return false;
++#endif
+ 	}
+ 
+ 	return true;
+-- 
+1.8.3.1
+

0038-make-the-given-terminal-as-controlling-terminal.patch

@@ -1,46 +0,0 @@
-From a1f81f063eb08dbf3ca99d298fef016b89d7d0d1 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Tue, 15 Jan 2019 23:24:21 -0500
-Subject: [PATCH 038/140] make the given terminal as controlling terminal
-
-Avoid warning when start cmd /bin/sh :
-sh: cannot set terminal process group (-1): Inappropriate ioctl for device
-sh: no job control in this shell
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/start.c | 10 ++++++++--
- 1 file changed, 8 insertions(+), 2 deletions(-)
-
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 708ab7f..ec61b32 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -1330,7 +1330,15 @@ static int do_start(void *data)
- 	 * setup on its console ie. the pty allocated in lxc_terminal_setup() so
- 	 * make sure that that pty is stdin,stdout,stderr.
- 	 */
-+	 setsid();
- 	 if (handler->conf->console.slave >= 0) {
-+		/* isulad:make the given terminal as controlling terminal to avoid warning
-+		* sh: cannot set terminal process group (-1): Inappropriate ioctl for device
-+		* sh: no job control in this shell */
-+		 if (ioctl(handler->conf->console.slave, TIOCSCTTY, NULL) < 0) {
-+			 ERROR("Faild to make the given terminal the controlling terminal of the calling process");
-+			 goto out_warn_father;
-+		 }
- 		 if (handler->daemonize || !handler->conf->is_execute)
- 			 ret = set_stdfds(handler->conf->console.slave);
- 		 else
-@@ -1377,8 +1385,6 @@ static int do_start(void *data)
- 		devnull_fd = -1;
- 	}
- 
--	setsid();
--
- 	if (handler->conf->init_cwd) {
- 		/* isulad: try to craete workdir if not exist */
- 		struct stat st;
--- 
-1.8.3.1
-

0039-net-adapt-to-isulad.patch

@@ -0,0 +1,51 @@
+From c474b4e528e591680c6f9cbbcb27bdfe4e5f94f9 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Thu, 16 Apr 2020 12:01:39 +0800
+Subject: [PATCH 39/49] net: adapt to isulad
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/commands_utils.c |  2 +-
+ src/lxc/network.c        | 10 +++++++++-
+ 2 files changed, 10 insertions(+), 2 deletions(-)
+
+diff --git a/src/lxc/commands_utils.c b/src/lxc/commands_utils.c
+index c5fc094..2f2670d 100644
+--- a/src/lxc/commands_utils.c
++++ b/src/lxc/commands_utils.c
+@@ -114,7 +114,7 @@ int lxc_make_abstract_socket_name(char *path, size_t pathlen,
+ 	}
+ 
+ 	ret = snprintf(offset, len, "%s/%s/%s", lxcpath, name, suffix);
+-	if (ret < 0 || (size_t)ret >= len)
++	if (ret < 0)
+ 		return log_error_errno(-1, errno, "Failed to create abstract socket name");
+ 
+ 	/*
+diff --git a/src/lxc/network.c b/src/lxc/network.c
+index b442ed5..19adb23 100644
+--- a/src/lxc/network.c
++++ b/src/lxc/network.c
+@@ -3448,10 +3448,18 @@ static int lxc_network_setup_in_child_namespaces_common(struct lxc_netdev *netde
+ 
+ 	/* set the network device up */
+ 	if (netdev->flags & IFF_UP) {
++
++#ifdef HAVE_ISULAD
++		if (netdev->name[0] != '\0') {
++			err = lxc_netdev_up(netdev->name);
++			if (err)
++				return log_error_errno(-1, -err, "Failed to set network device \"%s\" up", netdev->name);
++		}
++#else
+ 		err = lxc_netdev_up(netdev->name);
+ 		if (err)
+ 			return log_error_errno(-1, -err, "Failed to set network device \"%s\" up", netdev->name);
+-
++#endif
+ 		/* the network is up, make the loopback up too */
+ 		err = lxc_netdev_up("lo");
+ 		if (err)
+-- 
+1.8.3.1
+

0039-print-error-message-when-container-start-failed.patch

@@ -1,816 +0,0 @@
-From 24f908199916fc92cb7935bfccc19244d3a8d864 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Wed, 16 Jan 2019 14:38:38 +0800
-Subject: [PATCH 039/140] print error message when container start failed
-
-Signed-off-by: tanyifeng <tanyifeng1@huawei.com>
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/attach.c          |   6 +--
- src/lxc/cgroups/cgfsng.c  |  22 +++++++--
- src/lxc/cgroups/cgroup.c  |   4 +-
- src/lxc/cgroups/cgroup.h  |   2 +
- src/lxc/conf.c            | 122 ++++++++++++++++++++++++++--------------------
- src/lxc/conf.h            |   5 +-
- src/lxc/execute.c         |   3 +-
- src/lxc/lxccontainer.c    |  37 ++++++++++++--
- src/lxc/start.c           |  14 ++++--
- src/lxc/start.h           |   2 +-
- src/lxc/tools/lxc_start.c |   3 ++
- 11 files changed, 147 insertions(+), 73 deletions(-)
-
-diff --git a/src/lxc/attach.c b/src/lxc/attach.c
-index 4ccdd74..b44ea74 100644
---- a/src/lxc/attach.c
-+++ b/src/lxc/attach.c
-@@ -1396,7 +1396,7 @@ int lxc_attach(const char *name, const char *lxcpath,
- 
- 		/* Setup resource limits */
- 		if (!lxc_list_empty(&conf->limits)) {
--			ret = setup_resource_limits(&conf->limits, pid);
-+			ret = setup_resource_limits(&conf->limits, pid, -1);
- 			if (ret < 0)
- 				goto on_error;
- 		}
-@@ -1650,8 +1650,8 @@ int lxc_attach_run_command(void *payload, int msg_fd)
- 		}
- 	}
- 
--	/* isulad: write errorm messages */
--	lxc_write_error_message(msg_fd, "exec: \"%s\": %s", cmd->program, strerror(errno));
-+	/* isulad: write error messages */
-+	lxc_write_error_message(msg_fd, "exec: \"%s\": %s.", cmd->program, strerror(errno));
- 
- 	SYSERROR("Failed to exec \"%s\"", cmd->program);
- 	return ret;
-diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index bc1481d..47b12a6 100644
---- a/src/lxc/cgroups/cgfsng.c
-+++ b/src/lxc/cgroups/cgfsng.c
-@@ -1236,7 +1236,7 @@ static int mkdir_eexist_on_last(const char *dir, mode_t mode)
- 	return 0;
- }
- 
--static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname)
-+static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname, int errfd)
- {
- 	int ret;
- 
-@@ -1244,8 +1244,8 @@ static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname)
- 
- 	if (file_exists(h->container_full_path)) { // it must not already exist
- 		ERROR("Cgroup path \"%s\" already exist.", h->container_full_path);
--		//lxc_write_error_message(errfd, "%s:%d: Cgroup path \"%s\" already exist.",
--		//	__FILE__, __LINE__, h->fullcgpath);
-+		lxc_write_error_message(errfd, "%s:%d: Cgroup path \"%s\" already exist.",
-+			__FILE__, __LINE__, h->container_full_path);
- 		return false;
- 	}
- 
-@@ -1288,7 +1288,7 @@ __cgfsng_ops static bool cgfsng_payload_create(struct cgroup_ops *ops,
- 	}
- 
- 	for (i = 0; ops->hierarchies[i]; i++) {
--		if (!create_path_for_hierarchy(ops->hierarchies[i], container_cgroup)) {
-+		if (!create_path_for_hierarchy(ops->hierarchies[i], container_cgroup, ops->errfd)) {
- 			SYSERROR("Failed to create %s", ops->hierarchies[i]->container_full_path);
- 			return false;
- 		}
-@@ -2203,6 +2203,11 @@ static int cg_legacy_set_data(struct cgroup_ops *ops, const char *filename,
- 
- 	fullpath = must_make_path(h->container_full_path, filename, NULL);
- 	ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666);
-+	if (ret) {
-+		lxc_write_error_message(ops->errfd,
-+			"%s:%d: setting cgroup config for ready process caused \"failed to write %s to %s: %s\".",
-+			__FILE__, __LINE__, value, fullpath, strerror(errno));
-+	}
- 	free(fullpath);
- 	return ret;
- }
-@@ -2294,9 +2299,15 @@ static bool __cg_legacy_setup_limits(struct cgroup_ops *ops,
- 			}
- 			if (setvalue > readvalue) {
- 				ERROR("The maximum allowed cpu-shares is %s", value);
-+				lxc_write_error_message(ops->errfd,
-+					"%s:%d: setting cgroup config for ready process caused \"The maximum allowed cpu-shares is %s\".",
-+					__FILE__, __LINE__, value);
- 				goto out;
- 			} else if (setvalue < readvalue) {
- 				ERROR("The minimum allowed cpu-shares is %s", value);
-+				lxc_write_error_message(ops->errfd,
-+					"%s:%d: setting cgroup config for ready process caused \"The minimum allowed cpu-shares is %s\".",
-+					__FILE__, __LINE__, value);
- 				goto out;
- 			}
- 		}
-@@ -2699,7 +2710,7 @@ __cgfsng_ops static bool cgfsng_data_init(struct cgroup_ops *ops, struct lxc_han
- 	return true;
- }
- 
--struct cgroup_ops *cgfsng_ops_init(void)
-+struct cgroup_ops *cgfsng_ops_init(int errfd)
- {
- 	struct cgroup_ops *cgfsng_ops;
- 
-@@ -2715,6 +2726,7 @@ struct cgroup_ops *cgfsng_ops_init(void)
- 		return NULL;
- 	}
- 
-+	cgfsng_ops->errfd = errfd;
- 	cgfsng_ops->data_init = cgfsng_data_init;
- 	cgfsng_ops->destroy = cgfsng_payload_destroy;
- 	cgfsng_ops->payload_create = cgfsng_payload_create;
-diff --git a/src/lxc/cgroups/cgroup.c b/src/lxc/cgroups/cgroup.c
-index 8e7aef9..7442c31 100644
---- a/src/lxc/cgroups/cgroup.c
-+++ b/src/lxc/cgroups/cgroup.c
-@@ -38,13 +38,13 @@
- 
- lxc_log_define(cgroup, lxc);
- 
--extern struct cgroup_ops *cgfsng_ops_init(void);
-+extern struct cgroup_ops *cgfsng_ops_init(int errfd);
- 
- struct cgroup_ops *cgroup_init(struct lxc_handler *handler)
- {
- 	struct cgroup_ops *cgroup_ops;
- 
--	cgroup_ops = cgfsng_ops_init();
-+	cgroup_ops = cgfsng_ops_init(handler->conf->errpipe[1]);
- 	if (!cgroup_ops) {
- 		ERROR("Failed to initialize cgroup driver");
- 		return NULL;
-diff --git a/src/lxc/cgroups/cgroup.h b/src/lxc/cgroups/cgroup.h
-index fa4871e..b6116f6 100644
---- a/src/lxc/cgroups/cgroup.h
-+++ b/src/lxc/cgroups/cgroup.h
-@@ -92,6 +92,8 @@ struct cgroup_ops {
- 	char **cgroup_use;
- 	char *cgroup_pattern;
- 	char *container_cgroup;
-+	/* isulad: errfd */
-+	int errfd;
- 
- 	/* @hierarchies
- 	 * - A NULL-terminated array of struct hierarchy, one per legacy
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 88cebfd..8fa63f7 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -2414,6 +2414,8 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
- 		ret = mkdir_p(path, 0755);
- 		if (ret < 0 && errno != EEXIST) {
- 			SYSERROR("Failed to create directory \"%s\"", path);
-+			lxc_write_error_message(rootfs->errfd, "%s:%d: mkdir %s: %s.",
-+				__FILE__, __LINE__, path, strerror(errno));
- 			return -1;
- 		}
- 	}
-@@ -2435,12 +2437,17 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
- 	free(p1);
- 	if (ret < 0 && errno != EEXIST) {
- 		SYSERROR("Failed to create directory \"%s\"", path);
-+		lxc_write_error_message(rootfs->errfd, "%s:%d: mkdir %s: %s.",
-+				__FILE__, __LINE__, p2, strerror(errno));
- 		return -1;
- 	}
- 
- 	ret = mknod(path, S_IFREG | 0000, 0);
--	if (ret < 0 && errno != EEXIST)
-+	if (ret < 0 && errno != EEXIST) {
-+		lxc_write_error_message(rootfs->errfd, "%s:%d: open %s: %s.",
-+				__FILE__, __LINE__, path, strerror(errno));
- 		return -errno;
-+	}
- 
- 	return 0;
- }
-@@ -2550,7 +2557,9 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
- 	if (rootfs_path) {
- 		rpath = follow_symlink_in_scope(path, rootfs_path);
- 		if (!rpath) {
--			ERROR("Failed to get real path for '%s'", path);
-+			ERROR("Failed to get real path of '%s' in scope '%s'.", path, rootfs_path);
-+			lxc_write_error_message(rootfs->errfd, "%s:%d: failed to get real path of '%s' in scope '%s'.",
-+				__FILE__, __LINE__, path, rootfs_path);
- 			return -1;
- 		}
- 		dest = rpath;
-@@ -2558,6 +2567,8 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
- 		ret = check_mount_destination(rootfs_path, dest);
- 		if (ret) {
- 			ERROR("Mount destination is invalid: '%s'", dest);
-+			lxc_write_error_message(rootfs->errfd, "%s:%d: mount destination is invalid: '%s'.",
-+				__FILE__, __LINE__, dest);
- 			free(rpath);
- 			return -1;
- 		}
-@@ -2587,6 +2598,10 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
- 		ret = mount_entry(mntent->mnt_fsname, dest, mntent->mnt_type, mntflags,
- 				  pflags, mntdata, optional, dev, relative, rootfs_path);
- 	}
-+	if (ret < 0) {
-+		lxc_write_error_message(rootfs->errfd, "%s:%d: failed to mount %s as type %s.",
-+				__FILE__, __LINE__, mntent->mnt_fsname, mntent->mnt_type);
-+	}
- 
- 	free(mntdata);
- 	free(rpath);
-@@ -2949,7 +2964,7 @@ static int parse_resource(const char *res)
- 	return resid;
- }
- 
--int setup_resource_limits(struct lxc_list *limits, pid_t pid)
-+int setup_resource_limits(struct lxc_list *limits, pid_t pid, int errfd)
- {
- 	int resid;
- 	struct lxc_list *it;
-@@ -2966,7 +2981,10 @@ int setup_resource_limits(struct lxc_list *limits, pid_t pid)
- 
- #if HAVE_PRLIMIT || HAVE_PRLIMIT64
- 		if (prlimit(pid, resid, &lim->limit, NULL) != 0) {
--			SYSERROR("Failed to set limit %s", lim->resource);
-+			SYSERROR("Failed to set limit %s %lu %lu.", lim->resource, lim->limit.rlim_cur, lim->limit.rlim_max);
-+			lxc_write_error_message(errfd, "%s:%d: Failed to set limit %s %lu %lu: %s.",
-+					__FILE__, __LINE__, lim->resource,
-+					lim->limit.rlim_cur, lim->limit.rlim_max, strerror(errno));
- 			return -1;
- 		}
- 
-@@ -3989,6 +4007,8 @@ int lxc_setup(struct lxc_handler *handler)
- 	ret = lxc_setup_rootfs_prepare_root(lxc_conf, name, lxcpath);
- 	if (ret < 0) {
- 		ERROR("Failed to setup rootfs");
-+		lxc_write_error_message(lxc_conf->errpipe[1], "%s:%d: failed to setup rootfs %s.",
-+				__FILE__, __LINE__, lxc_conf->rootfs.path);
- 		return -1;
- 	}
- 
-@@ -3996,31 +4016,31 @@ int lxc_setup(struct lxc_handler *handler)
- 		ret = setup_utsname(lxc_conf->utsname);
- 		if (ret < 0) {
- 			ERROR("Failed to setup the utsname %s", name);
--			return -1;
-+			goto on_error;
- 		}
- 	}
- 
- 	ret = lxc_setup_keyring();
- 	if (ret < 0)
--		return -1;
-+		goto on_error;
- 
- 	ret = lxc_setup_network_in_child_namespaces(lxc_conf, &lxc_conf->network);
- 	if (ret < 0) {
- 		ERROR("Failed to setup network");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	ret = lxc_network_send_name_and_ifindex_to_parent(handler);
- 	if (ret < 0) {
- 		ERROR("Failed to send network device names and ifindices to parent");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	if (lxc_conf->autodev > 0) {
- 		ret = mount_autodev(name, &lxc_conf->rootfs, lxcpath);
- 		if (ret < 0) {
- 			ERROR("Failed to mount \"/dev\"");
--			return -1;
-+			goto on_error;
- 		}
- 	}
- 
-@@ -4030,13 +4050,14 @@ int lxc_setup(struct lxc_handler *handler)
- 	ret = lxc_mount_auto_mounts(lxc_conf, lxc_conf->auto_mounts & ~LXC_AUTO_CGROUP_MASK, handler);
- 	if (ret < 0) {
- 		ERROR("Failed to setup first automatic mounts");
--		return -1;
-+		goto on_error;
- 	}
- 
-+	lxc_conf->rootfs.errfd = lxc_conf->errpipe[1];
- 	ret = setup_mount(lxc_conf, &lxc_conf->rootfs, lxc_conf->fstab, name, lxcpath);
- 	if (ret < 0) {
- 		ERROR("Failed to setup mounts");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	if (lxc_conf->is_execute) {
-@@ -4047,13 +4068,13 @@ int lxc_setup(struct lxc_handler *handler)
- 			ret = snprintf(path, PATH_MAX, SBINDIR "/init.lxc.static");
- 			if (ret < 0 || ret >= PATH_MAX) {
- 				ERROR("Path to init.lxc.static too long");
--				return -1;
-+				goto on_error;
- 			}
- 
- 			fd = open(path, O_PATH | O_CLOEXEC);
- 			if (fd < 0) {
- 				SYSERROR("Unable to open lxc.init.static");
--				return -1;
-+				goto on_error;
- 			}
- 
- 			((struct execute_args *)handler->data)->init_fd = fd;
-@@ -4062,7 +4083,7 @@ int lxc_setup(struct lxc_handler *handler)
- 			ret = lxc_execute_bind_init(handler);
- 			if (ret < 0) {
- 				ERROR("Failed to bind-mount the lxc init system");
--				return -1;
-+				goto on_error;
- 			}
- 		}
- 	}
-@@ -4074,7 +4095,7 @@ int lxc_setup(struct lxc_handler *handler)
- 	ret = lxc_mount_auto_mounts(lxc_conf, lxc_conf->auto_mounts & LXC_AUTO_CGROUP_MASK, handler);
- 	if (ret < 0) {
- 		ERROR("Failed to setup remaining automatic mounts");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	/*isulad: move mount entries here, before we do lxc_fill_autodev and populate devices  */
-@@ -4083,27 +4104,27 @@ int lxc_setup(struct lxc_handler *handler)
- 					  &lxc_conf->mount_list, name, lxcpath);
- 		if (ret < 0) {
- 			ERROR("Failed to setup mount entries");
--			return -1;
-+			goto on_error;
- 		}
- 	}
- 
- 	ret = run_lxc_hooks(name, "mount", lxc_conf, NULL);
- 	if (ret < 0) {
- 		ERROR("Failed to run mount hooks");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	if (lxc_conf->autodev > 0) {
- 		ret = run_lxc_hooks(name, "autodev", lxc_conf, NULL);
- 		if (ret < 0) {
- 			ERROR("Failed to run autodev hooks");
--			return -1;
-+			goto on_error;
- 		}
- 
- 		ret = lxc_fill_autodev(&lxc_conf->rootfs);
- 		if (ret < 0) {
- 			ERROR("Failed to populate \"/dev\"");
--			return -1;
-+			goto on_error;
- 		}
- 	}
- 
-@@ -4111,64 +4132,64 @@ int lxc_setup(struct lxc_handler *handler)
- 	if (!lxc_list_empty(&lxc_conf->populate_devs)) {
- 		if (setup_populate_devs(&lxc_conf->rootfs, &lxc_conf->populate_devs)) {
- 			ERROR("Failed to setup devices in the container");
--			return -1;;
-+			goto on_error;
- 		}
- 	}
- 
- 	/* Make sure any start hooks are in the container */
- 	if (!verify_start_hooks(lxc_conf)) {
- 		ERROR("Failed to verify start hooks");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	ret = lxc_setup_console(&lxc_conf->rootfs, &lxc_conf->console,
- 				lxc_conf->ttys.dir);
- 	if (ret < 0) {
- 		ERROR("Failed to setup console");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	ret = lxc_setup_dev_symlinks(&lxc_conf->rootfs);
- 	if (ret < 0) {
- 		ERROR("Failed to setup \"/dev\" symlinks");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	ret = lxc_create_tmp_proc_mount(lxc_conf);
- 	if (ret < 0) {
- 		ERROR("Failed to \"/proc\" LSMs");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	ret = lxc_setup_rootfs_switch_root(&lxc_conf->rootfs);
- 	if (ret < 0) {
- 		ERROR("Failed to pivot root into rootfs");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	/* isulad: remount rootfs readonly if necessary */
- 	if (setup_rootfs_mountopts(&lxc_conf->rootfs)) {
- 		ERROR("failed to set rootfs for '%s'", name);
--		return -1;
-+		goto on_error;
- 	}
- 
- 	if (lxc_conf->rootfs.path) {
- 		ret = lxc_setup_devpts(lxc_conf);
- 		if (ret < 0) {
- 			ERROR("Failed to setup new devpts instance");
--			return -1;
-+			goto on_error;
- 		}
- 	}
- 
- 	ret = lxc_create_ttys(handler);
- 	if (ret < 0)
--		return -1;
-+		goto on_error;
- 
- 	//isulad: setup rootfs masked paths
- 	if (!lxc_list_empty(&lxc_conf->rootfs.maskedpaths)) {
- 		if (setup_rootfs_maskedpaths(&lxc_conf->rootfs.maskedpaths)) {
- 			ERROR("failed to setup maskedpaths");
--			return -1;
-+			goto on_error;
- 		}
- 	}
- 
-@@ -4176,7 +4197,7 @@ int lxc_setup(struct lxc_handler *handler)
- 	if (!lxc_list_empty(&lxc_conf->rootfs.ropaths)) {
- 		if (setup_rootfs_ropaths(&lxc_conf->rootfs.ropaths)) {
- 			ERROR("failed to setup readonlypaths");
--			return -1;
-+			goto on_error;
- 		}
- 	}
- 
-@@ -4186,7 +4207,7 @@ int lxc_setup(struct lxc_handler *handler)
- 	ret = setup_personality(lxc_conf->personality);
- 	if (ret < 0) {
- 		ERROR("Failed to set personality");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	/* Set sysctl value to a path under /proc/sys as determined from the
-@@ -4197,7 +4218,7 @@ int lxc_setup(struct lxc_handler *handler)
- 		ret = setup_sysctl_parameters(&lxc_conf->sysctls);
- 		if (ret < 0) {
- 			ERROR("Failed to setup sysctl parameters");
--			return -1;
-+			goto on_error;
- 		}
- 	}
- 
-@@ -4206,21 +4227,24 @@ int lxc_setup(struct lxc_handler *handler)
- 			ERROR("Container requests lxc.cap.drop and "
- 			      "lxc.cap.keep: either use lxc.cap.drop or "
- 			      "lxc.cap.keep, not both");
--			return -1;
-+			goto on_error;
- 		}
- 
- 		if (dropcaps_except(&lxc_conf->keepcaps)) {
- 			ERROR("Failed to keep capabilities");
--			return -1;
-+			goto on_error;
- 		}
- 	} else if (setup_caps(&lxc_conf->caps)) {
- 		ERROR("Failed to drop capabilities");
--		return -1;
-+		goto on_error;
- 	}
- 
- 	NOTICE("The container \"%s\" is set up", name);
- 
- 	return 0;
-+on_error:
-+	lxc_write_error_message(lxc_conf->errpipe[1], "Failed to setup lxc, please check the config file.");
-+	return -1;
- }
- 
- /* isulad drop caps for container*/
-@@ -4555,11 +4579,9 @@ void* wait_ocihook_timeout(void *arg)
- 			__FILE__, __LINE__, lxchook_names[conf->which],
- 			(double)conf->timeout);
- 
--		if (conf->errfd >= 0) {
--			lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\"",
--				__FILE__, __LINE__, lxchook_names[conf->which],
--				(double)conf->timeout);
--		}
-+		lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\".",
-+			__FILE__, __LINE__, lxchook_names[conf->which],
-+			(double)conf->timeout);
- 
- 		if (kill(conf->pid, SIGKILL) && errno != ESRCH) {
- 			ERROR("Send kill signal failed");
-@@ -4640,22 +4662,18 @@ static int run_ocihook_buffer(struct oci_hook_conf *oconf, char *inmsg)
- 		goto print_hook;
- 	} else if (WIFEXITED(ret) && WEXITSTATUS(ret) != 0) {
- 		ERROR("Script exited with status %d. output:%s", WEXITSTATUS(ret), output);
--		if (conf->errfd >= 0) {
--			lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"error running hook: exit status %d, output:%s\"",
--				__FILE__, __LINE__,
--				(conf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[conf->which],
--				WEXITSTATUS(ret), output);
--		}
-+		lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"error running hook: exit status %d, output:%s\".",
-+			__FILE__, __LINE__,
-+			(conf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[conf->which],
-+			WEXITSTATUS(ret), output);
- 
- 		goto print_hook;
- 	} else if (WIFSIGNALED(ret)) {
- 		ERROR("Script terminated by signal %d.", WTERMSIG(ret));
--		if (conf->errfd >= 0) {
--			lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"error running hook: Script terminated by signal %d\"",
--				__FILE__, __LINE__,
--				(conf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[conf->which],
--				WTERMSIG(ret));
--		}
-+		lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"error running hook: Script terminated by signal %d\".",
-+			__FILE__, __LINE__,
-+			(conf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[conf->which],
-+			WTERMSIG(ret));
- 
- 		goto print_hook;
- 	}
-diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index b92c48e..88f5b41 100644
---- a/src/lxc/conf.h
-+++ b/src/lxc/conf.h
-@@ -175,6 +175,8 @@ struct lxc_rootfs {
- 	struct lxc_list maskedpaths;
- 	/* isulad: ropaths */
- 	struct lxc_list ropaths;
-+	/* isulad: errfd */
-+	int errfd;
- };
- 
- /*
-@@ -462,7 +464,7 @@ extern int lxc_setup_rootfs_prepare_root(struct lxc_conf *conf,
- 					 const char *name, const char *lxcpath);
- extern int lxc_setup(struct lxc_handler *handler);
- extern int lxc_setup_parent(struct lxc_handler *handler);
--extern int setup_resource_limits(struct lxc_list *limits, pid_t pid);
-+extern int setup_resource_limits(struct lxc_list *limits, pid_t pid, int errfd);
- extern int find_unmapped_nsid(struct lxc_conf *conf, enum idtype idtype);
- extern int mapped_hostid(unsigned id, struct lxc_conf *conf,
- 			 enum idtype idtype);
-@@ -499,6 +501,7 @@ int lxc_clear_populate_devices(struct lxc_conf *c);
- int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
- int lxc_clear_rootfs_ro_paths(struct lxc_conf *c);
- int lxc_drop_caps(struct lxc_conf *conf);
-+void lxc_close_error_pipe(int *errpipe);
- 
- /* isulad add end */
- 
-diff --git a/src/lxc/execute.c b/src/lxc/execute.c
-index d388e63..3fc46c6 100644
---- a/src/lxc/execute.c
-+++ b/src/lxc/execute.c
-@@ -40,7 +40,7 @@
- 
- lxc_log_define(execute, start);
- 
--static int execute_start(struct lxc_handler *handler, void* data)
-+static int execute_start(struct lxc_handler *handler, void* data, int fd)
- {
- 	int argc_add, j;
- 	char **argv;
-@@ -91,6 +91,7 @@ static int execute_start(struct lxc_handler *handler, void* data)
- 	else
- 		execvp(argv[0], argv);
- 	SYSERROR("Failed to exec %s", argv[0]);
-+	lxc_write_error_message(fd, "Failed to exec: \"%s\": %s.", argv[0], strerror(errno));
- 
- 	free(argv);
- out1:
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index e99c41c..d641851 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -916,7 +916,9 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 		NULL,
- 	};
- 	char **init_cmd = NULL;
--	int keepfds[4] = {-1, -1, -1, -1};
-+	int keepfds[] = {-1, -1, -1, -1, -1};
-+	ssize_t size_read;
-+	char errbuf[BUFSIZ + 1] = {0};
- 
- 	/* container does exist */
- 	if (!c)
-@@ -962,7 +964,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 			argv = init_cmd = split_init_cmd(conf->init_cmd);
- 	}
- 
--    /* isulad: use init argv as init cmd */
-+	/* isulad: use init argv as init cmd */
- 	if (!argv) {
- 		argv = init_cmd = use_init_args(conf->init_argv, conf->init_argc);
- 	}
-@@ -986,10 +988,19 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 		char title[2048];
- 		pid_t pid;
- 
-+		//isulad: pipdfd for get error message of child or grandchild process.
-+		if (pipe2(conf->errpipe, O_CLOEXEC) != 0) {
-+			SYSERROR("Failed to init errpipe");
-+			free_init_cmd(init_cmd);
-+			lxc_free_handler(handler);
-+			return false;
-+		}
-+
- 		pid = fork();
- 		if (pid < 0) {
- 			free_init_cmd(init_cmd);
- 			lxc_free_handler(handler);
-+			lxc_close_error_pipe(conf->errpipe);
- 			return false;
- 		}
- 
-@@ -999,11 +1010,23 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 			 * the PID file, child will do the free and unlink.
- 			 */
- 			c->pidfile = NULL;
-+			close(conf->errpipe[1]);
-+			conf->errpipe[1] = -1;
- 
- 			/* Wait for container to tell us whether it started
- 			 * successfully.
- 			 */
- 			started = wait_on_daemonized_start(handler, pid);
-+			if (!started) {
-+				size_read = read(conf->errpipe[0], errbuf, BUFSIZ);
-+				if (size_read > 0) {
-+					conf->errmsg = strdup(errbuf);
-+					if (!conf->errmsg)
-+						ERROR("Out of memory");
-+				}
-+			}
-+			close(conf->errpipe[0]);
-+			conf->errpipe[0] = -1;
- 
- 			free_init_cmd(init_cmd);
- 			lxc_free_handler(handler);
-@@ -1039,6 +1062,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 		if (pid != 0) {
- 			free_init_cmd(init_cmd);
- 			lxc_free_handler(handler);
-+			lxc_close_error_pipe(conf->errpipe);
- 			_exit(EXIT_SUCCESS);
- 		}
- 
-@@ -1050,10 +1074,12 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 			SYSERROR("Failed to change to \"/\" directory");
- 			_exit(EXIT_FAILURE);
- 		}
--
-+		close(conf->errpipe[0]);
-+		conf->errpipe[0] = -1;
- 		keepfds[0] = handler->conf->maincmd_fd;
- 		keepfds[1] = handler->state_socket_pair[0];
- 		keepfds[2] = handler->state_socket_pair[1];
-+		keepfds[4] = conf->errpipe[1];
- 		ret = lxc_check_inherited(conf, true, keepfds,
- 					  sizeof(keepfds) / sizeof(keepfds[0]));
- 		if (ret < 0)
-@@ -1088,6 +1114,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 		if (w < 0 || (size_t)w >= sizeof(pidstr)) {
- 			free_init_cmd(init_cmd);
- 			lxc_free_handler(handler);
-+			lxc_close_error_pipe(conf->errpipe);
- 
- 			SYSERROR("Failed to write monitor pid to \"%s\"", c->pidfile);
- 
-@@ -1101,6 +1128,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
- 		if (ret < 0) {
- 			free_init_cmd(init_cmd);
- 			lxc_free_handler(handler);
-+			lxc_close_error_pipe(conf->errpipe);
- 
- 			SYSERROR("Failed to write monitor pid to \"%s\"", c->pidfile);
- 
-@@ -1159,6 +1187,9 @@ reboot:
- 	if (conf->exit_fd >= 0) {
- 		keepfds[3] = conf->exit_fd;
- 	}
-+	/* isulad: keep errpipe fd */
-+	if (c->daemonize)
-+		keepfds[4] = conf->errpipe[1];
- 	ret = lxc_check_inherited(conf, c->daemonize, keepfds,
- 				  sizeof(keepfds) / sizeof(keepfds[0]));
- 	if (ret < 0) {
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index ec61b32..3e6854f 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -892,6 +892,7 @@ int lxc_init(const char *name, struct lxc_handler *handler)
- 	ret = lxc_terminal_setup(conf);
- 	if (ret < 0) {
- 		ERROR("Failed to create console");
-+		lxc_write_error_message(conf->errpipe[1], "Failed to create console for container \"%s\".", name);
- 		goto out_restore_sigmask;
- 	}
- 	TRACE("Created console");
-@@ -1390,6 +1391,8 @@ static int do_start(void *data)
- 		struct stat st;
- 		if (stat(handler->conf->init_cwd, &st) < 0 && mkdir_p(handler->conf->init_cwd, 0755) < 0) {
- 			SYSERROR("Try to create directory \"%s\" as workdir failed", handler->conf->init_cwd);
-+			lxc_write_error_message(handler->conf->errpipe[1], "%s:%d: Failed to create workdir: %s.",
-+					__FILE__, __LINE__, strerror(errno));
- 			goto out_warn_father;
- 		}
- 		if (chdir(handler->conf->init_cwd)) {
-@@ -1489,7 +1492,7 @@ static int do_start(void *data)
- 	/* After this call, we are in error because this ops should not return
- 	 * as it execs.
- 	 */
--	handler->ops->start(handler, handler->data);
-+	handler->ops->start(handler, handler->data, handler->daemonize ? handler->conf->errpipe[1] : -1);
- 
- out_warn_father:
- 	/* We want the parent to know something went wrong, so we return a
-@@ -1898,7 +1901,7 @@ static int lxc_spawn(struct lxc_handler *handler)
- 		goto out_delete_net;
- 
- 	if (!lxc_list_empty(&conf->limits)) {
--		ret = setup_resource_limits(&conf->limits, handler->pid);
-+		ret = setup_resource_limits(&conf->limits, handler->pid, conf->errpipe[1]);
- 		if (ret < 0) {
- 			ERROR("Failed to setup resource limits");
- 			goto out_delete_net;
-@@ -1960,7 +1963,7 @@ static int lxc_spawn(struct lxc_handler *handler)
- 	}
- 
- 	if (START_TIMEOUT == global_timeout_state) {
--		//lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
-+		lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
- 		ERROR("Starting the container \"%s\" timeout.", name);
- 		goto out_delete_net;
- 	}
-@@ -2008,7 +2011,7 @@ static int lxc_spawn(struct lxc_handler *handler)
- 	}
- 
- 	if (START_TIMEOUT == global_timeout_state) {
--		//lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
-+		lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
- 		ERROR("Starting the container \"%s\" timeout.", name);
- 		goto out_abort;
- 	}
-@@ -2232,7 +2235,7 @@ struct start_args {
- 	char *const *argv;
- };
- 
--static int start(struct lxc_handler *handler, void* data)
-+static int start(struct lxc_handler *handler, void* data, int fd)
- {
- 	struct start_args *arg = data;
- 
-@@ -2240,6 +2243,7 @@ static int start(struct lxc_handler *handler, void* data)
- 
- 	execvp(arg->argv[0], arg->argv);
- 	SYSERROR("Failed to exec \"%s\"", arg->argv[0]);
-+	lxc_write_error_message(fd, "exec: \"%s\": %s.", arg->argv[0], strerror(errno));
- 	return 0;
- }
- 
-diff --git a/src/lxc/start.h b/src/lxc/start.h
-index 1d84325..ab72e6e 100644
---- a/src/lxc/start.h
-+++ b/src/lxc/start.h
-@@ -145,7 +145,7 @@ struct execute_args {
- };
- 
- struct lxc_operations {
--	int (*start)(struct lxc_handler *, void *);
-+	int (*start)(struct lxc_handler *, void *, int);
- 	int (*post_start)(struct lxc_handler *, void *);
- };
- 
-diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
-index f37f8a6..ec48701 100644
---- a/src/lxc/tools/lxc_start.c
-+++ b/src/lxc/tools/lxc_start.c
-@@ -392,6 +392,9 @@ int main(int argc, char *argv[])
- 	else
- 		err = c->start(c, 0, args) ? EXIT_SUCCESS : EXIT_FAILURE;
- 	if (err) {
-+		if (c->lxc_conf->errmsg)
-+			fprintf(stderr, "%s:%s:%s:%d starting container process caused \"%s\"", c->name,
-+			__FILE__, __func__, __LINE__, c->lxc_conf->errmsg);
- 		ERROR("The container failed to start");
- 
- 		if (my_args.daemonize)
--- 
-1.8.3.1
-

0040-add-timeout-200ms-for-cmds-send-to-lxc-monitor.patch

@@ -1,107 +0,0 @@
-From 04512fa5baae9aeff6ac8f120e718f339b56525f Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Wed, 16 Jan 2019 02:22:13 -0500
-Subject: [PATCH 040/140] add timeout(200ms) for cmds send to [lxc monitor]
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/af_unix.c  | 25 +++++++++++++++++++++++--
- src/lxc/af_unix.h  |  2 ++
- src/lxc/commands.c |  7 +++++--
- 3 files changed, 30 insertions(+), 4 deletions(-)
-
-diff --git a/src/lxc/af_unix.c b/src/lxc/af_unix.c
-index 02f32c4..24500a8 100644
---- a/src/lxc/af_unix.c
-+++ b/src/lxc/af_unix.c
-@@ -194,8 +194,9 @@ int lxc_abstract_unix_send_fds(int fd, int *sendfds, int num_sendfds,
- 	return ret;
- }
- 
--int lxc_abstract_unix_recv_fds(int fd, int *recvfds, int num_recvfds,
--			       void *data, size_t size)
-+/* isulad: add wait timeout Microseconds*/
-+int lxc_abstract_unix_recv_fds_timeout(int fd, int *recvfds, int num_recvfds,
-+			       void *data, size_t size, unsigned int timeout)
- {
- 	int ret;
- 	struct msghdr msg;
-@@ -204,6 +205,7 @@ int lxc_abstract_unix_recv_fds(int fd, int *recvfds, int num_recvfds,
- 	char buf[1] = {0};
- 	char *cmsgbuf;
- 	size_t cmsgbufsize = CMSG_SPACE(num_recvfds * sizeof(int));
-+	struct timeval out;
- 
- 	memset(&msg, 0, sizeof(msg));
- 	memset(&iov, 0, sizeof(iov));
-@@ -222,6 +224,19 @@ int lxc_abstract_unix_recv_fds(int fd, int *recvfds, int num_recvfds,
- 	msg.msg_iov = &iov;
- 	msg.msg_iovlen = 1;
- 
-+	if (timeout > 0) {
-+		memset(&out, 0, sizeof(out));
-+		out.tv_sec = timeout / 1000000;
-+		out.tv_usec = timeout % 1000000;
-+		ret = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO,
-+		        (const void *)&out, sizeof(out));
-+		if (ret < 0) {
-+			ERROR("Failed to set %u timeout on containter "
-+			     "state socket", timeout);
-+			goto out;
-+		}
-+	}
-+
- 	ret = recvmsg(fd, &msg, 0);
- 	if (ret <= 0)
- 		goto out;
-@@ -238,6 +253,12 @@ out:
- 	return ret;
- }
- 
-+int lxc_abstract_unix_recv_fds(int fd, int *recvfds, int num_recvfds,
-+			       void *data, size_t size)
-+{
-+	return lxc_abstract_unix_recv_fds_timeout(fd, recvfds, num_recvfds, data, size, 0);
-+}
-+
- int lxc_abstract_unix_send_credential(int fd, void *data, size_t size)
- {
- 	struct msghdr msg = {0};
-diff --git a/src/lxc/af_unix.h b/src/lxc/af_unix.h
-index f2c2fdc..74fd77f 100644
---- a/src/lxc/af_unix.h
-+++ b/src/lxc/af_unix.h
-@@ -37,5 +37,7 @@ extern int lxc_abstract_unix_recv_fds(int fd, int *recvfds, int num_recvfds,
- 				      void *data, size_t size);
- extern int lxc_abstract_unix_send_credential(int fd, void *data, size_t size);
- extern int lxc_abstract_unix_rcv_credential(int fd, void *data, size_t size);
-+extern int lxc_abstract_unix_recv_fds_timeout(int fd, int *recvfds, int num_recvfds,
-+			       void *data, size_t size, unsigned int timeout);
- 
- #endif /* __LXC_AF_UNIX_H */
-diff --git a/src/lxc/commands.c b/src/lxc/commands.c
-index 133384d..47a824a 100644
---- a/src/lxc/commands.c
-+++ b/src/lxc/commands.c
-@@ -126,13 +126,16 @@ static int lxc_cmd_rsp_recv(int sock, struct lxc_cmd_rr *cmd)
- 	int ret, rspfd;
- 	struct lxc_cmd_rsp *rsp = &cmd->rsp;
- 
--	ret = lxc_abstract_unix_recv_fds(sock, &rspfd, 1, rsp, sizeof(*rsp));
-+	/*isulad: add timeout 200ms to avoid long block due to [lxc monitor] error*/
-+	ret = lxc_abstract_unix_recv_fds_timeout(sock, &rspfd, 1, rsp, sizeof(*rsp), 200 * 1000);
- 	if (ret < 0) {
- 		SYSWARN("Failed to receive response for command \"%s\"",
- 		        lxc_cmd_str(cmd->req.cmd));
- 
--		if (errno == ECONNRESET)
-+		if (errno == ECONNRESET || errno == EAGAIN || errno == EWOULDBLOCK) {
-+			errno = ECONNRESET; /*isulad set errno ECONNRESET when timeout */
- 			return -1;
-+		}
- 
- 		return -1;
- 	}
--- 
-1.8.3.1
-

0040-cgfsng-make-container-full-path-in-cgfsng_get_cgroup.patch

@@ -0,0 +1,29 @@
+From beefa97e5dd3daec438b6d8237af167fde1097dd Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Thu, 16 Apr 2020 12:35:45 +0800
+Subject: [PATCH 40/49] cgfsng: make container full path in cgfsng_get_cgroup
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/cgroups/cgfsng.c | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
+index 76576c5..e2a8983 100644
+--- a/src/lxc/cgroups/cgfsng.c
++++ b/src/lxc/cgroups/cgfsng.c
+@@ -2612,6 +2612,11 @@ __cgfsng_ops static const char *cgfsng_get_cgroup(struct cgroup_ops *ops,
+ 		return log_warn_errno(NULL, ENOENT, "Failed to find hierarchy for controller \"%s\"",
+ 				      controller ? controller : "(null)");
+ 
++#ifdef HAVE_ISULAD
++	if (!h->container_full_path)
++		h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, ops->container_cgroup, NULL);
++#endif
++
+ 	return h->container_full_path
+ 		   ? h->container_full_path + strlen(h->mountpoint)
+ 		   : NULL;
+-- 
+1.8.3.1
+

0041-build-fix-some-bug-in-free-memory.patch

@@ -0,0 +1,52 @@
+From dcc50d14398f1bfcde7c41e7480928e6c98b52d9 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Thu, 16 Apr 2020 12:35:45 +0800
+Subject: [PATCH 41/49] build: fix some bug in free memory
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/lxccontainer.c | 10 ++++++++++
+ src/lxc/string_utils.c |  1 +
+ 2 files changed, 11 insertions(+)
+
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index ab10ac6..818848a 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -6006,11 +6006,21 @@ int list_active_containers(const char *lxcpath, char ***nret,
+ 				continue;
+ 		}
+ 
++#ifdef HAVE_ISULAD
++		if (ct_name && ct_name_cnt) {
++			if (array_contains(&ct_name, p, ct_name_cnt)) {
++				if (is_hashed)
++					free(p);
++				continue;
++			}
++		}
++#else
+ 		if (array_contains(&ct_name, p, ct_name_cnt)) {
+ 			if (is_hashed)
+ 				free(p);
+ 			continue;
+ 		}
++#endif
+ 
+ 		if (!add_to_array(&ct_name, p, ct_name_cnt)) {
+ 			if (is_hashed)
+diff --git a/src/lxc/string_utils.c b/src/lxc/string_utils.c
+index dcb1160..9118add 100644
+--- a/src/lxc/string_utils.c
++++ b/src/lxc/string_utils.c
+@@ -501,6 +501,7 @@ int lxc_grow_array(void ***array, size_t *capacity, size_t new_size, size_t capa
+ 	/* first time around, catch some trivial mistakes of the user
+ 	 * only initializing one of these */
+ 	if (!*array || !*capacity) {
++		free(*array);
+ 		*array = NULL;
+ 		*capacity = 0;
+ 	}
+-- 
+1.8.3.1
+

0041-return-1-when-_lxc_start-fails.patch

@@ -1,25 +0,0 @@
-From 4315f825763d752a9a9dfcb3ade70368634ee15d Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Wed, 16 Jan 2019 05:53:36 -0500
-Subject: [PATCH 041/140] return -1 when _lxc_start fails
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/start.c | 1 +
- 1 file changed, 1 insertion(+)
-
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 3e6854f..7bbcb00 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -2175,6 +2175,7 @@ int __lxc_start(const char *name, struct lxc_handler *handler,
- 
- 	if (!handler->init_died && handler->pid > 0) {
- 		ERROR("Child process is not killed");
-+		ret = -1;
- 		goto out_abort;
- 	}
- 
--- 
-1.8.3.1
-

0042-cgfsng-make-container-full-path-in-destory-cgroup.patch

@@ -0,0 +1,29 @@
+From a5cc738c0f322b35a759f83d8d631e657f8bd58b Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Thu, 16 Apr 2020 16:30:34 +0800
+Subject: [PATCH 42/49] cgfsng: make container full path in destory cgroup
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/cgroups/cgfsng.c | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+
+diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
+index e2a8983..1e1df3b 100644
+--- a/src/lxc/cgroups/cgfsng.c
++++ b/src/lxc/cgroups/cgfsng.c
+@@ -970,8 +970,9 @@ static int isulad_cgroup_tree_remove(struct hierarchy **hierarchies,
+ 		struct hierarchy *h = hierarchies[i];
+ 		int ret;
+ 
+-		if (!h->container_full_path)
+-			continue;
++		if (!h->container_full_path) {
++			h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, container_cgroup, NULL);
++		}
+ 
+ 		ret = lxc_rm_rf(h->container_full_path);
+ 		if (ret < 0) {
+-- 
+1.8.3.1
+

0042-lxc-seccomp-adopt-to-lxc3.0.patch

@@ -1,169 +0,0 @@
-From cd1cd3c8d36b5f689ceaac00965ab1a3e77c2f33 Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Thu, 17 Jan 2019 03:09:00 +0800
-Subject: [PATCH 042/140] lxc: seccomp adopt to lxc3.0
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/seccomp.c | 127 ++++++++++++++++++++++++++++--------------------------
- 1 file changed, 65 insertions(+), 62 deletions(-)
-
-diff --git a/src/lxc/seccomp.c b/src/lxc/seccomp.c
-index 1e14be1..27bdc22 100644
---- a/src/lxc/seccomp.c
-+++ b/src/lxc/seccomp.c
-@@ -936,86 +936,89 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 		     SCMP_ARCH_NATIVE, line, rule.action,
- 		     get_action_name(rule.action));
- 
--		if (ctx.architectures[0] != SCMP_ARCH_NATIVE) {
--			if (!do_resolve_add_rule(ctx.architectures[0], line,
--						 ctx.contexts[0], &rule))
--				goto bad_rule;
--
--			INFO("Added compat rule for arch %d for %s action %d(%s)",
--			     ctx.architectures[0], line, rule.action,
--			     get_action_name(rule.action));
--		}
-+		if (cur_rule_arch == lxc_seccomp_arch_all) {
-+			if (ctx.architectures[0] != SCMP_ARCH_NATIVE) {
-+				if (!do_resolve_add_rule(ctx.architectures[0], line,
-+							ctx.contexts[0], &rule))
-+					goto bad_rule;
-+
-+					INFO("Added compat rule for arch %d for %s action %d(%s)",
-+						ctx.architectures[0], line, rule.action,
-+						get_action_name(rule.action));
-+			}
- 
--		if (ctx.architectures[1] != SCMP_ARCH_NATIVE) {
--			if (!do_resolve_add_rule(ctx.architectures[1], line,
--						 ctx.contexts[1], &rule))
--				goto bad_rule;
-+			if (ctx.architectures[1] != SCMP_ARCH_NATIVE) {
-+				if (!do_resolve_add_rule(ctx.architectures[1], line,
-+							ctx.contexts[1], &rule))
-+					goto bad_rule;
- 
--			INFO("Added compat rule for arch %d for %s action %d(%s)",
--			     ctx.architectures[1], line, rule.action,
--			     get_action_name(rule.action));
--		}
-+				INFO("Added compat rule for arch %d for %s action %d(%s)",
-+					ctx.architectures[1], line, rule.action,
-+					get_action_name(rule.action));
-+			}
- 
--		if (ctx.architectures[2] != SCMP_ARCH_NATIVE) {
--			if (!do_resolve_add_rule(ctx.architectures[2], line,
-+			if (ctx.architectures[2] != SCMP_ARCH_NATIVE) {
-+				if (!do_resolve_add_rule(ctx.architectures[2], line,
- 						ctx.contexts[2], &rule))
--				goto bad_rule;
-+					goto bad_rule;
- 
--			INFO("Added native rule for arch %d for %s action %d(%s)",
--			     ctx.architectures[2], line, rule.action,
--			     get_action_name(rule.action));
-+				INFO("Added native rule for arch %d for %s action %d(%s)",
-+					ctx.architectures[2], line, rule.action,
-+					get_action_name(rule.action));
-+			}
- 		}
- 	}
- 
--	INFO("Merging compat seccomp contexts into main context");
--	if (ctx.contexts[0]) {
--		if (ctx.needs_merge[0]) {
--			ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[0]);
--			if (ret < 0) {
--				ERROR("Failed to merge first compat seccomp "
-+	if (cur_rule_arch == lxc_seccomp_arch_all) {
-+		INFO("Merging compat seccomp contexts into main context");
-+		if (ctx.contexts[0]) {
-+			if (ctx.needs_merge[0]) {
-+				ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[0]);
-+				if (ret < 0) {
-+					ERROR("Failed to merge first compat seccomp "
- 				      "context into main context");
--				goto bad;
--			}
-+					goto bad;
-+				}
- 
--			TRACE("Merged first compat seccomp context into main context");
--		} else {
--			seccomp_release(ctx.contexts[0]);
--			ctx.contexts[0] = NULL;
-+				TRACE("Merged first compat seccomp context into main context");
-+			} else {
-+				seccomp_release(ctx.contexts[0]);
-+				ctx.contexts[0] = NULL;
-+			}
- 		}
--	}
- 
--	if (ctx.contexts[1]) {
--		if (ctx.needs_merge[1]) {
--			ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[1]);
--			if (ret < 0) {
--				ERROR("Failed to merge first compat seccomp "
--				      "context into main context");
--				goto bad;
--			}
-+		if (ctx.contexts[1]) {
-+			if (ctx.needs_merge[1]) {
-+				ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[1]);
-+				if (ret < 0) {
-+					ERROR("Failed to merge first compat seccomp "
-+					      "context into main context");
-+					goto bad;
-+				}
- 
--			TRACE("Merged second compat seccomp context into main context");
--		} else {
--			seccomp_release(ctx.contexts[1]);
--			ctx.contexts[1] = NULL;
-+				TRACE("Merged second compat seccomp context into main context");
-+			} else {
-+				seccomp_release(ctx.contexts[1]);
-+				ctx.contexts[1] = NULL;
-+			}
- 		}
--	}
- 
--	if (ctx.contexts[2]) {
--		if (ctx.needs_merge[2]) {
--			ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[2]);
--			if (ret < 0) {
--				ERROR("Failed to merge third compat seccomp "
--				      "context into main context");
--				goto bad;
--			}
-+		if (ctx.contexts[2]) {
-+			if (ctx.needs_merge[2]) {
-+				ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[2]);
-+				if (ret < 0) {
-+					ERROR("Failed to merge third compat seccomp "
-+					      "context into main context");
-+					goto bad;
-+				}
- 
--			TRACE("Merged third compat seccomp context into main context");
--		} else {
--			seccomp_release(ctx.contexts[2]);
--			ctx.contexts[2] = NULL;
-+				TRACE("Merged third compat seccomp context into main context");
-+			} else {
-+				seccomp_release(ctx.contexts[2]);
-+				ctx.contexts[2] = NULL;
-+			}
- 		}
- 	}
--
- 	free(line);
- 	return 0;
- 
--- 
-1.8.3.1
-

0043-check-null-pointer-of-handler-to-fix-coredump-of-att.patch

@@ -1,28 +0,0 @@
-From b94affbd314be09dcd5927b15bee85459f6bc2ff Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Thu, 17 Jan 2019 10:19:37 +0800
-Subject: [PATCH 043/140] check null pointer of handler to fix coredump of
- attach
-
-Signed-off-by: tanyifeng <tanyifeng1@huawei.com>
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/cgroups/cgroup.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/src/lxc/cgroups/cgroup.c b/src/lxc/cgroups/cgroup.c
-index 7442c31..720a6c9 100644
---- a/src/lxc/cgroups/cgroup.c
-+++ b/src/lxc/cgroups/cgroup.c
-@@ -44,7 +44,7 @@ struct cgroup_ops *cgroup_init(struct lxc_handler *handler)
- {
- 	struct cgroup_ops *cgroup_ops;
- 
--	cgroup_ops = cgfsng_ops_init(handler->conf->errpipe[1]);
-+	cgroup_ops = cgfsng_ops_init(handler ? handler->conf->errpipe[1] : -1);
- 	if (!cgroup_ops) {
- 		ERROR("Failed to initialize cgroup driver");
- 		return NULL;
--- 
-1.8.3.1
-

0043-support-error-report.patch

@@ -0,0 +1,683 @@
+From 968c9e3e7715c080f23a1fd80c31d4bcf20d241b Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Thu, 16 Apr 2020 15:16:41 +0800
+Subject: [PATCH 43/49] support error report
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ src/lxc/attach.c          | 60 +++++++++++++++++++++++++++++++++++++++++++----
+ src/lxc/attach_options.h  | 12 ++++++++++
+ src/lxc/cgroups/cgfsng.c  |  1 +
+ src/lxc/conf.c            | 59 ++++++++++++++++++++++++++++++++++++++++++++++
+ src/lxc/conf.h            |  5 ++++
+ src/lxc/execute.c         |  7 ++++++
+ src/lxc/lxccontainer.c    | 47 +++++++++++++++++++++++++++++++++++++
+ src/lxc/start.c           | 21 +++++++++++++++++
+ src/lxc/start.h           |  4 ++++
+ src/lxc/tools/lxc_ls.c    |  8 +++++++
+ src/lxc/tools/lxc_start.c |  5 ++++
+ src/lxc/utils.c           | 16 +++++++++++++
+ 12 files changed, 241 insertions(+), 4 deletions(-)
+
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index 510c069..734cddd 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -677,9 +677,13 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 	bool needs_lsm = (options->namespaces & CLONE_NEWNS) &&
+ 			 (options->attach_flags & LXC_ATTACH_LSM) &&
+ 			 init_ctx->lsm_label;
+-
+ #ifdef HAVE_ISULAD
+-		/*isulad: set system umask */
++	int msg_fd = -1;
++
++	/*isulad: record errpipe fd*/
++	msg_fd = init_ctx->container->lxc_conf->errpipe[1];
++	init_ctx->container->lxc_conf->errpipe[1] = -1;
++	/*isulad: set system umask */
+ 	umask(init_ctx->container->lxc_conf->umask);
+ #endif
+ 
+@@ -963,10 +967,12 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 	payload->ipc_socket = -EBADF;
+ 	lxc_proc_put_context_info(init_ctx);
+ 	payload->init_ctx = NULL;
+-#endif
+-
++	_exit(payload->exec_function(payload->exec_payload, msg_fd));
++#else
+ 	/* We're done, so we can now do whatever the user intended us to do. */
+ 	_exit(payload->exec_function(payload->exec_payload));
++#endif
++
+ 
+ on_error:
+ 	lxc_put_attach_clone_payload(payload);
+@@ -1331,6 +1337,25 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 		return -1;
+ 	}
+ 
++#ifdef HAVE_ISULAD
++	/* isulad: pipdfd for get error message of child or grandchild process. */
++	if (pipe2(conf->errpipe, O_CLOEXEC) != 0) {
++		SYSERROR("Failed to init errpipe");
++		if (options->attach_flags & LXC_ATTACH_TERMINAL) {
++			lxc_terminal_delete(&terminal);
++			lxc_terminal_conf_free(&terminal);
++			if (exec_command.maincmd_fd != -1) {
++				close(exec_command.maincmd_fd);
++			}
++		}
++		close(ipc_sockets[0]);
++		close(ipc_sockets[1]);
++		free(cwd);
++		lxc_proc_put_context_info(init_ctx);
++		return -1;
++	}
++#endif
++
+ 	/* Create intermediate subprocess, two reasons:
+ 	 *       1. We can't setns() in the child itself, since we want to make
+ 	 *          sure we are properly attached to the pidns.
+@@ -1365,6 +1390,11 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 		/* close unneeded file descriptors */
+ 		close(ipc_sockets[1]);
+ 		free(cwd);
++#ifdef HAVE_ISULAD
++		/* isulad: close errpipe */
++		close(conf->errpipe[1]);
++		conf->errpipe[1] = -1;
++#endif
+ 		lxc_proc_close_ns_fd(init_ctx);
+ 		if (options->attach_flags & LXC_ATTACH_TERMINAL)
+ 			lxc_attach_terminal_close_slave(&terminal);
+@@ -1398,7 +1428,11 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 
+ 		/* Setup resource limits */
+ 		if (!lxc_list_empty(&conf->limits)) {
++#ifdef HAVE_ISULAD
++			ret = setup_resource_limits(&conf->limits, pid, -1);
++#else
+ 			ret = setup_resource_limits(&conf->limits, pid);
++#endif
+ 			if (ret < 0)
+ 				goto on_error;
+ 		}
+@@ -1561,6 +1595,12 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 	/* close unneeded file descriptors */
+ 	close_prot_errno_disarm(ipc_sockets[0]);
+ 
++#ifdef HAVE_ISULAD
++	/* isulad: close errpipe */
++	close(conf->errpipe[0]);
++	conf->errpipe[0] = -1;
++#endif
++
+ 	if (options->attach_flags & LXC_ATTACH_TERMINAL) {
+ 		lxc_attach_terminal_close_master(&terminal);
+ 		lxc_attach_terminal_close_peer(&terminal);
+@@ -1665,7 +1705,11 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 	_exit(EXIT_SUCCESS);
+ }
+ 
++#ifdef HAVE_ISULAD
++int lxc_attach_run_command(void *payload, int msg_fd)
++#else
+ int lxc_attach_run_command(void *payload)
++#endif
+ {
+ 	int ret = -1;
+ 	lxc_attach_command_t *cmd = payload;
+@@ -1681,11 +1725,19 @@ int lxc_attach_run_command(void *payload)
+ 			break;
+ 		}
+ 	}
++#ifdef HAVE_ISULAD
++	/* isulad: write error messages */
++	lxc_write_error_message(msg_fd, "exec: \"%s\": %s.", cmd->program, strerror(errno));
++#endif
+ 
+ 	return log_error_errno(ret, errno, "Failed to exec \"%s\"", cmd->program);
+ }
+ 
++#ifdef HAVE_ISULAD
++int lxc_attach_run_shell(void* payload, int msg_fd)
++#else
+ int lxc_attach_run_shell(void* payload)
++#endif
+ {
+ 	__do_free char *buf = NULL;
+ 	uid_t uid;
+diff --git a/src/lxc/attach_options.h b/src/lxc/attach_options.h
+index 5f01739..d5d4f44 100644
+--- a/src/lxc/attach_options.h
++++ b/src/lxc/attach_options.h
+@@ -49,7 +49,11 @@ enum {
+  *
+  * \return Function should return \c 0 on success, and any other value to denote failure.
+  */
++#ifdef HAVE_ISULAD
++typedef int (*lxc_attach_exec_t)(void* payload, int msg_fd);
++#else
+ typedef int (*lxc_attach_exec_t)(void* payload);
++#endif
+ 
+ /*!
+  * LXC attach options for \ref lxc_container \c attach().
+@@ -153,7 +157,11 @@ typedef struct lxc_attach_command_t {
+  *
+  * \return \c -1 on error, exit code of lxc_attach_command_t program on success.
+  */
++#ifdef HAVE_ISULAD
++extern int lxc_attach_run_command(void* payload, int msg_fd);
++#else
+ extern int lxc_attach_run_command(void* payload);
++#endif
+ 
+ /*!
+  * \brief Run a shell command in the container.
+@@ -162,7 +170,11 @@ extern int lxc_attach_run_command(void* payload);
+  *
+  * \return Exit code of shell.
+  */
++#ifdef HAVE_ISULAD
++extern int lxc_attach_run_shell(void* payload, int msg_fd);
++#else
+ extern int lxc_attach_run_shell(void* payload);
++#endif
+ 
+ #ifdef  __cplusplus
+ }
+diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
+index 1e1df3b..002f051 100644
+--- a/src/lxc/cgroups/cgfsng.c
++++ b/src/lxc/cgroups/cgfsng.c
+@@ -4233,6 +4233,7 @@ struct cgroup_ops *cgfsng_ops_init(struct lxc_conf *conf)
+ 
+ 	cgfsng_ops->data_init = cgfsng_data_init;
+ #ifdef HAVE_ISULAD
++	cgfsng_ops->errfd = conf ? conf->errpipe[1] : -1;
+ 	cgfsng_ops->payload_destroy = isulad_cgfsng_payload_destroy;
+ #else
+ 	cgfsng_ops->payload_destroy = cgfsng_payload_destroy;
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index e8ee749..d7a78bd 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -2275,8 +2275,15 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
+ 
+ 	if (hasmntopt(mntent, "create=dir")) {
+ 		ret = mkdir_p(path, 0755);
++#ifdef HAVE_ISULAD
++		if (ret < 0 && errno != EEXIST) {
++			lxc_write_error_message(rootfs->errfd, "%s:%d: mkdir %s: %s.", __FILE__, __LINE__, path, strerror(errno));
++			return log_error_errno(-1, errno, "Failed to create directory \"%s\"", path);
++		}
++#else
+ 		if (ret < 0 && errno != EEXIST)
+ 			return log_error_errno(-1, errno, "Failed to create directory \"%s\"", path);
++#endif
+ 	}
+ 
+ 	if (!hasmntopt(mntent, "create=file"))
+@@ -2293,12 +2300,26 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
+ 	p2 = dirname(p1);
+ 
+ 	ret = mkdir_p(p2, 0755);
++#ifdef HAVE_ISULAD
++	if (ret < 0 && errno != EEXIST) {
++		lxc_write_error_message(rootfs->errfd, "%s:%d: mkdir %s: %s.", __FILE__, __LINE__, path, strerror(errno));
++		return log_error_errno(-1, errno, "Failed to create directory \"%s\"", path);
++	}
++#else
+ 	if (ret < 0 && errno != EEXIST)
+ 		return log_error_errno(-1, errno, "Failed to create directory \"%s\"", path);
++#endif
+ 
+ 	ret = mknod(path, S_IFREG | 0000, 0);
++#ifdef HAVE_ISULAD
++	if (ret < 0 && errno != EEXIST) {
++		lxc_write_error_message(rootfs->errfd, "%s:%d: open %s: %s.", __FILE__, __LINE__, path, strerror(errno));
++		return -errno;
++	}
++#else
+ 	if (ret < 0 && errno != EEXIST)
+ 		return -errno;
++#endif
+ 
+ 	return 0;
+ }
+@@ -2960,7 +2981,11 @@ static int parse_resource(const char *res)
+ 	return resid;
+ }
+ 
++#ifdef HAVE_ISULAD
++int setup_resource_limits(struct lxc_list *limits, pid_t pid, int errfd)
++#else
+ int setup_resource_limits(struct lxc_list *limits, pid_t pid)
++#endif
+ {
+ 	int resid;
+ 	struct lxc_list *it;
+@@ -2974,8 +2999,17 @@ int setup_resource_limits(struct lxc_list *limits, pid_t pid)
+ 			return log_error(-1, "Unknown resource %s", lim->resource);
+ 
+ #if HAVE_PRLIMIT || HAVE_PRLIMIT64
++#if HAVE_ISULAD
++		if (prlimit(pid, resid, &lim->limit, NULL) != 0) {
++			lxc_write_error_message(errfd, "%s:%d: Failed to set limit %s %lu %lu: %s.",
++					__FILE__, __LINE__, lim->resource,
++					lim->limit.rlim_cur, lim->limit.rlim_max, strerror(errno));
++			return log_error_errno(-1, errno, "Failed to set limit %s", lim->resource);
++		}
++#else
+ 		if (prlimit(pid, resid, &lim->limit, NULL) != 0)
+ 			return log_error_errno(-1, errno, "Failed to set limit %s", lim->resource);
++#endif
+ 
+ 		TRACE("Setup \"%s\" limit", lim->resource);
+ #else
+@@ -3134,6 +3168,9 @@ struct lxc_conf *lxc_conf_init(void)
+ 	new->console.pipes[2][0] = -1;
+ 	new->console.pipes[2][1] = -1;
+ 	lxc_list_init(&new->console.fifos);
++	new->errmsg = NULL;
++	new->errpipe[0] = -1;
++	new->errpipe[1] = -1;
+ #endif
+ 
+ 	return new;
+@@ -4534,8 +4571,16 @@ int lxc_setup(struct lxc_handler *handler)
+ 	char *keyring_context = NULL;
+ 
+ 	ret = lxc_setup_rootfs_prepare_root(lxc_conf, name, lxcpath);
++#ifdef HAVE_ISULAD
++	if (ret < 0) {
++		lxc_write_error_message(lxc_conf->errpipe[1], "%s:%d: failed to setup rootfs %s.",
++				__FILE__, __LINE__, lxc_conf->rootfs.path);
++		return log_error(-1, "Failed to setup rootfs");
++	}
++#else
+ 	if (ret < 0)
+ 		return log_error(-1, "Failed to setup rootfs");
++#endif
+ 
+ 	if (handler->nsfd[LXC_NS_UTS] == -EBADF) {
+ 		ret = setup_utsname(lxc_conf->utsname);
+@@ -5243,6 +5288,8 @@ void lxc_conf_free(struct lxc_conf *conf)
+ 	lxc_clear_populate_devices(conf);
+ 	lxc_clear_rootfs_masked_paths(conf);
+ 	lxc_clear_rootfs_ro_paths(conf);
++	free(conf->errmsg);
++	lxc_close_error_pipe(conf->errpipe);
+ #endif
+ 	free(conf);
+ }
+@@ -6127,4 +6174,16 @@ int lxc_clear_rootfs_ro_paths(struct lxc_conf *c)
+ 	return 0;
+ }
+ 
++/*isulad: close error pipe */
++void lxc_close_error_pipe(int *errpipe)
++{
++	if (errpipe[0] >= 0) {
++		close(errpipe[0]);
++		errpipe[0] = -1;
++	}
++	if (errpipe[1] >= 0) {
++		close(errpipe[1]);
++		errpipe[1] = -1;
++	}
++}
+ #endif
+diff --git a/src/lxc/conf.h b/src/lxc/conf.h
+index 7b6fd3b..4b6409e 100644
+--- a/src/lxc/conf.h
++++ b/src/lxc/conf.h
+@@ -506,7 +506,11 @@ extern int lxc_setup_rootfs_prepare_root(struct lxc_conf *conf,
+ 					 const char *name, const char *lxcpath);
+ extern int lxc_setup(struct lxc_handler *handler);
+ extern int lxc_setup_parent(struct lxc_handler *handler);
++#ifdef HAVE_ISULAD
++extern int setup_resource_limits(struct lxc_list *limits, pid_t pid, int errfd);
++#else
+ extern int setup_resource_limits(struct lxc_list *limits, pid_t pid);
++#endif
+ extern int find_unmapped_nsid(const struct lxc_conf *conf, enum idtype idtype);
+ extern int mapped_hostid(unsigned id, const struct lxc_conf *conf,
+ 			 enum idtype idtype);
+@@ -557,5 +561,6 @@ int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
+ int lxc_clear_rootfs_ro_paths(struct lxc_conf *c);
+ int lxc_drop_caps(struct lxc_conf *conf);
+ int run_oci_hooks(const char *name, const char *hookname, struct lxc_conf *conf, const char *lxcpath);
++void lxc_close_error_pipe(int *errpipe);
+ #endif
+ #endif /* __LXC_CONF_H */
+diff --git a/src/lxc/execute.c b/src/lxc/execute.c
+index 59ff604..16c0fed 100644
+--- a/src/lxc/execute.c
++++ b/src/lxc/execute.c
+@@ -19,7 +19,11 @@
+ 
+ lxc_log_define(execute, start);
+ 
++#ifdef HAVE_ISULAD
++static int execute_start(struct lxc_handler *handler, void* data, int fd)
++#else
+ static int execute_start(struct lxc_handler *handler, void* data)
++#endif
+ {
+ 	int argc_add, j;
+ 	char **argv;
+@@ -71,6 +75,9 @@ static int execute_start(struct lxc_handler *handler, void* data)
+ 		execvp(argv[0], argv);
+ 	SYSERROR("Failed to exec %s", argv[0]);
+ 
++#ifdef HAVE_ISULAD
++	lxc_write_error_message(fd, "Failed to exec: \"%s\": %s.", argv[0], strerror(errno));
++#endif
+ 	free(argv);
+ out1:
+ 	return 1;
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index 818848a..ed09a59 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -972,6 +972,8 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+ 	char **init_cmd = NULL;
+ #ifdef HAVE_ISULAD
+ 	int keepfds[] = {-1, -1, -1, -1, -1};
++	ssize_t size_read;
++	char errbuf[BUFSIZ + 1] = {0};
+ #else
+ 	int keepfds[3] = {-1, -1, -1};
+ #endif
+@@ -1046,10 +1048,23 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+ 		char title[2048];
+ 		pid_t pid_first, pid_second;
+ 
++#ifdef HAVE_ISULAD
++		//isulad: pipdfd for get error message of child or grandchild process.
++		if (pipe2(conf->errpipe, O_CLOEXEC) != 0) {
++			SYSERROR("Failed to init errpipe");
++			free_init_cmd(init_cmd);
++			lxc_free_handler(handler);
++			return false;
++		}
++#endif
++
+ 		pid_first = fork();
+ 		if (pid_first < 0) {
+ 			free_init_cmd(init_cmd);
+ 			lxc_free_handler(handler);
++#ifdef HAVE_ISULAD
++			lxc_close_error_pipe(conf->errpipe);
++#endif
+ 			return false;
+ 		}
+ 
+@@ -1059,11 +1074,25 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+ 			 * the PID file, child will do the free and unlink.
+ 			 */
+ 			c->pidfile = NULL;
++#ifdef HAVE_ISULAD
++			close(conf->errpipe[1]);
++			conf->errpipe[1] = -1;
++#endif
+ 
+ 			/* Wait for container to tell us whether it started
+ 			 * successfully.
+ 			 */
+ 			started = wait_on_daemonized_start(handler, pid_first);
++#ifdef HAVE_ISULAD
++			if (!started) {
++				size_read = read(conf->errpipe[0], errbuf, BUFSIZ);
++				if (size_read > 0) {
++					conf->errmsg = safe_strdup(errbuf);
++				}
++			}
++			close(conf->errpipe[0]);
++			conf->errpipe[0] = -1;
++#endif
+ 
+ 			free_init_cmd(init_cmd);
+ 			lxc_free_handler(handler);
+@@ -1099,6 +1128,9 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+ 		if (pid_second != 0) {
+ 			free_init_cmd(init_cmd);
+ 			lxc_free_handler(handler);
++#ifdef HAVE_ISULAD
++			lxc_close_error_pipe(conf->errpipe);
++#endif
+ 			_exit(EXIT_SUCCESS);
+ 		}
+ 
+@@ -1114,6 +1146,11 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+ 		keepfds[0] = handler->conf->maincmd_fd;
+ 		keepfds[1] = handler->state_socket_pair[0];
+ 		keepfds[2] = handler->state_socket_pair[1];
++#ifdef HAVE_ISULAD
++		keepfds[4] = conf->errpipe[1];
++		close(conf->errpipe[0]);
++		conf->errpipe[0] = -1;
++#endif
+ 		ret = lxc_check_inherited(conf, true, keepfds,
+ 					  sizeof(keepfds) / sizeof(keepfds[0]));
+ 		if (ret < 0)
+@@ -1148,6 +1185,9 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+ 		if (w < 0 || (size_t)w >= sizeof(pidstr)) {
+ 			free_init_cmd(init_cmd);
+ 			lxc_free_handler(handler);
++#ifdef HAVE_ISULAD
++			lxc_close_error_pipe(conf->errpipe);
++#endif
+ 
+ 			SYSERROR("Failed to write monitor pid to \"%s\"", c->pidfile);
+ 
+@@ -1161,6 +1201,9 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
+ 		if (ret < 0) {
+ 			free_init_cmd(init_cmd);
+ 			lxc_free_handler(handler);
++#ifdef HAVE_ISULAD
++			lxc_close_error_pipe(conf->errpipe);
++#endif
+ 
+ 			SYSERROR("Failed to write monitor pid to \"%s\"", c->pidfile);
+ 
+@@ -1224,6 +1267,9 @@ reboot:
+ 	if (conf->exit_fd >= 0) {
+ 		keepfds[3] = conf->exit_fd;
+ 	}
++	/* isulad: keep errpipe fd */
++	if (c->daemonize)
++		keepfds[4] = conf->errpipe[1];
+ #endif
+ 
+ 	ret = lxc_check_inherited(conf, c->daemonize, keepfds,
+@@ -3250,6 +3296,7 @@ static bool container_destroy(struct lxc_container *c,
+ 			ERROR("Sprintf failed");
+ 			goto out;
+ 		}
++		c->error_string = safe_strdup(msg);
+ #endif
+ 		goto out;
+ 	}
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 0bc1143..134235f 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -836,6 +836,9 @@ int lxc_init(const char *name, struct lxc_handler *handler)
+ 	ret = lxc_terminal_setup(conf);
+ 	if (ret < 0) {
+ 		ERROR("Failed to create console");
++#ifdef HAVE_ISULAD
++		lxc_write_error_message(conf->errpipe[1], "Failed to create console for container \"%s\".", name);
++#endif
+ 		goto out_restore_sigmask;
+ 	}
+ 	TRACE("Created console");
+@@ -1470,6 +1473,9 @@ static int do_start(void *data)
+ 	/* Setup the container, ip, names, utsname, ... */
+ 	ret = lxc_setup(handler);
+ 	if (ret < 0) {
++#ifdef HAVE_ISULAD
++		lxc_write_error_message(handler->conf->errpipe[1], "Failed to setup lxc, please check the config file.");
++#endif
+ 		ERROR("Failed to setup container \"%s\"", handler->name);
+ 		goto out_warn_father;
+ 	}
+@@ -1763,7 +1769,11 @@ static int do_start(void *data)
+ 	/* After this call, we are in error because this ops should not return
+ 	 * as it execs.
+ 	 */
++#ifdef HAVE_ISULAD
++	handler->ops->start(handler, handler->data, handler->daemonize ? handler->conf->errpipe[1] : -1);
++#else
+ 	handler->ops->start(handler, handler->data);
++#endif
+ 
+ out_warn_father:
+ 	/* We want the parent to know something went wrong, so we return a
+@@ -2246,7 +2256,11 @@ static int lxc_spawn(struct lxc_handler *handler)
+ 		goto out_delete_net;
+ 
+ 	if (!lxc_list_empty(&conf->limits)) {
++#ifdef HAVE_ISULAD
++		ret = setup_resource_limits(&conf->limits, handler->pid, conf->errpipe[1]);
++#else
+ 		ret = setup_resource_limits(&conf->limits, handler->pid);
++#endif
+ 		if (ret < 0) {
+ 			ERROR("Failed to setup resource limits");
+ 			goto out_delete_net;
+@@ -2652,7 +2666,11 @@ struct start_args {
+ 	char *const *argv;
+ };
+ 
++#ifdef HAVE_ISULAD
++static int start(struct lxc_handler *handler, void* data, int fd)
++#else
+ static int start(struct lxc_handler *handler, void* data)
++#endif
+ {
+ 	struct start_args *arg = data;
+ 
+@@ -2660,6 +2678,9 @@ static int start(struct lxc_handler *handler, void* data)
+ 
+ 	execvp(arg->argv[0], arg->argv);
+ 	SYSERROR("Failed to exec \"%s\"", arg->argv[0]);
++#ifdef HAVE_ISULAD
++	lxc_write_error_message(fd, "exec: \"%s\": %s.", arg->argv[0], strerror(errno));
++#endif
+ 	return 0;
+ }
+ 
+diff --git a/src/lxc/start.h b/src/lxc/start.h
+index cea37bc..ebeeb72 100644
+--- a/src/lxc/start.h
++++ b/src/lxc/start.h
+@@ -141,7 +141,11 @@ struct execute_args {
+ };
+ 
+ struct lxc_operations {
++#ifdef HAVE_ISULAD
++	int (*start)(struct lxc_handler *, void *, int);
++#else
+ 	int (*start)(struct lxc_handler *, void *);
++#endif
+ 	int (*post_start)(struct lxc_handler *, void *);
+ };
+ 
+diff --git a/src/lxc/tools/lxc_ls.c b/src/lxc/tools/lxc_ls.c
+index 4be8564..e601f9d 100644
+--- a/src/lxc/tools/lxc_ls.c
++++ b/src/lxc/tools/lxc_ls.c
+@@ -106,7 +106,11 @@ struct wrapargs {
+ /*
+  * Takes struct wrapargs as argument.
+  */
++#ifdef HAVE_ISULAD
++static int ls_get_wrapper(void *wrap, int msgfd);
++#else
+ static int ls_get_wrapper(void *wrap);
++#endif
+ 
+ /*
+  * To calculate swap usage we should not simply check memory.usage_in_bytes and
+@@ -1005,7 +1009,11 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 	return 0;
+ }
+ 
++#ifdef HAVE_ISULAD
++static int ls_get_wrapper(void *wrap, int msgfd)
++#else
+ static int ls_get_wrapper(void *wrap)
++#endif
+ {
+ 	int ret = -1;
+ 	size_t len = 0;
+diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
+index b430a1e..4f2c8af 100644
+--- a/src/lxc/tools/lxc_start.c
++++ b/src/lxc/tools/lxc_start.c
+@@ -392,6 +392,11 @@ int main(int argc, char *argv[])
+ 	else
+ 		err = c->start(c, 0, args) ? EXIT_SUCCESS : EXIT_FAILURE;
+ 	if (err) {
++#ifdef HAVE_ISULAD
++		if (c->lxc_conf->errmsg)
++			fprintf(stderr, "%s:%s:%s:%d starting container process caused \"%s\"", c->name,
++					__FILE__, __func__, __LINE__, c->lxc_conf->errmsg);
++#endif
+ 		ERROR("The container failed to start");
+ 
+ 		if (my_args.daemonize)
+diff --git a/src/lxc/utils.c b/src/lxc/utils.c
+index 810b7fe..4e418fb 100644
+--- a/src/lxc/utils.c
++++ b/src/lxc/utils.c
+@@ -73,6 +73,9 @@ static int _recursive_rmdir(const char *dirname, dev_t pdev,
+ 	int ret;
+ 	struct dirent *direntp;
+ 	char pathname[PATH_MAX];
++#ifdef HAVE_ISULAD
++	int saved_errno = 0;
++#endif
+ 
+ 	dir = opendir(dirname);
+ 	if (!dir)
+@@ -135,6 +138,11 @@ static int _recursive_rmdir(const char *dirname, dev_t pdev,
+ 		} else {
+ 			ret = unlink(pathname);
+ 			if (ret < 0) {
++#ifdef HAVE_ISULAD
++				if (saved_errno == 0) {
++					saved_errno = errno;
++				}
++#endif
+ 				__do_close int fd = -EBADF;
+ 
+ 				fd = open(pathname, O_RDONLY | O_CLOEXEC | O_NONBLOCK);
+@@ -160,10 +168,18 @@ static int _recursive_rmdir(const char *dirname, dev_t pdev,
+ 	}
+ 
+ 	if (rmdir(dirname) < 0 && !btrfs_try_remove_subvol(dirname) && !hadexclude) {
++#ifdef HAVE_ISULAD
++		if (saved_errno == 0) {
++			saved_errno = errno;
++		}
++#endif
+ 		SYSERROR("Failed to delete \"%s\"", dirname);
+ 		failed = 1;
+ 	}
+ 
++#ifdef HAVE_ISULAD
++	errno = saved_errno;
++#endif
+ 	return failed ? -1 : 0;
+ }
+ 
+-- 
+1.8.3.1
+

0044-remove-filelock-in-destroy-dir.patch

@@ -0,0 +1,153 @@
+From 0441dc446ad9f0bc02c5ca7a76d793c8a7734fd9 Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Thu, 16 Apr 2020 15:39:50 +0800
+Subject: [PATCH 44/49] remove filelock in destroy dir
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ src/lxc/lxccontainer.c | 21 +++++++++++++++++++++
+ src/lxc/lxclock.c      | 27 +++++++++++++++++++++++++++
+ src/lxc/lxclock.h      |  4 ++++
+ src/lxc/start.c        |  7 +++++++
+ src/lxc/storage/dir.c  |  4 ++++
+ 5 files changed, 63 insertions(+)
+
+diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
+index ed09a59..6281a8a 100644
+--- a/src/lxc/lxccontainer.c
++++ b/src/lxc/lxccontainer.c
+@@ -3168,8 +3168,19 @@ static bool container_destroy(struct lxc_container *c,
+ 	bool bret = false;
+ 	int ret = 0;
+ 
++#ifdef HAVE_ISULAD
++	if (!c)
++		return false;
++	// isulad: if container is not defined, we need to remove disk lock file
++	// which is created in lxc_container_new.
++	if (!do_lxcapi_is_defined(c)) {
++		container_disk_removelock(c);
++		return false;
++	}
++#else
+ 	if (!c || !do_lxcapi_is_defined(c))
+ 		return false;
++#endif
+ 
+ 	conf = c->lxc_conf;
+ 	if (container_disk_lock(c))
+@@ -3310,13 +3321,23 @@ out:
+ 		free(path);
+ 
+ 	container_disk_unlock(c);
++#ifdef HAVE_ISULAD
++	if (bret && container_disk_removelock(c)) {
++		bret = false;
++	}
++#endif
+ 	return bret;
+ }
+ 
+ static bool do_lxcapi_destroy(struct lxc_container *c)
+ {
++#ifdef HAVE_ISULAD
++	if (!c)
++		return false;
++#else
+ 	if (!c || !lxcapi_is_defined(c))
+ 		return false;
++#endif
+ 
+ 	if (c->lxc_conf && c->lxc_conf->rootfs.managed) {
+ 		if (has_snapshots(c)) {
+diff --git a/src/lxc/lxclock.c b/src/lxc/lxclock.c
+index 318e5bf..bb0dca0 100644
+--- a/src/lxc/lxclock.c
++++ b/src/lxc/lxclock.c
+@@ -370,3 +370,30 @@ void container_disk_unlock(struct lxc_container *c)
+ 	lxcunlock(c->slock);
+ 	lxcunlock(c->privlock);
+ }
++
++#ifdef HAVE_ISULAD
++static int lxc_removelock(struct lxc_lock *l)
++{
++	int ret = 0;
++
++	if (l->type == LXC_LOCK_FLOCK) {
++		ret = unlink(l->u.f.fname);
++		if (ret && errno != ENOENT) {
++			SYSERROR("Error unlink %s", l->u.f.fname);
++			return ret;
++		}
++	}
++
++	return ret;
++}
++
++int container_disk_removelock(struct lxc_container *c)
++{
++	int ret;
++
++	ret = lxc_removelock(c->slock);
++	if (ret)
++		return ret;
++	return lxc_removelock(c->privlock);
++}
++#endif
+diff --git a/src/lxc/lxclock.h b/src/lxc/lxclock.h
+index 9f9bc3b..6a71d7c 100644
+--- a/src/lxc/lxclock.h
++++ b/src/lxc/lxclock.h
+@@ -154,4 +154,8 @@ extern int container_disk_lock(struct lxc_container *c);
+  */
+ extern void container_disk_unlock(struct lxc_container *c);
+ 
++#ifdef HAVE_ISULAD
++int container_disk_removelock(struct lxc_container *c);
++#endif
++
+ #endif
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 134235f..6779cee 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -2143,6 +2143,13 @@ static int lxc_spawn(struct lxc_handler *handler)
+ 	if (ret < 0)
+ 		SYSERROR("Failed to set environment variable: LXC_PID=%s", pidstr);
+ 
++#ifdef HAVE_ISULAD
++	if (handler->cgroup_ops->container_cgroup) {
++		if (setenv("LXC_CGROUP_PATH", handler->cgroup_ops->container_cgroup, 1))
++			SYSERROR("Failed to set environment variable: LXC_CGROUP_PATH=%s.", handler->cgroup_ops->container_cgroup);
++	}
++#endif
++
+ 	for (i = 0; i < LXC_NS_MAX; i++)
+ 		if (handler->ns_on_clone_flags & ns_info[i].clone_flag)
+ 			INFO("Cloned %s", ns_info[i].flag_name);
+diff --git a/src/lxc/storage/dir.c b/src/lxc/storage/dir.c
+index 1dc95f1..485572a 100644
+--- a/src/lxc/storage/dir.c
++++ b/src/lxc/storage/dir.c
+@@ -94,6 +94,9 @@ int dir_create(struct lxc_storage *bdev, const char *dest, const char *n,
+ 
+ int dir_destroy(struct lxc_storage *orig)
+ {
++#ifdef HAVE_ISULAD
++	// isulad: do not destroy rootfs for directory, it should be managed by caller
++#else
+ 	int ret;
+ 	const char *src;
+ 
+@@ -102,6 +105,7 @@ int dir_destroy(struct lxc_storage *orig)
+ 	ret = lxc_rmdir_onedev(src, NULL);
+ 	if (ret < 0)
+ 		return log_error_errno(ret, errno, "Failed to delete \"%s\"", src);
++#endif
+ 
+ 	return 0;
+ }
+-- 
+1.8.3.1
+

0044-support-space-in-volume-mount-and-env.patch

@@ -1,118 +0,0 @@
-From 24d0250197bab7b59c3437cce7845af1fc88df7e Mon Sep 17 00:00:00 2001
-From: tanyifeng <tanyifeng1@huawei.com>
-Date: Thu, 17 Jan 2019 10:18:23 +0800
-Subject: [PATCH 044/140] support space in --volume, --mount and --env
-
-Signed-off-by: tanyifeng <tanyifeng1@huawei.com>
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c      | 16 ++++++++++++++++
- src/lxc/confile.c   |  9 ++++++++-
- src/lxc/namespace.h |  1 +
- src/lxc/utils.h     |  3 +++
- 4 files changed, 28 insertions(+), 1 deletion(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 8fa63f7..48e31af 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -54,6 +54,7 @@
- #include <time.h>
- #include <unistd.h>
- #include <pthread.h>
-+#include <linux/fs.h>
- 
- #include "af_unix.h"
- #include "caps.h"
-@@ -2695,6 +2696,19 @@ static int mount_file_entries(const struct lxc_conf *conf,
- 	int ret = -1;
- 
- 	while (getmntent_r(file, &mntent, buf, sizeof(buf))) {
-+		/* Note: Workaround for volume file path with space*/
-+		mntent.mnt_fsname = lxc_string_replace(SPACE_MAGIC_STR, " ", mntent.mnt_fsname);
-+		if(!mntent.mnt_fsname) {
-+			SYSERROR("memory allocation error");
-+			return -1;
-+		}
-+		mntent.mnt_dir = lxc_string_replace(SPACE_MAGIC_STR, " ", mntent.mnt_dir);
-+		if(!mntent.mnt_dir) {
-+			SYSERROR("memory allocation error");
-+			free(mntent.mnt_fsname);
-+			return -1;
-+		}
-+		ERROR("mntent.mnt_fsname:%s, mntent.mnt_dir:%s", mntent.mnt_fsname, mntent.mnt_dir);
- 		if (!rootfs->path)
- 			ret = mount_entry_on_systemfs(&mntent);
- 		else if (mntent.mnt_dir[0] != '/')
-@@ -2703,6 +2717,8 @@ static int mount_file_entries(const struct lxc_conf *conf,
- 		else
- 			ret = mount_entry_on_absolute_rootfs(&mntent, rootfs,
- 					                     lxc_name, lxc_path);
-+		free(mntent.mnt_fsname);
-+		free(mntent.mnt_dir);
- 		if (ret < 0)
- 			return -1;
- 	}
-diff --git a/src/lxc/confile.c b/src/lxc/confile.c
-index db63b55..7e9d5c8 100644
---- a/src/lxc/confile.c
-+++ b/src/lxc/confile.c
-@@ -1036,6 +1036,7 @@ static int set_config_environment(const char *key, const char *value,
- 				  struct lxc_conf *lxc_conf, void *data)
- {
- 	struct lxc_list *list_item = NULL;
-+	char *replaced;
- 
- 	if (lxc_config_value_empty(value))
- 		return lxc_clear_environment(lxc_conf);
-@@ -1044,7 +1045,12 @@ static int set_config_environment(const char *key, const char *value,
- 	if (!list_item)
- 		goto on_error;
- 
--	list_item->elem = strdup(value);
-+	/* isulad: recover space replaced by SPACE_MAGIC_STR */
-+	replaced = lxc_string_replace(SPACE_MAGIC_STR, " ", value);
-+	if(!replaced)
-+		goto on_error;
-+
-+	list_item->elem = replaced;
- 
- 	if (!list_item->elem)
- 		goto on_error;
-@@ -3661,6 +3667,7 @@ static int get_config_environment(const char *key, char *retv, int inlen,
- 		memset(retv, 0, inlen);
- 
- 	lxc_list_for_each(it, &c->environment) {
-+
- 		strprint(retv, inlen, "%s\n", (char *)it->elem);
- 	}
- 
-diff --git a/src/lxc/namespace.h b/src/lxc/namespace.h
-index ab583da..9caaf89 100644
---- a/src/lxc/namespace.h
-+++ b/src/lxc/namespace.h
-@@ -26,6 +26,7 @@
- #include <sched.h>
- #include <unistd.h>
- #include <sys/syscall.h>
-+#include <sys/types.h>
- 
- #ifndef CLONE_PARENT_SETTID
- #define CLONE_PARENT_SETTID 0x00100000
-diff --git a/src/lxc/utils.h b/src/lxc/utils.h
-index 4313942..73ffdd9 100644
---- a/src/lxc/utils.h
-+++ b/src/lxc/utils.h
-@@ -43,6 +43,9 @@
- #include "raw_syscalls.h"
- #include "string_utils.h"
- 
-+/* isulad: replace space with SPACE_MAGIC_STR */
-+#define SPACE_MAGIC_STR "[#)"
-+
- /* returns 1 on success, 0 if there were any failures */
- extern int lxc_rmdir_onedev(const char *path, const char *exclude);
- extern int get_u16(unsigned short *val, const char *arg, int base);
--- 
-1.8.3.1
-

0045-add_terminal_fifos-Add-terminal-fifos-dynamically.patch

@@ -1,254 +0,0 @@
-From 934f4937ec177e9c95445b800ca11adbb7c3a0ef Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Thu, 17 Jan 2019 02:18:14 -0500
-Subject: [PATCH 045/140] add_terminal_fifos: Add terminal fifos dynamically
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/commands.c     | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++
- src/lxc/commands.h     |  4 ++++
- src/lxc/lxccontainer.c | 24 +++++++++++++++++++
- src/lxc/lxccontainer.h | 10 ++++++++
- src/lxc/terminal.c     | 47 ++++++++++++++++++++++++++++++++++++++
- src/lxc/terminal.h     |  1 +
- 6 files changed, 148 insertions(+)
-
-diff --git a/src/lxc/commands.c b/src/lxc/commands.c
-index 47a824a..46b2805 100644
---- a/src/lxc/commands.c
-+++ b/src/lxc/commands.c
-@@ -96,6 +96,7 @@ static const char *lxc_cmd_str(lxc_cmd_t cmd)
- 		[LXC_CMD_ADD_STATE_CLIENT]    = "add_state_client",
- 		[LXC_CMD_CONSOLE_LOG]         = "console_log",
- 		[LXC_CMD_SERVE_STATE_CLIENTS] = "serve_state_clients",
-+		[LXC_CMD_SET_TERMINAL_FIFOS]  = "set_terminal_fifos",
- 	};
- 
- 	if (cmd >= LXC_CMD_MAX)
-@@ -1056,6 +1057,66 @@ reap_client_fd:
- 	return 1;
- }
- 
-+/*
-+ * isulad: lxc_cmd_set_terminal_fifos: Set the fifos used for the container as terminal input/output
-+ *
-+ * @hashed_sock_name: hashed socket name
-+ *
-+ * Returns 0 when success, else when fail.
-+ */
-+int lxc_cmd_set_terminal_fifos(const char *name, const char *lxcpath, const char *in_fifo, const char *out_fifo)
-+{
-+	int ret = 0, stopped = 0;
-+	int len = 0;
-+	char *tmp = NULL;
-+
-+	if (!in_fifo || !out_fifo) {
-+		return -1;
-+	}
-+
-+	len = strlen(in_fifo) + strlen("&&&&") + strlen(out_fifo) + 1;
-+	tmp = malloc(len);
-+	if (!tmp)
-+		return -1;
-+	snprintf(tmp, len, "%s%s%s", in_fifo, "&&&&", out_fifo);
-+
-+	struct lxc_cmd_rr cmd = {
-+		.req = {
-+			.cmd = LXC_CMD_SET_TERMINAL_FIFOS,
-+			.datalen = strlen(tmp)+1,
-+			.data = tmp,
-+		},
-+	};
-+
-+	ret = lxc_cmd(name, &cmd, &stopped, lxcpath, NULL);
-+	if (ret < 0) {
-+		ERROR("Failed to send command to container");
-+		free(tmp);
-+		return -1;
-+	}
-+
-+	if (cmd.rsp.ret != 0) {
-+		ERROR("Command response error:%d", cmd.rsp.ret);
-+		free(tmp);
-+		return -1;
-+	}
-+
-+	free(tmp);
-+	return 0;
-+}
-+
-+static int lxc_cmd_set_terminal_fifos_callback(int fd, struct lxc_cmd_req *req,
-+					struct lxc_handler *handler)
-+{
-+	struct lxc_cmd_rsp rsp;
-+	memset(&rsp, 0, sizeof(rsp));
-+
-+	rsp.ret = lxc_terminal_add_fifos(handler->conf, req->data);;
-+
-+	return lxc_cmd_rsp_send(fd, &rsp);
-+
-+}
-+
- static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
- 			   struct lxc_handler *handler)
- {
-@@ -1075,6 +1136,7 @@ static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
- 		[LXC_CMD_ADD_STATE_CLIENT]    = lxc_cmd_add_state_client_callback,
- 		[LXC_CMD_CONSOLE_LOG]         = lxc_cmd_console_log_callback,
- 		[LXC_CMD_SERVE_STATE_CLIENTS] = lxc_cmd_serve_state_clients_callback,
-+		[LXC_CMD_SET_TERMINAL_FIFOS]  = lxc_cmd_set_terminal_fifos_callback,
- 	};
- 
- 	if (req->cmd >= LXC_CMD_MAX) {
-diff --git a/src/lxc/commands.h b/src/lxc/commands.h
-index 2c024b6..0c64544 100644
---- a/src/lxc/commands.h
-+++ b/src/lxc/commands.h
-@@ -46,6 +46,7 @@ typedef enum {
- 	LXC_CMD_ADD_STATE_CLIENT,
- 	LXC_CMD_CONSOLE_LOG,
- 	LXC_CMD_SERVE_STATE_CLIENTS,
-+	LXC_CMD_SET_TERMINAL_FIFOS,
- 	LXC_CMD_MAX,
- } lxc_cmd_t;
- 
-@@ -125,4 +126,7 @@ extern int lxc_try_cmd(const char *name, const char *lxcpath);
- extern int lxc_cmd_console_log(const char *name, const char *lxcpath,
- 			       struct lxc_console_log *log);
- 
-+extern int lxc_cmd_set_terminal_fifos(const char *name, const char *lxcpath,
-+				const char *in_fifo, const char *out_fifo);
-+
- #endif /* __commands_h */
-diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
-index d641851..bfbf223 100644
---- a/src/lxc/lxccontainer.c
-+++ b/src/lxc/lxccontainer.c
-@@ -5164,6 +5164,29 @@ static bool do_lxcapi_clean_container_resource(struct lxc_container *c, pid_t pi
- 
- WRAP_API_1(bool, lxcapi_clean_container_resource, pid_t)
- 
-+/* isulad add clean resources */
-+static bool do_lxcapi_add_terminal_fifo(struct lxc_container *c, const char *in_fifo, const char *out_fifo)
-+{
-+	bool ret = true;
-+
-+	if (!c || !c->lxc_conf || !in_fifo || !out_fifo)
-+		return false;
-+	if (container_mem_lock(c)) {
-+		ERROR("Error getting mem lock");
-+		return false;
-+	}
-+
-+	if (lxc_cmd_set_terminal_fifos(c->name, c->config_path, in_fifo, out_fifo)) {
-+		ERROR("Error set console fifos");
-+		ret = false;
-+	}
-+
-+	container_mem_unlock(c);
-+	return ret;
-+}
-+
-+WRAP_API_2(bool, lxcapi_add_terminal_fifo, const char *, const char *)
-+
- static struct lxc_container *do_lxc_container_new(const char *name, const char *configpath, bool load_config)
- {
- 	struct lxc_container *c;
-@@ -5299,6 +5322,7 @@ static struct lxc_container *do_lxc_container_new(const char *name, const char *
- 	c->set_container_info_file = lxcapi_set_container_info_file;
- 	c->set_start_timeout = lxcapi_set_start_timeout;
- 	c->clean_container_resource = lxcapi_clean_container_resource;
-+	c->add_terminal_fifos = lxcapi_add_terminal_fifo;
- 	/* isulad add end */
- 	return c;
- 
-diff --git a/src/lxc/lxccontainer.h b/src/lxc/lxccontainer.h
-index a00e0ec..c1d83ba 100644
---- a/src/lxc/lxccontainer.h
-+++ b/src/lxc/lxccontainer.h
-@@ -878,6 +878,16 @@ struct lxc_container {
- 	bool (*set_terminal_init_fifos)(struct lxc_container *c, const char *in, const char *out);
- 
- 	/*! isulad add
-+	 * \brief An API call to add the path of terminal fifos
-+	 *
-+	 * \param c Container.
-+	 * \param path Value of the console path..
-+	 *
-+	 * \return \c true on success, else \c false.
-+	 */
-+	bool (*add_terminal_fifos)(struct lxc_container *c, const char *in, const char *out);
-+
-+	/*! isulad add
- 	 * \brief An API call to set the path of info file
- 	 *
- 	 * \param c Container.
-diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
-index ee3aef2..a33830d 100644
---- a/src/lxc/terminal.c
-+++ b/src/lxc/terminal.c
-@@ -1473,3 +1473,50 @@ int lxc_terminal_map_ids(struct lxc_conf *c, struct lxc_terminal *terminal)
- 
- 	return 0;
- }
-+
-+/* isulad: add fifos dynamic*/
-+int lxc_terminal_add_fifos(struct lxc_conf *conf, const char *fifonames)
-+{
-+	int ret = 0;
-+	struct lxc_terminal *terminal = &conf->console;
-+	int fifofd_in = -1;
-+	char *tmp = NULL, *saveptr = NULL, *in = NULL, *out = NULL;
-+
-+	tmp = strdup(fifonames);
-+	if (!tmp) {
-+		ret = -1;
-+		goto free_out;
-+	}
-+
-+	in = strtok_r(tmp, "&&&&", &saveptr);
-+	if (!in) {
-+		ret = -1;
-+		goto free_out;
-+	}
-+	out = strtok_r(NULL, "&&&&",  &saveptr);
-+	if (!out) {
-+		ret = -1;
-+		goto free_out;
-+	}
-+
-+	fifofd_in = lxc_terminal_set_fifo(terminal, in, out);
-+	if (fifofd_in < 0) {
-+		ERROR("Faild to set fifos to console config");
-+		ret = -1;
-+		goto free_out;
-+	}
-+
-+	if (lxc_mainloop_add_handler(terminal->descr, fifofd_in,
-+				     lxc_terminal_io_cb, terminal)) {
-+		ERROR("console fifo not added to mainloop");
-+		lxc_terminal_delete_fifo(fifofd_in, &terminal->fifos);
-+		ret = -1;
-+		goto free_out;
-+	}
-+
-+free_out:
-+	if (tmp)
-+		free(tmp);
-+	return ret;
-+}
-+
-diff --git a/src/lxc/terminal.h b/src/lxc/terminal.h
-index d25da65..d006b80 100644
---- a/src/lxc/terminal.h
-+++ b/src/lxc/terminal.h
-@@ -310,5 +310,6 @@ extern int lxc_terminal_map_ids(struct lxc_conf *c,
- static bool lxc_terminal_is_fifo(int fd, struct lxc_list *list);
- /* isulad: if fd == -1, means delete all the fifos*/
- int lxc_terminal_delete_fifo(int fd, struct lxc_list *list);
-+int lxc_terminal_add_fifos(struct lxc_conf *conf, const char *fifonames);
- 
- #endif /* __LXC_TERMINAL_H */
--- 
-1.8.3.1
-

0045-restore-default-signal-handler.patch

@@ -0,0 +1,120 @@
+From 4a609606e050a9a4693541a965e34d8ac366f153 Mon Sep 17 00:00:00 2001
+From: haozi007 <liuhao27@huawei.com>
+Date: Thu, 16 Apr 2020 15:56:21 +0800
+Subject: [PATCH 45/49] restore default signal handler
+
+Signed-off-by: haozi007 <liuhao27@huawei.com>
+---
+ hooks/Makefile.am   |  3 +++
+ src/lxc/attach.c    | 16 ++++++++++++++++
+ src/lxc/initutils.c |  4 ++++
+ src/lxc/start.c     | 19 +++++++++++++++++++
+ 4 files changed, 42 insertions(+)
+
+diff --git a/hooks/Makefile.am b/hooks/Makefile.am
+index 5ae73d7..ddfd4bc 100644
+--- a/hooks/Makefile.am
++++ b/hooks/Makefile.am
+@@ -10,6 +10,8 @@ hooks_SCRIPTS = \
+ 	squid-deb-proxy-client \
+ 	nvidia
+ 
++
++if !HAVE_ISULAD
+ binhooks_PROGRAMS = \
+ 	unmount-namespace
+ 
+@@ -20,5 +22,6 @@ if IS_BIONIC
+ unmount_namespace_SOURCES += \
+ 	../src/include/lxcmntent.c ../src/include/lxcmntent.h
+ endif
++endif
+ 
+ EXTRA_DIST=$(hooks_SCRIPTS)
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index 734cddd..1dd2b47 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -679,12 +679,28 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 			 init_ctx->lsm_label;
+ #ifdef HAVE_ISULAD
+ 	int msg_fd = -1;
++	sigset_t mask;
+ 
+ 	/*isulad: record errpipe fd*/
+ 	msg_fd = init_ctx->container->lxc_conf->errpipe[1];
+ 	init_ctx->container->lxc_conf->errpipe[1] = -1;
+ 	/*isulad: set system umask */
+ 	umask(init_ctx->container->lxc_conf->umask);
++
++	/*isulad: restore default signal handlers and unblock all signals*/
++	for (int i = 1; i < NSIG; i++)
++		signal(i, SIG_DFL);
++
++	ret = sigfillset(&mask);
++	if (ret < 0) {
++		SYSERROR("Failed to fill signal mask");
++		goto on_error;;
++	}
++	ret = sigprocmask(SIG_UNBLOCK, &mask, NULL);
++	if (ret < 0) {
++		SYSERROR("Failed to set signal mask");
++		goto on_error;
++	}
+ #endif
+ 
+ 	/* A description of the purpose of this functionality is provided in the
+diff --git a/src/lxc/initutils.c b/src/lxc/initutils.c
+index 5549c2e..76f0048 100644
+--- a/src/lxc/initutils.c
++++ b/src/lxc/initutils.c
+@@ -54,12 +54,16 @@ const char *lxc_global_config_value(const char *option_name)
+ 		{ NULL, NULL },
+ 	};
+ 
++#ifdef HAVE_ISULAD
++	static const char *values[sizeof(options) / sizeof(options[0])] = {0};
++#else
+ 	/* placed in the thread local storage pool for non-bionic targets */
+ #ifdef HAVE_TLS
+ 	static thread_local const char *values[sizeof(options) / sizeof(options[0])] = {0};
+ #else
+ 	static const char *values[sizeof(options) / sizeof(options[0])] = {0};
+ #endif
++#endif
+ 
+ 	/* user_config_path is freed as soon as it is used */
+ 	char *user_config_path = NULL;
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 6779cee..5d2faee 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -1290,6 +1290,25 @@ static int do_start(void *data)
+ 
+ 	lxc_sync_fini_parent(handler);
+ 
++#ifdef HAVE_ISULAD
++	sigset_t mask;
++
++	/*isulad: restore default signal handlers and unblock all signals*/
++	for (int i = 1; i < NSIG; i++)
++		signal(i, SIG_DFL);
++
++	ret = sigfillset(&mask);
++	if (ret < 0) {
++		SYSERROR("Failed to fill signal mask");
++		goto out_warn_father;
++	}
++	ret = sigprocmask(SIG_UNBLOCK, &mask, NULL);
++	if (ret < 0) {
++		SYSERROR("Failed to set signal mask");
++		goto out_warn_father;
++	}
++#endif
++
+ 	if (lxc_abstract_unix_recv_fds(data_sock1, &status_fd, 1, NULL, 0) < 0) {
+ 		ERROR("Failed to receive status file descriptor to child process");
+ 		goto out_warn_father;
+-- 
+1.8.3.1
+

0046-Do-not-test-cgroup-writeable.patch

@@ -1,48 +0,0 @@
-From fc6da7b9465295da1dc1d40d835a8c7680207d66 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Thu, 17 Jan 2019 03:49:16 -0500
-Subject: [PATCH 046/140] Do not test cgroup writeable
-
-If we run isulad in docker without cgroup namespace, the base_cgroup will be docker/XXX..,
-mountpoint+base_cgroup may be not exist
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/cgroups/cgfsng.c | 8 ++++++--
- 1 file changed, 6 insertions(+), 2 deletions(-)
-
-diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index 47b12a6..6bfa693 100644
---- a/src/lxc/cgroups/cgfsng.c
-+++ b/src/lxc/cgroups/cgfsng.c
-@@ -581,7 +581,7 @@ static bool cg_legacy_handle_cpuset_hierarchy(struct hierarchy *h, char *cgname)
- 	if (slash)
- 		*slash = '/';
- 
--	ret = mkdir(cgpath, 0755);
-+	ret = mkdir_p(cgpath, 0755);
- 	if (ret < 0) {
- 		if (errno != EEXIST) {
- 			SYSERROR("Failed to create directory \"%s\"", cgpath);
-@@ -2489,13 +2489,17 @@ static bool cg_hybrid_init(struct cgroup_ops *ops)
- 
- 		trim(base_cgroup);
- 		prune_init_scope(base_cgroup);
-+
-+		/* isulad:
-+		 * do not test writeable, if we run isulad in docker without cgroup namespace.
-+		 * the base_cgroup will be docker/XXX.., mountpoint+base_cgroup may be not exist
- 		if (type == CGROUP2_SUPER_MAGIC)
- 			writeable = test_writeable_v2(mountpoint, base_cgroup);
- 		else
- 			writeable = test_writeable_v1(mountpoint, base_cgroup);
- 		if (!writeable)
- 			goto next;
--
-+		*/
- 		if (type == CGROUP2_SUPER_MAGIC) {
- 			char *cgv2_ctrl_path;
- 
--- 
-1.8.3.1
-

0046-add-support-systemd.patch

@@ -1,24 +1,20 @@
-From 2b8bafa86e5086ee9f85dba13a73fb3abdf8656f Mon Sep 17 00:00:00 2001
+From 1922cc534f04a7a006064e7eae2dea44d0000cc3 Mon Sep 17 00:00:00 2001
 From: tanyifeng <tanyifeng1@huawei.com>
 Date: Sat, 20 Apr 2019 22:40:18 +0800
-Subject: [PATCH 086/140] confile: add support systemd
+Subject: [PATCH 46/49] add support systemd
-
-lxc.isulad.systemd=true remount systemd cgroup path to rw
 
 Signed-off-by: zhangsong <zhangsong34@huawei.com>
-Signed-off-by: LiFeng <lifeng68@huawei.com>
 ---
- src/lxc/cgroups/cgfsng.c | 16 ++++++++++++++++
+ src/lxc/cgroups/cgfsng.c | 15 +++++++++++++++
  src/lxc/conf.c           |  1 +
- src/lxc/conf.h           |  1 +
  src/lxc/confile.c        | 30 ++++++++++++++++++++++++++++++
- 4 files changed, 48 insertions(+)
+ 3 files changed, 46 insertions(+)
 
 diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index cc08737..b1f56b0 100644
+index 002f051..4abaa86 100644
 --- a/src/lxc/cgroups/cgfsng.c
 +++ b/src/lxc/cgroups/cgfsng.c
-@@ -1567,6 +1567,7 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
+@@ -2096,6 +2096,7 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
  {
  	int i, ret;
  	char *tmpfspath = NULL;
@@ -26,11 +22,10 @@ index cc08737..b1f56b0 100644
  	bool has_cgns = false, retval = false, wants_force_mount = false;
  	char **merged = NULL;
  
-@@ -1711,10 +1712,25 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
+@@ -2242,10 +2243,24 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
- 			goto on_error;
  		}
  	}
-+
+ 
 +        // isulad: remount /sys/fs/cgroup/systemd to readwrite for system container
 +        if (handler->conf->systemd != NULL && strcmp(handler->conf->systemd, "true") == 0) {
 +                systemdpath = must_make_path(root, "/sys/fs/cgroup/systemd", NULL);
@@ -53,51 +48,39 @@ index cc08737..b1f56b0 100644
  	return retval;
  }
 diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 20eb840..8cdccf1 100644
+index d7a78bd..235965f 100644
 --- a/src/lxc/conf.c
 +++ b/src/lxc/conf.c
-@@ -5320,6 +5320,7 @@ void lxc_conf_free(struct lxc_conf *conf)
+@@ -5283,6 +5283,7 @@ void lxc_conf_free(struct lxc_conf *conf)
- 	if (conf->exit_fd != -1)
+ 	if (conf->exit_fd != -1) {
  		close(conf->exit_fd);
- 	free(conf->errmsg);
+ 	}
 +	free(conf->systemd);
- 	lxc_close_error_pipe(conf->errpipe);
+ 	lxc_clear_init_args(conf);
- 	/* isulad add end */
+ 	lxc_clear_init_groups(conf);
- 	free(conf);
+ 	lxc_clear_populate_devices(conf);
-diff --git a/src/lxc/conf.h b/src/lxc/conf.h
-index 11cf596..fb3c156 100644
---- a/src/lxc/conf.h
-+++ b/src/lxc/conf.h
-@@ -430,6 +430,7 @@ struct lxc_conf {
- 	char *errmsg; /* record error messages */
- 	int errpipe[2];//pipdfd for get error message of child or grandchild process.
- 	mode_t umask;  //umask value
-+	char *systemd; //systemd value
- 	/* isulad add end */
- };
- 
 diff --git a/src/lxc/confile.c b/src/lxc/confile.c
-index 60e6c46..93936cc 100644
+index 8790494..771f635 100644
 --- a/src/lxc/confile.c
 +++ b/src/lxc/confile.c
-@@ -157,6 +157,7 @@ lxc_config_define(init_args);
+@@ -154,6 +154,7 @@ lxc_config_define(populate_device);
- lxc_config_define(init_groups);
- lxc_config_define(populate_device);
  lxc_config_define(umask);
+ lxc_config_define(rootfs_masked_paths);
+ lxc_config_define(rootfs_ro_paths);
 +lxc_config_define(systemd);
- /*isulad add end*/
+ #endif
  
- 
+ /*
-@@ -251,6 +252,7 @@ static struct lxc_config_t config_jump_table[] = {
+@@ -274,6 +275,7 @@ static struct lxc_config_t config_jump_table[] = {
+ 	{ "lxc.isulad.umask",              set_config_umask,                       get_config_umask,                       clr_config_umask,                     },
  	{ "lxc.isulad.rootfs.maskedpaths", set_config_rootfs_masked_paths,         get_config_rootfs_masked_paths,         clr_config_rootfs_masked_paths,       },
  	{ "lxc.isulad.rootfs.ropaths",     set_config_rootfs_ro_paths,             get_config_rootfs_ro_paths,             clr_config_rootfs_ro_paths,           },
- 	{ "lxc.isulad.umask",              set_config_umask,                       get_config_umask,                       clr_config_umask,                     },
 +	{ "lxc.isulad.systemd",            set_config_systemd,                     get_config_systemd,                     clr_config_systemd,                   },
- 	/*isulad add end*/
+ #endif
  };
  
-@@ -2433,6 +2435,18 @@ static int set_config_umask(const char *key, const char *value,
+@@ -6587,4 +6589,32 @@ static inline int clr_config_rootfs_ro_paths(const char *key, struct lxc_conf *c
- 	}
+ 	return lxc_clear_rootfs_ro_paths(c);
  }
  
 +/* isulad: set config for systemd */
@@ -112,13 +95,6 @@ index 60e6c46..93936cc 100644
 +	return 0;
 +}
 +
- struct parse_line_conf {
- 	struct lxc_conf *conf;
- 	bool from_include;
-@@ -3210,6 +3224,13 @@ static int get_config_umask(const char *key, char *retv, int inlen,
- 	return lxc_get_conf_size_t(c, retv, inlen, c->umask);
- }
- 
 +/* isulad add: get systemd value*/
 +static int get_config_systemd(const char *key, char *retv, int inlen,
 +			      struct lxc_conf *c, void *data)
@@ -126,13 +102,6 @@ index 60e6c46..93936cc 100644
 +	return lxc_get_conf_str(retv, inlen, c->systemd);
 +}
 +
- static int get_config_tty_dir(const char *key, char *retv, int inlen,
- 			     struct lxc_conf *c, void *data)
- {
-@@ -4491,6 +4512,15 @@ static inline int clr_config_umask(const char *key, struct lxc_conf *c,
- 	return 0;
- }
- 
 +/* isulad add: clear systemd value */
 +static inline int clr_config_systemd(const char *key, struct lxc_conf *c,
 +				     void *data)
@@ -142,9 +111,7 @@ index 60e6c46..93936cc 100644
 +	return 0;
 +}
 +
- static int get_config_includefiles(const char *key, char *retv, int inlen,
+ #endif
- 				   struct lxc_conf *c, void *data)
- {
 -- 
 1.8.3.1
 

0047-Fix-memory-leak-in-lxc_global_config_value.patch

@@ -1,44 +0,0 @@
-From f238c892c331361985a2d8f0211b4fd78ce99682 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Thu, 17 Jan 2019 07:43:23 -0500
-Subject: [PATCH 047/140] Fix memory leak in lxc_global_config_value
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/cgroups/cgroup.c | 2 ++
- src/lxc/initutils.c      | 5 -----
- 2 files changed, 2 insertions(+), 5 deletions(-)
-
-diff --git a/src/lxc/cgroups/cgroup.c b/src/lxc/cgroups/cgroup.c
-index 720a6c9..8d559be 100644
---- a/src/lxc/cgroups/cgroup.c
-+++ b/src/lxc/cgroups/cgroup.c
-@@ -95,6 +95,8 @@ void cgroup_exit(struct cgroup_ops *ops)
- 	}
- 	free(ops->hierarchies);
- 
-+	free(ops);
-+
- 	return;
- }
- 
-diff --git a/src/lxc/initutils.c b/src/lxc/initutils.c
-index 09b521e..60147a5 100644
---- a/src/lxc/initutils.c
-+++ b/src/lxc/initutils.c
-@@ -74,12 +74,7 @@ const char *lxc_global_config_value(const char *option_name)
- 		{ NULL, NULL },
- 	};
- 
--	/* placed in the thread local storage pool for non-bionic targets */
--#ifdef HAVE_TLS
--	static thread_local const char *values[sizeof(options) / sizeof(options[0])] = {0};
--#else
- 	static const char *values[sizeof(options) / sizeof(options[0])] = {0};
--#endif
- 
- 	/* user_config_path is freed as soon as it is used */
- 	char *user_config_path = NULL;
--- 
-1.8.3.1
-

0047-support-namespaced-kernel-params-can-be-changed-in-s.patch

@@ -1,24 +1,23 @@
-From 140cadc75c14917c9d0aa3c6c65d84fd8b682adb Mon Sep 17 00:00:00 2001
+From 288b6934f79456f056a2043216bbfdde4342b694 Mon Sep 17 00:00:00 2001
 From: liuhao <liuhao27@huawei.com>
 Date: Fri, 26 Apr 2019 07:13:53 +0800
-Subject: [PATCH 091/140] lxc: support namespaced kernel params can be changed
+Subject: [PATCH 47/49] support namespaced kernel params can be changed in
- in system container
+ system container
-
-support namespaced kernel params can be changed in system container
 
 Signed-off-by: yangchenliang <yangchenliang@huawei.com>
 ---
- src/lxc/conf.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ src/lxc/conf.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- 1 file changed, 77 insertions(+)
+ 1 file changed, 71 insertions(+)
 
 diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 14d5d80..0f227aa 100644
+index 235965f..15d8e42 100644
 --- a/src/lxc/conf.c
 +++ b/src/lxc/conf.c
-@@ -1516,6 +1516,66 @@ error:
+@@ -1449,6 +1449,68 @@ error:
  	return false;
  }
  
++#ifdef HAVE_ISULAD
 +static bool remount_readwrite(const char *path)
 +{
 +	int ret, i;
@@ -54,13 +53,13 @@ index 14d5d80..0f227aa 100644
 +	return false;
 +}
 +
-+static int remount_proc_sys_mount_entries(struct lxc_list *mount_list)
++static int remount_proc_sys_mount_entries(struct lxc_list *mount_list, bool lsm_aa_allow_nesting)
 +{
 +	char buf[4096];
 +	FILE *file;
 +	struct mntent mntent;
 +
-+	file = make_anonymous_mount_file(mount_list);
++	file = make_anonymous_mount_file(mount_list, lsm_aa_allow_nesting);
 +	if (!file)
 +		return -1;
 +
@@ -78,41 +77,27 @@ index 14d5d80..0f227aa 100644
 +	fclose(file);
 +	return 0;
 +}
++#endif
 +
  // remount_readonly will bind over the top of an existing path and ensure that it is read-only.
  static bool remount_readonly(const char *path)
  {
-@@ -2699,6 +2759,13 @@ static int mount_file_entries(const struct lxc_conf *conf,
+@@ -4773,6 +4835,15 @@ int lxc_setup(struct lxc_handler *handler)
- 	int ret = -1;
- 
- 	while (getmntent_r(file, &mntent, buf, sizeof(buf))) {
-+		//isulad, system contaienr, skip "proc/sys/xxx" path
-+		if (conf->systemd != NULL && strcmp(conf->systemd, "true") == 0) {
-+			if (strstr(mntent.mnt_dir, "proc/sys") != NULL) {
-+				continue;
-+			}
-+		}
-+
- 		/* Note: Workaround for volume file path with space*/
- 		mntent.mnt_fsname = lxc_string_replace(SPACE_MAGIC_STR, " ", mntent.mnt_fsname);
- 		if(!mntent.mnt_fsname) {
-@@ -4254,6 +4321,16 @@ int lxc_setup(struct lxc_handler *handler)
  		}
  	}
  
 +	//isulad: system container, remount /proc/sys/xxx by mount_list
 +	if (lxc_conf->systemd != NULL && strcmp(lxc_conf->systemd, "true") == 0) {
 +		if (!lxc_list_empty(&lxc_conf->mount_list)) {
-+			if (remount_proc_sys_mount_entries(&lxc_conf->mount_list)) {
++			if (remount_proc_sys_mount_entries(&lxc_conf->mount_list, lxc_conf->lsm_aa_allow_nesting)) {
-+				ERROR("failed to remount /proc/sys");
++				return log_error(-1, "failed to remount /proc/sys");
-+				goto on_error;
 +			}
 +		}
 +	}
 +
- 	if (!lxc_list_empty(&lxc_conf->keepcaps)) {
+ 	// isulad: create link /etc/mtab for /proc/mounts
- 		if (!lxc_list_empty(&lxc_conf->caps)) {
+ 	if (create_mtab_link() != 0) {
- 			ERROR("Container requests lxc.cap.drop and "
+ 		return log_error(-1, "failed to create link /etc/mtab for target /proc/mounts");
 -- 
 1.8.3.1
 

0048-clear-ONLCR-flag-from-master-of-terminal.patch

@@ -1,59 +0,0 @@
-From fc95cef9efded165758645b8f04da7c0ffd6b762 Mon Sep 17 00:00:00 2001
-From: liuhao <liuhao27@huawei.com>
-Date: Thu, 17 Jan 2019 20:46:33 +0800
-Subject: [PATCH 048/140] clear ONLCR flag from master of terminal
-
-clear ONLCR flag from master of terminal
-
-Signed-off-by: liuhao <liuhao27@huawei.com>
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/terminal.c | 25 +++++++++++++++++++++++++
- 1 file changed, 25 insertions(+)
-
-diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
-index a33830d..95140e0 100644
---- a/src/lxc/terminal.c
-+++ b/src/lxc/terminal.c
-@@ -1066,6 +1066,24 @@ static int lxc_terminal_fifo_default(struct lxc_terminal *terminal)
- 	return lxc_terminal_set_fifo(terminal, terminal->init_fifo[0], terminal->init_fifo[1]);
- }
- 
-+/*
-+ * isulad: disable (XSI) Map NL to CR-NL on output. 
-+ * */
-+static int use_unix_newline(int master_fd)
-+{
-+	struct termios oldtios;
-+	int ret;
-+
-+	ret = tcgetattr(master_fd, &oldtios);
-+	if (ret < 0)
-+		return -1;
-+	oldtios.c_oflag &= ~ONLCR;
-+	ret = tcsetattr(master_fd, TCSAFLUSH, &oldtios);
-+	if (ret < 0)
-+		return -1;
-+	return 0;
-+}
-+
- int lxc_terminal_create(struct lxc_terminal *terminal)
- {
- 	int ret;
-@@ -1082,6 +1100,13 @@ int lxc_terminal_create(struct lxc_terminal *terminal)
- 		goto err;
- 	}
- 
-+	/* isulad: clear ONLCR flag */
-+	ret = use_unix_newline(terminal->master);
-+	if (ret < 0) {
-+		SYSERROR("Failed to clear ONLCR flag on terminal master");
-+		goto err;
-+	}
-+
- 	ret = fd_cloexec(terminal->master, true);
- 	if (ret < 0) {
- 		SYSERROR("Failed to set FD_CLOEXEC flag on terminal master");
--- 
-1.8.3.1
-

0048-don-t-use-the-unified-hierarchy-for-the-systemd-cgro.patch

@@ -1,22 +1,18 @@
-From 724ce7ab5bf6dd83ee034d13b2d60515632e1d67 Mon Sep 17 00:00:00 2001
+From 4dfcafd037464a64047b70ffea3b91a70514ae92 Mon Sep 17 00:00:00 2001
 From: liuhao <liuhao27@huawei.com>
 Date: Fri, 10 May 2019 11:05:42 +0800
-Subject: [PATCH 095/140] lxc: don't use the unified hierarchy for the systemd
+Subject: [PATCH 48/49] don't use the unified hierarchy for the systemd cgroup
- cgroup
-
-reason:don't use the unified hierarchy for the systemd cgroup
 
 Signed-off-by: zhangsong <zhangsong34@huawei.com>
-Signed-off-by: LiFeng <lifeng68@huawei.com>
 ---
  src/lxc/cgroups/cgfsng.c | 14 ++++++++++++++
  1 file changed, 14 insertions(+)
 
 diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
-index 2bf142f..eee7ed6 100644
+index 4abaa86..4a0961f 100644
 --- a/src/lxc/cgroups/cgfsng.c
 +++ b/src/lxc/cgroups/cgfsng.c
-@@ -1571,6 +1571,7 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
+@@ -2097,6 +2097,7 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
  	int i, ret;
  	char *tmpfspath = NULL;
  	char *systemdpath = NULL;
@@ -24,7 +20,7 @@ index 2bf142f..eee7ed6 100644
  	bool has_cgns = false, retval = false, wants_force_mount = false;
  	char **merged = NULL;
  
-@@ -1718,6 +1719,16 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
+@@ -2245,6 +2246,16 @@ __cgfsng_ops static bool cgfsng_mount(struct cgroup_ops *ops,
  
          // isulad: remount /sys/fs/cgroup/systemd to readwrite for system container
          if (handler->conf->systemd != NULL && strcmp(handler->conf->systemd, "true") == 0) {
@@ -41,7 +37,7 @@ index 2bf142f..eee7ed6 100644
                  systemdpath = must_make_path(root, "/sys/fs/cgroup/systemd", NULL);
                  ret = mount(systemdpath, systemdpath, "bind",
                              MS_NOSUID|MS_NODEV|MS_NOEXEC|MS_RELATIME|MS_BIND|MS_REMOUNT, NULL);
-@@ -1734,6 +1745,9 @@ on_error:
+@@ -2261,6 +2272,9 @@ on_error:
          if (systemdpath != NULL) {
                  free(systemdpath);
          }

0049-Add-100ms-timeout-for-console-epoll.patch

@@ -1,28 +0,0 @@
-From c144ed16a5a45c8dc2873bbb1016d8bc1fc02a9c Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Sat, 19 Jan 2019 02:05:17 -0500
-Subject: [PATCH 049/140] Add 100ms timeout for console epoll
-
-add 100ms timeout for console epoll to avoid lose console
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/start.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/src/lxc/start.c b/src/lxc/start.c
-index 7bbcb00..daf2af4 100644
---- a/src/lxc/start.c
-+++ b/src/lxc/start.c
-@@ -643,7 +643,7 @@ int lxc_poll(const char *name, struct lxc_handler *handler)
- 		goto out_mainloop_console;
- 
- 	if (has_console)
--		ret = lxc_mainloop(&descr_console, 0);
-+		ret = lxc_mainloop(&descr_console, 100);
- 
- out_mainloop_console:
- 	if (has_console) {
--- 
-1.8.3.1
-

0049-make-dev-bind-mount-from-host-tmpfs-for-system-conta.patch

@@ -0,0 +1,99 @@
+From e1e3ad7bc7cb1a26cca676ff5e4a5ceaf7eedaee Mon Sep 17 00:00:00 2001
+From: tanyifeng <tanyifeng1@huawei.com>
+Date: Wed, 15 May 2019 12:42:08 +0800
+Subject: [PATCH 49/49] make /dev bind mount from host tmpfs for system
+ container
+
+Signed-off-by: zhangsong <zhangsong34@huawei.com>
+---
+ src/lxc/conf.c | 33 +++++++++++++++++++++++++++++++--
+ 1 file changed, 31 insertions(+), 2 deletions(-)
+
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index 15d8e42..4d8fa2a 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -1074,8 +1074,13 @@ on_error:
+ /* Just create a path for /dev under $lxcpath/$name and in rootfs If we hit an
+  * error, log it but don't fail yet.
+  */
++#ifdef HAVE_ISULAD
++static int mount_autodev(const char *name, const struct lxc_rootfs *rootfs,
++			 int autodevtmpfssize, const char *lxcpath, char *systemd)
++#else
+ static int mount_autodev(const char *name, const struct lxc_rootfs *rootfs,
+ 			 int autodevtmpfssize, const char *lxcpath)
++#endif
+ {
+ 	__do_free char *path = NULL;
+ 	int ret;
+@@ -1088,6 +1093,7 @@ static int mount_autodev(const char *name, const struct lxc_rootfs *rootfs,
+ 	/* $(rootfs->mount) + "/dev/pts" + '\0' */
+ 	clen = (rootfs->path ? strlen(rootfs->mount) : 0) + 9;
+ 	path = must_realloc(NULL, clen);
++
+ 	sprintf(mount_options, "size=%d,mode=755", (autodevtmpfssize != 0) ? autodevtmpfssize : 500000);
+ 	DEBUG("Using mount options: %s", mount_options);
+ 
+@@ -1103,6 +1109,23 @@ static int mount_autodev(const char *name, const struct lxc_rootfs *rootfs,
+ 		goto reset_umask;
+ 	}
+ 
++#ifdef HAVE_ISULAD
++        if (systemd != NULL && !strcmp(systemd, "true")) {
++                ret = mount(path, path, "", MS_BIND, NULL);
++                if (ret < 0) {
++		        SYSERROR("Failed to bind mount path \"%s\"", path);
++		        goto reset_umask;
++                }
++        } else {
++	        ret = safe_mount("none", path, "tmpfs", 0, mount_options,
++			 rootfs->path ? rootfs->mount : NULL);
++	        if (ret < 0) {
++		        SYSERROR("Failed to mount tmpfs on \"%s\"", path);
++		        goto reset_umask;
++	        }
++	        TRACE("Mounted tmpfs on \"%s\"", path);
++        }
++#else
+ 	ret = safe_mount("none", path, "tmpfs", 0, mount_options,
+ 			 rootfs->path ? rootfs->mount : NULL );
+ 	if (ret < 0) {
+@@ -1110,6 +1133,7 @@ static int mount_autodev(const char *name, const struct lxc_rootfs *rootfs,
+ 		goto reset_umask;
+ 	}
+ 	TRACE("Mounted tmpfs on \"%s\"", path);
++#endif
+ 
+ 	ret = snprintf(path, clen, "%s/dev/pts", rootfs->path ? rootfs->mount : "");
+ 	if (ret < 0 || (size_t)ret >= clen) {
+@@ -4674,7 +4698,11 @@ int lxc_setup(struct lxc_handler *handler)
+ 	}
+ 
+ 	if (lxc_conf->autodev > 0) {
++#ifdef HAVE_ISULAD
++		ret = mount_autodev(name, &lxc_conf->rootfs, lxc_conf->autodevtmpfssize, lxcpath, lxc_conf->systemd);
++#else
+ 		ret = mount_autodev(name, &lxc_conf->rootfs, lxc_conf->autodevtmpfssize, lxcpath);
++#endif
+ 		if (ret < 0)
+ 			return log_error(-1, "Failed to mount \"/dev\"");
+ 	}
+@@ -4838,11 +4866,12 @@ int lxc_setup(struct lxc_handler *handler)
+ 	//isulad: system container, remount /proc/sys/xxx by mount_list
+ 	if (lxc_conf->systemd != NULL && strcmp(lxc_conf->systemd, "true") == 0) {
+ 		if (!lxc_list_empty(&lxc_conf->mount_list)) {
+-			if (remount_proc_sys_mount_entries(&lxc_conf->mount_list, lxc_conf->lsm_aa_allow_nesting)) {
++			if (remount_proc_sys_mount_entries(&lxc_conf->mount_list,
++							   lxc_conf->lsm_aa_allow_nesting)) {
+ 				return log_error(-1, "failed to remount /proc/sys");
+ 			}
+ 		}
+-	}
++        }
+ 
+ 	// isulad: create link /etc/mtab for /proc/mounts
+ 	if (create_mtab_link() != 0) {
+-- 
+1.8.3.1
+

0050-clean-add-init-fd-in-lxc_init_clean_handler.patch

@@ -0,0 +1,27 @@
+From d15c6e2871202d8d8b928a4b4e11537997c24fc1 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Thu, 16 Apr 2020 21:39:19 +0800
+Subject: [PATCH] clean: add init fd in lxc_init_clean_handler
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/start.c | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 5d2faee..938ca8e 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -2913,7 +2913,9 @@ static struct lxc_handler *lxc_init_clean_handler(char *name, char *lxcpath, str
+ 	handler->lxcpath = lxcpath;
+ 	handler->pinfd = -1;
+ 	handler->sigfd = -EBADF;
++	handler->pidfd = -EBADF;
+ 	handler->init_died = false;
++	handler->monitor_status_fd = -EBADF;
+ 	handler->pid = pid;
+ 	handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1;
+ 	if (handler->conf->reboot == REBOOT_NONE)
+-- 
+1.8.3.1
+

0050-seccomp-add-rules-for-specified-architecture-only.patch

@@ -1,408 +0,0 @@
-From 15d3b1e6c5e5ac896b15d04d486ba0c04ca39336 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Fri, 18 Jan 2019 02:11:11 -0500
-Subject: [PATCH 050/140] seccomp: add rules for specified architecture only
-
-LXC MR: https://github.com/lxc/lxc/pull/2786
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/seccomp.c | 234 ++++++++++++++++++++++++++++++------------------------
- 1 file changed, 132 insertions(+), 102 deletions(-)
-
-diff --git a/src/lxc/seccomp.c b/src/lxc/seccomp.c
-index 27bdc22..4a9143b 100644
---- a/src/lxc/seccomp.c
-+++ b/src/lxc/seccomp.c
-@@ -291,7 +291,7 @@ on_error:
- #endif
- 
- #if HAVE_DECL_SECCOMP_SYSCALL_RESOLVE_NAME_ARCH
--enum lxc_hostarch_t {
-+enum lxc_arch_t {
- 	lxc_seccomp_arch_all = 0,
- 	lxc_seccomp_arch_native,
- 	lxc_seccomp_arch_i386,
-@@ -345,8 +345,8 @@ int get_hostarch(void)
- 	return lxc_seccomp_arch_unknown;
- }
- 
--scmp_filter_ctx get_new_ctx(enum lxc_hostarch_t n_arch,
--			    uint32_t default_policy_action, bool *needs_merge)
-+scmp_filter_ctx get_new_ctx(enum lxc_arch_t n_arch,
-+			    uint32_t default_policy_action)
- {
- 	int ret;
- 	uint32_t arch;
-@@ -464,10 +464,7 @@ scmp_filter_ctx get_new_ctx(enum lxc_hostarch_t n_arch,
- 			return NULL;
- 		}
- 		TRACE("Removed native arch from main seccomp context");
--
--		*needs_merge = true;
- 	} else {
--		*needs_merge = false;
- 		TRACE("Arch %d already present in main seccomp context", (int)n_arch);
- 	}
- 
-@@ -550,6 +547,27 @@ bool do_resolve_add_rule(uint32_t arch, char *line, scmp_filter_ctx ctx,
- 	return true;
- }
- 
-+#define SCMP_ARCH_INDEX_MAX 3
-+
-+struct scmp_ctx_info {
-+	uint32_t architectures[SCMP_ARCH_INDEX_MAX];
-+	enum lxc_arch_t lxc_arch[SCMP_ARCH_INDEX_MAX];
-+	scmp_filter_ctx contexts[SCMP_ARCH_INDEX_MAX];
-+	bool needs_merge[SCMP_ARCH_INDEX_MAX];
-+};
-+
-+static int get_arch_index(enum lxc_arch_t arch, struct scmp_ctx_info *ctx)
-+{
-+	int i;
-+
-+	for (i = 0; i < SCMP_ARCH_INDEX_MAX; i++) {
-+		if (ctx->lxc_arch[i] == arch)
-+			return i;
-+	}
-+
-+	return -1;
-+}
-+
- /*
-  * v2 consists of
-  * [x86]
-@@ -568,15 +586,11 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- {
- 	int ret;
- 	char *p;
--	enum lxc_hostarch_t cur_rule_arch, native_arch;
-+	enum lxc_arch_t cur_rule_arch, native_arch;
- 	bool blacklist = false;
- 	uint32_t default_policy_action = -1, default_rule_action = -1;
- 	struct seccomp_v2_rule rule;
--	struct scmp_ctx_info {
--		uint32_t architectures[3];
--		scmp_filter_ctx contexts[3];
--		bool needs_merge[3];
--	} ctx;
-+	struct scmp_ctx_info ctx;
- 
- 	if (strncmp(line, "blacklist", 9) == 0)
- 		blacklist = true;
-@@ -617,23 +631,23 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 		cur_rule_arch = lxc_seccomp_arch_all;
- 
- 		ctx.architectures[0] = SCMP_ARCH_X86;
-+		ctx.lxc_arch[0] = lxc_seccomp_arch_i386;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_i386,
--					      default_policy_action,
--					      &ctx.needs_merge[0]);
-+					      default_policy_action);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
- 		ctx.architectures[1] = SCMP_ARCH_X32;
-+		ctx.lxc_arch[1] = lxc_seccomp_arch_x32;
- 		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_x32,
--					      default_policy_action,
--					      &ctx.needs_merge[1]);
-+					      default_policy_action);
- 		if (!ctx.contexts[1])
- 			goto bad;
- 
- 		ctx.architectures[2] = SCMP_ARCH_X86_64;
-+		ctx.lxc_arch[2] = lxc_seccomp_arch_amd64;
- 		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_amd64,
--					      default_policy_action,
--					      &ctx.needs_merge[2]);
-+					      default_policy_action);
- 		if (!ctx.contexts[2])
- 			goto bad;
- #ifdef SCMP_ARCH_PPC
-@@ -641,17 +655,17 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 		cur_rule_arch = lxc_seccomp_arch_all;
- 
- 		ctx.architectures[0] = SCMP_ARCH_PPC;
-+		ctx.lxc_arch[0] = lxc_seccomp_arch_ppc;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_ppc,
--					      default_policy_action,
--					      &ctx.needs_merge[0]);
-+					      default_policy_action);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
--		ctx.architectures[2] = SCMP_ARCH_PPC64;
--		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_ppc64,
--					      default_policy_action,
--					      &ctx.needs_merge[2]);
--		if (!ctx.contexts[2])
-+		ctx.architectures[1] = SCMP_ARCH_PPC64;
-+		ctx.lxc_arch[1] = lxc_seccomp_arch_ppc64;
-+		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_ppc64,
-+					      default_policy_action);
-+		if (!ctx.contexts[1])
- 			goto bad;
- #endif
- #ifdef SCMP_ARCH_ARM
-@@ -659,18 +673,18 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 		cur_rule_arch = lxc_seccomp_arch_all;
- 
- 		ctx.architectures[0] = SCMP_ARCH_ARM;
-+		ctx.lxc_arch[0] = lxc_seccomp_arch_arm;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_arm,
--					      default_policy_action,
--					      &ctx.needs_merge[0]);
-+					      default_policy_action);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
- #ifdef SCMP_ARCH_AARCH64
--		ctx.architectures[2] = SCMP_ARCH_AARCH64;
--		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_arm64,
--					      default_policy_action,
--					      &ctx.needs_merge[2]);
--		if (!ctx.contexts[2])
-+		ctx.architectures[1] = SCMP_ARCH_AARCH64;
-+		ctx.lxc_arch[1] = lxc_seccomp_arch_arm64;
-+		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_arm64,
-+					      default_policy_action);
-+		if (!ctx.contexts[1])
- 			goto bad;
- #endif
- #endif
-@@ -679,46 +693,46 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 		cur_rule_arch = lxc_seccomp_arch_all;
- 
- 		ctx.architectures[0] = SCMP_ARCH_MIPS;
-+		ctx.lxc_arch[0] = lxc_seccomp_arch_mips;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_mips,
--					      default_policy_action,
--					      &ctx.needs_merge[0]);
-+					      default_policy_action);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
- 		ctx.architectures[1] = SCMP_ARCH_MIPS64N32;
-+		ctx.lxc_arch[1] = lxc_seccomp_arch_mips64n32;
- 		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_mips64n32,
--					      default_policy_action,
--					      &ctx.needs_merge[1]);
-+					      default_policy_action);
- 		if (!ctx.contexts[1])
- 			goto bad;
- 
- 		ctx.architectures[2] = SCMP_ARCH_MIPS64;
-+		ctx.lxc_arch[2] = lxc_seccomp_arch_mips64;
- 		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_mips64,
--					      default_policy_action,
--					      &ctx.needs_merge[2]);
-+					      default_policy_action);
- 		if (!ctx.contexts[2])
- 			goto bad;
- 	} else if (native_arch == lxc_seccomp_arch_mipsel64) {
- 		cur_rule_arch = lxc_seccomp_arch_all;
- 
- 		ctx.architectures[0] = SCMP_ARCH_MIPSEL;
-+		ctx.lxc_arch[0] = lxc_seccomp_arch_mipsel;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_mipsel,
--					      default_policy_action,
--					      &ctx.needs_merge[0]);
-+					      default_policy_action);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
- 		ctx.architectures[1] = SCMP_ARCH_MIPSEL64N32;
-+		ctx.lxc_arch[1] = lxc_seccomp_arch_mipsel64n32;
- 		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_mipsel64n32,
--					      default_policy_action,
--					      &ctx.needs_merge[1]);
-+					      default_policy_action);
- 		if (!ctx.contexts[1])
- 			goto bad;
- 
- 		ctx.architectures[2] = SCMP_ARCH_MIPSEL64;
-+		ctx.lxc_arch[2] = lxc_seccomp_arch_mipsel64;
- 		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_mipsel64,
--					      default_policy_action,
--					      &ctx.needs_merge[2]);
-+					      default_policy_action);
- 		if (!ctx.contexts[2])
- 			goto bad;
- #endif
-@@ -928,97 +942,113 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 			goto bad_rule;
- 		}
- 
--		if (!do_resolve_add_rule(SCMP_ARCH_NATIVE, line,
--					 conf->seccomp_ctx, &rule))
--			goto bad_rule;
-+		if (cur_rule_arch == native_arch) {
-+			if (!do_resolve_add_rule(SCMP_ARCH_NATIVE, line,
-+						 conf->seccomp_ctx, &rule))
-+				goto bad_rule;
-+
-+			INFO("Added native rule for arch %d for %s action %d(%s)",
-+			     SCMP_ARCH_NATIVE, line, rule.action,
-+			     get_action_name(rule.action));
-+		} else if (cur_rule_arch != lxc_seccomp_arch_all) {
-+			int arch_index = get_arch_index(cur_rule_arch, &ctx);
-+			if (arch_index < 0)
-+				goto bad_arch;
- 
--		INFO("Added native rule for arch %d for %s action %d(%s)",
--		     SCMP_ARCH_NATIVE, line, rule.action,
--		     get_action_name(rule.action));
-+			if (!do_resolve_add_rule(ctx.architectures[arch_index], line,
-+						 ctx.contexts[arch_index], &rule))
-+				goto bad_rule;
- 
--		if (cur_rule_arch == lxc_seccomp_arch_all) {
-+			INFO("Added compat rule for arch %d for %s action %d(%s)",
-+			     ctx.architectures[arch_index], line, rule.action,
-+			     get_action_name(rule.action));
-+			ctx.needs_merge[arch_index] = true;
-+		} else {
- 			if (ctx.architectures[0] != SCMP_ARCH_NATIVE) {
- 				if (!do_resolve_add_rule(ctx.architectures[0], line,
--							ctx.contexts[0], &rule))
-+							 ctx.contexts[0], &rule))
- 					goto bad_rule;
- 
--					INFO("Added compat rule for arch %d for %s action %d(%s)",
--						ctx.architectures[0], line, rule.action,
--						get_action_name(rule.action));
-+				INFO("Added compat rule for arch %d for %s action %d(%s)",
-+				     ctx.architectures[0], line, rule.action,
-+				     get_action_name(rule.action));
-+				ctx.needs_merge[0] = true;
- 			}
- 
- 			if (ctx.architectures[1] != SCMP_ARCH_NATIVE) {
- 				if (!do_resolve_add_rule(ctx.architectures[1], line,
--							ctx.contexts[1], &rule))
-+							 ctx.contexts[1], &rule))
- 					goto bad_rule;
- 
- 				INFO("Added compat rule for arch %d for %s action %d(%s)",
--					ctx.architectures[1], line, rule.action,
--					get_action_name(rule.action));
-+				     ctx.architectures[1], line, rule.action,
-+				     get_action_name(rule.action));
-+				ctx.needs_merge[1] = true;
- 			}
- 
- 			if (ctx.architectures[2] != SCMP_ARCH_NATIVE) {
- 				if (!do_resolve_add_rule(ctx.architectures[2], line,
--						ctx.contexts[2], &rule))
-+							ctx.contexts[2], &rule))
- 					goto bad_rule;
- 
- 				INFO("Added native rule for arch %d for %s action %d(%s)",
--					ctx.architectures[2], line, rule.action,
--					get_action_name(rule.action));
-+				     ctx.architectures[2], line, rule.action,
-+				     get_action_name(rule.action));
-+				ctx.needs_merge[2] = true;
- 			}
- 		}
--	}
- 
--	if (cur_rule_arch == lxc_seccomp_arch_all) {
--		INFO("Merging compat seccomp contexts into main context");
--		if (ctx.contexts[0]) {
--			if (ctx.needs_merge[0]) {
--				ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[0]);
--				if (ret < 0) {
--					ERROR("Failed to merge first compat seccomp "
--				      "context into main context");
--					goto bad;
--				}
-+	}
- 
--				TRACE("Merged first compat seccomp context into main context");
--			} else {
--				seccomp_release(ctx.contexts[0]);
--				ctx.contexts[0] = NULL;
-+	INFO("Merging compat seccomp contexts into main context");
-+	if (ctx.contexts[0]) {
-+		if (ctx.needs_merge[0]) {
-+			ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[0]);
-+			if (ret < 0) {
-+				ERROR("%s - Failed to merge first compat seccomp "
-+				      "context into main context", strerror(-ret));
-+				goto bad;
- 			}
--		}
- 
--		if (ctx.contexts[1]) {
--			if (ctx.needs_merge[1]) {
--				ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[1]);
--				if (ret < 0) {
--					ERROR("Failed to merge first compat seccomp "
--					      "context into main context");
--					goto bad;
--				}
-+			TRACE("Merged first compat seccomp context into main context");
-+		} else {
-+			seccomp_release(ctx.contexts[0]);
-+			ctx.contexts[0] = NULL;
-+		}
-+	}
- 
--				TRACE("Merged second compat seccomp context into main context");
--			} else {
--				seccomp_release(ctx.contexts[1]);
--				ctx.contexts[1] = NULL;
-+	if (ctx.contexts[1]) {
-+		if (ctx.needs_merge[1]) {
-+			ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[1]);
-+			if (ret < 0) {
-+				ERROR("%s - Failed to merge second compat seccomp "
-+				      "context into main context", strerror(-ret));
-+				goto bad;
- 			}
--		}
- 
--		if (ctx.contexts[2]) {
--			if (ctx.needs_merge[2]) {
--				ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[2]);
--				if (ret < 0) {
--					ERROR("Failed to merge third compat seccomp "
--					      "context into main context");
--					goto bad;
--				}
-+			TRACE("Merged second compat seccomp context into main context");
-+		} else {
-+			seccomp_release(ctx.contexts[1]);
-+			ctx.contexts[1] = NULL;
-+		}
-+	}
- 
--				TRACE("Merged third compat seccomp context into main context");
--			} else {
--				seccomp_release(ctx.contexts[2]);
--				ctx.contexts[2] = NULL;
-+	if (ctx.contexts[2]) {
-+		if (ctx.needs_merge[2]) {
-+			ret = seccomp_merge(conf->seccomp_ctx, ctx.contexts[2]);
-+			if (ret < 0) {
-+				ERROR("%s - Failed to merge third compat seccomp "
-+				      "context into main context", strerror(-ret));
-+				goto bad;
- 			}
-+
-+			TRACE("Merged third compat seccomp context into main context");
-+		} else {
-+			seccomp_release(ctx.contexts[2]);
-+			ctx.contexts[2] = NULL;
- 		}
- 	}
-+
- 	free(line);
- 	return 0;
- 
--- 
-1.8.3.1
-

0051-if-ocihook-is-empty.patch

@@ -1,34 +0,0 @@
-From 629036e2732ed24c776b422271cef51cf8458f18 Mon Sep 17 00:00:00 2001
-From: liuhao <liuhao27@huawei.com>
-Date: Tue, 22 Jan 2019 11:25:45 +0800
-Subject: [PATCH 051/140] if ocihook is empty
-
-return success
-
-Signed-off-by: liuhao <liuhao27@huawei.com>
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c | 5 ++++-
- 1 file changed, 4 insertions(+), 1 deletion(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 48e31af..6ea8f9c 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -4782,9 +4782,12 @@ static int run_oci_hooks(const char *name, const char *lxcpath, struct lxc_conf
- 	int ret = 0;
- 	char *rootpath;
- 
--	if (!lc || !lc->ocihooks) {
-+	if (!lc) {
- 		return -1;
- 	}
-+	if (!lc->ocihooks) {
-+		return 0;
-+	}
- 
- 	rootpath = get_root_path(lc->rootfs.path, lc->rootfs.bdev_type);
- 	if (!rootpath) {
--- 
-1.8.3.1
-

0051-init-pids-add-init-fd-in-lxc_init_pids_handler.patch

@@ -0,0 +1,26 @@
+From 3931e7ef88aa2a8a67ca967ad1fe359df736ac86 Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Thu, 16 Apr 2020 22:15:30 +0800
+Subject: [PATCH] init pids: add init fd in lxc_init_pids_handler
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/start.c | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index 938ca8e..e099285 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -2967,6 +2967,8 @@ static struct lxc_handler *lxc_init_pids_handler(char *name, char *lxcpath, stru
+ 	handler->sigfd = -EBADF;
+ 	handler->init_died = false;
+ 	handler->state_socket_pair[0] = handler->state_socket_pair[1] = -1;
++	handler->monitor_status_fd = -EBADF;
++	handler->pidfd = -EBADF;
+ 	if (handler->conf->reboot == REBOOT_NONE)
+ 		lxc_list_init(&handler->conf->state_clients);
+ 
+-- 
+1.8.3.1
+

0052-Fix-seccomp-fail-when-all-specified-in-config.patch

@@ -1,192 +0,0 @@
-From 2fbc0027a81a8ec79008afeb5298f5ec99203ea4 Mon Sep 17 00:00:00 2001
-From: LiFeng <lifeng68@huawei.com>
-Date: Mon, 21 Jan 2019 23:28:43 -0500
-Subject: [PATCH 052/140] Fix seccomp fail when [all] specified in config
-
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/seccomp.c | 56 +++++++++++++++++++++++++++----------------------------
- 1 file changed, 28 insertions(+), 28 deletions(-)
-
-diff --git a/src/lxc/seccomp.c b/src/lxc/seccomp.c
-index 4a9143b..3218a60 100644
---- a/src/lxc/seccomp.c
-+++ b/src/lxc/seccomp.c
-@@ -346,7 +346,7 @@ int get_hostarch(void)
- }
- 
- scmp_filter_ctx get_new_ctx(enum lxc_arch_t n_arch,
--			    uint32_t default_policy_action)
-+			    uint32_t default_policy_action, uint32_t *architectures)
- {
- 	int ret;
- 	uint32_t arch;
-@@ -464,8 +464,10 @@ scmp_filter_ctx get_new_ctx(enum lxc_arch_t n_arch,
- 			return NULL;
- 		}
- 		TRACE("Removed native arch from main seccomp context");
-+		*architectures = arch;
- 	} else {
- 		TRACE("Arch %d already present in main seccomp context", (int)n_arch);
-+		*architectures = SCMP_ARCH_NATIVE;
- 	}
- 
- 	return ctx;
-@@ -630,41 +632,36 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 	if (native_arch == lxc_seccomp_arch_amd64) {
- 		cur_rule_arch = lxc_seccomp_arch_all;
- 
--		ctx.architectures[0] = SCMP_ARCH_X86;
- 		ctx.lxc_arch[0] = lxc_seccomp_arch_i386;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_i386,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[0]);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
--		ctx.architectures[1] = SCMP_ARCH_X32;
- 		ctx.lxc_arch[1] = lxc_seccomp_arch_x32;
- 		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_x32,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[1]);
- 		if (!ctx.contexts[1])
- 			goto bad;
- 
--		ctx.architectures[2] = SCMP_ARCH_X86_64;
- 		ctx.lxc_arch[2] = lxc_seccomp_arch_amd64;
- 		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_amd64,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[2]);
- 		if (!ctx.contexts[2])
- 			goto bad;
- #ifdef SCMP_ARCH_PPC
- 	} else if (native_arch == lxc_seccomp_arch_ppc64) {
- 		cur_rule_arch = lxc_seccomp_arch_all;
- 
--		ctx.architectures[0] = SCMP_ARCH_PPC;
- 		ctx.lxc_arch[0] = lxc_seccomp_arch_ppc;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_ppc,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[0]);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
--		ctx.architectures[1] = SCMP_ARCH_PPC64;
- 		ctx.lxc_arch[1] = lxc_seccomp_arch_ppc64;
- 		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_ppc64,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[1]);
- 		if (!ctx.contexts[1])
- 			goto bad;
- #endif
-@@ -672,18 +669,16 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 	} else if (native_arch == lxc_seccomp_arch_arm64) {
- 		cur_rule_arch = lxc_seccomp_arch_all;
- 
--		ctx.architectures[0] = SCMP_ARCH_ARM;
- 		ctx.lxc_arch[0] = lxc_seccomp_arch_arm;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_arm,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[0]);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
- #ifdef SCMP_ARCH_AARCH64
--		ctx.architectures[1] = SCMP_ARCH_AARCH64;
- 		ctx.lxc_arch[1] = lxc_seccomp_arch_arm64;
- 		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_arm64,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[1]);
- 		if (!ctx.contexts[1])
- 			goto bad;
- #endif
-@@ -692,47 +687,41 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 	} else if (native_arch == lxc_seccomp_arch_mips64) {
- 		cur_rule_arch = lxc_seccomp_arch_all;
- 
--		ctx.architectures[0] = SCMP_ARCH_MIPS;
- 		ctx.lxc_arch[0] = lxc_seccomp_arch_mips;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_mips,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[0]);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
--		ctx.architectures[1] = SCMP_ARCH_MIPS64N32;
- 		ctx.lxc_arch[1] = lxc_seccomp_arch_mips64n32;
- 		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_mips64n32,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[1]);
- 		if (!ctx.contexts[1])
- 			goto bad;
- 
--		ctx.architectures[2] = SCMP_ARCH_MIPS64;
- 		ctx.lxc_arch[2] = lxc_seccomp_arch_mips64;
- 		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_mips64,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[2]);
- 		if (!ctx.contexts[2])
- 			goto bad;
- 	} else if (native_arch == lxc_seccomp_arch_mipsel64) {
- 		cur_rule_arch = lxc_seccomp_arch_all;
--
--		ctx.architectures[0] = SCMP_ARCH_MIPSEL;
-+;
- 		ctx.lxc_arch[0] = lxc_seccomp_arch_mipsel;
- 		ctx.contexts[0] = get_new_ctx(lxc_seccomp_arch_mipsel,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[0]);
- 		if (!ctx.contexts[0])
- 			goto bad;
- 
--		ctx.architectures[1] = SCMP_ARCH_MIPSEL64N32;
- 		ctx.lxc_arch[1] = lxc_seccomp_arch_mipsel64n32;
- 		ctx.contexts[1] = get_new_ctx(lxc_seccomp_arch_mipsel64n32,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[1]);
- 		if (!ctx.contexts[1])
- 			goto bad;
- 
--		ctx.architectures[2] = SCMP_ARCH_MIPSEL64;
- 		ctx.lxc_arch[2] = lxc_seccomp_arch_mipsel64;
- 		ctx.contexts[2] = get_new_ctx(lxc_seccomp_arch_mipsel64,
--					      default_policy_action);
-+					      default_policy_action, &ctx.architectures[2]);
- 		if (!ctx.contexts[2])
- 			goto bad;
- #endif
-@@ -943,6 +932,7 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 		}
- 
- 		if (cur_rule_arch == native_arch) {
-+			/* add for native arch */
- 			if (!do_resolve_add_rule(SCMP_ARCH_NATIVE, line,
- 						 conf->seccomp_ctx, &rule))
- 				goto bad_rule;
-@@ -951,6 +941,7 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 			     SCMP_ARCH_NATIVE, line, rule.action,
- 			     get_action_name(rule.action));
- 		} else if (cur_rule_arch != lxc_seccomp_arch_all) {
-+			/* add for compat specified arch */
- 			int arch_index = get_arch_index(cur_rule_arch, &ctx);
- 			if (arch_index < 0)
- 				goto bad_arch;
-@@ -964,6 +955,15 @@ static int parse_config_v2(FILE *f, char *line, size_t *line_bufsz, struct lxc_c
- 			     get_action_name(rule.action));
- 			ctx.needs_merge[arch_index] = true;
- 		} else {
-+			/* add for all compat archs */
-+			if (!do_resolve_add_rule(SCMP_ARCH_NATIVE, line,
-+						 conf->seccomp_ctx, &rule))
-+				goto bad_rule;
-+
-+			INFO("Added native rule for arch %d for %s action %d(%s)",
-+			     SCMP_ARCH_NATIVE, line, rule.action,
-+			     get_action_name(rule.action));
-+
- 			if (ctx.architectures[0] != SCMP_ARCH_NATIVE) {
- 				if (!do_resolve_add_rule(ctx.architectures[0], line,
- 							 ctx.contexts[0], &rule))
--- 
-1.8.3.1
-

0052-setupdev-add-judge-whether-have-mount-dev-entry.patch

@@ -0,0 +1,132 @@
+From 68d3c92b40e049a257bf86dbb29fb274a5f1125e Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Sat, 18 Apr 2020 18:13:16 +0800
+Subject: [PATCH] setupdev: add judge whether have mount /dev entry
+
+reason: If user specify the Destination "/dev".
+        1.Should not populate devices
+        2.Should not setup devpts
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/conf.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---
+ 1 file changed, 59 insertions(+), 3 deletions(-)
+
+diff --git a/src/lxc/conf.c b/src/lxc/conf.c
+index 4d8fa2a..e8568d8 100644
+--- a/src/lxc/conf.c
++++ b/src/lxc/conf.c
+@@ -1942,6 +1942,9 @@ static int lxc_setup_dev_console(const struct lxc_rootfs *rootfs,
+ 	if (ret < 0 && errno != EEXIST)
+ 		return log_error_errno(-errno, errno, "Failed to create console");
+ 
++#ifdef HAVE_ISULAD
++	if (console->slave > 0) {
++#endif
+ 	ret = fchmod(console->slave, S_IXUSR | S_IXGRP);
+ 	if (ret < 0)
+ 		return log_error_errno(-errno, errno, "Failed to set mode \"0%o\" to \"%s\"", S_IXUSR | S_IXGRP, console->name);
+@@ -1949,7 +1952,9 @@ static int lxc_setup_dev_console(const struct lxc_rootfs *rootfs,
+ 	ret = safe_mount(console->name, path, "none", MS_BIND, 0, rootfs_path);
+ 	if (ret < 0)
+ 		return log_error_errno(-1, errno, "Failed to mount \"%s\" on \"%s\"", console->name, path);
+-
++#ifdef HAVE_ISULAD
++	}
++#endif
+ 	DEBUG("Mounted pts device \"%s\" onto \"%s\"", console->name, path);
+ 	return 0;
+ }
+@@ -2913,6 +2918,51 @@ static int setup_mount_entries(const struct lxc_conf *conf,
+ 	return mount_file_entries(conf, rootfs, f, lxc_name, lxc_path);
+ }
+ 
++#ifdef HAVE_ISULAD
++static bool have_dev_bind_mount_entry(FILE *file)
++{
++	bool have_bind_dev = false;
++	char buf[PATH_MAX];
++	struct mntent mntent;
++
++	while (getmntent_r(file, &mntent, buf, sizeof(buf))) {
++		mntent.mnt_dir = lxc_string_replace(SPACE_MAGIC_STR, " ", mntent.mnt_dir);
++		if(!mntent.mnt_dir) {
++			SYSERROR("memory allocation error");
++			continue;
++		}
++
++		if (strcmp(mntent.mnt_dir, "dev") == 0 && strcmp(mntent.mnt_type, "bind") == 0) {
++			have_bind_dev = true;
++		}
++
++		free(mntent.mnt_dir);
++		mntent.mnt_dir = NULL;
++
++		if (have_bind_dev)
++			return true;
++	}
++
++	return false;
++}
++
++// returns true if /dev needs to be set up.
++static bool need_setup_dev(const struct lxc_conf *conf, struct lxc_list *mount)
++{
++	__do_fclose FILE *f = NULL;
++
++	f = make_anonymous_mount_file(mount, conf->lsm_aa_allow_nesting);
++	if (!f)
++		return true;
++
++	if (have_dev_bind_mount_entry(f)) {
++		return false;
++	} else {
++		return true;
++	}
++}
++#endif
++
+ static int parse_cap(const char *cap)
+ {
+ 	size_t i;
+@@ -4655,6 +4705,9 @@ int lxc_setup(struct lxc_handler *handler)
+ 	const char *lxcpath = handler->lxcpath, *name = handler->name;
+ 	struct lxc_conf *lxc_conf = handler->conf;
+ 	char *keyring_context = NULL;
++#ifdef HAVE_ISULAD
++	bool setup_dev = true;
++#endif
+ 
+ 	ret = lxc_setup_rootfs_prepare_root(lxc_conf, name, lxcpath);
+ #ifdef HAVE_ISULAD
+@@ -4723,6 +4776,9 @@ int lxc_setup(struct lxc_handler *handler)
+ 					  &lxc_conf->mount_list, name, lxcpath);
+ 		if (ret < 0)
+ 			return log_error(-1, "Failed to setup mount entries");
++#ifdef HAVE_ISULAD
++		setup_dev = need_setup_dev(lxc_conf, &lxc_conf->mount_list);
++#endif
+ 	}
+ 
+ 	if (lxc_conf->is_execute) {
+@@ -4771,7 +4827,7 @@ int lxc_setup(struct lxc_handler *handler)
+ 
+ #ifdef HAVE_ISULAD
+ 	/* isulad: setup devices which will be populated in the container. */
+-	if (!lxc_list_empty(&lxc_conf->populate_devs)) {
++	if (!lxc_list_empty(&lxc_conf->populate_devs) && setup_dev) {
+ 		if (setup_populate_devs(&lxc_conf->rootfs, &lxc_conf->populate_devs) != 0) {
+ 			return log_error(-1, "Failed to setup devices in the container");
+ 		}
+@@ -4813,7 +4869,7 @@ int lxc_setup(struct lxc_handler *handler)
+ 	if (setup_rootfs_mountopts(&lxc_conf->rootfs)) {
+ 			return log_error(-1, "failed to set rootfs for '%s'", name);
+ 	}
+-	if (lxc_conf->rootfs.path) {
++	if (lxc_conf->rootfs.path != NULL && setup_dev) {
+ 		ret = lxc_setup_devpts(lxc_conf);
+ 		if (ret < 0) {
+ 			return log_error(-1, "Failed to setup new devpts instance for '%s'", name);
+-- 
+1.8.3.1
+

0053-attach-seprate-i-and-t-flags.patch

@@ -0,0 +1,288 @@
+From 36f64e652afc7fe3feac6d93468cbc4f3d53ec9a Mon Sep 17 00:00:00 2001
+From: LiFeng <lifeng68@huawei.com>
+Date: Tue, 21 Apr 2020 15:36:55 +0800
+Subject: [PATCH] attach: seprate -i and -t flags
+
+Signed-off-by: LiFeng <lifeng68@huawei.com>
+---
+ src/lxc/attach.c           | 100 ++++++++++++++++++++++++++++++++++++++++++++-
+ src/lxc/attach_options.h   |   2 +
+ src/lxc/start.c            |   1 +
+ src/lxc/terminal.c         |  14 ++++---
+ src/lxc/tools/lxc_attach.c |  17 +++++++-
+ 5 files changed, 125 insertions(+), 9 deletions(-)
+
+diff --git a/src/lxc/attach.c b/src/lxc/attach.c
+index 1dd2b47..5539fb1 100644
+--- a/src/lxc/attach.c
++++ b/src/lxc/attach.c
+@@ -653,6 +653,9 @@ struct attach_clone_payload {
+ 	struct lxc_proc_context_info *init_ctx;
+ 	lxc_attach_exec_t exec_function;
+ 	void *exec_payload;
++#ifdef HAVE_ISULAD
++	struct lxc_terminal *terminal;
++#endif
+ };
+ 
+ static void lxc_put_attach_clone_payload(struct attach_clone_payload *p)
+@@ -665,6 +668,49 @@ static void lxc_put_attach_clone_payload(struct attach_clone_payload *p)
+ 	}
+ }
+ 
++#ifdef HAVE_ISULAD
++static int isulad_set_attach_pipes(struct lxc_terminal *terminal)
++{
++	int ret = 0;
++	if (terminal->pipes[0][1] >= 0) {
++		close(terminal->pipes[0][1]);
++		terminal->pipes[0][1] = -1;
++	}
++
++	if (terminal->pipes[0][0] >= 0) {
++		ret = dup2(terminal->pipes[0][0], STDIN_FILENO);
++		if (ret < 0)
++			goto out;
++	}
++
++	if (terminal->pipes[1][0] >= 0) {
++		close(terminal->pipes[1][0]);
++		terminal->pipes[1][0] = -1;
++	}
++
++	if (terminal->pipes[1][1] >= 0) {
++		ret = dup2(terminal->pipes[1][1], STDOUT_FILENO);
++		if (ret < 0)
++			goto out;
++	}
++	if (terminal->pipes[2][0] >= 0) {
++		close(terminal->pipes[2][0]);
++		terminal->pipes[2][0] = -1;
++	}
++
++	if (terminal->pipes[2][1] >= 0) {
++		ret = dup2(terminal->pipes[2][1], STDERR_FILENO);
++		if (ret < 0)
++			goto out;
++	}
++
++	setsid();
++out:
++	return ret;
++}
++
++#endif
++
+ static int attach_child_main(struct attach_clone_payload *payload)
+ {
+ 	int lsm_fd, ret;
+@@ -933,6 +979,25 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 	}
+ 
+ 	if (options->attach_flags & LXC_ATTACH_TERMINAL) {
++
++#ifdef HAVE_ISULAD
++		/* isulad: dup2 pipe[0][0] to container stdin, pipe[1][1] to container stdout, pipe[2][1] to container stderr */
++		if (payload->terminal->disable_pty) {
++			ret = isulad_set_attach_pipes(payload->terminal);
++			if (ret < 0) {
++				SYSERROR("Failed to prepare terminal file pipes");
++				goto on_error;
++			}
++		}
++
++		if(!payload->terminal->disable_pty && payload->terminal_slave_fd >= 0) {
++			ret = lxc_terminal_prepare_login(payload->terminal_slave_fd);
++			if (ret < 0) {
++				SYSERROR("Failed to prepare terminal file descriptor %d", payload->terminal_slave_fd);
++				goto on_error;
++			}
++		}
++#else
+ 		ret = lxc_terminal_prepare_login(payload->terminal_slave_fd);
+ 		if (ret < 0) {
+ 			SYSERROR("Failed to prepare terminal file descriptor %d", payload->terminal_slave_fd);
+@@ -940,6 +1005,7 @@ static int attach_child_main(struct attach_clone_payload *payload)
+ 		}
+ 
+ 		TRACE("Prepared terminal file descriptor %d", payload->terminal_slave_fd);
++#endif
+ 	}
+ 
+ 	/* Avoid unnecessary syscalls. */
+@@ -1016,6 +1082,9 @@ static int lxc_attach_terminal(struct lxc_conf *conf,
+ 		free(terminal->init_fifo[2]);
+ 		terminal->init_fifo[2] = safe_strdup(options->init_fifo[2]);
+ 	}
++
++	terminal->disable_pty = options->disable_pty;
++	terminal->open_stdin = options->open_stdin;
+ #endif
+ 
+ 	ret = lxc_terminal_create(terminal);
+@@ -1410,6 +1479,21 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 		/* isulad: close errpipe */
+ 		close(conf->errpipe[1]);
+ 		conf->errpipe[1] = -1;
++		/* isulad: close pipe after clone */
++		if (terminal.pipes[0][0] >= 0) {
++			close(terminal.pipes[0][0]);
++			terminal.pipes[0][0] = -1;
++		}
++
++		if (terminal.pipes[1][1] >= 0) {
++			close(terminal.pipes[1][1]);
++			terminal.pipes[1][1] = -1;
++		}
++
++		if (terminal.pipes[2][1] >= 0) {
++			close(terminal.pipes[2][1]);
++			terminal.pipes[2][1] = -1;
++		}
+ #endif
+ 		lxc_proc_close_ns_fd(init_ctx);
+ 		if (options->attach_flags & LXC_ATTACH_TERMINAL)
+@@ -1568,6 +1652,15 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 		ret_parent = 0;
+ 		to_cleanup_pid = -1;
+ 
++	#ifdef HAVE_ISULAD
++		// iSulad: close stdin pipe if we do not want open_stdin with container stdin
++		if (!terminal.open_stdin) {
++			if (terminal.pipes[0][1] > 0) {
++				close(terminal.pipes[0][1]);
++				terminal.pipes[0][1] = -1;
++			}
++		}
++	#endif
+ 		if (options->attach_flags & LXC_ATTACH_TERMINAL) {
+ 			ret = lxc_mainloop(&descr, -1);
+ 			if (ret < 0) {
+@@ -1671,6 +1764,9 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 	payload.terminal_slave_fd = terminal.slave;
+ 	payload.exec_function = exec_function;
+ 	payload.exec_payload = exec_payload;
++#ifdef HAVE_ISULAD
++	payload.terminal = &terminal;
++#endif
+ 
+ 	pid = lxc_raw_clone(CLONE_PARENT, NULL);
+ 	if (pid < 0) {
+@@ -1697,9 +1793,9 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
+ 		_exit(EXIT_FAILURE);
+ 	}
+ 
+-	if (options->attach_flags & LXC_ATTACH_TERMINAL)
++	if (options->attach_flags & LXC_ATTACH_TERMINAL) {
+ 		lxc_attach_terminal_close_slave(&terminal);
+-
++	}
+ 	/* Tell grandparent the pid of the pid of the newly created child. */
+ 	ret = lxc_write_nointr(ipc_sockets[1], &pid, sizeof(pid));
+ 	if (ret != sizeof(pid)) {
+diff --git a/src/lxc/attach_options.h b/src/lxc/attach_options.h
+index d5d4f44..5767560 100644
+--- a/src/lxc/attach_options.h
++++ b/src/lxc/attach_options.h
+@@ -121,6 +121,8 @@ typedef struct lxc_attach_options_t {
+ 	char *init_fifo[3]; /* isulad: default fifos for the start */
+ 	int64_t timeout;/* isulad: Seconds for waiting on a container to attach/exec before it is killed*/
+ 	const char *suffix;
++	bool disable_pty;
++	bool open_stdin;
+ } lxc_attach_options_t;
+ 
+ /*! Default attach options to use */
+diff --git a/src/lxc/start.c b/src/lxc/start.c
+index e099285..68a6116 100644
+--- a/src/lxc/start.c
++++ b/src/lxc/start.c
+@@ -1789,6 +1789,7 @@ static int do_start(void *data)
+ 	 * as it execs.
+ 	 */
+ #ifdef HAVE_ISULAD
++	close_prot_errno_disarm(status_fd);
+ 	handler->ops->start(handler, handler->data, handler->daemonize ? handler->conf->errpipe[1] : -1);
+ #else
+ 	handler->ops->start(handler, handler->data);
+diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
+index c0a4d1a..57def93 100644
+--- a/src/lxc/terminal.c
++++ b/src/lxc/terminal.c
+@@ -750,21 +750,23 @@ int lxc_terminal_io_cb(int fd, uint32_t events, void *data,
+ 			return LXC_MAINLOOP_CONTINUE;
+ 		} else if (fd == terminal->pipes[1][0] || fd == terminal->pipes[2][0]) {
+ 			if (fd == terminal->pipes[1][0]) {
+-				w_log = isulad_lxc_terminal_write_log_file(terminal, "stdout", NULL, 0);
++				if (terminal->log_fd >= 0) {
++					w_log = isulad_lxc_terminal_write_log_file(terminal, "stdout", NULL, 0);
++				}
+ 				terminal->pipes[1][0] = -EBADF;
+ 			} else if (fd == terminal->pipes[2][0]) {
+-				w_log = isulad_lxc_terminal_write_log_file(terminal, "stderr", NULL, 0);
++				if (terminal->log_fd >= 0) {
++					w_log = isulad_lxc_terminal_write_log_file(terminal, "stderr", NULL, 0);
++				}
+ 				terminal->pipes[2][0] = -EBADF;
+ 			}
+-			if (w_log < 0)
+-				TRACE("Failed to write %d bytes to terminal log", r);
+ 			close(fd);
+-			return LXC_MAINLOOP_CONTINUE;
++			return LXC_MAINLOOP_CLOSE;
+ 		} else if (fd == terminal->pipes[0][1]) {
+ 			TRACE("closed stdin pipe of container stdin");
+ 			terminal->pipes[0][1] = -EBADF;
+ 			close(fd);
+-			return LXC_MAINLOOP_CONTINUE;
++			return LXC_MAINLOOP_CLOSE;
+ 		} else {
+ 			ERROR("Handler received unexpected file descriptor");
+ 		}
+diff --git a/src/lxc/tools/lxc_attach.c b/src/lxc/tools/lxc_attach.c
+index a855a8d..da7a7d2 100644
+--- a/src/lxc/tools/lxc_attach.c
++++ b/src/lxc/tools/lxc_attach.c
+@@ -82,6 +82,8 @@ static const struct option my_longopts[] = {
+ 	{"err-fifo", required_argument, 0, OPT_STDERR_FIFO},
+ 	{"suffix", required_argument, 0, OPT_ATTACH_SUFFIX},
+ 	{"timeout", required_argument, 0, OPT_ATTACH_TIMEOUT},
++	{"disable-pty", no_argument, 0, OPT_DISABLE_PTY},
++	{"open-stdin", no_argument, 0, OPT_OPEN_STDIN},
+ #endif
+ 	LXC_COMMON_OPTIONS
+ };
+@@ -312,9 +314,14 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
+ 		}
+ 		args->attach_timeout = (unsigned int)atoll(arg);
+ 		break;
++	case OPT_DISABLE_PTY:
++		args->disable_pty = 1;
++		break;
++	case OPT_OPEN_STDIN:
++		args->open_stdin = 1;
++		break;
+ #endif
+ 	}
+-
+ 	return 0;
+ }
+ 
+@@ -614,6 +621,14 @@ int main(int argc, char *argv[])
+ 
+ 	attach_options.suffix = my_args.suffix;
+ 
++	if (my_args.disable_pty) {
++		attach_options.disable_pty = true;
++	}
++
++	if (my_args.open_stdin) {
++		attach_options.open_stdin = true;
++	}
++
+ 	/* isulad: add do attach background */
+ 	if (attach_options.attach_flags & LXC_ATTACH_TERMINAL)
+ 		wexit = do_attach_foreground(c, &command, &attach_options, &errmsg);
+-- 
+1.8.3.1
+

0053-destroy-empty-cgroup-path-return-ture.patch

@@ -1,31 +0,0 @@
-From 55aa538a996fb98bd76ca41029c93147d22c1ecf Mon Sep 17 00:00:00 2001
-From: liuhao <liuhao27@huawei.com>
-Date: Tue, 22 Jan 2019 14:45:54 +0800
-Subject: [PATCH 053/140] destroy empty cgroup path return ture
-
-destroy empty cgroup path return ture
-
-Signed-off-by: liuhao <liuhao27@huawei.com>
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/utils.c | 4 ++++
- 1 file changed, 4 insertions(+)
-
-diff --git a/src/lxc/utils.c b/src/lxc/utils.c
-index c8fb993..24e975b 100644
---- a/src/lxc/utils.c
-+++ b/src/lxc/utils.c
-@@ -1776,6 +1776,10 @@ int recursive_destroy(char *dirname)
- 
- 	dir = opendir(dirname);
- 	if (!dir) {
-+		if (errno == ENOENT) {
-+			WARN("Destroy path: %s do not exist");
-+			return 0;
-+		}
- 		SYSERROR("Failed to open dir \"%s\"", dirname);
- 		return -1;
- 	}
--- 
-1.8.3.1
-

0054-fix-invalid-log-message.patch

@@ -1,42 +0,0 @@
-From 6ca37fb9e3838f3ddffec7e0325e37e6925f3562 Mon Sep 17 00:00:00 2001
-From: liuhao <liuhao27@huawei.com>
-Date: Tue, 22 Jan 2019 20:14:53 +0800
-Subject: [PATCH 054/140] fix invalid log message
-
-fix invalid log message
-
-Signed-off-by: liuhao <liuhao27@huawei.com>
-Signed-off-by: LiFeng <lifeng68@huawei.com>
----
- src/lxc/conf.c  | 1 -
- src/lxc/utils.c | 2 +-
- 2 files changed, 1 insertion(+), 2 deletions(-)
-
-diff --git a/src/lxc/conf.c b/src/lxc/conf.c
-index 6ea8f9c..1e403eb 100644
---- a/src/lxc/conf.c
-+++ b/src/lxc/conf.c
-@@ -2708,7 +2708,6 @@ static int mount_file_entries(const struct lxc_conf *conf,
- 			free(mntent.mnt_fsname);
- 			return -1;
- 		}
--		ERROR("mntent.mnt_fsname:%s, mntent.mnt_dir:%s", mntent.mnt_fsname, mntent.mnt_dir);
- 		if (!rootfs->path)
- 			ret = mount_entry_on_systemfs(&mntent);
- 		else if (mntent.mnt_dir[0] != '/')
-diff --git a/src/lxc/utils.c b/src/lxc/utils.c
-index 24e975b..4db61c5 100644
---- a/src/lxc/utils.c
-+++ b/src/lxc/utils.c
-@@ -1777,7 +1777,7 @@ int recursive_destroy(char *dirname)
- 	dir = opendir(dirname);
- 	if (!dir) {
- 		if (errno == ENOENT) {
--			WARN("Destroy path: %s do not exist");
-+			WARN("Destroy path: \"%s\" do not exist", dirname);
- 			return 0;
- 		}
- 		SYSERROR("Failed to open dir \"%s\"", dirname);
--- 
-1.8.3.1
-