lxc/0039-print-error-message-when-container-start-failed.patch

From 24f908199916fc92cb7935bfccc19244d3a8d864 Mon Sep 17 00:00:00 2001
From: tanyifeng <tanyifeng1@huawei.com>
Date: Wed, 16 Jan 2019 14:38:38 +0800
Subject: [PATCH 039/140] print error message when container start failed

Signed-off-by: tanyifeng <tanyifeng1@huawei.com>
Signed-off-by: LiFeng <lifeng68@huawei.com>
---
 src/lxc/attach.c          |   6 +--
 src/lxc/cgroups/cgfsng.c  |  22 +++++++--
 src/lxc/cgroups/cgroup.c  |   4 +-
 src/lxc/cgroups/cgroup.h  |   2 +
 src/lxc/conf.c            | 122 ++++++++++++++++++++++++++--------------------
 src/lxc/conf.h            |   5 +-
 src/lxc/execute.c         |   3 +-
 src/lxc/lxccontainer.c    |  37 ++++++++++++--
 src/lxc/start.c           |  14 ++++--
 src/lxc/start.h           |   2 +-
 src/lxc/tools/lxc_start.c |   3 ++
 11 files changed, 147 insertions(+), 73 deletions(-)

diff --git a/src/lxc/attach.c b/src/lxc/attach.c
index 4ccdd74..b44ea74 100644
--- a/src/lxc/attach.c
+++ b/src/lxc/attach.c
@@ -1396,7 +1396,7 @@ int lxc_attach(const char *name, const char *lxcpath,
 
 		/* Setup resource limits */
 		if (!lxc_list_empty(&conf->limits)) {
-			ret = setup_resource_limits(&conf->limits, pid);
+			ret = setup_resource_limits(&conf->limits, pid, -1);
 			if (ret < 0)
 				goto on_error;
 		}
@@ -1650,8 +1650,8 @@ int lxc_attach_run_command(void *payload, int msg_fd)
 		}
 	}
 
-	/* isulad: write errorm messages */
-	lxc_write_error_message(msg_fd, "exec: \"%s\": %s", cmd->program, strerror(errno));
+	/* isulad: write error messages */
+	lxc_write_error_message(msg_fd, "exec: \"%s\": %s.", cmd->program, strerror(errno));
 
 	SYSERROR("Failed to exec \"%s\"", cmd->program);
 	return ret;
diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
index bc1481d..47b12a6 100644
--- a/src/lxc/cgroups/cgfsng.c
+++ b/src/lxc/cgroups/cgfsng.c
@@ -1236,7 +1236,7 @@ static int mkdir_eexist_on_last(const char *dir, mode_t mode)
 	return 0;
 }
 
-static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname)
+static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname, int errfd)
 {
 	int ret;
 
@@ -1244,8 +1244,8 @@ static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname)
 
 	if (file_exists(h->container_full_path)) { // it must not already exist
 		ERROR("Cgroup path \"%s\" already exist.", h->container_full_path);
-		//lxc_write_error_message(errfd, "%s:%d: Cgroup path \"%s\" already exist.",
-		//	__FILE__, __LINE__, h->fullcgpath);
+		lxc_write_error_message(errfd, "%s:%d: Cgroup path \"%s\" already exist.",
+			__FILE__, __LINE__, h->container_full_path);
 		return false;
 	}
 
@@ -1288,7 +1288,7 @@ __cgfsng_ops static bool cgfsng_payload_create(struct cgroup_ops *ops,
 	}
 
 	for (i = 0; ops->hierarchies[i]; i++) {
-		if (!create_path_for_hierarchy(ops->hierarchies[i], container_cgroup)) {
+		if (!create_path_for_hierarchy(ops->hierarchies[i], container_cgroup, ops->errfd)) {
 			SYSERROR("Failed to create %s", ops->hierarchies[i]->container_full_path);
 			return false;
 		}
@@ -2203,6 +2203,11 @@ static int cg_legacy_set_data(struct cgroup_ops *ops, const char *filename,
 
 	fullpath = must_make_path(h->container_full_path, filename, NULL);
 	ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666);
+	if (ret) {
+		lxc_write_error_message(ops->errfd,
+			"%s:%d: setting cgroup config for ready process caused \"failed to write %s to %s: %s\".",
+			__FILE__, __LINE__, value, fullpath, strerror(errno));
+	}
 	free(fullpath);
 	return ret;
 }
@@ -2294,9 +2299,15 @@ static bool __cg_legacy_setup_limits(struct cgroup_ops *ops,
 			}
 			if (setvalue > readvalue) {
 				ERROR("The maximum allowed cpu-shares is %s", value);
+				lxc_write_error_message(ops->errfd,
+					"%s:%d: setting cgroup config for ready process caused \"The maximum allowed cpu-shares is %s\".",
+					__FILE__, __LINE__, value);
 				goto out;
 			} else if (setvalue < readvalue) {
 				ERROR("The minimum allowed cpu-shares is %s", value);
+				lxc_write_error_message(ops->errfd,
+					"%s:%d: setting cgroup config for ready process caused \"The minimum allowed cpu-shares is %s\".",
+					__FILE__, __LINE__, value);
 				goto out;
 			}
 		}
@@ -2699,7 +2710,7 @@ __cgfsng_ops static bool cgfsng_data_init(struct cgroup_ops *ops, struct lxc_han
 	return true;
 }
 
-struct cgroup_ops *cgfsng_ops_init(void)
+struct cgroup_ops *cgfsng_ops_init(int errfd)
 {
 	struct cgroup_ops *cgfsng_ops;
 
@@ -2715,6 +2726,7 @@ struct cgroup_ops *cgfsng_ops_init(void)
 		return NULL;
 	}
 
+	cgfsng_ops->errfd = errfd;
 	cgfsng_ops->data_init = cgfsng_data_init;
 	cgfsng_ops->destroy = cgfsng_payload_destroy;
 	cgfsng_ops->payload_create = cgfsng_payload_create;
diff --git a/src/lxc/cgroups/cgroup.c b/src/lxc/cgroups/cgroup.c
index 8e7aef9..7442c31 100644
--- a/src/lxc/cgroups/cgroup.c
+++ b/src/lxc/cgroups/cgroup.c
@@ -38,13 +38,13 @@
 
 lxc_log_define(cgroup, lxc);
 
-extern struct cgroup_ops *cgfsng_ops_init(void);
+extern struct cgroup_ops *cgfsng_ops_init(int errfd);
 
 struct cgroup_ops *cgroup_init(struct lxc_handler *handler)
 {
 	struct cgroup_ops *cgroup_ops;
 
-	cgroup_ops = cgfsng_ops_init();
+	cgroup_ops = cgfsng_ops_init(handler->conf->errpipe[1]);
 	if (!cgroup_ops) {
 		ERROR("Failed to initialize cgroup driver");
 		return NULL;
diff --git a/src/lxc/cgroups/cgroup.h b/src/lxc/cgroups/cgroup.h
index fa4871e..b6116f6 100644
--- a/src/lxc/cgroups/cgroup.h
+++ b/src/lxc/cgroups/cgroup.h
@@ -92,6 +92,8 @@ struct cgroup_ops {
 	char **cgroup_use;
 	char *cgroup_pattern;
 	char *container_cgroup;
+	/* isulad: errfd */
+	int errfd;
 
 	/* @hierarchies
 	 * - A NULL-terminated array of struct hierarchy, one per legacy
diff --git a/src/lxc/conf.c b/src/lxc/conf.c
index 88cebfd..8fa63f7 100644
--- a/src/lxc/conf.c
+++ b/src/lxc/conf.c
@@ -2414,6 +2414,8 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
 		ret = mkdir_p(path, 0755);
 		if (ret < 0 && errno != EEXIST) {
 			SYSERROR("Failed to create directory \"%s\"", path);
+			lxc_write_error_message(rootfs->errfd, "%s:%d: mkdir %s: %s.",
+				__FILE__, __LINE__, path, strerror(errno));
 			return -1;
 		}
 	}
@@ -2435,12 +2437,17 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
 	free(p1);
 	if (ret < 0 && errno != EEXIST) {
 		SYSERROR("Failed to create directory \"%s\"", path);
+		lxc_write_error_message(rootfs->errfd, "%s:%d: mkdir %s: %s.",
+				__FILE__, __LINE__, p2, strerror(errno));
 		return -1;
 	}
 
 	ret = mknod(path, S_IFREG | 0000, 0);
-	if (ret < 0 && errno != EEXIST)
+	if (ret < 0 && errno != EEXIST) {
+		lxc_write_error_message(rootfs->errfd, "%s:%d: open %s: %s.",
+				__FILE__, __LINE__, path, strerror(errno));
 		return -errno;
+	}
 
 	return 0;
 }
@@ -2550,7 +2557,9 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
 	if (rootfs_path) {
 		rpath = follow_symlink_in_scope(path, rootfs_path);
 		if (!rpath) {
-			ERROR("Failed to get real path for '%s'", path);
+			ERROR("Failed to get real path of '%s' in scope '%s'.", path, rootfs_path);
+			lxc_write_error_message(rootfs->errfd, "%s:%d: failed to get real path of '%s' in scope '%s'.",
+				__FILE__, __LINE__, path, rootfs_path);
 			return -1;
 		}
 		dest = rpath;
@@ -2558,6 +2567,8 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
 		ret = check_mount_destination(rootfs_path, dest);
 		if (ret) {
 			ERROR("Mount destination is invalid: '%s'", dest);
+			lxc_write_error_message(rootfs->errfd, "%s:%d: mount destination is invalid: '%s'.",
+				__FILE__, __LINE__, dest);
 			free(rpath);
 			return -1;
 		}
@@ -2587,6 +2598,10 @@ static inline int mount_entry_on_generic(struct mntent *mntent,
 		ret = mount_entry(mntent->mnt_fsname, dest, mntent->mnt_type, mntflags,
 				  pflags, mntdata, optional, dev, relative, rootfs_path);
 	}
+	if (ret < 0) {
+		lxc_write_error_message(rootfs->errfd, "%s:%d: failed to mount %s as type %s.",
+				__FILE__, __LINE__, mntent->mnt_fsname, mntent->mnt_type);
+	}
 
 	free(mntdata);
 	free(rpath);
@@ -2949,7 +2964,7 @@ static int parse_resource(const char *res)
 	return resid;
 }
 
-int setup_resource_limits(struct lxc_list *limits, pid_t pid)
+int setup_resource_limits(struct lxc_list *limits, pid_t pid, int errfd)
 {
 	int resid;
 	struct lxc_list *it;
@@ -2966,7 +2981,10 @@ int setup_resource_limits(struct lxc_list *limits, pid_t pid)
 
 #if HAVE_PRLIMIT || HAVE_PRLIMIT64
 		if (prlimit(pid, resid, &lim->limit, NULL) != 0) {
-			SYSERROR("Failed to set limit %s", lim->resource);
+			SYSERROR("Failed to set limit %s %lu %lu.", lim->resource, lim->limit.rlim_cur, lim->limit.rlim_max);
+			lxc_write_error_message(errfd, "%s:%d: Failed to set limit %s %lu %lu: %s.",
+					__FILE__, __LINE__, lim->resource,
+					lim->limit.rlim_cur, lim->limit.rlim_max, strerror(errno));
 			return -1;
 		}
 
@@ -3989,6 +4007,8 @@ int lxc_setup(struct lxc_handler *handler)
 	ret = lxc_setup_rootfs_prepare_root(lxc_conf, name, lxcpath);
 	if (ret < 0) {
 		ERROR("Failed to setup rootfs");
+		lxc_write_error_message(lxc_conf->errpipe[1], "%s:%d: failed to setup rootfs %s.",
+				__FILE__, __LINE__, lxc_conf->rootfs.path);
 		return -1;
 	}
 
@@ -3996,31 +4016,31 @@ int lxc_setup(struct lxc_handler *handler)
 		ret = setup_utsname(lxc_conf->utsname);
 		if (ret < 0) {
 			ERROR("Failed to setup the utsname %s", name);
-			return -1;
+			goto on_error;
 		}
 	}
 
 	ret = lxc_setup_keyring();
 	if (ret < 0)
-		return -1;
+		goto on_error;
 
 	ret = lxc_setup_network_in_child_namespaces(lxc_conf, &lxc_conf->network);
 	if (ret < 0) {
 		ERROR("Failed to setup network");
-		return -1;
+		goto on_error;
 	}
 
 	ret = lxc_network_send_name_and_ifindex_to_parent(handler);
 	if (ret < 0) {
 		ERROR("Failed to send network device names and ifindices to parent");
-		return -1;
+		goto on_error;
 	}
 
 	if (lxc_conf->autodev > 0) {
 		ret = mount_autodev(name, &lxc_conf->rootfs, lxcpath);
 		if (ret < 0) {
 			ERROR("Failed to mount \"/dev\"");
-			return -1;
+			goto on_error;
 		}
 	}
 
@@ -4030,13 +4050,14 @@ int lxc_setup(struct lxc_handler *handler)
 	ret = lxc_mount_auto_mounts(lxc_conf, lxc_conf->auto_mounts & ~LXC_AUTO_CGROUP_MASK, handler);
 	if (ret < 0) {
 		ERROR("Failed to setup first automatic mounts");
-		return -1;
+		goto on_error;
 	}
 
+	lxc_conf->rootfs.errfd = lxc_conf->errpipe[1];
 	ret = setup_mount(lxc_conf, &lxc_conf->rootfs, lxc_conf->fstab, name, lxcpath);
 	if (ret < 0) {
 		ERROR("Failed to setup mounts");
-		return -1;
+		goto on_error;
 	}
 
 	if (lxc_conf->is_execute) {
@@ -4047,13 +4068,13 @@ int lxc_setup(struct lxc_handler *handler)
 			ret = snprintf(path, PATH_MAX, SBINDIR "/init.lxc.static");
 			if (ret < 0 || ret >= PATH_MAX) {
 				ERROR("Path to init.lxc.static too long");
-				return -1;
+				goto on_error;
 			}
 
 			fd = open(path, O_PATH | O_CLOEXEC);
 			if (fd < 0) {
 				SYSERROR("Unable to open lxc.init.static");
-				return -1;
+				goto on_error;
 			}
 
 			((struct execute_args *)handler->data)->init_fd = fd;
@@ -4062,7 +4083,7 @@ int lxc_setup(struct lxc_handler *handler)
 			ret = lxc_execute_bind_init(handler);
 			if (ret < 0) {
 				ERROR("Failed to bind-mount the lxc init system");
-				return -1;
+				goto on_error;
 			}
 		}
 	}
@@ -4074,7 +4095,7 @@ int lxc_setup(struct lxc_handler *handler)
 	ret = lxc_mount_auto_mounts(lxc_conf, lxc_conf->auto_mounts & LXC_AUTO_CGROUP_MASK, handler);
 	if (ret < 0) {
 		ERROR("Failed to setup remaining automatic mounts");
-		return -1;
+		goto on_error;
 	}
 
 	/*isulad: move mount entries here, before we do lxc_fill_autodev and populate devices  */
@@ -4083,27 +4104,27 @@ int lxc_setup(struct lxc_handler *handler)
 					  &lxc_conf->mount_list, name, lxcpath);
 		if (ret < 0) {
 			ERROR("Failed to setup mount entries");
-			return -1;
+			goto on_error;
 		}
 	}
 
 	ret = run_lxc_hooks(name, "mount", lxc_conf, NULL);
 	if (ret < 0) {
 		ERROR("Failed to run mount hooks");
-		return -1;
+		goto on_error;
 	}
 
 	if (lxc_conf->autodev > 0) {
 		ret = run_lxc_hooks(name, "autodev", lxc_conf, NULL);
 		if (ret < 0) {
 			ERROR("Failed to run autodev hooks");
-			return -1;
+			goto on_error;
 		}
 
 		ret = lxc_fill_autodev(&lxc_conf->rootfs);
 		if (ret < 0) {
 			ERROR("Failed to populate \"/dev\"");
-			return -1;
+			goto on_error;
 		}
 	}
 
@@ -4111,64 +4132,64 @@ int lxc_setup(struct lxc_handler *handler)
 	if (!lxc_list_empty(&lxc_conf->populate_devs)) {
 		if (setup_populate_devs(&lxc_conf->rootfs, &lxc_conf->populate_devs)) {
 			ERROR("Failed to setup devices in the container");
-			return -1;;
+			goto on_error;
 		}
 	}
 
 	/* Make sure any start hooks are in the container */
 	if (!verify_start_hooks(lxc_conf)) {
 		ERROR("Failed to verify start hooks");
-		return -1;
+		goto on_error;
 	}
 
 	ret = lxc_setup_console(&lxc_conf->rootfs, &lxc_conf->console,
 				lxc_conf->ttys.dir);
 	if (ret < 0) {
 		ERROR("Failed to setup console");
-		return -1;
+		goto on_error;
 	}
 
 	ret = lxc_setup_dev_symlinks(&lxc_conf->rootfs);
 	if (ret < 0) {
 		ERROR("Failed to setup \"/dev\" symlinks");
-		return -1;
+		goto on_error;
 	}
 
 	ret = lxc_create_tmp_proc_mount(lxc_conf);
 	if (ret < 0) {
 		ERROR("Failed to \"/proc\" LSMs");
-		return -1;
+		goto on_error;
 	}
 
 	ret = lxc_setup_rootfs_switch_root(&lxc_conf->rootfs);
 	if (ret < 0) {
 		ERROR("Failed to pivot root into rootfs");
-		return -1;
+		goto on_error;
 	}
 
 	/* isulad: remount rootfs readonly if necessary */
 	if (setup_rootfs_mountopts(&lxc_conf->rootfs)) {
 		ERROR("failed to set rootfs for '%s'", name);
-		return -1;
+		goto on_error;
 	}
 
 	if (lxc_conf->rootfs.path) {
 		ret = lxc_setup_devpts(lxc_conf);
 		if (ret < 0) {
 			ERROR("Failed to setup new devpts instance");
-			return -1;
+			goto on_error;
 		}
 	}
 
 	ret = lxc_create_ttys(handler);
 	if (ret < 0)
-		return -1;
+		goto on_error;
 
 	//isulad: setup rootfs masked paths
 	if (!lxc_list_empty(&lxc_conf->rootfs.maskedpaths)) {
 		if (setup_rootfs_maskedpaths(&lxc_conf->rootfs.maskedpaths)) {
 			ERROR("failed to setup maskedpaths");
-			return -1;
+			goto on_error;
 		}
 	}
 
@@ -4176,7 +4197,7 @@ int lxc_setup(struct lxc_handler *handler)
 	if (!lxc_list_empty(&lxc_conf->rootfs.ropaths)) {
 		if (setup_rootfs_ropaths(&lxc_conf->rootfs.ropaths)) {
 			ERROR("failed to setup readonlypaths");
-			return -1;
+			goto on_error;
 		}
 	}
 
@@ -4186,7 +4207,7 @@ int lxc_setup(struct lxc_handler *handler)
 	ret = setup_personality(lxc_conf->personality);
 	if (ret < 0) {
 		ERROR("Failed to set personality");
-		return -1;
+		goto on_error;
 	}
 
 	/* Set sysctl value to a path under /proc/sys as determined from the
@@ -4197,7 +4218,7 @@ int lxc_setup(struct lxc_handler *handler)
 		ret = setup_sysctl_parameters(&lxc_conf->sysctls);
 		if (ret < 0) {
 			ERROR("Failed to setup sysctl parameters");
-			return -1;
+			goto on_error;
 		}
 	}
 
@@ -4206,21 +4227,24 @@ int lxc_setup(struct lxc_handler *handler)
 			ERROR("Container requests lxc.cap.drop and "
 			      "lxc.cap.keep: either use lxc.cap.drop or "
 			      "lxc.cap.keep, not both");
-			return -1;
+			goto on_error;
 		}
 
 		if (dropcaps_except(&lxc_conf->keepcaps)) {
 			ERROR("Failed to keep capabilities");
-			return -1;
+			goto on_error;
 		}
 	} else if (setup_caps(&lxc_conf->caps)) {
 		ERROR("Failed to drop capabilities");
-		return -1;
+		goto on_error;
 	}
 
 	NOTICE("The container \"%s\" is set up", name);
 
 	return 0;
+on_error:
+	lxc_write_error_message(lxc_conf->errpipe[1], "Failed to setup lxc, please check the config file.");
+	return -1;
 }
 
 /* isulad drop caps for container*/
@@ -4555,11 +4579,9 @@ void* wait_ocihook_timeout(void *arg)
 			__FILE__, __LINE__, lxchook_names[conf->which],
 			(double)conf->timeout);
 
-		if (conf->errfd >= 0) {
-			lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\"",
-				__FILE__, __LINE__, lxchook_names[conf->which],
-				(double)conf->timeout);
-		}
+		lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"hook ran past specified timeout of %.1fs\".",
+			__FILE__, __LINE__, lxchook_names[conf->which],
+			(double)conf->timeout);
 
 		if (kill(conf->pid, SIGKILL) && errno != ESRCH) {
 			ERROR("Send kill signal failed");
@@ -4640,22 +4662,18 @@ static int run_ocihook_buffer(struct oci_hook_conf *oconf, char *inmsg)
 		goto print_hook;
 	} else if (WIFEXITED(ret) && WEXITSTATUS(ret) != 0) {
 		ERROR("Script exited with status %d. output:%s", WEXITSTATUS(ret), output);
-		if (conf->errfd >= 0) {
-			lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"error running hook: exit status %d, output:%s\"",
-				__FILE__, __LINE__,
-				(conf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[conf->which],
-				WEXITSTATUS(ret), output);
-		}
+		lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"error running hook: exit status %d, output:%s\".",
+			__FILE__, __LINE__,
+			(conf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[conf->which],
+			WEXITSTATUS(ret), output);
 
 		goto print_hook;
 	} else if (WIFSIGNALED(ret)) {
 		ERROR("Script terminated by signal %d.", WTERMSIG(ret));
-		if (conf->errfd >= 0) {
-			lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"error running hook: Script terminated by signal %d\"",
-				__FILE__, __LINE__,
-				(conf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[conf->which],
-				WTERMSIG(ret));
-		}
+		lxc_write_error_message(conf->errfd, "%s:%d: running %s hook caused \"error running hook: Script terminated by signal %d\".",
+			__FILE__, __LINE__,
+			(conf->which >= NUM_LXC_HOOKS) ? "invalid type" : lxchook_names[conf->which],
+			WTERMSIG(ret));
 
 		goto print_hook;
 	}
diff --git a/src/lxc/conf.h b/src/lxc/conf.h
index b92c48e..88f5b41 100644
--- a/src/lxc/conf.h
+++ b/src/lxc/conf.h
@@ -175,6 +175,8 @@ struct lxc_rootfs {
 	struct lxc_list maskedpaths;
 	/* isulad: ropaths */
 	struct lxc_list ropaths;
+	/* isulad: errfd */
+	int errfd;
 };
 
 /*
@@ -462,7 +464,7 @@ extern int lxc_setup_rootfs_prepare_root(struct lxc_conf *conf,
 					 const char *name, const char *lxcpath);
 extern int lxc_setup(struct lxc_handler *handler);
 extern int lxc_setup_parent(struct lxc_handler *handler);
-extern int setup_resource_limits(struct lxc_list *limits, pid_t pid);
+extern int setup_resource_limits(struct lxc_list *limits, pid_t pid, int errfd);
 extern int find_unmapped_nsid(struct lxc_conf *conf, enum idtype idtype);
 extern int mapped_hostid(unsigned id, struct lxc_conf *conf,
 			 enum idtype idtype);
@@ -499,6 +501,7 @@ int lxc_clear_populate_devices(struct lxc_conf *c);
 int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
 int lxc_clear_rootfs_ro_paths(struct lxc_conf *c);
 int lxc_drop_caps(struct lxc_conf *conf);
+void lxc_close_error_pipe(int *errpipe);
 
 /* isulad add end */
 
diff --git a/src/lxc/execute.c b/src/lxc/execute.c
index d388e63..3fc46c6 100644
--- a/src/lxc/execute.c
+++ b/src/lxc/execute.c
@@ -40,7 +40,7 @@
 
 lxc_log_define(execute, start);
 
-static int execute_start(struct lxc_handler *handler, void* data)
+static int execute_start(struct lxc_handler *handler, void* data, int fd)
 {
 	int argc_add, j;
 	char **argv;
@@ -91,6 +91,7 @@ static int execute_start(struct lxc_handler *handler, void* data)
 	else
 		execvp(argv[0], argv);
 	SYSERROR("Failed to exec %s", argv[0]);
+	lxc_write_error_message(fd, "Failed to exec: \"%s\": %s.", argv[0], strerror(errno));
 
 	free(argv);
 out1:
diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
index e99c41c..d641851 100644
--- a/src/lxc/lxccontainer.c
+++ b/src/lxc/lxccontainer.c
@@ -916,7 +916,9 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		NULL,
 	};
 	char **init_cmd = NULL;
-	int keepfds[4] = {-1, -1, -1, -1};
+	int keepfds[] = {-1, -1, -1, -1, -1};
+	ssize_t size_read;
+	char errbuf[BUFSIZ + 1] = {0};
 
 	/* container does exist */
 	if (!c)
@@ -962,7 +964,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 			argv = init_cmd = split_init_cmd(conf->init_cmd);
 	}
 
-    /* isulad: use init argv as init cmd */
+	/* isulad: use init argv as init cmd */
 	if (!argv) {
 		argv = init_cmd = use_init_args(conf->init_argv, conf->init_argc);
 	}
@@ -986,10 +988,19 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		char title[2048];
 		pid_t pid;
 
+		//isulad: pipdfd for get error message of child or grandchild process.
+		if (pipe2(conf->errpipe, O_CLOEXEC) != 0) {
+			SYSERROR("Failed to init errpipe");
+			free_init_cmd(init_cmd);
+			lxc_free_handler(handler);
+			return false;
+		}
+
 		pid = fork();
 		if (pid < 0) {
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
+			lxc_close_error_pipe(conf->errpipe);
 			return false;
 		}
 
@@ -999,11 +1010,23 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 			 * the PID file, child will do the free and unlink.
 			 */
 			c->pidfile = NULL;
+			close(conf->errpipe[1]);
+			conf->errpipe[1] = -1;
 
 			/* Wait for container to tell us whether it started
 			 * successfully.
 			 */
 			started = wait_on_daemonized_start(handler, pid);
+			if (!started) {
+				size_read = read(conf->errpipe[0], errbuf, BUFSIZ);
+				if (size_read > 0) {
+					conf->errmsg = strdup(errbuf);
+					if (!conf->errmsg)
+						ERROR("Out of memory");
+				}
+			}
+			close(conf->errpipe[0]);
+			conf->errpipe[0] = -1;
 
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
@@ -1039,6 +1062,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		if (pid != 0) {
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
+			lxc_close_error_pipe(conf->errpipe);
 			_exit(EXIT_SUCCESS);
 		}
 
@@ -1050,10 +1074,12 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 			SYSERROR("Failed to change to \"/\" directory");
 			_exit(EXIT_FAILURE);
 		}
-
+		close(conf->errpipe[0]);
+		conf->errpipe[0] = -1;
 		keepfds[0] = handler->conf->maincmd_fd;
 		keepfds[1] = handler->state_socket_pair[0];
 		keepfds[2] = handler->state_socket_pair[1];
+		keepfds[4] = conf->errpipe[1];
 		ret = lxc_check_inherited(conf, true, keepfds,
 					  sizeof(keepfds) / sizeof(keepfds[0]));
 		if (ret < 0)
@@ -1088,6 +1114,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		if (w < 0 || (size_t)w >= sizeof(pidstr)) {
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
+			lxc_close_error_pipe(conf->errpipe);
 
 			SYSERROR("Failed to write monitor pid to \"%s\"", c->pidfile);
 
@@ -1101,6 +1128,7 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		if (ret < 0) {
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
+			lxc_close_error_pipe(conf->errpipe);
 
 			SYSERROR("Failed to write monitor pid to \"%s\"", c->pidfile);
 
@@ -1159,6 +1187,9 @@ reboot:
 	if (conf->exit_fd >= 0) {
 		keepfds[3] = conf->exit_fd;
 	}
+	/* isulad: keep errpipe fd */
+	if (c->daemonize)
+		keepfds[4] = conf->errpipe[1];
 	ret = lxc_check_inherited(conf, c->daemonize, keepfds,
 				  sizeof(keepfds) / sizeof(keepfds[0]));
 	if (ret < 0) {
diff --git a/src/lxc/start.c b/src/lxc/start.c
index ec61b32..3e6854f 100644
--- a/src/lxc/start.c
+++ b/src/lxc/start.c
@@ -892,6 +892,7 @@ int lxc_init(const char *name, struct lxc_handler *handler)
 	ret = lxc_terminal_setup(conf);
 	if (ret < 0) {
 		ERROR("Failed to create console");
+		lxc_write_error_message(conf->errpipe[1], "Failed to create console for container \"%s\".", name);
 		goto out_restore_sigmask;
 	}
 	TRACE("Created console");
@@ -1390,6 +1391,8 @@ static int do_start(void *data)
 		struct stat st;
 		if (stat(handler->conf->init_cwd, &st) < 0 && mkdir_p(handler->conf->init_cwd, 0755) < 0) {
 			SYSERROR("Try to create directory \"%s\" as workdir failed", handler->conf->init_cwd);
+			lxc_write_error_message(handler->conf->errpipe[1], "%s:%d: Failed to create workdir: %s.",
+					__FILE__, __LINE__, strerror(errno));
 			goto out_warn_father;
 		}
 		if (chdir(handler->conf->init_cwd)) {
@@ -1489,7 +1492,7 @@ static int do_start(void *data)
 	/* After this call, we are in error because this ops should not return
 	 * as it execs.
 	 */
-	handler->ops->start(handler, handler->data);
+	handler->ops->start(handler, handler->data, handler->daemonize ? handler->conf->errpipe[1] : -1);
 
 out_warn_father:
 	/* We want the parent to know something went wrong, so we return a
@@ -1898,7 +1901,7 @@ static int lxc_spawn(struct lxc_handler *handler)
 		goto out_delete_net;
 
 	if (!lxc_list_empty(&conf->limits)) {
-		ret = setup_resource_limits(&conf->limits, handler->pid);
+		ret = setup_resource_limits(&conf->limits, handler->pid, conf->errpipe[1]);
 		if (ret < 0) {
 			ERROR("Failed to setup resource limits");
 			goto out_delete_net;
@@ -1960,7 +1963,7 @@ static int lxc_spawn(struct lxc_handler *handler)
 	}
 
 	if (START_TIMEOUT == global_timeout_state) {
-		//lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
+		lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
 		ERROR("Starting the container \"%s\" timeout.", name);
 		goto out_delete_net;
 	}
@@ -2008,7 +2011,7 @@ static int lxc_spawn(struct lxc_handler *handler)
 	}
 
 	if (START_TIMEOUT == global_timeout_state) {
-		//lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
+		lxc_write_error_message(conf->errpipe[1], "Starting the container \"%s\" timeout.", name);
 		ERROR("Starting the container \"%s\" timeout.", name);
 		goto out_abort;
 	}
@@ -2232,7 +2235,7 @@ struct start_args {
 	char *const *argv;
 };
 
-static int start(struct lxc_handler *handler, void* data)
+static int start(struct lxc_handler *handler, void* data, int fd)
 {
 	struct start_args *arg = data;
 
@@ -2240,6 +2243,7 @@ static int start(struct lxc_handler *handler, void* data)
 
 	execvp(arg->argv[0], arg->argv);
 	SYSERROR("Failed to exec \"%s\"", arg->argv[0]);
+	lxc_write_error_message(fd, "exec: \"%s\": %s.", arg->argv[0], strerror(errno));
 	return 0;
 }
 
diff --git a/src/lxc/start.h b/src/lxc/start.h
index 1d84325..ab72e6e 100644
--- a/src/lxc/start.h
+++ b/src/lxc/start.h
@@ -145,7 +145,7 @@ struct execute_args {
 };
 
 struct lxc_operations {
-	int (*start)(struct lxc_handler *, void *);
+	int (*start)(struct lxc_handler *, void *, int);
 	int (*post_start)(struct lxc_handler *, void *);
 };
 
diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
index f37f8a6..ec48701 100644
--- a/src/lxc/tools/lxc_start.c
+++ b/src/lxc/tools/lxc_start.c
@@ -392,6 +392,9 @@ int main(int argc, char *argv[])
 	else
 		err = c->start(c, 0, args) ? EXIT_SUCCESS : EXIT_FAILURE;
 	if (err) {
+		if (c->lxc_conf->errmsg)
+			fprintf(stderr, "%s:%s:%s:%d starting container process caused \"%s\"", c->name,
+			__FILE__, __func__, __LINE__, c->lxc_conf->errmsg);
 		ERROR("The container failed to start");
 
 		if (my_args.daemonize)
-- 
1.8.3.1