lxc/0043-support-error-report.patch

From 968c9e3e7715c080f23a1fd80c31d4bcf20d241b Mon Sep 17 00:00:00 2001
From: haozi007 <liuhao27@huawei.com>
Date: Thu, 16 Apr 2020 15:16:41 +0800
Subject: [PATCH 43/49] support error report

Signed-off-by: haozi007 <liuhao27@huawei.com>
---
 src/lxc/attach.c          | 60 +++++++++++++++++++++++++++++++++++++++++++----
 src/lxc/attach_options.h  | 12 ++++++++++
 src/lxc/cgroups/cgfsng.c  |  1 +
 src/lxc/conf.c            | 59 ++++++++++++++++++++++++++++++++++++++++++++++
 src/lxc/conf.h            |  5 ++++
 src/lxc/execute.c         |  7 ++++++
 src/lxc/lxccontainer.c    | 47 +++++++++++++++++++++++++++++++++++++
 src/lxc/start.c           | 21 +++++++++++++++++
 src/lxc/start.h           |  4 ++++
 src/lxc/tools/lxc_ls.c    |  8 +++++++
 src/lxc/tools/lxc_start.c |  5 ++++
 src/lxc/utils.c           | 16 +++++++++++++
 12 files changed, 241 insertions(+), 4 deletions(-)

diff --git a/src/lxc/attach.c b/src/lxc/attach.c
index 510c069..734cddd 100644
--- a/src/lxc/attach.c
+++ b/src/lxc/attach.c
@@ -677,9 +677,13 @@ static int attach_child_main(struct attach_clone_payload *payload)
 	bool needs_lsm = (options->namespaces & CLONE_NEWNS) &&
 			 (options->attach_flags & LXC_ATTACH_LSM) &&
 			 init_ctx->lsm_label;
-
 #ifdef HAVE_ISULAD
-		/*isulad: set system umask */
+	int msg_fd = -1;
+
+	/*isulad: record errpipe fd*/
+	msg_fd = init_ctx->container->lxc_conf->errpipe[1];
+	init_ctx->container->lxc_conf->errpipe[1] = -1;
+	/*isulad: set system umask */
 	umask(init_ctx->container->lxc_conf->umask);
 #endif
 
@@ -963,10 +967,12 @@ static int attach_child_main(struct attach_clone_payload *payload)
 	payload->ipc_socket = -EBADF;
 	lxc_proc_put_context_info(init_ctx);
 	payload->init_ctx = NULL;
-#endif
-
+	_exit(payload->exec_function(payload->exec_payload, msg_fd));
+#else
 	/* We're done, so we can now do whatever the user intended us to do. */
 	_exit(payload->exec_function(payload->exec_payload));
+#endif
+
 
 on_error:
 	lxc_put_attach_clone_payload(payload);
@@ -1331,6 +1337,25 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
 		return -1;
 	}
 
+#ifdef HAVE_ISULAD
+	/* isulad: pipdfd for get error message of child or grandchild process. */
+	if (pipe2(conf->errpipe, O_CLOEXEC) != 0) {
+		SYSERROR("Failed to init errpipe");
+		if (options->attach_flags & LXC_ATTACH_TERMINAL) {
+			lxc_terminal_delete(&terminal);
+			lxc_terminal_conf_free(&terminal);
+			if (exec_command.maincmd_fd != -1) {
+				close(exec_command.maincmd_fd);
+			}
+		}
+		close(ipc_sockets[0]);
+		close(ipc_sockets[1]);
+		free(cwd);
+		lxc_proc_put_context_info(init_ctx);
+		return -1;
+	}
+#endif
+
 	/* Create intermediate subprocess, two reasons:
 	 *       1. We can't setns() in the child itself, since we want to make
 	 *          sure we are properly attached to the pidns.
@@ -1365,6 +1390,11 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
 		/* close unneeded file descriptors */
 		close(ipc_sockets[1]);
 		free(cwd);
+#ifdef HAVE_ISULAD
+		/* isulad: close errpipe */
+		close(conf->errpipe[1]);
+		conf->errpipe[1] = -1;
+#endif
 		lxc_proc_close_ns_fd(init_ctx);
 		if (options->attach_flags & LXC_ATTACH_TERMINAL)
 			lxc_attach_terminal_close_slave(&terminal);
@@ -1398,7 +1428,11 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
 
 		/* Setup resource limits */
 		if (!lxc_list_empty(&conf->limits)) {
+#ifdef HAVE_ISULAD
+			ret = setup_resource_limits(&conf->limits, pid, -1);
+#else
 			ret = setup_resource_limits(&conf->limits, pid);
+#endif
 			if (ret < 0)
 				goto on_error;
 		}
@@ -1561,6 +1595,12 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
 	/* close unneeded file descriptors */
 	close_prot_errno_disarm(ipc_sockets[0]);
 
+#ifdef HAVE_ISULAD
+	/* isulad: close errpipe */
+	close(conf->errpipe[0]);
+	conf->errpipe[0] = -1;
+#endif
+
 	if (options->attach_flags & LXC_ATTACH_TERMINAL) {
 		lxc_attach_terminal_close_master(&terminal);
 		lxc_attach_terminal_close_peer(&terminal);
@@ -1665,7 +1705,11 @@ int lxc_attach(struct lxc_container *container, lxc_attach_exec_t exec_function,
 	_exit(EXIT_SUCCESS);
 }
 
+#ifdef HAVE_ISULAD
+int lxc_attach_run_command(void *payload, int msg_fd)
+#else
 int lxc_attach_run_command(void *payload)
+#endif
 {
 	int ret = -1;
 	lxc_attach_command_t *cmd = payload;
@@ -1681,11 +1725,19 @@ int lxc_attach_run_command(void *payload)
 			break;
 		}
 	}
+#ifdef HAVE_ISULAD
+	/* isulad: write error messages */
+	lxc_write_error_message(msg_fd, "exec: \"%s\": %s.", cmd->program, strerror(errno));
+#endif
 
 	return log_error_errno(ret, errno, "Failed to exec \"%s\"", cmd->program);
 }
 
+#ifdef HAVE_ISULAD
+int lxc_attach_run_shell(void* payload, int msg_fd)
+#else
 int lxc_attach_run_shell(void* payload)
+#endif
 {
 	__do_free char *buf = NULL;
 	uid_t uid;
diff --git a/src/lxc/attach_options.h b/src/lxc/attach_options.h
index 5f01739..d5d4f44 100644
--- a/src/lxc/attach_options.h
+++ b/src/lxc/attach_options.h
@@ -49,7 +49,11 @@ enum {
  *
  * \return Function should return \c 0 on success, and any other value to denote failure.
  */
+#ifdef HAVE_ISULAD
+typedef int (*lxc_attach_exec_t)(void* payload, int msg_fd);
+#else
 typedef int (*lxc_attach_exec_t)(void* payload);
+#endif
 
 /*!
  * LXC attach options for \ref lxc_container \c attach().
@@ -153,7 +157,11 @@ typedef struct lxc_attach_command_t {
  *
  * \return \c -1 on error, exit code of lxc_attach_command_t program on success.
  */
+#ifdef HAVE_ISULAD
+extern int lxc_attach_run_command(void* payload, int msg_fd);
+#else
 extern int lxc_attach_run_command(void* payload);
+#endif
 
 /*!
  * \brief Run a shell command in the container.
@@ -162,7 +170,11 @@ extern int lxc_attach_run_command(void* payload);
  *
  * \return Exit code of shell.
  */
+#ifdef HAVE_ISULAD
+extern int lxc_attach_run_shell(void* payload, int msg_fd);
+#else
 extern int lxc_attach_run_shell(void* payload);
+#endif
 
 #ifdef  __cplusplus
 }
diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
index 1e1df3b..002f051 100644
--- a/src/lxc/cgroups/cgfsng.c
+++ b/src/lxc/cgroups/cgfsng.c
@@ -4233,6 +4233,7 @@ struct cgroup_ops *cgfsng_ops_init(struct lxc_conf *conf)
 
 	cgfsng_ops->data_init = cgfsng_data_init;
 #ifdef HAVE_ISULAD
+	cgfsng_ops->errfd = conf ? conf->errpipe[1] : -1;
 	cgfsng_ops->payload_destroy = isulad_cgfsng_payload_destroy;
 #else
 	cgfsng_ops->payload_destroy = cgfsng_payload_destroy;
diff --git a/src/lxc/conf.c b/src/lxc/conf.c
index e8ee749..d7a78bd 100644
--- a/src/lxc/conf.c
+++ b/src/lxc/conf.c
@@ -2275,8 +2275,15 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
 
 	if (hasmntopt(mntent, "create=dir")) {
 		ret = mkdir_p(path, 0755);
+#ifdef HAVE_ISULAD
+		if (ret < 0 && errno != EEXIST) {
+			lxc_write_error_message(rootfs->errfd, "%s:%d: mkdir %s: %s.", __FILE__, __LINE__, path, strerror(errno));
+			return log_error_errno(-1, errno, "Failed to create directory \"%s\"", path);
+		}
+#else
 		if (ret < 0 && errno != EEXIST)
 			return log_error_errno(-1, errno, "Failed to create directory \"%s\"", path);
+#endif
 	}
 
 	if (!hasmntopt(mntent, "create=file"))
@@ -2293,12 +2300,26 @@ static int mount_entry_create_dir_file(const struct mntent *mntent,
 	p2 = dirname(p1);
 
 	ret = mkdir_p(p2, 0755);
+#ifdef HAVE_ISULAD
+	if (ret < 0 && errno != EEXIST) {
+		lxc_write_error_message(rootfs->errfd, "%s:%d: mkdir %s: %s.", __FILE__, __LINE__, path, strerror(errno));
+		return log_error_errno(-1, errno, "Failed to create directory \"%s\"", path);
+	}
+#else
 	if (ret < 0 && errno != EEXIST)
 		return log_error_errno(-1, errno, "Failed to create directory \"%s\"", path);
+#endif
 
 	ret = mknod(path, S_IFREG | 0000, 0);
+#ifdef HAVE_ISULAD
+	if (ret < 0 && errno != EEXIST) {
+		lxc_write_error_message(rootfs->errfd, "%s:%d: open %s: %s.", __FILE__, __LINE__, path, strerror(errno));
+		return -errno;
+	}
+#else
 	if (ret < 0 && errno != EEXIST)
 		return -errno;
+#endif
 
 	return 0;
 }
@@ -2960,7 +2981,11 @@ static int parse_resource(const char *res)
 	return resid;
 }
 
+#ifdef HAVE_ISULAD
+int setup_resource_limits(struct lxc_list *limits, pid_t pid, int errfd)
+#else
 int setup_resource_limits(struct lxc_list *limits, pid_t pid)
+#endif
 {
 	int resid;
 	struct lxc_list *it;
@@ -2974,8 +2999,17 @@ int setup_resource_limits(struct lxc_list *limits, pid_t pid)
 			return log_error(-1, "Unknown resource %s", lim->resource);
 
 #if HAVE_PRLIMIT || HAVE_PRLIMIT64
+#if HAVE_ISULAD
+		if (prlimit(pid, resid, &lim->limit, NULL) != 0) {
+			lxc_write_error_message(errfd, "%s:%d: Failed to set limit %s %lu %lu: %s.",
+					__FILE__, __LINE__, lim->resource,
+					lim->limit.rlim_cur, lim->limit.rlim_max, strerror(errno));
+			return log_error_errno(-1, errno, "Failed to set limit %s", lim->resource);
+		}
+#else
 		if (prlimit(pid, resid, &lim->limit, NULL) != 0)
 			return log_error_errno(-1, errno, "Failed to set limit %s", lim->resource);
+#endif
 
 		TRACE("Setup \"%s\" limit", lim->resource);
 #else
@@ -3134,6 +3168,9 @@ struct lxc_conf *lxc_conf_init(void)
 	new->console.pipes[2][0] = -1;
 	new->console.pipes[2][1] = -1;
 	lxc_list_init(&new->console.fifos);
+	new->errmsg = NULL;
+	new->errpipe[0] = -1;
+	new->errpipe[1] = -1;
 #endif
 
 	return new;
@@ -4534,8 +4571,16 @@ int lxc_setup(struct lxc_handler *handler)
 	char *keyring_context = NULL;
 
 	ret = lxc_setup_rootfs_prepare_root(lxc_conf, name, lxcpath);
+#ifdef HAVE_ISULAD
+	if (ret < 0) {
+		lxc_write_error_message(lxc_conf->errpipe[1], "%s:%d: failed to setup rootfs %s.",
+				__FILE__, __LINE__, lxc_conf->rootfs.path);
+		return log_error(-1, "Failed to setup rootfs");
+	}
+#else
 	if (ret < 0)
 		return log_error(-1, "Failed to setup rootfs");
+#endif
 
 	if (handler->nsfd[LXC_NS_UTS] == -EBADF) {
 		ret = setup_utsname(lxc_conf->utsname);
@@ -5243,6 +5288,8 @@ void lxc_conf_free(struct lxc_conf *conf)
 	lxc_clear_populate_devices(conf);
 	lxc_clear_rootfs_masked_paths(conf);
 	lxc_clear_rootfs_ro_paths(conf);
+	free(conf->errmsg);
+	lxc_close_error_pipe(conf->errpipe);
 #endif
 	free(conf);
 }
@@ -6127,4 +6174,16 @@ int lxc_clear_rootfs_ro_paths(struct lxc_conf *c)
 	return 0;
 }
 
+/*isulad: close error pipe */
+void lxc_close_error_pipe(int *errpipe)
+{
+	if (errpipe[0] >= 0) {
+		close(errpipe[0]);
+		errpipe[0] = -1;
+	}
+	if (errpipe[1] >= 0) {
+		close(errpipe[1]);
+		errpipe[1] = -1;
+	}
+}
 #endif
diff --git a/src/lxc/conf.h b/src/lxc/conf.h
index 7b6fd3b..4b6409e 100644
--- a/src/lxc/conf.h
+++ b/src/lxc/conf.h
@@ -506,7 +506,11 @@ extern int lxc_setup_rootfs_prepare_root(struct lxc_conf *conf,
 					 const char *name, const char *lxcpath);
 extern int lxc_setup(struct lxc_handler *handler);
 extern int lxc_setup_parent(struct lxc_handler *handler);
+#ifdef HAVE_ISULAD
+extern int setup_resource_limits(struct lxc_list *limits, pid_t pid, int errfd);
+#else
 extern int setup_resource_limits(struct lxc_list *limits, pid_t pid);
+#endif
 extern int find_unmapped_nsid(const struct lxc_conf *conf, enum idtype idtype);
 extern int mapped_hostid(unsigned id, const struct lxc_conf *conf,
 			 enum idtype idtype);
@@ -557,5 +561,6 @@ int lxc_clear_rootfs_masked_paths(struct lxc_conf *c);
 int lxc_clear_rootfs_ro_paths(struct lxc_conf *c);
 int lxc_drop_caps(struct lxc_conf *conf);
 int run_oci_hooks(const char *name, const char *hookname, struct lxc_conf *conf, const char *lxcpath);
+void lxc_close_error_pipe(int *errpipe);
 #endif
 #endif /* __LXC_CONF_H */
diff --git a/src/lxc/execute.c b/src/lxc/execute.c
index 59ff604..16c0fed 100644
--- a/src/lxc/execute.c
+++ b/src/lxc/execute.c
@@ -19,7 +19,11 @@
 
 lxc_log_define(execute, start);
 
+#ifdef HAVE_ISULAD
+static int execute_start(struct lxc_handler *handler, void* data, int fd)
+#else
 static int execute_start(struct lxc_handler *handler, void* data)
+#endif
 {
 	int argc_add, j;
 	char **argv;
@@ -71,6 +75,9 @@ static int execute_start(struct lxc_handler *handler, void* data)
 		execvp(argv[0], argv);
 	SYSERROR("Failed to exec %s", argv[0]);
 
+#ifdef HAVE_ISULAD
+	lxc_write_error_message(fd, "Failed to exec: \"%s\": %s.", argv[0], strerror(errno));
+#endif
 	free(argv);
 out1:
 	return 1;
diff --git a/src/lxc/lxccontainer.c b/src/lxc/lxccontainer.c
index 818848a..ed09a59 100644
--- a/src/lxc/lxccontainer.c
+++ b/src/lxc/lxccontainer.c
@@ -972,6 +972,8 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 	char **init_cmd = NULL;
 #ifdef HAVE_ISULAD
 	int keepfds[] = {-1, -1, -1, -1, -1};
+	ssize_t size_read;
+	char errbuf[BUFSIZ + 1] = {0};
 #else
 	int keepfds[3] = {-1, -1, -1};
 #endif
@@ -1046,10 +1048,23 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		char title[2048];
 		pid_t pid_first, pid_second;
 
+#ifdef HAVE_ISULAD
+		//isulad: pipdfd for get error message of child or grandchild process.
+		if (pipe2(conf->errpipe, O_CLOEXEC) != 0) {
+			SYSERROR("Failed to init errpipe");
+			free_init_cmd(init_cmd);
+			lxc_free_handler(handler);
+			return false;
+		}
+#endif
+
 		pid_first = fork();
 		if (pid_first < 0) {
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
+#ifdef HAVE_ISULAD
+			lxc_close_error_pipe(conf->errpipe);
+#endif
 			return false;
 		}
 
@@ -1059,11 +1074,25 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 			 * the PID file, child will do the free and unlink.
 			 */
 			c->pidfile = NULL;
+#ifdef HAVE_ISULAD
+			close(conf->errpipe[1]);
+			conf->errpipe[1] = -1;
+#endif
 
 			/* Wait for container to tell us whether it started
 			 * successfully.
 			 */
 			started = wait_on_daemonized_start(handler, pid_first);
+#ifdef HAVE_ISULAD
+			if (!started) {
+				size_read = read(conf->errpipe[0], errbuf, BUFSIZ);
+				if (size_read > 0) {
+					conf->errmsg = safe_strdup(errbuf);
+				}
+			}
+			close(conf->errpipe[0]);
+			conf->errpipe[0] = -1;
+#endif
 
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
@@ -1099,6 +1128,9 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		if (pid_second != 0) {
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
+#ifdef HAVE_ISULAD
+			lxc_close_error_pipe(conf->errpipe);
+#endif
 			_exit(EXIT_SUCCESS);
 		}
 
@@ -1114,6 +1146,11 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		keepfds[0] = handler->conf->maincmd_fd;
 		keepfds[1] = handler->state_socket_pair[0];
 		keepfds[2] = handler->state_socket_pair[1];
+#ifdef HAVE_ISULAD
+		keepfds[4] = conf->errpipe[1];
+		close(conf->errpipe[0]);
+		conf->errpipe[0] = -1;
+#endif
 		ret = lxc_check_inherited(conf, true, keepfds,
 					  sizeof(keepfds) / sizeof(keepfds[0]));
 		if (ret < 0)
@@ -1148,6 +1185,9 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		if (w < 0 || (size_t)w >= sizeof(pidstr)) {
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
+#ifdef HAVE_ISULAD
+			lxc_close_error_pipe(conf->errpipe);
+#endif
 
 			SYSERROR("Failed to write monitor pid to \"%s\"", c->pidfile);
 
@@ -1161,6 +1201,9 @@ static bool do_lxcapi_start(struct lxc_container *c, int useinit, char * const a
 		if (ret < 0) {
 			free_init_cmd(init_cmd);
 			lxc_free_handler(handler);
+#ifdef HAVE_ISULAD
+			lxc_close_error_pipe(conf->errpipe);
+#endif
 
 			SYSERROR("Failed to write monitor pid to \"%s\"", c->pidfile);
 
@@ -1224,6 +1267,9 @@ reboot:
 	if (conf->exit_fd >= 0) {
 		keepfds[3] = conf->exit_fd;
 	}
+	/* isulad: keep errpipe fd */
+	if (c->daemonize)
+		keepfds[4] = conf->errpipe[1];
 #endif
 
 	ret = lxc_check_inherited(conf, c->daemonize, keepfds,
@@ -3250,6 +3296,7 @@ static bool container_destroy(struct lxc_container *c,
 			ERROR("Sprintf failed");
 			goto out;
 		}
+		c->error_string = safe_strdup(msg);
 #endif
 		goto out;
 	}
diff --git a/src/lxc/start.c b/src/lxc/start.c
index 0bc1143..134235f 100644
--- a/src/lxc/start.c
+++ b/src/lxc/start.c
@@ -836,6 +836,9 @@ int lxc_init(const char *name, struct lxc_handler *handler)
 	ret = lxc_terminal_setup(conf);
 	if (ret < 0) {
 		ERROR("Failed to create console");
+#ifdef HAVE_ISULAD
+		lxc_write_error_message(conf->errpipe[1], "Failed to create console for container \"%s\".", name);
+#endif
 		goto out_restore_sigmask;
 	}
 	TRACE("Created console");
@@ -1470,6 +1473,9 @@ static int do_start(void *data)
 	/* Setup the container, ip, names, utsname, ... */
 	ret = lxc_setup(handler);
 	if (ret < 0) {
+#ifdef HAVE_ISULAD
+		lxc_write_error_message(handler->conf->errpipe[1], "Failed to setup lxc, please check the config file.");
+#endif
 		ERROR("Failed to setup container \"%s\"", handler->name);
 		goto out_warn_father;
 	}
@@ -1763,7 +1769,11 @@ static int do_start(void *data)
 	/* After this call, we are in error because this ops should not return
 	 * as it execs.
 	 */
+#ifdef HAVE_ISULAD
+	handler->ops->start(handler, handler->data, handler->daemonize ? handler->conf->errpipe[1] : -1);
+#else
 	handler->ops->start(handler, handler->data);
+#endif
 
 out_warn_father:
 	/* We want the parent to know something went wrong, so we return a
@@ -2246,7 +2256,11 @@ static int lxc_spawn(struct lxc_handler *handler)
 		goto out_delete_net;
 
 	if (!lxc_list_empty(&conf->limits)) {
+#ifdef HAVE_ISULAD
+		ret = setup_resource_limits(&conf->limits, handler->pid, conf->errpipe[1]);
+#else
 		ret = setup_resource_limits(&conf->limits, handler->pid);
+#endif
 		if (ret < 0) {
 			ERROR("Failed to setup resource limits");
 			goto out_delete_net;
@@ -2652,7 +2666,11 @@ struct start_args {
 	char *const *argv;
 };
 
+#ifdef HAVE_ISULAD
+static int start(struct lxc_handler *handler, void* data, int fd)
+#else
 static int start(struct lxc_handler *handler, void* data)
+#endif
 {
 	struct start_args *arg = data;
 
@@ -2660,6 +2678,9 @@ static int start(struct lxc_handler *handler, void* data)
 
 	execvp(arg->argv[0], arg->argv);
 	SYSERROR("Failed to exec \"%s\"", arg->argv[0]);
+#ifdef HAVE_ISULAD
+	lxc_write_error_message(fd, "exec: \"%s\": %s.", arg->argv[0], strerror(errno));
+#endif
 	return 0;
 }
 
diff --git a/src/lxc/start.h b/src/lxc/start.h
index cea37bc..ebeeb72 100644
--- a/src/lxc/start.h
+++ b/src/lxc/start.h
@@ -141,7 +141,11 @@ struct execute_args {
 };
 
 struct lxc_operations {
+#ifdef HAVE_ISULAD
+	int (*start)(struct lxc_handler *, void *, int);
+#else
 	int (*start)(struct lxc_handler *, void *);
+#endif
 	int (*post_start)(struct lxc_handler *, void *);
 };
 
diff --git a/src/lxc/tools/lxc_ls.c b/src/lxc/tools/lxc_ls.c
index 4be8564..e601f9d 100644
--- a/src/lxc/tools/lxc_ls.c
+++ b/src/lxc/tools/lxc_ls.c
@@ -106,7 +106,11 @@ struct wrapargs {
 /*
  * Takes struct wrapargs as argument.
  */
+#ifdef HAVE_ISULAD
+static int ls_get_wrapper(void *wrap, int msgfd);
+#else
 static int ls_get_wrapper(void *wrap);
+#endif
 
 /*
  * To calculate swap usage we should not simply check memory.usage_in_bytes and
@@ -1005,7 +1009,11 @@ static int my_parser(struct lxc_arguments *args, int c, char *arg)
 	return 0;
 }
 
+#ifdef HAVE_ISULAD
+static int ls_get_wrapper(void *wrap, int msgfd)
+#else
 static int ls_get_wrapper(void *wrap)
+#endif
 {
 	int ret = -1;
 	size_t len = 0;
diff --git a/src/lxc/tools/lxc_start.c b/src/lxc/tools/lxc_start.c
index b430a1e..4f2c8af 100644
--- a/src/lxc/tools/lxc_start.c
+++ b/src/lxc/tools/lxc_start.c
@@ -392,6 +392,11 @@ int main(int argc, char *argv[])
 	else
 		err = c->start(c, 0, args) ? EXIT_SUCCESS : EXIT_FAILURE;
 	if (err) {
+#ifdef HAVE_ISULAD
+		if (c->lxc_conf->errmsg)
+			fprintf(stderr, "%s:%s:%s:%d starting container process caused \"%s\"", c->name,
+					__FILE__, __func__, __LINE__, c->lxc_conf->errmsg);
+#endif
 		ERROR("The container failed to start");
 
 		if (my_args.daemonize)
diff --git a/src/lxc/utils.c b/src/lxc/utils.c
index 810b7fe..4e418fb 100644
--- a/src/lxc/utils.c
+++ b/src/lxc/utils.c
@@ -73,6 +73,9 @@ static int _recursive_rmdir(const char *dirname, dev_t pdev,
 	int ret;
 	struct dirent *direntp;
 	char pathname[PATH_MAX];
+#ifdef HAVE_ISULAD
+	int saved_errno = 0;
+#endif
 
 	dir = opendir(dirname);
 	if (!dir)
@@ -135,6 +138,11 @@ static int _recursive_rmdir(const char *dirname, dev_t pdev,
 		} else {
 			ret = unlink(pathname);
 			if (ret < 0) {
+#ifdef HAVE_ISULAD
+				if (saved_errno == 0) {
+					saved_errno = errno;
+				}
+#endif
 				__do_close int fd = -EBADF;
 
 				fd = open(pathname, O_RDONLY | O_CLOEXEC | O_NONBLOCK);
@@ -160,10 +168,18 @@ static int _recursive_rmdir(const char *dirname, dev_t pdev,
 	}
 
 	if (rmdir(dirname) < 0 && !btrfs_try_remove_subvol(dirname) && !hadexclude) {
+#ifdef HAVE_ISULAD
+		if (saved_errno == 0) {
+			saved_errno = errno;
+		}
+#endif
 		SYSERROR("Failed to delete \"%s\"", dirname);
 		failed = 1;
 	}
 
+#ifdef HAVE_ISULAD
+	errno = saved_errno;
+#endif
 	return failed ? -1 : 0;
 }
 
-- 
1.8.3.1