From 4c68aefb92800112f48aa73a55f80b368b09c69f Mon Sep 17 00:00:00 2001
From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
Date: Wed, 20 Jul 2022 11:00:13 +0800
Subject: [PATCH] refactor patch code of json

Signed-off-by: zhangxiaoyu <zhangxiaoyu58@huawei.com>
---
 src/lxc/cgroups/isulad_cgfsng.c  | 3223 ++++++++++++++++++++++++++++++
 src/lxc/json/defs.c              |  205 ++
 src/lxc/json/defs.h              |   37 +
 src/lxc/json/json_common.c       | 1153 +++++++++++
 src/lxc/json/json_common.h       |  185 ++
 src/lxc/json/logger_json_file.c  |  246 +++
 src/lxc/json/logger_json_file.h  |   45 +
 src/lxc/json/oci_runtime_hooks.c |   52 +
 src/lxc/json/oci_runtime_hooks.h |   15 +
 src/lxc/json/oci_runtime_spec.c  |  195 ++
 src/lxc/json/oci_runtime_spec.h  |   37 +
 src/lxc/json/read-file.h         |   11 +
 12 files changed, 5404 insertions(+)
 create mode 100644 src/lxc/cgroups/isulad_cgfsng.c
 create mode 100644 src/lxc/json/defs.c
 create mode 100644 src/lxc/json/defs.h
 create mode 100755 src/lxc/json/json_common.c
 create mode 100755 src/lxc/json/json_common.h
 create mode 100644 src/lxc/json/logger_json_file.c
 create mode 100644 src/lxc/json/logger_json_file.h
 create mode 100644 src/lxc/json/oci_runtime_hooks.c
 create mode 100644 src/lxc/json/oci_runtime_hooks.h
 create mode 100644 src/lxc/json/oci_runtime_spec.c
 create mode 100644 src/lxc/json/oci_runtime_spec.h
 create mode 100644 src/lxc/json/read-file.h

diff --git a/src/lxc/cgroups/isulad_cgfsng.c b/src/lxc/cgroups/isulad_cgfsng.c
new file mode 100644
index 0000000..8a9656a
--- /dev/null
+++ b/src/lxc/cgroups/isulad_cgfsng.c
@@ -0,0 +1,3223 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+ * Author: lifeng
+ * Create: 2020-11-02
+ * Description: provide container definition
+ * lxc: linux Container library
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+  ******************************************************************************/
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <ctype.h>
+#include <dirent.h>
+#include <errno.h>
+#include <grp.h>
+#include <linux/kdev_t.h>
+#include <linux/types.h>
+#include <poll.h>
+#include <signal.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "af_unix.h"
+#include "caps.h"
+#include "cgroup.h"
+#include "cgroup2_devices.h"
+#include "cgroup_utils.h"
+#include "commands.h"
+#include "conf.h"
+#include "config.h"
+#include "log.h"
+#include "macro.h"
+#include "mainloop.h"
+#include "memory_utils.h"
+#include "storage/storage.h"
+#include "utils.h"
+
+#ifndef HAVE_STRLCPY
+#include "include/strlcpy.h"
+#endif
+
+#ifndef HAVE_STRLCAT
+#include "include/strlcat.h"
+#endif
+
+lxc_log_define(isulad_cgfsng, cgroup);
+
+/* Given a pointer to a null-terminated array of pointers, realloc to add one
+ * entry, and point the new entry to NULL. Do not fail. Return the index to the
+ * second-to-last entry - that is, the one which is now available for use
+ * (keeping the list null-terminated).
+ */
+static int append_null_to_list(void ***list)
+{
+	int newentry = 0;
+
+	if (*list)
+		for (; (*list)[newentry]; newentry++)
+			;
+
+	*list = must_realloc(*list, (newentry + 2) * sizeof(void **));
+	(*list)[newentry + 1] = NULL;
+	return newentry;
+}
+
+/* Given a null-terminated array of strings, check whether @entry is one of the
+ * strings.
+ */
+static bool string_in_list(char **list, const char *entry)
+{
+	if (!list)
+		return false;
+
+	for (int i = 0; list[i]; i++)
+		if (strcmp(list[i], entry) == 0)
+			return true;
+
+	return false;
+}
+
+/* Return a copy of @entry prepending "name=", i.e.  turn "systemd" into
+ * "name=systemd". Do not fail.
+ */
+static char *cg_legacy_must_prefix_named(char *entry)
+{
+	size_t len;
+	char *prefixed;
+
+	len = strlen(entry);
+	prefixed = must_realloc(NULL, len + 6);
+
+	memcpy(prefixed, "name=", STRLITERALLEN("name="));
+	memcpy(prefixed + STRLITERALLEN("name="), entry, len);
+	prefixed[len + 5] = '\0';
+
+	return prefixed;
+}
+
+/* Append an entry to the clist. Do not fail. @clist must be NULL the first time
+ * we are called.
+ *
+ * We also handle named subsystems here. Any controller which is not a kernel
+ * subsystem, we prefix "name=". Any which is both a kernel and named subsystem,
+ * we refuse to use because we're not sure which we have here.
+ * (TODO: We could work around this in some cases by just remounting to be
+ * unambiguous, or by comparing mountpoint contents with current cgroup.)
+ *
+ * The last entry will always be NULL.
+ */
+static void must_append_controller(char **klist, char **nlist, char ***clist,
+				   char *entry)
+{
+	int newentry;
+	char *copy;
+
+	if (string_in_list(klist, entry) && string_in_list(nlist, entry)) {
+		ERROR("Refusing to use ambiguous controller \"%s\"", entry);
+		ERROR("It is both a named and kernel subsystem");
+		return;
+	}
+
+	newentry = append_null_to_list((void ***)clist);
+
+	if (strncmp(entry, "name=", 5) == 0)
+		copy = must_copy_string(entry);
+	else if (string_in_list(klist, entry))
+		copy = must_copy_string(entry);
+	else
+		copy = cg_legacy_must_prefix_named(entry);
+
+	(*clist)[newentry] = copy;
+}
+
+/* Given a handler's cgroup data, return the struct hierarchy for the controller
+ * @c, or NULL if there is none.
+ */
+struct hierarchy *get_hierarchy(struct cgroup_ops *ops, const char *controller)
+{
+	if (!ops->hierarchies)
+		return log_trace_errno(NULL, errno, "There are no useable cgroup controllers");
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		if (!controller) {
+			/* This is the empty unified hierarchy. */
+			if (ops->hierarchies[i]->controllers &&
+			    !ops->hierarchies[i]->controllers[0])
+				return ops->hierarchies[i];
+			continue;
+		} else if (pure_unified_layout(ops) &&
+			   strcmp(controller, "devices") == 0) {
+			if (ops->unified->bpf_device_controller)
+				return ops->unified;
+			break;
+		}
+
+		if (string_in_list(ops->hierarchies[i]->controllers, controller))
+			return ops->hierarchies[i];
+	}
+
+	if (controller)
+		WARN("There is no useable %s controller", controller);
+	else
+		WARN("There is no empty unified cgroup hierarchy");
+
+	return ret_set_errno(NULL, ENOENT);
+}
+
+#define BATCH_SIZE 50
+static void batch_realloc(char **mem, size_t oldlen, size_t newlen)
+{
+	int newbatches = (newlen / BATCH_SIZE) + 1;
+	int oldbatches = (oldlen / BATCH_SIZE) + 1;
+
+	if (!*mem || newbatches > oldbatches)
+		*mem = must_realloc(*mem, newbatches * BATCH_SIZE);
+}
+
+static void append_line(char **dest, size_t oldlen, char *new, size_t newlen)
+{
+	size_t full = oldlen + newlen;
+
+	batch_realloc(dest, oldlen, full + 1);
+
+	memcpy(*dest + oldlen, new, newlen + 1);
+}
+
+/* Slurp in a whole file */
+static char *read_file(const char *fnam)
+{
+	__do_free char *buf = NULL, *line = NULL;
+	__do_fclose FILE *f = NULL;
+	size_t len = 0, fulllen = 0;
+	int linelen;
+
+	f = fopen(fnam, "re");
+	if (!f)
+		return NULL;
+
+	while ((linelen = getline(&line, &len, f)) != -1) {
+		append_line(&buf, fulllen, line, linelen);
+		fulllen += linelen;
+	}
+
+	return move_ptr(buf);
+}
+
+static inline bool is_unified_hierarchy(const struct hierarchy *h)
+{
+	return h->version == CGROUP2_SUPER_MAGIC;
+}
+
+/* Given two null-terminated lists of strings, return true if any string is in
+ * both.
+ */
+static bool controller_lists_intersect(char **l1, char **l2)
+{
+	if (!l1 || !l2)
+		return false;
+
+	for (int i = 0; l1[i]; i++)
+		if (string_in_list(l2, l1[i]))
+			return true;
+
+	return false;
+}
+
+/* For a null-terminated list of controllers @clist, return true if any of those
+ * controllers is already listed the null-terminated list of hierarchies @hlist.
+ * Realistically, if one is present, all must be present.
+ */
+static bool controller_list_is_dup(struct hierarchy **hlist, char **clist)
+{
+	if (!hlist)
+		return false;
+
+	for (int i = 0; hlist[i]; i++)
+		if (controller_lists_intersect(hlist[i]->controllers, clist))
+			return true;
+
+	return false;
+}
+
+/* Return true if the controller @entry is found in the null-terminated list of
+ * hierarchies @hlist.
+ */
+static bool controller_found(struct hierarchy **hlist, char *entry)
+{
+	if (!hlist)
+		return false;
+
+	for (int i = 0; hlist[i]; i++)
+		if (string_in_list(hlist[i]->controllers, entry))
+			return true;
+
+	return false;
+}
+
+/* Return true if all of the controllers which we require have been found.  The
+ * required list is  freezer and anything in lxc.cgroup.use.
+ */
+static bool all_controllers_found(struct cgroup_ops *ops)
+{
+	struct hierarchy **hlist;
+
+	if (!ops->cgroup_use)
+		return true;
+
+	hlist = ops->hierarchies;
+	for (char **cur = ops->cgroup_use; cur && *cur; cur++)
+		if (!controller_found(hlist, *cur))
+			return log_error(false, "No %s controller mountpoint found", *cur);
+
+	return true;
+}
+
+/* Get the controllers from a mountinfo line There are other ways we could get
+ * this info. For lxcfs, field 3 is /cgroup/controller-list. For cgroupfs, we
+ * could parse the mount options. But we simply assume that the mountpoint must
+ * be /sys/fs/cgroup/controller-list
+ */
+static char **cg_hybrid_get_controllers(char **klist, char **nlist, char *line,
+					int type)
+{
+	/* The fourth field is /sys/fs/cgroup/comma-delimited-controller-list
+	 * for legacy hierarchies.
+	 */
+	__do_free_string_list char **aret = NULL;
+	int i;
+	char *p2, *tok;
+	char *p = line, *sep = ",";
+
+	for (i = 0; i < 4; i++) {
+		p = strchr(p, ' ');
+		if (!p)
+			return NULL;
+		p++;
+	}
+
+	/* Note, if we change how mountinfo works, then our caller will need to
+	 * verify /sys/fs/cgroup/ in this field.
+	 */
+	if (strncmp(p, DEFAULT_CGROUP_MOUNTPOINT "/", 15) != 0)
+		return log_warn(NULL, "Found hierarchy not under " DEFAULT_CGROUP_MOUNTPOINT ": \"%s\"", p);
+
+	p += 15;
+	p2 = strchr(p, ' ');
+	if (!p2)
+		return log_error(NULL, "Corrupt mountinfo");
+	*p2 = '\0';
+
+	if (type == CGROUP_SUPER_MAGIC) {
+		__do_free char *dup = NULL;
+
+		/* strdup() here for v1 hierarchies. Otherwise
+		 * lxc_iterate_parts() will destroy mountpoints such as
+		 * "/sys/fs/cgroup/cpu,cpuacct".
+		 */
+		dup = must_copy_string(p);
+		if (!dup)
+			return NULL;
+
+		lxc_iterate_parts (tok, dup, sep)
+			must_append_controller(klist, nlist, &aret, tok);
+	}
+	*p2 = ' ';
+
+	return move_ptr(aret);
+}
+
+static char **cg_unified_make_empty_controller(void)
+{
+	__do_free_string_list char **aret = NULL;
+	int newentry;
+
+	newentry = append_null_to_list((void ***)&aret);
+	aret[newentry] = NULL;
+	return move_ptr(aret);
+}
+
+static char **cg_unified_get_controllers(const char *file)
+{
+	__do_free char *buf = NULL;
+	__do_free_string_list char **aret = NULL;
+	char *sep = " \t\n";
+	char *tok;
+
+	buf = read_file(file);
+	if (!buf)
+		return NULL;
+
+	lxc_iterate_parts(tok, buf, sep) {
+		int newentry;
+		char *copy;
+
+		newentry = append_null_to_list((void ***)&aret);
+		copy = must_copy_string(tok);
+		aret[newentry] = copy;
+	}
+
+	return move_ptr(aret);
+}
+
+static struct hierarchy *add_hierarchy(struct hierarchy ***h, char **clist, char *mountpoint,
+				       char *container_base_path, int type)
+{
+	struct hierarchy *new;
+	int newentry;
+
+	new = zalloc(sizeof(*new));
+	new->controllers = clist;
+	new->mountpoint = mountpoint;
+	new->container_base_path = container_base_path;
+	new->version = type;
+	new->cgfd_con = -EBADF;
+	new->cgfd_mon = -EBADF;
+
+	newentry = append_null_to_list((void ***)h);
+	(*h)[newentry] = new;
+	return new;
+}
+
+/* Get a copy of the mountpoint from @line, which is a line from
+ * /proc/self/mountinfo.
+ */
+static char *cg_hybrid_get_mountpoint(char *line)
+{
+	char *p = line, *sret = NULL;
+	size_t len;
+	char *p2;
+
+	for (int i = 0; i < 4; i++) {
+		p = strchr(p, ' ');
+		if (!p)
+			return NULL;
+		p++;
+	}
+
+	if (strncmp(p, DEFAULT_CGROUP_MOUNTPOINT "/", 15) != 0)
+		return NULL;
+
+	p2 = strchr(p + 15, ' ');
+	if (!p2)
+		return NULL;
+	*p2 = '\0';
+
+	len = strlen(p);
+	sret = must_realloc(NULL, len + 1);
+	memcpy(sret, p, len);
+	sret[len] = '\0';
+
+	return sret;
+}
+
+/* Given a multi-line string, return a null-terminated copy of the current line. */
+static char *copy_to_eol(char *p)
+{
+	char *p2, *sret;
+	size_t len;
+
+	p2 = strchr(p, '\n');
+	if (!p2)
+		return NULL;
+
+	len = p2 - p;
+	sret = must_realloc(NULL, len + 1);
+	memcpy(sret, p, len);
+	sret[len] = '\0';
+
+	return sret;
+}
+
+/* cgline: pointer to character after the first ':' in a line in a \n-terminated
+ * /proc/self/cgroup file. Check whether controller c is present.
+ */
+static bool controller_in_clist(char *cgline, char *c)
+{
+	__do_free char *tmp = NULL;
+	char *tok, *eol;
+	size_t len;
+
+	eol = strchr(cgline, ':');
+	if (!eol)
+		return false;
+
+	len = eol - cgline;
+	tmp = must_realloc(NULL, len + 1);
+	memcpy(tmp, cgline, len);
+	tmp[len] = '\0';
+
+	lxc_iterate_parts(tok, tmp, ",")
+		if (strcmp(tok, c) == 0)
+			return true;
+
+	return false;
+}
+
+/* @basecginfo is a copy of /proc/$$/cgroup. Return the current cgroup for
+ * @controller.
+ */
+static char *cg_hybrid_get_current_cgroup(char *basecginfo, char *controller,
+					  int type)
+{
+	char *p = basecginfo;
+
+	for (;;) {
+		bool is_cgv2_base_cgroup = false;
+
+		/* cgroup v2 entry in "/proc/<pid>/cgroup": "0::/some/path" */
+		if ((type == CGROUP2_SUPER_MAGIC) && (*p == '0'))
+			is_cgv2_base_cgroup = true;
+
+		p = strchr(p, ':');
+		if (!p)
+			return NULL;
+		p++;
+
+		if (is_cgv2_base_cgroup || (controller && controller_in_clist(p, controller))) {
+			p = strchr(p, ':');
+			if (!p)
+				return NULL;
+			p++;
+			return copy_to_eol(p);
+		}
+
+		p = strchr(p, '\n');
+		if (!p)
+			return NULL;
+		p++;
+	}
+}
+
+static void must_append_string(char ***list, char *entry)
+{
+	int newentry;
+	char *copy;
+
+	newentry = append_null_to_list((void ***)list);
+	copy = must_copy_string(entry);
+	(*list)[newentry] = copy;
+}
+
+static int get_existing_subsystems(char ***klist, char ***nlist)
+{
+	__do_free char *line = NULL;
+	__do_fclose FILE *f = NULL;
+	size_t len = 0;
+
+	f = fopen("/proc/self/cgroup", "re");
+	if (!f)
+		return -1;
+
+	while (getline(&line, &len, f) != -1) {
+		char *p, *p2, *tok;
+		p = strchr(line, ':');
+		if (!p)
+			continue;
+		p++;
+		p2 = strchr(p, ':');
+		if (!p2)
+			continue;
+		*p2 = '\0';
+
+		/* If the kernel has cgroup v2 support, then /proc/self/cgroup
+		 * contains an entry of the form:
+		 *
+		 *	0::/some/path
+		 *
+		 * In this case we use "cgroup2" as controller name.
+		 */
+		if ((p2 - p) == 0) {
+			must_append_string(klist, "cgroup2");
+			continue;
+		}
+
+		lxc_iterate_parts(tok, p, ",") {
+			if (strncmp(tok, "name=", 5) == 0)
+				must_append_string(nlist, tok);
+			else
+				must_append_string(klist, tok);
+		}
+	}
+
+	return 0;
+}
+
+static char *trim(char *s)
+{
+	size_t len;
+
+	len = strlen(s);
+	while ((len > 1) && (s[len - 1] == '\n'))
+		s[--len] = '\0';
+
+	return s;
+}
+
+static void lxc_cgfsng_print_hierarchies(struct cgroup_ops *ops)
+{
+	int i;
+	struct hierarchy **it;
+
+	if (!ops->hierarchies) {
+		TRACE("  No hierarchies found");
+		return;
+	}
+
+	TRACE("  Hierarchies:");
+	for (i = 0, it = ops->hierarchies; it && *it; it++, i++) {
+		int j;
+		char **cit;
+
+		TRACE("  %d: base_cgroup: %s", i, (*it)->container_base_path ? (*it)->container_base_path : "(null)");
+		TRACE("      mountpoint:  %s", (*it)->mountpoint ? (*it)->mountpoint : "(null)");
+		TRACE("      controllers:");
+		for (j = 0, cit = (*it)->controllers; cit && *cit; cit++, j++)
+			TRACE("      %d: %s", j, *cit);
+	}
+}
+
+static void lxc_cgfsng_print_basecg_debuginfo(char *basecginfo, char **klist,
+					      char **nlist)
+{
+	int k;
+	char **it;
+
+	TRACE("basecginfo is:");
+	TRACE("%s", basecginfo);
+
+	for (k = 0, it = klist; it && *it; it++, k++)
+		TRACE("kernel subsystem %d: %s", k, *it);
+
+	for (k = 0, it = nlist; it && *it; it++, k++)
+		TRACE("named subsystem %d: %s", k, *it);
+}
+
+struct generic_userns_exec_data {
+	struct hierarchy **hierarchies;
+	const char *container_cgroup;
+	struct lxc_conf *conf;
+	uid_t origuid; /* target uid in parent namespace */
+	char *path;
+};
+
+static int isulad_cgroup_tree_remove(struct hierarchy **hierarchies,
+			const char *container_cgroup)
+{
+	if (!container_cgroup || !hierarchies)
+		return 0;
+
+	for (int i = 0; hierarchies[i]; i++) {
+		struct hierarchy *h = hierarchies[i];
+		int ret;
+
+		if (!h->container_full_path) {
+			h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, container_cgroup, NULL);
+		}
+
+		ret = lxc_rm_rf(h->container_full_path);
+		if (ret < 0) {
+			SYSERROR("Failed to destroy \"%s\"", h->container_full_path);
+			return -1;
+		}
+
+		free_disarm(h->container_full_path);
+	}
+
+	return 0;
+}
+
+static int isulad_cgroup_tree_remove_wrapper(void *data)
+{
+	struct generic_userns_exec_data *arg = data;
+	uid_t nsuid = (arg->conf->root_nsuid_map != NULL) ? 0 : arg->conf->init_uid;
+	gid_t nsgid = (arg->conf->root_nsgid_map != NULL) ? 0 : arg->conf->init_gid;
+	int ret;
+
+	if (!lxc_setgroups(0, NULL) && errno != EPERM)
+		return log_error_errno(-1, errno, "Failed to setgroups(0, NULL)");
+
+	ret = setresgid(nsgid, nsgid, nsgid);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to setresgid(%d, %d, %d)",
+				       (int)nsgid, (int)nsgid, (int)nsgid);
+
+	ret = setresuid(nsuid, nsuid, nsuid);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to setresuid(%d, %d, %d)",
+				       (int)nsuid, (int)nsuid, (int)nsuid);
+
+	return isulad_cgroup_tree_remove(arg->hierarchies, arg->container_cgroup);
+}
+
+__cgfsng_ops static bool isulad_cgfsng_payload_destroy(struct cgroup_ops *ops,
+						struct lxc_handler *handler)
+{
+	int ret;
+
+	if (!ops) {
+		ERROR("Called with uninitialized cgroup operations");
+		return false;
+	}
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_payload_destroy");
+		return true;
+	}
+#endif
+
+	if (!ops->hierarchies) {
+		return false;
+	}
+
+	if (!handler) {
+		ERROR("Called with uninitialized handler");
+		return false;
+	}
+
+	if (!handler->conf) {
+		ERROR("Called with uninitialized conf");
+		return false;
+	}
+
+#ifdef HAVE_STRUCT_BPF_CGROUP_DEV_CTX
+	ret = bpf_program_cgroup_detach(handler->conf->cgroup2_devices);
+	if (ret < 0)
+		WARN("Failed to detach bpf program from cgroup");
+#endif
+
+	if (handler->conf && !lxc_list_empty(&handler->conf->id_map)) {
+		struct generic_userns_exec_data wrap = {
+			.conf			= handler->conf,
+			.container_cgroup	= ops->container_cgroup,
+			.hierarchies		= ops->hierarchies,
+			.origuid		= 0,
+		};
+		ret = userns_exec_1(handler->conf, isulad_cgroup_tree_remove_wrapper,
+				    &wrap, "cgroup_tree_remove_wrapper");
+	} else {
+		ret = isulad_cgroup_tree_remove(ops->hierarchies, ops->container_cgroup);
+	}
+	if (ret < 0) {
+		SYSWARN("Failed to destroy cgroups");
+		return false;
+	}
+
+	return true;
+}
+
+__cgfsng_ops static void isulad_cgfsng_monitor_destroy(struct cgroup_ops *ops,
+						struct lxc_handler *handler)
+{
+	return;
+}
+
+__cgfsng_ops static inline bool isulad_cgfsng_monitor_create(struct cgroup_ops *ops,
+						      struct lxc_handler *handler)
+{
+	return true;
+}
+
+static bool isulad_copy_parent_file(char *path, char *file)
+{
+	int ret;
+	int len = 0;
+	char *value = NULL;
+	char *current = NULL;
+	char *fpath = NULL;
+	char *lastslash = NULL;
+	char oldv;
+
+	fpath = must_make_path(path, file, NULL);
+	current = read_file(fpath);
+
+	if (current == NULL) {
+		SYSERROR("Failed to read file \"%s\"", fpath);
+		free(fpath);
+		return false;
+	}
+
+	if (strcmp(current, "\n") != 0) {
+		free(fpath);
+		free(current);
+		return true;
+	}
+
+	free(fpath);
+	free(current);
+
+	lastslash = strrchr(path, '/');
+	if (lastslash == NULL) {
+		ERROR("Failed to detect \"/\" in \"%s\"", path);
+		return false;
+	}
+	oldv = *lastslash;
+	*lastslash = '\0';
+	fpath = must_make_path(path, file, NULL);
+	*lastslash = oldv;
+	len = lxc_read_from_file(fpath, NULL, 0);
+	if (len <= 0)
+		goto on_error;
+
+	value = must_realloc(NULL, len + 1);
+	ret = lxc_read_from_file(fpath, value, len);
+	if (ret != len)
+		goto on_error;
+	free(fpath);
+
+	fpath = must_make_path(path, file, NULL);
+	ret = lxc_write_to_file(fpath, value, len, false, 0666);
+	if (ret < 0)
+		SYSERROR("Failed to write \"%s\" to file \"%s\"", value, fpath);
+	free(fpath);
+	free(value);
+	return ret >= 0;
+
+on_error:
+	SYSERROR("Failed to read file \"%s\"", fpath);
+	free(fpath);
+	free(value);
+	return false;
+}
+
+static bool build_sub_cpuset_cgroup_dir(char *cgpath)
+{
+	int ret;
+
+	ret = mkdir_p(cgpath, 0755);
+	if (ret < 0) {
+		if (errno != EEXIST) {
+			SYSERROR("Failed to create directory \"%s\"", cgpath);
+			return false;
+		}
+	}
+
+	/* copy parent's settings */
+	if (!isulad_copy_parent_file(cgpath, "cpuset.cpus")) {
+		SYSERROR("Failed to copy \"cpuset.cpus\" settings");
+		return false;
+	}
+
+	/* copy parent's settings */
+	if (!isulad_copy_parent_file(cgpath, "cpuset.mems")) {
+		SYSERROR("Failed to copy \"cpuset.mems\" settings");
+		return false;
+	}
+
+	return true;
+}
+
+static bool isulad_cg_legacy_handle_cpuset_hierarchy(struct hierarchy *h, char *cgname)
+{
+	char *cgpath, *slash;
+	bool sub_mk_success = false;
+
+	if (is_unified_hierarchy(h))
+		return true;
+
+	if (!string_in_list(h->controllers, "cpuset"))
+		return true;
+
+	cgname += strspn(cgname, "/");
+
+	slash = strchr(cgname, '/');
+
+	if (slash != NULL) {
+		while (slash) {
+			*slash = '\0';
+			cgpath = must_make_path(h->mountpoint, h->container_base_path, cgname, NULL);
+			sub_mk_success = build_sub_cpuset_cgroup_dir(cgpath);
+			free(cgpath);
+			*slash = '/';
+			if (!sub_mk_success) {
+				return false;
+			}
+			slash = strchr(slash + 1, '/');
+		}
+	}
+
+	cgpath = must_make_path(h->mountpoint, h->container_base_path, cgname, NULL);
+	sub_mk_success = build_sub_cpuset_cgroup_dir(cgpath);
+	free(cgpath);
+	if (!sub_mk_success) {
+		return false;
+	}
+
+	return true;
+}
+
+static int isulad_mkdir_eexist_on_last(const char *dir, mode_t mode)
+{
+	const char *tmp = dir;
+	const char *orig = dir;
+
+	do {
+		int ret;
+		size_t cur_len;
+		char *makeme;
+
+		dir = tmp + strspn(tmp, "/");
+		tmp = dir + strcspn(dir, "/");
+
+		errno = ENOMEM;
+		cur_len = dir - orig;
+		makeme = strndup(orig, cur_len);
+		if (!makeme)
+			return -1;
+
+		ret = mkdir(makeme, mode);
+		if (ret < 0) {
+			if (errno != EEXIST) {
+				SYSERROR("Failed to create directory \"%s\"", makeme);
+				free(makeme);
+				return -1;
+			}
+		}
+		free(makeme);
+
+	} while (tmp != dir);
+
+	return 0;
+}
+
+static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname, int errfd)
+{
+	int ret;
+	__do_free char *path = NULL;
+
+	path = must_make_path(h->mountpoint, h->container_base_path, cgname, NULL);
+
+	if (file_exists(path)) { // it must not already exist
+		ERROR("Cgroup path \"%s\" already exist.", path);
+		lxc_write_error_message(errfd, "%s:%d: Cgroup path \"%s\" already exist.",
+		                        __FILE__, __LINE__, path);
+		return false;
+	}
+
+	if (!isulad_cg_legacy_handle_cpuset_hierarchy(h, cgname)) {
+		ERROR("Failed to handle legacy cpuset controller");
+		return false;
+	}
+
+	ret = isulad_mkdir_eexist_on_last(path, 0755);
+	if (ret < 0) {
+		ERROR("Failed to create cgroup \"%s\"", path);
+		return false;
+	}
+
+	h->cgfd_con = lxc_open_dirfd(path);
+	if (h->cgfd_con < 0)
+		return log_error_errno(false, errno, "Failed to open %s", path);
+
+	if (h->container_full_path == NULL) {
+		h->container_full_path = move_ptr(path);
+	}
+
+	return true;
+}
+
+/* isulad: create hierarchies path, if fail, return the error */
+__cgfsng_ops static inline bool isulad_cgfsng_payload_create(struct cgroup_ops *ops,
+						      struct lxc_handler *handler)
+{
+	int i;
+	char *container_cgroup = ops->container_cgroup;
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_payload_create");
+		return true;
+	}
+#endif
+
+	if (!container_cgroup) {
+		ERROR("cgfsng_create container_cgroup is invalid");
+		return false;
+	}
+
+	for (i = 0; ops->hierarchies[i]; i++) {
+		if (!create_path_for_hierarchy(ops->hierarchies[i], container_cgroup, ops->errfd)) {
+			SYSERROR("Failed to create %s", ops->hierarchies[i]->container_full_path);
+			return false;
+		}
+	}
+
+	return true;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_monitor_enter(struct cgroup_ops *ops,
+					      struct lxc_handler *handler)
+{
+	return true;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_payload_enter(struct cgroup_ops *ops,
+					      struct lxc_handler *handler)
+{
+	int len;
+	char pidstr[INTTYPE_TO_STRLEN(pid_t)];
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_payload_enter");
+		return true;
+	}
+#endif
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!ops->container_cgroup)
+		return ret_set_errno(false, ENOENT);
+
+	if (!handler || !handler->conf)
+		return ret_set_errno(false, EINVAL);
+
+	len = snprintf(pidstr, sizeof(pidstr), "%d", handler->pid);
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		int ret;
+		char *fullpath;
+		int retry_count = 0;
+		int max_retry = 10;
+
+		fullpath = must_make_path(ops->hierarchies[i]->container_full_path,
+		                          "cgroup.procs", NULL);
+retry:
+		ret = lxc_write_to_file(fullpath, pidstr, len, false, 0666);
+		if (ret != 0) {
+			if (retry_count < max_retry) {
+				SYSERROR("Failed to enter cgroup \"%s\" with retry count:%d", fullpath, retry_count);
+				(void)isulad_cg_legacy_handle_cpuset_hierarchy(ops->hierarchies[i], ops->container_cgroup);
+				(void)isulad_mkdir_eexist_on_last(ops->hierarchies[i]->container_full_path, 0755);
+				usleep(100 * 1000); /* 100 millisecond */
+				retry_count++;
+				goto retry;
+			}
+			SYSERROR("Failed to enter cgroup \"%s\"", fullpath);
+			free(fullpath);
+			return false;
+		}
+		free(fullpath);
+	}
+
+	return true;
+}
+
+static int fchowmodat(int dirfd, const char *path, uid_t chown_uid,
+		      gid_t chown_gid, mode_t chmod_mode)
+{
+	int ret;
+
+	ret = fchownat(dirfd, path, chown_uid, chown_gid,
+		       AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW);
+	if (ret < 0)
+		return log_warn_errno(-1,
+				      errno, "Failed to fchownat(%d, %s, %d, %d, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW )",
+				      dirfd, path, (int)chown_uid,
+				      (int)chown_gid);
+
+	ret = fchmodat(dirfd, (*path != '\0') ? path : ".", chmod_mode, 0);
+	if (ret < 0)
+		return log_warn_errno(-1, errno, "Failed to fchmodat(%d, %s, %d, AT_SYMLINK_NOFOLLOW)",
+				      dirfd, path, (int)chmod_mode);
+
+	return 0;
+}
+
+/* chgrp the container cgroups to container group.  We leave
+ * the container owner as cgroup owner.  So we must make the
+ * directories 775 so that the container can create sub-cgroups.
+ *
+ * Also chown the tasks and cgroup.procs files.  Those may not
+ * exist depending on kernel version.
+ */
+static int chown_cgroup_wrapper(void *data)
+{
+	int ret;
+	uid_t destuid;
+	struct generic_userns_exec_data *arg = data;
+	uid_t nsuid = (arg->conf->root_nsuid_map != NULL) ? 0 : arg->conf->init_uid;
+	gid_t nsgid = (arg->conf->root_nsgid_map != NULL) ? 0 : arg->conf->init_gid;
+
+	if (!lxc_setgroups(0, NULL) && errno != EPERM)
+		return log_error_errno(-1, errno, "Failed to setgroups(0, NULL)");
+
+	ret = setresgid(nsgid, nsgid, nsgid);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to setresgid(%d, %d, %d)",
+				       (int)nsgid, (int)nsgid, (int)nsgid);
+
+	ret = setresuid(nsuid, nsuid, nsuid);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to setresuid(%d, %d, %d)",
+				       (int)nsuid, (int)nsuid, (int)nsuid);
+
+	destuid = get_ns_uid(arg->origuid);
+	if (destuid == LXC_INVALID_UID)
+		destuid = 0;
+
+	for (int i = 0; arg->hierarchies[i]; i++) {
+		int dirfd = arg->hierarchies[i]->cgfd_con;
+
+		(void)fchowmodat(dirfd, "", destuid, nsgid, 0775);
+
+		/*
+		 * Failures to chown() these are inconvenient but not
+		 * detrimental We leave these owned by the container launcher,
+		 * so that container root can write to the files to attach.  We
+		 * chmod() them 664 so that container systemd can write to the
+		 * files (which systemd in wily insists on doing).
+		 */
+
+		if (arg->hierarchies[i]->version == CGROUP_SUPER_MAGIC)
+			(void)fchowmodat(dirfd, "tasks", destuid, nsgid, 0664);
+
+		(void)fchowmodat(dirfd, "cgroup.procs", destuid, nsgid, 0664);
+
+		if (arg->hierarchies[i]->version != CGROUP2_SUPER_MAGIC)
+			continue;
+
+		for (char **p = arg->hierarchies[i]->cgroup2_chown; p && *p; p++)
+			(void)fchowmodat(dirfd, *p, destuid, nsgid, 0664);
+	}
+
+	return 0;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_chown(struct cgroup_ops *ops,
+				      struct lxc_conf *conf)
+{
+	struct generic_userns_exec_data wrap;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!ops->container_cgroup)
+		return ret_set_errno(false, ENOENT);
+
+	if (!conf)
+		return ret_set_errno(false, EINVAL);
+
+	if (lxc_list_empty(&conf->id_map))
+		return true;
+
+	wrap.origuid = geteuid();
+	wrap.path = NULL;
+	wrap.hierarchies = ops->hierarchies;
+	wrap.conf = conf;
+
+	if (userns_exec_1(conf, chown_cgroup_wrapper, &wrap, "chown_cgroup_wrapper") < 0)
+		return log_error_errno(false, errno, "Error requesting cgroup chown in new user namespace");
+
+	return true;
+}
+
+__cgfsng_ops void isulad_cgfsng_payload_finalize(struct cgroup_ops *ops)
+{
+	if (!ops)
+		return;
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_payload_finalize");
+		return;
+	}
+#endif
+
+	if (!ops->hierarchies)
+		return;
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		struct hierarchy *h = ops->hierarchies[i];
+		/*
+		 * we don't keep the fds for non-unified hierarchies around
+		 * mainly because we don't make use of them anymore after the
+		 * core cgroup setup is done but also because there are quite a
+		 * lot of them.
+		 */
+		if (!is_unified_hierarchy(h))
+			close_prot_errno_disarm(h->cgfd_con);
+	}
+}
+
+/* cgroup-full:* is done, no need to create subdirs */
+static inline bool cg_mount_needs_subdirs(int type)
+{
+	return !(type >= LXC_AUTO_CGROUP_FULL_RO);
+}
+
+/* After $rootfs/sys/fs/container/controller/the/cg/path has been created,
+ * remount controller ro if needed and bindmount the cgroupfs onto
+ * control/the/cg/path.
+ */
+static int cg_legacy_mount_controllers(int type, struct hierarchy *h,
+				       char *controllerpath, char *cgpath,
+				       const char *container_cgroup)
+{
+	__do_free char *sourcepath = NULL;
+	int ret, remount_flags;
+	int flags = MS_BIND;
+
+	if (type == LXC_AUTO_CGROUP_RO || type == LXC_AUTO_CGROUP_MIXED) {
+		ret = mount(controllerpath, controllerpath, "cgroup", MS_BIND, NULL);
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to bind mount \"%s\" onto \"%s\"",
+					       controllerpath, controllerpath);
+
+		remount_flags = add_required_remount_flags(controllerpath,
+							   controllerpath,
+							   flags | MS_REMOUNT);
+		ret = mount(controllerpath, controllerpath, "cgroup",
+			    remount_flags | MS_REMOUNT | MS_BIND | MS_RDONLY,
+			    NULL);
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to remount \"%s\" ro", controllerpath);
+
+		INFO("Remounted %s read-only", controllerpath);
+	}
+
+	sourcepath = must_make_path(h->mountpoint, h->container_base_path,
+				    container_cgroup, NULL);
+	if (type == LXC_AUTO_CGROUP_RO)
+		flags |= MS_RDONLY;
+
+	ret = mount(sourcepath, cgpath, "cgroup", flags, NULL);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to mount \"%s\" onto \"%s\"",
+				       h->controllers[0], cgpath);
+	INFO("Mounted \"%s\" onto \"%s\"", h->controllers[0], cgpath);
+
+	if (flags & MS_RDONLY) {
+		remount_flags = add_required_remount_flags(sourcepath, cgpath,
+							   flags | MS_REMOUNT);
+		ret = mount(sourcepath, cgpath, "cgroup", remount_flags, NULL);
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to remount \"%s\" ro", cgpath);
+		INFO("Remounted %s read-only", cgpath);
+	}
+
+	INFO("Completed second stage cgroup automounts for \"%s\"", cgpath);
+	return 0;
+}
+
+/* __cg_mount_direct
+ *
+ * Mount cgroup hierarchies directly without using bind-mounts. The main
+ * uses-cases are mounting cgroup hierarchies in cgroup namespaces and mounting
+ * cgroups for the LXC_AUTO_CGROUP_FULL option.
+ */
+static int __cg_mount_direct(int type, struct hierarchy *h,
+			     const char *controllerpath)
+{
+	 __do_free char *controllers = NULL;
+	 char *fstype = "cgroup2";
+	 unsigned long flags = 0;
+	 int ret;
+
+	 flags |= MS_NOSUID;
+	 flags |= MS_NOEXEC;
+	 flags |= MS_NODEV;
+	 flags |= MS_RELATIME;
+
+	 if (type == LXC_AUTO_CGROUP_RO || type == LXC_AUTO_CGROUP_FULL_RO)
+		 flags |= MS_RDONLY;
+
+	 if (h->version != CGROUP2_SUPER_MAGIC) {
+		 controllers = lxc_string_join(",", (const char **)h->controllers, false);
+		 if (!controllers)
+			 return -ENOMEM;
+		 fstype = "cgroup";
+	}
+
+	ret = mount("cgroup", controllerpath, fstype, flags, controllers);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to mount \"%s\" with cgroup filesystem type %s",
+				       controllerpath, fstype);
+
+	DEBUG("Mounted \"%s\" with cgroup filesystem type %s", controllerpath, fstype);
+	return 0;
+}
+
+static inline int cg_mount_in_cgroup_namespace(int type, struct hierarchy *h,
+					       const char *controllerpath)
+{
+	return __cg_mount_direct(type, h, controllerpath);
+}
+
+static inline int cg_mount_cgroup_full(int type, struct hierarchy *h,
+				       const char *controllerpath)
+{
+	if (type < LXC_AUTO_CGROUP_FULL_RO || type > LXC_AUTO_CGROUP_FULL_MIXED)
+		return 0;
+
+	return __cg_mount_direct(type, h, controllerpath);
+}
+
+__cgfsng_ops static bool isulad_cgfsng_mount(struct cgroup_ops *ops,
+                                      struct lxc_handler *handler,
+                                      const char *root, int type)
+{
+	int i, ret;
+	char *tmpfspath = NULL;
+	char *systemdpath = NULL;
+	char *unifiedpath = NULL;
+	bool has_cgns = false, retval = false, wants_force_mount = false;
+	char **merged = NULL;
+
+	if ((type & LXC_AUTO_CGROUP_MASK) == 0)
+		return true;
+
+	if (type & LXC_AUTO_CGROUP_FORCE) {
+		type &= ~LXC_AUTO_CGROUP_FORCE;
+		wants_force_mount = true;
+	}
+
+	if (!wants_force_mount) {
+		if (!lxc_list_empty(&handler->conf->keepcaps))
+			wants_force_mount = !in_caplist(CAP_SYS_ADMIN, &handler->conf->keepcaps);
+		else
+			wants_force_mount = in_caplist(CAP_SYS_ADMIN, &handler->conf->caps);
+	}
+
+	has_cgns = cgns_supported();
+	if (has_cgns && !wants_force_mount)
+		return true;
+
+	if (type == LXC_AUTO_CGROUP_NOSPEC)
+		type = LXC_AUTO_CGROUP_MIXED;
+	else if (type == LXC_AUTO_CGROUP_FULL_NOSPEC)
+		type = LXC_AUTO_CGROUP_FULL_MIXED;
+
+	/* Mount tmpfs */
+	tmpfspath = must_make_path(root, "/sys/fs/cgroup", NULL);
+	if (mkdir_p(tmpfspath, 0755) < 0) {
+		ERROR("Failed to create directory: %s", tmpfspath);
+		goto on_error;
+	}
+
+        if (ops->cgroup_layout == CGROUP_LAYOUT_UNIFIED) {
+                if (has_cgns && wants_force_mount) {
+                        /*
+                         * If cgroup namespaces are supported but the container
+                         * will not have CAP_SYS_ADMIN after it has started we
+                         * need to mount the cgroups manually.
+                         */
+                        return cg_mount_in_cgroup_namespace(type, ops->unified, tmpfspath) == 0;
+                }
+
+                return cg_mount_cgroup_full(type, ops->unified, tmpfspath) == 0;
+        }
+
+	ret = safe_mount(NULL, tmpfspath, "tmpfs",
+	                 MS_NOSUID | MS_NODEV | MS_NOEXEC | MS_RELATIME,
+	                 "size=10240k,mode=755", root, handler->conf->lsm_se_mount_context);
+	if (ret < 0)
+		goto on_error;
+
+	for (i = 0; ops->hierarchies[i]; i++) {
+		char *controllerpath = NULL;
+		char *path2 = NULL;
+		struct hierarchy *h = ops->hierarchies[i];
+		char *controller = strrchr(h->mountpoint, '/');
+
+		if (!controller)
+			continue;
+		controller++;
+
+		// isulad: symlink subcgroup
+		if (strchr(controller, ',') != NULL) {
+			int pret;
+			pret = lxc_append_string(&merged, controller);
+			if (pret < 0)
+				goto on_error;
+		}
+
+		controllerpath = must_make_path(tmpfspath, controller, NULL);
+		if (dir_exists(controllerpath)) {
+			free(controllerpath);
+			continue;
+		}
+
+		ret = mkdir(controllerpath, 0755);
+		if (ret < 0) {
+			SYSERROR("Error creating cgroup path: %s", controllerpath);
+			free(controllerpath);
+			goto on_error;
+		}
+
+		if (has_cgns && wants_force_mount) {
+			/* If cgroup namespaces are supported but the container
+			 * will not have CAP_SYS_ADMIN after it has started we
+			 * need to mount the cgroups manually.
+			 */
+			ret = cg_mount_in_cgroup_namespace(type, h, controllerpath);
+			free(controllerpath);
+			if (ret < 0)
+				goto on_error;
+
+			continue;
+		}
+
+		ret = cg_mount_cgroup_full(type, h, controllerpath);
+		if (ret < 0) {
+			free(controllerpath);
+			goto on_error;
+		}
+
+		if (!cg_mount_needs_subdirs(type)) {
+			free(controllerpath);
+			continue;
+		}
+
+		// isulad: ignore ops->container_cgroup so we will not see directory lxc after /sys/fs/cgroup/xxx in container,
+		// isulad: ignore h->container_base_path so we will not see subgroup of /sys/fs/cgroup/xxx/subgroup in container
+		path2 = must_make_path(controllerpath, NULL);
+		ret = mkdir_p(path2, 0755);
+		if (ret < 0) {
+			free(controllerpath);
+			free(path2);
+			goto on_error;
+		}
+
+		ret = cg_legacy_mount_controllers(type, h, controllerpath,
+		                                  path2, ops->container_cgroup);
+		free(controllerpath);
+		free(path2);
+		if (ret < 0)
+			goto on_error;
+	}
+
+	// isulad: symlink subcgroup
+	if (merged) {
+		char **mc = NULL;
+		for (mc = merged; *mc; mc++) {
+			char *token = NULL;
+			char *copy = must_copy_string(*mc);
+			lxc_iterate_parts(token, copy, ",") {
+				int mret;
+				char *link;
+				link = must_make_path(tmpfspath, token, NULL);
+				mret = symlink(*mc, link);
+				if (mret < 0 && errno != EEXIST) {
+					SYSERROR("Failed to create link %s for target %s", link, *mc);
+					free(copy);
+					free(link);
+					goto on_error;
+				}
+				free(link);
+			}
+			free(copy);
+		}
+	}
+
+
+	// isulad: remount /sys/fs/cgroup to readonly
+	if (type == LXC_AUTO_CGROUP_FULL_RO || type == LXC_AUTO_CGROUP_RO) {
+		ret = mount(tmpfspath, tmpfspath, "bind",
+		            MS_NOSUID|MS_NODEV|MS_NOEXEC|MS_RELATIME|MS_RDONLY|MS_BIND|MS_REMOUNT, NULL);
+		if (ret < 0) {
+			SYSERROR("Failed to remount /sys/fs/cgroup.");
+			goto on_error;
+		}
+	}
+
+	// isulad: remount /sys/fs/cgroup/systemd to readwrite for system container
+	if (handler->conf->systemd != NULL && strcmp(handler->conf->systemd, "true") == 0)
+	{
+		unifiedpath = must_make_path(root, "/sys/fs/cgroup/unified", NULL);
+		if (dir_exists(unifiedpath))
+		{
+			ret = umount2(unifiedpath, MNT_DETACH);
+			if (ret < 0)
+			{
+				SYSERROR("Failed to umount /sys/fs/cgroup/unified.");
+				goto on_error;
+			}
+		}
+
+		systemdpath = must_make_path(root, "/sys/fs/cgroup/systemd", NULL);
+		ret = mount(systemdpath, systemdpath, "bind",
+					MS_NOSUID | MS_NODEV | MS_NOEXEC | MS_RELATIME | MS_BIND | MS_REMOUNT, NULL);
+		if (ret < 0)
+		{
+			SYSERROR("Failed to remount /sys/fs/cgroup/systemd.");
+			goto on_error;
+		}
+	}
+
+	retval = true;
+
+on_error:
+	free(tmpfspath);
+	if (systemdpath != NULL)
+	{
+		free(systemdpath);
+	}
+	if (unifiedpath != NULL)
+	{
+		free(unifiedpath);
+	}
+	lxc_free_array((void **)merged, free);
+	return retval;
+}
+
+/* Only root needs to escape to the cgroup of its init. */
+__cgfsng_ops static bool isulad_cgfsng_escape(const struct cgroup_ops *ops,
+				       struct lxc_conf *conf)
+{
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!conf)
+		return ret_set_errno(false, EINVAL);
+
+	if (conf->cgroup_meta.relative || geteuid())
+		return true;
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		__do_free char *fullpath = NULL;
+		int ret;
+
+		fullpath =
+		    must_make_path(ops->hierarchies[i]->mountpoint,
+				   ops->hierarchies[i]->container_base_path,
+				   "cgroup.procs", NULL);
+		ret = lxc_write_to_file(fullpath, "0", 2, false, 0666);
+		if (ret != 0)
+			return log_error_errno(false, errno, "Failed to escape to cgroup \"%s\"", fullpath);
+	}
+
+	return true;
+}
+
+__cgfsng_ops static int isulad_cgfsng_num_hierarchies(struct cgroup_ops *ops)
+{
+	int i = 0;
+
+	if (!ops)
+		return ret_set_errno(-1, ENOENT);
+
+	if (!ops->hierarchies)
+		return 0;
+
+	for (; ops->hierarchies[i]; i++)
+		;
+
+	return i;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_get_hierarchies(struct cgroup_ops *ops, int n,
+						char ***out)
+{
+	int i;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return ret_set_errno(false, ENOENT);
+
+	/* sanity check n */
+	for (i = 0; i < n; i++)
+		if (!ops->hierarchies[i])
+			return ret_set_errno(false, ENOENT);
+
+	*out = ops->hierarchies[i]->controllers;
+
+	return true;
+}
+
+static bool cg_legacy_freeze(struct cgroup_ops *ops)
+{
+	struct hierarchy *h;
+
+	h = get_hierarchy(ops, "freezer");
+	if (!h)
+		return ret_set_errno(-1, ENOENT);
+
+	return lxc_write_openat(h->container_full_path, "freezer.state",
+				"FROZEN", STRLITERALLEN("FROZEN"));
+}
+
+static int freezer_cgroup_events_cb(int fd, uint32_t events, void *cbdata,
+				    struct lxc_epoll_descr *descr)
+{
+	__do_close int duped_fd = -EBADF;
+	__do_free char *line = NULL;
+	__do_fclose FILE *f = NULL;
+	int state = PTR_TO_INT(cbdata);
+	size_t len;
+	const char *state_string;
+
+	duped_fd = dup(fd);
+	if (duped_fd < 0)
+		return LXC_MAINLOOP_ERROR;
+
+	if (lseek(duped_fd, 0, SEEK_SET) < (off_t)-1)
+		return LXC_MAINLOOP_ERROR;
+
+	f = fdopen(duped_fd, "re");
+	if (!f)
+		return LXC_MAINLOOP_ERROR;
+	move_fd(duped_fd);
+
+	if (state == 1)
+		state_string = "frozen 1";
+	else
+		state_string = "frozen 0";
+
+	while (getline(&line, &len, f) != -1)
+		if (strncmp(line, state_string, STRLITERALLEN("frozen") + 2) == 0)
+			return LXC_MAINLOOP_CLOSE;
+
+	return LXC_MAINLOOP_CONTINUE;
+}
+
+static int cg_unified_freeze(struct cgroup_ops *ops, int timeout)
+{
+	__do_close int fd = -EBADF;
+	call_cleaner(lxc_mainloop_close) struct lxc_epoll_descr *descr_ptr = NULL;
+	int ret;
+	struct lxc_epoll_descr descr;
+	struct hierarchy *h;
+
+	h = ops->unified;
+	if (!h)
+		return ret_set_errno(-1, ENOENT);
+
+	if (!h->container_full_path)
+		return ret_set_errno(-1, EEXIST);
+
+	if (timeout != 0) {
+		__do_free char *events_file = NULL;
+
+		events_file = must_make_path(h->container_full_path, "cgroup.events", NULL);
+		fd = open(events_file, O_RDONLY | O_CLOEXEC);
+		if (fd < 0)
+			return log_error_errno(-1, errno, "Failed to open cgroup.events file");
+
+		ret = lxc_mainloop_open(&descr);
+		if (ret)
+			return log_error_errno(-1, errno, "Failed to create epoll instance to wait for container freeze");
+
+		/* automatically cleaned up now */
+		descr_ptr = &descr;
+
+		ret = lxc_mainloop_add_handler(&descr, fd, freezer_cgroup_events_cb, INT_TO_PTR((int){1}));
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to add cgroup.events fd handler to mainloop");
+	}
+
+	ret = lxc_write_openat(h->container_full_path, "cgroup.freeze", "1", 1);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to open cgroup.freeze file");
+
+	if (timeout != 0 && lxc_mainloop(&descr, timeout))
+		return log_error_errno(-1, errno, "Failed to wait for container to be frozen");
+
+	return 0;
+}
+
+__cgfsng_ops static int isulad_cgfsng_freeze(struct cgroup_ops *ops, int timeout)
+{
+	if (!ops->hierarchies)
+		return ret_set_errno(-1, ENOENT);
+
+	if (ops->cgroup_layout != CGROUP_LAYOUT_UNIFIED)
+		return cg_legacy_freeze(ops);
+
+	return cg_unified_freeze(ops, timeout);
+}
+
+static int cg_legacy_unfreeze(struct cgroup_ops *ops)
+{
+	struct hierarchy *h;
+
+	h = get_hierarchy(ops, "freezer");
+	if (!h)
+		return ret_set_errno(-1, ENOENT);
+
+	return lxc_write_openat(h->container_full_path, "freezer.state",
+				"THAWED", STRLITERALLEN("THAWED"));
+}
+
+static int cg_unified_unfreeze(struct cgroup_ops *ops, int timeout)
+{
+	__do_close int fd = -EBADF;
+	call_cleaner(lxc_mainloop_close)struct lxc_epoll_descr *descr_ptr = NULL;
+	int ret;
+	struct lxc_epoll_descr descr;
+	struct hierarchy *h;
+
+	h = ops->unified;
+	if (!h)
+		return ret_set_errno(-1, ENOENT);
+
+	if (!h->container_full_path)
+		return ret_set_errno(-1, EEXIST);
+
+	if (timeout != 0) {
+		__do_free char *events_file = NULL;
+
+		events_file = must_make_path(h->container_full_path, "cgroup.events", NULL);
+		fd = open(events_file, O_RDONLY | O_CLOEXEC);
+		if (fd < 0)
+			return log_error_errno(-1, errno, "Failed to open cgroup.events file");
+
+		ret = lxc_mainloop_open(&descr);
+		if (ret)
+			return log_error_errno(-1, errno, "Failed to create epoll instance to wait for container unfreeze");
+
+		/* automatically cleaned up now */
+		descr_ptr = &descr;
+
+		ret = lxc_mainloop_add_handler(&descr, fd, freezer_cgroup_events_cb, INT_TO_PTR((int){0}));
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to add cgroup.events fd handler to mainloop");
+	}
+
+	ret = lxc_write_openat(h->container_full_path, "cgroup.freeze", "0", 1);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to open cgroup.freeze file");
+
+	if (timeout != 0 && lxc_mainloop(&descr, timeout))
+		return log_error_errno(-1, errno, "Failed to wait for container to be unfrozen");
+
+	return 0;
+}
+
+__cgfsng_ops static int isulad_cgfsng_unfreeze(struct cgroup_ops *ops, int timeout)
+{
+	if (!ops->hierarchies)
+		return ret_set_errno(-1, ENOENT);
+
+	if (ops->cgroup_layout != CGROUP_LAYOUT_UNIFIED)
+		return cg_legacy_unfreeze(ops);
+
+	return cg_unified_unfreeze(ops, timeout);
+}
+
+__cgfsng_ops static const char *isulad_cgfsng_get_cgroup(struct cgroup_ops *ops,
+						  const char *controller)
+{
+	struct hierarchy *h;
+
+	h = get_hierarchy(ops, controller);
+	if (!h)
+		return log_warn_errno(NULL, ENOENT, "Failed to find hierarchy for controller \"%s\"",
+				      controller ? controller : "(null)");
+
+	if (!h->container_full_path)
+		h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, ops->container_cgroup, NULL);
+
+	return h->container_full_path
+		   ? h->container_full_path + strlen(h->mountpoint)
+		   : NULL;
+}
+
+__cgfsng_ops static const char *isulad_cgfsng_get_cgroup_full_path(struct cgroup_ops *ops,
+						  const char *controller)
+{
+	struct hierarchy *h;
+
+	h = get_hierarchy(ops, controller);
+	if (!h)
+		return log_warn_errno(NULL, ENOENT, "Failed to find hierarchy for controller \"%s\"",
+				      controller ? controller : "(null)");
+
+	if (!h->container_full_path)
+		h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, ops->container_cgroup, NULL);
+
+	return h->container_full_path;
+}
+
+/* Given a cgroup path returned from lxc_cmd_get_cgroup_path, build a full path,
+ * which must be freed by the caller.
+ */
+static inline char *build_full_cgpath_from_monitorpath(struct hierarchy *h,
+						       const char *inpath,
+						       const char *filename)
+{
+	return must_make_path(h->mountpoint, inpath, filename, NULL);
+}
+
+static int cgroup_attach_leaf(const struct lxc_conf *conf, int unified_fd, pid_t pid)
+{
+	int idx = 1;
+	int ret;
+	char pidstr[INTTYPE_TO_STRLEN(int64_t) + 1];
+	size_t pidstr_len;
+
+	/* Create leaf cgroup. */
+	ret = mkdirat(unified_fd, ".lxc", 0755);
+	if (ret < 0 && errno != EEXIST)
+		return log_error_errno(-1, errno, "Failed to create leaf cgroup \".lxc\"");
+
+	pidstr_len = sprintf(pidstr, INT64_FMT, (int64_t)pid);
+	ret = lxc_writeat(unified_fd, ".lxc/cgroup.procs", pidstr, pidstr_len);
+	if (ret < 0)
+		ret = lxc_writeat(unified_fd, "cgroup.procs", pidstr, pidstr_len);
+	if (ret == 0)
+		return 0;
+
+	/* this is a non-leaf node */
+	if (errno != EBUSY)
+		return log_error_errno(-1, errno, "Failed to attach to unified cgroup");
+
+	do {
+		bool rm = false;
+		char attach_cgroup[STRLITERALLEN(".lxc-/cgroup.procs") + INTTYPE_TO_STRLEN(int) + 1];
+		char *slash;
+
+		ret = snprintf(attach_cgroup, sizeof(attach_cgroup), ".lxc-%d/cgroup.procs", idx);
+		if (ret < 0 || (size_t)ret >= sizeof(attach_cgroup))
+			return ret_errno(EIO);
+
+		/*
+		 * This shouldn't really happen but the compiler might complain
+		 * that a short write would cause a buffer overrun. So be on
+		 * the safe side.
+		 */
+		if (ret < STRLITERALLEN(".lxc-/cgroup.procs"))
+			return log_error_errno(-EINVAL, EINVAL, "Unexpected short write would cause buffer-overrun");
+
+		slash = &attach_cgroup[ret] - STRLITERALLEN("/cgroup.procs");
+		*slash = '\0';
+
+		ret = mkdirat(unified_fd, attach_cgroup, 0755);
+		if (ret < 0 && errno != EEXIST)
+			return log_error_errno(-1, errno, "Failed to create cgroup %s", attach_cgroup);
+		if (ret == 0)
+			rm = true;
+
+		*slash = '/';
+
+		ret = lxc_writeat(unified_fd, attach_cgroup, pidstr, pidstr_len);
+		if (ret == 0)
+			return 0;
+
+		if (rm && unlinkat(unified_fd, attach_cgroup, AT_REMOVEDIR))
+			SYSERROR("Failed to remove cgroup \"%d(%s)\"", unified_fd, attach_cgroup);
+
+		/* this is a non-leaf node */
+		if (errno != EBUSY)
+			return log_error_errno(-1, errno, "Failed to attach to unified cgroup");
+
+		idx++;
+	} while (idx < 1000);
+
+	return log_error_errno(-1, errno, "Failed to attach to unified cgroup");
+}
+
+static int cgroup_attach_create_leaf(const struct lxc_conf *conf,
+				     int unified_fd, int *sk_fd)
+{
+	__do_close int sk = *sk_fd, target_fd0 = -EBADF, target_fd1 = -EBADF;
+	int target_fds[2];
+	ssize_t ret;
+
+	/* Create leaf cgroup. */
+	ret = mkdirat(unified_fd, ".lxc", 0755);
+	if (ret < 0 && errno != EEXIST)
+		return log_error_errno(-1, errno, "Failed to create leaf cgroup \".lxc\"");
+
+	target_fd0 = openat(unified_fd, ".lxc/cgroup.procs", O_WRONLY | O_CLOEXEC | O_NOFOLLOW);
+	if (target_fd0 < 0)
+		return log_error_errno(-errno, errno, "Failed to open \".lxc/cgroup.procs\"");
+	target_fds[0] = target_fd0;
+
+	target_fd1 = openat(unified_fd, "cgroup.procs", O_WRONLY | O_CLOEXEC | O_NOFOLLOW);
+	if (target_fd1 < 0)
+		return log_error_errno(-errno, errno, "Failed to open \".lxc/cgroup.procs\"");
+	target_fds[1] = target_fd1;
+
+	ret = lxc_abstract_unix_send_fds(sk, target_fds, 2, NULL, 0);
+	if (ret <= 0)
+		return log_error_errno(-errno, errno, "Failed to send \".lxc/cgroup.procs\" fds %d and %d",
+				       target_fd0, target_fd1);
+
+	return log_debug(0, "Sent target cgroup fds %d and %d", target_fd0, target_fd1);
+}
+
+static int cgroup_attach_move_into_leaf(const struct lxc_conf *conf,
+					int *sk_fd, pid_t pid)
+{
+	__do_close int sk = *sk_fd, target_fd0 = -EBADF, target_fd1 = -EBADF;
+	int target_fds[2];
+	char pidstr[INTTYPE_TO_STRLEN(int64_t) + 1];
+	size_t pidstr_len;
+	ssize_t ret;
+
+	ret = lxc_abstract_unix_recv_fds(sk, target_fds, 2, NULL, 0);
+	if (ret <= 0)
+		return log_error_errno(-1, errno, "Failed to receive target cgroup fd");
+	target_fd0 = target_fds[0];
+	target_fd1 = target_fds[1];
+
+	pidstr_len = sprintf(pidstr, INT64_FMT, (int64_t)pid);
+
+	ret = lxc_write_nointr(target_fd0, pidstr, pidstr_len);
+	if (ret > 0 && ret == pidstr_len)
+		return log_debug(0, "Moved process into target cgroup via fd %d", target_fd0);
+
+	ret = lxc_write_nointr(target_fd1, pidstr, pidstr_len);
+	if (ret > 0 && ret == pidstr_len)
+		return log_debug(0, "Moved process into target cgroup via fd %d", target_fd1);
+
+	return log_debug_errno(-1, errno, "Failed to move process into target cgroup via fd %d and %d",
+			       target_fd0, target_fd1);
+}
+
+struct userns_exec_unified_attach_data {
+	const struct lxc_conf *conf;
+	int unified_fd;
+	int sk_pair[2];
+	pid_t pid;
+};
+
+static int cgroup_unified_attach_child_wrapper(void *data)
+{
+	struct userns_exec_unified_attach_data *args = data;
+
+	if (!args->conf || args->unified_fd < 0 || args->pid <= 0 ||
+	    args->sk_pair[0] < 0 || args->sk_pair[1] < 0)
+		return ret_errno(EINVAL);
+
+	close_prot_errno_disarm(args->sk_pair[0]);
+	return cgroup_attach_create_leaf(args->conf, args->unified_fd,
+					 &args->sk_pair[1]);
+}
+
+static int cgroup_unified_attach_parent_wrapper(void *data)
+{
+	struct userns_exec_unified_attach_data *args = data;
+
+	if (!args->conf || args->unified_fd < 0 || args->pid <= 0 ||
+	    args->sk_pair[0] < 0 || args->sk_pair[1] < 0)
+		return ret_errno(EINVAL);
+
+	close_prot_errno_disarm(args->sk_pair[1]);
+	return cgroup_attach_move_into_leaf(args->conf, &args->sk_pair[0],
+					    args->pid);
+}
+
+int cgroup_attach(const struct lxc_conf *conf, const char *name,
+		  const char *lxcpath, pid_t pid)
+{
+	__do_close int unified_fd = -EBADF;
+	int ret;
+
+	if (!conf || !name || !lxcpath || pid <= 0)
+		return ret_errno(EINVAL);
+
+	unified_fd = lxc_cmd_get_cgroup2_fd(name, lxcpath);
+	if (unified_fd < 0)
+		return ret_errno(EBADF);
+
+	if (!lxc_list_empty(&conf->id_map)) {
+		struct userns_exec_unified_attach_data args = {
+			.conf		= conf,
+			.unified_fd	= unified_fd,
+			.pid		= pid,
+		};
+
+		ret = socketpair(PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0, args.sk_pair);
+		if (ret < 0)
+			return -errno;
+
+		ret = userns_exec_minimal(conf,
+					  cgroup_unified_attach_parent_wrapper,
+					  &args,
+					  cgroup_unified_attach_child_wrapper,
+					  &args);
+	} else {
+		ret = cgroup_attach_leaf(conf, unified_fd, pid);
+	}
+
+	return ret;
+}
+
+/* Technically, we're always at a delegation boundary here (This is especially
+ * true when cgroup namespaces are available.). The reasoning is that in order
+ * for us to have been able to start a container in the first place the root
+ * cgroup must have been a leaf node. Now, either the container's init system
+ * has populated the cgroup and kept it as a leaf node or it has created
+ * subtrees. In the former case we will simply attach to the leaf node we
+ * created when we started the container in the latter case we create our own
+ * cgroup for the attaching process.
+ */
+static int __cg_unified_attach(const struct hierarchy *h,
+			       const struct lxc_conf *conf, const char *name,
+			       const char *lxcpath, pid_t pid,
+			       const char *controller)
+{
+	__do_close int unified_fd = -EBADF;
+	__do_free char *path = NULL, *cgroup = NULL;
+	int ret;
+
+	if (!conf || !name || !lxcpath || pid <= 0)
+		return ret_errno(EINVAL);
+
+	ret = cgroup_attach(conf, name, lxcpath, pid);
+	if (ret == 0)
+		return log_trace(0, "Attached to unified cgroup via command handler");
+	if (ret != -EBADF)
+		return log_error_errno(ret, errno, "Failed to attach to unified cgroup");
+
+	/* Fall back to retrieving the path for the unified cgroup. */
+	cgroup = lxc_cmd_get_cgroup_path(name, lxcpath, controller);
+	/* not running */
+	if (!cgroup)
+		return 0;
+
+	path = must_make_path(h->mountpoint, cgroup, NULL);
+
+	unified_fd = open(path, O_PATH | O_DIRECTORY | O_CLOEXEC);
+	if (unified_fd < 0)
+		return ret_errno(EBADF);
+
+	if (!lxc_list_empty(&conf->id_map)) {
+		struct userns_exec_unified_attach_data args = {
+			.conf		= conf,
+			.unified_fd	= unified_fd,
+			.pid		= pid,
+		};
+
+		ret = socketpair(PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0, args.sk_pair);
+		if (ret < 0)
+			return -errno;
+
+		ret = userns_exec_minimal(conf,
+					  cgroup_unified_attach_parent_wrapper,
+					  &args,
+					  cgroup_unified_attach_child_wrapper,
+					  &args);
+	} else {
+		ret = cgroup_attach_leaf(conf, unified_fd, pid);
+	}
+
+	return ret;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_attach(struct cgroup_ops *ops,
+				       const struct lxc_conf *conf,
+				       const char *name, const char *lxcpath,
+				       pid_t pid)
+{
+	int len, ret;
+	char pidstr[INTTYPE_TO_STRLEN(pid_t)];
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_attach");
+		return true;
+	}
+#endif
+
+	if (!ops->hierarchies)
+		return true;
+
+	len = snprintf(pidstr, sizeof(pidstr), "%d", pid);
+	if (len < 0 || (size_t)len >= sizeof(pidstr))
+		return false;
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		__do_free char *fullpath = NULL, *path = NULL;
+		struct hierarchy *h = ops->hierarchies[i];
+
+		if (h->version == CGROUP2_SUPER_MAGIC) {
+			ret = __cg_unified_attach(h, conf, name, lxcpath, pid,
+						  h->controllers[0]);
+			if (ret < 0)
+				return false;
+
+			continue;
+		}
+
+		path = lxc_cmd_get_cgroup_path(name, lxcpath, h->controllers[0]);
+		/* not running */
+		if (!path)
+			return false;
+
+		fullpath = build_full_cgpath_from_monitorpath(h, path, "cgroup.procs");
+		ret = lxc_write_to_file(fullpath, pidstr, len, false, 0666);
+		if (ret < 0)
+			return log_error_errno(false, errno, "Failed to attach %d to %s",
+					       (int)pid, fullpath);
+	}
+
+	return true;
+}
+
+__cgfsng_ops static int isulad_cgfsng_get(struct cgroup_ops *ops, const char *filename,
+                                   char *value, size_t len, const char *name,
+                                   const char *lxcpath)
+{
+	int ret = -1;
+	size_t controller_len;
+	char *controller, *p, *path;
+	struct hierarchy *h;
+
+	controller_len = strlen(filename);
+	controller = alloca(controller_len + 1);
+	(void)strlcpy(controller, filename, controller_len + 1);
+
+	p = strchr(controller, '.');
+	if (p)
+		*p = '\0';
+
+	const char *ori_path = ops->get_cgroup(ops, controller);
+	if (ori_path == NULL) {
+		ERROR("Failed to get cgroup path:%s", controller);
+		return -1;
+	}
+	path = safe_strdup(ori_path);
+
+	h = get_hierarchy(ops, controller);
+	if (h) {
+		char *fullpath;
+
+		fullpath = build_full_cgpath_from_monitorpath(h, path, filename);
+		ret = lxc_read_from_file(fullpath, value, len);
+		free(fullpath);
+	}
+	free(path);
+
+	return ret;
+}
+
+static int device_cgroup_parse_access(struct device_item *device, const char *val)
+{
+	for (int count = 0; count < 3; count++, val++) {
+		switch (*val) {
+		case 'r':
+			device->access[count] = *val;
+			break;
+		case 'w':
+			device->access[count] = *val;
+			break;
+		case 'm':
+			device->access[count] = *val;
+			break;
+		case '\n':
+		case '\0':
+			count = 3;
+			break;
+		default:
+			return ret_errno(EINVAL);
+		}
+	}
+
+	return 0;
+}
+
+int device_cgroup_rule_parse(struct device_item *device, const char *key,
+				    const char *val)
+{
+	int count, ret;
+	char temp[50];
+
+	if (strcmp("devices.allow", key) == 0)
+		device->allow = 1;
+	else
+		device->allow = 0;
+
+	if (strcmp(val, "a") == 0) {
+		/* global rule */
+		device->type = 'a';
+		device->major = -1;
+		device->minor = -1;
+		device->global_rule = device->allow
+					  ? LXC_BPF_DEVICE_CGROUP_BLACKLIST
+					  : LXC_BPF_DEVICE_CGROUP_WHITELIST;
+		device->allow = -1;
+		return 0;
+	}
+
+	/* local rule */
+	device->global_rule = LXC_BPF_DEVICE_CGROUP_LOCAL_RULE;
+
+	switch (*val) {
+	case 'a':
+		__fallthrough;
+	case 'b':
+		__fallthrough;
+	case 'c':
+		device->type = *val;
+		break;
+	default:
+		return -1;
+	}
+
+	val++;
+	if (!isspace(*val))
+		return -1;
+	val++;
+	if (*val == '*') {
+		device->major = -1;
+		val++;
+	} else if (isdigit(*val)) {
+		memset(temp, 0, sizeof(temp));
+		for (count = 0; count < sizeof(temp) - 1; count++) {
+			temp[count] = *val;
+			val++;
+			if (!isdigit(*val))
+				break;
+		}
+		ret = lxc_safe_int(temp, &device->major);
+		if (ret)
+			return -1;
+	} else {
+		return -1;
+	}
+	if (*val != ':')
+		return -1;
+	val++;
+
+	/* read minor */
+	if (*val == '*') {
+		device->minor = -1;
+		val++;
+	} else if (isdigit(*val)) {
+		memset(temp, 0, sizeof(temp));
+		for (count = 0; count < sizeof(temp) - 1; count++) {
+			temp[count] = *val;
+			val++;
+			if (!isdigit(*val))
+				break;
+		}
+		ret = lxc_safe_int(temp, &device->minor);
+		if (ret)
+			return -1;
+	} else {
+		return -1;
+	}
+	if (!isspace(*val))
+		return -1;
+
+	return device_cgroup_parse_access(device, ++val);
+}
+
+__cgfsng_ops static int isulad_cgfsng_set(struct cgroup_ops *ops,
+                                   const char *filename, const char *value,
+                                   const char *name, const char *lxcpath)
+{
+	int ret = -1;
+	size_t controller_len;
+	char *controller, *p, *path;
+	struct hierarchy *h;
+
+	controller_len = strlen(filename);
+	controller = alloca(controller_len + 1);
+	(void)strlcpy(controller, filename, controller_len + 1);
+
+	p = strchr(controller, '.');
+	if (p)
+		*p = '\0';
+
+	const char *ori_path = ops->get_cgroup(ops, controller);
+	if (ori_path == NULL) {
+		ERROR("Failed to get cgroup path:%s", controller);
+		return -1;
+	}
+	path = safe_strdup(ori_path);
+
+	h = get_hierarchy(ops, controller);
+	if (h) {
+		char *fullpath;
+		fullpath = build_full_cgpath_from_monitorpath(h, path, filename);
+
+		if (strcmp(filename, "io.weight") == 0 || strcmp(filename, "io.bfq.weight") == 0) {
+			if (!file_exists(fullpath)) {
+				free(path);
+				free(fullpath);
+				return 0;
+			}
+		}
+
+		ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666);
+		free(fullpath);
+	}
+	free(path);
+
+	return ret;
+}
+
+/* take devices cgroup line
+ *    /dev/foo rwx
+ * and convert it to a valid
+ *    type major:minor mode
+ * line. Return <0 on error. Dest is a preallocated buffer long enough to hold
+ * the output.
+ */
+static int device_cgroup_rule_parse_devpath(struct device_item *device,
+					    const char *devpath)
+{
+	__do_free char *path = NULL;
+	char *mode = NULL;
+	int n_parts, ret;
+	char *p;
+	struct stat sb;
+
+	path = must_copy_string(devpath);
+
+	/*
+	 * Read path followed by mode. Ignore any trailing text.
+	 * A '    # comment' would be legal. Technically other text is not
+	 * legal, we could check for that if we cared to.
+	 */
+	for (n_parts = 1, p = path; *p; p++) {
+		if (*p != ' ')
+			continue;
+		*p = '\0';
+
+		if (n_parts != 1)
+			break;
+		p++;
+		n_parts++;
+
+		while (*p == ' ')
+			p++;
+
+		mode = p;
+
+		if (*p == '\0')
+			return ret_set_errno(-1, EINVAL);
+	}
+
+	if (device_cgroup_parse_access(device, mode) < 0)
+		return -1;
+
+	if (n_parts == 1)
+		return ret_set_errno(-1, EINVAL);
+
+	ret = stat(path, &sb);
+	if (ret < 0)
+		return ret_set_errno(-1, errno);
+
+	mode_t m = sb.st_mode & S_IFMT;
+	switch (m) {
+	case S_IFBLK:
+		device->type = 'b';
+		break;
+	case S_IFCHR:
+		device->type = 'c';
+		break;
+	default:
+		return log_error_errno(-1, EINVAL, "Unsupported device type %i for \"%s\"", m, path);
+	}
+
+	device->major = MAJOR(sb.st_rdev);
+	device->minor = MINOR(sb.st_rdev);
+	device->allow = 1;
+	device->global_rule = LXC_BPF_DEVICE_CGROUP_LOCAL_RULE;
+
+	return 0;
+}
+
+static int convert_devpath(const char *invalue, char *dest)
+{
+	struct device_item device = {0};
+	int ret;
+
+	ret = device_cgroup_rule_parse_devpath(&device, invalue);
+	if (ret < 0)
+		return -1;
+
+	ret = snprintf(dest, 50, "%c %d:%d %s", device.type, device.major,
+		       device.minor, device.access);
+	if (ret < 0 || ret >= 50)
+		return log_error_errno(-1, ENAMETOOLONG, "Error on configuration value \"%c %d:%d %s\" (max 50 chars)",
+				       device.type, device.major, device.minor, device.access);
+
+	return 0;
+}
+
+/* Called from setup_limits - here we have the container's cgroup_data because
+ * we created the cgroups.
+ */
+static int isulad_cg_legacy_get_data(struct cgroup_ops *ops, const char *filename,
+                              char *value, size_t len)
+{
+	char *fullpath = NULL;
+	char *p = NULL;
+	struct hierarchy *h = NULL;
+	int ret = 0;
+	char *controller = NULL;
+
+	len = strlen(filename);
+	if (SIZE_MAX - 1 < len) {
+		errno = EINVAL;
+		return -1;
+	}
+	controller = calloc(1, len + 1);
+	if (controller == NULL) {
+		errno = ENOMEM;
+		return -1;
+	}
+	(void)strlcpy(controller, filename, len + 1);
+
+	p = strchr(controller, '.');
+	if (p)
+		*p = '\0';
+
+
+	h = get_hierarchy(ops, controller);
+	if (!h) {
+		ERROR("Failed to setup limits for the \"%s\" controller. "
+		      "The controller seems to be unused by \"cgfsng\" cgroup "
+		      "driver or not enabled on the cgroup hierarchy",
+		      controller);
+		errno = ENOENT;
+		free(controller);
+		return -ENOENT;
+	}
+
+	fullpath = must_make_path(h->container_full_path, filename, NULL);
+	ret = lxc_read_from_file(fullpath, value, len);
+	free(fullpath);
+	free(controller);
+	return ret;
+}
+
+static int isulad_cg_legacy_set_data(struct cgroup_ops *ops, const char *filename,
+                              const char *value)
+{
+	size_t len;
+	char *fullpath, *p;
+	/* "b|c <2^64-1>:<2^64-1> r|w|m" = 47 chars max */
+	char converted_value[50];
+	struct hierarchy *h;
+	int ret = 0;
+	char *controller = NULL;
+	int retry_count = 0;
+	int max_retry = 10;
+	char *container_cgroup = ops->container_cgroup;
+
+	len = strlen(filename);
+	controller = alloca(len + 1);
+	(void)strlcpy(controller, filename, len + 1);
+
+	p = strchr(controller, '.');
+	if (p)
+		*p = '\0';
+
+	if (strcmp("devices.allow", filename) == 0 && value[0] == '/') {
+		ret = convert_devpath(value, converted_value);
+		if (ret < 0)
+			return ret;
+		value = converted_value;
+	}
+
+	h = get_hierarchy(ops, controller);
+	if (!h) {
+		ERROR("Failed to setup limits for the \"%s\" controller. "
+		      "The controller seems to be unused by \"cgfsng\" cgroup "
+		      "driver or not enabled on the cgroup hierarchy",
+		      controller);
+		errno = ENOENT;
+		return -ENOENT;
+	}
+
+	fullpath = must_make_path(h->container_full_path, filename, NULL);
+
+retry:
+	ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666);
+	if (ret != 0) {
+		if (retry_count < max_retry) {
+			SYSERROR("setting cgroup config for ready process caused \"failed to write %s to %s\".", value, fullpath);
+			(void)isulad_cg_legacy_handle_cpuset_hierarchy(h, container_cgroup);
+			(void)isulad_mkdir_eexist_on_last(h->container_full_path, 0755);
+			usleep(100 * 1000); /* 100 millisecond */
+			retry_count++;
+			goto retry;
+		}
+		lxc_write_error_message(ops->errfd,
+		                        "%s:%d: setting cgroup config for ready process caused \"failed to write %s to %s: %s\".",
+		                        __FILE__, __LINE__, value, fullpath, strerror(errno));
+	}
+	free(fullpath);
+	return ret;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_setup_limits_legacy(struct cgroup_ops *ops,
+						    struct lxc_conf *conf,
+						    bool do_devices)
+{
+	__do_free struct lxc_list *sorted_cgroup_settings = NULL;
+	struct lxc_list *cgroup_settings = &conf->cgroup;
+	struct lxc_list *iterator, *next;
+	struct lxc_cgroup *cg;
+	bool ret = false;
+	char value[21 + 1] = { 0 };
+	long long int readvalue, setvalue;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!conf)
+		return ret_set_errno(false, EINVAL);
+
+	cgroup_settings = &conf->cgroup;
+	if (lxc_list_empty(cgroup_settings))
+		return true;
+
+	if (!ops->hierarchies)
+		return ret_set_errno(false, EINVAL);
+
+	if (pure_unified_layout(ops))
+		return true;
+
+	sorted_cgroup_settings = sort_cgroup_settings(cgroup_settings);
+	if (!sorted_cgroup_settings)
+		return false;
+
+	lxc_list_for_each(iterator, sorted_cgroup_settings) {
+		cg = iterator->elem;
+
+		if (do_devices == !strncmp("devices", cg->subsystem, 7)) {
+			const char *cgvalue = cg->value;
+			if (strcmp(cg->subsystem, "files.limit") == 0) {
+				if (lxc_safe_long_long(cgvalue, &setvalue) != 0) {
+					SYSERROR("Invalid integer value %s", cgvalue);
+					goto out;
+				}
+				if (setvalue <= 0) {
+					cgvalue = "max";
+				}
+			}
+			if (isulad_cg_legacy_set_data(ops, cg->subsystem, cgvalue)) {
+				if (do_devices && (errno == EACCES || errno == EPERM)) {
+					SYSWARN("Failed to set \"%s\" to \"%s\"", cg->subsystem, cgvalue);
+					continue;
+				}
+				SYSERROR("Failed to set \"%s\" to \"%s\"", cg->subsystem, cgvalue);
+				goto out;
+			}
+			DEBUG("Set controller \"%s\" set to \"%s\"", cg->subsystem, cgvalue);
+		}
+
+		// isulad: check cpu shares
+		if (strcmp(cg->subsystem, "cpu.shares") == 0) {
+			if (isulad_cg_legacy_get_data(ops, cg->subsystem, value, sizeof(value) - 1) < 0) {
+				SYSERROR("Error get %s", cg->subsystem);
+				goto out;
+			}
+			trim(value);
+			if (lxc_safe_long_long(cg->value, &setvalue) != 0) {
+				SYSERROR("Invalid value %s", cg->value);
+				goto out;
+			}
+			if (lxc_safe_long_long(value, &readvalue) != 0) {
+				SYSERROR("Invalid value %s", value);
+				goto out;
+			}
+			if (setvalue > readvalue) {
+				ERROR("The maximum allowed cpu-shares is %s", value);
+				lxc_write_error_message(ops->errfd,
+				                        "%s:%d: setting cgroup config for ready process caused \"The maximum allowed cpu-shares is %s\".",
+				                        __FILE__, __LINE__, value);
+				goto out;
+			} else if (setvalue < readvalue) {
+				ERROR("The minimum allowed cpu-shares is %s", value);
+				lxc_write_error_message(ops->errfd,
+				                        "%s:%d: setting cgroup config for ready process caused \"The minimum allowed cpu-shares is %s\".",
+				                        __FILE__, __LINE__, value);
+				goto out;
+			}
+		}
+	}
+
+	ret = true;
+	INFO("Limits for the legacy cgroup hierarchies have been setup");
+out:
+	lxc_list_for_each_safe(iterator, sorted_cgroup_settings, next) {
+		lxc_list_del(iterator);
+		free(iterator);
+	}
+
+	return ret;
+}
+
+/*
+ * Some of the parsing logic comes from the original cgroup device v1
+ * implementation in the kernel.
+ */
+static int bpf_device_cgroup_prepare(struct cgroup_ops *ops,
+				     struct lxc_conf *conf, const char *key,
+				     const char *val)
+{
+#ifdef HAVE_STRUCT_BPF_CGROUP_DEV_CTX
+	struct device_item device_item = {0};
+	int ret;
+
+	if (strcmp("devices.allow", key) == 0 && *val == '/')
+		ret = device_cgroup_rule_parse_devpath(&device_item, val);
+	else
+		ret = device_cgroup_rule_parse(&device_item, key, val);
+	if (ret < 0)
+		return log_error_errno(-1, EINVAL, "Failed to parse device string %s=%s", key, val);
+
+	ret = bpf_list_add_device(conf, &device_item);
+	if (ret < 0)
+		return -1;
+#endif
+	return 0;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_setup_limits(struct cgroup_ops *ops,
+					     struct lxc_handler *handler)
+{
+	__do_free char *path = NULL;
+	struct lxc_list *cgroup_settings, *iterator;
+	struct hierarchy *h;
+	struct lxc_conf *conf;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!ops->container_cgroup)
+		return ret_set_errno(false, EINVAL);
+
+	if (!handler || !handler->conf)
+		return ret_set_errno(false, EINVAL);
+	conf = handler->conf;
+
+	if (lxc_list_empty(&conf->cgroup2))
+		return true;
+	cgroup_settings = &conf->cgroup2;
+
+	if (!pure_unified_layout(ops))
+		return true;
+
+	if (!ops->unified)
+		return false;
+	h = ops->unified;
+
+	lxc_list_for_each (iterator, cgroup_settings) {
+		struct lxc_cgroup *cg = iterator->elem;
+		int ret;
+
+		if (strncmp("devices", cg->subsystem, 7) == 0) {
+			ret = bpf_device_cgroup_prepare(ops, conf, cg->subsystem,
+							cg->value);
+		} else if (strcmp(cg->subsystem, "files.limit") == 0) {
+			long long int setvalue = 0;
+			const char *cgvalue = cg->value;
+
+			if (lxc_safe_long_long(cgvalue, &setvalue) != 0)
+				return log_error(false, "Invalid integer value %s", cgvalue);
+
+			if (setvalue <= 0)
+				cgvalue = "max";
+
+			ret = lxc_write_openat(h->container_full_path,
+					       cg->subsystem, cgvalue,
+					       strlen(cgvalue));
+			if (ret < 0)
+				return log_error_errno(false, errno, "Failed to set \"%s\" to \"%s\"",
+						       cg->subsystem, cgvalue);
+		} else {
+			if (strcmp(cg->subsystem, "io.weight") == 0 || strcmp(cg->subsystem, "io.bfq.weight") == 0) {
+				path = must_make_path(h->container_full_path, cg->subsystem, NULL);
+				if (!file_exists(path)) {
+					continue;
+				}
+			}
+			ret = lxc_write_openat(h->container_full_path,
+					       cg->subsystem, cg->value,
+					       strlen(cg->value));
+			if (ret < 0)
+				return log_error_errno(false, errno, "Failed to set \"%s\" to \"%s\"",
+						       cg->subsystem, cg->value);
+		}
+		TRACE("Set \"%s\" to \"%s\"", cg->subsystem, cg->value);
+	}
+
+	return log_info(true, "Limits for the unified cgroup hierarchy have been setup");
+}
+
+__cgfsng_ops bool isulad_cgfsng_devices_activate(struct cgroup_ops *ops,
+					  struct lxc_handler *handler)
+{
+#ifdef HAVE_STRUCT_BPF_CGROUP_DEV_CTX
+	__do_bpf_program_free struct bpf_program *devices = NULL;
+	int ret;
+	struct lxc_conf *conf;
+	struct hierarchy *unified;
+	struct lxc_list *it;
+	struct bpf_program *devices_old;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!ops->container_cgroup)
+		return ret_set_errno(false, EEXIST);
+
+	if (!handler || !handler->conf)
+		return ret_set_errno(false, EINVAL);
+	conf = handler->conf;
+
+	unified = ops->unified;
+	if (!unified || !unified->bpf_device_controller ||
+	    !unified->container_full_path || lxc_list_empty(&conf->devices))
+		return true;
+
+	devices = bpf_program_new(BPF_PROG_TYPE_CGROUP_DEVICE);
+	if (!devices)
+		return log_error_errno(false, ENOMEM, "Failed to create new bpf program");
+
+	ret = bpf_program_init(devices);
+	if (ret)
+		return log_error_errno(false, ENOMEM, "Failed to initialize bpf program");
+
+	lxc_list_for_each(it, &conf->devices) {
+		struct device_item *cur = it->elem;
+
+		ret = bpf_program_append_device(devices, cur);
+		if (ret)
+			return log_error_errno(false, ENOMEM, "Failed to add new rule to bpf device program: type %c, major %d, minor %d, access %s, allow %d, global_rule %d",
+					       cur->type,
+					       cur->major,
+					       cur->minor,
+					       cur->access,
+					       cur->allow,
+					       cur->global_rule);
+		TRACE("Added rule to bpf device program: type %c, major %d, minor %d, access %s, allow %d, global_rule %d",
+		      cur->type,
+		      cur->major,
+		      cur->minor,
+		      cur->access,
+		      cur->allow,
+		      cur->global_rule);
+	}
+
+	ret = bpf_program_finalize(devices);
+	if (ret)
+		return log_error_errno(false, ENOMEM, "Failed to finalize bpf program");
+
+	ret = bpf_program_cgroup_attach(devices, BPF_CGROUP_DEVICE,
+					unified->container_full_path,
+					BPF_F_ALLOW_MULTI);
+	if (ret)
+		return log_error_errno(false, ENOMEM, "Failed to attach bpf program");
+
+	/* Replace old bpf program. */
+	devices_old = move_ptr(conf->cgroup2_devices);
+	conf->cgroup2_devices = move_ptr(devices);
+	devices = move_ptr(devices_old);
+#endif
+	return true;
+}
+
+bool __cgfsng_delegate_controllers(struct cgroup_ops *ops, const char *cgroup)
+{
+	__do_free char *add_controllers = NULL, *base_path = NULL;
+	__do_free_string_list char **parts = NULL;
+	struct hierarchy *unified = ops->unified;
+	ssize_t parts_len;
+	char **it;
+	size_t full_len = 0;
+
+	if (!ops->hierarchies || !pure_unified_layout(ops) ||
+	    !unified->controllers[0])
+		return true;
+
+	/* For now we simply enable all controllers that we have detected by
+	 * creating a string like "+memory +pids +cpu +io".
+	 * TODO: In the near future we might want to support "-<controller>"
+	 * etc. but whether supporting semantics like this make sense will need
+	 * some thinking.
+	 */
+	for (it = unified->controllers; it && *it; it++) {
+		full_len += strlen(*it) + 2;
+		add_controllers = must_realloc(add_controllers, full_len + 1);
+
+		if (unified->controllers[0] == *it)
+			add_controllers[0] = '\0';
+
+		(void)strlcat(add_controllers, "+", full_len + 1);
+		(void)strlcat(add_controllers, *it, full_len + 1);
+
+		if ((it + 1) && *(it + 1))
+			(void)strlcat(add_controllers, " ", full_len + 1);
+	}
+
+	parts = lxc_string_split(cgroup, '/');
+	if (!parts)
+		return false;
+
+	parts_len = lxc_array_len((void **)parts);
+	if (parts_len > 0)
+		parts_len--;
+
+	base_path = must_make_path(unified->mountpoint, unified->container_base_path, NULL);
+	for (ssize_t i = -1; i < parts_len; i++) {
+		int ret;
+		__do_free char *target = NULL;
+
+		if (i >= 0)
+			base_path = must_append_path(base_path, parts[i], NULL);
+		target = must_make_path(base_path, "cgroup.subtree_control", NULL);
+		ret = lxc_writeat(-1, target, add_controllers, full_len);
+		if (ret < 0)
+			return log_error_errno(false, errno, "Could not enable \"%s\" controllers in the unified cgroup \"%s\"",
+					       add_controllers, target);
+		TRACE("Enable \"%s\" controllers in the unified cgroup \"%s\"", add_controllers, target);
+	}
+
+	return true;
+}
+
+__cgfsng_ops bool isulad_cgfsng_monitor_delegate_controllers(struct cgroup_ops *ops)
+{
+	return true;
+}
+
+__cgfsng_ops bool isulad_cgfsng_payload_delegate_controllers(struct cgroup_ops *ops)
+{
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_payload_delegate_controllers");
+		return true;
+	}
+#endif
+
+	return __cgfsng_delegate_controllers(ops, ops->container_cgroup);
+}
+
+static bool cgroup_use_wants_controllers(const struct cgroup_ops *ops,
+				       char **controllers)
+{
+	if (!ops->cgroup_use)
+		return true;
+
+	for (char **cur_ctrl = controllers; cur_ctrl && *cur_ctrl; cur_ctrl++) {
+		bool found = false;
+
+		for (char **cur_use = ops->cgroup_use; cur_use && *cur_use; cur_use++) {
+			if (strcmp(*cur_use, *cur_ctrl) != 0)
+				continue;
+
+			found = true;
+			break;
+		}
+
+		if (found)
+			continue;
+
+		return false;
+	}
+
+	return true;
+}
+
+static void cg_unified_delegate(char ***delegate)
+{
+	__do_free char *buf = NULL;
+	char *standard[] = {"cgroup.subtree_control", "cgroup.threads", NULL};
+	char *token;
+	int idx;
+
+	buf = read_file("/sys/kernel/cgroup/delegate");
+	if (!buf) {
+		for (char **p = standard; p && *p; p++) {
+			idx = append_null_to_list((void ***)delegate);
+			(*delegate)[idx] = must_copy_string(*p);
+		}
+		SYSWARN("Failed to read /sys/kernel/cgroup/delegate");
+		return;
+	}
+
+	lxc_iterate_parts (token, buf, " \t\n") {
+		/*
+		 * We always need to chown this for both cgroup and
+		 * cgroup2.
+		 */
+		if (strcmp(token, "cgroup.procs") == 0)
+			continue;
+
+		idx = append_null_to_list((void ***)delegate);
+		(*delegate)[idx] = must_copy_string(token);
+	}
+}
+
+/* At startup, parse_hierarchies finds all the info we need about cgroup
+ * mountpoints and current cgroups, and stores it in @d.
+ */
+static int cg_hybrid_init(struct cgroup_ops *ops, bool relative, bool unprivileged)
+{
+	__do_free char *basecginfo = NULL, *line = NULL;
+	__do_free_string_list char **klist = NULL, **nlist = NULL;
+	__do_fclose FILE *f = NULL;
+	int ret;
+	size_t len = 0;
+
+	/* Root spawned containers escape the current cgroup, so use init's
+	 * cgroups as our base in that case.
+	 */
+	if (!relative && (geteuid() == 0))
+		basecginfo = read_file("/proc/1/cgroup");
+	else
+		basecginfo = read_file("/proc/self/cgroup");
+	if (!basecginfo)
+		return ret_set_errno(-1, ENOMEM);
+
+	ret = get_existing_subsystems(&klist, &nlist);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to retrieve available legacy cgroup controllers");
+
+	f = fopen("/proc/self/mountinfo", "re");
+	if (!f)
+		return log_error_errno(-1, errno, "Failed to open \"/proc/self/mountinfo\"");
+
+	lxc_cgfsng_print_basecg_debuginfo(basecginfo, klist, nlist);
+
+	while (getline(&line, &len, f) != -1) {
+		__do_free char *base_cgroup = NULL, *mountpoint = NULL;
+		__do_free_string_list char **controller_list = NULL;
+		int type;
+		struct hierarchy *new;
+
+		type = get_cgroup_version(line);
+		if (type == 0)
+			continue;
+
+		if (type == CGROUP2_SUPER_MAGIC && ops->unified)
+			continue;
+
+		if (ops->cgroup_layout == CGROUP_LAYOUT_UNKNOWN) {
+			if (type == CGROUP2_SUPER_MAGIC)
+				ops->cgroup_layout = CGROUP_LAYOUT_UNIFIED;
+			else if (type == CGROUP_SUPER_MAGIC)
+				ops->cgroup_layout = CGROUP_LAYOUT_LEGACY;
+		} else if (ops->cgroup_layout == CGROUP_LAYOUT_UNIFIED) {
+			if (type == CGROUP_SUPER_MAGIC)
+				ops->cgroup_layout = CGROUP_LAYOUT_HYBRID;
+		} else if (ops->cgroup_layout == CGROUP_LAYOUT_LEGACY) {
+			if (type == CGROUP2_SUPER_MAGIC)
+				ops->cgroup_layout = CGROUP_LAYOUT_HYBRID;
+		}
+
+		controller_list = cg_hybrid_get_controllers(klist, nlist, line, type);
+		if (!controller_list && type == CGROUP_SUPER_MAGIC)
+			continue;
+
+		if (type == CGROUP_SUPER_MAGIC)
+			if (controller_list_is_dup(ops->hierarchies, controller_list)) {
+				TRACE("Skipping duplicating controller");
+				continue;
+			}
+
+		mountpoint = cg_hybrid_get_mountpoint(line);
+		if (!mountpoint) {
+			WARN("Failed parsing mountpoint from \"%s\"", line);
+			continue;
+		}
+
+		if (type == CGROUP_SUPER_MAGIC)
+			base_cgroup = cg_hybrid_get_current_cgroup(basecginfo, controller_list[0], CGROUP_SUPER_MAGIC);
+		else
+			base_cgroup = cg_hybrid_get_current_cgroup(basecginfo, NULL, CGROUP2_SUPER_MAGIC);
+		if (!base_cgroup) {
+			WARN("Failed to find current cgroup");
+			continue;
+		}
+
+		trim(base_cgroup);
+		prune_init_scope(base_cgroup);
+
+		/* isulad: do not test writeable, if we run isulad in docker without cgroup namespace.
+		 * the base_cgroup will be docker/XXX.., mountpoint+base_cgroup may be not exist */
+
+		/*
+		 * reason:base cgroup may be started with /system.slice when cg_hybrid_init
+		 *	read /proc/1/cgroup on host, and cgroup init will set all containers
+		 *	cgroup path under /sys/fs/cgroup/<controller>/system.slice/xxx/lxc
+		 *	directory, this is not consistent with docker. The default cgroup path
+		 *	should be under /sys/fs/cgroup/<controller>/lxc directory.
+		 */
+
+		if (strlen(base_cgroup) > 1 && base_cgroup[0] == '/') {
+			base_cgroup[1] = '\0';
+		}
+
+		if (type == CGROUP2_SUPER_MAGIC) {
+			char *cgv2_ctrl_path;
+
+			cgv2_ctrl_path = must_make_path(mountpoint, base_cgroup,
+							"cgroup.controllers",
+							NULL);
+
+			controller_list = cg_unified_get_controllers(cgv2_ctrl_path);
+			free(cgv2_ctrl_path);
+			if (!controller_list) {
+				controller_list = cg_unified_make_empty_controller();
+				TRACE("No controllers are enabled for "
+				      "delegation in the unified hierarchy");
+			}
+		}
+
+		/* Exclude all controllers that cgroup use does not want. */
+		if (!cgroup_use_wants_controllers(ops, controller_list)) {
+			TRACE("Skipping controller");
+			continue;
+		}
+
+		new = add_hierarchy(&ops->hierarchies, move_ptr(controller_list), move_ptr(mountpoint), move_ptr(base_cgroup), type);
+		if (type == CGROUP2_SUPER_MAGIC && !ops->unified) {
+			if (unprivileged)
+				cg_unified_delegate(&new->cgroup2_chown);
+			ops->unified = new;
+		}
+	}
+
+	TRACE("Writable cgroup hierarchies:");
+	lxc_cgfsng_print_hierarchies(ops);
+
+	/* verify that all controllers in cgroup.use and all crucial
+	 * controllers are accounted for
+	 */
+	if (!all_controllers_found(ops))
+		return log_error_errno(-1, ENOENT, "Failed to find all required controllers");
+
+	return 0;
+}
+
+/* Get current cgroup from /proc/self/cgroup for the cgroupfs v2 hierarchy. */
+static char *cg_unified_get_current_cgroup(bool relative)
+{
+	__do_free char *basecginfo = NULL;
+	char *copy;
+	char *base_cgroup;
+
+	if (!relative && (geteuid() == 0))
+		basecginfo = read_file("/proc/1/cgroup");
+	else
+		basecginfo = read_file("/proc/self/cgroup");
+	if (!basecginfo)
+		return NULL;
+
+	base_cgroup = strstr(basecginfo, "0::/");
+	if (!base_cgroup)
+		return NULL;
+
+	base_cgroup = base_cgroup + 3;
+	copy = copy_to_eol(base_cgroup);
+	if (!copy)
+		return NULL;
+
+	return trim(copy);
+}
+
+static int cg_unified_init(struct cgroup_ops *ops, bool relative,
+			   bool unprivileged)
+{
+	__do_free char *subtree_path = NULL;
+	int ret;
+	char *mountpoint;
+	char **delegatable;
+	struct hierarchy *new;
+	char *base_cgroup = NULL;
+
+	ret = unified_cgroup_hierarchy();
+	if (ret == -ENOMEDIUM)
+		return ret_errno(ENOMEDIUM);
+
+	if (ret != CGROUP2_SUPER_MAGIC)
+		return 0;
+
+	base_cgroup = cg_unified_get_current_cgroup(relative);
+	if (!base_cgroup)
+		return ret_errno(EINVAL);
+	if (!relative)
+		prune_init_scope(base_cgroup);
+
+	/*
+	 * We assume that the cgroup we're currently in has been delegated to
+	 * us and we are free to further delege all of the controllers listed
+	 * in cgroup.controllers further down the hierarchy.
+	 */
+	mountpoint = must_copy_string(DEFAULT_CGROUP_MOUNTPOINT);
+	subtree_path = must_make_path(mountpoint, base_cgroup, "cgroup.controllers", NULL);
+	delegatable = cg_unified_get_controllers(subtree_path);
+	if (!delegatable)
+		delegatable = cg_unified_make_empty_controller();
+	if (!delegatable[0]) {
+		TRACE("No controllers are enabled for delegation");
+#ifdef HAVE_ISULAD
+		ops->no_controller = true;
+#endif
+	}
+
+	/* TODO: If the user requested specific controllers via lxc.cgroup.use
+	 * we should verify here. The reason I'm not doing it right is that I'm
+	 * not convinced that lxc.cgroup.use will be the future since it is a
+	 * global property. I much rather have an option that lets you request
+	 * controllers per container.
+	 */
+
+	new = add_hierarchy(&ops->hierarchies, delegatable, mountpoint, base_cgroup, CGROUP2_SUPER_MAGIC);
+	if (unprivileged)
+		cg_unified_delegate(&new->cgroup2_chown);
+
+	if (bpf_devices_cgroup_supported())
+		new->bpf_device_controller = 1;
+
+	ops->cgroup_layout = CGROUP_LAYOUT_UNIFIED;
+	ops->unified = new;
+
+	return CGROUP2_SUPER_MAGIC;
+}
+
+static int isulad_cg_init(struct cgroup_ops *ops, struct lxc_conf *conf)
+{
+	int ret;
+	const char *tmp;
+	bool relative = conf->cgroup_meta.relative;
+
+	tmp = lxc_global_config_value("lxc.cgroup.use");
+	if (tmp) {
+		__do_free char *pin = NULL;
+		char *chop, *cur;
+
+		pin = must_copy_string(tmp);
+		chop = pin;
+
+		lxc_iterate_parts(cur, chop, ",")
+			must_append_string(&ops->cgroup_use, cur);
+	}
+
+	ret = cg_unified_init(ops, relative, !lxc_list_empty(&conf->id_map));
+	if (ret < 0)
+		return -1;
+
+	if (ret == CGROUP2_SUPER_MAGIC)
+		return 0;
+
+	return cg_hybrid_init(ops, relative, !lxc_list_empty(&conf->id_map));
+}
+
+__cgfsng_ops static int isulad_cgfsng_data_init(struct cgroup_ops *ops, struct lxc_conf *conf)
+{
+	const char *cgroup_pattern;
+	const char *cgroup_tree;
+	__do_free char *container_cgroup = NULL, *__cgroup_tree = NULL;
+	size_t len;
+
+	if (!ops)
+		return ret_set_errno(-1, ENOENT);
+
+	/* copy system-wide cgroup information */
+	cgroup_pattern = lxc_global_config_value("lxc.cgroup.pattern");
+	if (cgroup_pattern && strcmp(cgroup_pattern, "") != 0)
+		ops->cgroup_pattern = must_copy_string(cgroup_pattern);
+
+	if (conf->cgroup_meta.dir) {
+		cgroup_tree = conf->cgroup_meta.dir;
+		container_cgroup = must_concat(&len, cgroup_tree, "/", conf->name, NULL);
+	} else if (ops->cgroup_pattern) {
+		__cgroup_tree = lxc_string_replace("%n", conf->name, ops->cgroup_pattern);
+		if (!__cgroup_tree)
+			return ret_set_errno(-1, ENOMEM);
+
+		cgroup_tree = __cgroup_tree;
+		container_cgroup = must_concat(&len, cgroup_tree, NULL);
+	} else {
+		cgroup_tree = NULL;
+		container_cgroup = must_concat(&len, conf->name, NULL);
+	}
+	if (!container_cgroup)
+		return ret_set_errno(-1, ENOMEM);
+
+	ops->container_cgroup = move_ptr(container_cgroup);
+
+	return 0;
+}
+
+struct cgroup_ops *cgfsng_ops_init(struct lxc_conf *conf)
+{
+	__do_free struct cgroup_ops *cgfsng_ops = NULL;
+
+	cgfsng_ops = malloc(sizeof(struct cgroup_ops));
+	if (!cgfsng_ops)
+		return ret_set_errno(NULL, ENOMEM);
+
+	memset(cgfsng_ops, 0, sizeof(struct cgroup_ops));
+	cgfsng_ops->cgroup_layout = CGROUP_LAYOUT_UNKNOWN;
+
+	if (isulad_cg_init(cgfsng_ops, conf))
+		return NULL;
+
+	cgfsng_ops->data_init = isulad_cgfsng_data_init;
+
+	cgfsng_ops->errfd = conf ? conf->errpipe[1] : -1;
+	cgfsng_ops->get_cgroup_full_path = isulad_cgfsng_get_cgroup_full_path;
+	cgfsng_ops->payload_destroy = isulad_cgfsng_payload_destroy;
+	cgfsng_ops->monitor_destroy = isulad_cgfsng_monitor_destroy;
+	cgfsng_ops->monitor_create = isulad_cgfsng_monitor_create;
+	cgfsng_ops->monitor_enter = isulad_cgfsng_monitor_enter;
+	cgfsng_ops->monitor_delegate_controllers = isulad_cgfsng_monitor_delegate_controllers;
+	cgfsng_ops->payload_delegate_controllers = isulad_cgfsng_payload_delegate_controllers;
+	cgfsng_ops->payload_create = isulad_cgfsng_payload_create;
+	cgfsng_ops->payload_enter = isulad_cgfsng_payload_enter;
+	cgfsng_ops->payload_finalize = isulad_cgfsng_payload_finalize;
+	cgfsng_ops->escape = isulad_cgfsng_escape;
+	cgfsng_ops->num_hierarchies = isulad_cgfsng_num_hierarchies;
+	cgfsng_ops->get_hierarchies = isulad_cgfsng_get_hierarchies;
+	cgfsng_ops->get_cgroup = isulad_cgfsng_get_cgroup;
+	cgfsng_ops->get = isulad_cgfsng_get;
+	cgfsng_ops->set = isulad_cgfsng_set;
+	cgfsng_ops->freeze = isulad_cgfsng_freeze;
+	cgfsng_ops->unfreeze = isulad_cgfsng_unfreeze;
+	cgfsng_ops->setup_limits_legacy = isulad_cgfsng_setup_limits_legacy;
+	cgfsng_ops->setup_limits = isulad_cgfsng_setup_limits;
+	cgfsng_ops->driver = "isulad_cgfsng";
+	cgfsng_ops->version = "1.0.0";
+	cgfsng_ops->attach = isulad_cgfsng_attach;
+	cgfsng_ops->chown = isulad_cgfsng_chown;
+	cgfsng_ops->mount = isulad_cgfsng_mount;
+	cgfsng_ops->devices_activate = isulad_cgfsng_devices_activate;
+
+	return move_ptr(cgfsng_ops);
+}
diff --git a/src/lxc/json/defs.c b/src/lxc/json/defs.c
new file mode 100644
index 0000000..4bf569a
--- /dev/null
+++ b/src/lxc/json/defs.c
@@ -0,0 +1,205 @@
+// Generated from defs.json. Do not edit!
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+#include <read-file.h>
+#include "defs.h"
+
+defs_hook *make_defs_hook(yajl_val tree, struct parser_context *ctx, parser_error *err) {
+    defs_hook *ret = NULL;
+    *err = 0;
+    if (tree == NULL)
+        return ret;
+    ret = safe_malloc(sizeof(*ret));
+    {
+        yajl_val val = get_val(tree, "path", yajl_t_string);
+        if (val != NULL) {
+            char *str = YAJL_GET_STRING(val);
+            ret->path = safe_strdup(str ? str : "");
+        }
+    }
+    {
+        yajl_val tmp = get_val(tree, "args", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->args_len = YAJL_GET_ARRAY(tmp)->len;
+            if (YAJL_GET_ARRAY(tmp)->len > SIZE_MAX / sizeof(*ret->args) - 1) {
+                free_defs_hook(ret);
+                return NULL;
+            }
+            ret->args = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->args));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                if (val != NULL) {
+                    char *str = YAJL_GET_STRING(val);
+                    ret->args[i] = safe_strdup(str ? str : "");
+                }
+            }
+        }
+    }
+    {
+        yajl_val tmp = get_val(tree, "env", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->env_len = YAJL_GET_ARRAY(tmp)->len;
+            if (YAJL_GET_ARRAY(tmp)->len > SIZE_MAX / sizeof(*ret->env) - 1) {
+                free_defs_hook(ret);
+                return NULL;
+            }
+            ret->env = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->env));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                if (val != NULL) {
+                    char *str = YAJL_GET_STRING(val);
+                    ret->env[i] = safe_strdup(str ? str : "");
+                }
+            }
+        }
+    }
+    {
+        yajl_val val = get_val(tree, "timeout", yajl_t_number);
+        if (val != NULL) {
+            int invalid = common_safe_int(YAJL_GET_NUMBER(val), (int *)&ret->timeout);
+            if (invalid) {
+                if (asprintf(err, "Invalid value '%s' with type 'integer' for key 'timeout': %s", YAJL_GET_NUMBER(val), strerror(-invalid)) < 0)
+                    *err = safe_strdup("error allocating memory");
+                free_defs_hook(ret);
+                return NULL;
+            }
+        }
+    }
+    if (ret->path == NULL) {
+        if (asprintf(err, "Required field '%s' not present", "path") < 0)
+            *err = safe_strdup("error allocating memory");
+        free_defs_hook(ret);
+        return NULL;
+    }
+
+    if (tree->type == yajl_t_object && (ctx->options & PARSE_OPTIONS_STRICT)) {
+        int i;
+        for (i = 0; i < tree->u.object.len; i++)
+            if (strcmp(tree->u.object.keys[i], "path") &&
+                strcmp(tree->u.object.keys[i], "args") &&
+                strcmp(tree->u.object.keys[i], "env") &&
+                strcmp(tree->u.object.keys[i], "timeout")) {
+                if (ctx->stderr > 0)
+                    fprintf(ctx->stderr, "WARNING: unknown key found: %s\n", tree->u.object.keys[i]);
+            }
+        }
+    return ret;
+}
+
+void free_defs_hook(defs_hook *ptr) {
+    if (ptr == NULL)
+        return;
+    free(ptr->path);
+    ptr->path = NULL;
+    if (ptr->args != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->args_len; i++) {
+            if (ptr->args[i] != NULL) {
+                free(ptr->args[i]);
+                ptr->args[i] = NULL;
+            }
+        }
+        free(ptr->args);
+        ptr->args = NULL;
+    }
+    if (ptr->env != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->env_len; i++) {
+            if (ptr->env[i] != NULL) {
+                free(ptr->env[i]);
+                ptr->env[i] = NULL;
+            }
+        }
+        free(ptr->env);
+        ptr->env = NULL;
+    }
+    free(ptr);
+}
+
+yajl_gen_status gen_defs_hook(yajl_gen g, defs_hook *ptr, struct parser_context *ctx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    *err = 0;
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->path != NULL)) {
+        char *str = "";
+        stat = reformat_map_key(g, "path", strlen("path"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->path != NULL) {
+            str = ptr->path;
+        }
+        stat = reformat_string(g, str, strlen(str));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr != NULL && ptr->args != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "args", strlen("args"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->args != NULL) {
+            len = ptr->args_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = reformat_string(g, ptr->args[i], strlen(ptr->args[i]));
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr != NULL && ptr->env != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "env", strlen("env"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->env != NULL) {
+            len = ptr->env_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = reformat_string(g, ptr->env[i], strlen(ptr->env[i]));
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->timeout)) {
+        long long int num = 0;
+        stat = reformat_map_key(g, "timeout", strlen("timeout"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->timeout) {
+            num = (long long int)ptr->timeout;
+        }
+        stat = reformat_int(g, num);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    return yajl_gen_status_ok;
+}
diff --git a/src/lxc/json/defs.h b/src/lxc/json/defs.h
new file mode 100644
index 0000000..0bbd8ac
--- /dev/null
+++ b/src/lxc/json/defs.h
@@ -0,0 +1,37 @@
+// Generated from defs.json. Do not edit!
+#ifndef DEFS_SCHEMA_H
+#define DEFS_SCHEMA_H
+
+#include <sys/types.h>
+#include <stdint.h>
+#include "json_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    char *path;
+
+    char **args;
+    size_t args_len;
+
+    char **env;
+    size_t env_len;
+
+    int timeout;
+
+}
+defs_hook;
+
+void free_defs_hook(defs_hook *ptr);
+
+defs_hook *make_defs_hook(yajl_val tree, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_defs_hook(yajl_gen g, defs_hook *ptr, struct parser_context *ctx, parser_error *err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lxc/json/json_common.c b/src/lxc/json/json_common.c
new file mode 100755
index 0000000..ec20c59
--- /dev/null
+++ b/src/lxc/json/json_common.c
@@ -0,0 +1,1153 @@
+// Auto generated file. Do not edit!
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <errno.h>
+#include <limits.h>
+#include "json_common.h"
+
+#define MAX_NUM_STR_LEN 21
+
+yajl_gen_status reformat_number(void *ctx, const char *str, size_t len) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_number(g, str, len);
+}
+
+yajl_gen_status reformat_uint(void *ctx, long long unsigned int num) {
+    char numstr[MAX_NUM_STR_LEN];
+    int ret;
+
+    ret = snprintf(numstr, MAX_NUM_STR_LEN, "%llu", num);
+    if (ret < 0 || ret >= MAX_NUM_STR_LEN) {
+        return yajl_gen_in_error_state;
+    }
+    return reformat_number(ctx, (const char *)numstr, strlen(numstr));
+}
+
+yajl_gen_status reformat_int(void *ctx, long long int num) {
+    char numstr[MAX_NUM_STR_LEN];
+    int ret;
+
+    ret = snprintf(numstr, MAX_NUM_STR_LEN, "%lld", num);
+    if (ret < 0 || ret >= MAX_NUM_STR_LEN) {
+        return yajl_gen_in_error_state;
+    }
+    return reformat_number(ctx, (const char *)numstr, strlen(numstr));
+}
+
+yajl_gen_status reformat_double(void *ctx, double num) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_double(g, num);
+}
+
+yajl_gen_status reformat_string(void *ctx, const char *str, size_t len) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_string(g, (const unsigned char *)str, len);
+}
+
+yajl_gen_status reformat_null(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_null(g);
+}
+
+yajl_gen_status reformat_bool(void *ctx, int boolean) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_bool(g, boolean);
+}
+
+yajl_gen_status reformat_map_key(void *ctx, const char *str, size_t len) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_string(g, (const unsigned char *)str, len);
+}
+
+yajl_gen_status reformat_start_map(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_map_open(g);
+}
+
+yajl_gen_status reformat_end_map(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_map_close(g);
+}
+
+yajl_gen_status reformat_start_array(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_array_open(g);
+}
+
+yajl_gen_status reformat_end_array(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_array_close(g);
+}
+
+bool json_gen_init(yajl_gen *g, struct parser_context *ctx) {
+    *g = yajl_gen_alloc(NULL);
+    if (NULL == *g) {
+        return false;
+
+    }
+    yajl_gen_config(*g, yajl_gen_beautify, !(ctx->options & GEN_OPTIONS_SIMPLIFY));
+    yajl_gen_config(*g, yajl_gen_validate_utf8, !(ctx->options & GEN_OPTIONS_NOT_VALIDATE_UTF8));
+    return true;
+}
+
+yajl_val get_val(yajl_val tree, const char *name, yajl_type type) {
+    const char *path[] = { name, NULL };
+    return yajl_tree_get(tree, path, type);
+}
+
+void *safe_malloc(size_t size) {
+    void *ret = NULL;
+    if (size == 0) {
+        abort();
+    }
+    ret = calloc(1, size);
+    if (ret == NULL) {
+        abort();
+    }
+    return ret;
+}
+
+int common_safe_double(const char *numstr, double *converted) {
+    char *err_str = NULL;
+    double d;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    d = strtod(numstr, &err_str);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err_str == NULL || err_str == numstr || *err_str != '\0') {
+        return -EINVAL;
+    }
+
+    *converted = d;
+    return 0;
+}
+
+int common_safe_uint8(const char *numstr, uint8_t *converted) {
+    char *err = NULL;
+    unsigned long int uli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    uli = strtoul(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (uli > UINT8_MAX) {
+        return -ERANGE;
+    }
+
+    *converted = (uint8_t)uli;
+    return 0;
+}
+
+int common_safe_uint16(const char *numstr, uint16_t *converted) {
+    char *err = NULL;
+    unsigned long int uli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    uli = strtoul(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (uli > UINT16_MAX) {
+        return -ERANGE;
+    }
+
+    *converted = (uint16_t)uli;
+    return 0;
+}
+
+int common_safe_uint32(const char *numstr, uint32_t *converted) {
+    char *err = NULL;
+    unsigned long long int ull;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    ull = strtoull(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (ull > UINT32_MAX) {
+        return -ERANGE;
+    }
+
+    *converted = (uint32_t)ull;
+    return 0;
+}
+
+int common_safe_uint64(const char *numstr, uint64_t *converted) {
+    char *err = NULL;
+    unsigned long long int ull;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    ull = strtoull(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    *converted = (uint64_t)ull;
+    return 0;
+}
+
+int common_safe_uint(const char *numstr, unsigned int *converted) {
+    char *err = NULL;
+    unsigned long long int ull;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    ull = strtoull(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (ull > UINT_MAX) {
+        return -ERANGE;
+    }
+
+    *converted = (unsigned int)ull;
+    return 0;
+}
+
+int common_safe_int8(const char *numstr, int8_t *converted) {
+    char *err = NULL;
+    long int li;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    li = strtol(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (li > INT8_MAX || li < INT8_MIN) {
+        return -ERANGE;
+    }
+
+    *converted = (int8_t)li;
+    return 0;
+}
+
+int common_safe_int16(const char *numstr, int16_t *converted) {
+    char *err = NULL;
+    long int li;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    li = strtol(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (li > INT16_MAX || li < INT16_MIN) {
+        return -ERANGE;
+    }
+
+    *converted = (int16_t)li;
+    return 0;
+}
+
+int common_safe_int32(const char *numstr, int32_t *converted) {
+    char *err = NULL;
+    long long int lli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    lli = strtol(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (lli > INT32_MAX || lli < INT32_MIN) {
+        return -ERANGE;
+    }
+
+    *converted = (int32_t)lli;
+    return 0;
+}
+
+int common_safe_int64(const char *numstr, int64_t *converted) {
+    char *err = NULL;
+    long long int lli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    lli = strtoll(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    *converted = (int64_t)lli;
+    return 0;
+}
+
+int common_safe_int(const char *numstr, int *converted) {
+    char *err = NULL;
+    long long int lli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    lli = strtol(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (lli > INT_MAX || lli < INT_MIN) {
+        return -ERANGE;
+    }
+
+    *converted = (int)lli;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_int_int(void *ctx, json_map_int_int *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        char numstr[MAX_NUM_STR_LEN];
+        int nret;
+        nret = snprintf(numstr, MAX_NUM_STR_LEN, "%lld", (long long int)map->keys[i]);
+        if (nret < 0 || nret >= MAX_NUM_STR_LEN) {
+            if (!*err && asprintf(err, "Error to print string") < 0) {
+                *(err) = safe_strdup("error allocating memory");
+            }
+            return yajl_gen_in_error_state;
+        }
+        stat = reformat_string(g, numstr, strlen(numstr));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_int(g, map->values[i]);
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_int_int(json_map_int_int *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            // No need to free key for type int
+            // No need to free value for type int
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_int_int *make_json_map_int_int(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_int_int *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(int) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(int));
+        ret->values = safe_malloc((len + 1) * sizeof(int));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+
+            if (srckey != NULL) {
+                int invalid;
+                invalid = common_safe_int(srckey, &(ret->keys[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid key '%s' with type 'int': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_int(ret);
+                    return NULL;
+                }
+            }
+
+            if (srcval != NULL) {
+                int invalid;
+                if (!YAJL_IS_NUMBER(srcval)) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'int' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_int(ret);
+                    return NULL;
+                }
+                invalid = common_safe_int(YAJL_GET_NUMBER(srcval), &(ret->values[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'int' for key '%s': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_int(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_int_int(json_map_int_int *map, int key, int val) {
+    size_t len;
+    int *keys = NULL;
+    int *vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(int) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(int));
+    vals = safe_malloc(len * sizeof(int));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(int));
+        (void)memcpy(vals, map->values, map->len * sizeof(int));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = key;
+    map->values[map->len] = val;
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_int_bool(void *ctx, json_map_int_bool *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        char numstr[MAX_NUM_STR_LEN];
+        int nret;
+        nret = snprintf(numstr, MAX_NUM_STR_LEN, "%lld", (long long int)map->keys[i]);
+        if (nret < 0 || nret >= MAX_NUM_STR_LEN) {
+            if (!*err && asprintf(err, "Error to print string") < 0) {
+                *(err) = safe_strdup("error allocating memory");
+            }
+            return yajl_gen_in_error_state;
+        }
+        stat = reformat_string(g, numstr, strlen(numstr));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_bool(g, map->values[i]);
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_int_bool(json_map_int_bool *map) {
+    if (map != NULL) {
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_int_bool *make_json_map_int_bool(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_int_bool *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(int) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(int));
+        ret->values = safe_malloc((len + 1) * sizeof(bool));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+
+            if (srckey != NULL) {
+                int invalid;
+                invalid = common_safe_int(srckey, &(ret->keys[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid key '%s' with type 'int': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_bool(ret);
+                    return NULL;
+                }
+            }
+
+            if (srcval != NULL) {
+                if (YAJL_IS_TRUE(srcval)) {
+                    ret->values[i] = true;
+                } else if (YAJL_IS_FALSE(srcval)) {
+                    ret->values[i] = false;
+                } else {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'bool' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_bool(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_int_bool(json_map_int_bool *map, int key, bool val) {
+    size_t len;
+    int *keys = NULL;
+    bool *vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(int) - 1) < map->len || (SIZE_MAX / sizeof(bool) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(int));
+    vals = safe_malloc(len * sizeof(bool));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(int));
+        (void)memcpy(vals, map->values, map->len * sizeof(bool));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = key;
+    map->values[map->len] = val;
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_int_string(void *ctx, json_map_int_string *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        char numstr[MAX_NUM_STR_LEN];
+        int nret;
+        nret = snprintf(numstr, MAX_NUM_STR_LEN, "%lld", (long long int)map->keys[i]);
+        if (nret < 0 || nret >= MAX_NUM_STR_LEN) {
+            if (!*err && asprintf(err, "Error to print string") < 0) {
+                *(err) = safe_strdup("error allocating memory");
+            }
+            return yajl_gen_in_error_state;
+        }
+        stat = reformat_string(g, numstr, strlen(numstr));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_string(g, map->values[i], strlen(map->values[i]));;
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_int_string(json_map_int_string *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            // No need to free key for type int
+            free(map->values[i]);
+            map->values[i] = NULL;
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_int_string *make_json_map_int_string(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_int_string *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(char *) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(int));
+        ret->values = safe_malloc((len + 1) * sizeof(char *));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+
+            if (srckey != NULL) {
+                int invalid;
+                invalid = common_safe_int(srckey, &(ret->keys[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid key '%s' with type 'int': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_string(ret);
+                    return NULL;
+                }
+            }
+
+            if (srcval != NULL) {
+                if (!YAJL_IS_STRING(srcval)) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'string' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_string(ret);
+                    return NULL;
+                }
+                char *str = YAJL_GET_STRING(srcval);
+                ret->values[i] = safe_strdup(str ? str : "");
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_int_string(json_map_int_string *map, int key, const char *val) {
+    size_t len;
+    int *keys = NULL;
+    char **vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(int) - 1) < map->len || (SIZE_MAX / sizeof(char *) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(int));
+    vals = safe_malloc(len * sizeof(char *));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(int));
+        (void)memcpy(vals, map->values, map->len * sizeof(char *));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = key;
+    map->values[map->len] = safe_strdup(val ? val : "");
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_string_int(void *ctx, json_map_string_int *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        stat = reformat_string(g, map->keys[i], strlen(map->keys[i]));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_int(g, map->values[i]);
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_string_int(json_map_string_int *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            free(map->keys[i]);
+            map->keys[i] = NULL;
+            // No need to free value for type int
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_string_int *make_json_map_string_int(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_string_int *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(char *) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(char *));
+        ret->values = safe_malloc((len + 1) * sizeof(int));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+            ret->keys[i] = safe_strdup(srckey ? srckey : "");
+
+            if (srcval != NULL) {
+                int invalid;
+                if (!YAJL_IS_NUMBER(srcval)) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'int' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_string_int(ret);
+                    return NULL;
+                }
+                invalid = common_safe_int(YAJL_GET_NUMBER(srcval), &(ret->values[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'int' for key '%s': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_string_int(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_string_int(json_map_string_int *map, const char *key, int val) {
+    size_t len;
+    char **keys = NULL;
+    int *vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(char *) - 1) < map->len || (SIZE_MAX / sizeof(int) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(char *));
+    vals = safe_malloc(len * sizeof(int));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(char *));
+        (void)memcpy(vals, map->values, map->len * sizeof(int));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = safe_strdup(key ? key : "");
+    map->values[map->len] = val;
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_string_bool(void *ctx, json_map_string_bool *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        stat = reformat_string(g, map->keys[i], strlen(map->keys[i]));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_bool(g, map->values[i]);
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_string_bool(json_map_string_bool *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            free(map->keys[i]);
+            map->keys[i] = NULL;
+            // No need to free value for type bool
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_string_bool *make_json_map_string_bool(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_string_bool *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(char *) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(char *));
+        ret->values = safe_malloc((len + 1) * sizeof(bool));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+            ret->keys[i] = safe_strdup(srckey ? srckey : "");
+
+            if (srcval != NULL) {
+                if (YAJL_IS_TRUE(srcval)) {
+                    ret->values[i] = true;
+                } else if (YAJL_IS_FALSE(srcval)) {
+                    ret->values[i] = false;
+                } else {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'bool' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_string_bool(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    return ret;
+}
+
+int append_json_map_string_bool(json_map_string_bool *map, const char *key, bool val) {
+    size_t len;
+    char **keys = NULL;
+    bool *vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(char *) - 1) < map->len || (SIZE_MAX / sizeof(bool) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(char *));
+    vals = safe_malloc(len * sizeof(bool));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(char *));
+        (void)memcpy(vals, map->values, map->len * sizeof(bool));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = safe_strdup(key ? key : "");
+    map->values[map->len] = val;
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_string_string(void *ctx, json_map_string_string *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        stat = reformat_string(g, map->keys[i], strlen(map->keys[i]));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_string(g, map->values[i], strlen(map->values[i]));;
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_string_string(json_map_string_string *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            free(map->keys[i]);
+            map->keys[i] = NULL;
+            free(map->values[i]);
+            map->values[i] = NULL;
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_string_string *make_json_map_string_string(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_string_string *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(char *) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(char *));
+        ret->values = safe_malloc((len + 1) * sizeof(char *));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+            ret->keys[i] = safe_strdup(srckey ? srckey : "");
+
+            if (srcval != NULL) {
+                if (!YAJL_IS_STRING(srcval)) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'string' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_string_string(ret);
+                    return NULL;
+                }
+                char *str = YAJL_GET_STRING(srcval);
+                ret->values[i] = safe_strdup(str ? str : "");
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_string_string(json_map_string_string *map, const char *key, const char *val) {
+    size_t len, i;
+    char **keys = NULL;
+    char **vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    for (i = 0; i < map->len; i++) {
+        if (strcmp(map->keys[i], key) == 0) {
+            free(map->values[i]);
+            map->values[i] = safe_strdup(val ? val : "");
+            return 0;
+        }
+    }
+
+    if ((SIZE_MAX / sizeof(char *) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(char *));
+    vals = safe_malloc(len * sizeof(char *));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(char *));
+        (void)memcpy(vals, map->values, map->len * sizeof(char *));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = safe_strdup(key ? key : "");
+    map->values[map->len] = safe_strdup(val ? val : "");
+
+    map->len++;
+    return 0;
+}
diff --git a/src/lxc/json/json_common.h b/src/lxc/json/json_common.h
new file mode 100755
index 0000000..60aa5fd
--- /dev/null
+++ b/src/lxc/json/json_common.h
@@ -0,0 +1,185 @@
+// Auto generated file. Do not edit!
+#ifndef _JSON_COMMON_H
+#define _JSON_COMMON_H
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <yajl/yajl_tree.h>
+#include <yajl/yajl_gen.h>
+#include "utils.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+# undef linux
+
+//options to report error if there is unknown key found in json
+# define PARSE_OPTIONS_STRICT 0x01
+//options to generate all key and value
+# define GEN_OPTIONS_ALLKEYVALUE 0x02
+//options to generate simplify(no indent) json string
+# define GEN_OPTIONS_SIMPLIFY 0x04
+//options not to validate utf8 data
+# define GEN_OPTIONS_NOT_VALIDATE_UTF8 0x08
+
+#define GEN_SET_ERROR_AND_RETURN(stat, err) { \
+        if (*(err) == NULL) {\
+            if (asprintf(err, "%s: %s: %d: error generating json, errcode: %d", __FILE__, __func__, __LINE__, stat) < 0) { \
+                *(err) = safe_strdup("error allocating memory"); \
+            } \
+        }\
+        return stat; \
+    }
+
+typedef char *parser_error;
+
+struct parser_context {
+    unsigned int options;
+    FILE *stderr;
+};
+
+yajl_gen_status reformat_number(void *ctx, const char *str, size_t len);
+
+yajl_gen_status reformat_uint(void *ctx, long long unsigned int num);
+
+yajl_gen_status reformat_int(void *ctx, long long int num);
+
+yajl_gen_status reformat_double(void *ctx, double num);
+
+yajl_gen_status reformat_string(void *ctx, const char *str, size_t len);
+
+yajl_gen_status reformat_null(void *ctx);
+
+yajl_gen_status reformat_bool(void *ctx, int boolean);
+
+yajl_gen_status reformat_map_key(void *ctx, const char *str, size_t len);
+
+yajl_gen_status reformat_start_map(void *ctx);
+
+yajl_gen_status reformat_end_map(void *ctx);
+
+yajl_gen_status reformat_start_array(void *ctx);
+
+yajl_gen_status reformat_end_array(void *ctx);
+
+bool json_gen_init(yajl_gen *g, struct parser_context *ctx);
+
+yajl_val get_val(yajl_val tree, const char *name, yajl_type type);
+
+void *safe_malloc(size_t size);
+
+int common_safe_double(const char *numstr, double *converted);
+
+int common_safe_uint8(const char *numstr, uint8_t *converted);
+
+int common_safe_uint16(const char *numstr, uint16_t *converted);
+
+int common_safe_uint32(const char *numstr, uint32_t *converted);
+
+int common_safe_uint64(const char *numstr, uint64_t *converted);
+
+int common_safe_uint(const char *numstr, unsigned int *converted);
+
+int common_safe_int8(const char *numstr, int8_t *converted);
+
+int common_safe_int16(const char *numstr, int16_t *converted);
+
+int common_safe_int32(const char *numstr, int32_t *converted);
+
+int common_safe_int64(const char *numstr, int64_t *converted);
+
+int common_safe_int(const char *numstr, int *converted);
+
+typedef struct {
+    int *keys;
+    int *values;
+    size_t len;
+} json_map_int_int;
+
+void free_json_map_int_int(json_map_int_int *map);
+
+json_map_int_int *make_json_map_int_int(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_int_int(void *ctx, json_map_int_int *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_int_int(json_map_int_int *map, int key, int val);
+
+typedef struct {
+    int *keys;
+    bool *values;
+    size_t len;
+} json_map_int_bool;
+
+void free_json_map_int_bool(json_map_int_bool *map);
+
+json_map_int_bool *make_json_map_int_bool(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_int_bool(void *ctx, json_map_int_bool *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_int_bool(json_map_int_bool *map, int key, bool val);
+
+typedef struct {
+    int *keys;
+    char **values;
+    size_t len;
+} json_map_int_string;
+
+void free_json_map_int_string(json_map_int_string *map);
+
+json_map_int_string *make_json_map_int_string(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_int_string(void *ctx, json_map_int_string *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_int_string(json_map_int_string *map, int key, const char *val);
+
+typedef struct {
+    char **keys;
+    int *values;
+    size_t len;
+} json_map_string_int;
+
+void free_json_map_string_int(json_map_string_int *map);
+
+json_map_string_int *make_json_map_string_int(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_string_int(void *ctx, json_map_string_int *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_string_int(json_map_string_int *map, const char *key, int val);
+
+typedef struct {
+    char **keys;
+    bool *values;
+    size_t len;
+} json_map_string_bool;
+
+void free_json_map_string_bool(json_map_string_bool *map);
+
+json_map_string_bool *make_json_map_string_bool(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_string_bool(void *ctx, json_map_string_bool *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_string_bool(json_map_string_bool *map, const char *key, bool val);
+
+typedef struct {
+    char **keys;
+    char **values;
+    size_t len;
+} json_map_string_string;
+
+void free_json_map_string_string(json_map_string_string *map);
+
+json_map_string_string *make_json_map_string_string(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_string_string(void *ctx, json_map_string_string *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_string_string(json_map_string_string *map, const char *key, const char *val);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/lxc/json/logger_json_file.c b/src/lxc/json/logger_json_file.c
new file mode 100644
index 0000000..6abeef4
--- /dev/null
+++ b/src/lxc/json/logger_json_file.c
@@ -0,0 +1,246 @@
+// Generated from json-file.json. Do not edit!
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+#include <read-file.h>
+#include "logger_json_file.h"
+
+logger_json_file *make_logger_json_file(yajl_val tree, struct parser_context *ctx, parser_error *err) {
+	logger_json_file *ret = NULL;
+	*err = 0;
+	if (tree == NULL)
+		return ret;
+	ret = safe_malloc(sizeof(*ret));
+	{
+		yajl_val tmp = get_val(tree, "log", yajl_t_string);
+		if (tmp != NULL) {
+			char *str = YAJL_GET_STRING(tmp);
+			ret->log = (uint8_t *)safe_strdup(str ? str : "");
+			ret->log_len = str != NULL ? strlen(str) : 0;
+		}
+	}
+	{
+		yajl_val val = get_val(tree, "stream", yajl_t_string);
+		if (val != NULL) {
+			char *str = YAJL_GET_STRING(val);
+			ret->stream = safe_strdup(str ? str : "");
+		}
+	}
+	{
+		yajl_val val = get_val(tree, "time", yajl_t_string);
+		if (val != NULL) {
+			char *str = YAJL_GET_STRING(val);
+			ret->time = safe_strdup(str ? str : "");
+		}
+	}
+	{
+		yajl_val tmp = get_val(tree, "attrs", yajl_t_string);
+		if (tmp != NULL) {
+			char *str = YAJL_GET_STRING(tmp);
+			ret->attrs = (uint8_t *)safe_strdup(str ? str : "");
+			ret->attrs_len = str != NULL ? strlen(str) : 0;
+		}
+	}
+
+	if (tree->type == yajl_t_object && (ctx->options & PARSE_OPTIONS_STRICT)) {
+		int i;
+		for (i = 0; i < tree->u.object.len; i++)
+			if (strcmp(tree->u.object.keys[i], "log") &&
+			                strcmp(tree->u.object.keys[i], "stream") &&
+			                strcmp(tree->u.object.keys[i], "time") &&
+			                strcmp(tree->u.object.keys[i], "attrs")) {
+				if (ctx->stderr > 0)
+					fprintf(ctx->stderr, "WARNING: unknown key found: %s\n", tree->u.object.keys[i]);
+			}
+	}
+	return ret;
+}
+
+void free_logger_json_file(logger_json_file *ptr) {
+	if (ptr == NULL)
+		return;
+	free(ptr->log);
+	ptr->log = NULL;
+	free(ptr->stream);
+	ptr->stream = NULL;
+	free(ptr->time);
+	ptr->time = NULL;
+	free(ptr->attrs);
+	ptr->attrs = NULL;
+	free(ptr);
+}
+
+yajl_gen_status gen_logger_json_file(yajl_gen g, logger_json_file *ptr, struct parser_context *ctx, parser_error *err) {
+	yajl_gen_status stat = yajl_gen_status_ok;
+	*err = 0;
+	stat = reformat_start_map(g);
+	if (yajl_gen_status_ok != stat)
+		GEN_SET_ERROR_AND_RETURN(stat, err);
+	if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr != NULL && ptr->log != NULL && ptr->log_len)) {
+		const char *str = "";
+		size_t len = 0;
+		stat = reformat_map_key(g, "log", strlen("log"));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+		if (ptr != NULL && ptr->log != NULL) {
+			str = (const char *)ptr->log;
+			len = ptr->log_len;
+		}
+		stat = reformat_string(g, str, len);
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+	}
+	if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->stream != NULL)) {
+		char *str = "";
+		stat = reformat_map_key(g, "stream", strlen("stream"));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+		if (ptr != NULL && ptr->stream != NULL) {
+			str = ptr->stream;
+		}
+		stat = reformat_string(g, str, strlen(str));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+	}
+	if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->time != NULL)) {
+		char *str = "";
+		stat = reformat_map_key(g, "time", strlen("time"));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+		if (ptr != NULL && ptr->time != NULL) {
+			str = ptr->time;
+		}
+		stat = reformat_string(g, str, strlen(str));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+	}
+	if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr != NULL && ptr->attrs != NULL && ptr->attrs_len)) {
+		const char *str = "";
+		size_t len = 0;
+		stat = reformat_map_key(g, "attrs", strlen("attrs"));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+		if (ptr != NULL && ptr->attrs != NULL) {
+			str = (const char *)ptr->attrs;
+			len = ptr->attrs_len;
+		}
+		stat = reformat_string(g, str, len);
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+	}
+	stat = reformat_end_map(g);
+	if (yajl_gen_status_ok != stat)
+		GEN_SET_ERROR_AND_RETURN(stat, err);
+	return yajl_gen_status_ok;
+}
+
+
+logger_json_file *logger_json_file_parse_file(const char *filename, struct parser_context *ctx, parser_error *err) {
+	logger_json_file *ptr = NULL;
+	size_t filesize;
+	char *content = NULL;
+
+	if (filename == NULL || err == NULL)
+		return NULL;
+
+	*err = NULL;
+	content = read_file(filename, &filesize);
+	if (content == NULL) {
+		if (asprintf(err, "cannot read the file: %s", filename) < 0)
+			*err = safe_strdup("error allocating memory");
+		return NULL;
+	}
+	ptr = logger_json_file_parse_data(content, ctx, err);
+	free(content);
+	return ptr;
+}
+
+logger_json_file *logger_json_file_parse_file_stream(FILE *stream, struct parser_context *ctx, parser_error *err) {
+	logger_json_file *ptr = NULL;
+	size_t filesize;
+	char *content = NULL ;
+
+	if (stream == NULL || err == NULL)
+		return NULL;
+
+	*err = NULL;
+	content = fread_file(stream, &filesize);
+	if (content == NULL) {
+		*err = safe_strdup("cannot read the file");
+		return NULL;
+	}
+	ptr = logger_json_file_parse_data(content, ctx, err);
+	free(content);
+	return ptr;
+}
+
+logger_json_file *logger_json_file_parse_data(const char *jsondata, struct parser_context *ctx, parser_error *err) {
+	logger_json_file *ptr  = NULL;
+	yajl_val tree;
+	char errbuf[1024];
+	struct parser_context tmp_ctx;
+
+	if (jsondata == NULL || err == NULL)
+		return NULL;
+
+	*err = NULL;
+	if (ctx == NULL) {
+		ctx = &tmp_ctx;
+		memset(&tmp_ctx, 0, sizeof(tmp_ctx));
+	}
+	tree = yajl_tree_parse(jsondata, errbuf, sizeof(errbuf));
+	if (tree == NULL) {
+		if (asprintf(err, "cannot parse the data: %s", errbuf) < 0)
+			*err = safe_strdup("error allocating memory");
+		return NULL;
+	}
+	ptr = make_logger_json_file(tree, ctx, err);
+	yajl_tree_free(tree);
+	return ptr;
+}
+char *logger_json_file_generate_json(logger_json_file *ptr, struct parser_context *ctx, parser_error *err) {
+	yajl_gen g = NULL;
+	struct parser_context tmp_ctx;
+	const unsigned char *gen_buf = NULL;
+	char *json_buf = NULL;
+	size_t gen_len = 0;
+
+	if (ptr == NULL || err == NULL)
+		return NULL;
+
+	*err = NULL;
+	if (ctx == NULL) {
+		ctx = &tmp_ctx;
+		memset(&tmp_ctx, 0, sizeof(tmp_ctx));
+	}
+
+	if (!json_gen_init(&g, ctx)) {
+		*err = safe_strdup("Json_gen init failed");
+		goto out;
+	}
+	if (yajl_gen_status_ok != gen_logger_json_file(g, ptr, ctx, err)) {
+		if (*err == NULL)
+			*err = safe_strdup("Failed to generate json");
+		goto free_out;
+	}
+	yajl_gen_get_buf(g, &gen_buf, &gen_len);
+	if (gen_buf == NULL) {
+		*err = safe_strdup("Error to get generated json");
+		goto free_out;
+	}
+
+	if (gen_len == SIZE_MAX) {
+		*err = safe_strdup("Invalid buffer length");
+		goto free_out;
+	}
+	json_buf = safe_malloc(gen_len + 1);
+	(void)memcpy(json_buf, gen_buf, gen_len);
+	json_buf[gen_len] = '\0';
+
+free_out:
+	yajl_gen_clear(g);
+	yajl_gen_free(g);
+out:
+	return json_buf;
+}
diff --git a/src/lxc/json/logger_json_file.h b/src/lxc/json/logger_json_file.h
new file mode 100644
index 0000000..ad5af7b
--- /dev/null
+++ b/src/lxc/json/logger_json_file.h
@@ -0,0 +1,45 @@
+// Generated from json-file.json. Do not edit!
+#ifndef LOGGER_JSON_FILE_SCHEMA_H
+#define LOGGER_JSON_FILE_SCHEMA_H
+
+#include <sys/types.h>
+#include <stdint.h>
+#include "json_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    uint8_t *log;
+    size_t log_len;
+
+    char *stream;
+
+    char *time;
+
+    uint8_t *attrs;
+    size_t attrs_len;
+
+}
+logger_json_file;
+
+void free_logger_json_file(logger_json_file *ptr);
+
+logger_json_file *make_logger_json_file(yajl_val tree, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_logger_json_file(yajl_gen g, logger_json_file *ptr, struct parser_context *ctx, parser_error *err);
+
+logger_json_file *logger_json_file_parse_file(const char *filename, struct parser_context *ctx, parser_error *err);
+
+logger_json_file *logger_json_file_parse_file_stream(FILE *stream, struct parser_context *ctx, parser_error *err);
+
+logger_json_file *logger_json_file_parse_data(const char *jsondata, struct parser_context *ctx, parser_error *err);
+
+char *logger_json_file_generate_json(logger_json_file *ptr, struct parser_context *ctx, parser_error *err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lxc/json/oci_runtime_hooks.c b/src/lxc/json/oci_runtime_hooks.c
new file mode 100644
index 0000000..41ddb67
--- /dev/null
+++ b/src/lxc/json/oci_runtime_hooks.c
@@ -0,0 +1,52 @@
+/******************************************************************************
+ * Copyright (C), 1988-1999, Huawei Tech. Co., Ltd.
+ * FileName: oci_runtime_hooks.c
+ * Author: maoweiyong   Version: 0.1   Date: 2018-11-07
+ * Explanation: provide oci runtime hooks functions
+ ******************************************************************************/
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <read-file.h>
+#include "oci_runtime_hooks.h"
+
+#include "log.h"
+#include "utils.h"
+
+#define PARSE_ERR_BUFFER_SIZE 1024
+
+oci_runtime_spec_hooks *oci_runtime_spec_hooks_parse_file(const char *filename,
+                                                          struct parser_context *ctx, parser_error *err)
+{
+    yajl_val tree;
+    size_t filesize;
+
+    if (!filename || !err) {
+        return NULL;
+    }
+    *err = NULL;
+    struct parser_context tmp_ctx;
+    if (!ctx) {
+        ctx = &tmp_ctx;
+        memset(&tmp_ctx, 0, sizeof(tmp_ctx));
+    }
+    char *content = read_file(filename, &filesize);
+    char errbuf[PARSE_ERR_BUFFER_SIZE];
+    if (content == NULL) {
+        if (asprintf(err, "cannot read the file: %s", filename) < 0) {
+            *err = safe_strdup("error allocating memory");
+        }
+        return NULL;
+    }
+    tree = yajl_tree_parse(content, errbuf, sizeof(errbuf));
+    free(content);
+    if (tree == NULL) {
+        if (asprintf(err, "cannot parse the file: %s", errbuf) < 0) {
+            *err = safe_strdup("error allocating memory");
+        }
+        return NULL;
+    }
+    oci_runtime_spec_hooks *ptr = make_oci_runtime_spec_hooks(tree, ctx, err);
+    yajl_tree_free(tree);
+    return ptr;
+}
diff --git a/src/lxc/json/oci_runtime_hooks.h b/src/lxc/json/oci_runtime_hooks.h
new file mode 100644
index 0000000..bf570c9
--- /dev/null
+++ b/src/lxc/json/oci_runtime_hooks.h
@@ -0,0 +1,15 @@
+/******************************************************************************
+ * Copyright (C), 1988-1999, Huawei Tech. Co., Ltd.
+ * FileName: oci_runtime_hooks.h
+ * Author: tanyifeng  Version: 0.1   Date: 2018-11-08
+ * Explanation: provide container oci runtime hooks function definition
+ ******************************************************************************/
+#ifndef _CONTAINER_HOOKS_H
+# define _CONTAINER_HOOKS_H
+
+# include "oci_runtime_spec.h"
+
+oci_runtime_spec_hooks *oci_runtime_spec_hooks_parse_file(const char *filename,
+                                                          struct parser_context *ctx, parser_error *err);
+
+#endif
diff --git a/src/lxc/json/oci_runtime_spec.c b/src/lxc/json/oci_runtime_spec.c
new file mode 100644
index 0000000..fd342de
--- /dev/null
+++ b/src/lxc/json/oci_runtime_spec.c
@@ -0,0 +1,195 @@
+// Generated from spec.json. Do not edit!
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+#include <read-file.h>
+#include "oci_runtime_spec.h"
+
+oci_runtime_spec_hooks *make_oci_runtime_spec_hooks(yajl_val tree, struct parser_context *ctx, parser_error *err) {
+    oci_runtime_spec_hooks *ret = NULL;
+    *err = 0;
+    if (tree == NULL)
+        return ret;
+    ret = safe_malloc(sizeof(*ret));
+    {
+        yajl_val tmp = get_val(tree, "prestart", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->prestart_len = YAJL_GET_ARRAY(tmp)->len;
+            ret->prestart = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->prestart));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                ret->prestart[i] = make_defs_hook(val, ctx, err);
+                if (ret->prestart[i] == NULL) {
+                    free_oci_runtime_spec_hooks(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    {
+        yajl_val tmp = get_val(tree, "poststart", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->poststart_len = YAJL_GET_ARRAY(tmp)->len;
+            ret->poststart = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->poststart));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                ret->poststart[i] = make_defs_hook(val, ctx, err);
+                if (ret->poststart[i] == NULL) {
+                    free_oci_runtime_spec_hooks(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    {
+        yajl_val tmp = get_val(tree, "poststop", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->poststop_len = YAJL_GET_ARRAY(tmp)->len;
+            ret->poststop = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->poststop));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                ret->poststop[i] = make_defs_hook(val, ctx, err);
+                if (ret->poststop[i] == NULL) {
+                    free_oci_runtime_spec_hooks(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+
+    if (tree->type == yajl_t_object && (ctx->options & PARSE_OPTIONS_STRICT)) {
+        int i;
+        for (i = 0; i < tree->u.object.len; i++)
+            if (strcmp(tree->u.object.keys[i], "prestart") &&
+                strcmp(tree->u.object.keys[i], "poststart") &&
+                strcmp(tree->u.object.keys[i], "poststop")) {
+                if (ctx->stderr > 0)
+                    fprintf(ctx->stderr, "WARNING: unknown key found: %s\n", tree->u.object.keys[i]);
+            }
+        }
+    return ret;
+}
+
+void free_oci_runtime_spec_hooks(oci_runtime_spec_hooks *ptr) {
+    if (ptr == NULL)
+        return;
+    if (ptr->prestart != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->prestart_len; i++)
+            if (ptr->prestart[i] != NULL) {
+                free_defs_hook(ptr->prestart[i]);
+                ptr->prestart[i] = NULL;
+            }
+        free(ptr->prestart);
+        ptr->prestart = NULL;
+    }
+    if (ptr->poststart != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->poststart_len; i++)
+            if (ptr->poststart[i] != NULL) {
+                free_defs_hook(ptr->poststart[i]);
+                ptr->poststart[i] = NULL;
+            }
+        free(ptr->poststart);
+        ptr->poststart = NULL;
+    }
+    if (ptr->poststop != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->poststop_len; i++)
+            if (ptr->poststop[i] != NULL) {
+                free_defs_hook(ptr->poststop[i]);
+                ptr->poststop[i] = NULL;
+            }
+        free(ptr->poststop);
+        ptr->poststop = NULL;
+    }
+    free(ptr);
+}
+
+yajl_gen_status gen_oci_runtime_spec_hooks(yajl_gen g, oci_runtime_spec_hooks *ptr, struct parser_context *ctx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    *err = 0;
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->prestart != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "prestart", strlen("prestart"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->prestart != NULL) {
+            len = ptr->prestart_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = gen_defs_hook(g, ptr->prestart[i], ctx, err);
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->poststart != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "poststart", strlen("poststart"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->poststart != NULL) {
+            len = ptr->poststart_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = gen_defs_hook(g, ptr->poststart[i], ctx, err);
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->poststop != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "poststop", strlen("poststop"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->poststop != NULL) {
+            len = ptr->poststop_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = gen_defs_hook(g, ptr->poststop[i], ctx, err);
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    return yajl_gen_status_ok;
+}
diff --git a/src/lxc/json/oci_runtime_spec.h b/src/lxc/json/oci_runtime_spec.h
new file mode 100644
index 0000000..ef3f161
--- /dev/null
+++ b/src/lxc/json/oci_runtime_spec.h
@@ -0,0 +1,37 @@
+// Generated from spec.json. Do not edit!
+#ifndef OCI_RUNTIME_SPEC_SCHEMA_H
+#define OCI_RUNTIME_SPEC_SCHEMA_H
+
+#include <sys/types.h>
+#include <stdint.h>
+#include "json_common.h"
+#include "defs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    defs_hook **prestart;
+    size_t prestart_len;
+
+    defs_hook **poststart;
+    size_t poststart_len;
+
+    defs_hook **poststop;
+    size_t poststop_len;
+
+}
+oci_runtime_spec_hooks;
+
+void free_oci_runtime_spec_hooks(oci_runtime_spec_hooks *ptr);
+
+oci_runtime_spec_hooks *make_oci_runtime_spec_hooks(yajl_val tree, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_oci_runtime_spec_hooks(yajl_gen g, oci_runtime_spec_hooks *ptr, struct parser_context *ctx, parser_error *err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lxc/json/read-file.h b/src/lxc/json/read-file.h
new file mode 100644
index 0000000..5d6e0eb
--- /dev/null
+++ b/src/lxc/json/read-file.h
@@ -0,0 +1,11 @@
+#ifndef READ_FILE_H
+#define READ_FILE_H
+
+#include <stddef.h>
+#include <stdio.h>
+
+extern char *fread_file(FILE *stream, size_t *length);
+
+extern char *read_file(const char *path, size_t *length);
+
+#endif
-- 
2.25.1

