sync branch to openEuler-22.03-LTS-SP1

Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com> (cherry picked from commit 9186039f774168dfbacef04dac8ee56356149736)
2024-03-13 14:26:44 +08:00 · 2024-03-13 14:26:44 +08:00 · f2256bb8f5
commit f2256bb8f5
parent ee4f85074a
26 changed files with 95414 additions and 168 deletions
--- a/0001-KubeOS-add-oci-image-digests-check-when-upgrade-and-.patch
+++ b/0001-KubeOS-add-oci-image-digests-check-when-upgrade-and-.patch
@ -1,116 +0,0 @@
-From 42f5a3e38ea6e23f5aff146f65ad20025088fc84 Mon Sep 17 00:00:00 2001
-From: liyuanr <liyuanrong1@huawei.com>
-Date: Mon, 29 May 2023 11:12:52 +0800
-Subject: [PATCH] KubeOS: add oci image digests check when upgrade and fix the
- issue with the software version display
-
-add check of digests of the oci image for upgrade after
-os-agent pulls image when os upgrading.
-
-Fix the issue where the softwares version is empty
-
-Signed-off-by: liyuanr <liyuanrong1@huawei.com>
---
- Makefile                             |  2 +-
- cmd/agent/server/containerd_image.go |  3 ++
- cmd/agent/server/docker_image.go     |  3 ++
- cmd/agent/server/utils.go            | 44 ++++++++++++++++++++++++++++
- docs/quick-start.md                  |  8 ++---
- 5 files changed, 55 insertions(+), 5 deletions(-)
-
-diff --git a/Makefile b/Makefile
-index 9d9fbea..27cf175 100644
--- a/Makefile
-+++ b/Makefile
-@@ -30,7 +30,7 @@ endif
- 
- VERSION_FILE := ./VERSION
- VERSION := $(shell cat $(VERSION_FILE))
-PACKAGE:=openeuler.org/saiyan/pkg/version
-+PACKAGE:=openeuler.org/KubeOS/pkg/version
- BUILDFLAGS = -buildmode=pie -trimpath
- LDFLAGS = -w -s -buildid=IdByKubeOS -linkmode=external -extldflags=-static -extldflags=-zrelro -extldflags=-Wl,-z,now -X ${PACKAGE}.Version=${VERSION}
- ENV = CGO_CFLAGS="-fstack-protector-all" CGO_CPPFLAGS="-D_FORTIFY_SOURCE=2 -O2"
-diff --git a/cmd/agent/server/containerd_image.go b/cmd/agent/server/containerd_image.go
-index 0b614b5..b019b72 100644
--- a/cmd/agent/server/containerd_image.go
-+++ b/cmd/agent/server/containerd_image.go
-@@ -48,6 +48,9 @@ func (c conImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath prep
- 	if err := runCommand("crictl", "pull", imageName); err != nil {
- 		return "", err
- 	}
-+	if err := checkOCIImageDigestMatch("containerd", imageName, req.CheckSum); err != nil {
-+		return "", err
-+	}
- 	if err := checkAndCleanMount(mountPath); err != nil {
- 		logrus.Errorln("containerd clean environment error", err)
- 		return "", err
-diff --git a/cmd/agent/server/docker_image.go b/cmd/agent/server/docker_image.go
-index 2a52634..e6fa9d6 100644
--- a/cmd/agent/server/docker_image.go
-+++ b/cmd/agent/server/docker_image.go
-@@ -38,6 +38,9 @@ func (d dockerImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath p
- 	if err := runCommand("docker", "pull", imageName); err != nil {
- 		return "", err
- 	}
-+	if err := checkOCIImageDigestMatch("docker", imageName, req.CheckSum); err != nil {
-+		return "", err
-+	}
- 	containerName := "kubeos-temp"
- 	dockerPsCmd := "docker ps -a -f=name=" + containerName + "| awk 'NR==2' | awk '{print $1}'"
- 	existId, err := runCommandWithOut("bash", "-c", dockerPsCmd)
-diff --git a/cmd/agent/server/utils.go b/cmd/agent/server/utils.go
-index 111497c..092417b 100644
--- a/cmd/agent/server/utils.go
-+++ b/cmd/agent/server/utils.go
-@@ -264,3 +264,47 @@ func checkFileExist(path string) (bool, error) {
- 		return false, err
- 	}
- }
-+
-+func checkOCIImageDigestMatch(containerRuntime string, imageName string, checkSum string) error {
-+	var cmdOutput string
-+	var err error
-+	switch containerRuntime {
-+	case "containerd":
-+		cmdOutput, err = runCommandWithOut("crictl", "inspecti", "--output", "go-template",
-+			"--template", "{{.status.repoDigests}}", imageName)
-+		if err != nil {
-+			return err
-+		}
-+	case "docker":
-+		cmdOutput, err = runCommandWithOut("docker", "inspect", "--format", "{{.RepoDigests}}", imageName)
-+		if err != nil {
-+			return err
-+		}
-+	default:
-+		logrus.Errorln("containerRuntime ", containerRuntime, " cannot be recognized")
-+		return fmt.Errorf("containerRuntime %s cannot be recognized", containerRuntime)
-+	}
-+	// cmdOutput format is as follows:
-+	// [imageRepository/imageName:imageTag@sha256:digests]
-+	// parse the output and get digest
-+	var imageDigests string
-+	outArray := strings.Split(cmdOutput, "@")
-+	if strings.HasPrefix(outArray[len(outArray)-1], "sha256") {
-+		pasredArray := strings.Split(strings.TrimSuffix(outArray[len(outArray)-1], "]"), ":")
-+		// 2 is the expected length of the array after dividing "imageName:imageTag@sha256:digests" based on ':'
-+		rightLen := 2
-+		if len(pasredArray) == rightLen {
-+			digestIndex := 1 // 1 is the index of digest data in pasredArray
-+			imageDigests = pasredArray[digestIndex]
-+		}
-+	}
-+	if imageDigests == "" {
-+		logrus.Errorln("error when get ", imageName, " digests")
-+		return fmt.Errorf("error when get %s digests", imageName)
-+	}
-+	if imageDigests != checkSum {
-+		logrus.Errorln("checkSumFailed ", imageDigests, " mismatch to ", checkSum)
-+		return fmt.Errorf("checkSumFailed %s mismatch to %s", imageDigests, checkSum)
-+	}
-+	return nil
-+}
-- 
-2.33.0.windows.2
-
--- a/0001-build-rust-os-agent-remove-useless-dependency.patch
+++ b/0001-build-rust-os-agent-remove-useless-dependency.patch
@ -0,0 +1,56 @@
+From 6e62adfa80c33d9b1fc4445487cc15e721db07bc Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Wed, 10 Jan 2024 15:53:26 +0800
+Subject: [PATCH] build(rust os-agent): remove useless dependency
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/Cargo.lock         | 11 -----------
+ KubeOS-Rust/manager/Cargo.toml |  2 +-
+ 2 files changed, 1 insertion(+), 12 deletions(-)
+
+diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock
+index 4b7fc12..2087339 100644
+--- a/KubeOS-Rust/Cargo.lock
+++ b/KubeOS-Rust/Cargo.lock
+@@ -199,16 +199,6 @@ dependencies = [
+  "serde_json",
+ ]
+ 
+-[[package]]
+-name = "colored"
+-version = "2.1.0"
+-source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8"
+-dependencies = [
+- "lazy_static",
+- "windows-sys",
+-]
+-
+ [[package]]
+ name = "core-foundation"
+ version = "0.9.3"
+@@ -1209,7 +1199,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
+ checksum = "80f9fece9bd97ab74339fe19f4bcaf52b76dcc18e5364c7977c1838f76b38de9"
+ dependencies = [
+  "assert-json-diff",
+- "colored",
+  "httparse",
+  "lazy_static",
+  "log",
+diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml
+index 40672cc..9431fba 100644
+--- a/KubeOS-Rust/manager/Cargo.toml
+++ b/KubeOS-Rust/manager/Cargo.toml
+@@ -8,7 +8,7 @@ version = "0.1.0"
+ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+ [dev-dependencies]
+ mockall = { version = "=0.11.3" }
+-mockito = { version = "0.31.1" }
+mockito = { version = "0.31.1", default-features = false }
+ predicates = { version = "=2.0.1" }
+ tempfile = { version = "3.6.0" }
+ 
+-- 
+2.34.1
+
--- a/0002-KubeOS-support-generate-coredump.patch
+++ b/0002-KubeOS-support-generate-coredump.patch
@ -1,30 +0,0 @@
-From 20d0487753b045a0a0af19dffd0d5deec25fd672 Mon Sep 17 00:00:00 2001
-From: liyuanr <liyuanrong1@huawei.com>
-Date: Wed, 14 Jun 2023 16:00:03 +0800
-Subject: [PATCH] KubeOS:support generate coredump
-
-For Go language applications, generating coredump
-requires declaring the environment variable GOTRACEBACK=crash,
-so the service of os agent adds Environment=GOTRACEBACK=crash
-to support generating coredump
-
-Signed-off-by: liyuanr <liyuanrong1@huawei.com>
---
- files/os-agent.service | 1 +
- 1 file changed, 1 insertion(+)
-
-diff --git a/files/os-agent.service b/files/os-agent.service
-index cf71d08..f778804 100644
--- a/files/os-agent.service
-+++ b/files/os-agent.service
-@@ -12,6 +12,7 @@
- Description=Agent For KubeOS
- 
- [Service]
-+Environment=GOTRACEBACK=crash
- ExecStart=/usr/bin/os-agent
- KillMode=process
- Restart=on-failure
-- 
-2.33.0.windows.2
-
--- a/0002-docs-Add-the-content-of-the-user-guide.patch
+++ b/0002-docs-Add-the-content-of-the-user-guide.patch
@ -0,0 +1,617 @@
+From f34fe043179aec4bd2a9270fc25cef3f4377c152 Mon Sep 17 00:00:00 2001
+From: liyuanr <liyuanrong1@huawei.com>
+Date: Thu, 11 Jan 2024 20:25:05 +0800
+Subject: [PATCH 02/13] docs:Add the content of the user guide.
+
+Modify the format of the user guide,
+add precautions, and add configuration information.
+
+Signed-off-by: liyuanr <liyuanrong1@huawei.com>
+---
+ docs/quick-start.md | 399 +++++++++++++++++++++++++++-----------------
+ 1 file changed, 243 insertions(+), 156 deletions(-)
+
+diff --git a/docs/quick-start.md b/docs/quick-start.md
+index 0d4dc4b..9656fb9 100644
+--- a/docs/quick-start.md
+++ b/docs/quick-start.md
+@@ -1,18 +1,22 @@
+ # 快速使用指导
+ 
+-## 编译及部署
+[TOC]
+ 
+-### 编译指导
+## 编译指导
+ 
+ * 编译环境：openEuler Linux x86/AArch64
+
+ * 进行编译需要以下包：
+   * golang(大于等于1.15版本)
+   * make
+   * git
+  * rust(大于等于1.57版本)
+  * cargo(大于等于1.57版本)
+  * openssl-devel
+ 
+   ``` shell
+-  sudo yum install golang make git
+-  ```  
+  sudo yum install golang make git rust cargo openssl-devel
+  ```
+ 
+ * 使用git获取本项目的源码
+ 
+@@ -27,76 +31,101 @@
+ 
+   ```shell
+   cd KubeOS
+-  sudo make 
+-  ```  
+-
+-  * proxy及operator容器镜像构建
+-    * proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕
+-    * 请用户自行编写Dockerfile来构建镜像，请注意
+-      * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性
+-      * 需将operator和proxy拷贝到baseimage上
+-      * 请确保proxy属主和属组为root，文件权限为500
+-      * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500
+-      * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应
+-    * 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库
+-    * Dockerfile参考如下, Dockerfile也可以使用多阶段构建:
+-
+-      ``` dockerfile
+-      FROM your_baseimage
+-      COPY ./bin/proxy /proxy
+-      ENTRYPOINT ["/proxy"]
+-      FROM your_baseimage
+-      COPY --chown=6552:6552 ./bin/operator /operator
+-      ENTRYPOINT ["/operator"]
+-      ```
+  sudo make
+  # 编译生成的二进制在bin目录下，查看二进制
+  tree bin
+  bin
+  ├── operator
+  ├── os-agent
+  ├── proxy
+  ├── rust
+  │   ├── ...
+  │   └── release
+  │       ├── ...
+  │       ├── os-agent
+  │       └── proxy
+  ```
+ 
+-      ```shell
+-      # 指定proxy的镜像仓库，镜像名及版本
+-      export IMG_PROXY=your_imageRepository/proxy_imageName:version
+-      # 指定proxy的Dockerfile地址
+-      export DOCKERFILE_PROXY=your_dockerfile_proxy
+-      # 指定operator的镜像仓库，镜像名及版本
+-      export IMG_OPERATOR=your_imageRepository/operator_imageName:version
+-      # 指定operator的Dockerfile路径
+-      export DOCKERFILE_OPERATOR=your_dockerfile_operator
+-
+-      # 镜像构建
+-      docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} .
+-      docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} .
+-      # 推送镜像到镜像仓库
+-      docker push ${IMG_OPERATOR}
+-      docker push ${IMG_PROXY}
+-      ```
+  * ```bin/proxy```、```bin/os-agent```为go语言编写的proxy和os-agent，```bin/rust/release/proxy```、```bin/rust/release/os-agent```为rust语言编写的proxy和os-agent，二者功能一致。
+
+## 镜像构建指导
+
+### proxy及operator镜像构建指导
+
+* proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕
+
+* 请用户自行编写Dockerfile来构建镜像，请注意
+  * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性
+  * 需将operator和proxy拷贝到baseimage上
+  * 请确保proxy属主和属组为root，文件权限为500
+  * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500
+  * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应
+
+* 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库
+
+* Dockerfile参考如下, Dockerfile也可以使用多阶段构建:
+
+  ``` dockerfile
+  FROM your_baseimage
+  COPY ./bin/proxy /proxy
+  ENTRYPOINT ["/proxy"]
+  FROM your_baseimage
+  COPY --chown=6552:6552 ./bin/operator /operator
+  ENTRYPOINT ["/operator"]
+  ```
+ 
+-* OS虚拟机镜像制作
+-  * 制作注意事项
+-    * 请确保已安装qemu-img，bc，parted，tar，yum，docker
+-    * 容器OS镜像制作需要使用root权限
+-    * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库
+-    * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式
+-    * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间
+-    * 容器镜像制作时不支持用户自定义配置挂载文件
+-    * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。
+-    * 请确保os-agent属主和属组为root，建议os-agent文件权限为500
+-      * 容器OS虚拟机镜像制作
+-          进入scripts目录，执行脚本
+-
+-          ```shell
+-          cd scripts
+-          bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0'''
+-          ```
+-
+-        * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。
+-        * 容器 OS 镜像制作完成后，会在 scripts 目录下生成：
+-          * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。
+-          * qcow2 格式的系统镜像 system.qcow2。
+-          * 可用于升级的根文件系统分区镜像 update.img 。
+-        * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数
+-        * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool)
+-        * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。
+-  * 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。
+-
+-### 部署指导
+  ```shell
+  # 指定proxy的镜像仓库，镜像名及版本
+  export IMG_PROXY=your_imageRepository/proxy_imageName:version
+  # 指定proxy的Dockerfile地址
+  export DOCKERFILE_PROXY=your_dockerfile_proxy
+  # 指定operator的镜像仓库，镜像名及版本
+  export IMG_OPERATOR=your_imageRepository/operator_imageName:version
+  # 指定operator的Dockerfile路径
+  export DOCKERFILE_OPERATOR=your_dockerfile_operator
+  
+  # 镜像构建
+  docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} .
+  docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} .
+  # 推送镜像到镜像仓库
+  docker push ${IMG_OPERATOR}
+  docker push ${IMG_PROXY}
+  ```
+
+### KubeOS虚拟机镜像制作指导
+
+* 制作注意事项
+  * 请确保已安装qemu-img，bc，parted，tar，yum，docker
+  * 容器OS镜像制作需要使用root权限
+  * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库
+  * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式
+  * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间
+  * 容器镜像制作时不支持用户自定义配置挂载文件
+  * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。
+  * 请确保os-agent属主和属组为root，建议os-agent文件权限为500
+
+* 容器OS虚拟机镜像制作
+    进入scripts目录，执行脚本
+
+    ```shell
+    cd scripts
+    bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0'''
+    ```
+
+      * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。
+      * 容器 OS 镜像制作完成后，会在 scripts 目录下生成：
+        * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。
+        * qcow2 格式的系统镜像 system.qcow2。
+        * 可用于升级的根文件系统分区镜像 update.img 。
+      * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数
+      * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool)
+      * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。
+
+* 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。
+
+## 部署指导
+
+### os-operator和os-proxy部署指导
+ 
+ * 环境要求
+   * openEuler Linux x86/AArch64系统
+@@ -142,18 +171,35 @@
+     kubectl get pods -A
+     ```
+ 
+-### 使用指导
+## 使用指导
+ 
+ #### 注意事项
+ 
+-* 容器OS升级为所有软件包原子升级，默认不在容器OS内提供单包升级能力。
+-* 容器OS升级为双区升级的方式，不支持更多分区数量。
+-* 单节点的升级过程的日志可在节点的/var/log/message文件查看。
+-* 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。
+-* 使用docker镜像升级和mtls双向认证仅支持 openEuler 22.09 及之后的版本
+-* 不支持跨大版本升级
+-
+-#### 参数说明
+* 公共注意事项
+  * 仅支持虚拟机x86和arm64 UEFI场景。
+  * 当前不支持集群节点OS多版本管理，即集群中OS的CR只能为一个。
+  * 使用kubectl apply通过YAML创建或更新OS的CR时，不建议并发apply，当并发请求过多时，kube-apiserver会无法处理请求导致失败。
+  * 如用户配置了容器镜像仓的证书或密钥，请用户保证证书或密钥文件的权限最小。
+* 升级注意事项
+  * 升级为所有软件包原子升级，默认不提供单包升级能力。
+  * 升级为双区升级的方式，不支持更多分区数量。
+  * 当前暂不支持跨大版本升级。
+  * 单节点的升级过程的日志可在节点的 /var/log/messages 文件查看。
+  * 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。
+  * 节点上containerd如需配置ctr使用的私有镜像，请将配置文件host.toml按照ctr指导放在/etc/containerd/certs.d目录下。
+
+* 配置注意事项
+  * 用户自行指定配置内容，用户需保证配置内容安全可靠 ，尤其是持久化配置（kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next），KubeOS不对参数有效性进行检验。
+  * opstype=config时，若osversion与当前集群节点的OS版本不一致，配置不会进行。
+  * 当前仅支持kernel参数临时配置（kernel.sysctl）、持久化配置（kernel.sysctl.persist）和grub cmdline配置（grub.cmdline.current和grub.cmdline.next）。
+  * 持久化配置会写入persist持久化分区，升级重启后配置保留；kernel参数临时配置重启后不保留。
+  * 配置grub.cmdline.current或grub.cmdline.next时，如为单个参数（非key=value格式参数），请指定key为该参数，value为空。
+  * 进行配置删除（operation=delete）时，key=value形式的配置需保证key、value和实际配置一致。
+  * 配置不支持回退，如需回退，请修改配置版本和配置内容，重新下发配置。
+  * 配置出现错误，节点状态陷入config时，请将配置版本恢复成上一版本并重新下发配置，从而使节点恢复至idel状态。 但是请注意：出现错误前已经配置完成的参数无法恢复。
+  * 在配置grub.cmdline.current或grub.cmdline.next时，若需要将已存在的“key=value”格式的参数更新为只有key无value格式，比如将“rd.info=0”更新成rd.info，需要先删除“key=value”，然后在下一次配置时，添加key。不支持直接更新或者更新删除动作在同一次完成。
+
+#### OS CR参数说明
+ 
+ 在集群中创建类别为OS的定制对象，设置相应字段。类别OS来自于安装和部署章节创建的CRD对象，字段及说明如下：
+ 
+@@ -163,21 +209,21 @@
+ 
+   | 参数            |参数类型  | 参数说明                                                     | 使用说明 | 是否必选         |
+   | -------------- | ------ | ------------------------------------------------------------ | ----- | ---------------- |
+-  | imagetype      | string | 使用的升级镜像的类型           | 需为 docker ，containerd ，或者是 disk，其他值无效，且该参数仅在升级场景有效。<br> **注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。|是               |
+-  | opstype        | string | 进行的操作，升级,回退或者配置 | 需为 upgrade ，config 或者 rollback ，其他值无效 |是               |
+-  | osversion      | string | 用于升级或回退的镜像的OS版本          | 需为 KubeOS version , 例如: KubeOS 1.0.0|是               |
+-  | maxunavailable | int    | 同时进行升级或回退的节点数 | maxunavailable值设置为大于实际集群的节点数时也可正常部署，升级或回退时会按照集群内实际节点数进行|是               |
+-  | containerimage    | string | 用于升级的容器镜像               | 需要为容器镜像格式：[REPOSITORY/NAME[:TAG@DIGEST]](https://docs.docker.com/engine/reference/commandline/tag/#extended-description)，仅在使用容器镜像升级场景下有效|是               |
+-  | imageurl       | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如：<https://192.168.122.15/update.img> 仅在使用磁盘镜像升级场景下有效|是               |
+  | imagetype      | string | 升级镜像的类型           | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。<br> **注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是               |
+  | opstype        | string | 操作类型：升级,回退或者配置 | 仅支持upgrade ，config 或者 rollback |是               |
+  | osversion      | string | 升级/回退的目标版本  | osversion需与节点的目标os版本对应（节点上/etc/os-release中PRETTY_NAME字段或k8s检查到的节点os版本） 例如：KubeOS 1.0.0。 |是               |
+  | maxunavailable | int    | 每批同时进行升级/回退/配置的节点数。 | maxunavailable值大于实际节点数时，取实际节点数进行升级/回退/配置。 |是               |
+  | containerimage    | string | 用于升级的容器镜像               | 仅在imagetype是容器类型时生效，仅支持以下3种格式的容器镜像地址： repository/name repository/name@sha256:xxxx repository/name:tag |是               |
+  | imageurl       | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如：<https://192.168.122.15/update.img> ，仅在使用磁盘镜像升级场景下有效 |是               |
+   | checksum       | string | 用于升级的磁盘镜像校验的checksum(SHA-256)值或者是用于升级的容器镜像的digests值                      | 仅在升级场景下有效 |是               |
+   | flagSafe       | bool   | 当imageurl的地址使用http协议表示是否是安全的                 | 需为 true 或者 false ，仅在imageurl使用http协议时有效 |是               |
+   | mtls           | bool   | 用于表示与imageurl连接是否采用https双向认证     | 需为 true 或者 false ，仅在imageurl使用https协议时有效|是               |
+   | cacert         | string | https或者https双向认证时使用的根证书文件                       | 仅在imageurl使用https协议时有效| imageurl使用https协议时必选 |
+   | clientcert     | string | https双向认证时使用的客户端证书文件                          | 仅在使用https双向认证时有效|mtls为true时必选 |
+   | clientkey      | string | https双向认证时使用的客户端公钥                              | 仅在使用https双向认证时有效|mtls为true时必选 |
+-  | evictpodforce      | bool | 用于表示升级/回退时是否强制驱逐pod                            | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 |
+-  | sysconfigs      | / | 需要进行配置的参数值                            | 在配置或者升级或者回退机器时有效，在升级或者回退操作之后即机器重启之后起效，详细字段说明请见```配置（Settings）指导```| 可选 |
+-  | upgradeconfigs      | / | 需要升级前进行的配置的参数值                            | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 |  
+  | evictpodforce      | bool | 升级/回退时是否强制驱逐pod                            | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 |
+  | sysconfigs      | / | 配置设置                          | 1. “opstype=config”时只进行配置。  2.“opstype=upgrade/rollback”时，代表升级/回退后配置，即在升级/回退重启后进行配置。```配置（Settings）指导``` | “opstype=config”时必选 |
+  | upgradeconfigs | / | 升级前配置设置                       | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 |
+ 
+ #### 升级指导
+ 
+@@ -271,13 +317,13 @@
+             sysconfigs:
+                 version: edit.os.version
+                 configs:
+-                    - model: kernel.systcl
+                    - model: kernel.sysctl
+                     contents:
+                         - key: kernel param key1
+                           value: kernel param value1
+                         - key: kernel param key2
+                           value: kernel param value2
+-                    - model: kernel.systcl.persist
+                    - model: kernel.sysctl.persist
+                       configpath: persist file path
+                       contents:
+                         - key: kernel param key3
+@@ -287,7 +333,7 @@
+             upgradeconfigs:
+                 version: 1.0.0
+                 configs:
+-                    - model: kernel.systcl
+                    - model: kernel.sysctl
+                     contents:
+                         - key: kernel param key4
+                           value: kernel param value4          
+@@ -311,12 +357,13 @@
+     kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage'
+     ```
+ 
+-* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的 imageurl, osversion, checksum, maxunavailable, flagSafe 或者containerimage字段进行相应修改。
+* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改 
+ 
+ #### 配置（Settings）指导
+ 
+ * Settings参数说明:
+-  以进行配置时的示例yaml为例对配置的参数进行说明，示例yaml如下：
+
+  基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致： 
+ 
+   ```yaml
+   apiVersion: upgrade.openeuler.org/v1alpha1
+@@ -330,72 +377,97 @@
+     maxunavailable: edit.node.config.number
+     containerimage: ""
+     evictpodforce: false
+-    imageurl: ""
+     checksum: ""
+-    flagSafe: false
+-    mtls: false
+     sysconfigs:
+-        version: 1.0.0
+        version: edit.sysconfigs.version
+         configs:
+-            - model: kernel.systcl
+-              contents:
+            - model: kernel.sysctl
+              contents: 
+                 - key: kernel param key1
+                   value: kernel param value1
+                 - key: kernel param key2
+                   value: kernel param value2
+                   operation: delete
+-            - model: kernel.systcl.persist
+            - model: kernel.sysctl.persist
+               configpath: persist file path
+               contents:
+                 - key: kernel param key3
+-                  value: kernel param value3         
+                  value: kernel param value3
+            - model: grub.cmdline.current
+              contents:
+                - key: boot param key1
+                - key: boot param key2
+                  value: boot param value2
+                - key: boot param key3
+                  value: boot param value3
+                  operation: delete
+            - model: grub.cmdline.next
+              contents:
+                - key: boot param key4
+                - key: boot param key5
+                  value: boot param value5
+                - key: boot param key6
+                  value: boot param value6
+                  operation: delete         
+   ```
+ 
+-  * 配置的参数说明如下：
+-    * version: 配置的版本,通过版本差异触发配置，请修改配置后更新 version
+-    * configs: 具体配置内容
+-      * model: 进行的配置的类型，支持的配置类型请看[Settings 列表](#setting-列表)
+-      * configpath: 如为持久化配置，配置文件路径
+-      * contents: 配置参数的 key / value 和对参数的操作。
+-        * key / value: 请看[Settings 列表](#setting-列表)对支持的配置的 key / value的说明。
+-        * operation: 若不指定operation，则默认为添加或更新。若指定为delete，代表删除目前OS中已配置的参数。
+-        **注意：** 当operation为delete时，yaml中的key/value必须和OS上想删除参数的key/value**一致**，否则删除失败。
+-  * upgradeconfigs与sysconfig参数相同，upgradeconfig为升级前进行的配置，仅在升级/回滚场景起效，在升级/回滚操作执行前进行配置，只进行配置或者需要升级/回滚重启后执行配置，使用sysconfigs
+  配置的参数说明如下：
+
+  | 参数       | 参数类型 | 参数说明                    | 使用说明                                                     | 配置中是否必选          |
+  | ---------- | -------- | --------------------------- | ------------------------------------------------------------ | ----------------------- |
+  | version    | string   | 配置的版本                  | 通过version是否相等来判断配置是否触发，version为空（为""或者没有值）时同样进行判断，所以不配置sysconfigs/upgradeconfigs时，继存的version值会被清空并触发配置。 | 是                      |
+  | configs    | /        | 具体配置内容                | 包含具体配置项列表。                                         | 是                      |
+  | model      | string   | 配置的类型                  | 支持的配置类型请看附录下的```Settings列表```                 | 是                      |
+  | configpath | string   | 配置文件路径                | 仅在kernel.sysctl.persist配置类型中生效，请看附录下的```Settings列表```对配置文件路径的说明。 | 否                      |
+  | contents   | /        | 具体key/value的值及操作类型 | 包含具体配置参数列表。                                       | 是                      |
+  | key        | string   | 参数名称                    | key不能为空，不能包含"="，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中每种配置类型对key的说明。 | 是                      |
+  | value      | string   | 参数值                      | key=value形式的参数中，value不能为空，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中对每种配置类型对value的说明。 | key=value形式的参数必选 |
+  | operation  | string   | 对参数进行的操作            | 仅对kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next类型的参数生效。默认为添加或更新。仅支持配置为delete，代表删除已存在的参数（key=value需完全一致才能删除）。 | 否                      |
+
+  
+
+  * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置 
+
+ * 使用说明
+
+   * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如上，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml
+
+   * 查看配置之前的节点的配置的版本和节点状态（NODESTATUS状态为idle）
+ 
+     ```shell
+-    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version'
+    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version'
+     ```
+ 
+   * 执行命令，在集群中部署cr实例后，节点会根据配置的参数信息进行配置，再次查看节点状态(NODESTATUS变成config)
+ 
+     ```shell
+     kubectl apply -f upgrade_v1alpha1_os.yaml
+-    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version'
+    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version'
+     ```
+ 
+   * 再次查看节点的配置的版本确认节点是否配置完成(NODESTATUS恢复为idle)
+ 
+     ```shell
+-    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version'
+    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version'
+     ```
+ 
+-* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。
+* 如果后续需要再次配置，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。
+ 
+ #### 回退指导
+ 
+ * 回退场景
+-  * 虚拟机无法正常启动时，需要退回到上一可以启动的版本时进行回退操作，仅支持手动回退容器 OS 。
+-  * 虚拟机能够正常启动并且进入系统，需要将当前版本退回到老版本时进行回退操作，支持工具回退（类似升级方式）和手动回退，建议使用工具回退。
+-  * 配置出现错误，节点状态陷入config时，可以回退至上一个配置版本以恢复节点至idle状态。
+-  **注意**：在配置新版本时，出现错误前已经配置的参数无法回退。
+  * 虚拟机无法正常启动时，可在grub启动项页面手动切换启动项，使系统回退至上一版本（即手动回退）。
+  * 虚拟机能够正常启动并且进入系统时，支持工具回退和手动回退，建议使用工具回退。
+  * 工具回退有两种方式：
+    1. rollback模式直接回退至上一版本。
+    2. upgrade模式重新升级至上一版本
+ * 手动回退指导
+-  * 手动重启虚拟机，选择第二启动项进行回退，手动回退仅支持回退到本次升级之前的版本。
+  
+  * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。 
+ * 工具回退指导
+   * 回退至任意版本
+-  * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。
+    * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。
+
+     * YAML修改完成后执行更新命令，在集群中更新定制对象后，节点会根据配置的字段信息进行回退
+ 
+         ```shell
+@@ -444,13 +516,13 @@
+         sysconfigs:
+             version: previous config version
+             configs:
+-                - model: kernel.systcl
+                - model: kernel.sysctl
+                   contents:
+                     - key: kernel param key1
+                       value: kernel param value1
+                     - key: kernel param key2
+                       value: kernel param value2
+-                - model: kernel.systcl.persist
+                - model: kernel.sysctl.persist
+                   configpath: persist file path
+                   contents:
+                     - key: kernel param key3
+@@ -467,23 +539,21 @@
+   * 查看节点容器 OS 版本(回退OS版本)或节点config版本&节点状态为idle(回退config版本)，确认回退是否成功。
+ 
+     ```shell
+-    kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage'
+-
+-    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version'
+    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version'
+     ```
+ 
+-#### Admin容器
+## Admin容器镜像制作、部署和使用
+ 
+ KubeOS提供一个分离的包含sshd服务和hostshell工具的Admin容器，来帮助管理员在必要情况下登录KubeOS，其中的sshd服务由[sysmaster](https://gitee.com/openeuler/sysmaster)/systemd拉起。Admin容器部署后用户可通过ssh连接到节点的Admin容器，进入Admin容器后执行hostshell命令获取host的root shell。
+ 
+-##### 部署方法
+### admin容器镜像制作
+ 
+-以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/everything/aarch64/Packages/)到scripts/admin-container目录下。
+以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-SP1-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/update/aarch64/Packages/)到scripts/admin-container目录下。
+ 
+-**修改**admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/docker_img)下载。
+修改admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts-sp1可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/docker_img)下载。
+ 
+ ```Dockerfile
+-FROM openeuler-22.03-lts
+FROM openeuler-22.03-lts-sp1
+ 
+ RUN yum -y install openssh-clients util-linux
+ 
+@@ -514,7 +584,9 @@ bash -x kbimg.sh create admin-image -f admin-container/Dockerfile -d your_imageR
+ docker push your_imageRepository/admin_imageName:version
+ ```
+ 
+-在master节点上部署Admin容器，需要提供ssh公钥来免密登录，**修改**并应用如下示例yaml文件:
+### admin容器部署
+
+在master节点上部署Admin容器，需要提供ssh公钥来免密登录，修改并应用如下示例yaml文件:
+ 
+ ```yaml
+ apiVersion: v1
+@@ -583,6 +655,8 @@ spec:
+       control-plane: admin-container-sysmaster
+ ```
+ 
+### admin容器使用
+
+ ssh到Admin容器，然后执行hostshell命令进入host root shell, 如：
+ 
+ ```shell
+@@ -590,7 +664,7 @@ ssh -p your-exposed-port root@your.worker.node.ip
+ hostshell
+ ```
+ 
+-##### hostshell
+#### hostshell说明
+ 
+ 为了保证KubeOS的轻便性，许多工具或命令没有安装在KubeOS内。因此，用户可以在制作Admin容器时，将期望使用的二进制文件放在容器内的如/usr/bin目录下。hostshell工具在执行时会将容器下的/usr/bin, /usr/sbin, /usr/local/bin, /usr/local/sbin路径添加到host root shell的环境变量。
+ 
+@@ -605,11 +679,10 @@ hostshell
+ 
+ #### kernel Settings
+ 
+-* kenerl.sysctl: 设置内核参数，key/value 表示内核参数的 key/value, 示例如下:
+-
+* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下:
+     ```yaml
+     configs:
+-      - model: kernel.systcl
+      - model: kernel.sysctl
+         contents:
+             - key: user.max_user_namespaces
+               value: 16384
+@@ -617,12 +690,10 @@ hostshell
+               value: 0
+               operation: delete
+     ```
+-
+-* kernel.sysctl.persist: 设置持久化内核参数，key/value 表示内核参数的 key/value, configpath为配置修改/新建的文件路径，如不指定configpath默认修改/etc/sysctl.conf
+-
+* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下:
+     ```yaml
+     configs:
+-      - model: kernel.systcl.persist
+      - model: kernel.sysctl.persist
+         configpath : /etc/persist.conf
+         contents:
+             - key: user.max_user_namespaces
+@@ -637,22 +708,38 @@ hostshell
+ * grub.cmdline: 设置grub.cfg文件中的内核引导参数，该行参数在grub.cfg文件中类似如下示例：
+ 
+   ```shell
+-      linux   /boot/vmlinuz root=UUID=5b1aaf5d-5b25-4e4b-a0d3-3d4c8d2e6a6e ro consoleblank=600 console=tty0 console=ttyS0,115200n8 selinux=1 panic=3
+-  ```  
+  linux   /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3
+  ```
+ 
+-  key/value 表示如上示例中内核引导参数的 key=value。
+-  **注意：** 当该参数有多个等号，如root=UUID=some-uuid时，配置时的key为第一个等号前的所有字符，value为第一个等号后的所有字符。
+-  配置方法示例如下：
+* KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置：
+
+  - grub.cmdline.current：对当前分区的启动项参数进行配置。
+  - grub.cmdline.next：对下一分区的启动项参数进行配置。
+
+* 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。
+
+* grub.cmdline.current/next支持“key=value”（value不能为空），也支持单key。若value中有“=”，例如“root=UUID=some-uuid”，key应设置为第一个“=”前的所有字符，value为第一个“=”后的所有字符。 配置方法示例如下：
+ 
+     ```yaml
+     configs:
+-      - model: grub.cmdline
+-        contents:
+-            - key: selinux
+-              value: 0
+-            - key: root
+-              value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94
+-            - key: panic
+-              value: 3
+-              operation: delete
+    - model: grub.cmdline.current
+      contents:
+          - key: selinux
+            value: "0"
+          - key: root
+            value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94
+          - key: panic
+            value: "3"
+            operation: delete
+          - key: crash_kexec_post_notifiers
+    - model: grub.cmdline.next
+      contents:
+          - key: selinux
+            value: "0"
+          - key: root
+            value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94
+          - key: panic
+            value: "3"
+            operation: delete
+          - key: crash_kexec_post_notifiers
+     ```
+-- 
+2.34.1
+
--- a/0003-Remove-cleanup-method-and-related-code.patch
+++ b/0003-Remove-cleanup-method-and-related-code.patch
@ -0,0 +1,143 @@
+From 8cd9d4dd70ec0601490d0b1fd997ba4e05aff420 Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Tue, 16 Jan 2024 17:25:17 +0800
+Subject: [PATCH 03/13] Remove cleanup method and related code
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/agent/src/rpc/agent.rs          |  3 ---
+ KubeOS-Rust/agent/src/rpc/agent_impl.rs     | 14 +---------
+ KubeOS-Rust/cli/src/method/cleanup.rs       | 29 ---------------------
+ KubeOS-Rust/cli/src/method/mod.rs           |  1 -
+ KubeOS-Rust/manager/src/api/agent_status.rs | 11 +-------
+ 5 files changed, 2 insertions(+), 56 deletions(-)
+ delete mode 100644 KubeOS-Rust/cli/src/method/cleanup.rs
+
+diff --git a/KubeOS-Rust/agent/src/rpc/agent.rs b/KubeOS-Rust/agent/src/rpc/agent.rs
+index 13775af..2496bfb 100644
+--- a/KubeOS-Rust/agent/src/rpc/agent.rs
+++ b/KubeOS-Rust/agent/src/rpc/agent.rs
+@@ -22,9 +22,6 @@ pub trait Agent {
+     #[rpc(name = "upgrade")]
+     fn upgrade(&self) -> RpcResult<Response>;
+ 
+-    #[rpc(name = "cleanup")]
+-    fn cleanup(&self) -> RpcResult<Response>;
+-
+     #[rpc(name = "configure")]
+     fn configure(&self, req: ConfigureRequest) -> RpcResult<Response>;
+ 
+diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+index 7101d0d..bc1eabd 100644
+--- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+@@ -17,7 +17,7 @@ use log::{debug, error, info};
+ use manager::{
+     api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest},
+     sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH},
+-    utils::{clean_env, get_partition_info, switch_boot_menuentry, PreparePath, RealCommandExecutor},
+    utils::{get_partition_info, switch_boot_menuentry, RealCommandExecutor},
+ };
+ use nix::{sys::reboot::RebootMode, unistd::sync};
+ 
+@@ -40,10 +40,6 @@ impl Agent for AgentImpl {
+         RpcFunction::call(|| self.upgrade_impl())
+     }
+ 
+-    fn cleanup(&self) -> RpcResult<Response> {
+-        RpcFunction::call(|| self.cleanup_impl())
+-    }
+-
+     fn configure(&self, req: ConfigureRequest) -> RpcResult<Response> {
+         RpcFunction::call(|| self.configure_impl(req))
+     }
+@@ -94,14 +90,6 @@ impl AgentImpl {
+         Ok(Response { status: AgentStatus::Upgraded })
+     }
+ 
+-    pub fn cleanup_impl(&self) -> Result<Response> {
+-        let _lock = self.mutex.lock().unwrap();
+-        info!("Start to cleanup");
+-        let paths = PreparePath::default();
+-        clean_env(paths.update_path, paths.mount_path, paths.image_path)?;
+-        Ok(Response { status: AgentStatus::CleanedUp })
+-    }
+-
+     pub fn configure_impl(&self, mut req: ConfigureRequest) -> Result<Response> {
+         let _lock = self.mutex.lock().unwrap();
+         debug!("Received a 'configure' request: {:?}", req);
+diff --git a/KubeOS-Rust/cli/src/method/cleanup.rs b/KubeOS-Rust/cli/src/method/cleanup.rs
+deleted file mode 100644
+index d1d7dbe..0000000
+--- a/KubeOS-Rust/cli/src/method/cleanup.rs
+++ /dev/null
+@@ -1,29 +0,0 @@
+-/*
+- * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+- * KubeOS is licensed under the Mulan PSL v2.
+- * You can use this software according to the terms and conditions of the Mulan PSL v2.
+- * You may obtain a copy of Mulan PSL v2 at:
+- *     http://license.coscl.org.cn/MulanPSL2
+- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+- * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+- * PURPOSE.
+- * See the Mulan PSL v2 for more details.
+- */
+-
+-use kubeos_manager::api;
+-use serde_json::value::RawValue;
+-
+-use crate::method::callable_method::RpcMethod;
+-
+-#[derive(Default)]
+-pub struct CleanupMethod {}
+-
+-impl RpcMethod for CleanupMethod {
+-    type Response = api::Response;
+-    fn command_name(&self) -> &'static str {
+-        "cleanup"
+-    }
+-    fn command_params(&self) -> Vec<Box<RawValue>> {
+-        vec![]
+-    }
+-}
+diff --git a/KubeOS-Rust/cli/src/method/mod.rs b/KubeOS-Rust/cli/src/method/mod.rs
+index b04b0fd..e1f38bc 100644
+--- a/KubeOS-Rust/cli/src/method/mod.rs
+++ b/KubeOS-Rust/cli/src/method/mod.rs
+@@ -11,7 +11,6 @@
+  */
+ 
+ pub mod callable_method;
+-pub mod cleanup;
+ pub mod configure;
+ pub mod prepare_upgrade;
+ pub mod request;
+diff --git a/KubeOS-Rust/manager/src/api/agent_status.rs b/KubeOS-Rust/manager/src/api/agent_status.rs
+index e466a50..bb16e6b 100644
+--- a/KubeOS-Rust/manager/src/api/agent_status.rs
+++ b/KubeOS-Rust/manager/src/api/agent_status.rs
+@@ -12,19 +12,10 @@
+ 
+ use serde::{Deserialize, Serialize};
+ 
+-#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
+#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]
+ pub enum AgentStatus {
+-    Unknown,
+-    NotApplied,
+     UpgradeReady,
+     Upgraded,
+     Rollbacked,
+     Configured,
+-    CleanedUp,
+-}
+-
+-impl Default for AgentStatus {
+-    fn default() -> Self {
+-        Self::Unknown
+-    }
+ }
+-- 
+2.34.1
+
--- a/0004-test-rust-proxy-add-drain-integration-test.patch
+++ b/0004-test-rust-proxy-add-drain-integration-test.patch
@ -0,0 +1,459 @@
+From 916ca24576d33dc024944a7ed18aaa39e95753f9 Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Thu, 18 Jan 2024 11:13:20 +0800
+Subject: [PATCH 04/13] test(rust proxy):add drain integration test
+
+move drain into a lib for integration test.
+use kind to deploy a cluster for integration test.
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/proxy/Cargo.toml                  |   7 ++
+ .../proxy/src/controller/controller.rs        |   4 +-
+ KubeOS-Rust/proxy/src/controller/mod.rs       |   1 -
+ KubeOS-Rust/proxy/src/controller/values.rs    |  12 ---
+ .../proxy/src/{controller => }/drain.rs       |  12 ++-
+ KubeOS-Rust/proxy/tests/common/mod.rs         |  63 +++++++++++
+ KubeOS-Rust/proxy/tests/drain_test.rs         |  41 +++++++
+ .../proxy/tests/setup/kind-config.yaml        |   5 +
+ KubeOS-Rust/proxy/tests/setup/resources.yaml  | 102 ++++++++++++++++++
+ .../proxy/tests/setup/setup_test_env.sh       |  81 ++++++++++++++
+ 10 files changed, 309 insertions(+), 19 deletions(-)
+ rename KubeOS-Rust/proxy/src/{controller => }/drain.rs (97%)
+ create mode 100644 KubeOS-Rust/proxy/tests/common/mod.rs
+ create mode 100644 KubeOS-Rust/proxy/tests/drain_test.rs
+ create mode 100644 KubeOS-Rust/proxy/tests/setup/kind-config.yaml
+ create mode 100644 KubeOS-Rust/proxy/tests/setup/resources.yaml
+ create mode 100644 KubeOS-Rust/proxy/tests/setup/setup_test_env.sh
+
+diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml
+index 9a148e8..72eb6b9 100644
+--- a/KubeOS-Rust/proxy/Cargo.toml
+++ b/KubeOS-Rust/proxy/Cargo.toml
+@@ -6,6 +6,13 @@ name = "proxy"
+ version = "0.1.0"
+ 
+ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "drain"
+path = "src/drain.rs"
+
+[[bin]]
+name = "proxy"
+path = "src/main.rs"
+ 
+ [dependencies]
+ anyhow = "1.0.44"
+diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs
+index e7ee9f9..b2bb332 100644
+--- a/KubeOS-Rust/proxy/src/controller/controller.rs
+++ b/KubeOS-Rust/proxy/src/controller/controller.rs
+@@ -13,6 +13,7 @@
+ use std::{collections::HashMap, env};
+ 
+ use anyhow::Result;
+use drain::drain_os;
+ use k8s_openapi::api::core::v1::Node;
+ use kube::{
+     api::{Api, PostParams},
+@@ -29,7 +30,6 @@ use super::{
+     agentclient::{AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo},
+     apiclient::ApplyApi,
+     crd::{Configs, Content, OSInstance, OS},
+-    drain::drain_os,
+     utils::{check_version, get_config_version, ConfigOperation, ConfigType},
+     values::{
+         LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE,
+@@ -340,7 +340,7 @@ impl<T: ApplyApi, U: AgentMethod> ProxyController<T, U> {
+     }
+ 
+     async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> {
+-        use crate::controller::drain::error::DrainError::*;
+        use drain::error::DrainError::*;
+         match drain_os(&self.k8s_client.clone(), node_name, force).await {
+             Err(DeletePodsError { errors, .. }) => Err(Error::DrainNodeError { value: errors.join("; ") }),
+             _ => Ok(()),
+diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs
+index 384d74b..73be45c 100644
+--- a/KubeOS-Rust/proxy/src/controller/mod.rs
+++ b/KubeOS-Rust/proxy/src/controller/mod.rs
+@@ -16,7 +16,6 @@ mod apiclient;
+ mod apiserver_mock;
+ mod controller;
+ mod crd;
+-mod drain;
+ mod utils;
+ mod values;
+ 
+diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs
+index fe43851..dec905a 100644
+--- a/KubeOS-Rust/proxy/src/controller/values.rs
+++ b/KubeOS-Rust/proxy/src/controller/values.rs
+@@ -31,15 +31,3 @@ pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock";
+ pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) };
+ 
+ pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) };
+-
+-pub const MAX_EVICT_POD_NUM: usize = 5;
+-
+-pub const EVERY_EVICTION_RETRY: Duration = Duration::from_secs(5);
+-
+-pub const EVERY_DELETION_CHECK: Duration = Duration::from_secs(5);
+-
+-pub const TIMEOUT: Duration = Duration::from_secs(u64::MAX);
+-
+-pub const RETRY_BASE_DELAY: Duration = Duration::from_millis(100);
+-pub const RETRY_MAX_DELAY: Duration = Duration::from_secs(20);
+-pub const MAX_RETRIES_TIMES: usize = 10;
+diff --git a/KubeOS-Rust/proxy/src/controller/drain.rs b/KubeOS-Rust/proxy/src/drain.rs
+similarity index 97%
+rename from KubeOS-Rust/proxy/src/controller/drain.rs
+rename to KubeOS-Rust/proxy/src/drain.rs
+index ddc38ae..09cf662 100644
+--- a/KubeOS-Rust/proxy/src/controller/drain.rs
+++ b/KubeOS-Rust/proxy/src/drain.rs
+@@ -29,10 +29,14 @@ use self::error::{
+     DrainError::{DeletePodsError, GetPodListsError, WaitDeletionError},
+     EvictionError::{EvictionErrorNoRetry, EvictionErrorRetry},
+ };
+-use super::values::{
+-    EVERY_DELETION_CHECK, EVERY_EVICTION_RETRY, MAX_EVICT_POD_NUM, MAX_RETRIES_TIMES, RETRY_BASE_DELAY,
+-    RETRY_MAX_DELAY, TIMEOUT,
+-};
+
+pub const MAX_EVICT_POD_NUM: usize = 5;
+pub const EVERY_EVICTION_RETRY: Duration = Duration::from_secs(5);
+pub const EVERY_DELETION_CHECK: Duration = Duration::from_secs(5);
+pub const TIMEOUT: Duration = Duration::from_secs(u64::MAX);
+pub const RETRY_BASE_DELAY: Duration = Duration::from_millis(100);
+pub const RETRY_MAX_DELAY: Duration = Duration::from_secs(20);
+pub const MAX_RETRIES_TIMES: usize = 10;
+ 
+ pub async fn drain_os(client: &Client, node_name: &str, force: bool) -> Result<(), error::DrainError> {
+     let pods_list = get_pods_deleted(client, node_name, force).await?;
+diff --git a/KubeOS-Rust/proxy/tests/common/mod.rs b/KubeOS-Rust/proxy/tests/common/mod.rs
+new file mode 100644
+index 0000000..8257759
+--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/common/mod.rs
+@@ -0,0 +1,63 @@
+use std::process::{Command, Stdio};
+
+use anyhow::Result;
+use k8s_openapi::api::core::v1::Node;
+use kube::{
+    api::ResourceExt,
+    client::Client,
+    config::{Config, KubeConfigOptions, Kubeconfig},
+    Api,
+};
+use manager::utils::{CommandExecutor, RealCommandExecutor};
+
+pub const CLUSTER: &str = "kubeos-test";
+
+pub fn run_command(cmd: &str, args: &[&str]) -> Result<()> {
+    let output = Command::new(cmd).args(args).stdout(Stdio::inherit()).stderr(Stdio::inherit()).output()?;
+    if !output.status.success() {
+        println!("failed to run command: {} {}\n", cmd, args.join(" "));
+    }
+    Ok(())
+}
+
+pub async fn setup() -> Result<Client> {
+    // set PATH variable
+    let path = std::env::var("PATH").unwrap();
+    let new_path = format!("{}:{}", path, "../../bin");
+    std::env::set_var("PATH", new_path);
+
+    // create cluster
+    let executor = RealCommandExecutor {};
+    println!("Creating cluster");
+    run_command("bash", &["./tests/setup/setup_test_env.sh"]).expect("failed to create cluster");
+
+    // connect to the cluster
+    let kind_config = executor.run_command_with_output("kind", &["get", "kubeconfig", "-n", CLUSTER]).unwrap();
+    let kubeconfig = Kubeconfig::from_yaml(kind_config.as_str()).expect("failed to parse kubeconfig");
+    let options = KubeConfigOptions::default();
+    let config = Config::from_custom_kubeconfig(kubeconfig, &&options).await.expect("failed to create config");
+    let client = Client::try_from(config).expect("failed to create client");
+    // list all nodes
+    let nodes: Api<Node> = Api::all(client.clone());
+    let node_list = nodes.list(&Default::default()).await.expect("failed to list nodes");
+    for n in node_list {
+        println!("Found Node: {}", n.name());
+    }
+    // check node status
+    let node = nodes.get("kubeos-test-worker").await.unwrap();
+    let status = node.status.unwrap();
+    let conditions = status.conditions.unwrap();
+    for c in conditions {
+        if c.type_ == "Ready" {
+            assert_eq!(c.status, "True");
+        }
+    }
+    println!("Cluster ready");
+    Ok(client)
+}
+
+pub fn clean_env() {
+    let executor = RealCommandExecutor {};
+    println!("Cleaning cluster");
+    executor.run_command("kind", &["delete", "clusters", CLUSTER]).expect("failed to clean cluster");
+}
+diff --git a/KubeOS-Rust/proxy/tests/drain_test.rs b/KubeOS-Rust/proxy/tests/drain_test.rs
+new file mode 100644
+index 0000000..2f4f150
+--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/drain_test.rs
+@@ -0,0 +1,41 @@
+mod common;
+
+use common::*;
+use drain::drain_os;
+use k8s_openapi::api::core::v1::{Node, Pod};
+use kube::Api;
+
+#[tokio::test]
+#[ignore = "integration test"]
+async fn test_drain() {
+    let client = setup().await.unwrap();
+    // drain node
+    let nodes: Api<Node> = Api::all(client.clone());
+    let node_name = "kubeos-test-worker";
+    println!("cordon node");
+    nodes.cordon(node_name).await.unwrap();
+    println!("drain node");
+    drain_os(&client, node_name, true).await.unwrap();
+
+    // assert unschedulable
+    println!("check node unschedulable");
+    let node = nodes.get(node_name).await.unwrap();
+    if let Some(spec) = node.spec {
+        assert_eq!(spec.unschedulable, Some(true));
+    } else {
+        panic!("node spec is none");
+    }
+    // list all pods on kubeos-test-worker node and all pods should belong to daemonset
+    println!("list all pods on kubeos-test-worker node");
+    let pods: Api<Pod> = Api::all(client.clone());
+    let pod_list = pods.list(&Default::default()).await.unwrap();
+    // check the pod is from daemonset
+    for p in pod_list {
+        if p.spec.unwrap().node_name.unwrap() == node_name {
+            assert_eq!(p.metadata.owner_references.unwrap()[0].kind, "DaemonSet");
+        }
+    }
+    nodes.uncordon(node_name).await.unwrap();
+
+    clean_env()
+}
+diff --git a/KubeOS-Rust/proxy/tests/setup/kind-config.yaml b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml
+new file mode 100644
+index 0000000..0fe29e7
+--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml
+@@ -0,0 +1,5 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+\ No newline at end of file
+diff --git a/KubeOS-Rust/proxy/tests/setup/resources.yaml b/KubeOS-Rust/proxy/tests/setup/resources.yaml
+new file mode 100644
+index 0000000..0e449d5
+--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/setup/resources.yaml
+@@ -0,0 +1,102 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: example-daemonset
+spec:
+  selector:
+    matchLabels:
+      name: example-daemonset
+  template:
+    metadata:
+      labels:
+        name: example-daemonset
+    spec:
+      containers:
+      - name: busybox
+        image: busybox:stable
+        command: ["/bin/sh", "-c", "sleep 3600"]
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-with-local-storage
+spec:
+  containers:
+  - name: busybox
+    image: busybox:stable
+    command: ["/bin/sh", "-c", "sleep 3600"]
+    volumeMounts:
+    - mountPath: "/data"
+      name: local-volume
+  volumes:
+  - name: local-volume
+    emptyDir: {}
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: standalone-pod
+spec:
+  containers:
+  - name: busybox
+    image: busybox:stable
+    command: ["/bin/sh", "-c", "sleep 3600"]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: example-deployment
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: example
+  template:
+    metadata:
+      labels:
+        app: example
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:alpine
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 1
+            preference:
+              matchExpressions:
+              - key: "node-role.kubernetes.io/control-plane"
+                operator: DoesNotExist
+      tolerations:
+      - key: "node-role.kubernetes.io/master"
+        operator: "Exists"
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: "Exists"
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: example-pdb
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: example
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: resource-intensive-pod
+spec:
+  containers:
+  - name: busybox
+    image: busybox:stable
+    command: ["/bin/sh", "-c", "sleep 3600"]
+    resources:
+      requests:
+        memory: "256Mi"
+        cpu: "500m"
+      limits:
+        memory: "512Mi"
+        cpu: "1000m"
+
+diff --git a/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh
+new file mode 100644
+index 0000000..d24d8e0
+--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh
+@@ -0,0 +1,81 @@
+#!/bin/bash
+# this bash script executes in proxy directory
+
+set -Eeuxo pipefail
+
+# Define variables
+KIND_VERSION="v0.19.0"
+KUBECTL_VERSION="v1.24.15"
+KIND_CLUSTER_NAME="kubeos-test"
+DOCKER_IMAGES=("busybox:stable" "nginx:alpine" "kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab")
+NODE_IMAGE="kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab"
+RESOURCE="./tests/setup/resources.yaml"
+KIND_CONFIG="./tests/setup/kind-config.yaml"
+BIN_PATH="../../bin/"
+ARCH=$(uname -m)
+
+# Install kind and kubectl
+install_bins() {
+    # if bin dir not exist then create
+    if [ ! -d "${BIN_PATH}" ]; then
+        mkdir -p "${BIN_PATH}"
+    fi
+    if [ ! -f "${BIN_PATH}"kind ]; then
+        echo "Installing Kind..."
+        # For AMD64 / x86_64
+        if [ "$ARCH" = x86_64 ]; then
+            # add proxy if you are behind proxy
+            curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-amd64
+        fi
+        # For ARM64
+        if [ "$ARCH" = aarch64 ]; then
+            curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-arm64
+        fi
+        chmod +x "${BIN_PATH}"kind
+    fi
+    if [ ! -f "${BIN_PATH}"kubectl ]; then
+        echo "Installing kubectl..."
+        if [ "$ARCH" = x86_64 ]; then
+            curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
+        fi
+        if [ "$ARCH" = aarch64 ]; then
+            curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/arm64/kubectl"
+        fi
+        chmod +x "${BIN_PATH}"kubectl
+    fi
+    export PATH=$PATH:"${BIN_PATH}"
+}
+
+# Create Kind Cluster
+create_cluster() {
+    echo "Creating Kind cluster..."
+    for image in "${DOCKER_IMAGES[@]}"; do
+        docker pull "$image"
+    done
+    kind create cluster --name "${KIND_CLUSTER_NAME}" --config "${KIND_CONFIG}" --image "${NODE_IMAGE}"
+}
+
+# Load Docker image into Kind cluster
+load_docker_image() {
+    echo "Loading Docker image into Kind cluster..."
+    DOCKER_IMAGE=$(printf "%s " "${DOCKER_IMAGES[@]:0:2}")
+    kind load docker-image ${DOCKER_IMAGE} --name "${KIND_CLUSTER_NAME}"
+}
+
+# Apply Kubernetes resource files
+apply_k8s_resources() {
+    echo "Applying Kubernetes resources from ${RESOURCE}..."
+    kubectl apply -f "${RESOURCE}"
+    echo "Waiting for nodes getting ready..."
+    sleep 40s
+}
+
+main() {
+    export no_proxy=localhost,127.0.0.1
+    install_bins
+    create_cluster
+    load_docker_image
+    apply_k8s_resources
+}
+
+main
+-- 
+2.34.1
+
--- a/0005-refactor-rust-os-agent-fix-code-check.patch
+++ b/0005-refactor-rust-os-agent-fix-code-check.patch
@ -0,0 +1,35 @@
+From adb73fcebd1d7698a2fffdb791c2e9d08e949d9c Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Wed, 17 Jan 2024 14:39:34 +0800
+Subject: [PATCH 05/13] refactor(rust os-agent): fix code check
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/cli/src/method/request.rs | 9 ++++-----
+ 1 file changed, 4 insertions(+), 5 deletions(-)
+
+diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs
+index 2dc1ffb..b4a24aa 100644
+--- a/KubeOS-Rust/cli/src/method/request.rs
+++ b/KubeOS-Rust/cli/src/method/request.rs
+@@ -36,13 +36,12 @@ pub fn parse_error(error: Error) -> anyhow::Error {
+             debug!("Json parse error: {:?}", e);
+             anyhow!("Failed to parse response")
+         },
+-        Error::Rpc(ref e) => match e.message == "Method not found" {
+-            true => {
+        Error::Rpc(ref e) => {
+            if e.message == "Method not found" {
+                 anyhow!("Method is unimplemented")
+-            },
+-            false => {
+            } else {
+                 anyhow!("{}", e.message)
+-            },
+            }
+         },
+         _ => {
+             debug!("{:?}", error);
+-- 
+2.34.1
+
--- a/0006-fix-agent-proxy-transform-log-timestamp-to-human-rea.patch
+++ b/0006-fix-agent-proxy-transform-log-timestamp-to-human-rea.patch
@ -0,0 +1,245 @@
+From 25a856bf7bfa7f2de32ecb7b4c8d6997a2835f76 Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Sat, 25 Nov 2023 15:18:23 +0800
+Subject: [PATCH 06/13] fix(agent, proxy): transform log timestamp to
+ human-readable format
+
+Originally, the log of controllers is timestamp which is hard to read. Now, transform the log into more human-readable format.
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ cmd/operator/controllers/os_controller_test.go |  3 +--
+ cmd/operator/controllers/suite_test.go         | 12 +++++++++++-
+ cmd/operator/main.go                           | 13 +++++++++++++
+ cmd/proxy/controllers/suite_test.go            | 11 ++++++++++-
+ cmd/proxy/main.go                              | 13 +++++++++++++
+ go.mod                                         |  3 ++-
+ go.sum                                         |  8 ++++++++
+ 7 files changed, 58 insertions(+), 5 deletions(-)
+
+diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go
+index 6cc2760..8c5d198 100644
+--- a/cmd/operator/controllers/os_controller_test.go
+++ b/cmd/operator/controllers/os_controller_test.go
+@@ -23,7 +23,6 @@ import (
+ 	"github.com/google/uuid"
+ 	. "github.com/onsi/ginkgo/v2"
+ 	. "github.com/onsi/gomega"
+-	corev1 "k8s.io/api/core/v1"
+ 	v1 "k8s.io/api/core/v1"
+ 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ 	"k8s.io/apimachinery/pkg/labels"
+@@ -912,7 +911,7 @@ func Test_getNodes(t *testing.T) {
+ 	tests := []struct {
+ 		name    string
+ 		args    args
+-		want    []corev1.Node
+		want    []v1.Node
+ 		wantErr bool
+ 	}{
+ 		{
+diff --git a/cmd/operator/controllers/suite_test.go b/cmd/operator/controllers/suite_test.go
+index aa6deea..67fc9e7 100644
+--- a/cmd/operator/controllers/suite_test.go
+++ b/cmd/operator/controllers/suite_test.go
+@@ -16,9 +16,13 @@ import (
+ 	"context"
+ 	"path/filepath"
+ 	"testing"
+	"time"
+ 
+ 	. "github.com/onsi/ginkgo/v2"
+ 	. "github.com/onsi/gomega"
+	zaplogfmt "github.com/sykesm/zap-logfmt"
+	uzap "go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+ 	"k8s.io/client-go/kubernetes/scheme"
+ 	"k8s.io/client-go/rest"
+ 	ctrl "sigs.k8s.io/controller-runtime"
+@@ -46,7 +50,13 @@ func TestAPIs(t *testing.T) {
+ }
+ 
+ var _ = BeforeSuite(func() {
+-	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+	configLog := uzap.NewProductionEncoderConfig()
+	configLog.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {
+		encoder.AppendString(ts.UTC().Format(time.RFC3339Nano))
+	}
+	logfmtEncoder := zaplogfmt.NewEncoder(configLog)
+	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Encoder(logfmtEncoder)))
+
+ 	ctx, cancel = context.WithCancel(context.TODO())
+ 
+ 	By("bootstrapping test environment")
+diff --git a/cmd/operator/main.go b/cmd/operator/main.go
+index 8249ad2..6b90b26 100644
+--- a/cmd/operator/main.go
+++ b/cmd/operator/main.go
+@@ -14,12 +14,17 @@ package main
+ 
+ import (
+ 	"os"
+	"time"
+ 
+	zaplogfmt "github.com/sykesm/zap-logfmt"
+	uzap "go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+ 	"k8s.io/apimachinery/pkg/runtime"
+ 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+ 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+ 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+ 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+ 
+ 	upgradev1 "openeuler.org/KubeOS/api/v1alpha1"
+ 	"openeuler.org/KubeOS/cmd/operator/controllers"
+@@ -41,6 +46,14 @@ func init() {
+ }
+ 
+ func main() {
+	configLog := uzap.NewProductionEncoderConfig()
+	configLog.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {
+		encoder.AppendString(ts.UTC().Format(time.RFC3339Nano))
+	}
+	logfmtEncoder := zaplogfmt.NewEncoder(configLog)
+	logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder))
+	ctrl.SetLogger(logger)
+
+ 	mgr, err := common.NewControllerManager(setupLog, scheme)
+ 	if err != nil {
+ 		setupLog.Error(err, "unable to start manager")
+diff --git a/cmd/proxy/controllers/suite_test.go b/cmd/proxy/controllers/suite_test.go
+index 00eebbf..767fe95 100644
+--- a/cmd/proxy/controllers/suite_test.go
+++ b/cmd/proxy/controllers/suite_test.go
+@@ -16,9 +16,13 @@ import (
+ 	"context"
+ 	"path/filepath"
+ 	"testing"
+	"time"
+ 
+ 	. "github.com/onsi/ginkgo/v2"
+ 	. "github.com/onsi/gomega"
+	zaplogfmt "github.com/sykesm/zap-logfmt"
+	uzap "go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+ 	"k8s.io/client-go/kubernetes/scheme"
+ 	"k8s.io/client-go/rest"
+ 	ctrl "sigs.k8s.io/controller-runtime"
+@@ -48,7 +52,12 @@ func TestAPIs(t *testing.T) {
+ }
+ 
+ var _ = BeforeSuite(func() {
+-	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+	configLog := uzap.NewProductionEncoderConfig()
+	configLog.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {
+		encoder.AppendString(ts.UTC().Format(time.RFC3339Nano))
+	}
+	logfmtEncoder := zaplogfmt.NewEncoder(configLog)
+	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Encoder(logfmtEncoder)))
+ 	ctx, cancel = context.WithCancel(context.TODO())
+ 
+ 	By("bootstrapping test environment")
+diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go
+index 3a537d9..e606083 100644
+--- a/cmd/proxy/main.go
+++ b/cmd/proxy/main.go
+@@ -15,12 +15,17 @@ package main
+ import (
+ 	"os"
+ 	"path/filepath"
+	"time"
+ 
+	zaplogfmt "github.com/sykesm/zap-logfmt"
+	uzap "go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+ 	"k8s.io/apimachinery/pkg/runtime"
+ 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+ 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+ 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+ 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+ 
+ 	upgradev1 "openeuler.org/KubeOS/api/v1alpha1"
+ 	"openeuler.org/KubeOS/cmd/agent/server"
+@@ -44,6 +49,14 @@ func init() {
+ }
+ 
+ func main() {
+	configLog := uzap.NewProductionEncoderConfig()
+	configLog.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) {
+		encoder.AppendString(ts.UTC().Format(time.RFC3339Nano))
+	}
+	logfmtEncoder := zaplogfmt.NewEncoder(configLog)
+	logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder))
+	ctrl.SetLogger(logger)
+
+ 	var err error
+ 	mgr, err := common.NewControllerManager(setupLog, scheme)
+ 	if err != nil {
+diff --git a/go.mod b/go.mod
+index 057292c..72ca978 100644
+--- a/go.mod
+++ b/go.mod
+@@ -9,6 +9,8 @@ require (
+ 	github.com/onsi/ginkgo/v2 v2.1.4
+ 	github.com/onsi/gomega v1.20.0
+ 	github.com/sirupsen/logrus v1.8.1
+	github.com/sykesm/zap-logfmt v0.0.4
+	go.uber.org/zap v1.19.1
+ 	google.golang.org/grpc v1.49.0
+ 	google.golang.org/protobuf v1.28.1
+ 	k8s.io/api v0.24.0
+@@ -82,7 +84,6 @@ require (
+ 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
+ 	go.uber.org/atomic v1.7.0 // indirect
+ 	go.uber.org/multierr v1.6.0 // indirect
+-	go.uber.org/zap v1.19.1 // indirect
+ 	golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
+ 	golang.org/x/net v0.10.0 // indirect
+ 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+diff --git a/go.sum b/go.sum
+index 6bd1ba1..325cd88 100644
+--- a/go.sum
+++ b/go.sum
+@@ -516,6 +516,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
+ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/sykesm/zap-logfmt v0.0.4 h1:U2WzRvmIWG1wDLCFY3sz8UeEmsdHQjHFNlIdmroVFaI=
+github.com/sykesm/zap-logfmt v0.0.4/go.mod h1:AuBd9xQjAe3URrWT1BBDk2v2onAZHkZkWRMiYZXiZWA=
+ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+@@ -559,15 +561,19 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe
+ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc=
+ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
+ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+ go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+ go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+ go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
+ go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI=
+@@ -812,6 +818,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
+ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+ golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+-- 
+2.34.1
+
--- a/0007-build-update-vendor.patch
+++ b/0007-build-update-vendor.patch
@ -0,0 +1,706 @@
+From e1b4b7a7008855920f866d0fa4c16d09f9341baf Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Sat, 25 Nov 2023 15:18:36 +0800
+Subject: [PATCH 07/13] build: update vendor
+
+use zapfmt to transform timestamp to human-readable format
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ .../github.com/sykesm/zap-logfmt/.gitignore   |   1 +
+ .../github.com/sykesm/zap-logfmt/.travis.yml  |  12 +
+ vendor/github.com/sykesm/zap-logfmt/LICENSE   |  21 +
+ vendor/github.com/sykesm/zap-logfmt/README.md |  76 +++
+ .../github.com/sykesm/zap-logfmt/encoder.go   | 527 ++++++++++++++++++
+ vendor/modules.txt                            |   3 +
+ 6 files changed, 640 insertions(+)
+ create mode 100644 vendor/github.com/sykesm/zap-logfmt/.gitignore
+ create mode 100644 vendor/github.com/sykesm/zap-logfmt/.travis.yml
+ create mode 100644 vendor/github.com/sykesm/zap-logfmt/LICENSE
+ create mode 100644 vendor/github.com/sykesm/zap-logfmt/README.md
+ create mode 100644 vendor/github.com/sykesm/zap-logfmt/encoder.go
+
+diff --git a/vendor/github.com/sykesm/zap-logfmt/.gitignore b/vendor/github.com/sykesm/zap-logfmt/.gitignore
+new file mode 100644
+index 0000000..7a6353d
+--- /dev/null
+++ b/vendor/github.com/sykesm/zap-logfmt/.gitignore
+@@ -0,0 +1 @@
+.envrc
+diff --git a/vendor/github.com/sykesm/zap-logfmt/.travis.yml b/vendor/github.com/sykesm/zap-logfmt/.travis.yml
+new file mode 100644
+index 0000000..7ce1f7a
+--- /dev/null
+++ b/vendor/github.com/sykesm/zap-logfmt/.travis.yml
+@@ -0,0 +1,12 @@
+language: go
+
+matrix:
+  include:
+  - go: "1.13.x"
+    install: true
+  - go: "1.14.x"
+    install: true
+  - go: "1.15.x"
+    install: true
+
+script: go test -race ./...
+diff --git a/vendor/github.com/sykesm/zap-logfmt/LICENSE b/vendor/github.com/sykesm/zap-logfmt/LICENSE
+new file mode 100644
+index 0000000..43a1c32
+--- /dev/null
+++ b/vendor/github.com/sykesm/zap-logfmt/LICENSE
+@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Jonathan Sternberg
+Copyright (c) 2019 Matthew Sykes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+diff --git a/vendor/github.com/sykesm/zap-logfmt/README.md b/vendor/github.com/sykesm/zap-logfmt/README.md
+new file mode 100644
+index 0000000..751f288
+--- /dev/null
+++ b/vendor/github.com/sykesm/zap-logfmt/README.md
+@@ -0,0 +1,76 @@
+# Logfmt Encoder
+
+This package provides a logfmt encoder for [zap][zap].
+
+It is a fork of [github.com/jsternberg/zap-logfmt][jsternberg] that improves
+the handling of reflected fields and encodes arrays and objects instead of
+dropping them from logs. While logging simple fields is preferred for many
+reasons, having ugly data is often better than missing data.
+
+[![Build Status](https://travis-ci.org/sykesm/zap-logfmt.svg?branch=master)](https://travis-ci.org/sykesm/zap-logfmt)
+[![GoDoc](https://godoc.org/github.com/sykesm/zap-logfmt?status.svg)](https://godoc.org/github.com/sykesm/zap-logfmt)
+
+## Usage
+
+The encoder is easy to configure. Simply create a new core with an instance of
+the logfmt encoder and use it with your preferred logging interface.
+
+```go
+package main
+
+import (
+	"os"
+
+	"github.com/sykesm/zap-logfmt"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+func main() {
+	config := zap.NewProductionEncoderConfig()
+	logger := zap.New(zapcore.NewCore(
+		zaplogfmt.NewEncoder(config),
+		os.Stdout,
+		zapcore.DebugLevel,
+	))
+	logger.Info("Hello World")
+}
+```
+
+## Arrays, Objects, and Reflected Fields
+
+While it's best to avoid complex data types in log fields, there are times
+when they sneak in. When complex fields are included in log records, they will
+be encoded, but they won't be very pretty.
+
+### Arrays
+
+Arrays are encoded as a comma separated list of values within square brackets.
+This format is very similar to JSON encoding. Arrays of simple scalars remain
+quite readable but including elements that require quoting will result in very
+ugly records.
+
+### Objects
+
+Objects are encoded as a space separated list of _key=value_ pairs. Because
+this format includes an equals sign, the encoded object will require quoting.
+If any value in the object requires quoting, the required escapes will make
+the encoded field pretty difficult for humans to read.
+
+### Channels and Functions
+
+Channels and functions are encoded as their type and their address. There
+aren't many meaningful ways to log channels and functions...
+
+### Maps and Structs
+
+Maps and structs are encoded as strings that contain the result of `fmt.Sprint`.
+
+## Namespaces
+
+Namespaces are supported. If a namespace is opened, all of the keys will
+be prepended with the namespace name. For example, with the namespace
+`foo` and the key `bar`, you would get a key of `foo.bar`.
+
+[zap]: https://github.com/uber-go/zap
+[jsternberg]: https://github.com/jsternberg/zap-logfmt
+diff --git a/vendor/github.com/sykesm/zap-logfmt/encoder.go b/vendor/github.com/sykesm/zap-logfmt/encoder.go
+new file mode 100644
+index 0000000..9e960cc
+--- /dev/null
+++ b/vendor/github.com/sykesm/zap-logfmt/encoder.go
+@@ -0,0 +1,527 @@
+// Package zaplogfmt provides a zap encoder that formats log entries in
+// "logfmt" format.
+package zaplogfmt
+
+import (
+	"bytes"
+	"encoding"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"math"
+	"reflect"
+	"strings"
+	"sync"
+	"time"
+	"unicode/utf8"
+
+	"go.uber.org/zap/buffer"
+	"go.uber.org/zap/zapcore"
+)
+
+var (
+	logfmtPool = sync.Pool{
+		New: func() interface{} { return &logfmtEncoder{} },
+	}
+	bufferpool = buffer.NewPool()
+)
+
+func getEncoder() *logfmtEncoder {
+	return logfmtPool.Get().(*logfmtEncoder)
+}
+
+func putEncoder(enc *logfmtEncoder) {
+	enc.EncoderConfig = nil
+	enc.buf = nil
+	enc.namespaces = nil
+	enc.arrayLiteral = false
+	logfmtPool.Put(enc)
+}
+
+type logfmtEncoder struct {
+	*zapcore.EncoderConfig
+	buf          *buffer.Buffer
+	namespaces   []string
+	arrayLiteral bool
+}
+
+// NewEncoder creates an encoder writes logfmt formatted log entries.
+func NewEncoder(cfg zapcore.EncoderConfig) zapcore.Encoder {
+	return &logfmtEncoder{
+		EncoderConfig: &cfg,
+		buf:           bufferpool.Get(),
+	}
+}
+
+func (enc *logfmtEncoder) AddArray(key string, arr zapcore.ArrayMarshaler) error {
+	enc.addKey(key)
+	return enc.AppendArray(arr)
+}
+
+func (enc *logfmtEncoder) AddBinary(key string, value []byte) {
+	enc.AddString(key, base64.StdEncoding.EncodeToString(value))
+}
+
+func (enc *logfmtEncoder) AddBool(key string, value bool) {
+	enc.addKey(key)
+	enc.AppendBool(value)
+}
+
+func (enc *logfmtEncoder) AddByteString(key string, value []byte) {
+	enc.addKey(key)
+	enc.AppendByteString(value)
+}
+
+func (enc *logfmtEncoder) AddComplex64(k string, v complex64) { enc.AddComplex128(k, complex128(v)) }
+func (enc *logfmtEncoder) AddComplex128(key string, value complex128) {
+	enc.addKey(key)
+	enc.AppendComplex128(value)
+}
+
+func (enc *logfmtEncoder) AddDuration(key string, value time.Duration) {
+	enc.addKey(key)
+	enc.AppendDuration(value)
+}
+
+func (enc *logfmtEncoder) AddFloat32(key string, value float32) {
+	enc.addKey(key)
+	enc.AppendFloat32(value)
+}
+
+func (enc *logfmtEncoder) AddFloat64(key string, value float64) {
+	enc.addKey(key)
+	enc.AppendFloat64(value)
+}
+
+func (enc *logfmtEncoder) AddInt(k string, v int)     { enc.AddInt64(k, int64(v)) }
+func (enc *logfmtEncoder) AddInt8(k string, v int8)   { enc.AddInt64(k, int64(v)) }
+func (enc *logfmtEncoder) AddInt32(k string, v int32) { enc.AddInt64(k, int64(v)) }
+func (enc *logfmtEncoder) AddInt16(k string, v int16) { enc.AddInt64(k, int64(v)) }
+func (enc *logfmtEncoder) AddInt64(key string, value int64) {
+	enc.addKey(key)
+	enc.AppendInt64(value)
+}
+
+func (enc *logfmtEncoder) AddObject(key string, obj zapcore.ObjectMarshaler) error {
+	enc.addKey(key)
+	return enc.AppendObject(obj)
+}
+
+func (enc *logfmtEncoder) AddReflected(key string, value interface{}) error {
+	enc.addKey(key)
+	return enc.AppendReflected(value)
+}
+
+func (enc *logfmtEncoder) AddString(key, value string) {
+	enc.addKey(key)
+	enc.AppendString(value)
+}
+
+func (enc *logfmtEncoder) AddTime(key string, value time.Time) {
+	enc.addKey(key)
+	enc.AppendTime(value)
+}
+
+func (enc *logfmtEncoder) AddUint(k string, v uint)       { enc.AddUint64(k, uint64(v)) }
+func (enc *logfmtEncoder) AddUint8(k string, v uint8)     { enc.AddUint64(k, uint64(v)) }
+func (enc *logfmtEncoder) AddUint32(k string, v uint32)   { enc.AddUint64(k, uint64(v)) }
+func (enc *logfmtEncoder) AddUint16(k string, v uint16)   { enc.AddUint64(k, uint64(v)) }
+func (enc *logfmtEncoder) AddUintptr(k string, v uintptr) { enc.AddUint64(k, uint64(v)) }
+func (enc *logfmtEncoder) AddUint64(key string, value uint64) {
+	enc.addKey(key)
+	enc.AppendUint64(value)
+}
+
+func (enc *logfmtEncoder) AppendArray(arr zapcore.ArrayMarshaler) error {
+	marshaler := enc.clone()
+	marshaler.namespaces = nil
+	marshaler.arrayLiteral = true
+
+	marshaler.buf.AppendByte('[')
+	err := arr.MarshalLogArray(marshaler)
+	if err == nil {
+		marshaler.buf.AppendByte(']')
+		enc.AppendByteString(marshaler.buf.Bytes())
+	} else {
+		enc.AppendByteString(nil)
+	}
+	marshaler.buf.Free()
+	putEncoder(marshaler)
+	return err
+}
+
+func (enc *logfmtEncoder) AppendBool(value bool) {
+	if value {
+		enc.AppendString("true")
+	} else {
+		enc.AppendString("false")
+	}
+}
+
+func (enc *logfmtEncoder) AppendByteString(value []byte) {
+	enc.addSeparator()
+
+	needsQuotes := bytes.IndexFunc(value, needsQuotedValueRune) != -1
+	if needsQuotes {
+		enc.buf.AppendByte('"')
+	}
+	enc.safeAddByteString(value)
+	if needsQuotes {
+		enc.buf.AppendByte('"')
+	}
+}
+
+func (enc *logfmtEncoder) AppendComplex64(v complex64) { enc.AppendComplex128(complex128(v)) }
+func (enc *logfmtEncoder) AppendComplex128(value complex128) {
+	enc.addSeparator()
+
+	// Cast to a platform-independent, fixed-size type.
+	r, i := float64(real(value)), float64(imag(value))
+	enc.buf.AppendFloat(r, 64)
+	enc.buf.AppendByte('+')
+	enc.buf.AppendFloat(i, 64)
+	enc.buf.AppendByte('i')
+}
+
+func (enc *logfmtEncoder) AppendDuration(value time.Duration) {
+	cur := enc.buf.Len()
+	if enc.EncodeDuration != nil {
+		enc.EncodeDuration(value, enc)
+	}
+	if cur == enc.buf.Len() {
+		enc.AppendInt64(int64(value))
+	}
+}
+
+func (enc *logfmtEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), 32) }
+func (enc *logfmtEncoder) AppendFloat64(v float64) { enc.appendFloat(v, 64) }
+func (enc *logfmtEncoder) appendFloat(val float64, bitSize int) {
+	enc.addSeparator()
+
+	switch {
+	case math.IsNaN(val):
+		enc.buf.AppendString(`NaN`)
+	case math.IsInf(val, 1):
+		enc.buf.AppendString(`+Inf`)
+	case math.IsInf(val, -1):
+		enc.buf.AppendString(`-Inf`)
+	default:
+		enc.buf.AppendFloat(val, bitSize)
+	}
+}
+
+func (enc *logfmtEncoder) AppendInt(v int)     { enc.AppendInt64(int64(v)) }
+func (enc *logfmtEncoder) AppendInt8(v int8)   { enc.AppendInt64(int64(v)) }
+func (enc *logfmtEncoder) AppendInt16(v int16) { enc.AppendInt64(int64(v)) }
+func (enc *logfmtEncoder) AppendInt32(v int32) { enc.AppendInt64(int64(v)) }
+func (enc *logfmtEncoder) AppendInt64(value int64) {
+	enc.addSeparator()
+	enc.buf.AppendInt(value)
+}
+
+func (enc *logfmtEncoder) AppendObject(obj zapcore.ObjectMarshaler) error {
+	marshaler := enc.clone()
+	marshaler.namespaces = nil
+
+	err := obj.MarshalLogObject(marshaler)
+	if err == nil {
+		enc.AppendByteString(marshaler.buf.Bytes())
+	} else {
+		enc.AppendByteString(nil)
+	}
+	marshaler.buf.Free()
+	putEncoder(marshaler)
+	return err
+}
+
+func (enc *logfmtEncoder) AppendReflected(value interface{}) error {
+	switch v := value.(type) {
+	case nil:
+		enc.AppendString("null")
+	case error:
+		enc.AppendString(v.Error())
+	case []byte:
+		enc.AppendByteString(v)
+	case fmt.Stringer:
+		enc.AppendString(v.String())
+	case encoding.TextMarshaler:
+		b, err := v.MarshalText()
+		if err != nil {
+			return err
+		}
+		enc.AppendString(string(b))
+	case json.Marshaler:
+		b, err := v.MarshalJSON()
+		if err != nil {
+			return err
+		}
+		enc.AppendString(string(b))
+	default:
+		rvalue := reflect.ValueOf(value)
+		switch rvalue.Kind() {
+		case reflect.Bool:
+			enc.AppendBool(rvalue.Bool())
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			enc.AppendInt64(rvalue.Int())
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+			enc.AppendUint64(rvalue.Uint())
+		case reflect.Float32:
+			enc.appendFloat(rvalue.Float(), 32)
+		case reflect.Float64:
+			enc.AppendFloat64(rvalue.Float())
+		case reflect.String:
+			enc.AppendString(rvalue.String())
+		case reflect.Complex64, reflect.Complex128:
+			enc.AppendComplex128(rvalue.Complex())
+		case reflect.Chan, reflect.Func:
+			enc.AppendString(fmt.Sprintf("%T(%p)", value, value))
+		case reflect.Map, reflect.Struct:
+			enc.AppendString(fmt.Sprint(value))
+		case reflect.Array, reflect.Slice:
+			enc.AppendArray(zapcore.ArrayMarshalerFunc(func(ae zapcore.ArrayEncoder) error {
+				for i := 0; i < rvalue.Len(); i++ {
+					ae.AppendReflected(rvalue.Index(i).Interface())
+				}
+				return nil
+			}))
+		case reflect.Interface, reflect.Ptr:
+			return enc.AppendReflected(rvalue.Elem().Interface())
+		}
+	}
+	return nil
+}
+
+func (enc *logfmtEncoder) AppendString(value string) {
+	enc.addSeparator()
+
+	needsQuotes := strings.IndexFunc(value, needsQuotedValueRune) != -1
+	if needsQuotes {
+		enc.buf.AppendByte('"')
+	}
+	enc.safeAddString(value)
+	if needsQuotes {
+		enc.buf.AppendByte('"')
+	}
+}
+
+func (enc *logfmtEncoder) AppendTime(value time.Time) {
+	cur := enc.buf.Len()
+	if enc.EncodeTime != nil {
+		enc.EncodeTime(value, enc)
+	}
+	if cur == enc.buf.Len() {
+		enc.AppendInt64(value.UnixNano())
+	}
+}
+
+func (enc *logfmtEncoder) AppendUint(v uint)       { enc.AppendUint64(uint64(v)) }
+func (enc *logfmtEncoder) AppendUint8(v uint8)     { enc.AppendUint64(uint64(v)) }
+func (enc *logfmtEncoder) AppendUint16(v uint16)   { enc.AppendUint64(uint64(v)) }
+func (enc *logfmtEncoder) AppendUint32(v uint32)   { enc.AppendUint64(uint64(v)) }
+func (enc *logfmtEncoder) AppendUintptr(v uintptr) { enc.AppendUint64(uint64(v)) }
+func (enc *logfmtEncoder) AppendUint64(value uint64) {
+	enc.addSeparator()
+	enc.buf.AppendUint(value)
+}
+
+func (enc *logfmtEncoder) Clone() zapcore.Encoder {
+	clone := enc.clone()
+	clone.buf.Write(enc.buf.Bytes())
+	return clone
+}
+
+func (enc *logfmtEncoder) clone() *logfmtEncoder {
+	clone := getEncoder()
+	clone.EncoderConfig = enc.EncoderConfig
+	clone.buf = bufferpool.Get()
+	clone.namespaces = enc.namespaces
+	return clone
+}
+
+func (enc *logfmtEncoder) OpenNamespace(key string) {
+	key = strings.Map(keyRuneFilter, key)
+	enc.namespaces = append(enc.namespaces, key)
+}
+
+func (enc *logfmtEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) {
+	final := enc.clone()
+	if final.TimeKey != "" {
+		final.AddTime(final.TimeKey, ent.Time)
+	}
+	if final.LevelKey != "" {
+		final.addKey(final.LevelKey)
+		cur := final.buf.Len()
+		if final.EncodeLevel != nil {
+			final.EncodeLevel(ent.Level, final)
+		}
+		if cur == final.buf.Len() {
+			// User-supplied EncodeLevel was a no-op. Fall back to strings to keep
+			// output valid.
+			final.AppendString(ent.Level.String())
+		}
+	}
+	if ent.LoggerName != "" && final.NameKey != "" {
+		final.addKey(final.NameKey)
+		cur := final.buf.Len()
+		if final.EncodeName != nil {
+			final.EncodeName(ent.LoggerName, final)
+		}
+		if cur == final.buf.Len() {
+			// User-supplied EncodeName was a no-op. Fall back to strings to
+			// keep output valid.
+			final.AppendString(ent.LoggerName)
+		}
+	}
+	if ent.Caller.Defined && final.CallerKey != "" {
+		final.addKey(final.CallerKey)
+		cur := final.buf.Len()
+		if final.EncodeCaller != nil {
+			final.EncodeCaller(ent.Caller, final)
+		}
+		if cur == final.buf.Len() {
+			// User-supplied EncodeCaller was a no-op. Fall back to strings to
+			// keep output valid.
+			final.AppendString(ent.Caller.String())
+		}
+	}
+	if final.MessageKey != "" {
+		final.addKey(enc.MessageKey)
+		final.AppendString(ent.Message)
+	}
+	if enc.buf.Len() > 0 {
+		if final.buf.Len() > 0 {
+			final.buf.AppendByte(' ')
+		}
+		final.buf.Write(enc.buf.Bytes())
+	}
+	addFields(final, fields)
+	if ent.Stack != "" && final.StacktraceKey != "" {
+		final.AddString(final.StacktraceKey, ent.Stack)
+	}
+	if final.LineEnding != "" {
+		final.buf.AppendString(final.LineEnding)
+	} else {
+		final.buf.AppendString(zapcore.DefaultLineEnding)
+	}
+
+	ret := final.buf
+	putEncoder(final)
+	return ret, nil
+}
+
+func (enc *logfmtEncoder) addSeparator() {
+	if !enc.arrayLiteral {
+		return
+	}
+
+	last := enc.buf.Len() - 1
+	if last >= 0 && enc.buf.Bytes()[last] != '[' {
+		enc.buf.AppendByte(',')
+	}
+}
+
+func (enc *logfmtEncoder) addKey(key string) {
+	key = strings.Map(keyRuneFilter, key)
+	if enc.buf.Len() > 0 {
+		enc.buf.AppendByte(' ')
+	}
+	for _, ns := range enc.namespaces {
+		enc.safeAddString(ns)
+		enc.buf.AppendByte('.')
+	}
+	enc.safeAddString(key)
+	enc.buf.AppendByte('=')
+}
+
+// safeAddString JSON-escapes a string and appends it to the internal buffer.
+// Unlike the standard library's encoder, it doesn't attempt to protect the
+// user from browser vulnerabilities or JSONP-related problems.
+func (enc *logfmtEncoder) safeAddString(s string) {
+	for i := 0; i < len(s); {
+		if enc.tryAddRuneSelf(s[i]) {
+			i++
+			continue
+		}
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if enc.tryAddRuneError(r, size) {
+			i++
+			continue
+		}
+		enc.buf.AppendString(s[i : i+size])
+		i += size
+	}
+}
+
+// safeAddByteString is no-alloc equivalent of safeAddString(string(s)) for s []byte.
+func (enc *logfmtEncoder) safeAddByteString(s []byte) {
+	for i := 0; i < len(s); {
+		if enc.tryAddRuneSelf(s[i]) {
+			i++
+			continue
+		}
+		r, size := utf8.DecodeRune(s[i:])
+		if enc.tryAddRuneError(r, size) {
+			i++
+			continue
+		}
+		enc.buf.Write(s[i : i+size])
+		i += size
+	}
+}
+
+// tryAddRuneSelf appends b if it is valid UTF-8 character represented in a single byte.
+func (enc *logfmtEncoder) tryAddRuneSelf(b byte) bool {
+	if b >= utf8.RuneSelf {
+		return false
+	}
+	if 0x20 <= b && b != '\\' && b != '"' {
+		enc.buf.AppendByte(b)
+		return true
+	}
+	switch b {
+	case '\\', '"':
+		enc.buf.AppendByte('\\')
+		enc.buf.AppendByte(b)
+	case '\n':
+		enc.buf.AppendByte('\\')
+		enc.buf.AppendByte('n')
+	case '\r':
+		enc.buf.AppendByte('\\')
+		enc.buf.AppendByte('r')
+	case '\t':
+		enc.buf.AppendByte('\\')
+		enc.buf.AppendByte('t')
+	default:
+		// Encode bytes < 0x20, except for the escape sequences above.
+		const _hex = "0123456789abcdef"
+		enc.buf.AppendString(`\u00`)
+		enc.buf.AppendByte(_hex[b>>4])
+		enc.buf.AppendByte(_hex[b&0xF])
+	}
+	return true
+}
+
+func (enc *logfmtEncoder) tryAddRuneError(r rune, size int) bool {
+	if r == utf8.RuneError && size == 1 {
+		enc.buf.AppendString(`\ufffd`)
+		return true
+	}
+	return false
+}
+
+func needsQuotedValueRune(r rune) bool {
+	return r <= ' ' || r == '=' || r == '"' || r == utf8.RuneError
+}
+
+func keyRuneFilter(r rune) rune {
+	if needsQuotedValueRune(r) {
+		return -1
+	}
+	return r
+}
+
+func addFields(enc zapcore.ObjectEncoder, fields []zapcore.Field) {
+	for i := range fields {
+		fields[i].AddTo(enc)
+	}
+}
+diff --git a/vendor/modules.txt b/vendor/modules.txt
+index c6048c9..67935ef 100644
+--- a/vendor/modules.txt
+++ b/vendor/modules.txt
+@@ -245,6 +245,9 @@ github.com/spf13/pflag
+ ## explicit; go 1.13
+ github.com/stretchr/testify/assert
+ github.com/stretchr/testify/require
+# github.com/sykesm/zap-logfmt v0.0.4
+## explicit; go 1.13
+github.com/sykesm/zap-logfmt
+ # github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca
+ ## explicit
+ github.com/xlab/treeprint
+-- 
+2.34.1
+
--- a/0008-test-rust-os-agent-add-os-agent-unit-tests.patch
+++ b/0008-test-rust-os-agent-add-os-agent-unit-tests.patch
@ -0,0 +1,817 @@
+From e30fcb3c11ba4290b892e7307976b2c2a64c8fee Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Tue, 16 Jan 2024 20:05:26 +0800
+Subject: [PATCH 08/13] test(rust os-agent): add os-agent unit tests
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/agent/src/rpc/agent_impl.rs       | 28 +++++----
+ KubeOS-Rust/cli/src/client.rs                 | 23 ++-----
+ KubeOS-Rust/cli/src/method/callable_method.rs | 28 +++++++++
+ KubeOS-Rust/cli/src/method/configure.rs       | 31 ++++++++++
+ KubeOS-Rust/cli/src/method/prepare_upgrade.rs | 37 +++++++++++
+ KubeOS-Rust/cli/src/method/request.rs         | 37 +++++++++++
+ KubeOS-Rust/cli/src/method/rollback.rs        | 13 ++++
+ KubeOS-Rust/cli/src/method/upgrade.rs         | 13 ++++
+ KubeOS-Rust/manager/src/api/types.rs          | 58 +++++++++++++++++
+ KubeOS-Rust/manager/src/sys_mgmt/config.rs    | 62 +++++++++----------
+ .../manager/src/sys_mgmt/containerd_image.rs  | 22 +------
+ .../manager/src/sys_mgmt/disk_image.rs        |  7 ++-
+ .../manager/src/sys_mgmt/docker_image.rs      |  4 +-
+ KubeOS-Rust/manager/src/utils/common.rs       | 43 ++++++++++---
+ .../manager/src/utils/container_image.rs      | 46 ++++++++++++--
+ KubeOS-Rust/manager/src/utils/partition.rs    | 15 +++++
+ 16 files changed, 366 insertions(+), 101 deletions(-)
+
+diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+index bc1eabd..8aef414 100644
+--- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+@@ -13,7 +13,7 @@
+ use std::{sync::Mutex, thread, time::Duration};
+ 
+ use anyhow::{bail, Result};
+-use log::{debug, error, info};
+use log::{debug, info};
+ use manager::{
+     api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest},
+     sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH},
+@@ -101,7 +101,6 @@ impl AgentImpl {
+                 debug!("Found configuration type: \"{}\"", config_type);
+                 configuration.set_config(config)?;
+             } else {
+-                error!("Unknown configuration type: \"{}\"", config_type);
+                 bail!("Unknown configuration type: \"{}\"", config_type);
+             }
+         }
+@@ -123,7 +122,7 @@ impl AgentImpl {
+         Ok(Response { status: AgentStatus::Rollbacked })
+     }
+ 
+-    pub fn reboot(&self) -> Result<()> {
+    fn reboot(&self) -> Result<()> {
+         info!("Wait to reboot");
+         thread::sleep(Duration::from_secs(1));
+         sync();
+@@ -144,7 +143,15 @@ mod test {
+     use super::*;
+ 
+     #[test]
+-    fn configure_impl_tests() {
+    fn test_reboot() {
+        let mut agent = AgentImpl::default();
+        agent.disable_reboot = true;
+        let res = agent.reboot();
+        assert!(res.is_ok());
+    }
+
+    #[test]
+    fn test_configure() {
+         let agent = AgentImpl::default();
+         let req = ConfigureRequest {
+             configs: vec![Sysconfig {
+@@ -153,7 +160,7 @@ mod test {
+                 contents: HashMap::new(),
+             }],
+         };
+-        let res = agent.configure_impl(req).unwrap();
+        let res = agent.configure(req).unwrap();
+         assert_eq!(res, Response { status: AgentStatus::Configured });
+ 
+         let req = ConfigureRequest {
+@@ -163,17 +170,12 @@ mod test {
+                 contents: HashMap::new(),
+             }],
+         };
+-        let res = agent.configure_impl(req);
+        let res = agent.configure(req);
+         assert!(res.is_err());
+     }
+ 
+     #[test]
+-    fn upgrade_impl_tests() {
+-        let _ = env_logger::builder()
+-            .target(env_logger::Target::Stdout)
+-            .filter_level(log::LevelFilter::Trace)
+-            .is_test(true)
+-            .try_init();
+    fn test_prepare_upgrade() {
+         let agent = AgentImpl::default();
+         let req = UpgradeRequest {
+             version: "v2".into(),
+@@ -185,7 +187,7 @@ mod test {
+             mtls: false,
+             certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+         };
+-        let res = agent.prepare_upgrade_impl(req);
+        let res = agent.prepare_upgrade(req);
+         assert!(res.is_err());
+     }
+ }
+diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs
+index ce45cdd..9765a42 100644
+--- a/KubeOS-Rust/cli/src/client.rs
+++ b/KubeOS-Rust/cli/src/client.rs
+@@ -43,27 +43,14 @@ impl Client {
+ 
+ #[cfg(test)]
+ mod test {
+-    use kubeos_manager::api;
+-
+     use super::*;
+-    use crate::method::{callable_method::RpcMethod, configure::ConfigureMethod};
+-
+     #[test]
+-    #[ignore]
+     fn test_client() {
+-        let socket_path = "/home/yuhang/os-agent-rust.sock";
+        let socket_path = "/tmp/KubeOS-test.sock";
+         let cli = Client::new(socket_path);
+-
+-        let configured = api::AgentStatus::Configured;
+-        let resp = api::Response { status: configured };
+-        let config_request = api::ConfigureRequest {
+-            configs: vec![api::Sysconfig {
+-                model: "kernel.sysctl".into(),
+-                config_path: "".into(),
+-                contents: std::collections::hash_map::HashMap::new(),
+-            }],
+-        };
+-        let config_resp = ConfigureMethod::new(config_request).call(&cli).unwrap();
+-        assert_eq!(resp, config_resp);
+        let command = "example_command";
+        let params = vec![];
+        let request = cli.send_request(cli.build_request(command, &params));
+        assert!(request.is_err());
+     }
+ }
+diff --git a/KubeOS-Rust/cli/src/method/callable_method.rs b/KubeOS-Rust/cli/src/method/callable_method.rs
+index c46614b..a174b5b 100644
+--- a/KubeOS-Rust/cli/src/method/callable_method.rs
+++ b/KubeOS-Rust/cli/src/method/callable_method.rs
+@@ -24,3 +24,31 @@ pub trait RpcMethod {
+         response.result().map_err(parse_error)
+     }
+ }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::client;
+
+    #[derive(Default)]
+    struct DummyMethod;
+
+    impl RpcMethod for DummyMethod {
+        type Response = String;
+
+        fn command_name(&self) -> &'static str {
+            "dummy_command"
+        }
+
+        fn command_params(&self) -> Vec<Box<RawValue>> {
+            vec![]
+        }
+    }
+
+    #[test]
+    fn test_call() {
+        let client = client::Client::new("/tmp/KubeOS-test.sock");
+        let result = DummyMethod::default().call(&client);
+        assert!(result.is_err());
+    }
+}
+diff --git a/KubeOS-Rust/cli/src/method/configure.rs b/KubeOS-Rust/cli/src/method/configure.rs
+index d137106..cca752d 100644
+--- a/KubeOS-Rust/cli/src/method/configure.rs
+++ b/KubeOS-Rust/cli/src/method/configure.rs
+@@ -39,3 +39,34 @@ impl RpcMethod for ConfigureMethod {
+         vec![to_raw_value(&self.req).unwrap()]
+     }
+ }
+#[cfg(test)]
+mod tests {
+    use kubeos_manager::api::{ConfigureRequest, Sysconfig};
+
+    use super::*;
+
+    #[test]
+    fn test_configure_method() {
+        let req = ConfigureRequest { configs: vec![] };
+        let mut method = ConfigureMethod::new(req);
+
+        // Test set_configure_request method
+        let new_req = ConfigureRequest {
+            configs: vec![Sysconfig {
+                model: "model".to_string(),
+                config_path: "config_path".to_string(),
+                contents: Default::default(),
+            }],
+        };
+        method.set_configure_request(new_req);
+
+        // Test command_name method
+        assert_eq!(method.command_name(), "configure");
+
+        // Test command_params method
+        let expected_params =
+            "RawValue({\"configs\":[{\"model\":\"model\",\"config_path\":\"config_path\",\"contents\":{}}]})";
+        let actual_params = format!("{:?}", method.command_params()[0]);
+        assert_eq!(actual_params, expected_params);
+    }
+}
+diff --git a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs
+index 91dae79..f2034f6 100644
+--- a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs
+++ b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs
+@@ -39,3 +39,40 @@ impl RpcMethod for PrepareUpgradeMethod {
+         vec![to_raw_value(&self.req).unwrap()]
+     }
+ }
+#[cfg(test)]
+mod tests {
+    use kubeos_manager::api::{CertsInfo, UpgradeRequest};
+
+    use super::*;
+
+    #[test]
+    fn test_prepare_upgrade_method() {
+        let req = UpgradeRequest {
+            version: "v1".into(),
+            check_sum: "".into(),
+            image_type: "".into(),
+            container_image: "".into(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        let mut method = PrepareUpgradeMethod::new(req);
+        let new_req = UpgradeRequest {
+            version: "v2".into(),
+            check_sum: "xxx".into(),
+            image_type: "xxx".into(),
+            container_image: "xxx".into(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        method.set_prepare_upgrade_request(new_req);
+        assert_eq!(method.command_name(), "prepare_upgrade");
+
+        let expected_params = "RawValue({\"version\":\"v2\",\"check_sum\":\"xxx\",\"image_type\":\"xxx\",\"container_image\":\"xxx\",\"image_url\":\"\",\"flag_safe\":false,\"mtls\":false,\"certs\":{\"ca_cert\":\"\",\"client_cert\":\"\",\"client_key\":\"\"}})";
+        let actual_params = format!("{:?}", method.command_params()[0]);
+        assert_eq!(actual_params, expected_params);
+    }
+}
+diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs
+index 2dc1ffb..ff75afd 100644
+--- a/KubeOS-Rust/cli/src/method/request.rs
+++ b/KubeOS-Rust/cli/src/method/request.rs
+@@ -50,3 +50,40 @@ pub fn parse_error(error: Error) -> anyhow::Error {
+         },
+     }
+ }
+
+#[cfg(test)]
+mod tests {
+    use jsonrpc::error::RpcError;
+    use serde::de::Error as DeError;
+
+    use super::*;
+
+    #[test]
+    fn test_parse_error() {
+        // Test Error::Transport
+        let transport_error =
+            Error::Transport(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "Connection timeout")));
+        let result = parse_error(transport_error);
+        assert_eq!(result.to_string(), "Cannot connect to KubeOS os-agent unix socket, Connection timeout");
+
+        // Test Error::Json
+        let json_error = Error::Json(serde_json::Error::custom("Failed to parse response"));
+        let result = parse_error(json_error);
+        assert_eq!(result.to_string(), "Failed to parse response");
+
+        // Test Error::Rpc with "Method not found" message
+        let rpc_error = Error::Rpc(RpcError { code: -32601, message: "Method not found".to_string(), data: None });
+        let result = parse_error(rpc_error);
+        assert_eq!(result.to_string(), "Method is unimplemented");
+
+        // Test Error::Rpc with other message
+        let rpc_error = Error::Rpc(RpcError { code: -32603, message: "Internal server error".to_string(), data: None });
+        let result = parse_error(rpc_error);
+        assert_eq!(result.to_string(), "Internal server error");
+
+        // Test other Error variant
+        let other_error = Error::VersionMismatch;
+        let result = parse_error(other_error);
+        assert_eq!(result.to_string(), "Response is invalid");
+    }
+}
+diff --git a/KubeOS-Rust/cli/src/method/rollback.rs b/KubeOS-Rust/cli/src/method/rollback.rs
+index 55aa751..7945f4b 100644
+--- a/KubeOS-Rust/cli/src/method/rollback.rs
+++ b/KubeOS-Rust/cli/src/method/rollback.rs
+@@ -27,3 +27,16 @@ impl RpcMethod for RollbackMethod {
+         vec![]
+     }
+ }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_rollback_method() {
+        let method = RollbackMethod::default();
+        assert_eq!(method.command_name(), "rollback");
+        let expected_params = "[]";
+        let actual_params = format!("{:?}", method.command_params());
+        assert_eq!(actual_params, expected_params);
+    }
+}
+diff --git a/KubeOS-Rust/cli/src/method/upgrade.rs b/KubeOS-Rust/cli/src/method/upgrade.rs
+index a9692ca..f2f94cd 100644
+--- a/KubeOS-Rust/cli/src/method/upgrade.rs
+++ b/KubeOS-Rust/cli/src/method/upgrade.rs
+@@ -27,3 +27,16 @@ impl RpcMethod for UpgradeMethod {
+         vec![]
+     }
+ }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_upgrade_method() {
+        let method = UpgradeMethod::default();
+        assert_eq!(method.command_name(), "upgrade");
+        let expected_params = "[]";
+        let actual_params = format!("{:?}", method.command_params());
+        assert_eq!(actual_params, expected_params);
+    }
+}
+diff --git a/KubeOS-Rust/manager/src/api/types.rs b/KubeOS-Rust/manager/src/api/types.rs
+index 28ee97d..98aeaa3 100644
+--- a/KubeOS-Rust/manager/src/api/types.rs
+++ b/KubeOS-Rust/manager/src/api/types.rs
+@@ -80,3 +80,61 @@ impl<T: CommandExecutor> ImageType<T> {
+ pub trait ImageHandler<T: CommandExecutor> {
+     fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result<UpgradeImageManager<T>>;
+ }
+
+#[cfg(test)]
+mod tests {
+    use anyhow::Result;
+    use mockall::mock;
+
+    use super::*;
+    use crate::utils::PreparePath;
+
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    #[test]
+    fn test_download_image() {
+        let req = UpgradeRequest {
+            version: "KubeOS v2".to_string(),
+            image_type: "containerd".to_string(),
+            container_image: "kubeos-temp".to_string(),
+            check_sum: "22222".to_string(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+
+        let mut mock_executor1 = MockCommandExec::new();
+        mock_executor1.expect_run_command().returning(|_, _| Ok(()));
+        mock_executor1.expect_run_command_with_output().returning(|_, _| Ok(String::new()));
+        let c_handler = CtrImageHandler::new(PreparePath::default(), mock_executor1);
+        let image_type = ImageType::Containerd(c_handler);
+        let result = image_type.download_image(&req);
+        assert!(result.is_err());
+
+        let mut mock_executor2 = MockCommandExec::new();
+        mock_executor2.expect_run_command().returning(|_, _| Ok(()));
+        mock_executor2.expect_run_command_with_output().returning(|_, _| Ok(String::new()));
+        let docker_handler = DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor2);
+        let image_type = ImageType::Docker(docker_handler);
+        let result = image_type.download_image(&req);
+        assert!(result.is_err());
+
+        let mut mock_executor3 = MockCommandExec::new();
+        mock_executor3.expect_run_command().returning(|_, _| Ok(()));
+        mock_executor3.expect_run_command_with_output().returning(|_, _| Ok(String::new()));
+        let disk_handler = DiskImageHandler::new(PreparePath::default(), mock_executor3, "test".into());
+        let image_type = ImageType::Disk(disk_handler);
+        let result = image_type.download_image(&req);
+        assert!(result.is_err());
+    }
+}
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+index cb5fad1..48517b4 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+@@ -186,7 +186,7 @@ fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String
+         return config_kv.join("=");
+     }
+     info!("Delete configuration {}={}", key, old_value);
+-    String::from("")
+    String::new()
+ }
+ 
+ fn handle_update_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String {
+@@ -413,11 +413,20 @@ mod tests {
+         let mut tmp_file = tempfile::NamedTempFile::new().unwrap();
+         writeln!(tmp_file, "{}", comment).unwrap();
+         writeln!(tmp_file, "a=0").unwrap();
+        writeln!(tmp_file, "d=4").unwrap();
+        writeln!(tmp_file, "e=5").unwrap();
+        writeln!(tmp_file, "g=7").unwrap();
+         let kernel_sysctl_persist = KernelSysctlPersist {};
+         let config_detail = HashMap::from([
+             ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }),
+             ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }),
+             ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }),
+            ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }),
+            ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }),
+            ("f".to_string(), KeyInfo { value: "".to_string(), operation: "add".to_string() }),
+            ("g".to_string(), KeyInfo { value: "7".to_string(), operation: "delete".to_string() }),
+            ("".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }),
+            ("s=x".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }),
+         ]);
+         let mut config = Sysconfig {
+             model: KERNEL_SYSCTL_PERSIST.to_string(),
+@@ -426,33 +435,16 @@ mod tests {
+         };
+         kernel_sysctl_persist.set_config(&mut config).unwrap();
+         let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap();
+-        let expected_res = format!("{}\n{}\n{}\n", comment, "a=1", "c=3");
+        let expected_res = format!("{}\n{}\n{}\n{}\n{}\n", comment, "a=1", "d=4", "e=5", "c=3");
+         assert_eq!(result, expected_res);
+-
+-        // test config_path is empty
+-        // remember modify DEFAULT_KERNEL_CONFIG_PATH first
+-        // let config_detail = HashMap::from([
+-        //     (
+-        //         "aaa".to_string(),
+-        //         KeyInfo {
+-        //             value: "3".to_string(),
+-        //             operation: "add".to_string(),
+-        //         },
+-        //     ),
+-        //     (
+-        //         "bbb".to_string(),
+-        //         KeyInfo {
+-        //             value: "1".to_string(),
+-        //             operation: "delete".to_string(),
+-        //         },
+-        //     ),
+-        // ]);
+-        // config.config_path = "".to_string();
+-        // config.contents = config_detail;
+-        // kernel_sysctl_persist.set_config(&mut config).unwrap();
+-        // let result = fs::read_to_string(crate::sys_mgmt::DEFAULT_KERNEL_CONFIG_PATH).unwrap();
+-        // let expected_res = format!("{}\n", "aaa=3",);
+-        // assert_eq!(result, expected_res);
+        let mut config = Sysconfig {
+            model: KERNEL_SYSCTL_PERSIST.to_string(),
+            config_path: String::from("/tmp/kubeos-test-kernel-sysctl-persist.txt"),
+            contents: HashMap::new(),
+        };
+        kernel_sysctl_persist.set_config(&mut config).unwrap();
+        assert!(is_file_exist(&config.config_path));
+        delete_file_or_dir(&config.config_path).unwrap();
+     }
+ 
+     #[test]
+@@ -492,7 +484,7 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri
+         initrd  /boot/initramfs.img
+ }";
+         writeln!(tmp_file, "{}", grub_cfg).unwrap();
+-        let config_first_part = HashMap::from([
+        let config_second_part = HashMap::from([
+             ("debug".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }),
+             ("quiet".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }),
+             ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }),
+@@ -506,15 +498,16 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri
+         let mut config = Sysconfig {
+             model: GRUB_CMDLINE_CURRENT.to_string(),
+             config_path: String::new(),
+-            contents: config_first_part,
+            contents: config_second_part,
+         };
+         grub_cmdline.set_config(&mut config).unwrap();
+         grub_cmdline.is_cur_partition = false;
+-        let config_second = HashMap::from([
+        let config_first_part = HashMap::from([
+             ("pci".to_string(), KeyInfo { value: "nomis".to_string(), operation: "".to_string() }),
+-            ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }),
+            ("quiet".to_string(), KeyInfo { value: "11".to_string(), operation: "delete".to_string() }),
+            ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "update".to_string() }),
+         ]);
+-        config.contents = config_second;
+        config.contents = config_first_part;
+         config.model = GRUB_CMDLINE_NEXT.to_string();
+         grub_cmdline.set_config(&mut config).unwrap();
+         let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap();
+@@ -540,6 +533,11 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri
+ }
+ ";
+         assert_eq!(result, expected_res);
+
+        // test grub.cfg not exist
+        grub_cmdline.grub_path = "/tmp/grub-KubeOS-test.cfg".to_string();
+        let res = grub_cmdline.set_config(&mut config);
+        assert!(res.is_err());
+     }
+ 
+     #[test]
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+index 0b50ad6..dd7036f 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+@@ -48,7 +48,7 @@ impl Default for CtrImageHandler<RealCommandExecutor> {
+ 
+ impl<T: CommandExecutor> CtrImageHandler<T> {
+     #[cfg(test)]
+-    fn new(paths: PreparePath, executor: T) -> Self {
+    pub fn new(paths: PreparePath, executor: T) -> Self {
+         Self { paths, executor }
+     }
+ 
+@@ -301,24 +301,4 @@ mod tests {
+ 
+         assert!(result.is_ok());
+     }
+-
+-    #[test]
+-    #[ignore]
+-    fn test_download_image() {
+-        init();
+-        let ctr = CtrImageHandler { paths: PreparePath::default(), executor: RealCommandExecutor {} };
+-        let update_req = UpgradeRequest {
+-            version: "KubeOS v2".to_string(),
+-            image_type: "containerd".to_string(),
+-            container_image: "docker.io/library/busybox:latest".to_string(),
+-            check_sum: "".to_string(),
+-            image_url: "".to_string(),
+-            flag_safe: false,
+-            mtls: false,
+-            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+-        };
+-        ctr.download_image(&update_req).unwrap();
+-        let tar_path = "/persist/KubeOS-Update/os.tar";
+-        assert_eq!(true, Path::new(tar_path).exists());
+-    }
+ }
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
+index 4ccb603..a120db8 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
+@@ -41,7 +41,7 @@ impl Default for DiskImageHandler<RealCommandExecutor> {
+ 
+ impl<T: CommandExecutor> DiskImageHandler<T> {
+     #[cfg(test)]
+-    fn new(paths: PreparePath, executor: T, certs_path: String) -> Self {
+    pub fn new(paths: PreparePath, executor: T, certs_path: String) -> Self {
+         Self { paths, executor, certs_path }
+     }
+ 
+@@ -392,11 +392,14 @@ mod tests {
+             .with_body("This is a test txt file for KubeOS test.\n")
+             .create();
+         handler.download_image(&upgrade_request).unwrap();
+-
+         assert_eq!(true, handler.paths.image_path.exists());
+         assert_eq!(
+             fs::read(handler.paths.image_path.to_str().unwrap()).unwrap(),
+             "This is a test txt file for KubeOS test.\n".as_bytes()
+         );
+
+        let _m = mockito::mock("GET", "/test.txt").with_status(404).with_body("Not found").create();
+        let res = handler.download_image(&upgrade_request);
+        assert!(res.is_err())
+     }
+ }
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+index 121e257..177dfeb 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+@@ -33,7 +33,7 @@ impl Default for DockerImageHandler<RealCommandExecutor> {
+ 
+ impl<T: CommandExecutor> DockerImageHandler<T> {
+     #[cfg(test)]
+-    fn new(paths: PreparePath, container_name: String, executor: T) -> Self {
+    pub fn new(paths: PreparePath, container_name: String, executor: T) -> Self {
+         Self { paths, container_name, executor }
+     }
+ 
+@@ -129,6 +129,8 @@ mod tests {
+         let result =
+             DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor).check_and_rm_container();
+         assert!(result.is_ok());
+
+        assert_eq!(DockerImageHandler::default().container_name, "kubeos-temp");
+     }
+ 
+     #[test]
+diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs
+index 301a8c8..da8c8c3 100644
+--- a/KubeOS-Rust/manager/src/utils/common.rs
+++ b/KubeOS-Rust/manager/src/utils/common.rs
+@@ -23,14 +23,25 @@ use nix::{mount, mount::MntFlags};
+ use super::executor::CommandExecutor;
+ use crate::sys_mgmt::{MOUNT_DIR, OS_IMAGE_NAME, PERSIST_DIR, ROOTFS_ARCHIVE, UPDATE_DIR};
+ 
+/// * persist_path: /persist
+///
+/// * update_path: /persist/KubeOS-Update
+///
+/// * mount_path: /persist/KubeOS-Update/kubeos-update
+///
+/// * tar_path: /persist/KubeOS-Update/os.tar
+///
+/// * image_path: /persist/update.img
+///
+/// * rootfs_file: os.tar
+ #[derive(Clone)]
+ pub struct PreparePath {
+-    pub persist_path: PathBuf, // persist_path: /persist
+-    pub update_path: PathBuf,  // update_path: /persist/KubeOS-Update
+-    pub mount_path: PathBuf,   // mount_path: /persist/KubeOS-Update/kubeos-update
+-    pub tar_path: PathBuf,     // tar_path: /persist/KubeOS-Update/os.tar
+-    pub image_path: PathBuf,   // image_path: /persist/update.img
+-    pub rootfs_file: String,   // rootfs_file: os.tar
+    pub persist_path: PathBuf,
+    pub update_path: PathBuf,
+    pub mount_path: PathBuf,
+    pub tar_path: PathBuf,
+    pub image_path: PathBuf,
+    pub rootfs_file: String,
+ }
+ 
+ impl Default for PreparePath {
+@@ -72,7 +83,7 @@ pub fn check_disk_size<P: AsRef<Path>>(need_bytes: i64, path: P) -> Result<()> {
+     Ok(())
+ }
+ 
+-// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img
+/// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img
+ pub fn clean_env<P>(update_path: P, mount_path: P, image_path: P) -> Result<()>
+ where
+     P: AsRef<Path>,
+@@ -160,6 +171,7 @@ mod tests {
+     use tempfile::{NamedTempFile, TempDir};
+ 
+     use super::*;
+    use crate::utils::RealCommandExecutor;
+ 
+     // Mock the CommandExecutor trait
+     mock! {
+@@ -278,10 +290,23 @@ mod tests {
+     }
+ 
+     #[test]
+-    #[ignore]
+     fn test_get_boot_mode() {
+         init();
+         let boot_mode = get_boot_mode();
+-        assert!(boot_mode == "uefi");
+        let executor = RealCommandExecutor {};
+        let res = executor.run_command("ls", &["/sys/firmware/efi"]);
+        if res.is_ok() {
+            assert!(boot_mode == "uefi");
+        } else {
+            assert!(boot_mode == "bios");
+        }
+    }
+
+    #[test]
+    fn test_is_command_available() {
+        init();
+        let executor = RealCommandExecutor {};
+        assert_eq!(is_command_available("ls", &executor), true);
+        assert_eq!(is_command_available("aaaabb", &executor), false);
+     }
+ }
+diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs
+index a54fc19..7c3aa02 100644
+--- a/KubeOS-Rust/manager/src/utils/container_image.rs
+++ b/KubeOS-Rust/manager/src/utils/container_image.rs
+@@ -197,12 +197,24 @@ mod tests {
+         let out1 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap();
+         let expect_output = "1111";
+         assert_eq!(out1, expect_output);
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok("invalid output".to_string()));
+        let out2 = get_oci_image_digest(container_runtime, image_name, &mock);
+        assert!(out2.is_err());
+ 
+         let container_runtime = "crictl";
+         let command_output2 = "[docker.io/nginx@sha256:1111]";
+         mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string()));
+-        let out2 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap();
+-        assert_eq!(out2, expect_output);
+        let out3 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap();
+        assert_eq!(out3, expect_output);
+
+        let out4 = get_oci_image_digest("invalid", image_name, &mock);
+        assert!(out4.is_err());
+
+        let container_runtime = "crictl";
+        let command_output3 = "[docker.io/nginx:sha256:1111]";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string()));
+        let out5 = get_oci_image_digest(container_runtime, image_name, &mock);
+        assert!(out5.is_err());
+     }
+ 
+     #[test]
+@@ -211,11 +223,13 @@ mod tests {
+         let mut mock = MockCommandExec::new();
+         let image_name = "docker.io/nginx:latest";
+         let container_runtime = "crictl";
+-        let command_output = "[docker.io/nginx@sha256:1111]";
+-        let check_sum = "1111";
+-        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output.to_string()));
+        let command_output = "[docker.io/nginx@sha256:1a2b]";
+        let check_sum = "1A2B";
+        mock.expect_run_command_with_output().times(2).returning(|_, _| Ok(command_output.to_string()));
+         let result = check_oci_image_digest(container_runtime, image_name, check_sum, &mock);
+         assert!(result.is_ok());
+        let result = check_oci_image_digest(container_runtime, image_name, "1111", &mock);
+        assert!(result.is_err());
+     }
+ 
+     #[test]
+@@ -251,4 +265,26 @@ mod tests {
+         let result = pull_image("aaa", image_name, &mock_executor);
+         assert!(result.is_err());
+     }
+
+    #[test]
+    fn test_remove_image_if_exist() {
+        init();
+        let mut mock_executor = MockCommandExec::new();
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| cmd == "ctr" && args.contains(&"check")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(String::from("something")));
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "ctr" && args.contains(&"rm")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+        let image_name = "docker.io/nginx:latest";
+        let res = remove_image_if_exist("ctr", image_name, &mock_executor);
+        assert!(res.is_ok());
+
+        let res = remove_image_if_exist("invalid", image_name, &mock_executor);
+        assert!(res.is_err());
+    }
+ }
+diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs
+index 0419159..fcfa2d8 100644
+--- a/KubeOS-Rust/manager/src/utils/partition.rs
+++ b/KubeOS-Rust/manager/src/utils/partition.rs
+@@ -22,6 +22,7 @@ pub struct PartitionInfo {
+     pub fs_type: String,
+ }
+ 
+/// get_partition_info returns the current partition info and the next partition info.
+ pub fn get_partition_info<T: CommandExecutor>(executor: &T) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> {
+     let lsblk = executor.run_command_with_output("lsblk", &["-lno", "NAME,MOUNTPOINTS,FSTYPE"])?;
+     // After split whitespace, the root directory line should have 3 elements, which are "sda2 / ext4".
+@@ -93,5 +94,19 @@ mod tests {
+             PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() },
+         );
+         assert_eq!(res, expect_res);
+
+        let command_output2 = "sda\nsda1 /boot/efi vfat\nsda2   ext4\nsda3 / ext4\nsda4 /persist ext4\nsr0  iso9660\n";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string()));
+        let res = get_partition_info(&mock).unwrap();
+        let expect_res = (
+            PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() },
+            PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() },
+        );
+        assert_eq!(res, expect_res);
+
+        let command_output3 = "";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string()));
+        let res = get_partition_info(&mock);
+        assert!(res.is_err());
+     }
+ }
+-- 
+2.34.1
+
--- a/0009-fix-checksum-comparison-log-format.patch
+++ b/0009-fix-checksum-comparison-log-format.patch
@ -0,0 +1,52 @@
+From 23b76996928a59e5472a288e40980fc8ba4b1e34 Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Wed, 24 Jan 2024 10:39:27 +0800
+Subject: [PATCH 09/13] fix: checksum comparison log format
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs | 9 +++++----
+ 1 file changed, 5 insertions(+), 4 deletions(-)
+
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
+index 4ccb603..f60d714 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
+@@ -72,13 +72,14 @@ impl<T: CommandExecutor> DiskImageHandler<T> {
+ 
+     fn checksum_match(&self, file_path: &str, check_sum: &str) -> Result<()> {
+         trace!("Start to check checksum");
+        let check_sum = check_sum.to_ascii_lowercase();
+         let file = fs::read(file_path)?;
+         let mut hasher = Sha256::new();
+         hasher.update(file);
+         let hash = hasher.finalize();
+         // sha256sum -b /persist/update.img
+-        let cal_sum = format!("{:X}", hash);
+-        if cal_sum.to_lowercase() != check_sum.to_lowercase() {
+        let cal_sum = format!("{:X}", hash).to_ascii_lowercase();
+        if cal_sum != check_sum {
+             delete_file_or_dir(file_path)?;
+             bail!("Checksum {} mismatch to {}", cal_sum, check_sum);
+         }
+@@ -302,7 +303,7 @@ mod tests {
+         handler.paths.image_path = tmp_file.path().to_path_buf();
+         let mut req = UpgradeRequest {
+             version: "v2".into(),
+-            check_sum: "98ea7aff44631d183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(),
+            check_sum: "98Ea7aff44631D183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(),
+             image_type: "disk".into(),
+             container_image: "".into(),
+             image_url: "http://localhost:8080/aaa.txt".to_string(),
+@@ -313,7 +314,7 @@ mod tests {
+         assert_eq!(handler.paths.image_path.exists(), true);
+         handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum).unwrap();
+ 
+-        req.check_sum = "1234567890".into();
+        req.check_sum = "1234567Abc".into();
+         let res = handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum);
+         assert!(res.is_err());
+     }
+-- 
+2.34.1
+
--- a/0010-fix-check-image-name-is-valid-regex.patch
+++ b/0010-fix-check-image-name-is-valid-regex.patch
@ -0,0 +1,106 @@
+From f2c736335868873cd0cb7562a7ba95ee7c19a315 Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Thu, 25 Jan 2024 11:57:16 +0800
+Subject: [PATCH 10/13] fix: check image name is valid regex
+
+the regex for checking the validity of the  container image image is wrong in case of "IP:PORT@sha256:111"
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ .../manager/src/utils/container_image.rs      | 73 ++++++++++++++++---
+ 1 file changed, 62 insertions(+), 11 deletions(-)
+
+diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs
+index a54fc19..dc31925 100644
+--- a/KubeOS-Rust/manager/src/utils/container_image.rs
+++ b/KubeOS-Rust/manager/src/utils/container_image.rs
+@@ -17,7 +17,7 @@ use regex::Regex;
+ use super::executor::CommandExecutor;
+ 
+ pub fn is_valid_image_name(image: &str) -> Result<()> {
+-    let pattern = r"^(?P<Registry>[a-z0-9\-.]+\.[a-z0-9\-]+:?[0-9]*)?/?((?P<Name>[a-zA-Z0-9-_]+?)|(?P<UserName>[a-zA-Z0-9-_]+?)/(?P<ImageName>[a-zA-Z-_]+?))(?P<Tag>(?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$";
+    let pattern = r"^((?:[\w.-]+)(?::\d+)?/)*(?:[\w.-]+)((?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$";
+     let reg_ex = Regex::new(pattern)?;
+     if !reg_ex.is_match(image) {
+         bail!("Invalid image name: {}", image);
+@@ -172,16 +172,67 @@ mod tests {
+     #[test]
+     fn test_is_valid_image_name() {
+         init();
+-        let out = is_valid_image_name("nginx").unwrap();
+-        assert_eq!(out, ());
+-        let out =
+-            is_valid_image_name("docker.example.com:5000/gmr/alpine@sha256:11111111111111111111111111111111").unwrap();
+-        assert_eq!(out, ());
+-        let out =
+-            is_valid_image_name("sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8");
+-        match out {
+-            Ok(_) => assert_eq!(true, false),
+-            Err(_) => assert_eq!(true, true),
+        let correct_images = vec![
+            "alpine",
+            "alpine:latest",
+            "localhost/latest",
+            "library/alpine",
+            "localhost:1234/test",
+            "test:1234/blaboon",
+            "alpine:3.7",
+            "docker.example.edu/gmr/alpine:3.7",
+            "docker.example.com:5000/gmr/alpine@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8",
+            "docker.example.co.uk/gmr/alpine/test2:latest",
+            "registry.dobby.org/dobby/dobby-servers/arthound:2019-08-08",
+            "owasp/zap:3.8.0",
+            "registry.dobby.co/dobby/dobby-servers/github-run:2021-10-04",
+            "docker.elastic.co/kibana/kibana:7.6.2",
+            "registry.dobby.org/dobby/dobby-servers/lerphound:latest",
+            "registry.dobby.org/dobby/dobby-servers/marbletown-poc:2021-03-29",
+            "marbles/marbles:v0.38.1",
+            "registry.dobby.org/dobby/dobby-servers/loophole@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8",
+            "sonatype/nexon:3.30.0",
+            "prom/node-exporter:v1.1.1",
+            "sosedoff/pgweb@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8",
+            "sosedoff/pgweb:latest",
+            "registry.dobby.org/dobby/dobby-servers/arpeggio:2021-06-01",
+            "registry.dobby.org/dobby/antique-penguin:release-production",
+            "dalprodictus/halcon:6.7.5",
+            "antigua/antigua:v31",
+            "weblate/weblate:4.7.2-1",
+            "redis:4.0.01-alpine",
+            "registry.dobby.com/dobby/dobby-servers/github-run:latest",
+            "192.168.122.123:5000/kubeos-x86_64:2023-01",
+        ];
+        let wrong_images = vec![
+            "alpine;v1.0",
+            "alpine:latest@sha256:11111111111111111111111111111111",
+            "alpine|v1.0",
+            "alpine&v1.0",
+            "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8",
+            "192.168.122.123:5000/kubeos-x86_64:2023-01@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a",
+            "192.168.122.123:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a",
+            "myimage$%^&",
+            ":myimage",
+            "/myimage",
+            "myimage/",
+            "myimage:",
+            "myimage@@latest",
+            "myimage::tag",
+            "registry.com//myimage:tag",
+            " myimage",
+            "myimage ",
+            "registry.com/:tag",
+            "myimage:",
+            "",
+            ":tag",
+            "IP:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a",
+        ];
+        for image in correct_images {
+            assert!(is_valid_image_name(image).is_ok());
+        }
+        for image in wrong_images {
+            assert!(is_valid_image_name(image).is_err());
+         }
+     }
+ 
+-- 
+2.34.1
+
--- a/0011-Bump-version-from-1.0.4-to-1.0.5.patch
+++ b/0011-Bump-version-from-1.0.4-to-1.0.5.patch
@ -0,0 +1,116 @@
+diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock
+index 2087339..c44c152 100644
+--- a/KubeOS-Rust/Cargo.lock
+++ b/KubeOS-Rust/Cargo.lock
+@@ -189,7 +189,7 @@ dependencies = [
+ 
+ [[package]]
+ name = "cli"
+-version = "0.1.0"
+version = "1.0.5"
+ dependencies = [
+  "anyhow",
+  "jsonrpc",
+@@ -1092,7 +1092,7 @@ dependencies = [
+ 
+ [[package]]
+ name = "manager"
+-version = "0.1.0"
+version = "1.0.5"
+ dependencies = [
+  "anyhow",
+  "env_logger",
+@@ -1335,7 +1335,7 @@ dependencies = [
+ 
+ [[package]]
+ name = "os-agent"
+-version = "0.1.0"
+version = "1.0.5"
+ dependencies = [
+  "anyhow",
+  "env_logger",
+@@ -1498,7 +1498,7 @@ dependencies = [
+ 
+ [[package]]
+ name = "proxy"
+-version = "0.1.0"
+version = "1.0.5"
+ dependencies = [
+  "anyhow",
+  "assert-json-diff",
+diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml
+index 6db4df4..739bbbc 100644
+--- a/KubeOS-Rust/agent/Cargo.toml
+++ b/KubeOS-Rust/agent/Cargo.toml
+@@ -3,7 +3,7 @@ description = "KubeOS os-agent"
+ edition = "2021"
+ license = "MulanPSL-2.0"
+ name = "os-agent"
+-version = "0.1.0"
+version = "1.0.5"
+ 
+ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+ [dependencies]
+diff --git a/KubeOS-Rust/cli/Cargo.toml b/KubeOS-Rust/cli/Cargo.toml
+index 1c46db3..c3c14c6 100644
+--- a/KubeOS-Rust/cli/Cargo.toml
+++ b/KubeOS-Rust/cli/Cargo.toml
+@@ -3,7 +3,7 @@ description = "KubeOS os-agent client"
+ edition = "2021"
+ license = "MulanPSL-2.0"
+ name = "cli"
+-version = "0.1.0"
+version = "1.0.5"
+ 
+ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+ [dependencies]
+diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml
+index 9431fba..311a87c 100644
+--- a/KubeOS-Rust/manager/Cargo.toml
+++ b/KubeOS-Rust/manager/Cargo.toml
+@@ -3,7 +3,7 @@ description = "KubeOS os-agent manager"
+ edition = "2021"
+ license = "MulanPSL-2.0"
+ name = "manager"
+-version = "0.1.0"
+version = "1.0.5"
+ 
+ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+ [dev-dependencies]
+diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml
+index 72eb6b9..fe657af 100644
+--- a/KubeOS-Rust/proxy/Cargo.toml
+++ b/KubeOS-Rust/proxy/Cargo.toml
+@@ -3,7 +3,7 @@ description = "KubeOS os-proxy"
+ edition = "2021"
+ license = "MulanPSL-2.0"
+ name = "proxy"
+-version = "0.1.0"
+version = "1.0.5"
+ 
+ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+ [lib]
+@@ -18,14 +18,14 @@ path = "src/main.rs"
+ anyhow = "1.0.44"
+ async-trait = "0.1"
+ chrono = { version = "0.4", default-features = false, features = ["std"] }
+-cli = { version = "0.1.0", path = "../cli" }
+cli = { version = "1.0.5", path = "../cli" }
+ env_logger = "0.9.0"
+ futures = "0.3.17"
+ h2 = "=0.3.16"
+ k8s-openapi = { version = "0.13.1", features = ["v1_22"] }
+ kube = { version = "0.66.0", features = ["derive", "runtime"] }
+ log = "=0.4.15"
+-manager = { version = "0.1.0", path = "../manager" }
+manager = { version = "1.0.5", path = "../manager" }
+ regex = "=1.7.3"
+ reqwest = { version = "=0.11.18", default-features = false, features = [
+     "json",
+diff --git a/VERSION b/VERSION
+index ee90284..90a27f9 100644
+--- a/VERSION
+++ b/VERSION
+@@ -1 +1 @@
+-1.0.4
+1.0.5
--- a/0012-perf-crd-improve-default-display-of-crd-under-kubect.patch
+++ b/0012-perf-crd-improve-default-display-of-crd-under-kubect.patch
@ -0,0 +1,73 @@
+From 2c90e2e1fae25a8324cf527a71d7501b1d0b7921 Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Fri, 26 Jan 2024 10:53:40 +0800
+Subject: [PATCH 12/13] perf(crd): improve default display of crd under kubectl
+ get
+
+kubectl get os|osinstance doesn't display any valuable information. Adding default printer columns in crd.yaml to display more helpful information such as osversion, config version etc.
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ .../config/crd/upgrade.openeuler.org_os.yaml  | 13 ++++++++++++
+ .../upgrade.openeuler.org_osinstances.yaml    | 21 +++++++++++++++++++
+ 2 files changed, 34 insertions(+)
+
+diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml
+index 3bb1333..2dd822e 100644
+--- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml
+++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml
+@@ -17,6 +17,19 @@ spec:
+   scope: Namespaced
+   versions:
+   - name: v1alpha1
+    additionalPrinterColumns:
+    - name: OS VERSION
+      jsonPath: .spec.osversion
+      type: string
+      description: The version of OS
+    - name: SYSCONFIG VERSION
+      type: string
+      jsonPath: .spec.sysconfigs.version
+      description: The version of sysconfig
+    - name: UPGRADECONFIG VERSION
+      type: string
+      jsonPath: .spec.upgradeconfigs.version
+      description: The version of upgradeconfig
+     schema:
+       openAPIV3Schema:
+         description: OS is a specification for OS in the cluster
+diff --git a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml
+index 3fa70c0..df9119b 100644
+--- a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml
+++ b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml
+@@ -17,6 +17,27 @@ spec:
+   scope: Namespaced
+   versions:
+   - name: v1alpha1
+    additionalPrinterColumns:
+    - name: NODESTATUS
+      type: string
+      jsonPath: .spec.nodestatus
+      description: The status of node
+    - name: SYSCONFIG-VERSION-CURRENT
+      type: string
+      jsonPath: .status.sysconfigs.version
+      description: The current status of sysconfig
+    - name: SYSCONFIG-VERSION-DESIRED
+      type: string
+      jsonPath: .spec.sysconfigs.version
+      description: The expected version of sysconfig
+    - name: UPGRADECONFIG-VERSION-CURRENT
+      type: string
+      jsonPath: .status.upgradeconfigs.version
+      description: The current version of upgradeconfig
+    - name: UPGRADECONFIG-VERSION-DESIRED
+      type: string
+      jsonPath: .spec.upgradeconfigs.version
+      description: The expected version of upgradeconfig
+     schema:
+       openAPIV3Schema:
+         description: OSInstance defines some infomation of a node
+-- 
+2.34.1
+
--- a/0013-fix-logs-content-grammar-and-format.patch
+++ b/0013-fix-logs-content-grammar-and-format.patch
@ -0,0 +1,196 @@
+From fed39fc1ff83f016828d10c8fcbbf26762236dfa Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Wed, 24 Jan 2024 15:49:53 +0800
+Subject: [PATCH 13/13] fix: logs content, grammar and format
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/agent/src/function.rs                    | 3 ++-
+ KubeOS-Rust/manager/src/sys_mgmt/config.rs           | 8 ++++----
+ KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs | 2 +-
+ KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs       | 2 +-
+ KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs     | 6 +++---
+ KubeOS-Rust/manager/src/utils/common.rs              | 8 +++-----
+ KubeOS-Rust/manager/src/utils/executor.rs            | 7 ++++---
+ KubeOS-Rust/manager/src/utils/image_manager.rs       | 2 +-
+ 8 files changed, 19 insertions(+), 19 deletions(-)
+
+diff --git a/KubeOS-Rust/agent/src/function.rs b/KubeOS-Rust/agent/src/function.rs
+index 2c97347..9789d95 100644
+--- a/KubeOS-Rust/agent/src/function.rs
+++ b/KubeOS-Rust/agent/src/function.rs
+@@ -25,7 +25,8 @@ impl RpcFunction {
+         F: FnOnce() -> anyhow::Result<T>,
+     {
+         (f)().map_err(|e| {
+-            error!("{:?}", e);
+            let error_message = format!("{:#}", e);
+            error!("{}", error_message.replace('\n', " ").replace('\r', ""));
+             Error { code: ErrorCode::ServerError(RPC_OP_ERROR), message: format!("{:?}", e), data: None }
+         })
+     }
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+index 48517b4..a629756 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+@@ -66,7 +66,7 @@ pub struct GrubCmdline {
+ 
+ impl Configuration for KernelSysctl {
+     fn set_config(&self, config: &mut Sysconfig) -> Result<()> {
+-        info!("Start set kernel.sysctl");
+        info!("Start setting kernel.sysctl");
+         for (key, key_info) in config.contents.iter() {
+             let proc_path = self.get_proc_path(key);
+             if key_info.operation == "delete" {
+@@ -99,7 +99,7 @@ impl KernelSysctl {
+ 
+ impl Configuration for KernelSysctlPersist {
+     fn set_config(&self, config: &mut Sysconfig) -> Result<()> {
+-        info!("Start set kernel.sysctl.persist");
+        info!("Start setting kernel.sysctl.persist");
+         let mut config_path = &values::DEFAULT_KERNEL_CONFIG_PATH.to_string();
+         if !config.config_path.is_empty() {
+             config_path = &config.config_path;
+@@ -247,9 +247,9 @@ fn handle_add_key(expect_configs: &HashMap<String, KeyInfo>, is_only_key_valid:
+ impl Configuration for GrubCmdline {
+     fn set_config(&self, config: &mut Sysconfig) -> Result<()> {
+         if self.is_cur_partition {
+-            info!("Start set grub.cmdline.current configuration");
+            info!("Start setting grub.cmdline.current configuration");
+         } else {
+-            info!("Start set grub.cmdline.next configuration");
+            info!("Start setting grub.cmdline.next configuration");
+         }
+         if !is_file_exist(&self.grub_path) {
+             bail!("Failed to find grub.cfg file");
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+index dd7036f..5b0d0b7 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+@@ -72,7 +72,7 @@ impl<T: CommandExecutor> CtrImageHandler<T> {
+             .mount_path
+             .to_str()
+             .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?;
+-        info!("Start get rootfs {}", image_name);
+        info!("Start getting rootfs {}", image_name);
+         self.check_and_unmount(mount_path)?;
+         self.executor
+             .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?;
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
+index 7c64bf0..6d836dc 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
+@@ -71,7 +71,7 @@ impl<T: CommandExecutor> DiskImageHandler<T> {
+     }
+ 
+     fn checksum_match(&self, file_path: &str, check_sum: &str) -> Result<()> {
+-        trace!("Start to check checksum");
+        info!("Start checking image checksum");
+         let check_sum = check_sum.to_ascii_lowercase();
+         let file = fs::read(file_path)?;
+         let mut hasher = Sha256::new();
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+index 177dfeb..a8bbee2 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+@@ -42,16 +42,16 @@ impl<T: CommandExecutor> DockerImageHandler<T> {
+         is_valid_image_name(image_name)?;
+         let cli = "docker";
+         remove_image_if_exist(cli, image_name, &self.executor)?;
+-        info!("Start pull image {}", image_name);
+        info!("Start pulling image {}", image_name);
+         pull_image(cli, image_name, &self.executor)?;
+-        info!("Start check image digest");
+        info!("Start checking image digest");
+         check_oci_image_digest(cli, image_name, &req.check_sum, &self.executor)?;
+         Ok(())
+     }
+ 
+     fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> {
+         let image_name = &req.container_image;
+-        info!("Start get rootfs {}", image_name);
+        info!("Start getting rootfs {}", image_name);
+         self.check_and_rm_container()?;
+         debug!("Create container {}", self.container_name);
+         let container_id =
+diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs
+index da8c8c3..a6d62a0 100644
+--- a/KubeOS-Rust/manager/src/utils/common.rs
+++ b/KubeOS-Rust/manager/src/utils/common.rs
+@@ -79,7 +79,6 @@ pub fn check_disk_size<P: AsRef<Path>>(need_bytes: i64, path: P) -> Result<()> {
+     if available_space < need_bytes {
+         bail!("Space is not enough for downloading");
+     }
+-    info!("There is enough disk space to upgrade");
+     Ok(())
+ }
+ 
+@@ -88,9 +87,8 @@ pub fn clean_env<P>(update_path: P, mount_path: P, image_path: P) -> Result<()>
+ where
+     P: AsRef<Path>,
+ {
+-    info!("Clean up the residual upgrade environment");
+     if is_mounted(&mount_path)? {
+-        debug!("Umount {}", mount_path.as_ref().display());
+        debug!("Umount \"{}\"", mount_path.as_ref().display());
+         if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) {
+             bail!("Failed to umount {} in clean_env: {}", mount_path.as_ref().display(), errno);
+         }
+@@ -104,10 +102,10 @@ where
+ pub fn delete_file_or_dir<P: AsRef<Path>>(path: P) -> Result<()> {
+     if is_file_exist(&path) {
+         if fs::metadata(&path)?.is_file() {
+-            debug!("Delete file {}", path.as_ref().display());
+            info!("Delete file \"{}\"", path.as_ref().display());
+             fs::remove_file(&path)?;
+         } else {
+-            debug!("Delete directory {}", path.as_ref().display());
+            info!("Delete directory \"{}\"", path.as_ref().display());
+             fs::remove_dir_all(&path)?;
+         }
+     }
+diff --git a/KubeOS-Rust/manager/src/utils/executor.rs b/KubeOS-Rust/manager/src/utils/executor.rs
+index 8f4cb25..c87bf2a 100644
+--- a/KubeOS-Rust/manager/src/utils/executor.rs
+++ b/KubeOS-Rust/manager/src/utils/executor.rs
+@@ -28,8 +28,9 @@ impl CommandExecutor for RealCommandExecutor {
+         trace!("run_command: {} {:?}", name, args);
+         let output = Command::new(name).args(args).output()?;
+         if !output.status.success() {
+            let stdout = String::from_utf8_lossy(&output.stdout);
+             let error_message = String::from_utf8_lossy(&output.stderr);
+-            bail!("Failed to run command: {} {:?}, stderr: {}", name, args, error_message);
+            bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message);
+         }
+         debug!("run_command: {} {:?} done", name, args);
+         Ok(())
+@@ -38,11 +39,11 @@ impl CommandExecutor for RealCommandExecutor {
+     fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String> {
+         trace!("run_command_with_output: {} {:?}", name, args);
+         let output = Command::new(name).args(args).output()?;
+        let stdout = String::from_utf8_lossy(&output.stdout).to_string();
+         if !output.status.success() {
+             let error_message = String::from_utf8_lossy(&output.stderr);
+-            bail!("Failed to run command: {} {:?}, stderr: {}", name, args, error_message);
+            bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message);
+         }
+-        let stdout = String::from_utf8_lossy(&output.stdout).to_string();
+         debug!("run_command_with_output: {} {:?} done", name, args);
+         Ok(stdout.trim_end_matches('\n').to_string())
+     }
+diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs
+index dc82323..90806cf 100644
+--- a/KubeOS-Rust/manager/src/utils/image_manager.rs
+++ b/KubeOS-Rust/manager/src/utils/image_manager.rs
+@@ -89,7 +89,7 @@ impl<T: CommandExecutor> UpgradeImageManager<T> {
+         self.format_image()?;
+         self.mount_image()?;
+         self.extract_tar_to_image()?;
+-        // Pass empty image_path to clean_env to avoid delete image file
+        // Pass empty image_path to clean_env but avoid deleting the upgrade image
+         clean_env(&self.paths.update_path, &self.paths.mount_path, &PathBuf::new())?;
+         Ok(self)
+     }
+-- 
+2.34.1
+
--- a/0014-fix-os-agent-add-context-when-returning-error.patch
+++ b/0014-fix-os-agent-add-context-when-returning-error.patch
@ -0,0 +1,144 @@
+From a9d6ff63344023e7ecc7d90904f694714514e19e Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Mon, 29 Jan 2024 15:28:29 +0800
+Subject: [PATCH 1/4] fix(os-agent):add context when returning error
+
+In some cases, the error log only contains "No such file" information. It is hard to identify the origin of the error. Provide more context when propagating error.
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/manager/src/sys_mgmt/config.rs     | 18 ++++++++++++------
+ .../manager/src/sys_mgmt/containerd_image.rs   |  6 +++---
+ .../manager/src/sys_mgmt/docker_image.rs       |  6 +++---
+ KubeOS-Rust/manager/src/utils/common.rs        |  8 ++++----
+ 4 files changed, 22 insertions(+), 16 deletions(-)
+
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+index a629756..33efdca 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+@@ -105,9 +105,10 @@ impl Configuration for KernelSysctlPersist {
+             config_path = &config.config_path;
+         }
+         debug!("kernel.sysctl.persist config_path: \"{}\"", config_path);
+-        create_config_file(config_path)?;
+-        let configs = get_and_set_configs(&mut config.contents, config_path)?;
+-        write_configs_to_file(config_path, &configs)?;
+        create_config_file(config_path).with_context(|| format!("Failed to find config path"))?;
+        let configs = get_and_set_configs(&mut config.contents, config_path)
+            .with_context(|| format!("Failed to set persist kernel configs"))?;
+        write_configs_to_file(config_path, &configs).with_context(|| format!("Failed to write configs to file"))?;
+         Ok(())
+     }
+ }
+@@ -254,10 +255,15 @@ impl Configuration for GrubCmdline {
+         if !is_file_exist(&self.grub_path) {
+             bail!("Failed to find grub.cfg file");
+         }
+-        let config_partition =
+-            if cfg!(test) { self.is_cur_partition } else { self.get_config_partition(RealCommandExecutor {})? };
+        let config_partition = if cfg!(test) {
+            self.is_cur_partition
+        } else {
+            self.get_config_partition(RealCommandExecutor {})
+                .with_context(|| format!("Failed to get config partition"))?
+        };
+         debug!("Config_partition: {} (false means partition A, true means partition B)", config_partition);
+-        let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition)?;
+        let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition)
+            .with_context(|| format!("Failed to set grub configs"))?;
+         write_configs_to_file(&self.grub_path, &configs)?;
+         Ok(())
+     }
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+index 5b0d0b7..727949b 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+@@ -12,7 +12,7 @@
+ 
+ use std::{fs, os::unix::fs::PermissionsExt, path::Path};
+ 
+-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, Context, Result};
+ use log::{debug, info};
+ 
+ use crate::{
+@@ -73,12 +73,12 @@ impl<T: CommandExecutor> CtrImageHandler<T> {
+             .to_str()
+             .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?;
+         info!("Start getting rootfs {}", image_name);
+-        self.check_and_unmount(mount_path)?;
+        self.check_and_unmount(mount_path).with_context(|| format!("Failed to clean containerd environment"))?;
+         self.executor
+             .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?;
+         // copy os.tar from mount_path to its partent dir
+         self.copy_file(self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission)?;
+-        self.check_and_unmount(mount_path)?;
+        self.check_and_unmount(mount_path).with_context(|| format!("Failed to clean containerd environment"))?;
+         Ok(())
+     }
+ 
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+index a8bbee2..f697142 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+@@ -1,4 +1,4 @@
+-use anyhow::Result;
+use anyhow::{Context, Result};
+ use log::{debug, info, trace};
+ 
+ use crate::{
+@@ -52,7 +52,7 @@ impl<T: CommandExecutor> DockerImageHandler<T> {
+     fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> {
+         let image_name = &req.container_image;
+         info!("Start getting rootfs {}", image_name);
+-        self.check_and_rm_container()?;
+        self.check_and_rm_container().with_context(|| format!("Failed to remove kubeos-temp container"))?;
+         debug!("Create container {}", self.container_name);
+         let container_id =
+             self.executor.run_command_with_output("docker", &["create", "--name", &self.container_name, image_name])?;
+@@ -65,7 +65,7 @@ impl<T: CommandExecutor> DockerImageHandler<T> {
+                 self.paths.update_path.to_str().unwrap(),
+             ],
+         )?;
+-        self.check_and_rm_container()?;
+        self.check_and_rm_container().with_context(|| format!("Failed to remove kubeos-temp container"))?;
+         Ok(())
+     }
+ 
+diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs
+index a6d62a0..9baf99e 100644
+--- a/KubeOS-Rust/manager/src/utils/common.rs
+++ b/KubeOS-Rust/manager/src/utils/common.rs
+@@ -16,7 +16,7 @@ use std::{
+     path::{Path, PathBuf},
+ };
+ 
+-use anyhow::{anyhow, bail, Result};
+use anyhow::{anyhow, bail, Context, Result};
+ use log::{debug, info, trace};
+ use nix::{mount, mount::MntFlags};
+ 
+@@ -85,7 +85,7 @@ pub fn check_disk_size<P: AsRef<Path>>(need_bytes: i64, path: P) -> Result<()> {
+ /// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img
+ pub fn clean_env<P>(update_path: P, mount_path: P, image_path: P) -> Result<()>
+ where
+-    P: AsRef<Path>,
+    P: AsRef<Path> + std::fmt::Debug,
+ {
+     if is_mounted(&mount_path)? {
+         debug!("Umount \"{}\"", mount_path.as_ref().display());
+@@ -94,8 +94,8 @@ where
+         }
+     }
+     // losetup -D?
+-    delete_file_or_dir(update_path)?;
+-    delete_file_or_dir(image_path)?;
+    delete_file_or_dir(&update_path).with_context(|| format!("Failed to delete {:?}", update_path))?;
+    delete_file_or_dir(&image_path).with_context(|| format!("Failed to delete {:?}", image_path))?;
+     Ok(())
+ }
+ 
+-- 
+2.34.1
+
--- a/0015-proxy-Add-unit-tests-and-delete-useless-dependencies.patch
+++ b/0015-proxy-Add-unit-tests-and-delete-useless-dependencies.patch
--- a/0016-proxy-fix-code-review-issues.patch
+++ b/0016-proxy-fix-code-review-issues.patch
@ -0,0 +1,256 @@
+From 1c5fcb965561dd7fb48118ca50952a5323ae93be Mon Sep 17 00:00:00 2001
+From: liyuanr <liyuanrong1@huawei.com>
+Date: Tue, 30 Jan 2024 15:05:20 +0800
+Subject: [PATCH 3/4] proxy: fix code review issues
+
+1. Fixed the enumeration naming problem.
+2. Resolved the problem of redundant return keywords.
+3. Fix unnecessary reference issues.
+4. Fix unnecessary matches and replace them with if let.
+5. Fix unnecessary copying of bool values.
+
+Signed-off-by: liyuanr <liyuanrong1@huawei.com>
+---
+ .../proxy/src/controller/controller.rs        | 47 +++++++++----------
+ KubeOS-Rust/proxy/src/controller/utils.rs     | 12 ++---
+ 2 files changed, 28 insertions(+), 31 deletions(-)
+
+diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs
+index c21f304..ad44380 100644
+--- a/KubeOS-Rust/proxy/src/controller/controller.rs
+++ b/KubeOS-Rust/proxy/src/controller/controller.rs
+@@ -59,7 +59,7 @@ pub async fn reconcile<T: ApplyApi, U: AgentCall>(
+         .ok_or(Error::MissingSubResource { value: String::from("node.status.node_info") })?
+         .os_image;
+     debug!("os expected osversion is {},actual osversion is {}", os_cr.spec.osversion, node_os_image);
+-    if check_version(&os_cr.spec.osversion, &node_os_image) {
+    if check_version(&os_cr.spec.osversion, node_os_image) {
+         match ConfigType::SysConfig.check_config_version(&os, &osinstance) {
+             ConfigOperation::Reassign => {
+                 debug!("start reassign");
+@@ -92,8 +92,7 @@ pub async fn reconcile<T: ApplyApi, U: AgentCall>(
+         if os_cr.spec.opstype == NODE_STATUS_CONFIG {
+             return Err(Error::UpgradeBeforeConfig);
+         }
+-        match ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) {
+-            ConfigOperation::Reassign => {
+        if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance)  {
+                 debug!("start reassign");
+                 proxy_controller
+                     .refresh_node(
+@@ -104,8 +103,6 @@ pub async fn reconcile<T: ApplyApi, U: AgentCall>(
+                     )
+                     .await?;
+                 return Ok(REQUEUE_NORMAL);
+-            }
+-            _ => {}
+         }
+         if node.labels().contains_key(LABEL_UPGRADING) {
+             if osinstance.spec.nodestatus == NODE_STATUS_IDLE {
+@@ -159,14 +156,14 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+         match osi_api.get(node_name).await {
+             Ok(osi) => {
+                 debug!("osinstance is exist {:?}", osi.name());
+-                return Ok(());
+                Ok(())
+             }
+             Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => {
+                 info!("Create OSInstance {}", node_name);
+                 self.controller_client.create_osinstance(node_name, namespace).await?;
+                 Ok(())
+             }
+-            Err(err) => Err(Error::KubeError { source: err }),
+            Err(err) => Err(Error::KubeClient { source: err }),
+         }
+     }
+ 
+@@ -243,7 +240,7 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+         let namespace = &osinstance
+             .namespace()
+             .ok_or(Error::MissingObjectKey { resource: "osinstance".to_string(), value: "namespace".to_string() })?;
+-        self.controller_client.update_osinstance_status(&osinstance.name(), &namespace, &osinstance.status).await?;
+        self.controller_client.update_osinstance_status(&osinstance.name(), namespace, &osinstance.status).await?;
+         Ok(())
+     }
+ 
+@@ -256,7 +253,7 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+                     match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }) {
+                         Ok(_resp) => {}
+                         Err(e) => {
+-                            return Err(Error::AgentError { source: e });
+                            return Err(Error::Agent { source: e });
+                         }
+                     }
+                 }
+@@ -278,9 +275,9 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+                     image_type: os_cr.spec.imagetype.clone(),
+                     check_sum: os_cr.spec.checksum.clone(),
+                     container_image: os_cr.spec.containerimage.clone(),
+-                    flagsafe: os_cr.spec.flagsafe.clone(),
+                    flagsafe: os_cr.spec.flagsafe,
+                     imageurl: os_cr.spec.imageurl.clone(),
+-                    mtls: os_cr.spec.mtls.clone(),
+                    mtls: os_cr.spec.mtls,
+                     cacert: os_cr.spec.cacert.clone().unwrap_or_default(),
+                     clientcert: os_cr.spec.clientcert.clone().unwrap_or_default(),
+                     clientkey: os_cr.spec.clientkey.clone().unwrap_or_default(),
+@@ -289,14 +286,14 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+                 match self.agent_client.prepare_upgrade_method(upgrade_info) {
+                     Ok(_resp) => {}
+                     Err(e) => {
+-                        return Err(Error::AgentError { source: e });
+                        return Err(Error::Agent { source: e });
+                     }
+                 }
+                 self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?;
+                 match self.agent_client.upgrade_method() {
+                     Ok(_resp) => {}
+                     Err(e) => {
+-                        return Err(Error::AgentError { source: e });
+                        return Err(Error::Agent { source: e });
+                     }
+                 }
+             }
+@@ -306,12 +303,12 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+                 match self.agent_client.rollback_method() {
+                     Ok(_resp) => {}
+                     Err(e) => {
+-                        return Err(Error::AgentError { source: e });
+                        return Err(Error::Agent { source: e });
+                     }
+                 }
+             }
+             _ => {
+-                return Err(Error::OperationError { value: os_cr.spec.opstype.clone() });
+                return Err(Error::Operation { value: os_cr.spec.opstype.clone() });
+             }
+         }
+         Ok(())
+@@ -336,7 +333,7 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+     async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> {
+         use drain::error::DrainError::*;
+         match drain_os(&self.k8s_client.clone(), node_name, force).await {
+-            Err(DeletePodsError { errors, .. }) => Err(Error::DrainNodeError { value: errors.join("; ") }),
+            Err(DeletePodsError { errors, .. }) => Err(Error::DrainNode { value: errors.join("; ") }),
+             _ => Ok(()),
+         }
+     }
+@@ -365,13 +362,13 @@ fn convert_to_agent_config(configs: Configs) -> Option<Vec<Sysconfig>> {
+                 }
+             };
+         }
+-        if agent_configs.len() == 0 {
+        if agent_configs.is_empty() {
+             info!("no contents in all models, no need to configure");
+             return None;
+         }
+         return Some(agent_configs);
+     }
+-    return None;
+    None
+ }
+ 
+ fn convert_to_config_hashmap(contents: Vec<Content>) -> Option<HashMap<String, KeyInfo>> {
+@@ -381,7 +378,7 @@ fn convert_to_config_hashmap(contents: Vec<Content>) -> Option<HashMap<String, K
+             KeyInfo { value: content.value.unwrap_or_default(), operation: content.operation.unwrap_or_default() };
+         contents_tmp.insert(content.key.unwrap_or_default(), key_info);
+     }
+-    return Some(contents_tmp);
+    Some(contents_tmp)
+ }
+ 
+ pub mod reconciler_error {
+@@ -391,19 +388,19 @@ pub mod reconciler_error {
+     #[derive(Error, Debug)]
+     pub enum Error {
+         #[error("Kubernetes reported error: {source}")]
+-        KubeError {
+        KubeClient {
+             #[from]
+             source: kube::Error,
+         },
+ 
+         #[error("Create/Patch OSInstance reported error: {source}")]
+-        ApplyApiError {
+        ApplyApi {
+             #[from]
+             source: apiclient_error::Error,
+         },
+ 
+         #[error("Cannot get environment NODE_NAME, error: {source}")]
+-        EnvError {
+        Env {
+             #[from]
+             source: std::env::VarError,
+         },
+@@ -415,16 +412,16 @@ pub mod reconciler_error {
+         MissingSubResource { value: String },
+ 
+         #[error("operation {} cannot be recognized", value)]
+-        OperationError { value: String },
+        Operation { value: String },
+ 
+         #[error("Expect OS Version is not same with Node OS Version, please upgrade first")]
+         UpgradeBeforeConfig,
+ 
+         #[error("os-agent reported error:{source}")]
+-        AgentError { source: agent_error::Error },
+        Agent { source: agent_error::Error },
+ 
+         #[error("Error when drain node, error reported: {}", value)]
+-        DrainNodeError { value: String },
+        DrainNode { value: String },
+     }
+ }
+ 
+diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs
+index 0f56878..82d960b 100644
+--- a/KubeOS-Rust/proxy/src/controller/utils.rs
+++ b/KubeOS-Rust/proxy/src/controller/utils.rs
+@@ -39,7 +39,7 @@ impl ConfigType {
+         debug!("start check_config_version");
+         let node_status = &osinstance.spec.nodestatus;
+         if node_status == NODE_STATUS_IDLE {
+-            debug!("======node status is idle======");
+            debug!("node status is idle");
+             return ConfigOperation::DoNothing;
+         };
+         match self {
+@@ -47,7 +47,7 @@ impl ConfigType {
+                 let os_config_version = get_config_version(os.spec.upgradeconfigs.as_ref());
+                 let osi_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref());
+                 debug!(
+-                    "=======os upgradeconfig version is{},osinstance spec upragdeconfig version is{}",
+                    "os upgradeconfig version is{},osinstance spec upragdeconfig version is{}",
+                     os_config_version, osi_config_version
+                 );
+                 if !check_version(&os_config_version, &osi_config_version) {
+@@ -61,7 +61,7 @@ impl ConfigType {
+                 let os_config_version = get_config_version(os.spec.sysconfigs.as_ref());
+                 let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref());
+                 debug!(
+-                    "=======os sysconfig version is{},osinstance spec sysconfig version is{}",
+                    "os sysconfig version is{},osinstance spec sysconfig version is{}",
+                     os_config_version, osi_config_version
+                 );
+                 if !check_version(&os_config_version, &osi_config_version) {
+@@ -108,13 +108,13 @@ impl ConfigType {
+             }
+         }
+         debug!(
+-            "=======osinstance soec config version is {},status config version is {}",
+            "osinstance soec config version is {},status config version is {}",
+             spec_config_version, status_config_version
+         );
+         if spec_config_version != status_config_version && osinstance.spec.nodestatus != NODE_STATUS_IDLE {
+-            return ConfigInfo { need_config: true, configs: configs };
+            return ConfigInfo { need_config: true, configs };
+         }
+-        return ConfigInfo { need_config: false, configs: None };
+        ConfigInfo { need_config: false, configs: None }
+     }
+     pub fn set_osi_status_config(&self, osinstance: &mut OSInstance) {
+         match self {
+-- 
+2.34.1
+
--- a/0017-fix-clippy-warnings-and-fmt-code.patch
+++ b/0017-fix-clippy-warnings-and-fmt-code.patch
@ -0,0 +1,917 @@
+From 79f0f59f967be0bcb05d60461af5417141cbe5ab Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Tue, 30 Jan 2024 16:05:56 +0800
+Subject: [PATCH 4/4] fix: clippy warnings and fmt code
+
+fix some clippy warnings and fmt code
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/agent/src/rpc/agent_impl.rs       |  8 +-
+ KubeOS-Rust/cli/src/client.rs                 |  2 +-
+ KubeOS-Rust/manager/src/sys_mgmt/config.rs    | 21 ++---
+ .../manager/src/sys_mgmt/containerd_image.rs  |  9 +-
+ .../manager/src/sys_mgmt/docker_image.rs      |  4 +-
+ .../proxy/src/controller/apiserver_mock.rs    | 67 +++++++++------
+ .../proxy/src/controller/controller.rs        | 75 ++++++++--------
+ KubeOS-Rust/proxy/src/controller/mod.rs       |  2 +-
+ KubeOS-Rust/proxy/src/controller/utils.rs     | 12 +--
+ KubeOS-Rust/proxy/src/drain.rs                | 86 +++++++++----------
+ KubeOS-Rust/proxy/src/main.rs                 |  2 +-
+ 11 files changed, 148 insertions(+), 140 deletions(-)
+
+diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+index 8aef414..5f3a325 100644
+--- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+@@ -56,7 +56,7 @@ impl Default for AgentImpl {
+ }
+ 
+ impl AgentImpl {
+-    pub fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result<Response> {
+    fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result<Response> {
+         let _lock = self.mutex.lock().unwrap();
+         debug!("Received an 'prepare upgrade' request: {:?}", req);
+         info!("Start preparing for upgrading to version: {}", req.version);
+@@ -75,7 +75,7 @@ impl AgentImpl {
+         Ok(Response { status: AgentStatus::UpgradeReady })
+     }
+ 
+-    pub fn upgrade_impl(&self) -> Result<Response> {
+    fn upgrade_impl(&self) -> Result<Response> {
+         let _lock = self.mutex.lock().unwrap();
+         info!("Start to upgrade");
+         let command_executor = RealCommandExecutor {};
+@@ -90,7 +90,7 @@ impl AgentImpl {
+         Ok(Response { status: AgentStatus::Upgraded })
+     }
+ 
+-    pub fn configure_impl(&self, mut req: ConfigureRequest) -> Result<Response> {
+    fn configure_impl(&self, mut req: ConfigureRequest) -> Result<Response> {
+         let _lock = self.mutex.lock().unwrap();
+         debug!("Received a 'configure' request: {:?}", req);
+         info!("Start to configure");
+@@ -107,7 +107,7 @@ impl AgentImpl {
+         Ok(Response { status: AgentStatus::Configured })
+     }
+ 
+-    pub fn rollback_impl(&self) -> Result<Response> {
+    fn rollback_impl(&self) -> Result<Response> {
+         let _lock = self.mutex.lock().unwrap();
+         info!("Start to rollback");
+         let command_executor = RealCommandExecutor {};
+diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs
+index 9765a42..37518bd 100644
+--- a/KubeOS-Rust/cli/src/client.rs
+++ b/KubeOS-Rust/cli/src/client.rs
+@@ -30,7 +30,7 @@ impl Client {
+         Client { json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)) }
+     }
+ 
+-    pub fn build_request<'a>(&self, command: &'a str, params: &'a Vec<Box<RawValue>>) -> Request<'a> {
+    pub fn build_request<'a>(&self, command: &'a str, params: &'a [Box<RawValue>]) -> Request<'a> {
+         let json_rpc_request = self.json_rpc_client.build_request(command, params);
+         let request = Request(json_rpc_request);
+         request
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+index 33efdca..138df9d 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
+@@ -105,10 +105,10 @@ impl Configuration for KernelSysctlPersist {
+             config_path = &config.config_path;
+         }
+         debug!("kernel.sysctl.persist config_path: \"{}\"", config_path);
+-        create_config_file(config_path).with_context(|| format!("Failed to find config path"))?;
+        create_config_file(config_path).with_context(|| format!("Failed to find config path \"{}\"", config_path))?;
+         let configs = get_and_set_configs(&mut config.contents, config_path)
+-            .with_context(|| format!("Failed to set persist kernel configs"))?;
+-        write_configs_to_file(config_path, &configs).with_context(|| format!("Failed to write configs to file"))?;
+            .with_context(|| format!("Failed to set persist kernel configs \"{}\"", config_path))?;
+        write_configs_to_file(config_path, &configs).with_context(|| "Failed to write configs to file".to_string())?;
+         Ok(())
+     }
+ }
+@@ -125,7 +125,7 @@ fn create_config_file(config_path: &str) -> Result<()> {
+ }
+ 
+ fn get_and_set_configs(expect_configs: &mut HashMap<String, KeyInfo>, config_path: &str) -> Result<Vec<String>> {
+-    let f = File::open(config_path)?;
+    let f = File::open(config_path).with_context(|| format!("Failed to open config path \"{}\"", config_path))?;
+     let mut configs_write = Vec::new();
+     for line in io::BufReader::new(f).lines() {
+         let line = line?;
+@@ -169,7 +169,7 @@ fn write_configs_to_file(config_path: &str, configs: &Vec<String>) -> Result<()>
+     Ok(())
+ }
+ 
+-fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String {
+fn handle_delete_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String {
+     let key = config_kv[0];
+     if config_kv.len() == 1 && new_config_info.value.is_empty() {
+         info!("Delete configuration key: \"{}\"", key);
+@@ -190,7 +190,7 @@ fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String
+     String::new()
+ }
+ 
+-fn handle_update_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String {
+fn handle_update_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String {
+     let key = config_kv[0];
+     if !new_config_info.operation.is_empty() {
+         warn!(
+@@ -259,12 +259,13 @@ impl Configuration for GrubCmdline {
+             self.is_cur_partition
+         } else {
+             self.get_config_partition(RealCommandExecutor {})
+-                .with_context(|| format!("Failed to get config partition"))?
+                .with_context(|| "Failed to get config partition".to_string())?
+         };
+         debug!("Config_partition: {} (false means partition A, true means partition B)", config_partition);
+         let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition)
+-            .with_context(|| format!("Failed to set grub configs"))?;
+-        write_configs_to_file(&self.grub_path, &configs)?;
+            .with_context(|| "Failed to set grub configs".to_string())?;
+        write_configs_to_file(&self.grub_path, &configs)
+            .with_context(|| "Failed to write configs to file".to_string())?;
+         Ok(())
+     }
+ }
+@@ -286,7 +287,7 @@ fn get_and_set_grubcfg(
+     grub_path: &str,
+     config_partition: bool,
+ ) -> Result<Vec<String>> {
+-    let f = File::open(grub_path)?;
+    let f = File::open(grub_path).with_context(|| format!("Failed to open grub.cfg \"{}\"", grub_path))?;
+     let re_find_cur_linux = r"^\s*linux.*root=.*";
+     let re = Regex::new(re_find_cur_linux)?;
+     let mut configs_write = Vec::new();
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+index 727949b..80caf29 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
+@@ -73,12 +73,12 @@ impl<T: CommandExecutor> CtrImageHandler<T> {
+             .to_str()
+             .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?;
+         info!("Start getting rootfs {}", image_name);
+-        self.check_and_unmount(mount_path).with_context(|| format!("Failed to clean containerd environment"))?;
+        self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?;
+         self.executor
+             .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?;
+         // copy os.tar from mount_path to its partent dir
+         self.copy_file(self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission)?;
+-        self.check_and_unmount(mount_path).with_context(|| format!("Failed to clean containerd environment"))?;
+        self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?;
+         Ok(())
+     }
+ 
+@@ -103,10 +103,7 @@ impl<T: CommandExecutor> CtrImageHandler<T> {
+ 
+ #[cfg(test)]
+ mod tests {
+-    use std::{
+-        io::Write,
+-        path::{Path, PathBuf},
+-    };
+    use std::{io::Write, path::PathBuf};
+ 
+     use mockall::mock;
+     use tempfile::NamedTempFile;
+diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+index f697142..4d97552 100644
+--- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
+@@ -52,7 +52,7 @@ impl<T: CommandExecutor> DockerImageHandler<T> {
+     fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> {
+         let image_name = &req.container_image;
+         info!("Start getting rootfs {}", image_name);
+-        self.check_and_rm_container().with_context(|| format!("Failed to remove kubeos-temp container"))?;
+        self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?;
+         debug!("Create container {}", self.container_name);
+         let container_id =
+             self.executor.run_command_with_output("docker", &["create", "--name", &self.container_name, image_name])?;
+@@ -65,7 +65,7 @@ impl<T: CommandExecutor> DockerImageHandler<T> {
+                 self.paths.update_path.to_str().unwrap(),
+             ],
+         )?;
+-        self.check_and_rm_container().with_context(|| format!("Failed to remove kubeos-temp container"))?;
+        self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?;
+         Ok(())
+     }
+ 
+diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs
+index ef5977c..2b182ca 100644
+--- a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs
+++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs
+@@ -1,3 +1,35 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::collections::BTreeMap;
+
+use anyhow::Result;
+use cli::{
+    client::Client,
+    method::{
+        callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod,
+        rollback::RollbackMethod, upgrade::UpgradeMethod,
+    },
+};
+use http::{Request, Response};
+use hyper::{body::to_bytes, Body};
+use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod};
+use kube::{
+    api::ObjectMeta,
+    core::{ListMeta, ObjectList},
+    Client as KubeClient, Resource, ResourceExt,
+};
+use mockall::mock;
+
+ use self::mock_error::Error;
+ use super::{
+     agentclient::*,
+@@ -10,23 +42,6 @@ use crate::controller::{
+     values::{LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE},
+     ProxyController,
+ };
+-use anyhow::Result;
+-use cli::client::Client;
+-use cli::method::{
+-    callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod,
+-    rollback::RollbackMethod, upgrade::UpgradeMethod,
+-};
+-use http::{Request, Response};
+-use hyper::{body::to_bytes, Body};
+-use k8s_openapi::api::core::v1::Pod;
+-use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo};
+-use kube::{
+-    api::ObjectMeta,
+-    core::{ListMeta, ObjectList},
+-};
+-use kube::{Client as KubeClient, Resource, ResourceExt};
+-use mockall::mock;
+-use std::collections::BTreeMap;
+ 
+ type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
+ pub struct ApiServerVerifier(ApiServerHandle);
+@@ -66,7 +81,7 @@ impl ApiServerVerifier {
+                         .unwrap()
+                         .handler_node_get(osi)
+                         .await
+-                }
+                },
+                 Testcases::UpgradeNormal(osi) => {
+                     self.handler_osinstance_get_exist(osi.clone())
+                         .await
+@@ -85,7 +100,7 @@ impl ApiServerVerifier {
+                         .unwrap()
+                         .handler_node_pod_list_get(osi)
+                         .await
+-                }
+                },
+                 Testcases::UpgradeUpgradeconfigsVersionMismatch(osi) => {
+                     self.handler_osinstance_get_exist(osi.clone())
+                         .await
+@@ -104,7 +119,7 @@ impl ApiServerVerifier {
+                         .unwrap()
+                         .handler_osinstance_patch_nodestatus_idle(osi)
+                         .await
+-                }
+                },
+                 Testcases::UpgradeOSInstaceNodestatusConfig(osi) => {
+                     self.handler_osinstance_get_exist(osi.clone())
+                         .await
+@@ -114,7 +129,7 @@ impl ApiServerVerifier {
+                         .unwrap()
+                         .handler_node_get_with_label(osi.clone())
+                         .await
+-                }
+                },
+                 Testcases::UpgradeOSInstaceNodestatusIdle(osi) => {
+                     self.handler_osinstance_get_exist(osi.clone())
+                         .await
+@@ -130,7 +145,7 @@ impl ApiServerVerifier {
+                         .unwrap()
+                         .handler_node_uncordon(osi)
+                         .await
+-                }
+                },
+                 Testcases::ConfigNormal(osi) => {
+                     self.handler_osinstance_get_exist(osi.clone())
+                         .await
+@@ -146,7 +161,7 @@ impl ApiServerVerifier {
+                         .unwrap()
+                         .handler_osinstance_patch_nodestatus_idle(osi)
+                         .await
+-                }
+                },
+                 Testcases::ConfigVersionMismatchReassign(osi) => {
+                     self.handler_osinstance_get_exist(osi.clone())
+                         .await
+@@ -159,7 +174,7 @@ impl ApiServerVerifier {
+                         .unwrap()
+                         .handler_osinstance_patch_nodestatus_idle(osi)
+                         .await
+-                }
+                },
+                 Testcases::ConfigVersionMismatchUpdate(osi) => {
+                     self.handler_osinstance_get_exist(osi.clone())
+                         .await
+@@ -172,7 +187,7 @@ impl ApiServerVerifier {
+                         .unwrap()
+                         .handler_osinstance_patch_spec_sysconfig_v2(osi)
+                         .await
+-                }
+                },
+                 Testcases::Rollback(osi) => {
+                     self.handler_osinstance_get_exist(osi.clone())
+                         .await
+@@ -191,7 +206,7 @@ impl ApiServerVerifier {
+                         .unwrap()
+                         .handler_node_pod_list_get(osi)
+                         .await
+-                }
+                },
+             }
+             .expect("Case completed without errors");
+         })
+diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs
+index ad44380..80a85d1 100644
+--- a/KubeOS-Rust/proxy/src/controller/controller.rs
+++ b/KubeOS-Rust/proxy/src/controller/controller.rs
+@@ -72,7 +72,7 @@ pub async fn reconcile<T: ApplyApi, U: AgentCall>(
+                     )
+                     .await?;
+                 return Ok(REQUEUE_NORMAL);
+-            }
+            },
+             ConfigOperation::UpdateConfig => {
+                 debug!("start update config");
+                 osinstance.spec.sysconfigs = os_cr.spec.sysconfigs.clone();
+@@ -81,8 +81,8 @@ pub async fn reconcile<T: ApplyApi, U: AgentCall>(
+                     .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec)
+                     .await?;
+                 return Ok(REQUEUE_ERROR);
+-            }
+-            _ => {}
+            },
+            _ => {},
+         }
+         proxy_controller.set_config(&mut osinstance, ConfigType::SysConfig).await?;
+         proxy_controller
+@@ -92,17 +92,17 @@ pub async fn reconcile<T: ApplyApi, U: AgentCall>(
+         if os_cr.spec.opstype == NODE_STATUS_CONFIG {
+             return Err(Error::UpgradeBeforeConfig);
+         }
+-        if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance)  {
+-                debug!("start reassign");
+-                proxy_controller
+-                    .refresh_node(
+-                        node,
+-                        osinstance,
+-                        &get_config_version(os_cr.spec.upgradeconfigs.as_ref()),
+-                        ConfigType::UpgradeConfig,
+-                    )
+-                    .await?;
+-                return Ok(REQUEUE_NORMAL);
+        if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) {
+            debug!("start reassign");
+            proxy_controller
+                .refresh_node(
+                    node,
+                    osinstance,
+                    &get_config_version(os_cr.spec.upgradeconfigs.as_ref()),
+                    ConfigType::UpgradeConfig,
+                )
+                .await?;
+            return Ok(REQUEUE_NORMAL);
+         }
+         if node.labels().contains_key(LABEL_UPGRADING) {
+             if osinstance.spec.nodestatus == NODE_STATUS_IDLE {
+@@ -157,12 +157,12 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+             Ok(osi) => {
+                 debug!("osinstance is exist {:?}", osi.name());
+                 Ok(())
+-            }
+            },
+             Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => {
+                 info!("Create OSInstance {}", node_name);
+                 self.controller_client.create_osinstance(node_name, namespace).await?;
+                 Ok(())
+-            }
+            },
+             Err(err) => Err(Error::KubeClient { source: err }),
+         }
+     }
+@@ -251,15 +251,15 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+             match config_info.configs.and_then(convert_to_agent_config) {
+                 Some(agent_configs) => {
+                     match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }) {
+-                        Ok(_resp) => {}
+                        Ok(_resp) => {},
+                         Err(e) => {
+                             return Err(Error::Agent { source: e });
+-                        }
+                        },
+                     }
+-                }
+                },
+                 None => {
+                     info!("config is none, No content can be configured.");
+-                }
+                },
+             };
+             self.update_osi_status(osinstance, config_type).await?;
+         }
+@@ -284,32 +284,32 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+                 };
+ 
+                 match self.agent_client.prepare_upgrade_method(upgrade_info) {
+-                    Ok(_resp) => {}
+                    Ok(_resp) => {},
+                     Err(e) => {
+                         return Err(Error::Agent { source: e });
+-                    }
+                    },
+                 }
+                 self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?;
+                 match self.agent_client.upgrade_method() {
+-                    Ok(_resp) => {}
+                    Ok(_resp) => {},
+                     Err(e) => {
+                         return Err(Error::Agent { source: e });
+-                    }
+                    },
+                 }
+-            }
+            },
+             OPERATION_TYPE_ROLLBACK => {
+                 self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?;
+ 
+                 match self.agent_client.rollback_method() {
+-                    Ok(_resp) => {}
+                    Ok(_resp) => {},
+                     Err(e) => {
+                         return Err(Error::Agent { source: e });
+-                    }
+                    },
+                 }
+-            }
+            },
+             _ => {
+                 return Err(Error::Operation { value: os_cr.spec.opstype.clone() });
+-            }
+            },
+         }
+         Ok(())
+     }
+@@ -320,12 +320,12 @@ impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+         node_api.cordon(node_name).await?;
+         info!("Cordon node Successfully{}, start drain nodes", node_name);
+         match self.drain_node(node_name, evict_pod_force).await {
+-            Ok(()) => {}
+            Ok(()) => {},
+             Err(e) => {
+                 node_api.uncordon(node_name).await?;
+                 info!("Drain node {} error, uncordon node successfully", node_name);
+                 return Err(e);
+-            }
+            },
+         }
+         Ok(())
+     }
+@@ -351,7 +351,7 @@ fn convert_to_agent_config(configs: Configs) -> Option<Vec<Sysconfig>> {
+                         contents: contents_tmp,
+                     };
+                     agent_configs.push(config_tmp)
+-                }
+                },
+                 None => {
+                     info!(
+                         "model {} which has configpath {} do not has any contents no need to configure",
+@@ -359,7 +359,7 @@ fn convert_to_agent_config(configs: Configs) -> Option<Vec<Sysconfig>> {
+                         config.configpath.unwrap_or_default()
+                     );
+                     continue;
+-                }
+                },
+             };
+         }
+         if agent_configs.is_empty() {
+@@ -427,11 +427,14 @@ pub mod reconciler_error {
+ 
+ #[cfg(test)]
+ mod test {
+-    use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS};
+-    use crate::controller::apiserver_mock::{timeout_after_5s, MockAgentCallClient, Testcases};
+-    use crate::controller::ControllerClient;
+     use std::env;
+ 
+    use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS};
+    use crate::controller::{
+        apiserver_mock::{timeout_after_5s, MockAgentCallClient, Testcases},
+        ControllerClient,
+    };
+
+     #[tokio::test]
+     async fn test_create_osinstance_with_no_upgrade_or_configuration() {
+         let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs
+index e30c8df..b8a4e6e 100644
+--- a/KubeOS-Rust/proxy/src/controller/mod.rs
+++ b/KubeOS-Rust/proxy/src/controller/mod.rs
+@@ -21,6 +21,6 @@ mod values;
+ 
+ pub use agentclient::{AgentCallClient, AgentClient};
+ pub use apiclient::ControllerClient;
+-pub use controller::{error_policy, reconcile, reconciler_error::Error, ProxyController};
+pub use controller::{error_policy, reconcile, ProxyController};
+ pub use crd::OS;
+ pub use values::SOCK_PATH;
+diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs
+index 82d960b..148ca24 100644
+--- a/KubeOS-Rust/proxy/src/controller/utils.rs
+++ b/KubeOS-Rust/proxy/src/controller/utils.rs
+@@ -56,7 +56,7 @@ impl ConfigType {
+                     );
+                     return ConfigOperation::Reassign;
+                 }
+-            }
+            },
+             ConfigType::SysConfig => {
+                 let os_config_version = get_config_version(os.spec.sysconfigs.as_ref());
+                 let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref());
+@@ -78,7 +78,7 @@ impl ConfigType {
+                         return ConfigOperation::UpdateConfig;
+                     }
+                 }
+-            }
+            },
+         };
+         ConfigOperation::DoNothing
+     }
+@@ -96,7 +96,7 @@ impl ConfigType {
+                     status_config_version = get_config_version(None);
+                 }
+                 configs = osinstance.spec.upgradeconfigs.clone();
+-            }
+            },
+             ConfigType::SysConfig => {
+                 spec_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref());
+                 if let Some(osinstance_status) = osinstance.status.as_ref() {
+@@ -105,7 +105,7 @@ impl ConfigType {
+                     status_config_version = get_config_version(None);
+                 }
+                 configs = osinstance.spec.sysconfigs.clone();
+-            }
+            },
+         }
+         debug!(
+             "osinstance soec config version is {},status config version is {}",
+@@ -127,7 +127,7 @@ impl ConfigType {
+                         sysconfigs: None,
+                     })
+                 }
+-            }
+            },
+             ConfigType::SysConfig => {
+                 if let Some(osi_status) = &mut osinstance.status {
+                     osi_status.sysconfigs = osinstance.spec.sysconfigs.clone();
+@@ -135,7 +135,7 @@ impl ConfigType {
+                     osinstance.status =
+                         Some(OSInstanceStatus { upgradeconfigs: None, sysconfigs: osinstance.spec.sysconfigs.clone() })
+                 }
+-            }
+            },
+         }
+     }
+ }
+diff --git a/KubeOS-Rust/proxy/src/drain.rs b/KubeOS-Rust/proxy/src/drain.rs
+index 72836f9..64417df 100644
+--- a/KubeOS-Rust/proxy/src/drain.rs
+++ b/KubeOS-Rust/proxy/src/drain.rs
+@@ -66,7 +66,7 @@ async fn get_pods_deleted(
+         Ok(pods @ ObjectList { .. }) => pods,
+         Err(err) => {
+             return Err(GetPodListsError { source: err, node_name: node_name.to_string() });
+-        }
+        },
+     };
+     let mut filterd_pods_list: Vec<Pod> = Vec::new();
+     let mut filterd_err: Vec<String> = Vec::new();
+@@ -81,7 +81,7 @@ async fn get_pods_deleted(
+             filterd_pods_list.push(pod);
+         }
+     }
+-    if filterd_err.len() > 0 {
+    if !filterd_err.is_empty() {
+         return Err(DeletePodsError { errors: filterd_err });
+     }
+     Ok(filterd_pods_list.into_iter())
+@@ -189,14 +189,14 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e
+                 let name = (&p).name_any();
+                 info!("Pod {} deleted.", name);
+                 break;
+-            }
+            },
+             Ok(_) => {
+                 info!("Pod '{}' is not yet deleted. Waiting {}s.", pod.name_any(), EVERY_DELETION_CHECK.as_secs_f64());
+-            }
+            },
+             Err(kube::Error::Api(e)) if e.code == response_error_not_found => {
+                 info!("Pod {} is deleted.", pod.name_any());
+                 break;
+-            }
+            },
+             Err(e) => {
+                 error!(
+                     "Get pod {} reported error: '{}', whether pod is deleted cannot be determined, waiting {}s.",
+@@ -204,7 +204,7 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e
+                     e,
+                     EVERY_DELETION_CHECK.as_secs_f64()
+                 );
+-            }
+            },
+         }
+         if start_time.elapsed() > TIMEOUT {
+             return Err(WaitDeletionError { pod_name: pod.name_any(), max_wait: TIMEOUT });
+@@ -223,25 +223,25 @@ fn get_pod_api_with_namespace(client: &kube::Client, pod: &Pod) -> Api<Pod> {
+ }
+ 
+ trait NameAny {
+-    fn name_any(self: &Self) -> String;
+    fn name_any(&self) -> String;
+ }
+ 
+ impl NameAny for &Pod {
+-    fn name_any(self: &Self) -> String {
+    fn name_any(&self) -> String {
+         self.metadata.name.clone().or_else(|| self.metadata.generate_name.clone()).unwrap_or_default()
+     }
+ }
+ trait PodFilter {
+-    fn filter(self: &Self, pod: &Pod) -> Box<FilterResult>;
+    fn filter(&self, pod: &Pod) -> Box<FilterResult>;
+ }
+ 
+ struct FinishedOrFailedFilter {}
+ impl PodFilter for FinishedOrFailedFilter {
+-    fn filter(self: &Self, pod: &Pod) -> Box<FilterResult> {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+         return match pod.status.as_ref() {
+             Some(PodStatus { phase: Some(phase), .. }) if phase == "Failed" || phase == "Succeeded" => {
+                 FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay)
+-            }
+            },
+             _ => FilterResult::create_filter_result(false, "", PodDeleteStatus::Okay),
+         };
+     }
+@@ -251,7 +251,7 @@ struct DaemonFilter {
+     force: bool,
+ }
+ impl PodFilter for DaemonFilter {
+-    fn filter(self: &Self, pod: &Pod) -> Box<FilterResult> {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+         if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() {
+             return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay);
+         }
+@@ -269,25 +269,25 @@ impl PodFilter for DaemonFilter {
+                     let description = format!("Cannot drain Pod '{}': Pod is member of a DaemonSet", pod.name_any());
+                     Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error })
+                 }
+-            }
+            },
+             _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay),
+         };
+     }
+ }
+ impl DaemonFilter {
+     fn new(force: bool) -> DaemonFilter {
+-        return DaemonFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force: force };
+        DaemonFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force }
+     }
+ }
+ 
+ struct MirrorFilter {}
+ impl PodFilter for MirrorFilter {
+-    fn filter(self: &Self, pod: &Pod) -> Box<FilterResult> {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+         return match pod.metadata.annotations.as_ref() {
+             Some(annotations) if annotations.contains_key("kubernetes.io/config.mirror") => {
+                 let description = format!("Ignore Pod '{}': Pod is a static Mirror Pod", pod.name_any());
+                 FilterResult::create_filter_result(false, &description.to_string(), PodDeleteStatus::Warning)
+-            }
+            },
+             _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay),
+         };
+     }
+@@ -298,7 +298,7 @@ struct LocalStorageFilter {
+     force: bool,
+ }
+ impl PodFilter for LocalStorageFilter {
+-    fn filter(self: &Self, pod: &Pod) -> Box<FilterResult> {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+         if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() {
+             return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay);
+         }
+@@ -312,14 +312,14 @@ impl PodFilter for LocalStorageFilter {
+                     let description = format!("Cannot drain Pod '{}': Pod has local Storage", pod.name_any());
+                     Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error })
+                 }
+-            }
+            },
+             _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay),
+         };
+     }
+ }
+ impl LocalStorageFilter {
+     fn new(force: bool) -> LocalStorageFilter {
+-        return LocalStorageFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force: force };
+        LocalStorageFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force }
+     }
+ }
+ struct UnreplicatedFilter {
+@@ -327,7 +327,7 @@ struct UnreplicatedFilter {
+     force: bool,
+ }
+ impl PodFilter for UnreplicatedFilter {
+-    fn filter(self: &Self, pod: &Pod) -> Box<FilterResult> {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+         if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() {
+             return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay);
+         }
+@@ -338,18 +338,18 @@ impl PodFilter for UnreplicatedFilter {
+             return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay);
+         }
+ 
+-        return if !is_replicated && self.force {
+        if !is_replicated && self.force {
+             let description = format!("Force drain Pod '{}': Pod is unreplicated", pod.name_any());
+             Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning })
+         } else {
+             let description = format!("Cannot drain Pod '{}': Pod is unreplicated", pod.name_any());
+             Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error })
+-        };
+        }
+     }
+ }
+ impl UnreplicatedFilter {
+     fn new(force: bool) -> UnreplicatedFilter {
+-        return UnreplicatedFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force: force };
+        UnreplicatedFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force }
+     }
+ }
+ 
+@@ -357,7 +357,7 @@ struct DeletedFilter {
+     delete_wait_timeout: Duration,
+ }
+ impl PodFilter for DeletedFilter {
+-    fn filter(self: &Self, pod: &Pod) -> Box<FilterResult> {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+         let now = Instant::now().elapsed();
+         return match pod.metadata.deletion_timestamp.as_ref() {
+             Some(time)
+@@ -365,7 +365,7 @@ impl PodFilter for DeletedFilter {
+                     && now - Duration::from_secs(time.0.timestamp() as u64) >= self.delete_wait_timeout =>
+             {
+                 FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay)
+-            }
+            },
+             _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay),
+         };
+     }
+@@ -379,14 +379,14 @@ struct CombinedFilter {
+     unreplicated_filter: UnreplicatedFilter,
+ }
+ impl PodFilter for CombinedFilter {
+-    fn filter(self: &Self, pod: &Pod) -> Box<FilterResult> {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+         let mut filter_res = self.deleted_filter.filter(pod);
+         if !filter_res.result {
+             info!("{}", filter_res.desc);
+             return Box::new(FilterResult {
+                 result: filter_res.result,
+                 desc: filter_res.desc.clone(),
+-                status: filter_res.status.clone(),
+                status: filter_res.status,
+             });
+         }
+         filter_res = self.daemon_filter.filter(pod);
+@@ -395,7 +395,7 @@ impl PodFilter for CombinedFilter {
+             return Box::new(FilterResult {
+                 result: filter_res.result,
+                 desc: filter_res.desc.clone(),
+-                status: filter_res.status.clone(),
+                status: filter_res.status,
+             });
+         }
+         filter_res = self.mirror_filter.filter(pod);
+@@ -404,7 +404,7 @@ impl PodFilter for CombinedFilter {
+             return Box::new(FilterResult {
+                 result: filter_res.result,
+                 desc: filter_res.desc.clone(),
+-                status: filter_res.status.clone(),
+                status: filter_res.status,
+             });
+         }
+         filter_res = self.local_storage_filter.filter(pod);
+@@ -413,7 +413,7 @@ impl PodFilter for CombinedFilter {
+             return Box::new(FilterResult {
+                 result: filter_res.result,
+                 desc: filter_res.desc.clone(),
+-                status: filter_res.status.clone(),
+                status: filter_res.status,
+             });
+         }
+         filter_res = self.unreplicated_filter.filter(pod);
+@@ -422,22 +422,22 @@ impl PodFilter for CombinedFilter {
+             return Box::new(FilterResult {
+                 result: filter_res.result,
+                 desc: filter_res.desc.clone(),
+-                status: filter_res.status.clone(),
+                status: filter_res.status,
+             });
+         }
+ 
+-        return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay);
+        FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay)
+     }
+ }
+ impl CombinedFilter {
+     fn new(force: bool) -> CombinedFilter {
+-        return CombinedFilter {
+        CombinedFilter {
+             deleted_filter: DeletedFilter { delete_wait_timeout: TIMEOUT },
+             daemon_filter: DaemonFilter::new(force),
+             mirror_filter: MirrorFilter {},
+             local_storage_filter: LocalStorageFilter::new(force),
+             unreplicated_filter: UnreplicatedFilter::new(force),
+-        };
+        }
+     }
+ }
+ 
+@@ -454,7 +454,7 @@ struct FilterResult {
+ }
+ impl FilterResult {
+     fn create_filter_result(result: bool, desc: &str, status: PodDeleteStatus) -> Box<FilterResult> {
+-        Box::new(FilterResult { result: result, desc: desc.to_string(), status: status })
+        Box::new(FilterResult { result, desc: desc.to_string(), status })
+     }
+ }
+ 
+@@ -468,13 +468,11 @@ impl ErrorHandleStrategy {
+         let backoff =
+             ExponentialBackoff::from_millis(RETRY_BASE_DELAY.as_millis() as u64).max_delay(RETRY_MAX_DELAY).map(jitter);
+ 
+-        return match self {
+-            Self::TolerateStrategy => {
+-                return backoff.take(0);
+-            }
+        match self {
+            Self::TolerateStrategy => backoff.take(0),
+ 
+             Self::RetryStrategy => backoff.take(MAX_RETRIES_TIMES),
+-        };
+        }
+     }
+ }
+ 
+@@ -482,13 +480,7 @@ impl tokio_retry::Condition<error::EvictionError> for ErrorHandleStrategy {
+     fn should_retry(&mut self, error: &error::EvictionError) -> bool {
+         match self {
+             Self::TolerateStrategy => false,
+-            Self::RetryStrategy => {
+-                if let error::EvictionError::EvictionErrorRetry { .. } = error {
+-                    true
+-                } else {
+-                    false
+-                }
+-            }
+            Self::RetryStrategy => matches!(error, error::EvictionError::EvictionErrorRetry { .. }),
+         }
+     }
+ }
+diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs
+index ad36b64..5c122ba 100644
+--- a/KubeOS-Rust/proxy/src/main.rs
+++ b/KubeOS-Rust/proxy/src/main.rs
+@@ -39,7 +39,7 @@ async fn main() -> Result<()> {
+         .run(reconcile, error_policy, Context::new(proxy_controller))
+         .for_each(|res| async move {
+             match res {
+-                Ok(_o) => {}
+                Ok(_o) => {},
+                 Err(e) => error!("reconcile failed: {}", e.to_string()),
+             }
+         })
+-- 
+2.34.1
+
--- a/0018-Bump-tokio-to-1.28.0.patch
+++ b/0018-Bump-tokio-to-1.28.0.patch
@ -0,0 +1,127 @@
+From d6a1854785e9e4ec854654baf024a32ce72d4962 Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Mon, 5 Feb 2024 10:04:46 +0800
+Subject: [PATCH 1/3] Bump tokio to 1.28.0
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/Cargo.lock       | 42 ++++++++++--------------------------
+ KubeOS-Rust/proxy/Cargo.toml |  6 ++++--
+ 2 files changed, 15 insertions(+), 33 deletions(-)
+
+diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock
+index 2342c7b2..004ef234 100644
+--- a/KubeOS-Rust/Cargo.lock
+++ b/KubeOS-Rust/Cargo.lock
+@@ -1133,24 +1133,14 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+ 
+ [[package]]
+ name = "mio"
+-version = "0.7.14"
+version = "0.8.10"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
+checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"
+ dependencies = [
+  "libc",
+  "log",
+- "miow",
+- "ntapi",
+- "winapi",
+-]
+-
+-[[package]]
+-name = "miow"
+-version = "0.3.7"
+-source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
+-dependencies = [
+- "winapi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "windows-sys",
+ ]
+ 
+ [[package]]
+@@ -1234,15 +1224,6 @@ version = "0.3.0"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+ checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
+ 
+-[[package]]
+-name = "ntapi"
+-version = "0.3.7"
+-source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
+-dependencies = [
+- "winapi",
+-]
+-
+ [[package]]
+ name = "num-traits"
+ version = "0.2.17"
+@@ -2077,21 +2058,20 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+ 
+ [[package]]
+ name = "tokio"
+-version = "1.14.0"
+version = "1.28.0"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144"
+checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f"
+ dependencies = [
+  "autocfg",
+  "bytes",
+  "libc",
+- "memchr",
+  "mio",
+  "num_cpus",
+- "once_cell",
+  "pin-project-lite",
+  "signal-hook-registry",
+ "socket2",
+  "tokio-macros",
+- "winapi",
+ "windows-sys",
+ ]
+ 
+ [[package]]
+@@ -2106,13 +2086,13 @@ dependencies = [
+ 
+ [[package]]
+ name = "tokio-macros"
+-version = "1.8.2"
+version = "2.1.0"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+ dependencies = [
+  "proc-macro2",
+  "quote",
+- "syn 1.0.109",
+ "syn 2.0.37",
+ ]
+ 
+ [[package]]
+diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml
+index 94e3b3c8..429c5fdb 100644
+--- a/KubeOS-Rust/proxy/Cargo.toml
+++ b/KubeOS-Rust/proxy/Cargo.toml
+@@ -35,7 +35,10 @@ serde_json = "1.0.68"
+ socket2 = "=0.4.9"
+ thiserror = "1.0.29"
+ thread_local = "=1.1.4"
+-tokio = { version = "=1.14.0", features = ["macros", "rt-multi-thread"] }
+tokio = { version = "=1.28.0", default-features = false, features = [
+    "macros",
+    "rt-multi-thread",
+] }
+ tokio-retry = "0.3"
+ 
+ [dev-dependencies]
+@@ -44,4 +47,3 @@ http = "0.2.9"
+ hyper = "0.14.25"
+ tower-test = "0.4.0"
+ mockall = { version = "=0.11.3" }
+-
+-- 
+2.34.1
+
--- a/0019-build-update-vendor.patch
+++ b/0019-build-update-vendor.patch
--- a/0020-fix-mutex-locking-in-agent_impl.rs.patch
+++ b/0020-fix-mutex-locking-in-agent_impl.rs.patch
@ -0,0 +1,84 @@
+From 0b4843d4514cd8b7e653990025d0ecd5e80d56ba Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Tue, 20 Feb 2024 10:18:27 +0800
+Subject: [PATCH 1/2] fix: mutex locking in agent_impl.rs
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/agent/src/rpc/agent_impl.rs | 32 +++++++++++++++++++++----
+ 1 file changed, 28 insertions(+), 4 deletions(-)
+
+diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+index 5f3a3259..ab826413 100644
+--- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
+@@ -57,7 +57,10 @@ impl Default for AgentImpl {
+ 
+ impl AgentImpl {
+     fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result<Response> {
+-        let _lock = self.mutex.lock().unwrap();
+        let lock = self.mutex.try_lock();
+        if lock.is_err() {
+            bail!("os-agent is processing another request");
+        }
+         debug!("Received an 'prepare upgrade' request: {:?}", req);
+         info!("Start preparing for upgrading to version: {}", req.version);
+ 
+@@ -76,7 +79,10 @@ impl AgentImpl {
+     }
+ 
+     fn upgrade_impl(&self) -> Result<Response> {
+-        let _lock = self.mutex.lock().unwrap();
+        let lock = self.mutex.try_lock();
+        if lock.is_err() {
+            bail!("os-agent is processing another request");
+        }
+         info!("Start to upgrade");
+         let command_executor = RealCommandExecutor {};
+         let (_, next_partition_info) = get_partition_info(&command_executor)?;
+@@ -91,7 +97,10 @@ impl AgentImpl {
+     }
+ 
+     fn configure_impl(&self, mut req: ConfigureRequest) -> Result<Response> {
+-        let _lock = self.mutex.lock().unwrap();
+        let lock = self.mutex.try_lock();
+        if lock.is_err() {
+            bail!("os-agent is processing another request");
+        }
+         debug!("Received a 'configure' request: {:?}", req);
+         info!("Start to configure");
+         let config_map = &*CONFIG_TEMPLATE;
+@@ -108,7 +117,10 @@ impl AgentImpl {
+     }
+ 
+     fn rollback_impl(&self) -> Result<Response> {
+-        let _lock = self.mutex.lock().unwrap();
+        let lock = self.mutex.try_lock();
+        if lock.is_err() {
+            bail!("os-agent is processing another request");
+        }
+         info!("Start to rollback");
+         let command_executor = RealCommandExecutor {};
+         let (_, next_partition_info) = get_partition_info(&command_executor)?;
+@@ -172,6 +184,18 @@ mod test {
+         };
+         let res = agent.configure(req);
+         assert!(res.is_err());
+
+        // test lock
+        let _lock = agent.mutex.lock().unwrap();
+        let req = ConfigureRequest {
+            configs: vec![Sysconfig {
+                model: "kernel.sysctl".to_string(),
+                config_path: "".to_string(),
+                contents: HashMap::new(),
+            }],
+        };
+        let res = agent.configure(req);
+        assert!(res.is_err());
+     }
+ 
+     #[test]
+-- 
+2.34.1
+
--- a/0021-fix-partition-info-retrieval-in-get_partition_info-f.patch
+++ b/0021-fix-partition-info-retrieval-in-get_partition_info-f.patch
@ -0,0 +1,38 @@
+From 3bbb48b9e2569514caa88b9d67aa14d67a48432f Mon Sep 17 00:00:00 2001
+From: Yuhang Wei <weiyuhang3@huawei.com>
+Date: Tue, 20 Feb 2024 10:18:42 +0800
+Subject: [PATCH 2/2] fix: partition info retrieval in get_partition_info
+ function
+
+Signed-off-by: Yuhang Wei <weiyuhang3@huawei.com>
+---
+ KubeOS-Rust/manager/src/utils/partition.rs | 7 ++++++-
+ 1 file changed, 6 insertions(+), 1 deletion(-)
+
+diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs
+index fcfa2d8b..799b4b35 100644
+--- a/KubeOS-Rust/manager/src/utils/partition.rs
+++ b/KubeOS-Rust/manager/src/utils/partition.rs
+@@ -50,7 +50,7 @@ pub fn get_partition_info<T: CommandExecutor>(executor: &T) -> Result<(Partition
+             }
+         }
+     }
+-    if cur_partition.device.is_empty() {
+    if cur_partition.menuentry.is_empty() {
+         bail!("Failed to get partition info, lsblk output: {}", lsblk);
+     }
+     Ok((cur_partition, next_partition))
+@@ -108,5 +108,10 @@ mod tests {
+         mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string()));
+         let res = get_partition_info(&mock);
+         assert!(res.is_err());
+
+        let command_output4 = "sda4 / ext4";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output4.to_string()));
+        let res = get_partition_info(&mock);
+        assert!(res.is_err());
+     }
+ }
+-- 
+2.34.1
+
--- a/KubeOS.spec
+++ b/KubeOS.spec
@ -1,16 +1,37 @@
 # Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved.

 Name:           KubeOS
-Version:        1.0.3
-Release:        3
+Version:        1.0.5
+Release:        4
 Summary:        O&M platform used to update the whole OS as an entirety
 License:        Mulan PSL v2
 Source0:        https://gitee.com/openeuler/KubeOS/repository/archive/v%{version}.tar.gz
-Patch1:         0001-KubeOS-add-oci-image-digests-check-when-upgrade-and-.patch
-Patch2:         0002-KubeOS-support-generate-coredump.patch
+Patch1:         0001-build-rust-os-agent-remove-useless-dependency.patch
+Patch2:         0002-docs-Add-the-content-of-the-user-guide.patch
+Patch3:         0003-Remove-cleanup-method-and-related-code.patch
+Patch4:         0004-test-rust-proxy-add-drain-integration-test.patch
+Patch5:         0005-refactor-rust-os-agent-fix-code-check.patch
+Patch6:         0006-fix-agent-proxy-transform-log-timestamp-to-human-rea.patch
+Patch7:         0007-build-update-vendor.patch
+Patch8:         0008-test-rust-os-agent-add-os-agent-unit-tests.patch
+Patch9:         0009-fix-checksum-comparison-log-format.patch
+Patch10:        0010-fix-check-image-name-is-valid-regex.patch
+Patch11:        0011-Bump-version-from-1.0.4-to-1.0.5.patch
+Patch12:        0012-perf-crd-improve-default-display-of-crd-under-kubect.patch
+Patch13:        0013-fix-logs-content-grammar-and-format.patch
+Patch14:        0014-fix-os-agent-add-context-when-returning-error.patch
+Patch15:        0015-proxy-Add-unit-tests-and-delete-useless-dependencies.patch
+Patch16:        0016-proxy-fix-code-review-issues.patch
+Patch17:        0017-fix-clippy-warnings-and-fmt-code.patch
+Patch18:        0018-Bump-tokio-to-1.28.0.patch
+Patch19:        0019-build-update-vendor.patch
+Patch20:        0020-fix-mutex-locking-in-agent_impl.rs.patch
+Patch21:        0021-fix-partition-info-retrieval-in-get_partition_info-f.patch
+
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
-BuildRequires:  make
+BuildRequires:  make rust cargo openssl-devel
 BuildRequires:  golang >= 1.13
+
 %description
 This is an O&M platform used to update the whole OS as an entirety,
 it should be running in kubernetes environment.
@ -30,14 +51,24 @@ The scripts package includes scripts which could build the os image and binaries
 %{nil}

 %build
+mkdir ./KubeOS-Rust/.cargo
+cat << EOF >> ./KubeOS-Rust/.cargo/config
+
+[source.crates-io]
+replace-with = "vendored-sources"
+
+[source.vendored-sources]
+directory = "vendor"
+EOF
+
 make

 %install
 install -d %{buildroot}%{_bindir}
 #install binary
 install -d -m 0740 %{buildroot}/opt/kubeOS/bin
-install -p -m 0500 ./bin/os-agent %{buildroot}/opt/kubeOS/bin
-install -p -m 0500 ./bin/proxy %{buildroot}/opt/kubeOS/bin
+install -p -m 0500 ./bin/rust/release/os-agent %{buildroot}/opt/kubeOS/bin
+install -p -m 0500 ./bin/rust/release/proxy %{buildroot}/opt/kubeOS/bin
 install -p -m 0500 ./bin/operator %{buildroot}/opt/kubeOS/bin

 #install artifacts
@ -72,8 +103,8 @@ install -p -m 0600 ./files/os-agent.service %{buildroot}/opt/kubeOS/files
 install -p -m 0600 ./files/os-release %{buildroot}/opt/kubeOS/files

 %files
-%attr(0500,root,root) /opt/kubeOS/bin/os-agent
 %defattr(-,root,root,0500)
+%attr(0500,root,root) /opt/kubeOS/bin/os-agent
 %attr(0600,root,root) /opt/kubeOS/files/boot-efi.mount
 %attr(0600,root,root) /opt/kubeOS/files/etc.mount
 %attr(0600,root,root) /opt/kubeOS/files/persist.mount
@ -103,28 +134,63 @@ install -p -m 0600 ./files/os-release %{buildroot}/opt/kubeOS/files
 %attr(0500,root,root) /opt/kubeOS/scripts/00bootup/module-setup.sh
 %attr(0500,root,root) /opt/kubeOS/scripts/00bootup/mount.sh

-
 %clean
 rm -rfv %{buildroot}

 %changelog
-* Wed June 14 2023 liyuanrong<liyuanrong1@huawei.com> - 1.0.3-3
+* Fri Feb 23 2024 Yuhang Wei<weiyuhang3@huawei.com> - 1.0.5-4
 - Type:requirement
 - CVE:NA
 - SUG:restart
- DESC:support generate coredump
+- DESC:fix code check

-* Tue May 30 2023 liyuanrong<liyuanrong1@huawei.com> - 1.0.3-2
+* Mon Feb 05 2024 Yuhang Wei<weiyuhang3@huawei.com> - 1.0.5-3
 - Type:requirement
 - CVE:NA
 - SUG:restart
- DESC:add oci image digests check when upgrade and fix the issue with the software version display
+- DESC:bump tokio to 1.28.0

-* Tue May 16 2023 liyuanrong<liyuanrong1@huawei.com> - 1.0.3-1
+* Mon Jan 29 2024 Yuhang Wei<weiyuhang3@huawei.com> - 1.0.5-2
 - Type:requirement
 - CVE:NA
 - SUG:restart
- DESC:update to 1.0.3-1
+- DESC:sync code from source master branch
+
+* Wed Jan 10 2024 Yuhang Wei<weiyuhang3@huawei.com> - 1.0.5-1
+- Type:requirement
+- CVE:NA
+- SUG:restart
+- DESC:update version to 1.0.5.1
+
+* Tue Sep 05 2023 Yuhang Wei<weiyuhang3@huawei.com> - 1.0.4-4
+- Type:requirement
+- CVE:NA
+- SUG:restart
+- DESC:fix proxy requeue and update image label
+
+* Thu Aug 24 2023 Yuhang Wei<weiyuhang3@huawei.com> - 1.0.4-3
+- Type:requirement
+- CVE:NA
+- SUG:restart
+- DESC:sync code from source master branch
+
+* Mon Aug 07 2023 liyuanrong<liyuanrong1@huawei.com> - 1.0.4-2
+- Type:requirement
+- CVE:NA
+- SUG:restart
+- DESC:fix the hostshell cannot obtain the lib
+
+* Wed Aug 02 2023 liyuanrong<liyuanrong1@huawei.com> - 1.0.4-1
+- Type:requirement
+- CVE:NA
+- SUG:restart
+- DESC:update version to 1.0.4.1
+
+* Thu Dec 08 2022 liyuanrong<liyuanrong1@huawei.com> - 1.0.2-9
+- Type:requirement
+- CVE:NA
+- SUG:restart
+- DESC:sync code from source, support containerd upgrade,settings and admin-container

 * Thu Dec 08 2022 liyuanrong<liyuanrong1@huawei.com> - 1.0.2-8
 - Type:requirement
@ -174,12 +240,6 @@ rm -rfv %{buildroot}
 - SUG:restart
 - DESC:update to 1.0.1-8

-* Fri Dec 09 2022 liyuanrong<liyuanrong1@huawei.com> - 1.0.1-6
- Type:requirement
- CVE:NA
- SUG:restart
- DESC:add yaml
-
 * Fri Dec 17 2021 liyuanrong<liyuanrong1@huawei.com> - 1.0.1-5
 - Type:requirement
 - CVE:NA
@ -198,7 +258,7 @@ rm -rfv %{buildroot}
 - SUG:restart
 - DESC:fix bugs of checks in generate.sh and change module path

-* Fri Oct 30 2021 liyuanrong<liyuanrong1@huawei.com> - 1.0.1-2
+* Sat Oct 30 2021 liyuanrong<liyuanrong1@huawei.com> - 1.0.1-2
 - Type:requirement
 - CVE:NA
 - SUG:restart
--- a/v1.0.3.tar.gz
+++ b/v1.0.3.tar.gz
--- a/v1.0.5.tar.gz
+++ b/v1.0.5.tar.gz