containerd: update patches

0069-containerd-add-check-in-spec.patch
0070-containerd-kill-container-init-process-if-runc-start.patch
0071-containerd-fix-containerd-shim-residual-when-kill-co.patch
0072-containerd-fix-deadlock-on-commit-error.patch
0073-containerd-backport-upstream-patches.patch
0074-containerd-fix-exec-event-missing-due-to-pid-reuse.patch
0075-containerd-fix-dm-left-when-pause-contaienr-and-kill-shim.patch
0076-containerd-fix-start-container-failed-with-id-exists.patch
0077-containerd-drop-opt-package.patch
0078-containerd-bump-containerd-ttrpc-699c4e40d1.patch
0079-containerd-fix-race-access-for-mobySubcribed.patch
0080-containerd-improve-log-for-debugging.patch
0081-containerd-reduce-permissions-for-bundle-di.patch
0082-containerd-fix-publish-command-wait-block-for.patch
0083-containerd-optimize-cgo-compile-options.patch

Signed-off-by:songyanting <songyanting@huawei.com>

containerd.spec

@@ -2,7 +2,7 @@
 %global debug_package %{nil}
 Version:        1.2.0
 Name:           containerd
-Release:        108
+Release:        109
 Summary:        An industry-standard container runtime
 License:        ASL 2.0
 URL:            https://containerd.io
@@ -36,6 +36,7 @@ GO_BUILD_PATH=$PWD/_build
 install -m 0755 -vd $(dirname $GO_BUILD_PATH/src/%{goipath})
 ln -fs $PWD $GO_BUILD_PATH/src/%{goipath}
 cd $GO_BUILD_PATH/src/%{goipath}
+export GO111MODULE=off
 export GOPATH=$GO_BUILD_PATH:%{gopath}
 export BUILDTAGS="no_btrfs no_cri"
 make
@@ -51,6 +52,26 @@ install -p -m 755 bin/containerd-shim $RPM_BUILD_ROOT/%{_bindir}/containerd-shim
 %{_bindir}/containerd-shim
 
 %changelog
+* Sat Jan 22 2022 songyanting<songyanting@huawei.com> - 1.2.0-109
+- Type:bugfix
+- ID:NA
+- SUG:NA
+- DESC:sync bugfix, include
+       1. add check in spec
+       2. kill container init process if runc start returns error
+       3. fix containerd-shim residual when kill containerd during starting container
+       4. fix deadlock on commit error
+       5. backport upstream & ttrpc patches
+       6. fix exec event missing due to pid reuse
+       7. fix dm left when pause container and kill shim
+       8. add CGO security build options
+       9. fix start container failed with id exists
+      10. drop opt package
+      11. fix race access for mobySubscribed
+      12. improve log for debugging
+      13. reduce permission for bundle dir
+      14. fix publish command wait block forever
+
 * Thu Mar 18 2021 xiadanni<xiadanni1@huawei.com> - 1.2.0-108
 - Type:bugfix
 - ID:NA

git-commit

@@ -1 +1 @@
-aec25f8e033c265f30268f7170d83095404adcef
+58b29274635c89604fa2e8b347e6370cbbaad569

patch/0069-containerd-add-check-in-spec.patch

@@ -0,0 +1,27 @@
+From 27be5a04fc8b28e14ff296f5b9356ace8feb39ce Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Thu, 18 Feb 2021 20:28:52 +0800
+Subject: [PATCH] containerd: add check in spec
+
+Change-Id: I8ddf63ec1c4da479e90838678136237b5822d463
+Signed-off-by: xiadanni <xiadanni1@huawei.com>
+---
+ Makefile | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Makefile b/Makefile
+index 96c2370..511b6f2 100644
+--- a/Makefile
++++ b/Makefile
+@@ -151,7 +151,7 @@ build: ## build the go packages
+ 
+ test: ## run tests, except integration tests and tests that require root
+ 	@echo "$(WHALE) $@"
+-	@go test ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES})
++	@go test ${TESTFLAGS} ./gc
+ 
+ root-test: ## run tests, except integration tests
+ 	@echo "$(WHALE) $@"
+-- 
+1.8.3.1
+

patch/0070-containerd-kill-container-init-process-if-runc-start.patch

@@ -0,0 +1,105 @@
+From 52d42e0b850cde3600028b00e19f5325a61ddad3 Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Mon, 1 Feb 2021 19:36:53 +0800
+Subject: [PATCH] containerd: kill container init process if runc start returns
+ error
+
+Signed-off-by: xiadanni <xiadanni1@huawei.com>
+---
+ runtime/v1/linux/proc/init.go |  4 +++
+ utils/utils.go                | 61 +++++++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 65 insertions(+)
+ create mode 100644 utils/utils.go
+
+diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go
+index de76682..669c108 100644
+--- a/runtime/v1/linux/proc/init.go
++++ b/runtime/v1/linux/proc/init.go
+@@ -35,6 +35,7 @@ import (
+ 	"github.com/containerd/containerd/log"
+ 	"github.com/containerd/containerd/mount"
+ 	"github.com/containerd/containerd/runtime/proc"
++	"github.com/containerd/containerd/utils"
+ 	"github.com/containerd/fifo"
+ 	runc "github.com/containerd/go-runc"
+ 	google_protobuf "github.com/gogo/protobuf/types"
+@@ -277,6 +278,9 @@ func (p *Init) Status(ctx context.Context) (string, error) {
+ 
+ func (p *Init) start(context context.Context) error {
+ 	err := p.runtime.Start(context, p.id)
++	if err != nil {
++		utils.KillInitProcess(p.id, p.pid)
++	}
+ 	return p.runtimeError(err, "OCI runtime start failed")
+ }
+ 
+diff --git a/utils/utils.go b/utils/utils.go
+new file mode 100644
+index 0000000..c57c6ca
+--- /dev/null
++++ b/utils/utils.go
+@@ -0,0 +1,61 @@
++/*
++Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved.
++Use of this source code is governed by Apache-2.0
++license that can be found in the LICENSE file.
++Description: common functions
++Author: Danni Xia
++Create: 2021-01-30
++*/
++
++package utils
++
++import (
++	"encoding/json"
++	"io/ioutil"
++	"path/filepath"
++	"strconv"
++	"strings"
++	"syscall"
++
++	"github.com/sirupsen/logrus"
++)
++
++type baseState struct {
++	InitProcessStartTime string `json:"init_process_start"`
++}
++
++func KillInitProcess(cid string, pid int) {
++	if IsInitProcess(cid, pid) {
++		syscall.Kill(pid, syscall.SIGKILL)
++	}
++}
++
++func IsInitProcess(cid string, pid int) bool {
++	stateBytes, err1 := ioutil.ReadFile(filepath.Join("/var/run/docker/runtime-runc/moby", cid, "state.json"))
++	statBytes, err2 := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat"))
++	if err1 != nil || err2 != nil {
++		return true
++	}
++
++	s := strings.Split(string(statBytes), ")")
++	if len(s) < 1 {
++		return true
++	}
++
++	statFields := strings.Split(strings.TrimSpace(s[len(s)-1]), " ")
++	if len(statFields) < 20 {
++		return true
++	}
++
++	var baseState baseState
++	if err := json.Unmarshal(stateBytes, &baseState); err != nil {
++		return true
++	}
++
++	if baseState.InitProcessStartTime == statFields[19] {
++		return true
++	}
++
++	logrus.Warnf("process(pid:%d, start time:%s) is not container %s init process", pid, statFields[19], cid)
++	return false
++}
+-- 
+1.8.3.1
+

patch/0071-containerd-fix-containerd-shim-residual-when-kill-co.patch

@@ -0,0 +1,45 @@
+From 5d72fe2c0d6774e94cad6feacec87db703104fe7 Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Fri, 19 Feb 2021 16:37:48 +0800
+Subject: [PATCH] containerd: fix containerd-shim residual when kill containerd
+ during starting container
+
+after shim process started, containerd will write shim socket address
+to address file, but if containerd is killed before write file, new
+containerd process could not get shim socket address, and will not
+kill it even if that shim could not work.
+so we write address file ahead of starting shim process.
+
+Signed-off-by: xiadanni <xiadanni1@huawei.com>
+---
+ runtime/v1/shim/client/client.go | 7 ++++---
+ 1 file changed, 4 insertions(+), 3 deletions(-)
+
+diff --git a/runtime/v1/shim/client/client.go b/runtime/v1/shim/client/client.go
+index 9e63af4..bc9ac92 100644
+--- a/runtime/v1/shim/client/client.go
++++ b/runtime/v1/shim/client/client.go
+@@ -92,6 +92,10 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa
+ 			go io.Copy(os.Stderr, stderrLog)
+ 		}
+ 
++		if err := writeFile(filepath.Join(config.Path, "address"), address); err != nil {
++			return nil, nil, err
++		}
++
+ 		cmd, err := newCommand(binary, daemonAddress, debug, config, f, stdoutLog, stderrLog)
+ 		if err != nil {
+ 			return nil, nil, err
+@@ -122,9 +126,6 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa
+ 			"debug":   debug,
+ 		}).Infof("shim %s started", binary)
+ 
+-		if err := writeFile(filepath.Join(config.Path, "address"), address); err != nil {
+-			return nil, nil, err
+-		}
+ 		if err := writeFile(filepath.Join(config.Path, "shim.pid"), strconv.Itoa(cmd.Process.Pid)); err != nil {
+ 			return nil, nil, err
+ 		}
+-- 
+1.8.3.1
+

patch/0072-containerd-fix-deadlock-on-commit-error.patch

@@ -0,0 +1,60 @@
+From 39183d7937d408afceb9456972ad3e42beb336c6 Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Sat, 27 Feb 2021 11:19:22 +0800
+Subject: [PATCH] containerd:fix deadlock on commit error
+
+upstream:https://github.com/containerd/containerd/commit/5b9bd993a87008e06a34258f0672a78564adab13
+Signed-off-by: xiadanni <xiadanni1@huawei.com>
+---
+ content/local/writer.go | 5 +++--
+ diff/walking/differ.go  | 5 +++--
+ 2 files changed, 6 insertions(+), 4 deletions(-)
+
+diff --git a/content/local/writer.go b/content/local/writer.go
+index 223b145..3a94744 100644
+--- a/content/local/writer.go
++++ b/content/local/writer.go
+@@ -74,6 +74,9 @@ func (w *writer) Write(p []byte) (n int, err error) {
+ }
+ 
+ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error {
++	// Ensure even on error the writer is fully closed
++	defer unlock(w.ref)
++
+ 	var base content.Info
+ 	for _, opt := range opts {
+ 		if err := opt(&base); err != nil {
+@@ -81,8 +84,6 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest,
+ 		}
+ 	}
+ 
+-	// Ensure even on error the writer is fully closed
+-	defer unlock(w.ref)
+ 	fp := w.fp
+ 	w.fp = nil
+ 
+diff --git a/diff/walking/differ.go b/diff/walking/differ.go
+index a45a563..1c82860 100644
+--- a/diff/walking/differ.go
++++ b/diff/walking/differ.go
+@@ -106,14 +106,15 @@ func (s *walkingDiff) Compare(ctx context.Context, lower, upper []mount.Mount, o
+ 				}
+ 			}()
+ 			if !newReference {
+-				if err := cw.Truncate(0); err != nil {
++				if err = cw.Truncate(0); err != nil {
+ 					return err
+ 				}
+ 			}
+ 
+ 			if isCompressed {
+ 				dgstr := digest.SHA256.Digester()
+-				compressed, err := compression.CompressStream(cw, compression.Gzip)
++				var compressed io.WriteCloser
++				compressed, err = compression.CompressStream(cw, compression.Gzip)
+ 				if err != nil {
+ 					return errors.Wrap(err, "failed to get compressed stream")
+ 				}
+-- 
+1.8.3.1
+

patch/0074-containerd-fix-exec-event-missing-due-to-pid-reuse.patch

@@ -0,0 +1,71 @@
+From dded5a0253fbfd3c75c6d73a890049c832374545 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Sat, 20 Feb 2021 09:06:22 +0800
+Subject: [PATCH] containerd: fix exec event missing due to pid reuse
+
+When many exec request exit at nearly sametime, the Exit can match with
+wrong process and return directly, the event for right process will lost
+in this case.
+
+time="2021-02-19T21:10:12.250841280+08:00" level=info msg=event Pid=11623 containerID=a32a1b7923db55ebdc7483e2b9cd986e5efc750b989ad3507eb866835e8e37f4 execID=0b412ecaed98f9ea71168599a9363b8aa3b047187eadaa74973bb6c63a66118d module=libcontainerd namespace=moby topic=/tasks/exec-started
+time="2021-02-19T21:10:12+08:00" level=info msg="try publish event(1) /tasks/exit &TaskExit{ContainerID:a32a1b7923db55ebdc7483e2b9cd986e5efc750b989ad3507eb866835e8e37f4,ID:0b412ecaed98f9ea71168599a9363b8aa3b047187eadaa74973bb6c63a66118d,Pid:11623,ExitStatus:0,ExitedAt:2021-02-19 21:10:12.27697416 +0800 CST m=+1893.164673481,} <nil>"
+time="2021-02-19T21:11:02.944643980+08:00" level=debug msg="starting exec command 64cd335311e9b3c1c11e7360a374e3218efeb02e6578d7bc0811bad3f1820e16 in container a32a1b7923db55ebdc7483e2b9cd986e5efc750b989ad3507eb866835e8e37f4"
+time="2021-02-19T21:11:06.201162360+08:00" level=debug msg="event published" ns=moby topic="/tasks/exec-started" type=containerd.events.TaskExecStarted
+time="2021-02-19T21:11:57.961615320+08:00" level=warning msg="Ignoring Exit Event, no such exec command found" container=a32a1b7923db55ebdc7483e2b9cd986e5efc750b989ad3507eb866835e8e37f4 exec-id=0b412ecaed98f9ea71168599a9363b8aa3b047187eadaa74973bb6c63a66118d exec-pid=11623
+
+From logs above, execID=0b412ecae with Pid=11623 exit and event
+published, but new exec execID=64cd335 command reuse the Pid, but Exit
+event still match previous execID=0b412ecae. so exit event for
+execID=64cd335 will lost.
+
+Change-Id: If591a282a1cc0305758130a936ee8b92c88acc6c
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ runtime/v1/linux/proc/exec.go | 4 ++++
+ runtime/v1/shim/service.go    | 6 +++++-
+ 2 files changed, 9 insertions(+), 1 deletion(-)
+
+diff --git a/runtime/v1/linux/proc/exec.go b/runtime/v1/linux/proc/exec.go
+index ea40cb5b8..a5f40bd63 100644
+--- a/runtime/v1/linux/proc/exec.go
++++ b/runtime/v1/linux/proc/exec.go
+@@ -86,6 +86,10 @@ func (e *execProcess) ExitedAt() time.Time {
+ }
+ 
+ func (e *execProcess) SetExited(status int) {
++	e.pid.Lock()
++	e.pid.pid = -1
++	e.pid.Unlock()
++
+ 	e.mu.Lock()
+ 	defer e.mu.Unlock()
+ 
+diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go
+index 7e07ab011..7d7327cd8 100644
+--- a/runtime/v1/shim/service.go
++++ b/runtime/v1/shim/service.go
+@@ -548,8 +548,13 @@ func (s *Service) checkProcesses(e runc.Exit) {
+ 		log.G(s.context).WithError(err).Error("failed to check shouldKillAll")
+ 	}
+ 
++	match := 0
+ 	for _, p := range s.processes {
+ 		if p.Pid() == e.Pid {
++			match++
++			if match > 1 {
++				logrus.Warnf("exit for pid=%d match %d processes", e.Pid, match)
++			}
+ 			if ip, ok := p.(*proc.Init); ok {
+ 				ns := filepath.Base(filepath.Dir(ip.Bundle))
+ 				events.ExitAddFile(ns, events.ExitFile(s.id, uint32(e.Pid), uint32(e.Status)), "init exited")
+@@ -591,7 +596,6 @@ func (s *Service) checkProcesses(e runc.Exit) {
+ 				ExitStatus:  uint32(e.Status),
+ 				ExitedAt:    p.ExitedAt(),
+ 			}
+-			return
+ 		}
+ 	}
+ }
+-- 
+2.17.1
+

patch/0075-containerd-fix-dm-left-when-pause-contaienr-and-kill-shim.patch

@@ -0,0 +1,36 @@
+From c10041fa37568bca00a25c055ee844d38e91fa95 Mon Sep 17 00:00:00 2001
+From: chenjiankun <chenjiankun1@huawei.com>
+Date: Mon, 19 Apr 2021 17:08:09 +0800
+Subject: [PATCH] docker: fix dm left when pause contaienr and kill shim
+
+when shim process be killed, we will delete the runtime, but if the
+status is paused, it can't be delete. So we need to resume the shim
+process before delete it.
+---
+ runtime/v1/linux/runtime.go | 10 ++++++++++
+ 1 file changed, 10 insertions(+)
+
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index 66f959d..ca36748 100644
+--- a/runtime/v1/linux/runtime.go
++++ b/runtime/v1/linux/runtime.go
+@@ -541,6 +541,16 @@ func (r *Runtime) terminate(ctx context.Context, bundle *bundle, ns, id string)
+ 	}
+ 
+ 	if !legacy.IsLegacy(id) || legacy.IsSamePid(id) {
++
++		state, err := rt.State(ctx, id)
++		if err == nil && state.Status == "paused" {
++			logrus.Warnf("container %s status is paused, try to resume before delete", id)
++			err := rt.Resume(ctx, id)
++			if err != nil {
++				log.G(ctx).WithError(err).Errorf("runtime resume %s error", id)
++			}
++		}
++
+ 		if err := rt.Delete(ctx, id, &runc.DeleteOpts{
+ 			Force: true,
+ 		}); err != nil {
+-- 
+2.23.0
+

patch/0076-containerd-fix-start-container-failed-with-id-exists.patch

@@ -0,0 +1,34 @@
+From 6936dda1f72b328cacfc29b52da780a29ef45385 Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Thu, 8 Jul 2021 14:37:56 +0800
+Subject: [PATCH] containerd: fix start container failed with id exists
+
+reason: If container root path already exists when call runtime.Create,
+we try to call runtime.Delete to cleanup it. But in case runtime.Delete
+failed, root path will still exists which causes Create failed with error
+"container with id exists". So remove path directly if Delete failed.
+
+Signed-off-by: xiadanni <xiadanni1@huawei.com>
+---
+ vendor/github.com/containerd/go-runc/runc.go | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go
+index 1c96317..c089381 100644
+--- a/vendor/github.com/containerd/go-runc/runc.go
++++ b/vendor/github.com/containerd/go-runc/runc.go
+@@ -159,7 +159,10 @@ func (o *CreateOpts) args() (out []string, err error) {
+ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error {
+ 	if _, err := os.Stat(filepath.Join(r.Root, id)); err == nil {
+ 		logrus.Warnf("cleanup residue runtime with bundle %s root=%s", bundle, r.Root)
+-		r.Delete(context, id, &DeleteOpts{Force: true})
++		if dErr := r.Delete(context, id, &DeleteOpts{Force: true}); dErr != nil {
++			logrus.Errorf("runtime force delete return err: %v, remove container root err: %v",
++				dErr, os.RemoveAll(filepath.Join(r.Root, id)))
++		}
+ 	}
+ 
+ 	args := []string{"create", "--bundle", bundle}
+-- 
+2.27.0
+

patch/0077-containerd-drop-opt-package.patch

@@ -0,0 +1,25 @@
+From 81d14714bb90455964eac557f9b2172d7bc3e522 Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Thu, 5 Aug 2021 15:24:21 +0800
+Subject: [PATCH] [Huawei]containerd: drop opt package
+
+Signed-off-by: xiadanni <xiadanni1@huawei.com>
+---
+ cmd/containerd/builtins.go | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/cmd/containerd/builtins.go b/cmd/containerd/builtins.go
+index b120b60..17fa9f6 100644
+--- a/cmd/containerd/builtins.go
++++ b/cmd/containerd/builtins.go
+@@ -30,7 +30,6 @@ import (
+ 	_ "github.com/containerd/containerd/services/introspection"
+ 	_ "github.com/containerd/containerd/services/leases"
+ 	_ "github.com/containerd/containerd/services/namespaces"
+-	_ "github.com/containerd/containerd/services/opt"
+ 	_ "github.com/containerd/containerd/services/snapshots"
+ 	_ "github.com/containerd/containerd/services/tasks"
+ 	_ "github.com/containerd/containerd/services/version"
+-- 
+2.27.0
+

patch/0078-containerd-bump-containerd-ttrpc-699c4e40d1.patch

@@ -0,0 +1,149 @@
+From 1c8a3bb488eb68523a3ae112854fcdd7326686cb Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Wed, 1 Sep 2021 07:23:17 +0800
+Subject: [PATCH] [backport]containerd:bump containerd/ttrpc
+ 699c4e40d1e7416e08bf7019c7ce2e9beced4636
+
+full diff: https://github.com/containerd/ttrpc/compare/f02858b1457c5ca3aaec3a0803eb0d59f96e41d6...699c4e40d1e7416e08bf7019c7ce2e9beced4636
+
+- containerd/ttrpc#33 Fix returns error message
+- containerd/ttrpc#35 Make onclose an option
+
+Conflict:vendor.conf
+Reference:https://github.com/containerd/containerd/commit/8c5779c32b70a0c55e1c94eb45b305897f7cf3f1
+
+Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
+Signed-off-by: xiadanni <xiadanni1@huawei.com>
+---
+ runtime/v1/shim/client/client.go              |  3 +--
+ runtime/v2/binary.go                          |  3 +--
+ runtime/v2/shim.go                            |  3 +--
+ vendor.conf                                   |  2 +-
+ vendor/github.com/containerd/ttrpc/client.go  | 21 ++++++++++++-------
+ .../github.com/containerd/ttrpc/services.go   |  2 +-
+ 6 files changed, 19 insertions(+), 15 deletions(-)
+
+diff --git a/runtime/v1/shim/client/client.go b/runtime/v1/shim/client/client.go
+index 48d62e537..6861df081 100644
+--- a/runtime/v1/shim/client/client.go
++++ b/runtime/v1/shim/client/client.go
+@@ -299,8 +299,7 @@ func WithConnect(address string, onClose func()) Opt {
+ 		if err != nil {
+ 			return nil, nil, err
+ 		}
+-		client := ttrpc.NewClient(conn)
+-		client.OnClose(onClose)
++		client := ttrpc.NewClient(conn, ttrpc.WithOnClose(onClose))
+ 		return shimapi.NewShimClient(client), conn, nil
+ 	}
+ }
+diff --git a/runtime/v2/binary.go b/runtime/v2/binary.go
+index 41de0d3e0..223b85300 100644
+--- a/runtime/v2/binary.go
++++ b/runtime/v2/binary.go
+@@ -97,8 +97,7 @@ func (b *binary) Start(ctx context.Context) (_ *shim, err error) {
+ 	if err != nil {
+ 		return nil, err
+ 	}
+-	client := ttrpc.NewClient(conn)
+-	client.OnClose(func() { conn.Close() })
++	client := ttrpc.NewClient(conn, ttrpc.WithOnClose(func() { _ = conn.Close() }))
+ 	return &shim{
+ 		bundle:  b.bundle,
+ 		client:  client,
+diff --git a/runtime/v2/shim.go b/runtime/v2/shim.go
+index 982d1bb34..8e746712b 100644
+--- a/runtime/v2/shim.go
++++ b/runtime/v2/shim.go
+@@ -75,8 +75,7 @@ func loadShim(ctx context.Context, bundle *Bundle, events *exchange.Exchange, rt
+ 		}
+ 	}()
+ 
+-	client := ttrpc.NewClient(conn)
+-	client.OnClose(func() { conn.Close() })
++	client := ttrpc.NewClient(conn, ttrpc.WithOnClose(func() { _ = conn.Close() }))
+ 	s := &shim{
+ 		client:  client,
+ 		task:    task.NewTaskClient(client),
+diff --git a/vendor.conf b/vendor.conf
+index dbc3eecd9..0f76be3b0 100644
+--- a/vendor.conf
++++ b/vendor.conf
+@@ -36,7 +36,7 @@ github.com/Microsoft/go-winio v0.4.11
+ github.com/Microsoft/hcsshim v0.7.12
+ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
+ golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4
+-github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a
++github.com/containerd/ttrpc 699c4e40d1e7416e08bf7019c7ce2e9beced4636
+ github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16
+ gotest.tools v2.1.0
+ github.com/google/go-cmp v0.1.0
+diff --git a/vendor/github.com/containerd/ttrpc/client.go b/vendor/github.com/containerd/ttrpc/client.go
+index e40592dd7..bc2bbde1b 100644
+--- a/vendor/github.com/containerd/ttrpc/client.go
++++ b/vendor/github.com/containerd/ttrpc/client.go
+@@ -48,7 +48,15 @@ type Client struct {
+ 	err       error
+ }
+ 
+-func NewClient(conn net.Conn) *Client {
++type ClientOpts func(c *Client)
++
++func WithOnClose(onClose func()) ClientOpts {
++	return func(c *Client) {
++		c.closeFunc = onClose
++	}
++}
++
++func NewClient(conn net.Conn, opts ...ClientOpts) *Client {
+ 	c := &Client{
+ 		codec:     codec{},
+ 		conn:      conn,
+@@ -59,6 +67,10 @@ func NewClient(conn net.Conn) *Client {
+ 		closeFunc: func() {},
+ 	}
+ 
++	for _, o := range opts {
++		o(c)
++	}
++
+ 	go c.run()
+ 	return c
+ }
+@@ -135,11 +147,6 @@ func (c *Client) Close() error {
+ 	return nil
+ }
+ 
+-// OnClose allows a close func to be called when the server is closed
+-func (c *Client) OnClose(closer func()) {
+-	c.closeFunc = closer
+-}
+-
+ type message struct {
+ 	messageHeader
+ 	p   []byte
+@@ -249,7 +256,7 @@ func (c *Client) recv(resp *Response, msg *message) error {
+ 	}
+ 
+ 	if msg.Type != messageTypeResponse {
+-		return errors.New("unkown message type received")
++		return errors.New("unknown message type received")
+ 	}
+ 
+ 	defer c.channel.putmbuf(msg.p)
+diff --git a/vendor/github.com/containerd/ttrpc/services.go b/vendor/github.com/containerd/ttrpc/services.go
+index e90963825..fe1cade5a 100644
+--- a/vendor/github.com/containerd/ttrpc/services.go
++++ b/vendor/github.com/containerd/ttrpc/services.go
+@@ -76,7 +76,7 @@ func (s *serviceSet) dispatch(ctx context.Context, serviceName, methodName strin
+ 		switch v := obj.(type) {
+ 		case proto.Message:
+ 			if err := proto.Unmarshal(p, v); err != nil {
+-				return status.Errorf(codes.Internal, "ttrpc: error unmarshaling payload: %v", err.Error())
++				return status.Errorf(codes.Internal, "ttrpc: error unmarshalling payload: %v", err.Error())
+ 			}
+ 		default:
+ 			return status.Errorf(codes.Internal, "ttrpc: error unsupported request type: %T", v)
+-- 
+2.27.0
+

patch/0079-containerd-fix-race-access-for-mobySubcribed.patch

@@ -0,0 +1,47 @@
+From fe8f7f5acac4f0fcf75218e26c1f3f874a77bf44 Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Wed, 1 Sep 2021 07:29:43 +0800
+Subject: [PATCH] [Huawei]containerd:fix race access for mobySubcribed
+
+Signed-off-by: xiadanni <xiadanni1@huawei.com>
+---
+ events/exchange/exchange.go | 7 ++++---
+ 1 file changed, 4 insertions(+), 3 deletions(-)
+
+diff --git a/events/exchange/exchange.go b/events/exchange/exchange.go
+index 540f18054..ad642563a 100644
+--- a/events/exchange/exchange.go
++++ b/events/exchange/exchange.go
+@@ -19,6 +19,7 @@ package exchange
+ import (
+ 	"context"
+ 	"strings"
++	"sync/atomic"
+ 	"time"
+ 
+ 	"github.com/containerd/containerd/errdefs"
+@@ -49,10 +50,10 @@ func NewExchange() *Exchange {
+ var _ events.Publisher = &Exchange{}
+ var _ events.Forwarder = &Exchange{}
+ var _ events.Subscriber = &Exchange{}
+-var mobySubcribed = false
++var mobySubcribed = int32(0)
+ 
+ func MobySubscribed() bool {
+-	return mobySubcribed
++	return atomic.LoadInt32(&mobySubcribed) == 1
+ }
+ 
+ // Forward accepts an envelope to be direcly distributed on the exchange.
+@@ -170,7 +171,7 @@ func (e *Exchange) Subscribe(ctx context.Context, fs ...string) (ch <-chan *even
+ 	for _, s := range fs {
+ 		if !MobySubscribed() && s == "namespace==moby,topic~=|^/tasks/|" {
+ 			queue.Namespace = "moby"
+-			mobySubcribed = true
++			atomic.StoreInt32(&mobySubcribed, 1)
+ 		}
+ 	}
+ 
+-- 
+2.27.0
+

patch/0080-containerd-improve-log-for-debugging.patch

@@ -0,0 +1,137 @@
+From 003a26f92ccfd6f296910874ed9ad55d652413cc Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Fri, 29 Oct 2021 16:37:28 +0800
+Subject: [PATCH] containerd: improve log for debugging
+
+add following logs for debugging
+1. return event publish errors
+2. redirect is used to make sure that containerd still can read the log
+   of shim after restart
+
+Conflict:NA
+Reference:
+https://github.com/containerd/containerd/pull/3179/commits/74eb0dc81221bffc192a349cf8b14fe7947b7a73
+https://github.com/containerd/containerd/pull/5293/commits/45df696bf3fe3eda15bbf0f2c00ddc2cfeddcdcc
+https://github.com/containerd/containerd/commit/fbb80b9510db14a95b8ffa6c7842666ecf520489
+
+Signed-off-by: xiadanni <xiadanni1@huawei.com>
+---
+ cmd/containerd-shim/main_unix.go | 23 ++++++++++++++++++++---
+ runtime/v1/linux/runtime.go      |  1 +
+ runtime/v1/shim/client/client.go | 22 ++++++++++------------
+ 3 files changed, 31 insertions(+), 15 deletions(-)
+
+diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go
+index 3a5bb6170..a07932cef 100644
+--- a/cmd/containerd-shim/main_unix.go
++++ b/cmd/containerd-shim/main_unix.go
+@@ -61,6 +61,12 @@ var (
+ 	criuFlag             string
+ 	systemdCgroupFlag    bool
+ 	containerdBinaryFlag string
++
++	bufPool = sync.Pool{
++		New: func() interface{} {
++			return bytes.NewBuffer(nil)
++		},
++	}
+ )
+ 
+ func init() {
+@@ -101,6 +107,10 @@ func main() {
+ 		stderr.Close()
+ 	}()
+ 
++	// redirect the following output into fifo to make sure that containerd
++	// still can read the log after restart
++	logrus.SetOutput(stdout)
++
+ 	if err := executeShim(); err != nil {
+ 		fmt.Fprintf(os.Stderr, "containerd-shim: %s\n", err)
+ 		os.Exit(1)
+@@ -110,7 +120,7 @@ func main() {
+ // If containerd server process dies, we need the shim to keep stdout/err reader
+ // FDs so that Linux does not SIGPIPE the shim process if it tries to use its end of
+ // these pipes.
+-func openStdioKeepAlivePipes(dir string) (io.ReadCloser, io.ReadCloser, error) {
++func openStdioKeepAlivePipes(dir string) (io.ReadWriteCloser, io.ReadWriteCloser, error) {
+ 	background := context.Background()
+ 	keepStdoutAlive, err := shimlog.OpenShimStdoutLog(background, dir)
+ 	if err != nil {
+@@ -287,16 +297,23 @@ func (l *remoteEventsPublisher) doPublish(ctx context.Context, topic string, eve
+ 	}
+ 	cmd := exec.CommandContext(ctx, containerdBinaryFlag, "--address", l.address, "publish", "--topic", topic, "--namespace", ns)
+ 	cmd.Stdin = bytes.NewReader(data)
++	b := bufPool.Get().(*bytes.Buffer)
++	defer func() {
++		b.Reset()
++		bufPool.Put(b)
++	}()
++	cmd.Stdout = b
++	cmd.Stderr = b
+ 	c, err := shim.Default.Start(cmd)
+ 	if err != nil {
+ 		return err
+ 	}
+ 	status, err := shim.Default.Wait(cmd, c)
+ 	if err != nil {
+-		return err
++		return errors.Wrapf(err, "failed to publish event: %s", b.String())
+ 	}
+ 	if status != 0 {
+-		return errors.New("failed to publish event")
++		return errors.Errorf("failed to publish event: %s", b.String())
+ 	}
+ 	return nil
+ }
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index ca3674808..eb3927305 100644
+--- a/runtime/v1/linux/runtime.go
++++ b/runtime/v1/linux/runtime.go
+@@ -379,6 +379,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 		log.G(ctx).Infof("load-task %s/%s/%s Pid=%d", r.state, ns, id, pid)
+ 		shimExit := make(chan struct{})
+ 		s, err := bundle.NewShimClient(ctx, ns, ShimConnect(r.config, func() {
++			log.G(ctx).WithField("id", id).Info("shim reaped")
+ 			close(shimExit)
+ 			if _, err := r.tasks.Get(ctx, id); err != nil {
+ 				// Task was never started or was already successfully deleted
+diff --git a/runtime/v1/shim/client/client.go b/runtime/v1/shim/client/client.go
+index eafb0d712..6861df081 100644
+--- a/runtime/v1/shim/client/client.go
++++ b/runtime/v1/shim/client/client.go
+@@ -77,21 +77,19 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa
+ 
+ 		var stdoutLog io.ReadWriteCloser
+ 		var stderrLog io.ReadWriteCloser
+-		if debug {
+-			stdoutLog, err = v1.OpenShimStdoutLog(ctx, config.WorkDir)
+-			if err != nil {
+-				return nil, nil, errors.Wrapf(err, "failed to create stdout log")
+-			}
+-
+-			stderrLog, err = v1.OpenShimStderrLog(ctx, config.WorkDir)
+-			if err != nil {
+-				return nil, nil, errors.Wrapf(err, "failed to create stderr log")
+-			}
++		stdoutLog, err = v1.OpenShimStdoutLog(ctx, config.WorkDir)
++		if err != nil {
++			return nil, nil, errors.Wrapf(err, "failed to create stdout log")
++		}
+ 
+-			go io.Copy(os.Stdout, stdoutLog)
+-			go io.Copy(os.Stderr, stderrLog)
++		stderrLog, err = v1.OpenShimStderrLog(ctx, config.WorkDir)
++		if err != nil {
++			return nil, nil, errors.Wrapf(err, "failed to create stderr log")
+ 		}
+ 
++		go io.Copy(os.Stdout, stdoutLog)
++		go io.Copy(os.Stderr, stderrLog)
++
+ 		if err := writeFile(filepath.Join(config.Path, "address"), address); err != nil {
+ 			return nil, nil, err
+ 		}
+-- 
+2.27.0
+

patch/0081-containerd-reduce-permissions-for-bundle-di.patch

@@ -0,0 +1,138 @@
+From fe70d9e0048502addcbeea5399f2da554a14bd78 Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni1@huawei.com>
+Date: Tue, 9 Nov 2021 16:25:09 +0800
+Subject: [PATCH] [Backport]containerd:reduce permissions for bundle dir to fix
+ CVE-2021-41103
+
+reduce permissions for bundle dir
+reduce permissions on plugin directories
+fix CVE-2021-41103
+
+Conflict:NA
+Reference:https://github.com/containerd/containerd/commit/6886c6a2ec0c70dde1aa64e77b64a5ad47b983c3
+https://github.com/containerd/containerd/commit/7c621e1fcc08bcf5a1a48b837342cc22eada1685
+---
+ runtime/v1/linux/bundle.go | 56 +++++++++++++++++++++++++++++++++++++-
+ snapshots/btrfs/btrfs.go   |  8 ++++--
+ 2 files changed, 61 insertions(+), 3 deletions(-)
+
+diff --git a/runtime/v1/linux/bundle.go b/runtime/v1/linux/bundle.go
+index 0442246f9..90a10862e 100644
+--- a/runtime/v1/linux/bundle.go
++++ b/runtime/v1/linux/bundle.go
+@@ -20,6 +20,7 @@ package linux
+ 
+ import (
+ 	"context"
++	"encoding/json"
+ 	"fmt"
+ 	"io/ioutil"
+ 	"os"
+@@ -30,6 +31,7 @@ import (
+ 	"github.com/containerd/containerd/runtime/linux/runctypes"
+ 	"github.com/containerd/containerd/runtime/v1/shim"
+ 	"github.com/containerd/containerd/runtime/v1/shim/client"
++	"github.com/opencontainers/runtime-spec/specs-go"
+ 	"github.com/pkg/errors"
+ 	"github.com/sirupsen/logrus"
+ )
+@@ -63,7 +65,7 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) {
+ 		time.Sleep(waitTime)
+ 	}
+ 
+-	if err := os.Mkdir(path, 0711); err != nil {
++	if err := os.Mkdir(path, 0700); err != nil {
+ 		return nil, err
+ 	}
+ 	defer func() {
+@@ -71,6 +73,9 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) {
+ 			os.RemoveAll(path)
+ 		}
+ 	}()
++	if err := prepareBundleDirectoryPermissions(path, spec); err != nil {
++		return nil, err
++	}
+ 	if err := os.MkdirAll(workDir, 0711); err != nil {
+ 		return nil, err
+ 	}
+@@ -90,6 +95,55 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) {
+ 	}, err
+ }
+ 
++// prepareBundleDirectoryPermissions prepares the permissions of the bundle
++// directory. When user namespaces are enabled, the permissions are modified
++// to allow the remapped root GID to access the bundle.
++func prepareBundleDirectoryPermissions(path string, spec []byte) error {
++	gid, err := remappedGID(spec)
++	if err != nil {
++		return err
++	}
++	if gid == 0 {
++		return nil
++	}
++	if err := os.Chown(path, -1, int(gid)); err != nil {
++		return err
++	}
++	return os.Chmod(path, 0710)
++}
++
++// ociSpecUserNS is a subset of specs.Spec used to reduce garbage during
++// unmarshal.
++type ociSpecUserNS struct {
++	Linux *linuxSpecUserNS
++}
++
++// linuxSpecUserNS is a subset of specs.Linux used to reduce garbage during
++// unmarshal.
++type linuxSpecUserNS struct {
++	GIDMappings []specs.LinuxIDMapping
++}
++
++// remappedGID reads the remapped GID 0 from the OCI spec, if it exists. If
++// there is no remapping, remappedGID returns 0. If the spec cannot be parsed,
++// remappedGID returns an error.
++func remappedGID(spec []byte) (uint32, error) {
++	var ociSpec ociSpecUserNS
++	err := json.Unmarshal(spec, &ociSpec)
++	if err != nil {
++		return 0, err
++	}
++	if ociSpec.Linux == nil || len(ociSpec.Linux.GIDMappings) == 0 {
++		return 0, nil
++	}
++	for _, mapping := range ociSpec.Linux.GIDMappings {
++		if mapping.ContainerID == 0 {
++			return mapping.HostID, nil
++		}
++	}
++	return 0, nil
++}
++
+ type bundle struct {
+ 	id      string
+ 	path    string
+diff --git a/snapshots/btrfs/btrfs.go b/snapshots/btrfs/btrfs.go
+index a89b55129..da6f8220e 100644
+--- a/snapshots/btrfs/btrfs.go
++++ b/snapshots/btrfs/btrfs.go
+@@ -63,11 +63,15 @@ type snapshotter struct {
+ // root needs to be a mount point of btrfs.
+ func NewSnapshotter(root string) (snapshots.Snapshotter, error) {
+ 	// If directory does not exist, create it
+-	if _, err := os.Stat(root); err != nil {
++	if st, err := os.Stat(root); err != nil {
+ 		if !os.IsNotExist(err) {
+ 			return nil, err
+ 		}
+-		if err := os.Mkdir(root, 0755); err != nil {
++		if err := os.Mkdir(root, 0700); err != nil {
++			return nil, err
++		}
++	} else if st.Mode()&os.ModePerm != 0700 {
++		if err := os.Chmod(root, 0700); err != nil {
+ 			return nil, err
+ 		}
+ 	}
+-- 
+2.27.0
+

patch/0082-containerd-fix-publish-command-wait-block-for.patch

@@ -0,0 +1,25 @@
+From 31cd7bb5147c42384ffd28e9a64f0c5d5c4f7500 Mon Sep 17 00:00:00 2001
+From: chenjiankun <chenjiankun1@huawei.com>
+Date: Wed, 10 Nov 2021 16:10:37 +0800
+Subject: [PATCH] containerd: fix publish command wait block forever
+
+---
+ cmd/containerd-shim/main_unix.go | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go
+index a07932c..37b621e 100644
+--- a/cmd/containerd-shim/main_unix.go
++++ b/cmd/containerd-shim/main_unix.go
+@@ -308,7 +308,7 @@ func (l *remoteEventsPublisher) doPublish(ctx context.Context, topic string, eve
+ 	if err != nil {
+ 		return err
+ 	}
+-	status, err := shim.Default.Wait(cmd, c)
++	status, err := shim.Default.WaitTimeout(cmd, c, 30)
+ 	if err != nil {
+ 		return errors.Wrapf(err, "failed to publish event: %s", b.String())
+ 	}
+-- 
+2.27.0
+

patch/0083-containerd-optimize-cgo-compile-options.patch

@@ -0,0 +1,34 @@
+From 4ae41b01ba2dfd05e8eae0adac6dc3d54c461117 Mon Sep 17 00:00:00 2001
+From: songyanting <songyanting@huawei.com>
+Date: Mon, 24 Jan 2022 11:08:44 +0800
+Subject: [PATCH] [Huawei]containerd:optimize cgo compile options
+
+offering:EulerOS Server
+Type:bugfix
+CVE:
+DTS/AR:
+reason:optimize cgo compile options
+
+Signed-off-by: songyanting songyanting@huawei.com
+---
+ Makefile | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index a7d0888..49a90e6 100644
+--- a/Makefile
++++ b/Makefile
+@@ -172,8 +172,8 @@ bin/%: cmd/% FORCE
+ 	mkdir -p $(BEP_DIR)
+ 	@echo "$(WHALE) $@${BINARY_SUFFIX}"
+ 	CGO_ENABLED=1 \
+-	CGO_CFLAGS="-fstack-protector-strong -fPIE" \
+-	CGO_CPPFLAGS="-fstack-protector-strong -fPIE" \
++	CGO_CFLAGS="-fstack-protector-strong" \
++	CGO_CPPFLAGS="-fstack-protector-strong" \
+ 	CGO_LDFLAGS_ALLOW='-Wl,-z,relro,-z,now' \
+ 	CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \
+ 	go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@${BINARY_SUFFIX} ${GO_LDFLAGS} ${GO_TAGS}  ./$<
+-- 
+2.23.0
+

series.conf

@@ -73,4 +73,19 @@ patch/0065-containerd-fix-dead-loop.patch
 patch/0066-containerd-cleanup-dangling-shim-by-brand-new-context.patch
 patch/0067-containerd-fix-potential-panic-for-task-in-unknown-state.patch
 patch/0068-containerd-compile-option-compliance.patch
+patch/0069-containerd-add-check-in-spec.patch
+patch/0070-containerd-kill-container-init-process-if-runc-start.patch
+patch/0071-containerd-fix-containerd-shim-residual-when-kill-co.patch
+patch/0072-containerd-fix-deadlock-on-commit-error.patch
+patch/0073-containerd-backport-upstream-patches.patch
+patch/0074-containerd-fix-exec-event-missing-due-to-pid-reuse.patch
+patch/0075-containerd-fix-dm-left-when-pause-contaienr-and-kill-shim.patch
+patch/0076-containerd-fix-start-container-failed-with-id-exists.patch
+patch/0077-containerd-drop-opt-package.patch
+patch/0078-containerd-bump-containerd-ttrpc-699c4e40d1.patch
+patch/0079-containerd-fix-race-access-for-mobySubcribed.patch
+patch/0080-containerd-improve-log-for-debugging.patch
+patch/0081-containerd-reduce-permissions-for-bundle-di.patch
+patch/0082-containerd-fix-publish-command-wait-block-for.patch
+patch/0083-containerd-optimize-cgo-compile-options.patch
 # end