!1 first commit

Merge pull request !1 from openeuler-basic/master
2019-12-30 14:15:03 +08:00 · 2019-12-30 14:15:03 +08:00 · 4848bdbebd
commit 4848bdbebd
parent cc4fbfeb56 7b8aa4184d
50 changed files with 3918 additions and 36 deletions
--- a/README.en.md
+++ b/README.en.md
@ -1,36 +0,0 @@
-# containerd
-
-#### Description
-{**When you're done, you can delete the content in this README and update the file with details for others getting started with your repository**}
-
-#### Software Architecture
-Software architecture description
-
-#### Installation
-
-1.  xxxx
-2.  xxxx
-3.  xxxx
-
-#### Instructions
-
-1.  xxxx
-2.  xxxx
-3.  xxxx
-
-#### Contribution
-
-1.  Fork the repository
-2.  Create Feat_xxx branch
-3.  Commit your code
-4.  Create Pull Request
-
-
-#### Gitee Feature
-
-1.  You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md
-2.  Gitee blog [blog.gitee.com](https://blog.gitee.com)
-3.  Explore open source project [https://gitee.com/explore](https://gitee.com/explore)
-4.  The most valuable open source project [GVP](https://gitee.com/gvp)
-5.  The manual of Gitee [https://gitee.com/help](https://gitee.com/help)
-6.  The most popular members  [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)
--- a/36
+++ b/36
@ -0,0 +1,36 @@
+#! /bin/bash
+
+# Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved.
+# Description: This shell script is used to apply patches for the project
+# Author: zhangyu235@huawei.com
+# Create: 2019-05-17
+
+set -ex
+
+pkg=containerd-1.2.0
+cwd=$PWD
+src=$cwd/$pkg
+
+unzip $pkg.zip
+
+cd $src
+git init
+git add .
+git config user.name 'build'
+git config user.email 'build@obs.com'
+git commit -m 'init build'
+cd $cwd
+
+series=$cwd/series.conf
+while IPF= read -r line
+do
+	if [[ "$line" =~ ^patch* ]]; then
+		echo git apply $cwd/$line
+		cd $src && git apply $cwd/$line
+	fi
+done <"$series"
+
+cd $cwd
+cp -rf $src/* .
+mv $src/.git $src/git
+rm -rf containerd-1.2.0
--- a/containerd-1.2.0.zip
+++ b/containerd-1.2.0.zip
--- a/containerd.spec
+++ b/containerd.spec
@ -0,0 +1,43 @@
+%global goipath github.com/containerd/containerd
+%global debug_package %{nil}
+Version:        1.2.0
+Name:           containerd
+Release:        100
+Summary:        An industry-standard container runtime
+License:        ASL 2.0
+URL:            https://containerd.io
+Source0:        %{name}-%{version}.tar.gz
+
+BuildRequires:  golang glibc-static make btrfs-progs-devel
+
+%description
+containerd is an industry-standard container runtime with an emphasis on
+simplicity, robustness and portability.  It is available as a daemon for Linux
+and Windows, which can manage the complete container lifecycle of its host
+system: image transfer and storage, container execution and supervision,
+low-level storage and network attachments, etc.
+
+%prep
+%setup -c -n containerd
+
+%build
+./apply-patch
+
+GO_BUILD_PATH=$PWD/_build
+install -m 0755 -vd $(dirname $GO_BUILD_PATH/src/%{goipath})
+ln -fs $PWD $GO_BUILD_PATH/src/%{goipath}
+cd $GO_BUILD_PATH/src/%{goipath}
+export GOPATH=$GO_BUILD_PATH:%{gopath}
+export BUILDTAGS="no_btrfs no_cri"
+make
+
+%install
+install -d $RPM_BUILD_ROOT/%{_bindir}
+install -p -m 755 bin/containerd $RPM_BUILD_ROOT/%{_bindir}/containerd
+install -p -m 755 bin/containerd-shim $RPM_BUILD_ROOT/%{_bindir}/containerd-shim
+
+%files
+%{_bindir}/containerd
+%{_bindir}/containerd-shim
+
+%changelog
--- a/patch/0001-grpc-vendor-grpc-fix-grpc-map-panic.patch
+++ b/patch/0001-grpc-vendor-grpc-fix-grpc-map-panic.patch
@ -0,0 +1,63 @@
+From fe090d706a522392e30dd4c44447f915ec99c1a0 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Sat, 22 Dec 2018 15:16:53 +0800
+Subject: [PATCH 01/27] vendor: grpc: fix grpc map panic
+
+reason: Fix grpc map panic
+
+cherry-pick from containerd-0.2.8
+
+a8cdda827867cec97568318368a7aa40097d0487
+
+Fix grpc map panic
+
+Description:
+ In golang, if we read/write map in different goroutine, it may panic.
+ We need to add lock to protect the map data when read/write the map.
+
+Now the grpc map is only protected by a mutex while register, not
+protected in reading process(handleStream function).
+
+This MR will use a RWMutex to protect this map.
+
+Change-Id: I786bd99234461c40fcb57621fd7c1fb4faa0c208
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ vendor/google.golang.org/grpc/server.go | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/vendor/google.golang.org/grpc/server.go b/vendor/google.golang.org/grpc/server.go
+index 4969331..77f7840 100644
+--- a/vendor/google.golang.org/grpc/server.go
+++ b/vendor/google.golang.org/grpc/server.go
+@@ -90,7 +90,7 @@ type service struct {
+ type Server struct {
+ 	opts options
+ 
+-	mu     sync.Mutex // guards following
+	mu     sync.RWMutex // guards following
+ 	lis    map[net.Listener]bool
+ 	conns  map[io.Closer]bool
+ 	serve  bool
+@@ -438,6 +438,8 @@ type ServiceInfo struct {
+ // Service names include the package names, in the form of <package>.<service>.
+ func (s *Server) GetServiceInfo() map[string]ServiceInfo {
+ 	ret := make(map[string]ServiceInfo)
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+ 	for n, srv := range s.m {
+ 		methods := make([]MethodInfo, 0, len(srv.md)+len(srv.sd))
+ 		for m := range srv.md {
+@@ -1221,7 +1223,9 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str
+ 	}
+ 	service := sm[:pos]
+ 	method := sm[pos+1:]
+	s.mu.RLock()
+ 	srv, ok := s.m[service]
+	s.mu.RUnlock()
+ 	if !ok {
+ 		if unknownDesc := s.opts.unknownStreamDesc; unknownDesc != nil {
+ 			s.processStreamingRPC(t, stream, nil, unknownDesc, trInfo)
+-- 
+2.7.4.3
+
--- a/patch/0002-sys-sys-count-steal-time-when-calculating-Sys.patch
+++ b/patch/0002-sys-sys-count-steal-time-when-calculating-Sys.patch
@ -0,0 +1,44 @@
+From 003dc7956765712fdf4a893c2d541af2e2d0f300 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Sat, 22 Dec 2018 15:44:50 +0800
+Subject: [PATCH 02/27] sys:  sys: count steal time when calculating
+ SystemCPUUsage
+
+reason: count steal time when calculating SystemCPUUsage
+
+cherry-pick from containerd-0.2.8
+
+13f22eecd33d30520ace277822ac5f0acb387e75
+
+containerd: count steal time when calculating SystemCPUUsage
+
+[Changelog]: when counting docker stat in virtual machines, now containerd do not count steal time when calculating SystemCPUUsage, which causes that cpuusage value larger than its actua$
+[Author]git
+
+Change-Id: I2b62c9508cbdc444d514116f4bea5aad3d292af5
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ sys/proc.go | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/sys/proc.go b/sys/proc.go
+index 496eb1f..82a6351 100644
+--- a/sys/proc.go
+++ b/sys/proc.go
+@@ -61,11 +61,11 @@ func GetSystemCPUUsage() (uint64, error) {
+ 		parts := strings.Fields(line)
+ 		switch parts[0] {
+ 		case "cpu":
+-			if len(parts) < 8 {
+			if len(parts) < 9 {
+ 				return 0, fmt.Errorf("bad format of cpu stats")
+ 			}
+ 			var totalClockTicks uint64
+-			for _, i := range parts[1:8] {
+			for _, i := range parts[1:9] {
+ 				v, err := strconv.ParseUint(i, 10, 64)
+ 				if err != nil {
+ 					return 0, fmt.Errorf("error parsing cpu stats")
+-- 
+2.7.4.3
+
--- a/patch/0003-oci-oci-add-files-cgroups-support.patch
+++ b/patch/0003-oci-oci-add-files-cgroups-support.patch
@ -0,0 +1,51 @@
+From c9cc468949d80c663524f5b764e2c661af13bca2 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Sat, 22 Dec 2018 16:25:07 +0800
+Subject: [PATCH 03/27] oci:  oci: add files cgroups support
+
+reason: Add file fds limit
+
+cherry-pick from containerd-0.2.8
+
+29b822599b86f823d5a9f94df1cdceea485e0b19
+
+Add file fds limit
+
+With the patch(https://lwn.net/Articles/604129/),we can limit the
+num of open files in container.
+
+Change-Id: I72b45430dd7535727c4af9e190bbb345ba8ee316
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ vendor/github.com/opencontainers/runtime-spec/specs-go/config.go | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+
+diff --git a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go
+index f32698c..ac24cde 100644
+--- a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go
+++ b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go
+@@ -314,6 +314,12 @@ type LinuxPids struct {
+ 	Limit int64 `json:"limit"`
+ }
+ 
+// Files for Linux cgroup 'files' resource management (https://lwn.net/Articles/604129/)
+type Files struct {
+	// Maximum number of open files".
+	Limit *int64 `json:"limit,omitempty"`
+}
+
+ // LinuxNetwork identification and priority configuration
+ type LinuxNetwork struct {
+ 	// Set class identifier for container's network packets
+@@ -340,6 +346,8 @@ type LinuxResources struct {
+ 	CPU *LinuxCPU `json:"cpu,omitempty"`
+ 	// Task resource restriction configuration.
+ 	Pids *LinuxPids `json:"pids,omitempty"`
+	// Files resource restriction configuration.
+	Files *Files `json:"files,omitempty"`
+ 	// BlockIO restriction configuration
+ 	BlockIO *LinuxBlockIO `json:"blockIO,omitempty"`
+ 	// Hugetlb limit (in bytes)
+-- 
+2.7.4.3
+
--- a/patch/0004-runv-vendor-runv-compatibility.patch
+++ b/patch/0004-runv-vendor-runv-compatibility.patch
@ -0,0 +1,63 @@
+From 5fa863a6ea74ed24cfcc0c16eaa5e5a4e77387ec Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Wed, 26 Dec 2018 12:08:20 +0800
+Subject: [PATCH 04/27] runv:  vendor: runv compatibility
+
+reason: fix manslaughter of runtime delete process
+
+cherry-pick from containerd-0.2.8
+
+reference:
+
+7906753998667b5a9fa9a996f4a0e41d4736d5c1
+
+contaierd-17: fix manslaughter of runtime delete process
+
+fix manslaughter of runtime delete process
+
+f82956a89ca7d7cea3bdd5fcd4d4fd70c313f378
+
+containerd-17: fix qemu remaining when dockerd restart
+
+fix qemu remaining when dockerd restart and container start concurrency
+
+Change-Id: Id23456e90961041194c946a289ae790327b874c8
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ vendor/github.com/containerd/go-runc/command_linux.go | 10 +++++++++-
+ 1 file changed, 9 insertions(+), 1 deletion(-)
+
+diff --git a/vendor/github.com/containerd/go-runc/command_linux.go b/vendor/github.com/containerd/go-runc/command_linux.go
+index 71b52f9..6ad27be 100644
+--- a/vendor/github.com/containerd/go-runc/command_linux.go
+++ b/vendor/github.com/containerd/go-runc/command_linux.go
+@@ -20,9 +20,17 @@ import (
+ 	"context"
+ 	"os"
+ 	"os/exec"
+	"strings"
+ 	"syscall"
+ )
+ 
+func (r *Runc) isrunv() bool {
+	if strings.Contains(r.Command, "runv") {
+		return true
+	}
+	return false
+}
+
+ func (r *Runc) command(context context.Context, args ...string) *exec.Cmd {
+ 	command := r.Command
+ 	if command == "" {
+@@ -33,7 +41,7 @@ func (r *Runc) command(context context.Context, args ...string) *exec.Cmd {
+ 		Setpgid: r.Setpgid,
+ 	}
+ 	cmd.Env = os.Environ()
+-	if r.PdeathSignal != 0 {
+	if r.PdeathSignal != 0 && !r.isrunv() {
+ 		cmd.SysProcAttr.Pdeathsig = r.PdeathSignal
+ 	}
+ 
+-- 
+2.7.4.3
+
--- a/patch/0005-containerd-add-spec-for-build.patch
+++ b/patch/0005-containerd-add-spec-for-build.patch
@ -0,0 +1,69 @@
+From 8e46f370733951e6decec6dd36b0c13308ced2c2 Mon Sep 17 00:00:00 2001
+From: caihaomin <caihaomin@huawei.com>
+Date: Mon, 21 Jan 2019 22:31:05 +0800
+Subject: [PATCH 05/27] containerd: add spec for build
+
+reason:add spec for build
+
+Change-Id: I42d9d32e4898c006194df1ead4735155b4785584
+Signed-off-by: caihaomin <caihaomin@huawei.com>
+---
+ hack/containerd.spec | 46 ++++++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 46 insertions(+)
+ create mode 100644 hack/containerd.spec
+
+diff --git a/hack/containerd.spec b/hack/containerd.spec
+new file mode 100644
+index 0000000..f53c37b
+--- /dev/null
+++ b/hack/containerd.spec
+@@ -0,0 +1,46 @@
+%global goipath github.com/containerd/containerd
+%global debug_package %{nil}
+Version:        1.2.0
+
+Name:           containerd
+Release:        1%{?dist}
+Summary:        An industry-standard container runtime
+License:        ASL 2.0
+URL:            https://containerd.io
+Source0:        containerd-1.2.0.tar.gz
+
+BuildRequires:  golang glibc-static make
+BuildRequires:  btrfs-progs-devel
+
+
+%description
+containerd is an industry-standard container runtime with an emphasis on
+simplicity, robustness and portability.  It is available as a daemon for Linux
+and Windows, which can manage the complete container lifecycle of its host
+system: image transfer and storage, container execution and supervision,
+low-level storage and network attachments, etc.
+
+
+%prep
+%setup -c -n containerd
+
+%build
+GO_BUILD_PATH=$PWD/_build
+install -m 0755 -vd $(dirname $GO_BUILD_PATH/src/%{goipath})
+ln -fs $PWD $GO_BUILD_PATH/src/%{goipath}
+cd $GO_BUILD_PATH/src/%{goipath}
+export GOPATH=$GO_BUILD_PATH:%{gopath}
+export BUILDTAGS="no_btrfs no_cri"
+make
+
+%install
+install -d $RPM_BUILD_ROOT/%{_bindir}
+install -p -m 755 bin/containerd $RPM_BUILD_ROOT/%{_bindir}/containerd
+install -p -m 755 bin/containerd-shim $RPM_BUILD_ROOT/%{_bindir}/containerd-shim
+
+%files
+%{_bindir}/containerd
+%{_bindir}/containerd-shim
+
+
+%changelog
+-- 
+2.7.4.3
+
--- a/patch/0006-shim-optimize-shim-lock-in-runtime-v1.patch
+++ b/patch/0006-shim-optimize-shim-lock-in-runtime-v1.patch
@ -0,0 +1,320 @@
+From 31621148229d56835575189c71e80339fba9f1fc Mon Sep 17 00:00:00 2001
+From: lujingxiao <lujingxiao@huawei.com>
+Date: Wed, 23 Jan 2019 14:55:27 +0800
+Subject: [PATCH 06/27] shim:  optimize shim lock in runtime v1
+
+reason: apply lock only around process map of shim service,
+avoid lock affect other procs operations.
+
+Cherry-pick from upstream c206da795
+
+Change-Id: I33f0f6b3537673533fdb60afb7a0295ac9665f11
+Signed-off-by: Ace-Tang <aceapril@126.com>
+Signed-off-by: lujingxiao <lujingxiao@huawei.com>
+---
+ runtime/v1/shim/service.go | 144 +++++++++++++++++++++++----------------------
+ 1 file changed, 75 insertions(+), 69 deletions(-)
+
+diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go
+index d76d580..679982a 100644
+--- a/runtime/v1/shim/service.go
+++ b/runtime/v1/shim/service.go
+@@ -114,9 +114,6 @@ type Service struct {
+ 
+ // Create a new initial process and container with the underlying OCI runtime
+ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *shimapi.CreateTaskResponse, err error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-
+ 	var mounts []proc.Mount
+ 	for _, m := range r.Rootfs {
+ 		mounts = append(mounts, proc.Mount{
+@@ -158,6 +155,10 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
+ 			return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m)
+ 		}
+ 	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+ 	process, err := newInit(
+ 		ctx,
+ 		s.config.Path,
+@@ -187,11 +188,9 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
+ 
+ // Start a process
+ func (s *Service) Start(ctx context.Context, r *shimapi.StartRequest) (*shimapi.StartResponse, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-	p := s.processes[r.ID]
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process %s", r.ID)
+	p, err := s.getExecProcess(r.ID)
+	if err != nil {
+		return nil, err
+ 	}
+ 	if err := p.Start(ctx); err != nil {
+ 		return nil, err
+@@ -204,16 +203,16 @@ func (s *Service) Start(ctx context.Context, r *shimapi.StartRequest) (*shimapi.
+ 
+ // Delete the initial process and container
+ func (s *Service) Delete(ctx context.Context, r *ptypes.Empty) (*shimapi.DeleteResponse, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-	p := s.processes[s.id]
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	p, err := s.getInitProcess()
+	if err != nil {
+		return nil, err
+ 	}
+ 	if err := p.Delete(ctx); err != nil {
+ 		return nil, err
+ 	}
+	s.mu.Lock()
+ 	delete(s.processes, s.id)
+	s.mu.Unlock()
+ 	s.platform.Close()
+ 	return &shimapi.DeleteResponse{
+ 		ExitStatus: uint32(p.ExitStatus()),
+@@ -227,11 +226,9 @@ func (s *Service) DeleteProcess(ctx context.Context, r *shimapi.DeleteProcessReq
+ 	if r.ID == s.id {
+ 		return nil, status.Errorf(codes.InvalidArgument, "cannot delete init process with DeleteProcess")
+ 	}
+-	s.mu.Lock()
+-	p := s.processes[r.ID]
+-	s.mu.Unlock()
+-	if p == nil {
+-		return nil, errors.Wrapf(errdefs.ErrNotFound, "process %s", r.ID)
+	p, err := s.getExecProcess(r.ID)
+	if err != nil {
+		return nil, err
+ 	}
+ 	if err := p.Delete(ctx); err != nil {
+ 		return nil, err
+@@ -249,13 +246,14 @@ func (s *Service) DeleteProcess(ctx context.Context, r *shimapi.DeleteProcessReq
+ // Exec an additional process inside the container
+ func (s *Service) Exec(ctx context.Context, r *shimapi.ExecProcessRequest) (*ptypes.Empty, error) {
+ 	s.mu.Lock()
+-	defer s.mu.Unlock()
+ 
+ 	if p := s.processes[r.ID]; p != nil {
+		s.mu.Unlock()
+ 		return nil, errdefs.ToGRPCf(errdefs.ErrAlreadyExists, "id %s", r.ID)
+ 	}
+ 
+ 	p := s.processes[s.id]
+	s.mu.Unlock()
+ 	if p == nil {
+ 		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+ 	}
+@@ -271,14 +269,14 @@ func (s *Service) Exec(ctx context.Context, r *shimapi.ExecProcessRequest) (*pty
+ 	if err != nil {
+ 		return nil, errdefs.ToGRPC(err)
+ 	}
+	s.mu.Lock()
+ 	s.processes[r.ID] = process
+	s.mu.Unlock()
+ 	return empty, nil
+ }
+ 
+ // ResizePty of a process
+ func (s *Service) ResizePty(ctx context.Context, r *shimapi.ResizePtyRequest) (*ptypes.Empty, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+ 	if r.ID == "" {
+ 		return nil, errdefs.ToGRPCf(errdefs.ErrInvalidArgument, "id not provided")
+ 	}
+@@ -286,7 +284,9 @@ func (s *Service) ResizePty(ctx context.Context, r *shimapi.ResizePtyRequest) (*
+ 		Width:  uint16(r.Width),
+ 		Height: uint16(r.Height),
+ 	}
+	s.mu.Lock()
+ 	p := s.processes[r.ID]
+	s.mu.Unlock()
+ 	if p == nil {
+ 		return nil, errors.Errorf("process does not exist %s", r.ID)
+ 	}
+@@ -298,11 +298,9 @@ func (s *Service) ResizePty(ctx context.Context, r *shimapi.ResizePtyRequest) (*
+ 
+ // State returns runtime state information for a process
+ func (s *Service) State(ctx context.Context, r *shimapi.StateRequest) (*shimapi.StateResponse, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-	p := s.processes[r.ID]
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process id %s", r.ID)
+	p, err := s.getExecProcess(r.ID)
+	if err != nil {
+		return nil, err
+ 	}
+ 	st, err := p.Status(ctx)
+ 	if err != nil {
+@@ -338,11 +336,9 @@ func (s *Service) State(ctx context.Context, r *shimapi.StateRequest) (*shimapi.
+ 
+ // Pause the container
+ func (s *Service) Pause(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-	p := s.processes[s.id]
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	p, err := s.getInitProcess()
+	if err != nil {
+		return nil, err
+ 	}
+ 	if err := p.(*proc.Init).Pause(ctx); err != nil {
+ 		return nil, err
+@@ -352,11 +348,9 @@ func (s *Service) Pause(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, er
+ 
+ // Resume the container
+ func (s *Service) Resume(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-	p := s.processes[s.id]
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	p, err := s.getInitProcess()
+	if err != nil {
+		return nil, err
+ 	}
+ 	if err := p.(*proc.Init).Resume(ctx); err != nil {
+ 		return nil, err
+@@ -366,12 +360,10 @@ func (s *Service) Resume(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, e
+ 
+ // Kill a process with the provided signal
+ func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Empty, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+ 	if r.ID == "" {
+-		p := s.processes[s.id]
+-		if p == nil {
+-			return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+		p, err := s.getInitProcess()
+		if err != nil {
+			return nil, err
+ 		}
+ 		if err := p.Kill(ctx, r.Signal, r.All); err != nil {
+ 			return nil, errdefs.ToGRPC(err)
+@@ -379,9 +371,9 @@ func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Emp
+ 		return empty, nil
+ 	}
+ 
+-	p := s.processes[r.ID]
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process id %s not found", r.ID)
+	p, err := s.getExecProcess(r.ID)
+	if err != nil {
+		return nil, err
+ 	}
+ 	if err := p.Kill(ctx, r.Signal, r.All); err != nil {
+ 		return nil, errdefs.ToGRPC(err)
+@@ -422,11 +414,9 @@ func (s *Service) ListPids(ctx context.Context, r *shimapi.ListPidsRequest) (*sh
+ 
+ // CloseIO of a process
+ func (s *Service) CloseIO(ctx context.Context, r *shimapi.CloseIORequest) (*ptypes.Empty, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-	p := s.processes[r.ID]
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process does not exist %s", r.ID)
+	p, err := s.getExecProcess(r.ID)
+	if err != nil {
+		return nil, err
+ 	}
+ 	if stdin := p.Stdin(); stdin != nil {
+ 		if err := stdin.Close(); err != nil {
+@@ -438,11 +428,9 @@ func (s *Service) CloseIO(ctx context.Context, r *shimapi.CloseIORequest) (*ptyp
+ 
+ // Checkpoint the container
+ func (s *Service) Checkpoint(ctx context.Context, r *shimapi.CheckpointTaskRequest) (*ptypes.Empty, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-	p := s.processes[s.id]
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	p, err := s.getInitProcess()
+	if err != nil {
+		return nil, err
+ 	}
+ 	var options runctypes.CheckpointOptions
+ 	if r.Options != nil {
+@@ -475,11 +463,9 @@ func (s *Service) ShimInfo(ctx context.Context, r *ptypes.Empty) (*shimapi.ShimI
+ 
+ // Update a running container
+ func (s *Service) Update(ctx context.Context, r *shimapi.UpdateTaskRequest) (*ptypes.Empty, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-	p := s.processes[s.id]
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	p, err := s.getInitProcess()
+	if err != nil {
+		return nil, err
+ 	}
+ 	if err := p.(*proc.Init).Update(ctx, r.Resources); err != nil {
+ 		return nil, errdefs.ToGRPC(err)
+@@ -489,11 +475,9 @@ func (s *Service) Update(ctx context.Context, r *shimapi.UpdateTaskRequest) (*pt
+ 
+ // Wait for a process to exit
+ func (s *Service) Wait(ctx context.Context, r *shimapi.WaitRequest) (*shimapi.WaitResponse, error) {
+-	s.mu.Lock()
+-	p := s.processes[r.ID]
+-	s.mu.Unlock()
+-	if p == nil {
+-		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	p, err := s.getExecProcess(r.ID)
+	if err != nil {
+		return nil, err
+ 	}
+ 	p.Wait()
+ 
+@@ -563,11 +547,9 @@ func shouldKillAllOnExit(bundlePath string) (bool, error) {
+ }
+ 
+ func (s *Service) getContainerPids(ctx context.Context, id string) ([]uint32, error) {
+-	s.mu.Lock()
+-	defer s.mu.Unlock()
+-	p := s.processes[s.id]
+-	if p == nil {
+-		return nil, errors.Wrapf(errdefs.ErrFailedPrecondition, "container must be created")
+	p, err := s.getInitProcess()
+	if err != nil {
+		return nil, err
+ 	}
+ 
+ 	ps, err := p.(*proc.Init).Runtime().Ps(ctx, id)
+@@ -589,6 +571,30 @@ func (s *Service) forward(publisher events.Publisher) {
+ 	}
+ }
+ 
+// getInitProcess returns initial process
+func (s *Service) getInitProcess() (rproc.Process, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	p := s.processes[s.id]
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	return p, nil
+}
+
+// getExecProcess returns exec process
+func (s *Service) getExecProcess(id string) (rproc.Process, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	p := s.processes[id]
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process %s does not exist", id)
+	}
+	return p, nil
+}
+
+ func getTopic(ctx context.Context, e interface{}) string {
+ 	switch e.(type) {
+ 	case *eventstypes.TaskCreate:
+-- 
+2.7.4.3
+
--- a/patch/0007-shim-Increase-reaper-buffer-size-and-non-bl.patch
+++ b/patch/0007-shim-Increase-reaper-buffer-size-and-non-bl.patch
@ -0,0 +1,109 @@
+From 2e143a25ff02800afb569352c407cf71a9c0312b Mon Sep 17 00:00:00 2001
+From: lujingxiao <lujingxiao@huawei.com>
+Date: Wed, 23 Jan 2019 14:56:19 +0800
+Subject: [PATCH 07/27] shim:  Increase reaper buffer size and
+ non-blocking send
+
+reason: Fixes #2709
+
+This increases the buffer size for process exit subscribers. It also
+implements a non-blocking send on the subscriber channel.  It is better
+to drop an exit even than it is to block a shim for one slow subscriber.
+
+Cherry-pick from upstream 232a063496
+
+Change-Id: Ibf9f06cc82945a8592fb02a87816d69d5dac2b6b
+Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
+Signed-off-by: lujingxiao <lujingxiao@huawei.com>
+---
+ runtime/v1/shim/reaper.go      | 14 +++++++++++---
+ runtime/v2/shim/reaper_unix.go | 14 +++++++++++---
+ 2 files changed, 22 insertions(+), 6 deletions(-)
+
+diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go
+index 2937f1a..10d5c30 100644
+--- a/runtime/v1/shim/reaper.go
+++ b/runtime/v1/shim/reaper.go
+@@ -26,12 +26,13 @@ import (
+ 	"github.com/containerd/containerd/sys"
+ 	runc "github.com/containerd/go-runc"
+ 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+ )
+ 
+ // ErrNoSuchProcess is returned when the process no longer exists
+ var ErrNoSuchProcess = errors.New("no such process")
+ 
+-const bufferSize = 32
+const bufferSize = 2048
+ 
+ // Reap should be called when the process receives an SIGCHLD.  Reap will reap
+ // all exited processes and close their wait channels
+@@ -41,13 +42,20 @@ func Reap() error {
+ 	Default.Lock()
+ 	for c := range Default.subscribers {
+ 		for _, e := range exits {
+-			c <- runc.Exit{
+			select {
+			case c <- runc.Exit{
+ 				Timestamp: now,
+ 				Pid:       e.Pid,
+ 				Status:    e.Status,
+			}:
+			default:
+				logrus.WithFields(logrus.Fields{
+					"subscriber": c,
+					"pid":        e.Pid,
+					"status":     e.Status,
+				}).Warn("failed to send exit to subscriber")
+ 			}
+ 		}
+-
+ 	}
+ 	Default.Unlock()
+ 	return err
+diff --git a/runtime/v2/shim/reaper_unix.go b/runtime/v2/shim/reaper_unix.go
+index 2937f1a..10d5c30 100644
+--- a/runtime/v2/shim/reaper_unix.go
+++ b/runtime/v2/shim/reaper_unix.go
+@@ -26,12 +26,13 @@ import (
+ 	"github.com/containerd/containerd/sys"
+ 	runc "github.com/containerd/go-runc"
+ 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+ )
+ 
+ // ErrNoSuchProcess is returned when the process no longer exists
+ var ErrNoSuchProcess = errors.New("no such process")
+ 
+-const bufferSize = 32
+const bufferSize = 2048
+ 
+ // Reap should be called when the process receives an SIGCHLD.  Reap will reap
+ // all exited processes and close their wait channels
+@@ -41,13 +42,20 @@ func Reap() error {
+ 	Default.Lock()
+ 	for c := range Default.subscribers {
+ 		for _, e := range exits {
+-			c <- runc.Exit{
+			select {
+			case c <- runc.Exit{
+ 				Timestamp: now,
+ 				Pid:       e.Pid,
+ 				Status:    e.Status,
+			}:
+			default:
+				logrus.WithFields(logrus.Fields{
+					"subscriber": c,
+					"pid":        e.Pid,
+					"status":     e.Status,
+				}).Warn("failed to send exit to subscriber")
+ 			}
+ 		}
+-
+ 	}
+ 	Default.Unlock()
+ 	return err
+-- 
+2.7.4.3
+
--- a/patch/0008-runtime-Use-named-pipes-for-shim-logs.patch
+++ b/patch/0008-runtime-Use-named-pipes-for-shim-logs.patch
@ -0,0 +1,578 @@
+From 9bdd5d485c6796c44356ae9482df8de467463feb Mon Sep 17 00:00:00 2001
+From: lujingxiao <lujingxiao@huawei.com>
+Date: Wed, 23 Jan 2019 14:57:41 +0800
+Subject: [PATCH 08/27] runtime:  Use named pipes for shim logs
+
+reason: TestDaemonRestart hangs if shim_debug is enabled
+Relating to issue [#2606](https://github.com/containerd/containerd/issues/2606)
+
+Co-authored-by: Oliver Stenbom <ostenbom@pivotal.io>
+Co-authored-by: Georgi Sabev <georgethebeatle@gmail.com>
+Co-authored-by: Giuseppe Capizzi <gcapizzi@pivotal.io>
+Co-authored-by: Danail Branekov <danailster@gmail.com>
+
+Cherry-pick from upstream 1d4105cacf
+
+Change-Id: I0038401dda88c234750e8d1378a4dd97230400b0
+Signed-off-by: Oliver Stenbom <ostenbom@pivotal.io>
+Signed-off-by: Georgi Sabev <georgethebeatle@gmail.com>
+Signed-off-by: Giuseppe Capizzi <gcapizzi@pivotal.io>
+Signed-off-by: Danail Branekov <danailster@gmail.com>
+Signed-off-by: lujingxiao <lujingxiao@huawei.com>
+---
+ client_test.go                   |  49 +++++++--
+ cmd/containerd-shim/main_unix.go |  28 ++++++
+ container_linux_test.go          | 209 +++++++++++++++++++++++++++++++++++++++
+ runtime/v1/linux/runtime.go      |  26 +++++
+ runtime/v1/shim.go               |  38 +++++++
+ runtime/v1/shim/client/client.go |  34 +++++--
+ 6 files changed, 370 insertions(+), 14 deletions(-)
+ create mode 100644 runtime/v1/shim.go
+
+diff --git a/client_test.go b/client_test.go
+index a6b1d59..1a4cf39 100644
+--- a/client_test.go
+++ b/client_test.go
+@@ -21,6 +21,8 @@ import (
+ 	"context"
+ 	"flag"
+ 	"fmt"
+	"io"
+	"io/ioutil"
+ 	"os"
+ 	"os/exec"
+ 	"testing"
+@@ -36,11 +38,12 @@ import (
+ )
+ 
+ var (
+-	address       string
+-	noDaemon      bool
+-	noCriu        bool
+-	supportsCriu  bool
+-	testNamespace = "testing"
+	address           string
+	noDaemon          bool
+	noCriu            bool
+	supportsCriu      bool
+	testNamespace     = "testing"
+	ctrdStdioFilePath string
+ 
+ 	ctrd = &daemon{}
+ )
+@@ -76,13 +79,26 @@ func TestMain(m *testing.M) {
+ 	if !noDaemon {
+ 		sys.ForceRemoveAll(defaultRoot)
+ 
+-		err := ctrd.start("containerd", address, []string{
+		stdioFile, err := ioutil.TempFile("", "")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "could not create a new stdio temp file: %s\n", err)
+			os.Exit(1)
+		}
+		defer func() {
+			stdioFile.Close()
+			os.Remove(stdioFile.Name())
+		}()
+		ctrdStdioFilePath = stdioFile.Name()
+		stdioWriter := io.MultiWriter(stdioFile, buf)
+
+		err = ctrd.start("containerd", address, []string{
+ 			"--root", defaultRoot,
+ 			"--state", defaultState,
+ 			"--log-level", "debug",
+-		}, buf, buf)
+			"--config", createShimDebugConfig(),
+		}, stdioWriter, stdioWriter)
+ 		if err != nil {
+-			fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String())
+			fmt.Fprintf(os.Stderr, "%s: %s\n", err, buf.String())
+ 			os.Exit(1)
+ 		}
+ 	}
+@@ -137,6 +153,7 @@ func TestMain(m *testing.M) {
+ 				fmt.Fprintln(os.Stderr, "failed to wait for containerd", err)
+ 			}
+ 		}
+
+ 		if err := sys.ForceRemoveAll(defaultRoot); err != nil {
+ 			fmt.Fprintln(os.Stderr, "failed to remove test root dir", err)
+ 			os.Exit(1)
+@@ -343,3 +360,19 @@ func TestClientReconnect(t *testing.T) {
+ 		t.Errorf("client closed returned error %v", err)
+ 	}
+ }
+
+func createShimDebugConfig() string {
+	f, err := ioutil.TempFile("", "containerd-config-")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to create config file: %s\n", err)
+		os.Exit(1)
+	}
+	defer f.Close()
+
+	if _, err := f.WriteString("[plugins.linux]\n\tshim_debug = true\n"); err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to write to config file %s: %s\n", f.Name(), err)
+		os.Exit(1)
+	}
+
+	return f.Name()
+}
+diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go
+index ca0a90a..6c59cd1 100644
+--- a/cmd/containerd-shim/main_unix.go
+++ b/cmd/containerd-shim/main_unix.go
+@@ -23,6 +23,7 @@ import (
+ 	"context"
+ 	"flag"
+ 	"fmt"
+	"io"
+ 	"net"
+ 	"os"
+ 	"os/exec"
+@@ -36,6 +37,7 @@ import (
+ 
+ 	"github.com/containerd/containerd/events"
+ 	"github.com/containerd/containerd/namespaces"
+	shimlog "github.com/containerd/containerd/runtime/v1"
+ 	"github.com/containerd/containerd/runtime/v1/linux/proc"
+ 	"github.com/containerd/containerd/runtime/v1/shim"
+ 	shimapi "github.com/containerd/containerd/runtime/v1/shim/v1"
+@@ -92,12 +94,38 @@ func main() {
+ 		runtime.GOMAXPROCS(2)
+ 	}
+ 
+	stdout, stderr, err := openStdioKeepAlivePipes(workdirFlag)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "containerd-shim: %s\n", err)
+		os.Exit(1)
+	}
+	defer func() {
+		stdout.Close()
+		stderr.Close()
+	}()
+
+ 	if err := executeShim(); err != nil {
+ 		fmt.Fprintf(os.Stderr, "containerd-shim: %s\n", err)
+ 		os.Exit(1)
+ 	}
+ }
+ 
+// If containerd server process dies, we need the shim to keep stdout/err reader
+// FDs so that Linux does not SIGPIPE the shim process if it tries to use its end of
+// these pipes.
+func openStdioKeepAlivePipes(dir string) (io.ReadCloser, io.ReadCloser, error) {
+	background := context.Background()
+	keepStdoutAlive, err := shimlog.OpenShimStdoutLog(background, dir)
+	if err != nil {
+		return nil, nil, err
+	}
+	keepStderrAlive, err := shimlog.OpenShimStderrLog(background, dir)
+	if err != nil {
+		return nil, nil, err
+	}
+	return keepStdoutAlive, keepStderrAlive, nil
+}
+
+ func executeShim() error {
+ 	// start handling signals as soon as possible so that things are properly reaped
+ 	// or if runtime exits before we hit the handler
+diff --git a/container_linux_test.go b/container_linux_test.go
+index 60b0336..fa764d7 100644
+--- a/container_linux_test.go
+++ b/container_linux_test.go
+@@ -24,7 +24,9 @@ import (
+ 	"fmt"
+ 	"io"
+ 	"io/ioutil"
+	"os"
+ 	"os/exec"
+	"path/filepath"
+ 	"runtime"
+ 	"strings"
+ 	"sync"
+@@ -258,6 +260,213 @@ func TestDaemonRestart(t *testing.T) {
+ 	<-statusC
+ }
+ 
+func TestShimDoesNotLeakPipes(t *testing.T) {
+	containerdPid := ctrd.cmd.Process.Pid
+	initialPipes, err := numPipes(containerdPid)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client, err := newClient(t, address)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	var (
+		image       Image
+		ctx, cancel = testContext()
+		id          = t.Name()
+	)
+	defer cancel()
+
+	image, err = client.GetImage(ctx, testImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "30")))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	task, err := container.NewTask(ctx, empty())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	exitChannel, err := task.Wait(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := task.Start(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
+		t.Fatal(err)
+	}
+
+	<-exitChannel
+
+	if _, err := task.Delete(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := container.Delete(ctx, WithSnapshotCleanup); err != nil {
+		t.Fatal(err)
+	}
+
+	currentPipes, err := numPipes(containerdPid)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if initialPipes != currentPipes {
+		t.Errorf("Pipes have leaked after container has been deleted. Initially there were %d pipes, after container deletion there were %d pipes", initialPipes, currentPipes)
+	}
+}
+
+func numPipes(pid int) (int, error) {
+	cmd := exec.Command("sh", "-c", fmt.Sprintf("lsof -p %d | grep pipe", pid))
+
+	var stdout bytes.Buffer
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err != nil {
+		return 0, err
+	}
+	return strings.Count(stdout.String(), "\n"), nil
+}
+
+func TestDaemonReconnectsToShimIOPipesOnRestart(t *testing.T) {
+	client, err := newClient(t, address)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	var (
+		image       Image
+		ctx, cancel = testContext()
+		id          = t.Name()
+	)
+	defer cancel()
+
+	image, err = client.GetImage(ctx, testImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "30")))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer container.Delete(ctx, WithSnapshotCleanup)
+
+	task, err := container.NewTask(ctx, empty())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer task.Delete(ctx)
+
+	_, err = task.Wait(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := task.Start(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ctrd.Restart(nil); err != nil {
+		t.Fatal(err)
+	}
+
+	waitCtx, waitCancel := context.WithTimeout(ctx, 2*time.Second)
+	serving, err := client.IsServing(waitCtx)
+	waitCancel()
+	if !serving {
+		t.Fatalf("containerd did not start within 2s: %v", err)
+	}
+
+	// After we restared containerd we write some messages to the log pipes, simulating shim writing stuff there.
+	// Then we make sure that these messages are available on the containerd log thus proving that the server reconnected to the log pipes
+	runtimeVersion := getRuntimeVersion()
+	logDirPath := getLogDirPath(runtimeVersion, id)
+
+	switch runtimeVersion {
+	case "v1":
+		writeToFile(t, filepath.Join(logDirPath, "shim.stdout.log"), fmt.Sprintf("%s writing to stdout\n", id))
+		writeToFile(t, filepath.Join(logDirPath, "shim.stderr.log"), fmt.Sprintf("%s writing to stderr\n", id))
+	case "v2":
+		writeToFile(t, filepath.Join(logDirPath, "log"), fmt.Sprintf("%s writing to log\n", id))
+	}
+
+	statusC, err := task.Wait(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
+		t.Fatal(err)
+	}
+
+	<-statusC
+
+	stdioContents, err := ioutil.ReadFile(ctrdStdioFilePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	switch runtimeVersion {
+	case "v1":
+		if !strings.Contains(string(stdioContents), fmt.Sprintf("%s writing to stdout", id)) {
+			t.Fatal("containerd did not connect to the shim stdout pipe")
+		}
+		if !strings.Contains(string(stdioContents), fmt.Sprintf("%s writing to stderr", id)) {
+			t.Fatal("containerd did not connect to the shim stderr pipe")
+		}
+	case "v2":
+		if !strings.Contains(string(stdioContents), fmt.Sprintf("%s writing to log", id)) {
+			t.Fatal("containerd did not connect to the shim log pipe")
+		}
+	}
+}
+
+func writeToFile(t *testing.T, filePath, message string) {
+	writer, err := os.OpenFile(filePath, os.O_WRONLY, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := writer.WriteString(message); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func getLogDirPath(runtimeVersion, id string) string {
+	switch runtimeVersion {
+	case "v1":
+		return filepath.Join(defaultRoot, "io.containerd.runtime.v1.linux", testNamespace, id)
+	case "v2":
+		return filepath.Join(defaultState, "io.containerd.runtime.v2.task", testNamespace, id)
+	default:
+		panic(fmt.Errorf("Unsupported runtime version %s", runtimeVersion))
+	}
+}
+
+func getRuntimeVersion() string {
+	switch rt := os.Getenv("TEST_RUNTIME"); rt {
+	case "io.containerd.runc.v1":
+		return "v2"
+	default:
+		return "v1"
+	}
+}
+
+ func TestContainerPTY(t *testing.T) {
+ 	t.Parallel()
+ 
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index d19b8e5..e1b3cac 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -21,6 +21,7 @@ package linux
+ import (
+ 	"context"
+ 	"fmt"
+	"io"
+ 	"io/ioutil"
+ 	"os"
+ 	"path/filepath"
+@@ -40,6 +41,7 @@ import (
+ 	"github.com/containerd/containerd/plugin"
+ 	"github.com/containerd/containerd/runtime"
+ 	"github.com/containerd/containerd/runtime/linux/runctypes"
+	"github.com/containerd/containerd/runtime/v1"
+ 	"github.com/containerd/containerd/runtime/v1/linux/proc"
+ 	shim "github.com/containerd/containerd/runtime/v1/shim/v1"
+ 	runc "github.com/containerd/go-runc"
+@@ -341,6 +343,30 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			continue
+ 		}
+ 
+		logDirPath := filepath.Join(r.root, ns, id)
+
+		shimStdoutLog, err := v1.OpenShimStdoutLog(ctx, logDirPath)
+		if err != nil {
+			log.G(ctx).WithError(err).WithFields(logrus.Fields{
+				"id":         id,
+				"namespace":  ns,
+				"logDirPath": logDirPath,
+			}).Error("opening shim stdout log pipe")
+			continue
+		}
+		go io.Copy(os.Stdout, shimStdoutLog)
+
+		shimStderrLog, err := v1.OpenShimStderrLog(ctx, logDirPath)
+		if err != nil {
+			log.G(ctx).WithError(err).WithFields(logrus.Fields{
+				"id":         id,
+				"namespace":  ns,
+				"logDirPath": logDirPath,
+			}).Error("opening shim stderr log pipe")
+			continue
+		}
+		go io.Copy(os.Stderr, shimStderrLog)
+
+ 		t, err := newTask(id, ns, pid, s, r.events, r.tasks, bundle)
+ 		if err != nil {
+ 			log.G(ctx).WithError(err).Error("loading task type")
+diff --git a/runtime/v1/shim.go b/runtime/v1/shim.go
+new file mode 100644
+index 0000000..3942968
+--- /dev/null
+++ b/runtime/v1/shim.go
+@@ -0,0 +1,38 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package v1
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+
+	"github.com/containerd/fifo"
+	"golang.org/x/sys/unix"
+)
+
+// OpenShimStdoutLog opens the shim log for reading
+func OpenShimStdoutLog(ctx context.Context, logDirPath string) (io.ReadWriteCloser, error) {
+	return fifo.OpenFifo(ctx, filepath.Join(logDirPath, "shim.stdout.log"), unix.O_RDWR|unix.O_CREAT|unix.O_NONBLOCK, 0700)
+}
+
+// OpenShimStderrLog opens the shim log
+func OpenShimStderrLog(ctx context.Context, logDirPath string) (io.ReadWriteCloser, error) {
+	return fifo.OpenFifo(ctx, filepath.Join(logDirPath, "shim.stderr.log"), unix.O_RDWR|unix.O_CREAT|unix.O_NONBLOCK, 0700)
+}
+diff --git a/runtime/v1/shim/client/client.go b/runtime/v1/shim/client/client.go
+index 015d88c..ef74030 100644
+--- a/runtime/v1/shim/client/client.go
+++ b/runtime/v1/shim/client/client.go
+@@ -37,6 +37,7 @@ import (
+ 
+ 	"github.com/containerd/containerd/events"
+ 	"github.com/containerd/containerd/log"
+	v1 "github.com/containerd/containerd/runtime/v1"
+ 	"github.com/containerd/containerd/runtime/v1/shim"
+ 	shimapi "github.com/containerd/containerd/runtime/v1/shim/v1"
+ 	"github.com/containerd/containerd/sys"
+@@ -62,7 +63,24 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa
+ 		}
+ 		defer f.Close()
+ 
+-		cmd, err := newCommand(binary, daemonAddress, debug, config, f)
+		var stdoutLog io.ReadWriteCloser
+		var stderrLog io.ReadWriteCloser
+		if debug {
+			stdoutLog, err = v1.OpenShimStdoutLog(ctx, config.WorkDir)
+			if err != nil {
+				return nil, nil, errors.Wrapf(err, "failed to create stdout log")
+			}
+
+			stderrLog, err = v1.OpenShimStderrLog(ctx, config.WorkDir)
+			if err != nil {
+				return nil, nil, errors.Wrapf(err, "failed to create stderr log")
+			}
+
+			go io.Copy(os.Stdout, stdoutLog)
+			go io.Copy(os.Stderr, stderrLog)
+		}
+
+		cmd, err := newCommand(binary, daemonAddress, debug, config, f, stdoutLog, stderrLog)
+ 		if err != nil {
+ 			return nil, nil, err
+ 		}
+@@ -77,6 +95,12 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa
+ 		go func() {
+ 			cmd.Wait()
+ 			exitHandler()
+			if stdoutLog != nil {
+				stderrLog.Close()
+			}
+			if stdoutLog != nil {
+				stderrLog.Close()
+			}
+ 		}()
+ 		log.G(ctx).WithFields(logrus.Fields{
+ 			"pid":     cmd.Process.Pid,
+@@ -104,7 +128,7 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa
+ 	}
+ }
+ 
+-func newCommand(binary, daemonAddress string, debug bool, config shim.Config, socket *os.File) (*exec.Cmd, error) {
+func newCommand(binary, daemonAddress string, debug bool, config shim.Config, socket *os.File, stdout, stderr io.Writer) (*exec.Cmd, error) {
+ 	selfExe, err := os.Executable()
+ 	if err != nil {
+ 		return nil, err
+@@ -137,10 +161,8 @@ func newCommand(binary, daemonAddress string, debug bool, config shim.Config, so
+ 	cmd.SysProcAttr = getSysProcAttr()
+ 	cmd.ExtraFiles = append(cmd.ExtraFiles, socket)
+ 	cmd.Env = append(os.Environ(), "GOMAXPROCS=2")
+-	if debug {
+-		cmd.Stdout = os.Stdout
+-		cmd.Stderr = os.Stderr
+-	}
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+ 	return cmd, nil
+ }
+ 
+-- 
+2.7.4.3
+
--- a/patch/0009-runtime-fix-pipe-in-broken-may-cause-shim-l.patch
+++ b/patch/0009-runtime-fix-pipe-in-broken-may-cause-shim-l.patch
@ -0,0 +1,38 @@
+From 77b025a48d9dc89666ef7c03709ef1fc2a4d0b34 Mon Sep 17 00:00:00 2001
+From: lujingxiao <lujingxiao@huawei.com>
+Date: Wed, 23 Jan 2019 15:00:12 +0800
+Subject: [PATCH 09/27] runtime:  fix pipe in broken may cause shim
+ lock forever for runtime v2
+
+reason: fix pipe in broken may cause shim lock forever for runtime v2
+
+Cherry-pick from upstream b3438f7a6f
+
+Change-Id: I3c324050531a1e68a5c3a688a51408a121a3f9f1
+Signed-off-by: Lifubang <lifubang@acmcoder.com>
+Signed-off-by: lujingxiao <lujingxiao@huawei.com>
+---
+ runtime/v2/runc/service_linux.go | 7 ++++---
+ 1 file changed, 4 insertions(+), 3 deletions(-)
+
+diff --git a/runtime/v2/runc/service_linux.go b/runtime/v2/runc/service_linux.go
+index 5e30cfc..19d1fec 100644
+--- a/runtime/v2/runc/service_linux.go
+++ b/runtime/v2/runc/service_linux.go
+@@ -49,9 +49,10 @@ func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console
+ 		cwg.Add(1)
+ 		go func() {
+ 			cwg.Done()
+-			p := bufPool.Get().(*[]byte)
+-			defer bufPool.Put(p)
+-			io.CopyBuffer(epollConsole, in, *p)
+			bp := bufPool.Get().(*[]byte)
+			defer bufPool.Put(bp)
+			io.CopyBuffer(epollConsole, in, *bp)
+			epollConsole.Shutdown(p.epoller.CloseConsole)
+ 		}()
+ 	}
+ 
+-- 
+2.7.4.3
+
--- a/patch/0010-runtime-fix-pipe-in-broken-may-cause-shim-l.patch
+++ b/patch/0010-runtime-fix-pipe-in-broken-may-cause-shim-l.patch
@ -0,0 +1,52 @@
+From d0e57aafce7c98b3c9b3004c862d5a15180df86c Mon Sep 17 00:00:00 2001
+From: lujingxiao <lujingxiao@huawei.com>
+Date: Wed, 23 Jan 2019 15:03:08 +0800
+Subject: [PATCH 10/27] runtime:  fix pipe in broken may cause shim
+ lock forever for runtime v1
+
+reason: fix pipe in broken may cause shim lock forever for runtime v1
+
+Cherry-pick from upstream e76a8879eb
+
+Change-Id: Ie603b36f92c4a6cc41777a9cd1e6a19b8584eaf1
+Signed-off-by: Lifubang <lifubang@acmcoder.com>
+Signed-off-by: lujingxiao <lujingxiao@huawei.com>
+---
+ runtime/v1/shim/service_linux.go | 8 +++++---
+ runtime/v2/runc/service_linux.go | 1 +
+ 2 files changed, 6 insertions(+), 3 deletions(-)
+
+diff --git a/runtime/v1/shim/service_linux.go b/runtime/v1/shim/service_linux.go
+index 18ae650..307e20d 100644
+--- a/runtime/v1/shim/service_linux.go
+++ b/runtime/v1/shim/service_linux.go
+@@ -49,9 +49,11 @@ func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console
+ 		cwg.Add(1)
+ 		go func() {
+ 			cwg.Done()
+-			p := bufPool.Get().(*[]byte)
+-			defer bufPool.Put(p)
+-			io.CopyBuffer(epollConsole, in, *p)
+			bp := bufPool.Get().(*[]byte)
+			defer bufPool.Put(bp)
+			io.CopyBuffer(epollConsole, in, *bp)
+			// we need to shutdown epollConsole when pipe broken
+			epollConsole.Shutdown(p.epoller.CloseConsole)
+ 		}()
+ 	}
+ 
+diff --git a/runtime/v2/runc/service_linux.go b/runtime/v2/runc/service_linux.go
+index 19d1fec..1161673 100644
+--- a/runtime/v2/runc/service_linux.go
+++ b/runtime/v2/runc/service_linux.go
+@@ -52,6 +52,7 @@ func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console
+ 			bp := bufPool.Get().(*[]byte)
+ 			defer bufPool.Put(bp)
+ 			io.CopyBuffer(epollConsole, in, *bp)
+			// we need to shutdown epollConsole when pipe broken
+ 			epollConsole.Shutdown(p.epoller.CloseConsole)
+ 		}()
+ 	}
+-- 
+2.7.4.3
+
--- a/patch/0011-runtime-Add-timeout-and-cancel-to-shim-fifo.patch
+++ b/patch/0011-runtime-Add-timeout-and-cancel-to-shim-fifo.patch
@ -0,0 +1,95 @@
+From 8eb1ab31006f3079d1bf95b4ab089e049a4f45f2 Mon Sep 17 00:00:00 2001
+From: lujingxiao <lujingxiao@huawei.com>
+Date: Wed, 23 Jan 2019 15:04:03 +0800
+Subject: [PATCH 11/27] runtime:  Add timeout and cancel to shim fifo
+ open
+
+reason: Add timeout and cancel to shim fifo open
+There is still a special case where the client side fails to open or
+load causes things to be slow and the shim can lock up when this
+happens.  This adds a timeout to the context for this case to abort fifo
+creation.
+
+Cherry-pick from upstream 18f57e20b0
+
+Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
+(cherry picked from commit a2a4241979f615eb0a1084c7638c21f830f48ac5)
+Signed-off-by: Andrew Hsu <andrewhsu@docker.com>
+Signed-off-by: lujingxiao <lujingxiao@huawei.com>
+
+Change-Id: Ic7f285b149f97f4d6526b3f2c28b6ac6790332b0
+---
+ runtime/v1/linux/proc/exec.go | 5 +++++
+ runtime/v1/linux/proc/init.go | 5 +++++
+ 2 files changed, 10 insertions(+)
+
+diff --git a/runtime/v1/linux/proc/exec.go b/runtime/v1/linux/proc/exec.go
+index 96c425d..715a977 100644
+--- a/runtime/v1/linux/proc/exec.go
+++ b/runtime/v1/linux/proc/exec.go
+@@ -172,22 +172,27 @@ func (e *execProcess) start(ctx context.Context) (err error) {
+ 		e.stdin = sc
+ 	}
+ 	var copyWaitGroup sync.WaitGroup
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ 	if socket != nil {
+ 		console, err := socket.ReceiveMaster()
+ 		if err != nil {
+			cancel()
+ 			return errors.Wrap(err, "failed to retrieve console master")
+ 		}
+ 		if e.console, err = e.parent.Platform.CopyConsole(ctx, console, e.stdio.Stdin, e.stdio.Stdout, e.stdio.Stderr, &e.wg, &copyWaitGroup); err != nil {
+			cancel()
+ 			return errors.Wrap(err, "failed to start console copy")
+ 		}
+ 	} else if !e.stdio.IsNull() {
+ 		if err := copyPipes(ctx, e.io, e.stdio.Stdin, e.stdio.Stdout, e.stdio.Stderr, &e.wg, &copyWaitGroup); err != nil {
+			cancel()
+ 			return errors.Wrap(err, "failed to start io pipe copy")
+ 		}
+ 	}
+ 	copyWaitGroup.Wait()
+ 	pid, err := runc.ReadPidFile(opts.PidFile)
+ 	if err != nil {
+		cancel()
+ 		return errors.Wrap(err, "failed to retrieve OCI runtime exec pid")
+ 	}
+ 	e.pid = pid
+diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go
+index 5bf5f83..5b23671 100644
+--- a/runtime/v1/linux/proc/init.go
+++ b/runtime/v1/linux/proc/init.go
+@@ -168,18 +168,22 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error {
+ 		p.closers = append(p.closers, sc)
+ 	}
+ 	var copyWaitGroup sync.WaitGroup
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ 	if socket != nil {
+ 		console, err := socket.ReceiveMaster()
+ 		if err != nil {
+			cancel()
+ 			return errors.Wrap(err, "failed to retrieve console master")
+ 		}
+ 		console, err = p.Platform.CopyConsole(ctx, console, r.Stdin, r.Stdout, r.Stderr, &p.wg, &copyWaitGroup)
+ 		if err != nil {
+			cancel()
+ 			return errors.Wrap(err, "failed to start console copy")
+ 		}
+ 		p.console = console
+ 	} else if !hasNoIO(r) {
+ 		if err := copyPipes(ctx, p.io, r.Stdin, r.Stdout, r.Stderr, &p.wg, &copyWaitGroup); err != nil {
+			cancel()
+ 			return errors.Wrap(err, "failed to start io pipe copy")
+ 		}
+ 	}
+@@ -187,6 +191,7 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error {
+ 	copyWaitGroup.Wait()
+ 	pid, err := runc.ReadPidFile(pidFile)
+ 	if err != nil {
+		cancel()
+ 		return errors.Wrap(err, "failed to retrieve OCI runtime container pid")
+ 	}
+ 	p.pid = pid
+-- 
+2.7.4.3
+
--- a/patch/0012-bump-bump-containerd-to-1.2.0.2.patch
+++ b/patch/0012-bump-bump-containerd-to-1.2.0.2.patch
@ -0,0 +1,36 @@
+From ea92cca7c1d4dfbd6a563588a6ea9b56a764fc39 Mon Sep 17 00:00:00 2001
+From: lujingxiao <lujingxiao@huawei.com>
+Date: Wed, 23 Jan 2019 15:31:56 +0800
+Subject: [PATCH 12/27] bump: bump containerd to 1.2.0.2
+
+reason: bump containerd to 1.2.0.2 after cherry-picked patches from
+upstream:
+- runtime:  Add timeout and cancel to shim fifo open
+- runtime:  fix pipe in broken may cause shim lock forever for runtime v1
+- runtime:  fix pipe in broken may cause shim lock forever for runtime v2
+- runtime:  Use named pipes for shim logs
+- shim:  Increase reaper buffer size and non-blocking send
+- shim:  optimize shim lock in runtime v1
+
+Change-Id: Ibd7574e2ab18a2f783c694931101e1459bc779ad
+Signed-off-by: lujingxiao <lujingxiao@huawei.com>
+---
+ hack/containerd.spec | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/hack/containerd.spec b/hack/containerd.spec
+index f53c37b..c7d358d 100644
+--- a/hack/containerd.spec
+++ b/hack/containerd.spec
+@@ -3,7 +3,7 @@
+ Version:        1.2.0
+ 
+ Name:           containerd
+-Release:        1%{?dist}
+Release:        2%{?dist}
+ Summary:        An industry-standard container runtime
+ License:        ASL 2.0
+ URL:            https://containerd.io
+-- 
+2.7.4.3
+
--- a/patch/0013-log-support-log-init-pid-to-start-event-log.patch
+++ b/patch/0013-log-support-log-init-pid-to-start-event-log.patch
@ -0,0 +1,50 @@
+From d4d3f8a239f4b4afd009d954453e585704ddb112 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Thu, 24 Jan 2019 11:55:10 +0800
+Subject: [PATCH 13/27] log:  support log init pid to start event log
+
+reason: DFX support start event with init pid
+
+Change-Id: I8ae9c7a9652f694680979965829682416aed4055
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ hack/containerd.spec     | 2 +-
+ runtime/v1/linux/task.go | 2 ++
+ 2 files changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/hack/containerd.spec b/hack/containerd.spec
+index c7d358d..462d35e 100644
+--- a/hack/containerd.spec
+++ b/hack/containerd.spec
+@@ -3,7 +3,7 @@
+ Version:        1.2.0
+ 
+ Name:           containerd
+-Release:        2%{?dist}
+Release:        3%{?dist}
+ Summary:        An industry-standard container runtime
+ License:        ASL 2.0
+ URL:            https://containerd.io
+diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go
+index 38da35c..1c650c4 100644
+--- a/runtime/v1/linux/task.go
+++ b/runtime/v1/linux/task.go
+@@ -36,6 +36,7 @@ import (
+ 	"github.com/containerd/typeurl"
+ 	"github.com/gogo/protobuf/types"
+ 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+ )
+ 
+ // Task on a linux based system
+@@ -131,6 +132,7 @@ func (t *Task) Start(ctx context.Context) error {
+ 		t.cg = cg
+ 		t.mu.Unlock()
+ 	}
+	logrus.Infof("publish event %s for container %s with pid %d", runtime.TaskStartEventTopic, t.id, t.pid)
+ 	t.events.Publish(ctx, runtime.TaskStartEventTopic, &eventstypes.TaskStart{
+ 		ContainerID: t.id,
+ 		Pid:         uint32(t.pid),
+-- 
+2.7.4.3
+
--- a/patch/0014-event-resend-exit-event-when-detect-container.patch
+++ b/patch/0014-event-resend-exit-event-when-detect-container.patch
@ -0,0 +1,84 @@
+From 200ae6f4b733f8a869aac36a730da90e79213387 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Sun, 10 Feb 2019 18:40:59 +0800
+Subject: [PATCH 14/27] event:  resend exit event when detect
+ containerd restarted
+
+reason: testCE_docker_containerd_ABN.026.sh
+fix docker stop no effect.
+
+Change-Id: I024b2f6a03d74fcbb5623c696212dcbfb624b285
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ cmd/containerd-shim/main_unix.go | 38 +++++++++++++++++++++++++++++++++++++-
+ 1 file changed, 37 insertions(+), 1 deletion(-)
+
+diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go
+index 6c59cd1..d1f41b0 100644
+--- a/cmd/containerd-shim/main_unix.go
+++ b/cmd/containerd-shim/main_unix.go
+@@ -24,12 +24,14 @@ import (
+ 	"flag"
+ 	"fmt"
+ 	"io"
+	"io/ioutil"
+ 	"net"
+ 	"os"
+ 	"os/exec"
+ 	"os/signal"
+ 	"runtime"
+ 	"runtime/debug"
+	"strconv"
+ 	"strings"
+ 	"sync"
+ 	"syscall"
+@@ -263,7 +265,7 @@ type remoteEventsPublisher struct {
+ 	address string
+ }
+ 
+-func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event events.Event) error {
+func (l *remoteEventsPublisher) doPublish(ctx context.Context, topic string, event events.Event) error {
+ 	ns, _ := namespaces.Namespace(ctx)
+ 	encoded, err := typeurl.MarshalAny(event)
+ 	if err != nil {
+@@ -288,3 +290,37 @@ func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event
+ 	}
+ 	return nil
+ }
+
+func getContainerdPid() int {
+	pidFile := "/var/run/docker/containerd/containerd.pid"
+	data, err := ioutil.ReadFile(pidFile)
+	if err != nil {
+		return -1
+	}
+	pid, err := strconv.Atoi(string(data))
+	if err != nil {
+		return -1
+	}
+	return pid
+}
+
+func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event events.Event) error {
+	old := getContainerdPid()
+	for i := 1; i <= 10; i++ {
+		err := l.doPublish(ctx, topic, event)
+		logrus.Infof("try publish event(%d) %s %v %v", i, topic, event, err)
+		if err == nil {
+			new := getContainerdPid()
+			if old == new {
+				return nil
+			}
+			logrus.Warnf("containerd pid %d changed to %d", old, new)
+			old = new
+		}
+		if i == 10 {
+			return err
+		}
+		time.Sleep(time.Duration(i) * time.Second)
+	}
+	return nil
+}
+-- 
+2.7.4.3
+
--- a/patch/0015-restore-cleanup-container-pid-1.patch
+++ b/patch/0015-restore-cleanup-container-pid-1.patch
@ -0,0 +1,122 @@
+From fd1c8dda8cc02b9aef28f1e3e4e51ab216338e2b Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Sun, 10 Feb 2019 15:40:52 +0800
+Subject: [PATCH 15/27] restore:  cleanup container pid=-1
+
+reason: fix testCE_docker_hook_spec_ABN.050.sh
+when containerd killed during task create, see Runtime.Create(). the
+defer function will not execute, so shim residual. cleanup shim for
+container pid=-1
+
+Change-Id: Ie9a7f6dff5f8a922cc97c5fcf44664ab60ac1a7a
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ runtime/v1/linux/runtime.go | 10 +++++++---
+ runtime/v1/linux/task.go    | 26 ++++++++++++++++++++++++--
+ 2 files changed, 31 insertions(+), 5 deletions(-)
+
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index e1b3cac..3b66304 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -316,6 +316,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			continue
+ 		}
+ 		id := path.Name()
+		log.G(ctx).Infof("load-task %s", id)
+ 		bundle := loadBundle(
+ 			id,
+ 			filepath.Join(r.state, ns, id),
+@@ -372,6 +373,12 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			log.G(ctx).WithError(err).Error("loading task type")
+ 			continue
+ 		}
+		if pid == -1 {
+			_, err := t.DeleteForce(ctx)
+			log.G(ctx).Warnf("delete force %s Pid=-1 error=%v", id, err)
+			continue
+		}
+		log.G(ctx).Infof("load-task %s Pid=%d done", id, pid)
+ 		o = append(o, t)
+ 	}
+ 	return o, nil
+@@ -380,9 +387,6 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ func (r *Runtime) cleanupAfterDeadShim(ctx context.Context, bundle *bundle, ns, id string, pid int) error {
+ 	ctx = namespaces.WithNamespace(ctx, ns)
+ 	if err := r.terminate(ctx, bundle, ns, id); err != nil {
+-		if r.config.ShimDebug {
+-			return errors.Wrap(err, "failed to terminate task, leaving bundle for debugging")
+-		}
+ 		log.G(ctx).WithError(err).Warn("failed to terminate task")
+ 	}
+ 
+diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go
+index 1c650c4..6995156 100644
+--- a/runtime/v1/linux/task.go
+++ b/runtime/v1/linux/task.go
+@@ -21,6 +21,7 @@ package linux
+ import (
+ 	"context"
+ 	"sync"
+	"time"
+ 
+ 	"github.com/containerd/cgroups"
+ 	eventstypes "github.com/containerd/containerd/api/events"
+@@ -37,6 +38,7 @@ import (
+ 	"github.com/gogo/protobuf/types"
+ 	"github.com/pkg/errors"
+ 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+ )
+ 
+ // Task on a linux based system
+@@ -86,10 +88,13 @@ func (t *Task) Namespace() string {
+ }
+ 
+ // Delete the task and return the exit status
+-func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) {
+func (t *Task) delete(ctx context.Context, force bool) (*runtime.Exit, error) {
+ 	rsp, err := t.shim.Delete(ctx, empty)
+ 	if err != nil {
+-		return nil, errdefs.FromGRPC(err)
+		log.G(ctx).WithError(err).Error("failed to delete container, force=%t", force)
+		if !force {
+			return nil, errdefs.FromGRPC(err)
+		}
+ 	}
+ 	t.tasks.Delete(ctx, t.id)
+ 	if err := t.shim.KillShim(ctx); err != nil {
+@@ -98,6 +103,14 @@ func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) {
+ 	if err := t.bundle.Delete(); err != nil {
+ 		log.G(ctx).WithError(err).Error("failed to delete bundle")
+ 	}
+
+	if rsp == nil {
+		rsp = &shim.DeleteResponse{}
+		rsp.ExitStatus = 128 + uint32(unix.SIGKILL)
+		rsp.ExitedAt = time.Now().UTC()
+		rsp.Pid = 0
+	}
+
+ 	t.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{
+ 		ContainerID: t.id,
+ 		ExitStatus:  rsp.ExitStatus,
+@@ -111,6 +124,15 @@ func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) {
+ 	}, nil
+ }
+ 
+// Delete the task and return the exit status
+func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) {
+	return t.delete(ctx, false)
+}
+
+func (t *Task) DeleteForce(ctx context.Context) (*runtime.Exit, error) {
+	return t.delete(ctx, true)
+}
+
+ // Start the task
+ func (t *Task) Start(ctx context.Context) error {
+ 	t.mu.Lock()
+-- 
+2.7.4.3
+
--- a/patch/0016-create-runc-delete-force-before-create.patch
+++ b/patch/0016-create-runc-delete-force-before-create.patch
@ -0,0 +1,31 @@
+From e7827a737c42861afd6b41e2e7dc953c249278fc Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Mon, 11 Feb 2019 17:40:31 +0800
+Subject: [PATCH 16/27] create:  runc delete force before create
+
+reason: testCE_docker_hook_spec_ABN.051.sh
+kill -9 shim will generate residual runc files, cleanup runc files using
+runc delete before create.
+
+Change-Id: I3efa3c4d0989ba8d688bcb6f35ba543b6ab91b2d
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ vendor/github.com/containerd/go-runc/runc.go | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go
+index 96262af..e688881 100644
+--- a/vendor/github.com/containerd/go-runc/runc.go
+++ b/vendor/github.com/containerd/go-runc/runc.go
+@@ -138,6 +138,8 @@ func (o *CreateOpts) args() (out []string, err error) {
+ 
+ // Create creates a new container and returns its pid if it was created successfully
+ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error {
+	r.Delete(context, id, &DeleteOpts{Force: true})
+
+ 	args := []string{"create", "--bundle", bundle}
+ 	if opts != nil {
+ 		oargs, err := opts.args()
+-- 
+2.7.4.3
+
--- a/patch/0017-exit-using-init.exit-indicate-container-is-ex.patch
+++ b/patch/0017-exit-using-init.exit-indicate-container-is-ex.patch
@ -0,0 +1,65 @@
+From f83e391aef03283b30431a960b66f720cf0d9dd3 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Mon, 11 Feb 2019 20:12:15 +0800
+Subject: [PATCH 17/27] exit:  using init.exit indicate container is
+ exiting
+
+reason: testCE_docker_hook_spec_ABN.053.sh
+kill dockerd during docker stop in post-stophook, containerd will load
+task and treat as ok when shim response client. add init.exit to forbid
+load exiting task.
+
+Change-Id: I8f03cd51088d43d4fb457b32981f3eebd8558f84
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ runtime/v1/linux/proc/init.go | 1 +
+ runtime/v1/linux/runtime.go   | 5 +++++
+ runtime/v1/shim/service.go    | 4 +++-
+ 3 files changed, 9 insertions(+), 1 deletion(-)
+
+diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go
+index 5b23671..caa31c3 100644
+--- a/runtime/v1/linux/proc/init.go
+++ b/runtime/v1/linux/proc/init.go
+@@ -43,6 +43,7 @@ import (
+ 
+ // InitPidFile name of the file that contains the init pid
+ const InitPidFile = "init.pid"
+const InitExit = "init.exit"
+ 
+ // Init represents an initial process for a container
+ type Init struct {
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index 3b66304..123d675 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -378,6 +378,11 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			log.G(ctx).Warnf("delete force %s Pid=-1 error=%v", id, err)
+ 			continue
+ 		}
+		if _, err := os.Stat(filepath.Join(bundle.path, proc.InitExit)); err == nil {
+			_, err := t.DeleteForce(ctx)
+			log.G(ctx).Warnf("delete force %s Pid=%d(exiting) error=%v", id, pid, err)
+			continue
+		}
+ 		log.G(ctx).Infof("load-task %s Pid=%d done", id, pid)
+ 		o = append(o, t)
+ 	}
+diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go
+index 679982a..8c7984f 100644
+--- a/runtime/v1/shim/service.go
+++ b/runtime/v1/shim/service.go
+@@ -504,7 +504,9 @@ func (s *Service) checkProcesses(e runc.Exit) {
+ 
+ 	for _, p := range s.processes {
+ 		if p.Pid() == e.Pid {
+-
+			if ip, ok := p.(*proc.Init); ok {
+				ioutil.WriteFile(filepath.Join(ip.Bundle, proc.InitExit), []byte(fmt.Sprintf("%d", e.Pid)), 0600)
+			}
+ 			if shouldKillAll {
+ 				if ip, ok := p.(*proc.Init); ok {
+ 					// Ensure all children are killed
+-- 
+2.7.4.3
+
--- a/patch/0018-containerd-shim-Dump-log-to-file-when-docker-.patch
+++ b/patch/0018-containerd-shim-Dump-log-to-file-when-docker-.patch
@ -0,0 +1,42 @@
+From 7f483b7d5a6bd88ea35f5dcf1a5fea5d165044fe Mon Sep 17 00:00:00 2001
+From: lixiang172 <lixiang172@huawei.com>
+Date: Tue, 12 Feb 2019 15:22:06 +0800
+Subject: [PATCH 18/27] containerd-shim:  Dump log to file when docker
+ received signal
+
+reason: Dump stack log to file when docker received "kill -SIGUSR1
+PID" signal
+The name of log files is "shim-stack-[time].log".
+The log file can be found at:
+/run/docker/containerd/daemon/io.containerd.runtime.v1.linux/moby/container-id/shim-stack-[time].log
+
+Change-Id: I6d7e03c9a0fd36e9a76f1dd45cfd5312985d03f8
+Signed-off-by: lixiang172 <lixiang172@huawei.com>
+---
+ cmd/containerd-shim/main_unix.go | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go
+index d1f41b0..38b3eb4 100644
+--- a/cmd/containerd-shim/main_unix.go
+++ b/cmd/containerd-shim/main_unix.go
+@@ -246,6 +246,8 @@ func handleSignals(logger *logrus.Entry, signals chan os.Signal, server *ttrpc.S
+ 	}
+ }
+ 
+const stacksLogNameTemplate = "shim-stacks-%s.log"
+
+ func dumpStacks(logger *logrus.Entry) {
+ 	var (
+ 		buf       []byte
+@@ -258,6 +260,7 @@ func dumpStacks(logger *logrus.Entry) {
+ 		bufferLen *= 2
+ 	}
+ 	buf = buf[:stackSize]
+	ioutil.WriteFile(fmt.Sprintf(stacksLogNameTemplate, strings.Replace(time.Now().Format(time.RFC3339), ":", "", -1)), buf, 0600)
+ 	logger.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
+ }
+ 
+-- 
+2.7.4.3
+
--- a/patch/0019-restore-check-shim-alive-when-containerd-is-r.patch
+++ b/patch/0019-restore-check-shim-alive-when-containerd-is-r.patch
@ -0,0 +1,47 @@
+From 112c2ef89b1085e95959285ce5328af5d74ba8db Mon Sep 17 00:00:00 2001
+From: xueshaojia <xueshaojia@huawei.com>
+Date: Thu, 14 Feb 2019 10:48:14 +0800
+Subject: [PATCH 19/27] restore:  check shim alive when containerd is
+ restarted
+
+reason: fix docker_containerd-shim:testCE_docker_containerd_shim_ABN.021.sh
+    When containerd is restarted, it will load all tasks.In some cases, the
+    containerd-shim is killed and the sock file will exist for a while.
+    Containerd should check the containerd-shim is available using the sock file.
+    If the containerd-shim server not responses, do r.cleanupAfterDeadShim
+
+Change-Id: I448c8caefa8c1252bd5cdcff79deb8eff1005903
+Signed-off-by: xueshaojia <xueshaojia@huawei.com>
+---
+ runtime/v1/linux/runtime.go | 15 +++++++++++++++
+ 1 file changed, 15 insertions(+)
+
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index 123d675..477cda0 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -343,6 +343,21 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			}
+ 			continue
+ 		}
+		ctxContact, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+		alive, err := s.IsAlive(ctxContact)
+		if !alive {
+			log.G(ctx).WithError(err).WithFields(logrus.Fields{
+				"id":        id,
+				"namespace": ns,
+			}).Error("contacting to shim")
+			err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid)
+			if err != nil {
+				log.G(ctx).WithError(err).WithField("bundle", bundle.path).
+					Error("cleaning up after dead shim")
+			}
+			continue
+		}
+ 
+ 		logDirPath := filepath.Join(r.root, ns, id)
+ 
+-- 
+2.7.4.3
+
--- a/patch/0020-events-resend-pending-exit-events-on-restore.patch
+++ b/patch/0020-events-resend-pending-exit-events-on-restore.patch
@ -0,0 +1,357 @@
+From 27762e8d75c00c8898c725873c17a23105ba5b7c Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Tue, 12 Feb 2019 17:03:11 +0800
+Subject: [PATCH 20/27] events:  resend pending exit events on restore
+
+reason: fix exit event may lost.
+testCE_docker_containerd_ABN.026.sh
+
+Change-Id: I5bcdf06ad4ee7b8a0ca782e610186f52e3d79bbd
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ events/events.go                            | 13 +++++
+ events/exchange/exchange.go                 | 12 +++++
+ events/exit.go                              | 79 +++++++++++++++++++++++++++++
+ runtime/v1/linux/runtime.go                 | 56 +++++++++++++++++---
+ runtime/v1/linux/task.go                    | 10 ++--
+ runtime/v1/shim/service.go                  |  2 +
+ vendor/github.com/docker/go-events/queue.go |  8 +++
+ 7 files changed, 167 insertions(+), 13 deletions(-)
+ create mode 100644 events/exit.go
+
+diff --git a/events/events.go b/events/events.go
+index b7eb86f..aa07236 100644
+--- a/events/events.go
+++ b/events/events.go
+@@ -22,6 +22,7 @@ import (
+ 
+ 	"github.com/containerd/typeurl"
+ 	"github.com/gogo/protobuf/types"
+	apievents "github.com/containerd/containerd/api/events"
+ )
+ 
+ // Envelope provides the packaging for an event.
+@@ -32,6 +33,18 @@ type Envelope struct {
+ 	Event     *types.Any
+ }
+ 
+func (e *Envelope) ExitFile() string {
+	decoded, err := typeurl.UnmarshalAny(e.Event)
+	if err != nil {
+		return ""
+	}
+
+	if e, ok := decoded.(*apievents.TaskExit); ok {
+		return ExitFile(e.ContainerID, e.Pid, e.ExitStatus)
+	}
+
+	return ""
+}
+ // Field returns the value for the given fieldpath as a string, if defined.
+ // If the value is not defined, the second value will be false.
+ func (e *Envelope) Field(fieldpath []string) (string, bool) {
+diff --git a/events/exchange/exchange.go b/events/exchange/exchange.go
+index 95d21b7..540f180 100644
+--- a/events/exchange/exchange.go
+++ b/events/exchange/exchange.go
+@@ -49,6 +49,11 @@ func NewExchange() *Exchange {
+ var _ events.Publisher = &Exchange{}
+ var _ events.Forwarder = &Exchange{}
+ var _ events.Subscriber = &Exchange{}
+var mobySubcribed = false
+
+func MobySubscribed() bool {
+	return mobySubcribed
+}
+ 
+ // Forward accepts an envelope to be direcly distributed on the exchange.
+ //
+@@ -161,6 +166,13 @@ func (e *Exchange) Subscribe(ctx context.Context, fs ...string) (ch <-chan *even
+ 	}
+ 
+ 	e.broadcaster.Add(dst)
+	logrus.Infof("subscribe ctx=%v fs=%v", ctx, fs)
+	for _, s := range fs {
+		if !MobySubscribed() && s == "namespace==moby,topic~=|^/tasks/|" {
+			queue.Namespace = "moby"
+			mobySubcribed = true
+		}
+	}
+ 
+ 	go func() {
+ 		defer closeAll()
+diff --git a/events/exit.go b/events/exit.go
+new file mode 100644
+index 0000000..e1ce089
+--- /dev/null
+++ b/events/exit.go
+@@ -0,0 +1,79 @@
+package events
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"github.com/sirupsen/logrus"
+)
+
+const ExitDir = "/var/run/docker/containerd/exit"
+const ExitStatusDefault = 137
+
+func ExitFile(cid string, pid uint32, status uint32) string {
+	return fmt.Sprintf("%s.%d.%d", cid, pid, status)
+}
+
+func ExitInfo(ef string) (string, uint32, uint32) {
+	s := strings.Split(ef, ".")
+	if len(s) != 3 {
+		return "", 0, 0
+	}
+
+	cid := s[0]
+	pid, err := strconv.ParseUint(s[1], 10, 32)
+	if err != nil {
+		return "", 0, 0
+	}
+	status, err := strconv.ParseUint(s[2], 10, 32)
+	if err != nil {
+		return "", 0, 0
+	}
+
+	return cid, uint32(pid), uint32(status)
+}
+
+func ExitAddFile(ns string, ef string, reason string) {
+	os.MkdirAll(filepath.Join(ExitDir, ns), 0700)
+	err := ioutil.WriteFile(filepath.Join(ExitDir, ns, ef), []byte{}, 0600)
+	logrus.Infof("exit-add %s/%s [reason: %s] error=%v", ns, ef, reason, err)
+}
+
+func ExitDelFile(ns string, ef string) {
+	err := os.RemoveAll(filepath.Join(ExitDir, ns, ef))
+	logrus.Infof("exit-del %s/%s error=%v", ns, ef, err)
+}
+
+func ExitGetFile(ns string, cid string, pid uint32, status uint32) string {
+	ef := ExitFile(cid, pid, status)
+	if _, err := os.Stat(filepath.Join(ExitDir, ns, ef)); err == nil {
+		return ef
+	}
+	return ""
+}
+
+func ExitGetFiles(ns string) []string {
+	files, err := ioutil.ReadDir(filepath.Join(ExitDir, ns))
+	if err != nil {
+		return []string{}
+	}
+
+	names := []string{}
+	for _, f := range files {
+		names = append(names, f.Name())
+	}
+
+	return names
+}
+
+func ExitPending(ns string, cid string, pid uint32) bool {
+	for _, ef := range ExitGetFiles(ns) {
+		if strings.Contains(ef, fmt.Sprintf("%s.%d", cid, pid)) {
+			return true
+		}
+	}
+	return false
+}
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index 477cda0..add4d52 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -31,6 +31,7 @@ import (
+ 	"github.com/containerd/containerd/api/types"
+ 	"github.com/containerd/containerd/containers"
+ 	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/events"
+ 	"github.com/containerd/containerd/events/exchange"
+ 	"github.com/containerd/containerd/identifiers"
+ 	"github.com/containerd/containerd/log"
+@@ -129,6 +130,7 @@ func New(ic *plugin.InitContext) (interface{}, error) {
+ 			return nil, err
+ 		}
+ 	}
+	go r.resendExitEvents(ic.Context, "moby")
+ 	return r, nil
+ }
+ 
+@@ -175,7 +177,8 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
+ 	}
+ 	defer func() {
+ 		if err != nil {
+-			bundle.Delete()
+			errd := bundle.Delete()
+			log.G(ctx).WithError(err).Errorf("revert: delete bundle error=%v", errd)
+ 		}
+ 	}()
+ 
+@@ -218,9 +221,8 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
+ 	}
+ 	defer func() {
+ 		if err != nil {
+-			if kerr := s.KillShim(ctx); kerr != nil {
+-				log.G(ctx).WithError(err).Error("failed to kill shim")
+-			}
+			kerr := s.KillShim(ctx)
+			log.G(ctx).WithError(err).Errorf("revert: kill shim error=%v", kerr)
+ 		}
+ 	}()
+ 
+@@ -305,6 +307,41 @@ func (r *Runtime) Get(ctx context.Context, id string) (runtime.Task, error) {
+ 	return r.tasks.Get(ctx, id)
+ }
+ 
+func (r *Runtime) resendExitEvents(ctx context.Context, ns string) {
+	for {
+		time.Sleep(time.Second)
+		efs := events.ExitGetFiles(ns)
+		if len(efs) == 0 {
+			break
+		}
+
+		if !exchange.MobySubscribed() {
+			logrus.Infof("waiting moby event stream ...")
+			continue
+		}
+		time.Sleep(time.Second)
+
+		for _, ef := range efs {
+			cid, pid, status := events.ExitInfo(ef)
+			if cid == "" {
+				continue
+			}
+
+			e := &eventstypes.TaskExit{
+				ContainerID: cid,
+				ID:          cid,
+				ExitStatus:  status,
+				ExitedAt:    time.Now().UTC(),
+				Pid:         uint32(pid),
+			}
+
+			ctx := namespaces.WithNamespace(context.Background(), ns)
+			err := r.events.Publish(ctx, runtime.TaskExitEventTopic, e)
+			logrus.Infof("resend exit event %v error=%v", e, err)
+		}
+	}
+}
+
+ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 	dir, err := ioutil.ReadDir(filepath.Join(r.state, ns))
+ 	if err != nil {
+@@ -388,13 +425,16 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			log.G(ctx).WithError(err).Error("loading task type")
+ 			continue
+ 		}
+-		if pid == -1 {
+-			_, err := t.DeleteForce(ctx)
+-			log.G(ctx).Warnf("delete force %s Pid=-1 error=%v", id, err)
+		if pid <= 0 {
+			_, err := t.DeleteForce(ctx, 0)
+			log.G(ctx).Warnf("delete force %s Pid=%d error=%v", id, pid, err)
+ 			continue
+ 		}
+ 		if _, err := os.Stat(filepath.Join(bundle.path, proc.InitExit)); err == nil {
+-			_, err := t.DeleteForce(ctx)
+			if !events.ExitPending(ns, t.id, uint32(pid)) {
+				events.ExitAddFile(ns, events.ExitFile(t.id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task")
+			}
+			_, err := t.DeleteForce(ctx, uint32(pid))
+ 			log.G(ctx).Warnf("delete force %s Pid=%d(exiting) error=%v", id, pid, err)
+ 			continue
+ 		}
+diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go
+index 6995156..b692ae7 100644
+--- a/runtime/v1/linux/task.go
+++ b/runtime/v1/linux/task.go
+@@ -88,7 +88,7 @@ func (t *Task) Namespace() string {
+ }
+ 
+ // Delete the task and return the exit status
+-func (t *Task) delete(ctx context.Context, force bool) (*runtime.Exit, error) {
+func (t *Task) delete(ctx context.Context, force bool, pid uint32) (*runtime.Exit, error) {
+ 	rsp, err := t.shim.Delete(ctx, empty)
+ 	if err != nil {
+ 		log.G(ctx).WithError(err).Error("failed to delete container, force=%t", force)
+@@ -108,7 +108,7 @@ func (t *Task) delete(ctx context.Context, force bool) (*runtime.Exit, error) {
+ 		rsp = &shim.DeleteResponse{}
+ 		rsp.ExitStatus = 128 + uint32(unix.SIGKILL)
+ 		rsp.ExitedAt = time.Now().UTC()
+-		rsp.Pid = 0
+		rsp.Pid = pid
+ 	}
+ 
+ 	t.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{
+@@ -126,11 +126,11 @@ func (t *Task) delete(ctx context.Context, force bool) (*runtime.Exit, error) {
+ 
+ // Delete the task and return the exit status
+ func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) {
+-	return t.delete(ctx, false)
+	return t.delete(ctx, false, 0)
+ }
+ 
+-func (t *Task) DeleteForce(ctx context.Context) (*runtime.Exit, error) {
+-	return t.delete(ctx, true)
+func (t *Task) DeleteForce(ctx context.Context, pid uint32) (*runtime.Exit, error) {
+	return t.delete(ctx, true, pid)
+ }
+ 
+ // Start the task
+diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go
+index 8c7984f..a2eb35b 100644
+--- a/runtime/v1/shim/service.go
+++ b/runtime/v1/shim/service.go
+@@ -505,6 +505,8 @@ func (s *Service) checkProcesses(e runc.Exit) {
+ 	for _, p := range s.processes {
+ 		if p.Pid() == e.Pid {
+ 			if ip, ok := p.(*proc.Init); ok {
+				ns := filepath.Base(filepath.Dir(ip.Bundle))
+				events.ExitAddFile(ns, events.ExitFile(s.id, uint32(e.Pid), uint32(e.Status)), "init exited")
+ 				ioutil.WriteFile(filepath.Join(ip.Bundle, proc.InitExit), []byte(fmt.Sprintf("%d", e.Pid)), 0600)
+ 			}
+ 			if shouldKillAll {
+diff --git a/vendor/github.com/docker/go-events/queue.go b/vendor/github.com/docker/go-events/queue.go
+index 4bb770a..0608e7e 100644
+--- a/vendor/github.com/docker/go-events/queue.go
+++ b/vendor/github.com/docker/go-events/queue.go
+@@ -5,12 +5,14 @@ import (
+ 	"sync"
+ 
+ 	"github.com/sirupsen/logrus"
+	topevents "github.com/containerd/containerd/events"
+ )
+ 
+ // Queue accepts all messages into a queue for asynchronous consumption
+ // by a sink. It is unbounded and thread safe but the sink must be reliable or
+ // events will be dropped.
+ type Queue struct {
+	Namespace string
+ 	dst    Sink
+ 	events *list.List
+ 	cond   *sync.Cond
+@@ -83,6 +85,12 @@ func (eq *Queue) run() {
+ 				"event": event,
+ 				"sink":  eq.dst,
+ 			}).WithError(err).Debug("eventqueue: dropped event")
+		} else {
+			if e, ok := event.(*topevents.Envelope); ok {
+				if ef := e.ExitFile(); ef != "" {
+					topevents.ExitDelFile(eq.Namespace, ef)
+				}
+			}
+ 		}
+ 	}
+ }
+-- 
+2.7.4.3
+
--- a/patch/0021-containerd-Update-the-version-info-of-contain.patch
+++ b/patch/0021-containerd-Update-the-version-info-of-contain.patch
@ -0,0 +1,59 @@
+From 818ef5fe43d3b9b4c53301800d545ce4c775afff Mon Sep 17 00:00:00 2001
+From: lixiang172 <lixiang172@huawei.com>
+Date: Tue, 12 Feb 2019 11:37:37 +0800
+Subject: [PATCH 21/27] containerd:  Update the version info of
+ containerd
+
+reason: Update the version info after type "containerd -v"
+The version info now is defined by "containerd.spec" rather than
+"version.go"
+
+Change-Id: I04c6b78737e09f93a3e84a100c88be19294a5c4f
+Signed-off-by: lixiang172 <lixiang172@huawei.com>
+---
+ Makefile           | 8 ++++----
+ version/version.go | 2 +-
+ 2 files changed, 5 insertions(+), 5 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 35021fd..e38dfb3 100644
+--- a/Makefile
+++ b/Makefile
+@@ -20,8 +20,8 @@ ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+ DESTDIR=/usr/local
+ 
+ # Used to populate variables in version package.
+-VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always)
+-REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
+VERSION=$(shell echo version:)$(shell grep '^Version' ${ROOTDIR}/hack/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/').$(shell grep '^Release:' ${ROOTDIR}/hack/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+REVISION=$(shell echo commit:)$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
+ 
+ ifneq "$(strip $(shell command -v go 2>/dev/null))" ""
+ 	GOOS ?= $(shell go env GOOS)
+@@ -77,8 +77,8 @@ MANPAGES=ctr.1 containerd.1 containerd-config.1 containerd-config.toml.5
+ # Build tags seccomp and apparmor are needed by CRI plugin.
+ BUILDTAGS ?= seccomp apparmor
+ GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",)
+-GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PKG) $(EXTRA_LDFLAGS)'
+-SHIM_GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PKG) -extldflags "-static"'
+GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)'
+SHIM_GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"'
+ 
+ #Replaces ":" (*nix), ";" (windows) with newline for easy parsing
+ GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n")
+diff --git a/version/version.go b/version/version.go
+index b2874bf..04b7097 100644
+--- a/version/version.go
+++ b/version/version.go
+@@ -18,7 +18,7 @@ package version
+ 
+ var (
+ 	// Package is filled at linking time
+-	Package = "github.com/containerd/containerd"
+	Package = ""
+ 
+ 	// Version holds the complete version number. Filled in at linking time.
+ 	Version = "1.2.0+unknown"
+-- 
+2.7.4.3
+
--- a/patch/0022-containerd-bump-version-1.2.0.4.patch
+++ b/patch/0022-containerd-bump-version-1.2.0.4.patch
@ -0,0 +1,29 @@
+From bea413085725db89439817284b63bb4061e62753 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Wed, 13 Feb 2019 22:03:08 +0800
+Subject: [PATCH 22/27] containerd: bump version 1.2.0.4
+
+reason: bump version
+
+Change-Id: Iee2348e931a723929ccfe63b3539c812514acc90
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ hack/containerd.spec | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/hack/containerd.spec b/hack/containerd.spec
+index 462d35e..f8d9084 100644
+--- a/hack/containerd.spec
+++ b/hack/containerd.spec
+@@ -3,7 +3,7 @@
+ Version:        1.2.0
+ 
+ Name:           containerd
+-Release:        3%{?dist}
+Release:        4%{?dist}
+ Summary:        An industry-standard container runtime
+ License:        ASL 2.0
+ URL:            https://containerd.io
+-- 
+2.7.4.3
+
--- a/patch/0023-containerd-set-create-and-exec-timeout.patch
+++ b/patch/0023-containerd-set-create-and-exec-timeout.patch
@ -0,0 +1,218 @@
+From 006bc6d0a9e0c233d0d14de53de0b18799c67081 Mon Sep 17 00:00:00 2001
+From: xiadanni <xiadanni@huawei.com>
+Date: Fri, 15 Feb 2019 06:00:52 +0800
+Subject: [PATCH 23/27] containerd:  set create and exec timeout
+
+reason:set create and exec timeout to avild block when command failed
+
+Change-Id: I6bc55f4ccc953bdc1d926ab940f0900811d68760
+Signed-off-by: xiadanni <xiadanni@huawei.com>
+---
+ hack/containerd.spec                            |  2 +-
+ runtime/v1/shim/reaper.go                       | 50 +++++++++++++++++++++++++
+ runtime/v2/shim/reaper_unix.go                  |  4 ++
+ vendor/github.com/containerd/go-runc/monitor.go |  6 +++
+ vendor/github.com/containerd/go-runc/runc.go    | 31 +++++++++++++--
+ 5 files changed, 88 insertions(+), 5 deletions(-)
+
+diff --git a/hack/containerd.spec b/hack/containerd.spec
+index f8d9084..f39c57a 100644
+--- a/hack/containerd.spec
+++ b/hack/containerd.spec
+@@ -3,7 +3,7 @@
+ Version:        1.2.0
+ 
+ Name:           containerd
+-Release:        4%{?dist}
+Release:        5%{?dist}
+ Summary:        An industry-standard container runtime
+ License:        ASL 2.0
+ URL:            https://containerd.io
+diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go
+index 10d5c30..a2b90fe 100644
+--- a/runtime/v1/shim/reaper.go
+++ b/runtime/v1/shim/reaper.go
+@@ -19,8 +19,13 @@
+ package shim
+ 
+ import (
+	"io/ioutil"
+ 	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+ 	"sync"
+	"syscall"
+ 	"time"
+ 
+ 	"github.com/containerd/containerd/sys"
+@@ -100,6 +105,34 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) {
+ 	return -1, ErrNoSuchProcess
+ }
+ 
+// WaitTimeout is used to skip the blocked command and kill the left process.
+func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, error) {
+	sch := make(chan int)
+	ech := make(chan error)
+	go func() {
+		for e := range ec {
+			if e.Pid == c.Process.Pid {
+				// make sure we flush all IO
+				c.Wait()
+				m.Unsubscribe(ec)
+				sch <- e.Status
+				return
+			}
+		}
+	}()
+	select {
+	case <-time.After(time.Duration(sec) * time.Second):
+		if SameProcess(c, c.Process.Pid) {
+			syscall.Kill(c.Process.Pid, syscall.SIGKILL)
+		}
+		return 0, errors.Errorf("timeout %ds for cmd(pid= %d): %s, %s", sec, c.Process.Pid, c.Path, c.Args)
+	case status := <-sch:
+		return status, nil
+	case err := <-ech:
+		return -1, err
+	}
+}
+
+ // Subscribe to process exit changes
+ func (m *Monitor) Subscribe() chan runc.Exit {
+ 	c := make(chan runc.Exit, bufferSize)
+@@ -116,3 +149,20 @@ func (m *Monitor) Unsubscribe(c chan runc.Exit) {
+ 	close(c)
+ 	m.Unlock()
+ }
+
+func SameProcess(cmd *exec.Cmd, pid int) bool {
+	bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline"))
+	if err != nil {
+		return false
+	}
+	for i := range bytes {
+		if bytes[i] == 0 {
+			bytes[i] = 32
+		}
+	}
+	cmdline := string(bytes)
+	if strings.EqualFold(cmdline, strings.Join(cmd.Args, " ")+" ") {
+		return true
+	}
+	return false
+}
+diff --git a/runtime/v2/shim/reaper_unix.go b/runtime/v2/shim/reaper_unix.go
+index 10d5c30..8bd7dd1 100644
+--- a/runtime/v2/shim/reaper_unix.go
+++ b/runtime/v2/shim/reaper_unix.go
+@@ -100,6 +100,10 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) {
+ 	return -1, ErrNoSuchProcess
+ }
+ 
+func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, error) {
+	return m.Wait(c, ec)
+}
+
+ // Subscribe to process exit changes
+ func (m *Monitor) Subscribe() chan runc.Exit {
+ 	c := make(chan runc.Exit, bufferSize)
+diff --git a/vendor/github.com/containerd/go-runc/monitor.go b/vendor/github.com/containerd/go-runc/monitor.go
+index ff06a3f..2c184d2 100644
+--- a/vendor/github.com/containerd/go-runc/monitor.go
+++ b/vendor/github.com/containerd/go-runc/monitor.go
+@@ -40,6 +40,7 @@ type Exit struct {
+ type ProcessMonitor interface {
+ 	Start(*exec.Cmd) (chan Exit, error)
+ 	Wait(*exec.Cmd, chan Exit) (int, error)
+	WaitTimeout(*exec.Cmd, chan Exit, int64) (int, error)
+ }
+ 
+ type defaultMonitor struct {
+@@ -74,3 +75,8 @@ func (m *defaultMonitor) Wait(c *exec.Cmd, ec chan Exit) (int, error) {
+ 	e := <-ec
+ 	return e.Status, nil
+ }
+
+func (m *defaultMonitor) WaitTimeout(c *exec.Cmd, ec chan Exit, sec int64) (int, error) {
+	e := <-ec
+	return e.Status, nil
+}
+\ No newline at end of file
+diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go
+index e688881..fc64e8a 100644
+--- a/vendor/github.com/containerd/go-runc/runc.go
+++ b/vendor/github.com/containerd/go-runc/runc.go
+@@ -52,6 +52,8 @@ const (
+ 	Text Format = "text"
+ 	// DefaultCommand is the default command for Runc
+ 	DefaultCommand = "runc"
+	execTimeout    = 30
+	createTimeout  = 120
+ )
+ 
+ // Runc is the client to the runc cli
+@@ -155,7 +157,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp
+ 	cmd.ExtraFiles = opts.ExtraFiles
+ 
+ 	if cmd.Stdout == nil && cmd.Stderr == nil {
+-		data, err := cmdOutput(cmd, true)
+		data, err := cmdOutputTimeout(cmd, true, createTimeout)
+ 		if err != nil {
+ 			return fmt.Errorf("%s: %s", err, data)
+ 		}
+@@ -172,7 +174,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp
+ 			}
+ 		}
+ 	}
+-	status, err := Monitor.Wait(cmd, ec)
+	status, err := Monitor.WaitTimeout(cmd, ec, createTimeout)
+ 	if err == nil && status != 0 {
+ 		err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
+ 	}
+@@ -234,7 +236,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts
+ 		opts.Set(cmd)
+ 	}
+ 	if cmd.Stdout == nil && cmd.Stderr == nil {
+-		data, err := cmdOutput(cmd, true)
+		data, err := cmdOutputTimeout(cmd, true, execTimeout)
+ 		if err != nil {
+ 			return fmt.Errorf("%s: %s", err, data)
+ 		}
+@@ -251,7 +253,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts
+ 			}
+ 		}
+ 	}
+-	status, err := Monitor.Wait(cmd, ec)
+	status, err := Monitor.WaitTimeout(cmd, ec, execTimeout)
+ 	if err == nil && status != 0 {
+ 		err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
+ 	}
+@@ -707,3 +709,24 @@ func cmdOutput(cmd *exec.Cmd, combined bool) ([]byte, error) {
+ 
+ 	return b.Bytes(), err
+ }
+
+func cmdOutputTimeout(cmd *exec.Cmd, combined bool, timeout int64) ([]byte, error) {
+	b := getBuf()
+	defer putBuf(b)
+
+	cmd.Stdout = b
+	if combined {
+		cmd.Stderr = b
+	}
+	ec, err := Monitor.Start(cmd)
+	if err != nil {
+		return nil, err
+	}
+
+	status, err := Monitor.WaitTimeout(cmd, ec, timeout)
+	if err == nil && status != 0 {
+		err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
+	}
+
+	return b.Bytes(), err
+}
+-- 
+2.7.4.3
+
--- a/patch/0024-create-cleanup-runc-dirty-files-on-start.patch
+++ b/patch/0024-create-cleanup-runc-dirty-files-on-start.patch
@ -0,0 +1,54 @@
+From f96039fcd94c5bc75dcec297668418811d60e785 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Tue, 19 Feb 2019 11:53:41 +0800
+Subject: [PATCH 24/27] create:  cleanup runc dirty files on start
+
+reason: add check before cleanup runtime dirty files.
+
+Change-Id: I6f218fd8d19ed65d8b13ae1ea744b80574279f83
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ hack/containerd.spec                         | 2 +-
+ vendor/github.com/containerd/go-runc/runc.go | 6 +++++-
+ 2 files changed, 6 insertions(+), 2 deletions(-)
+
+diff --git a/hack/containerd.spec b/hack/containerd.spec
+index f39c57a..869012a 100644
+--- a/hack/containerd.spec
+++ b/hack/containerd.spec
+@@ -3,7 +3,7 @@
+ Version:        1.2.0
+ 
+ Name:           containerd
+-Release:        5%{?dist}
+Release:        6%{?dist}
+ Summary:        An industry-standard container runtime
+ License:        ASL 2.0
+ URL:            https://containerd.io
+diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go
+index fc64e8a..e66ea5b 100644
+--- a/vendor/github.com/containerd/go-runc/runc.go
+++ b/vendor/github.com/containerd/go-runc/runc.go
+@@ -30,6 +30,7 @@ import (
+ 	"strings"
+ 	"syscall"
+ 	"time"
+	"github.com/sirupsen/logrus"
+ 
+ 	specs "github.com/opencontainers/runtime-spec/specs-go"
+ )
+@@ -140,7 +141,10 @@ func (o *CreateOpts) args() (out []string, err error) {
+ 
+ // Create creates a new container and returns its pid if it was created successfully
+ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error {
+-	r.Delete(context, id, &DeleteOpts{Force: true})
+	if _, err := os.Stat(filepath.Join(r.Root, id)); err == nil {
+		logrus.Warnf("cleanup residue runtime with bundle %s root=%s", bundle, r.Root)
+		r.Delete(context, id, &DeleteOpts{Force: true})
+	}
+ 
+ 	args := []string{"create", "--bundle", bundle}
+ 	if opts != nil {
+-- 
+2.7.4.3
+
--- a/patch/0025-restore-skip-load-task-in-creating.patch
+++ b/patch/0025-restore-skip-load-task-in-creating.patch
@ -0,0 +1,74 @@
+From 869ceecb455640da5e90f7827f75275665e93e95 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Sat, 23 Feb 2019 15:51:24 +0800
+Subject: [PATCH 25/27] restore:  skip load task in creating
+
+load task in creating will stuck containerd restore process.
+
+Change-Id: I2f8b77a88d78597ef2be5122708fc8ab16fad956
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ runtime/v1/linux/runtime.go | 5 ++---
+ runtime/v1/shim/service.go  | 6 ++++++
+ 2 files changed, 8 insertions(+), 3 deletions(-)
+
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index add4d52..5647f94 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -353,7 +353,6 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			continue
+ 		}
+ 		id := path.Name()
+-		log.G(ctx).Infof("load-task %s", id)
+ 		bundle := loadBundle(
+ 			id,
+ 			filepath.Join(r.state, ns, id),
+@@ -361,6 +360,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 		)
+ 		ctx = namespaces.WithNamespace(ctx, ns)
+ 		pid, _ := runc.ReadPidFile(filepath.Join(bundle.path, proc.InitPidFile))
+		log.G(ctx).Infof("load-task %s/%s/%s Pid=%d", r.state, ns, id, pid)
+ 		s, err := bundle.NewShimClient(ctx, ns, ShimConnect(r.config, func() {
+ 			err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid)
+ 			if err != nil {
+@@ -426,8 +426,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			continue
+ 		}
+ 		if pid <= 0 {
+-			_, err := t.DeleteForce(ctx, 0)
+-			log.G(ctx).Warnf("delete force %s Pid=%d error=%v", id, pid, err)
+			log.G(ctx).Warnf("skip load task in creating %s", id)
+ 			continue
+ 		}
+ 		if _, err := os.Stat(filepath.Join(bundle.path, proc.InitExit)); err == nil {
+diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go
+index a2eb35b..d7fdcaf 100644
+--- a/runtime/v1/shim/service.go
+++ b/runtime/v1/shim/service.go
+@@ -26,6 +26,7 @@ import (
+ 	"os"
+ 	"path/filepath"
+ 	"sync"
+	"time"
+ 
+ 	"github.com/containerd/console"
+ 	eventstypes "github.com/containerd/containerd/api/events"
+@@ -140,9 +141,14 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
+ 	rootfs := filepath.Join(r.Bundle, "rootfs")
+ 	defer func() {
+ 		if err != nil {
+			logrus.Errorf("create init %s failed error=%v", r.ID, err)
+ 			if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
+ 				log.G(ctx).WithError(err2).Warn("Failed to cleanup rootfs mount")
+ 			}
+			go func() {
+				time.Sleep(10*time.Second)
+				os.Exit(0)
+			}()
+ 		}
+ 	}()
+ 	for _, rm := range mounts {
+-- 
+2.7.4.3
+
--- a/patch/0026-exit-optimize-init.exit-record.patch
+++ b/patch/0026-exit-optimize-init.exit-record.patch
@ -0,0 +1,96 @@
+From c26316153098e72a9b30668befc36fcfcba3b76f Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Sat, 23 Feb 2019 15:55:21 +0800
+Subject: [PATCH 26/27] exit:  optimize init.exit record
+
+Change-Id: If1319f7d87defed16d1113337957f36b7320e9b9
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ events/exit.go                | 21 +++++++++++++++++++++
+ runtime/v1/linux/proc/init.go |  1 -
+ runtime/v1/linux/runtime.go   |  2 +-
+ runtime/v1/shim/service.go    |  2 +-
+ 4 files changed, 23 insertions(+), 3 deletions(-)
+
+diff --git a/events/exit.go b/events/exit.go
+index e1ce089..772dc24 100644
+--- a/events/exit.go
+++ b/events/exit.go
+@@ -7,11 +7,13 @@ import (
+ 	"path/filepath"
+ 	"strconv"
+ 	"strings"
+
+ 	"github.com/sirupsen/logrus"
+ )
+ 
+ const ExitDir = "/var/run/docker/containerd/exit"
+ const ExitStatusDefault = 137
+const InitExit = "init.exit"
+ 
+ func ExitFile(cid string, pid uint32, status uint32) string {
+ 	return fmt.Sprintf("%s.%d.%d", cid, pid, status)
+@@ -77,3 +79,22 @@ func ExitPending(ns string, cid string, pid uint32) bool {
+ 	}
+ 	return false
+ }
+
+func InitExitWrite(bundle string, pid int) {
+	if _, err := os.Stat(bundle); err != nil {
+		logrus.Infof("skip write init.exit %s error=%v", bundle, err)
+		return
+	}
+	err := ioutil.WriteFile(filepath.Join(bundle, InitExit), []byte(fmt.Sprintf("%d", pid)), 0600)
+	if err != nil {
+		logrus.Infof("failed write init.exit error=%s", bundle, err)
+	}
+}
+
+func InitExitExist(bundle string) bool {
+	if _, err := os.Stat(filepath.Join(bundle, InitExit)); err == nil {
+		return true
+	}
+	return false
+}
+
+diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go
+index caa31c3..5b23671 100644
+--- a/runtime/v1/linux/proc/init.go
+++ b/runtime/v1/linux/proc/init.go
+@@ -43,7 +43,6 @@ import (
+ 
+ // InitPidFile name of the file that contains the init pid
+ const InitPidFile = "init.pid"
+-const InitExit = "init.exit"
+ 
+ // Init represents an initial process for a container
+ type Init struct {
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index 5647f94..e92904e 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -429,7 +429,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			log.G(ctx).Warnf("skip load task in creating %s", id)
+ 			continue
+ 		}
+-		if _, err := os.Stat(filepath.Join(bundle.path, proc.InitExit)); err == nil {
+		if events.InitExitExist(bundle.path) {
+ 			if !events.ExitPending(ns, t.id, uint32(pid)) {
+ 				events.ExitAddFile(ns, events.ExitFile(t.id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task")
+ 			}
+diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go
+index d7fdcaf..f421fde 100644
+--- a/runtime/v1/shim/service.go
+++ b/runtime/v1/shim/service.go
+@@ -513,7 +513,7 @@ func (s *Service) checkProcesses(e runc.Exit) {
+ 			if ip, ok := p.(*proc.Init); ok {
+ 				ns := filepath.Base(filepath.Dir(ip.Bundle))
+ 				events.ExitAddFile(ns, events.ExitFile(s.id, uint32(e.Pid), uint32(e.Status)), "init exited")
+-				ioutil.WriteFile(filepath.Join(ip.Bundle, proc.InitExit), []byte(fmt.Sprintf("%d", e.Pid)), 0600)
+				events.InitExitWrite(ip.Bundle, e.Pid)
+ 			}
+ 			if shouldKillAll {
+ 				if ip, ok := p.(*proc.Init); ok {
+-- 
+2.7.4.3
+
--- a/patch/0027-log-make-tester-happy.patch
+++ b/patch/0027-log-make-tester-happy.patch
@ -0,0 +1,48 @@
+From a275b359b2e85d8f353eab12d538a94609171918 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Sat, 23 Feb 2019 18:32:00 +0800
+Subject: [PATCH 27/27] log:  make tester happy
+
+reason: make tester happy
+ check_docker_error /tmp/tmp_11955/log2 b3357887148bc59212d30dba46d3eea9490cfe94594fa00aa7706c7addb92d91
+ grep docker /tmp/tmp_11955/log2
+ grep error
+ grep b3357887148bc59212d30dba46d3eea9490cfe94594fa00aa7706c7addb92d91
+ grep -w 'container did not start before the specified timeout'
+
+Change-Id: Iddd40bd42212bf09f52c17f28119a6b5364f4de7
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ hack/containerd.spec      | 2 +-
+ runtime/v1/shim/reaper.go | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/hack/containerd.spec b/hack/containerd.spec
+index 869012a..05f68c7 100644
+--- a/hack/containerd.spec
+++ b/hack/containerd.spec
+@@ -3,7 +3,7 @@
+ Version:        1.2.0
+ 
+ Name:           containerd
+-Release:        6%{?dist}
+Release:        7%{?dist}
+ Summary:        An industry-standard container runtime
+ License:        ASL 2.0
+ URL:            https://containerd.io
+diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go
+index a2b90fe..529a533 100644
+--- a/runtime/v1/shim/reaper.go
+++ b/runtime/v1/shim/reaper.go
+@@ -125,7 +125,7 @@ func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, e
+ 		if SameProcess(c, c.Process.Pid) {
+ 			syscall.Kill(c.Process.Pid, syscall.SIGKILL)
+ 		}
+-		return 0, errors.Errorf("timeout %ds for cmd(pid= %d): %s, %s", sec, c.Process.Pid, c.Path, c.Args)
+		return 0, errors.Errorf("container did not start before the specified timeout %ds for cmd(pid=%d): %s, %s", sec, c.Process.Pid, c.Path, c.Args)
+ 	case status := <-sch:
+ 		return status, nil
+ 	case err := <-ech:
+-- 
+2.7.4.3
+
--- a/patch/0028-restore-delete-task-in-containerd-restoring.patch
+++ b/patch/0028-restore-delete-task-in-containerd-restoring.patch
@ -0,0 +1,33 @@
+From 1130a0bc101c3f59c99eb850b24d0799c216d677 Mon Sep 17 00:00:00 2001
+From: xiadanni1 <xiadanni1@huawei.com>
+Date: Fri, 22 Mar 2019 21:22:08 +0800
+Subject: [PATCH] restore:  delete task in containerd restoring
+
+reason: delete task quickly when containerd is restoring to avoid container restart fail.
+
+Change-Id: Ide5e8c9bbd873addc6c35b9604e4cda03ca78b5e
+Signed-off-by: xiadanni1 <xiadanni1@huawei.com>
+---
+ runtime/v1/linux/runtime.go | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index e92904e..2a45aaa 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -426,7 +426,11 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			continue
+ 		}
+ 		if pid <= 0 {
+-			log.G(ctx).Warnf("skip load task in creating %s", id)
+			go func() {
+				log.G(ctx).Infof("del task in creating %s", id)
+				t.DeleteForce(ctx, uint32(pid))
+				log.G(ctx).Infof("del task in creating %s done", id)
+			}()
+ 			continue
+ 		}
+ 		if events.InitExitExist(bundle.path) {
+-- 
+1.8.3.1
+
--- a/patch/0029-restore-delete-task-asynchronously.patch
+++ b/patch/0029-restore-delete-task-asynchronously.patch
@ -0,0 +1,35 @@
+From de14f9d00033a9596823e0ea953437f5f244cb74 Mon Sep 17 00:00:00 2001
+From: xiadanni1 <xiadanni1@huawei.com>
+Date: Sat, 23 Mar 2019 07:18:57 +0800
+Subject: [PATCH] restore:  delete task asynchronously
+
+reason: set delete task to asynchronous to avoid containerd be killed when delete is blocking.
+        testCE_docker_hook_spec_ABN.059.sh
+
+Change-Id: I5fae8e60987b9617a835ea07710ca3c842efab14
+Signed-off-by: xiadanni1 <xiadanni1@huawei.com>
+---
+ runtime/v1/linux/runtime.go | 7 +++++--
+ 1 file changed, 5 insertions(+), 2 deletions(-)
+
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index 2a45aaa..cca72fe 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -437,8 +437,11 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			if !events.ExitPending(ns, t.id, uint32(pid)) {
+ 				events.ExitAddFile(ns, events.ExitFile(t.id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task")
+ 			}
+-			_, err := t.DeleteForce(ctx, uint32(pid))
+-			log.G(ctx).Warnf("delete force %s Pid=%d(exiting) error=%v", id, pid, err)
+			go func(){
+				log.G(ctx).Infof("delete force %s start, Pid=%d(exiting)", id, pid)
+				_, err := t.DeleteForce(ctx, uint32(pid))
+				log.G(ctx).Infof("delete force %s done, Pid=%d(exiting) error=%v", id, pid, err)
+			}()
+ 			continue
+ 		}
+ 		log.G(ctx).Infof("load-task %s Pid=%d done", id, pid)
+-- 
+1.8.3.1
+
--- a/patch/0030-event-fix-events-lost-when-loadTask-failed.patch
+++ b/patch/0030-event-fix-events-lost-when-loadTask-failed.patch
@ -0,0 +1,45 @@
+From 375689497320d105aa2ed026710e20d9b0bd2a72 Mon Sep 17 00:00:00 2001
+From: jiangpengfei9 <jiangpengfei9@huawei.com>
+Date: Mon, 1 Apr 2019 13:08:50 -0400
+Subject: [PATCH] event: fix events lost when loadTask failed
+
+reason: If containerd-shim and containerd process is killed, container will exit,
+however containerd exit event which generates when containerd restart to reload
+tasks can not publish to dockerd, because at the time of loading tasks the connection
+between dockerd and containerd isn't established.
+
+So we add this unpublish exit event to file and resend this event after grpc connection
+is established.
+
+Signed-off-by: jiangpengfei9 <jiangpengfei9@huawei.com>
+---
+ runtime/v1/linux/runtime.go | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index cca72fe..af823b2 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -373,6 +373,9 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 				"id":        id,
+ 				"namespace": ns,
+ 			}).Error("connecting to shim")
+			if !events.ExitPending(ns, id, uint32(pid)) {
+				events.ExitAddFile(ns, events.ExitFile(id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task")
+			}
+ 			err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid)
+ 			if err != nil {
+ 				log.G(ctx).WithError(err).WithField("bundle", bundle.path).
+@@ -388,6 +391,9 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 				"id":        id,
+ 				"namespace": ns,
+ 			}).Error("contacting to shim")
+			if !events.ExitPending(ns, id, uint32(pid)) {
+				events.ExitAddFile(ns, events.ExitFile(id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task")
+			}
+ 			err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid)
+ 			if err != nil {
+ 				log.G(ctx).WithError(err).WithField("bundle", bundle.path).
+-- 
+1.8.3.1
+
--- a/patch/0031-containerd-enable-relro-flags.patch
+++ b/patch/0031-containerd-enable-relro-flags.patch
@ -0,0 +1,28 @@
+From 2db6e4cda2e042fab327493c0fa095723d7c0352 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Mon, 15 Apr 2019 10:58:07 +0800
+Subject: [PATCH] containerd:  enable relro flags
+
+Change-Id: I5f32e7bf794842a14e1644f7aa3115a65b1bc698
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ Makefile | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/Makefile b/Makefile
+index e38dfb38..921b2d50 100644
+--- a/Makefile
+++ b/Makefile
+@@ -77,7 +77,8 @@ MANPAGES=ctr.1 containerd.1 containerd-config.1 containerd-config.toml.5
+ # Build tags seccomp and apparmor are needed by CRI plugin.
+ BUILDTAGS ?= seccomp apparmor
+ GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",)
+-GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)'
+GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' \
+	-ldflags=-extldflags=-zrelro -ldflags=-extldflags=-znow
+ SHIM_GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"'
+ 
+ #Replaces ":" (*nix), ";" (windows) with newline for easy parsing
+-- 
+2.17.1
+
--- a/patch/0032-containerd-enable-bep-ldflags.patch
+++ b/patch/0032-containerd-enable-bep-ldflags.patch
@ -0,0 +1,45 @@
+From da6ea77f9f47c740fe85e7e4d34889e131135b81 Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Mon, 15 Apr 2019 23:44:55 +0800
+Subject: [PATCH] containerd:  enable bep ldflags
+
+Change-Id: I820b100aa1420fc399878a905de14fb6a25ca1a4
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ Makefile | 12 ++++++++----
+ 1 file changed, 8 insertions(+), 4 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 921b2d50..612330b4 100644
+--- a/Makefile
+++ b/Makefile
+@@ -77,9 +77,12 @@ MANPAGES=ctr.1 containerd.1 containerd-config.1 containerd-config.toml.5
+ # Build tags seccomp and apparmor are needed by CRI plugin.
+ BUILDTAGS ?= seccomp apparmor
+ GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",)
+-GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' \
+-	-ldflags=-extldflags=-zrelro -ldflags=-extldflags=-znow
+-SHIM_GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"'
+
+BEP_DIR=/tmp/containerd-build-bep
+BEP_FLAGS=-tmpdir=/tmp/containerd-build-bep
+
+GO_LDFLAGS=-ldflags '-s -w -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)'
+SHIM_GO_LDFLAGS=-ldflags '-s -w $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"'
+ 
+ #Replaces ":" (*nix), ";" (windows) with newline for easy parsing
+ GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n")
+@@ -166,8 +169,9 @@ FORCE:
+ 
+ # Build a binary from a cmd.
+ bin/%: cmd/% FORCE
+	mkdir -p $(BEP_DIR)
+ 	@echo "$(WHALE) $@${BINARY_SUFFIX}"
+-	@go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@${BINARY_SUFFIX} ${GO_LDFLAGS} ${GO_TAGS}  ./$<
+	go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@${BINARY_SUFFIX} ${GO_LDFLAGS} ${GO_TAGS}  ./$<
+ 
+ bin/containerd-shim: cmd/containerd-shim FORCE # set !cgo and omit pie for a static shim build: https://github.com/golang/go/issues/17789#issuecomment-258542220
+ 	@echo "$(WHALE) bin/containerd-shim"
+-- 
+2.17.1
+
--- a/patch/0033-containerd-fix-opened-file-not-close.patch
+++ b/patch/0033-containerd-fix-opened-file-not-close.patch
@ -0,0 +1,28 @@
+From b5806942e2938d4800298df276f1a095b859bacb Mon Sep 17 00:00:00 2001
+From: xiadanni1 <xiadanni1@huawei.com>
+Date: Fri, 19 Apr 2019 22:05:18 +0800
+Subject: [PATCH] containerd:  fix opened file not close
+
+reason: fix opened file not close
+
+Change-Id: I69f53255eabd3dd2e87a61ba963fa8027870e014
+Signed-off-by: xiadanni1 <xiadanni1@huawei.com>
+---
+ runtime/v1/linux/proc/utils.go | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/runtime/v1/linux/proc/utils.go b/runtime/v1/linux/proc/utils.go
+index 3d0334c..ab9f5fa 100644
+--- a/runtime/v1/linux/proc/utils.go
+++ b/runtime/v1/linux/proc/utils.go
+@@ -41,6 +41,7 @@ func getLastRuntimeError(r *runc.Runc) (string, error) {
+ 	if err != nil {
+ 		return "", err
+ 	}
+	defer f.Close()
+ 
+ 	var (
+ 		errMsg string
+-- 
+1.8.3.1
+
--- a/patch/0034-containerd-add-buildid-in-Makefile.patch
+++ b/patch/0034-containerd-add-buildid-in-Makefile.patch
@ -0,0 +1,28 @@
+From e61f2c1664c91b5c8a8cb48641002c7c471c1d45 Mon Sep 17 00:00:00 2001
+From: zhangyu235 <zhangyu235@huawei.com>
+Date: Tue, 23 Apr 2019 12:24:50 +0800
+Subject: [PATCH] containerd: add buildid in Makefile
+
+Change-Id: I1c2ff035db2a02d125139b9ff170f91e81181541
+---
+ Makefile | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 612330b..a400899 100644
+--- a/Makefile
+++ b/Makefile
+@@ -81,8 +81,8 @@ GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",)
+ BEP_DIR=/tmp/containerd-build-bep
+ BEP_FLAGS=-tmpdir=/tmp/containerd-build-bep
+ 
+-GO_LDFLAGS=-ldflags '-s -w -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)'
+-SHIM_GO_LDFLAGS=-ldflags '-s -w $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"'
+GO_LDFLAGS=-ldflags '-s -w -buildid=IdByIsula -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)'
+SHIM_GO_LDFLAGS=-ldflags '-s -w -buildid=IdByIsula $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"'
+ 
+ #Replaces ":" (*nix), ";" (windows) with newline for easy parsing
+ GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n")
+-- 
+2.7.4.3
+
--- a/patch/0035-containerd-fix-the-path-of-containerd.spec-in.patch
+++ b/patch/0035-containerd-fix-the-path-of-containerd.spec-in.patch
@ -0,0 +1,82 @@
+From 8f97c7a7353c05a8b64ef9ee522ee62fba66a608 Mon Sep 17 00:00:00 2001
+From: zhangyu235 <zhangyu235@huawei.com>
+Date: Sun, 5 May 2019 19:50:56 +0800
+Subject: [PATCH] containerd: fix the path of containerd.spec in
+ Makefile
+
+Change-Id: I4ec87e5ddf256574513f977e53e4bdf050e0169c
+Signed-off-by: zhangyu235 <zhangyu235@huawei.com>
+---
+ Makefile             |  2 +-
+ hack/containerd.spec | 46 ----------------------------------------------
+ 2 files changed, 1 insertion(+), 47 deletions(-)
+ delete mode 100644 hack/containerd.spec
+
+diff --git a/Makefile b/Makefile
+index a400899..5de5cf7 100644
+--- a/Makefile
+++ b/Makefile
+@@ -20,7 +20,7 @@ ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+ DESTDIR=/usr/local
+ 
+ # Used to populate variables in version package.
+-VERSION=$(shell echo version:)$(shell grep '^Version' ${ROOTDIR}/hack/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/').$(shell grep '^Release:' ${ROOTDIR}/hack/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+VERSION=$(shell echo version:)$(shell grep '^Version' ${ROOTDIR}/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/').$(shell grep '^Release:' ${ROOTDIR}/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+ REVISION=$(shell echo commit:)$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
+ 
+ ifneq "$(strip $(shell command -v go 2>/dev/null))" ""
+diff --git a/hack/containerd.spec b/hack/containerd.spec
+deleted file mode 100644
+index 05f68c7..0000000
+--- a/hack/containerd.spec
+++ /dev/null
+@@ -1,46 +0,0 @@
+-%global goipath github.com/containerd/containerd
+-%global debug_package %{nil}
+-Version:        1.2.0
+-
+-Name:           containerd
+-Release:        7%{?dist}
+-Summary:        An industry-standard container runtime
+-License:        ASL 2.0
+-URL:            https://containerd.io
+-Source0:        containerd-1.2.0.tar.gz
+-
+-BuildRequires:  golang glibc-static make
+-BuildRequires:  btrfs-progs-devel
+-
+-
+-%description
+-containerd is an industry-standard container runtime with an emphasis on
+-simplicity, robustness and portability.  It is available as a daemon for Linux
+-and Windows, which can manage the complete container lifecycle of its host
+-system: image transfer and storage, container execution and supervision,
+-low-level storage and network attachments, etc.
+-
+-
+-%prep
+-%setup -c -n containerd
+-
+-%build
+-GO_BUILD_PATH=$PWD/_build
+-install -m 0755 -vd $(dirname $GO_BUILD_PATH/src/%{goipath})
+-ln -fs $PWD $GO_BUILD_PATH/src/%{goipath}
+-cd $GO_BUILD_PATH/src/%{goipath}
+-export GOPATH=$GO_BUILD_PATH:%{gopath}
+-export BUILDTAGS="no_btrfs no_cri"
+-make
+-
+-%install
+-install -d $RPM_BUILD_ROOT/%{_bindir}
+-install -p -m 755 bin/containerd $RPM_BUILD_ROOT/%{_bindir}/containerd
+-install -p -m 755 bin/containerd-shim $RPM_BUILD_ROOT/%{_bindir}/containerd-shim
+-
+-%files
+-%{_bindir}/containerd
+-%{_bindir}/containerd-shim
+-
+-
+-%changelog
+-- 
+2.7.4.3
+
--- a/patch/0036-containerd-support-container-start-timeout-se.patch
+++ b/patch/0036-containerd-support-container-start-timeout-se.patch
@ -0,0 +1,69 @@
+From 1980e34108cf2fab407c4e0b45cb07fc06e15642 Mon Sep 17 00:00:00 2001
+From: lixiang172 <lixiang172@huawei.com>
+Date: Thu, 9 May 2019 21:36:56 +0800
+Subject: [PATCH] containerd:  support container start timeout setting
+
+Change-Id: I8c958a1c16ed6c7a86e4c6299ad1ef81c7476120
+Signed-off-by: lixiang172 <lixiang172@huawei.com>
+---
+ vendor/github.com/containerd/go-runc/runc.go | 24 ++++++++++++++++++++++--
+ 1 file changed, 22 insertions(+), 2 deletions(-)
+
+diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go
+index e66ea5b..6323bf2 100644
+--- a/vendor/github.com/containerd/go-runc/runc.go
+++ b/vendor/github.com/containerd/go-runc/runc.go
+@@ -30,9 +30,9 @@ import (
+ 	"strings"
+ 	"syscall"
+ 	"time"
+-	"github.com/sirupsen/logrus"
+ 
+ 	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+ )
+ 
+ // Format is the type of log formatting options avaliable
+@@ -54,7 +54,10 @@ const (
+ 	// DefaultCommand is the default command for Runc
+ 	DefaultCommand = "runc"
+ 	execTimeout    = 30
+-	createTimeout  = 120
+)
+
+var (
+	createTimeout int64 = 120
+ )
+ 
+ // Runc is the client to the runc cli
+@@ -72,6 +75,15 @@ type Runc struct {
+ 	Rootless      *bool // nil stands for "auto"
+ }
+ 
+func init() {
+	runtimeTimeout, err := convertTime(os.Getenv("DOCKER_RUNTIME_START_TIMEOUT"))
+	if err != nil {
+		logrus.Warnf("init error, wrong runtimeTimeout format: %v", err)
+	} else {
+		createTimeout = runtimeTimeout
+	}
+}
+
+ // List returns all containers created inside the provided runc root directory
+ func (r *Runc) List(context context.Context) ([]*Container, error) {
+ 	data, err := cmdOutput(r.command(context, "list", "--format=json"), false)
+@@ -734,3 +746,11 @@ func cmdOutputTimeout(cmd *exec.Cmd, combined bool, timeout int64) ([]byte, erro
+ 
+ 	return b.Bytes(), err
+ }
+
+func convertTime(timeout string) (int64, error) {
+	timeDura, err := time.ParseDuration(timeout)
+	if err != nil {
+		return 0, err
+	}
+	return timeDura.Nanoseconds() / 1e9, nil
+}
+-- 
+1.8.3.1
+
--- a/patch/0037-containerd-Fix-fd-leak-of-shim-log.patch
+++ b/patch/0037-containerd-Fix-fd-leak-of-shim-log.patch
@ -0,0 +1,55 @@
+From 26c6307f1cab31105583ef22c2da8fe44a8d45e4 Mon Sep 17 00:00:00 2001
+From: zhangyu235 <zhangyu235@huawei.com>
+Date: Fri, 17 May 2019 16:52:06 +0800
+Subject: [PATCH] containerd: Fix fd leak of shim log
+
+reason:Open shim v2 log with the flag `O_RDWR` will cause the `Read()` block
+forever even if the pipe has been closed on the shim side. Then the
+`io.Copy()` would never return and lead to a fd leak.
+Fix typo when closing shim v1 log which causes the `stdouLog` leak.
+Update `numPipes` function in test case to get the opened FIFO
+correctly.
+
+Cherry-pick from upstream cf6e00854
+Reference from https://github.com/containerd/containerd/pull/3266
+
+Change-Id: If83a4ca9b9ec0079ac0f0015d1f6768581571030
+Signed-off-by: Li Yuxuan <liyuxuan04@baidu.com>
+Signed-off-by: zhangyu235 <zhangyu235@huawei.com>
+---
+ container_linux_test.go          | 2 +-
+ runtime/v1/shim/client/client.go | 4 ++--
+ 2 files changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/container_linux_test.go b/container_linux_test.go
+index fa764d7..fdf6349 100644
+--- a/container_linux_test.go
+++ b/container_linux_test.go
+@@ -329,7 +329,7 @@ func TestShimDoesNotLeakPipes(t *testing.T) {
+ }
+ 
+ func numPipes(pid int) (int, error) {
+-	cmd := exec.Command("sh", "-c", fmt.Sprintf("lsof -p %d | grep pipe", pid))
+	cmd := exec.Command("sh", "-c", fmt.Sprintf("lsof -p %d | grep FIFO", pid))
+ 
+ 	var stdout bytes.Buffer
+ 	cmd.Stdout = &stdout
+diff --git a/runtime/v1/shim/client/client.go b/runtime/v1/shim/client/client.go
+index ef74030..a819be6 100644
+--- a/runtime/v1/shim/client/client.go
+++ b/runtime/v1/shim/client/client.go
+@@ -96,9 +96,9 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa
+ 			cmd.Wait()
+ 			exitHandler()
+ 			if stdoutLog != nil {
+-				stderrLog.Close()
+				stdoutLog.Close()
+ 			}
+-			if stdoutLog != nil {
+			if stderrLog != nil {
+ 				stderrLog.Close()
+ 			}
+ 		}()
+-- 
+2.7.4.3
+
--- a/patch/0037-containerd-fix-shim-std-logs-not-close-after-.patch
+++ b/patch/0037-containerd-fix-shim-std-logs-not-close-after-.patch
@ -0,0 +1,59 @@
+From d13733a390a987006bd5febb7d28a2d1c7873af2 Mon Sep 17 00:00:00 2001
+From: zhangyu235 <zhangyu235@huawei.com>
+Date: Thu, 30 May 2019 09:27:00 +0800
+Subject: [PATCH] containerd: fix shim std logs not close after shim
+ exit
+
+reason:fix shim std logs not close after shim exit
+
+Change-Id: I980fb17b1d46de099b81529ea46681cf9f4bf09c
+Signed-off-by: zhangyu235 <zhangyu235@huawei.com>
+---
+ runtime/v1/linux/runtime.go | 16 +++++++++++++++-
+ 1 file changed, 15 insertions(+), 1 deletion(-)
+
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index af823b2..66914fe 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -361,7 +361,9 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 		ctx = namespaces.WithNamespace(ctx, ns)
+ 		pid, _ := runc.ReadPidFile(filepath.Join(bundle.path, proc.InitPidFile))
+ 		log.G(ctx).Infof("load-task %s/%s/%s Pid=%d", r.state, ns, id, pid)
+		shimExit := make(chan struct{})
+ 		s, err := bundle.NewShimClient(ctx, ns, ShimConnect(r.config, func() {
+			close(shimExit)
+ 			err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid)
+ 			if err != nil {
+ 				log.G(ctx).WithError(err).WithField("bundle", bundle.path).
+@@ -426,6 +428,18 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 		}
+ 		go io.Copy(os.Stderr, shimStderrLog)
+ 
+		go func() {
+			select {
+			case <-shimExit:
+				if shimStdoutLog != nil {
+					shimStdoutLog.Close()
+				}
+				if shimStderrLog != nil {
+					shimStderrLog.Close()
+				}
+			}
+		}()
+
+ 		t, err := newTask(id, ns, pid, s, r.events, r.tasks, bundle)
+ 		if err != nil {
+ 			log.G(ctx).WithError(err).Error("loading task type")
+@@ -443,7 +457,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
+ 			if !events.ExitPending(ns, t.id, uint32(pid)) {
+ 				events.ExitAddFile(ns, events.ExitFile(t.id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task")
+ 			}
+-			go func(){
+			go func() {
+ 				log.G(ctx).Infof("delete force %s start, Pid=%d(exiting)", id, pid)
+ 				_, err := t.DeleteForce(ctx, uint32(pid))
+ 				log.G(ctx).Infof("delete force %s done, Pid=%d(exiting) error=%v", id, pid, err)
+-- 
+2.7.4.3
+
--- a/patch/0038-containerd-support-kill-D-state-container.patch
+++ b/patch/0038-containerd-support-kill-D-state-container.patch
@ -0,0 +1,65 @@
+From 8ab02b5aecb0fa04ad747988d838e1c4de535222 Mon Sep 17 00:00:00 2001
+From: Jing Rui <jingrui@huawei.com>
+Date: Tue, 18 Jun 2019 00:12:41 +0800
+Subject: [PATCH] containerd:  support kill D state container
+
+Change-Id: I057553f2b8d3f57b71e5ea79930067bb7071e524
+Signed-off-by: Jing Rui <jingrui@huawei.com>
+---
+ runtime/v1/shim/service.go | 21 +++++++++++++++++++++
+ 1 file changed, 21 insertions(+)
+
+diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go
+index f421fdef..8adaf35b 100644
+--- a/runtime/v1/shim/service.go
+++ b/runtime/v1/shim/service.go
+@@ -26,6 +26,7 @@ import (
+ 	"os"
+ 	"path/filepath"
+ 	"sync"
+	"syscall"
+ 	"time"
+ 
+ 	"github.com/containerd/console"
+@@ -366,11 +367,30 @@ func (s *Service) Resume(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, e
+ 
+ // Kill a process with the provided signal
+ func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Empty, error) {
+	delayKill := func(p rproc.Process) {
+		if s.id != p.ID() || r.Signal != uint32(syscall.SIGKILL) {
+			return
+		}
+
+		for i := 1; i < 5; i++ {
+			time.Sleep(10 * time.Second)
+			err := p.Kill(ctx, r.Signal, r.All)
+			logrus.Infof("delay kill %s retry %d error=%v", s.id, i, err)
+		}
+
+		logrus.Infof("force exit shim %s ...", s.id)
+		p.SetExited(137)
+		err := p.Delete(ctx)
+		logrus.Infof("force exit shim %s error=%v", s.id, err)
+		os.Exit(0)
+	}
+
+ 	if r.ID == "" {
+ 		p, err := s.getInitProcess()
+ 		if err != nil {
+ 			return nil, err
+ 		}
+		go delayKill(p)
+ 		if err := p.Kill(ctx, r.Signal, r.All); err != nil {
+ 			return nil, errdefs.ToGRPC(err)
+ 		}
+@@ -381,6 +401,7 @@ func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Emp
+ 	if err != nil {
+ 		return nil, err
+ 	}
+	go delayKill(p)
+ 	if err := p.Kill(ctx, r.Signal, r.All); err != nil {
+ 		return nil, errdefs.ToGRPC(err)
+ 	}
+-- 
+2.17.1
+
--- a/patch/0039-containerd-modify-containerd-shim-to-ad.patch
+++ b/patch/0039-containerd-modify-containerd-shim-to-ad.patch
@ -0,0 +1,51 @@
+From 5eef82c3c41eabb532cd7520acf7e8587b76d8b5 Mon Sep 17 00:00:00 2001
+From: jiangpengfei <jiangpengfei9@huawei.com>
+Date: Wed, 10 Jul 2019 15:07:46 -0400
+Subject: [PATCH] containerd: modify containerd-shim to adapt runv
+ runtime
+
+reason: containerd-shim pass a too long runtime root path to runv runtime, which cause hyperstartgrpc.sock
+file absolute path exceed the max length of Unix Socket(max length is 108).
+
+Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
+---
+ runtime/v1/linux/proc/init.go | 11 ++++++++++-
+ 1 file changed, 10 insertions(+), 1 deletion(-)
+
+diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go
+index 5b23671..d464147 100644
+--- a/runtime/v1/linux/proc/init.go
+++ b/runtime/v1/linux/proc/init.go
+@@ -44,6 +44,9 @@ import (
+ // InitPidFile name of the file that contains the init pid
+ const InitPidFile = "init.pid"
+ 
+// Default runv runtime root dir
+const defaultRunvRoot = "/run/runv"
+
+ // Init represents an initial process for a container
+ type Init struct {
+ 	wg sync.WaitGroup
+@@ -83,12 +86,18 @@ func NewRunc(root, path, namespace, runtime, criu string, systemd bool) *runc.Ru
+ 	if root == "" {
+ 		root = RuncRoot
+ 	}
+
+	rootPath := filepath.Join(root, namespace)
+	if strings.Contains(runtime, "runv") {
+		rootPath = defaultRunvRoot
+	}
+
+ 	return &runc.Runc{
+ 		Command:       runtime,
+ 		Log:           filepath.Join(path, "log.json"),
+ 		LogFormat:     runc.JSON,
+ 		PdeathSignal:  syscall.SIGKILL,
+-		Root:          filepath.Join(root, namespace),
+		Root:          rootPath,
+ 		Criu:          criu,
+ 		SystemdCgroup: systemd,
+ 	}
+-- 
+1.8.3.1
+
--- a/patch/0040-containerd-add-shim-exit-when-bundle-dir-does.patch
+++ b/patch/0040-containerd-add-shim-exit-when-bundle-dir-does.patch
@ -0,0 +1,47 @@
+From 07605707cce769e4f4c79b700586b5c59ec0b15a Mon Sep 17 00:00:00 2001
+From: xiadanni1 <xiadanni1@huawei.com>
+Date: Sat, 13 Jul 2019 06:32:54 +0800
+Subject: [PATCH] containerd:  add shim exit when bundle dir does not
+ exist
+
+reason: when bundle dir is deleted, containerd-shim should exit to avoid
+shim.sock is occupied when container restart next time.
+
+Change-Id: I956412598e17d15f25b91afe1cbb9e24463f04be
+Signed-off-by: xiadanni1 <xiadanni1@huawei.com>
+---
+ runtime/v1/shim/service.go | 12 +++++++++++-
+ 1 file changed, 11 insertions(+), 1 deletion(-)
+
+diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go
+index 8adaf35..ac545ea 100644
+--- a/runtime/v1/shim/service.go
+++ b/runtime/v1/shim/service.go
+@@ -141,13 +141,23 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
+ 	}
+ 	rootfs := filepath.Join(r.Bundle, "rootfs")
+ 	defer func() {
+		go func() {
+			for i := 0; i < 60; i++ {
+				time.Sleep(time.Second)
+				_, err := os.Stat(r.Bundle)
+				if os.IsNotExist(err) {
+					logrus.Errorf("bundle dir: %v does not exist, containerd-shim exit", r.Bundle)
+					os.Exit(0)
+				}
+			}
+		}()
+ 		if err != nil {
+ 			logrus.Errorf("create init %s failed error=%v", r.ID, err)
+ 			if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
+ 				log.G(ctx).WithError(err2).Warn("Failed to cleanup rootfs mount")
+ 			}
+ 			go func() {
+-				time.Sleep(10*time.Second)
+				time.Sleep(10 * time.Second)
+ 				os.Exit(0)
+ 			}()
+ 		}
+-- 
+1.8.3.1
+
--- a/patch/0041-containerd-fix-containerd-call-runv-delete-directly.patch
+++ b/patch/0041-containerd-fix-containerd-call-runv-delete-directly.patch
@ -0,0 +1,80 @@
+From be9c04e9a90be92437c12ce90c8ff6d4ec1d83b3 Mon Sep 17 00:00:00 2001
+From: jiangpengfei <jiangpengfei9@huawei.com>
+Date: Thu, 18 Jul 2019 07:57:52 -0400
+Subject: [PATCH] containerd: fix containerd call runv delete directly
+ use wrong --root parameters
+
+reason: When containerd-shim process is killed abnormaly, containerd will exec runv
+delete command directly, however it will use the wrong --root parameters which is not
+compatible with runv runtime.
+
+Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
+---
+ runtime/v1/linux/proc/init.go |  4 ++--
+ runtime/v1/linux/runtime.go   | 10 +++++++++-
+ 2 files changed, 11 insertions(+), 3 deletions(-)
+
+diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go
+index d464147..44d3f58 100644
+--- a/runtime/v1/linux/proc/init.go
+++ b/runtime/v1/linux/proc/init.go
+@@ -45,7 +45,7 @@ import (
+ const InitPidFile = "init.pid"
+ 
+ // Default runv runtime root dir
+-const defaultRunvRoot = "/run/runv"
+const DefaultRunvRoot = "/run/runv"
+ 
+ // Init represents an initial process for a container
+ type Init struct {
+@@ -89,7 +89,7 @@ func NewRunc(root, path, namespace, runtime, criu string, systemd bool) *runc.Ru
+ 
+ 	rootPath := filepath.Join(root, namespace)
+ 	if strings.Contains(runtime, "runv") {
+-		rootPath = defaultRunvRoot
+		rootPath = DefaultRunvRoot
+ 	}
+ 
+ 	return &runc.Runc{
+diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go
+index 66914fe..f8e3074 100644
+--- a/runtime/v1/linux/runtime.go
+++ b/runtime/v1/linux/runtime.go
+@@ -25,6 +25,7 @@ import (
+ 	"io/ioutil"
+ 	"os"
+ 	"path/filepath"
+	"strings"
+ 	"time"
+ 
+ 	eventstypes "github.com/containerd/containerd/api/events"
+@@ -506,6 +507,7 @@ func (r *Runtime) terminate(ctx context.Context, bundle *bundle, ns, id string)
+ 	if err != nil {
+ 		return err
+ 	}
+
+ 	if err := rt.Delete(ctx, id, &runc.DeleteOpts{
+ 		Force: true,
+ 	}); err != nil {
+@@ -539,11 +541,17 @@ func (r *Runtime) getRuntime(ctx context.Context, ns, id string) (*runc.Runc, er
+ 		}
+ 	}
+ 
+	rootPath := filepath.Join(root, ns)
+
+	if strings.Contains(cmd, "runv") {
+		rootPath = proc.DefaultRunvRoot
+	}
+
+ 	return &runc.Runc{
+ 		Command:      cmd,
+ 		LogFormat:    runc.JSON,
+ 		PdeathSignal: unix.SIGKILL,
+-		Root:         filepath.Join(root, ns),
+		Root:         rootPath,
+ 		Debug:        r.config.ShimDebug,
+ 	}, nil
+ }
+-- 
+1.8.3.1
+
--- a/patch/0042-containerd-close-inherit-shim.sock-fd-to-adap.patch
+++ b/patch/0042-containerd-close-inherit-shim.sock-fd-to-adap.patch
@ -0,0 +1,34 @@
+From dcef6fcbdc78f7e9c14bdcd58e79d3eac8bc1c1b Mon Sep 17 00:00:00 2001
+From: jiangpengfei <jiangpengfei9@huawei.com>
+Date: Thu, 18 Jul 2019 15:44:12 -0400
+Subject: [PATCH] containerd: close inherit shim.sock fd to adapt runv
+
+reason: runv create prcess is created by containerd-shim process and will
+inherit the abstract unix socket shim.sock fd from containerd-shim.
+If pause container restart, qemu and runv-proxy process are still running,
+and shim.sock fd doesn't close, so pause container can not reuse the shim.sock
+path and restart failed!
+
+Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
+---
+ cmd/containerd-shim/main_unix.go | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go
+index 38b3eb4..89f6be9 100644
+--- a/cmd/containerd-shim/main_unix.go
+++ b/cmd/containerd-shim/main_unix.go
+@@ -189,6 +189,10 @@ func serve(ctx context.Context, server *ttrpc.Server, path string) error {
+ 	)
+ 	if path == "" {
+ 		l, err = net.FileListener(os.NewFile(3, "socket"))
+		_, _, errnoValue := unix.Syscall(unix.SYS_FCNTL, 3, uintptr(unix.F_SETFD), unix.FD_CLOEXEC)
+		if errnoValue != 0 {
+			logrus.Errorf("SYS_FCNTL set fd 3 FD_CLOEXEC flag failed: %v", errnoValue)
+		}
+ 		path = "[inherited from parent]"
+ 	} else {
+ 		if len(path) > 106 {
+-- 
+1.8.3.1
+
--- a/patch/0043-containerd-run-state-with-timeout-10s.patch
+++ b/patch/0043-containerd-run-state-with-timeout-10s.patch
@ -0,0 +1,77 @@
+From 7b9e8a793fa6c0ec67effac0bc53d55c275e13be Mon Sep 17 00:00:00 2001
+From: jingrui <jingrui@huawei.com>
+Date: Thu, 25 Jul 2019 19:29:50 +0800
+Subject: [PATCH] containerd:  run state with timeout 10s
+
+Change-Id: Idf55f750c2e7c6a9268318f519f1c8bc1595e09e
+Signed-off-by: jingrui <jingrui@huawei.com>
+---
+ Makefile                 |  4 ++--
+ runtime/v1/linux/task.go |  3 ---
+ services/tasks/local.go  | 11 +++++++++++
+ 3 files changed, 13 insertions(+), 5 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 5de5cf75..9e7f3ae3 100644
+--- a/Makefile
+++ b/Makefile
+@@ -81,8 +81,8 @@ GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",)
+ BEP_DIR=/tmp/containerd-build-bep
+ BEP_FLAGS=-tmpdir=/tmp/containerd-build-bep
+ 
+-GO_LDFLAGS=-ldflags '-s -w -buildid=IdByIsula -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)'
+-SHIM_GO_LDFLAGS=-ldflags '-s -w -buildid=IdByIsula $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"'
+GO_LDFLAGS=-ldflags ' -buildid=IdByIsula -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)'
+SHIM_GO_LDFLAGS=-ldflags ' -buildid=IdByIsula $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"'
+ 
+ #Replaces ":" (*nix), ";" (windows) with newline for easy parsing
+ GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n")
+diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go
+index b692ae78..d2bbb764 100644
+--- a/runtime/v1/linux/task.go
+++ b/runtime/v1/linux/task.go
+@@ -92,9 +92,6 @@ func (t *Task) delete(ctx context.Context, force bool, pid uint32) (*runtime.Exi
+ 	rsp, err := t.shim.Delete(ctx, empty)
+ 	if err != nil {
+ 		log.G(ctx).WithError(err).Error("failed to delete container, force=%t", force)
+-		if !force {
+-			return nil, errdefs.FromGRPC(err)
+-		}
+ 	}
+ 	t.tasks.Delete(ctx, t.id)
+ 	if err := t.shim.KillShim(ctx); err != nil {
+diff --git a/services/tasks/local.go b/services/tasks/local.go
+index ce9ee59d..990e8411 100644
+--- a/services/tasks/local.go
+++ b/services/tasks/local.go
+@@ -47,6 +47,7 @@ import (
+ 	ptypes "github.com/gogo/protobuf/types"
+ 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+ 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+ 	bolt "go.etcd.io/bbolt"
+ 	"google.golang.org/grpc"
+ 	"google.golang.org/grpc/codes"
+@@ -185,9 +186,19 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.
+ 	if err := l.monitor.Monitor(c); err != nil {
+ 		return nil, errors.Wrap(err, "monitor task")
+ 	}
+
+	ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
+	defer cancel()
+
+ 	state, err := c.State(ctx)
+ 	if err != nil {
+ 		log.G(ctx).Error(err)
+		go func() {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+			defer cancel()
+			_, err := c.Delete(ctx)
+			logrus.Errorf("failed get pid, delete force error=%v", err)
+		}()
+ 	}
+ 	return &api.CreateTaskResponse{
+ 		ContainerID: r.ContainerID,
+-- 
+2.17.1
+
--- a/patch/0044-containerd-add-copyright.patch
+++ b/patch/0044-containerd-add-copyright.patch
@ -0,0 +1,38 @@
+From 80972f7d142540b886068d67a49794aaa7232fb5 Mon Sep 17 00:00:00 2001
+From: lixiang <lixiang172@huawei.com>
+Date: Fri, 6 Sep 2019 15:16:21 +0800
+Subject: [PATCH] containerd: add copyright
+
+reason: add copyright
+
+Change-Id: I93ef565c6bf10d6f8cb66d956dddbfbd14477138
+Signed-off-by: lixiang <lixiang172@huawei.com>
+---
+ events/exit.go | 10 +++++++++-
+ 1 file changed, 9 insertions(+), 1 deletion(-)
+
+diff --git a/events/exit.go b/events/exit.go
+index 772dc24..d3b3027 100644
+--- a/events/exit.go
+++ b/events/exit.go
+@@ -1,3 +1,12 @@
+/*
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+Use of this source code is governed by Apache-2.0
+license that can be found in the LICENSE file
+Description: common functions
+Author: jingrui
+Create: 2019-02-12
+*/
+
+ package events
+ 
+ import (
+@@ -97,4 +106,3 @@ func InitExitExist(bundle string) bool {
+ 	}
+ 	return false
+ }
+-
+-- 
+1.8.3.1
+
--- a/series.conf
+++ b/series.conf
@ -0,0 +1,46 @@
+patch/0006-shim-optimize-shim-lock-in-runtime-v1.patch
+patch/0007-shim-Increase-reaper-buffer-size-and-non-bl.patch
+patch/0008-runtime-Use-named-pipes-for-shim-logs.patch
+patch/0009-runtime-fix-pipe-in-broken-may-cause-shim-l.patch
+patch/0010-runtime-fix-pipe-in-broken-may-cause-shim-l.patch
+patch/0011-runtime-Add-timeout-and-cancel-to-shim-fifo.patch
+patch/0037-containerd-Fix-fd-leak-of-shim-log.patch
+
+patch/0001-grpc-vendor-grpc-fix-grpc-map-panic.patch
+patch/0002-sys-sys-count-steal-time-when-calculating-Sys.patch
+patch/0003-oci-oci-add-files-cgroups-support.patch
+patch/0004-runv-vendor-runv-compatibility.patch
+patch/0005-containerd-add-spec-for-build.patch
+patch/0012-bump-bump-containerd-to-1.2.0.2.patch
+patch/0013-log-support-log-init-pid-to-start-event-log.patch
+patch/0014-event-resend-exit-event-when-detect-container.patch
+patch/0015-restore-cleanup-container-pid-1.patch
+patch/0016-create-runc-delete-force-before-create.patch
+patch/0017-exit-using-init.exit-indicate-container-is-ex.patch
+patch/0018-containerd-shim-Dump-log-to-file-when-docker-.patch
+patch/0019-restore-check-shim-alive-when-containerd-is-r.patch
+patch/0020-events-resend-pending-exit-events-on-restore.patch
+patch/0021-containerd-Update-the-version-info-of-contain.patch
+patch/0022-containerd-bump-version-1.2.0.4.patch
+patch/0023-containerd-set-create-and-exec-timeout.patch
+patch/0024-create-cleanup-runc-dirty-files-on-start.patch
+patch/0025-restore-skip-load-task-in-creating.patch
+patch/0026-exit-optimize-init.exit-record.patch
+patch/0027-log-make-tester-happy.patch
+patch/0028-restore-delete-task-in-containerd-restoring.patch
+patch/0029-restore-delete-task-asynchronously.patch
+patch/0030-event-fix-events-lost-when-loadTask-failed.patch
+patch/0031-containerd-enable-relro-flags.patch
+patch/0032-containerd-enable-bep-ldflags.patch
+patch/0033-containerd-fix-opened-file-not-close.patch
+patch/0034-containerd-add-buildid-in-Makefile.patch
+patch/0035-containerd-fix-the-path-of-containerd.spec-in.patch
+patch/0036-containerd-support-container-start-timeout-se.patch
+patch/0037-containerd-fix-shim-std-logs-not-close-after-.patch
+patch/0038-containerd-support-kill-D-state-container.patch
+patch/0039-containerd-modify-containerd-shim-to-ad.patch
+patch/0040-containerd-add-shim-exit-when-bundle-dir-does.patch
+patch/0041-containerd-fix-containerd-call-runv-delete-directly.patch
+patch/0042-containerd-close-inherit-shim.sock-fd-to-adap.patch
+patch/0043-containerd-run-state-with-timeout-10s.patch
+patch/0044-containerd-add-copyright.patch