From fd1c8dda8cc02b9aef28f1e3e4e51ab216338e2b Mon Sep 17 00:00:00 2001 From: jingrui Date: Sun, 10 Feb 2019 15:40:52 +0800 Subject: [PATCH 15/27] restore: cleanup container pid=-1 reason: fix testCE_docker_hook_spec_ABN.050.sh when containerd killed during task create, see Runtime.Create(). the defer function will not execute, so shim residual. cleanup shim for container pid=-1 Change-Id: Ie9a7f6dff5f8a922cc97c5fcf44664ab60ac1a7a Signed-off-by: jingrui --- runtime/v1/linux/runtime.go | 10 +++++++--- runtime/v1/linux/task.go | 26 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go index e1b3cac..3b66304 100644 --- a/runtime/v1/linux/runtime.go +++ b/runtime/v1/linux/runtime.go @@ -316,6 +316,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { continue } id := path.Name() + log.G(ctx).Infof("load-task %s", id) bundle := loadBundle( id, filepath.Join(r.state, ns, id), @@ -372,6 +373,12 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { log.G(ctx).WithError(err).Error("loading task type") continue } + if pid == -1 { + _, err := t.DeleteForce(ctx) + log.G(ctx).Warnf("delete force %s Pid=-1 error=%v", id, err) + continue + } + log.G(ctx).Infof("load-task %s Pid=%d done", id, pid) o = append(o, t) } return o, nil @@ -380,9 +387,6 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { func (r *Runtime) cleanupAfterDeadShim(ctx context.Context, bundle *bundle, ns, id string, pid int) error { ctx = namespaces.WithNamespace(ctx, ns) if err := r.terminate(ctx, bundle, ns, id); err != nil { - if r.config.ShimDebug { - return errors.Wrap(err, "failed to terminate task, leaving bundle for debugging") - } log.G(ctx).WithError(err).Warn("failed to terminate task") } diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go index 1c650c4..6995156 100644 --- a/runtime/v1/linux/task.go +++ b/runtime/v1/linux/task.go @@ -21,6 +21,7 @@ package linux import ( "context" "sync" + "time" "github.com/containerd/cgroups" eventstypes "github.com/containerd/containerd/api/events" @@ -37,6 +38,7 @@ import ( "github.com/gogo/protobuf/types" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // Task on a linux based system @@ -86,10 +88,13 @@ func (t *Task) Namespace() string { } // Delete the task and return the exit status -func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { +func (t *Task) delete(ctx context.Context, force bool) (*runtime.Exit, error) { rsp, err := t.shim.Delete(ctx, empty) if err != nil { - return nil, errdefs.FromGRPC(err) + log.G(ctx).WithError(err).Error("failed to delete container, force=%t", force) + if !force { + return nil, errdefs.FromGRPC(err) + } } t.tasks.Delete(ctx, t.id) if err := t.shim.KillShim(ctx); err != nil { @@ -98,6 +103,14 @@ func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { if err := t.bundle.Delete(); err != nil { log.G(ctx).WithError(err).Error("failed to delete bundle") } + + if rsp == nil { + rsp = &shim.DeleteResponse{} + rsp.ExitStatus = 128 + uint32(unix.SIGKILL) + rsp.ExitedAt = time.Now().UTC() + rsp.Pid = 0 + } + t.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{ ContainerID: t.id, ExitStatus: rsp.ExitStatus, @@ -111,6 +124,15 @@ func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { }, nil } +// Delete the task and return the exit status +func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { + return t.delete(ctx, false) +} + +func (t *Task) DeleteForce(ctx context.Context) (*runtime.Exit, error) { + return t.delete(ctx, true) +} + // Start the task func (t *Task) Start(ctx context.Context) error { t.mu.Lock() -- 2.7.4.3