From 006bc6d0a9e0c233d0d14de53de0b18799c67081 Mon Sep 17 00:00:00 2001 From: xiadanni Date: Fri, 15 Feb 2019 06:00:52 +0800 Subject: [PATCH 23/27] containerd: set create and exec timeout reason:set create and exec timeout to avild block when command failed Change-Id: I6bc55f4ccc953bdc1d926ab940f0900811d68760 Signed-off-by: xiadanni --- hack/containerd.spec | 2 +- runtime/v1/shim/reaper.go | 50 +++++++++++++++++++++++++ runtime/v2/shim/reaper_unix.go | 4 ++ vendor/github.com/containerd/go-runc/monitor.go | 6 +++ vendor/github.com/containerd/go-runc/runc.go | 31 +++++++++++++-- 5 files changed, 88 insertions(+), 5 deletions(-) diff --git a/hack/containerd.spec b/hack/containerd.spec index f8d9084..f39c57a 100644 --- a/hack/containerd.spec +++ b/hack/containerd.spec @@ -3,7 +3,7 @@ Version: 1.2.0 Name: containerd -Release: 4%{?dist} +Release: 5%{?dist} Summary: An industry-standard container runtime License: ASL 2.0 URL: https://containerd.io diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go index 10d5c30..a2b90fe 100644 --- a/runtime/v1/shim/reaper.go +++ b/runtime/v1/shim/reaper.go @@ -19,8 +19,13 @@ package shim import ( + "io/ioutil" "os/exec" + "path/filepath" + "strconv" + "strings" "sync" + "syscall" "time" "github.com/containerd/containerd/sys" @@ -100,6 +105,34 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) { return -1, ErrNoSuchProcess } +// WaitTimeout is used to skip the blocked command and kill the left process. +func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, error) { + sch := make(chan int) + ech := make(chan error) + go func() { + for e := range ec { + if e.Pid == c.Process.Pid { + // make sure we flush all IO + c.Wait() + m.Unsubscribe(ec) + sch <- e.Status + return + } + } + }() + select { + case <-time.After(time.Duration(sec) * time.Second): + if SameProcess(c, c.Process.Pid) { + syscall.Kill(c.Process.Pid, syscall.SIGKILL) + } + return 0, errors.Errorf("timeout %ds for cmd(pid= %d): %s, %s", sec, c.Process.Pid, c.Path, c.Args) + case status := <-sch: + return status, nil + case err := <-ech: + return -1, err + } +} + // Subscribe to process exit changes func (m *Monitor) Subscribe() chan runc.Exit { c := make(chan runc.Exit, bufferSize) @@ -116,3 +149,20 @@ func (m *Monitor) Unsubscribe(c chan runc.Exit) { close(c) m.Unlock() } + +func SameProcess(cmd *exec.Cmd, pid int) bool { + bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline")) + if err != nil { + return false + } + for i := range bytes { + if bytes[i] == 0 { + bytes[i] = 32 + } + } + cmdline := string(bytes) + if strings.EqualFold(cmdline, strings.Join(cmd.Args, " ")+" ") { + return true + } + return false +} diff --git a/runtime/v2/shim/reaper_unix.go b/runtime/v2/shim/reaper_unix.go index 10d5c30..8bd7dd1 100644 --- a/runtime/v2/shim/reaper_unix.go +++ b/runtime/v2/shim/reaper_unix.go @@ -100,6 +100,10 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) { return -1, ErrNoSuchProcess } +func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, error) { + return m.Wait(c, ec) +} + // Subscribe to process exit changes func (m *Monitor) Subscribe() chan runc.Exit { c := make(chan runc.Exit, bufferSize) diff --git a/vendor/github.com/containerd/go-runc/monitor.go b/vendor/github.com/containerd/go-runc/monitor.go index ff06a3f..2c184d2 100644 --- a/vendor/github.com/containerd/go-runc/monitor.go +++ b/vendor/github.com/containerd/go-runc/monitor.go @@ -40,6 +40,7 @@ type Exit struct { type ProcessMonitor interface { Start(*exec.Cmd) (chan Exit, error) Wait(*exec.Cmd, chan Exit) (int, error) + WaitTimeout(*exec.Cmd, chan Exit, int64) (int, error) } type defaultMonitor struct { @@ -74,3 +75,8 @@ func (m *defaultMonitor) Wait(c *exec.Cmd, ec chan Exit) (int, error) { e := <-ec return e.Status, nil } + +func (m *defaultMonitor) WaitTimeout(c *exec.Cmd, ec chan Exit, sec int64) (int, error) { + e := <-ec + return e.Status, nil +} \ No newline at end of file diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go index e688881..fc64e8a 100644 --- a/vendor/github.com/containerd/go-runc/runc.go +++ b/vendor/github.com/containerd/go-runc/runc.go @@ -52,6 +52,8 @@ const ( Text Format = "text" // DefaultCommand is the default command for Runc DefaultCommand = "runc" + execTimeout = 30 + createTimeout = 120 ) // Runc is the client to the runc cli @@ -155,7 +157,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp cmd.ExtraFiles = opts.ExtraFiles if cmd.Stdout == nil && cmd.Stderr == nil { - data, err := cmdOutput(cmd, true) + data, err := cmdOutputTimeout(cmd, true, createTimeout) if err != nil { return fmt.Errorf("%s: %s", err, data) } @@ -172,7 +174,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp } } } - status, err := Monitor.Wait(cmd, ec) + status, err := Monitor.WaitTimeout(cmd, ec, createTimeout) if err == nil && status != 0 { err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0]) } @@ -234,7 +236,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts opts.Set(cmd) } if cmd.Stdout == nil && cmd.Stderr == nil { - data, err := cmdOutput(cmd, true) + data, err := cmdOutputTimeout(cmd, true, execTimeout) if err != nil { return fmt.Errorf("%s: %s", err, data) } @@ -251,7 +253,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts } } } - status, err := Monitor.Wait(cmd, ec) + status, err := Monitor.WaitTimeout(cmd, ec, execTimeout) if err == nil && status != 0 { err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0]) } @@ -707,3 +709,24 @@ func cmdOutput(cmd *exec.Cmd, combined bool) ([]byte, error) { return b.Bytes(), err } + +func cmdOutputTimeout(cmd *exec.Cmd, combined bool, timeout int64) ([]byte, error) { + b := getBuf() + defer putBuf(b) + + cmd.Stdout = b + if combined { + cmd.Stderr = b + } + ec, err := Monitor.Start(cmd) + if err != nil { + return nil, err + } + + status, err := Monitor.WaitTimeout(cmd, ec, timeout) + if err == nil && status != 0 { + err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0]) + } + + return b.Bytes(), err +} -- 2.7.4.3