219 lines
6.5 KiB
Diff
219 lines
6.5 KiB
Diff
|
|
From 006bc6d0a9e0c233d0d14de53de0b18799c67081 Mon Sep 17 00:00:00 2001
|
||
|
|
From: xiadanni <xiadanni@huawei.com>
|
||
|
|
Date: Fri, 15 Feb 2019 06:00:52 +0800
|
||
|
|
Subject: [PATCH 23/27] containerd: set create and exec timeout
|
||
|
|
|
||
|
|
reason:set create and exec timeout to avild block when command failed
|
||
|
|
|
||
|
|
Change-Id: I6bc55f4ccc953bdc1d926ab940f0900811d68760
|
||
|
|
Signed-off-by: xiadanni <xiadanni@huawei.com>
|
||
|
|
---
|
||
|
|
hack/containerd.spec | 2 +-
|
||
|
|
runtime/v1/shim/reaper.go | 50 +++++++++++++++++++++++++
|
||
|
|
runtime/v2/shim/reaper_unix.go | 4 ++
|
||
|
|
vendor/github.com/containerd/go-runc/monitor.go | 6 +++
|
||
|
|
vendor/github.com/containerd/go-runc/runc.go | 31 +++++++++++++--
|
||
|
|
5 files changed, 88 insertions(+), 5 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/hack/containerd.spec b/hack/containerd.spec
|
||
|
|
index f8d9084..f39c57a 100644
|
||
|
|
--- a/hack/containerd.spec
|
||
|
|
+++ b/hack/containerd.spec
|
||
|
|
@@ -3,7 +3,7 @@
|
||
|
|
Version: 1.2.0
|
||
|
|
|
||
|
|
Name: containerd
|
||
|
|
-Release: 4%{?dist}
|
||
|
|
+Release: 5%{?dist}
|
||
|
|
Summary: An industry-standard container runtime
|
||
|
|
License: ASL 2.0
|
||
|
|
URL: https://containerd.io
|
||
|
|
diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go
|
||
|
|
index 10d5c30..a2b90fe 100644
|
||
|
|
--- a/runtime/v1/shim/reaper.go
|
||
|
|
+++ b/runtime/v1/shim/reaper.go
|
||
|
|
@@ -19,8 +19,13 @@
|
||
|
|
package shim
|
||
|
|
|
||
|
|
import (
|
||
|
|
+ "io/ioutil"
|
||
|
|
"os/exec"
|
||
|
|
+ "path/filepath"
|
||
|
|
+ "strconv"
|
||
|
|
+ "strings"
|
||
|
|
"sync"
|
||
|
|
+ "syscall"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/containerd/containerd/sys"
|
||
|
|
@@ -100,6 +105,34 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) {
|
||
|
|
return -1, ErrNoSuchProcess
|
||
|
|
}
|
||
|
|
|
||
|
|
+// WaitTimeout is used to skip the blocked command and kill the left process.
|
||
|
|
+func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, error) {
|
||
|
|
+ sch := make(chan int)
|
||
|
|
+ ech := make(chan error)
|
||
|
|
+ go func() {
|
||
|
|
+ for e := range ec {
|
||
|
|
+ if e.Pid == c.Process.Pid {
|
||
|
|
+ // make sure we flush all IO
|
||
|
|
+ c.Wait()
|
||
|
|
+ m.Unsubscribe(ec)
|
||
|
|
+ sch <- e.Status
|
||
|
|
+ return
|
||
|
|
+ }
|
||
|
|
+ }
|
||
|
|
+ }()
|
||
|
|
+ select {
|
||
|
|
+ case <-time.After(time.Duration(sec) * time.Second):
|
||
|
|
+ if SameProcess(c, c.Process.Pid) {
|
||
|
|
+ syscall.Kill(c.Process.Pid, syscall.SIGKILL)
|
||
|
|
+ }
|
||
|
|
+ return 0, errors.Errorf("timeout %ds for cmd(pid= %d): %s, %s", sec, c.Process.Pid, c.Path, c.Args)
|
||
|
|
+ case status := <-sch:
|
||
|
|
+ return status, nil
|
||
|
|
+ case err := <-ech:
|
||
|
|
+ return -1, err
|
||
|
|
+ }
|
||
|
|
+}
|
||
|
|
+
|
||
|
|
// Subscribe to process exit changes
|
||
|
|
func (m *Monitor) Subscribe() chan runc.Exit {
|
||
|
|
c := make(chan runc.Exit, bufferSize)
|
||
|
|
@@ -116,3 +149,20 @@ func (m *Monitor) Unsubscribe(c chan runc.Exit) {
|
||
|
|
close(c)
|
||
|
|
m.Unlock()
|
||
|
|
}
|
||
|
|
+
|
||
|
|
+func SameProcess(cmd *exec.Cmd, pid int) bool {
|
||
|
|
+ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline"))
|
||
|
|
+ if err != nil {
|
||
|
|
+ return false
|
||
|
|
+ }
|
||
|
|
+ for i := range bytes {
|
||
|
|
+ if bytes[i] == 0 {
|
||
|
|
+ bytes[i] = 32
|
||
|
|
+ }
|
||
|
|
+ }
|
||
|
|
+ cmdline := string(bytes)
|
||
|
|
+ if strings.EqualFold(cmdline, strings.Join(cmd.Args, " ")+" ") {
|
||
|
|
+ return true
|
||
|
|
+ }
|
||
|
|
+ return false
|
||
|
|
+}
|
||
|
|
diff --git a/runtime/v2/shim/reaper_unix.go b/runtime/v2/shim/reaper_unix.go
|
||
|
|
index 10d5c30..8bd7dd1 100644
|
||
|
|
--- a/runtime/v2/shim/reaper_unix.go
|
||
|
|
+++ b/runtime/v2/shim/reaper_unix.go
|
||
|
|
@@ -100,6 +100,10 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) {
|
||
|
|
return -1, ErrNoSuchProcess
|
||
|
|
}
|
||
|
|
|
||
|
|
+func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, error) {
|
||
|
|
+ return m.Wait(c, ec)
|
||
|
|
+}
|
||
|
|
+
|
||
|
|
// Subscribe to process exit changes
|
||
|
|
func (m *Monitor) Subscribe() chan runc.Exit {
|
||
|
|
c := make(chan runc.Exit, bufferSize)
|
||
|
|
diff --git a/vendor/github.com/containerd/go-runc/monitor.go b/vendor/github.com/containerd/go-runc/monitor.go
|
||
|
|
index ff06a3f..2c184d2 100644
|
||
|
|
--- a/vendor/github.com/containerd/go-runc/monitor.go
|
||
|
|
+++ b/vendor/github.com/containerd/go-runc/monitor.go
|
||
|
|
@@ -40,6 +40,7 @@ type Exit struct {
|
||
|
|
type ProcessMonitor interface {
|
||
|
|
Start(*exec.Cmd) (chan Exit, error)
|
||
|
|
Wait(*exec.Cmd, chan Exit) (int, error)
|
||
|
|
+ WaitTimeout(*exec.Cmd, chan Exit, int64) (int, error)
|
||
|
|
}
|
||
|
|
|
||
|
|
type defaultMonitor struct {
|
||
|
|
@@ -74,3 +75,8 @@ func (m *defaultMonitor) Wait(c *exec.Cmd, ec chan Exit) (int, error) {
|
||
|
|
e := <-ec
|
||
|
|
return e.Status, nil
|
||
|
|
}
|
||
|
|
+
|
||
|
|
+func (m *defaultMonitor) WaitTimeout(c *exec.Cmd, ec chan Exit, sec int64) (int, error) {
|
||
|
|
+ e := <-ec
|
||
|
|
+ return e.Status, nil
|
||
|
|
+}
|
||
|
|
\ No newline at end of file
|
||
|
|
diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go
|
||
|
|
index e688881..fc64e8a 100644
|
||
|
|
--- a/vendor/github.com/containerd/go-runc/runc.go
|
||
|
|
+++ b/vendor/github.com/containerd/go-runc/runc.go
|
||
|
|
@@ -52,6 +52,8 @@ const (
|
||
|
|
Text Format = "text"
|
||
|
|
// DefaultCommand is the default command for Runc
|
||
|
|
DefaultCommand = "runc"
|
||
|
|
+ execTimeout = 30
|
||
|
|
+ createTimeout = 120
|
||
|
|
)
|
||
|
|
|
||
|
|
// Runc is the client to the runc cli
|
||
|
|
@@ -155,7 +157,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp
|
||
|
|
cmd.ExtraFiles = opts.ExtraFiles
|
||
|
|
|
||
|
|
if cmd.Stdout == nil && cmd.Stderr == nil {
|
||
|
|
- data, err := cmdOutput(cmd, true)
|
||
|
|
+ data, err := cmdOutputTimeout(cmd, true, createTimeout)
|
||
|
|
if err != nil {
|
||
|
|
return fmt.Errorf("%s: %s", err, data)
|
||
|
|
}
|
||
|
|
@@ -172,7 +174,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
- status, err := Monitor.Wait(cmd, ec)
|
||
|
|
+ status, err := Monitor.WaitTimeout(cmd, ec, createTimeout)
|
||
|
|
if err == nil && status != 0 {
|
||
|
|
err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
|
||
|
|
}
|
||
|
|
@@ -234,7 +236,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts
|
||
|
|
opts.Set(cmd)
|
||
|
|
}
|
||
|
|
if cmd.Stdout == nil && cmd.Stderr == nil {
|
||
|
|
- data, err := cmdOutput(cmd, true)
|
||
|
|
+ data, err := cmdOutputTimeout(cmd, true, execTimeout)
|
||
|
|
if err != nil {
|
||
|
|
return fmt.Errorf("%s: %s", err, data)
|
||
|
|
}
|
||
|
|
@@ -251,7 +253,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
- status, err := Monitor.Wait(cmd, ec)
|
||
|
|
+ status, err := Monitor.WaitTimeout(cmd, ec, execTimeout)
|
||
|
|
if err == nil && status != 0 {
|
||
|
|
err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
|
||
|
|
}
|
||
|
|
@@ -707,3 +709,24 @@ func cmdOutput(cmd *exec.Cmd, combined bool) ([]byte, error) {
|
||
|
|
|
||
|
|
return b.Bytes(), err
|
||
|
|
}
|
||
|
|
+
|
||
|
|
+func cmdOutputTimeout(cmd *exec.Cmd, combined bool, timeout int64) ([]byte, error) {
|
||
|
|
+ b := getBuf()
|
||
|
|
+ defer putBuf(b)
|
||
|
|
+
|
||
|
|
+ cmd.Stdout = b
|
||
|
|
+ if combined {
|
||
|
|
+ cmd.Stderr = b
|
||
|
|
+ }
|
||
|
|
+ ec, err := Monitor.Start(cmd)
|
||
|
|
+ if err != nil {
|
||
|
|
+ return nil, err
|
||
|
|
+ }
|
||
|
|
+
|
||
|
|
+ status, err := Monitor.WaitTimeout(cmd, ec, timeout)
|
||
|
|
+ if err == nil && status != 0 {
|
||
|
|
+ err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0])
|
||
|
|
+ }
|
||
|
|
+
|
||
|
|
+ return b.Bytes(), err
|
||
|
|
+}
|
||
|
|
--
|
||
|
|
2.7.4.3
|
||
|
|
|