packages/kata-containers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
kata-containers/runtime/patches/0056-kata_runtime-support-the-blkio-in-host-cgroups.patch

319 lines
10 KiB
Diff
Raw Normal View History

kata-containers: move all kata related source repo into one repo kata-containers reason: in order to make manage kata-containers related source code more easy, we decide to move all kata related source repo into kata-containers repo. Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
2020-12-22 12:23:42 +08:00
From f4b899b933a3a30fc378ceb4d8855778cd783d9a Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Fri, 18 Sep 2020 16:54:11 +0800
Subject: [PATCH 4/5] kata_runtime: support the blkio in host cgroups
reason: support the blkio in host cgroups, run with paras
--annotation io.kubernetes.docker.type=podsandbox --annotation io.katacontainers.blkio_cgroup=
'{"blkiocgroup":[{"path":"/dev/sda","limits":[{"type":"throttle_read_bps","value":400}]}]}'
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
vendor/github.com/containerd/cgroups/blkio.go | 6 ++
virtcontainers/cgroups.go | 102 ++++++++++++++++++++++++++
virtcontainers/pkg/annotations/annotations.go | 3 +
virtcontainers/pkg/oci/utils.go | 50 ++++++++++++-
virtcontainers/sandbox.go | 6 ++
virtcontainers/utils/utils.go | 22 ++++++
6 files changed, 186 insertions(+), 3 deletions(-)
diff --git a/vendor/github.com/containerd/cgroups/blkio.go b/vendor/github.com/containerd/cgroups/blkio.go
index 7c498de..485ff7b 100644
--- a/vendor/github.com/containerd/cgroups/blkio.go
+++ b/vendor/github.com/containerd/cgroups/blkio.go
@@ -23,6 +23,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
+ "reflect"
"strconv"
"strings"
@@ -55,6 +56,11 @@ func (b *blkioController) Create(path string, resources *specs.LinuxResources) e
return nil
}
for _, t := range createBlkioSettings(resources.BlockIO) {
+ ptr := reflect.ValueOf(t.value)
+ if ptr.Kind() == reflect.Ptr && ptr.IsNil() {
+ continue
+ }
+
if t.value != nil {
if err := ioutil.WriteFile(
filepath.Join(b.Path(path), fmt.Sprintf("blkio.%s", t.name)),
diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go
index 65d2001..e8c5a7b 100644
--- a/virtcontainers/cgroups.go
+++ b/virtcontainers/cgroups.go
@@ -9,6 +9,7 @@ package virtcontainers
import (
"bufio"
"context"
+ "encoding/json"
"fmt"
"os"
"path/filepath"
@@ -16,6 +17,8 @@ import (
"strings"
"github.com/containerd/cgroups"
+ "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
+ "github.com/kata-containers/runtime/virtcontainers/utils"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
)
@@ -32,8 +35,46 @@ const (
cgroupKataPath = "/kata/"
vcpuCgroupName = "vcpu"
emulatorCgroupName = "emulator"
+
+ // BlkioThrottleReadBps is the key to fetch throttle_read_bps
+ BlkioThrottleReadBps = "throttle_read_bps"
+
+ // BlkioThrottleWriteBps is the key to fetch throttle_write_bps
+ BlkioThrottleWriteBps = "throttle_write_bps"
+
+ // BlkioThrottleReadIOPS is the key to fetch throttle_read_iops
+ BlkioThrottleReadIOPS = "throttle_read_iops"
+
+ // BlkioThrottleWriteIOPS is the key to fetch throttle_write_iops
+ BlkioThrottleWriteIOPS = "throttle_write_iops"
+
+ // BlkioWeight is the key to fetch blkio_weight
+ BlkioWeight = "blkio_weight"
+
+ // BlkioLeafWeight is the key to fetch blkio_leaf_weight
+ BlkioLeafWeight = "blkio_leaf_weight"
)
+// BlkioCgroup for Linux cgroup 'blkio' data exchange
+type BlkioCgroup struct {
+ // Items specifies per cgroup values
+ Items []BlockIOCgroupItem `json:"blkiocgroup,omitempty"`
+}
+
+type BlockIOCgroupItem struct {
+ // Path represent path of blkio device
+ Path string `json:"path,omitempty"`
+ // Limits specifies the blkio type and value
+ Limits []IOLimit `json:"limits,omitempty"`
+}
+
+type IOLimit struct {
+ // Type specifies IO type
+ Type string `json:"type,omitempty"`
+ // Value specifies rate or weight value
+ Value uint64 `json:"value,omitempty"`
+}
+
var cgroupsLoadFunc = cgroups.Load
var cgroupsNewFunc = cgroups.New
@@ -372,3 +413,64 @@ func isInSlice(i int, s []int) bool {
}
return false
}
+
+func (s *Sandbox) blockIOResource() *specs.LinuxBlockIO {
+ value, ok := s.config.Annotations[annotations.BlkioCgroupTypeKey]
+ if !ok {
+ return nil
+ }
+
+ var blkioCgroupParse BlkioCgroup
+ var linuxBlkio specs.LinuxBlockIO
+ if err := json.Unmarshal([]byte(value), &blkioCgroupParse); err != nil {
+ s.Logger().Errorf("blkio_cgroup Unmarshal error:%v", err)
+ return nil
+ }
+
+ for _, item := range blkioCgroupParse.Items {
+ if item.Limits == nil {
+ s.Logger().Errorf("%v:limits have none data", item)
+ return nil
+ }
+ for _, limit := range item.Limits {
+ if item.Path != "" {
+ major, minor, err := utils.GetDeviceByPath(item.Path)
+ if err != nil {
+ s.Logger().Errorf("failed to find major and minor of device %s: %v", item.Path, err)
+ return nil
+ }
+ td := specs.LinuxThrottleDevice{
+ Rate: limit.Value,
+ }
+ td.Major = major
+ td.Minor = minor
+ switch limit.Type {
+ case BlkioThrottleReadBps:
+ linuxBlkio.ThrottleReadBpsDevice = append(linuxBlkio.ThrottleReadBpsDevice, td)
+ case BlkioThrottleWriteBps:
+ linuxBlkio.ThrottleWriteBpsDevice = append(linuxBlkio.ThrottleWriteBpsDevice, td)
+ case BlkioThrottleReadIOPS:
+ linuxBlkio.ThrottleReadIOPSDevice = append(linuxBlkio.ThrottleReadIOPSDevice, td)
+ case BlkioThrottleWriteIOPS:
+ linuxBlkio.ThrottleWriteIOPSDevice = append(linuxBlkio.ThrottleWriteIOPSDevice, td)
+ default:
+ s.Logger().Errorf("the type of throtlle device：%s is not surpport", limit.Type)
+ return nil
+ }
+ } else {
+ switch limit.Type {
+ case BlkioWeight:
+ weightUint16 := uint16(limit.Value)
+ linuxBlkio.Weight = &weightUint16
+ case BlkioLeafWeight:
+ weightLeafUint16 := uint16(limit.Value)
+ linuxBlkio.LeafWeight = &weightLeafUint16
+ default:
+ s.Logger().Errorf("the type:%s is not one of the supported types of blkio_weight and blkio_leaf_weight", limit.Type)
+ return nil
+ }
+ }
+ }
+ }
+ return &linuxBlkio
+}
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
index 96c4ef2..c35993e 100644
--- a/virtcontainers/pkg/annotations/annotations.go
+++ b/virtcontainers/pkg/annotations/annotations.go
@@ -21,6 +21,9 @@ const (
ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type"
SandboxConfigPathKey = kataAnnotationsPrefix + "config_path"
+
+ // BlkioCgroupTypeKey is the annotation key to fetch sandbox blkio cgroup values
+ BlkioCgroupTypeKey = kataAnnotationsPrefix + "blkio_cgroup"
)
// Annotations related to Hypervisor configuration
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index e8ef41b..643753b 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -38,9 +38,10 @@ type annotationContainerType struct {
type annotationHandler func(value string) error
var annotationHandlerList = map[string]annotationHandler{
- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
- vcAnnotations.StaticMemTypeKey: validateSandboxMem,
- vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
+ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS,
+ vcAnnotations.BlkioCgroupTypeKey: validateBlkioCgroup,
}
var (
@@ -1076,6 +1077,7 @@ func validateOtherSandboxAnnotations(annotation, value string) error {
// addOtherSandboxAnnotation add self defined annotation for sandbox
func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
otherSandboxAnnotationsKey := []string{
+ vcAnnotations.BlkioCgroupTypeKey,
vcAnnotations.StaticCPUTypeKey,
vcAnnotations.StaticMemTypeKey,
}
@@ -1137,6 +1139,48 @@ func validateSandboxDNS(value string) error {
return nil
}
+func validateBlkioCgroup(value string) error {
+ var linuxBlkioCgroup vc.BlkioCgroup
+ if err := json.Unmarshal([]byte(value), &linuxBlkioCgroup); err != nil {
+ return fmt.Errorf("blkio_cgroup Unmarshal error:%v, value is %s", err, value)
+ }
+
+ if linuxBlkioCgroup.Items == nil {
+ return fmt.Errorf("BlkioCgroup:%v fetch none data", linuxBlkioCgroup)
+ }
+
+ // pathNull used to judge path for blkio.weight/blkio.leaf_weight only once
+ pathNull := false
+ for _, item := range linuxBlkioCgroup.Items {
+ if pathNull {
+ return fmt.Errorf("too many paths for blkio.weight/blkio.leaf_weight")
+ }
+ if item.Path == "" && !pathNull {
+ pathNull = true
+ if len(item.Limits) > 2 || len(item.Limits) == 0 {
+ return fmt.Errorf("the format of values for blkio.weight or blkio.leaf_weight is wrong")
+ }
+
+ for _, ioLimit := range item.Limits {
+ switch ioLimit.Type {
+ case vc.BlkioWeight:
+ // Blkio.weight, between 10 and 1000
+ if ioLimit.Value < 10 || ioLimit.Value > 1000 {
+ return fmt.Errorf("blkio.weight:%v must be between 10 and 1000", ioLimit.Value)
+ }
+ case vc.BlkioLeafWeight:
+ return fmt.Errorf("the blkio.leaf_weight is not supported now")
+ default:
+ return fmt.Errorf("the type of blkio device：%s is not surpport", ioLimit.Type)
+ }
+ }
+ } else if _, _, err := utils.GetDeviceByPath(item.Path); err != nil {
+ return fmt.Errorf("failed to find major and minor of device %s: %v", item.Path, err)
+ }
+ }
+ return nil
+}
+
func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) {
if s == nil {
return "", fmt.Errorf("spec is nil")
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index ca4e700..9284f99 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -2422,6 +2422,12 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error {
}
// limit blkio resource to "<path>"
+ blkioResources := specs.LinuxResources{
+ BlockIO: s.blockIOResource(),
+ }
+ if err := applyResourceLimit(&blkioResources, vcpuCgroupPath); err != nil {
+ return err
+ }
// limit files resource
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
index 36ac67a..d4dad40 100644
--- a/virtcontainers/utils/utils.go
+++ b/virtcontainers/utils/utils.go
@@ -370,6 +370,28 @@ func GetPhysicalCPUNumber() int {
return cpuNum
}
+// GetDeviceByPath returns the device No.
+func GetDeviceByPath(path string) (int64, int64, error) {
+ if path == "" {
+ return -1, -1, fmt.Errorf("Path cannot be empty")
+ }
+
+ path, err := filepath.EvalSymlinks(path)
+ if err != nil {
+ return -1, -1, err
+ }
+
+ stat := syscall.Stat_t{}
+ err = syscall.Stat(path, &stat)
+ if err != nil {
+ return -1, -1, err
+ }
+
+ major := int64(stat.Rdev / 256)
+ minor := int64(stat.Rdev % 256)
+ return major, minor, nil
+}
+
func RoundVCPUNumber(value string) (int, error) {
cpuNum, err := strconv.ParseFloat(value, 64)
if err != nil || cpuNum < minCPUs {
--
1.8.3.1
Reference in New Issue Copy Permalink