rubik/patch/0015-rubik-set-container-engine-when-getting-container-id.patch
vegbir a1d5d5f63d rubik: support nri & optimize
Signed-off-by: vegbir <yangjiaqi16@huawei.com>
2024-09-23 03:35:19 +00:00

486 lines
18 KiB
Diff

From fe585042518f7c1a1da09028fe56177fa573ebbc Mon Sep 17 00:00:00 2001
From: vegbir <yangjiaqi16@huawei.com>
Date: Wed, 4 Sep 2024 10:35:50 +0800
Subject: [PATCH 2/2] rubik: set container engine when getting container id
reason:
bugfix: `/proc/self/cgroup` is not a sure way to get container engine because of the different cgroup path format.
So we have two approaches to get container engines: `proc/self/cgroup` or containerdID passed by k8s.
reconstruct: We abstract the cgroup driver, i.e., systemd &cgroupfs, to simplify the code for splicing cgroup paths.
Currently, only the path to the apiserver has been simplified, and the nri method needs to be reconstructed.
Signed-off-by: vegbir <yangjiaqi16@huawei.com>
---
pkg/core/typedef/cgroup/cgroupfs/driver.go | 37 +++++++++
pkg/core/typedef/cgroup/common.go | 5 --
pkg/core/typedef/cgroup/driver.go | 48 +++++++++++
pkg/core/typedef/cgroup/systemd/driver.go | 51 ++++++++++++
pkg/core/typedef/containerinfo.go | 34 ++++----
pkg/core/typedef/nrirawpod.go | 63 +--------------
pkg/core/typedef/rawpod.go | 92 ++++++----------------
7 files changed, 175 insertions(+), 155 deletions(-)
create mode 100644 pkg/core/typedef/cgroup/cgroupfs/driver.go
create mode 100644 pkg/core/typedef/cgroup/driver.go
create mode 100644 pkg/core/typedef/cgroup/systemd/driver.go
diff --git a/pkg/core/typedef/cgroup/cgroupfs/driver.go b/pkg/core/typedef/cgroup/cgroupfs/driver.go
new file mode 100644
index 0000000..4256deb
--- /dev/null
+++ b/pkg/core/typedef/cgroup/cgroupfs/driver.go
@@ -0,0 +1,37 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
+// rubik licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+// http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Jiaqi Yang
+// Date: 2024-09-03
+// Description: This file is used for cgroupfs driver
+
+package cgroupfs
+
+import (
+ "path/filepath"
+
+ "isula.org/rubik/pkg/common/constant"
+)
+
+const Name = "cgroupfs"
+
+type Driver struct{}
+
+func (d *Driver) Name() string {
+ return Name
+}
+
+func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string {
+ // When using cgroupfs as cgroup driver:
+ // 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3
+ // 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3
+ // 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3
+
+ return filepath.Join(constant.KubepodsCgroup, qosClass, constant.PodCgroupNamePrefix+id)
+}
diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go
index 668f951..5252e7d 100644
--- a/pkg/core/typedef/cgroup/common.go
+++ b/pkg/core/typedef/cgroup/common.go
@@ -27,11 +27,6 @@ import (
var rootDir = constant.DefaultCgroupRoot
var cgroupDriver = constant.CgroupDriverCgroupfs
-// SetCgroupDriver is the setter of global cgroup driver
-func SetCgroupDriver(driver string) {
- cgroupDriver = driver
-}
-
// GetCgroupDriver is the getter of global cgroup driver
func GetCgroupDriver() string {
return cgroupDriver
diff --git a/pkg/core/typedef/cgroup/driver.go b/pkg/core/typedef/cgroup/driver.go
new file mode 100644
index 0000000..b8cd4d5
--- /dev/null
+++ b/pkg/core/typedef/cgroup/driver.go
@@ -0,0 +1,48 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
+// rubik licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+// http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Jiaqi Yang
+// Date: 2024-09-03
+// Description: This file is used for cgroup driver
+
+package cgroup
+
+import (
+ "isula.org/rubik/pkg/core/typedef/cgroup/cgroupfs"
+ "isula.org/rubik/pkg/core/typedef/cgroup/systemd"
+)
+
+type Driver interface {
+ Name() string
+ ConcatPodCgroupPath(qosClass string, id string) string
+}
+
+var driver Driver = &cgroupfs.Driver{}
+
+// SetCgroupDriver is the setter of global cgroup driver
+func SetCgroupDriver(driverTyp string) {
+ cgroupDriver = driverTyp
+ switch driverTyp {
+ case systemd.Name:
+ driver = &systemd.Driver{}
+ case cgroupfs.Name:
+ driver = &cgroupfs.Driver{}
+ }
+}
+
+func Type() string {
+ return driver.Name()
+}
+func ConcatPodCgroupPath(qosClass, id string) string {
+ return driver.ConcatPodCgroupPath(qosClass, id)
+}
+
+func ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string {
+ return driver.ConcatPodCgroupPath(podCgroupPath, containerScope)
+}
diff --git a/pkg/core/typedef/cgroup/systemd/driver.go b/pkg/core/typedef/cgroup/systemd/driver.go
new file mode 100644
index 0000000..740b7ba
--- /dev/null
+++ b/pkg/core/typedef/cgroup/systemd/driver.go
@@ -0,0 +1,51 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
+// rubik licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+// http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Jiaqi Yang
+// Date: 2024-09-03
+// Description: This file is used for system cgroup driver
+
+package systemd
+
+import (
+ "path/filepath"
+ "strings"
+
+ "isula.org/rubik/pkg/common/constant"
+)
+
+const Name = "systemd"
+
+type Driver struct{}
+
+func (d *Driver) Name() string {
+ return Name
+}
+
+func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string {
+ // When using systemd as cgroup driver:
+ // 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
+ // 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
+ // 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/
+ const suffix = ".slice"
+ var (
+ prefix = constant.KubepodsCgroup
+ podPath = constant.KubepodsCgroup + suffix
+ )
+ if qosClass != "" {
+ podPath = filepath.Join(podPath, constant.KubepodsCgroup+"-"+qosClass+suffix)
+ prefix = strings.Join([]string{prefix, qosClass}, "-")
+ }
+ return filepath.Join(podPath,
+ strings.Join([]string{prefix, constant.PodCgroupNamePrefix + strings.Replace(id, "-", "_", -1) + suffix}, "-"))
+}
+
+func (d *Driver) ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string {
+ return filepath.Join(podCgroupPath, containerScope+".scope")
+}
diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go
index 841c800..ec04ed8 100644
--- a/pkg/core/typedef/containerinfo.go
+++ b/pkg/core/typedef/containerinfo.go
@@ -85,33 +85,30 @@ type ContainerInfo struct {
PodSandboxId string `json:"podisandid,omitempty"`
}
+func containerPath(id, podCgroupPath string) string {
+ if cgroup.Type() == constant.CgroupDriverSystemd {
+ return filepath.Join(podCgroupPath, containerEngineScopes[currentContainerEngines]+"-"+id+".scope")
+ }
+ // In the case of cgroupfs, the path of crio contains a special prefix
+ if containerEngineScopes[currentContainerEngines] == constant.ContainerEngineCrio {
+ return filepath.Join(podCgroupPath, constant.ContainerEngineCrio+"-"+id)
+ }
+ return filepath.Join(podCgroupPath, id)
+}
+
// NewContainerInfo creates a ContainerInfo instance
func NewContainerInfo(id, podCgroupPath string, rawContainer *RawContainer) *ContainerInfo {
- scopeName := containerEngineScopes[currentContainerEngines]
requests, limits := rawContainer.GetResourceMaps()
- var path string
- if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd {
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
- path = filepath.Join(podCgroupPath, scopeName+"-"+id+".scope")
- }
- } else {
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
- path = filepath.Join(podCgroupPath, id)
- case constant.ContainerEngineCrio:
- path = filepath.Join(podCgroupPath, scopeName+"-"+id)
- }
- }
return &ContainerInfo{
Name: rawContainer.status.Name,
ID: id,
- Hierarchy: cgroup.Hierarchy{Path: path},
+ Hierarchy: cgroup.Hierarchy{Path: containerPath(id, podCgroupPath)},
RequestResources: requests,
- LimitResources: limits}
+ LimitResources: limits,
+ }
}
-func fixContainerEngine(containerID string) {
+func getEngineFromContainerID(containerID string) {
for engine, prefix := range supportEnginesPrefixMap {
if strings.HasPrefix(containerID, prefix) {
currentContainerEngines = engine
@@ -119,7 +116,6 @@ func fixContainerEngine(containerID string) {
return
}
}
- currentContainerEngines = UNDEFINED
}
// DeepCopy returns deepcopy object.
diff --git a/pkg/core/typedef/nrirawpod.go b/pkg/core/typedef/nrirawpod.go
index 0749c8a..d060923 100644
--- a/pkg/core/typedef/nrirawpod.go
+++ b/pkg/core/typedef/nrirawpod.go
@@ -49,10 +49,6 @@ const (
fileMode os.FileMode = 0666
)
-func init() {
- setContainerEnginesOnce.Do(FixContainerEngine)
-}
-
// convert NRIRawPod structure to PodInfo structure
func (pod *NRIRawPod) ConvertNRIRawPod2PodInfo() *PodInfo {
if pod == nil {
@@ -87,7 +83,6 @@ func (pod *NRIRawPod) GetQosClass() string {
// get pod cgroupPath
func (pod *NRIRawPod) CgroupPath() string {
- var path string
id := pod.Uid
qosClassPath := ""
@@ -100,61 +95,7 @@ func (pod *NRIRawPod) CgroupPath() string {
default:
return ""
}
- /*
- Kubernetes defines three different pods:
- 1. Burstable: pod requests are less than the value of limits and not 0;
- 2. BestEffort: pod requests and limits are both 0;
- 3. Guaranteed: pod requests are equal to the value set by limits;
-
- When using cgroupfs as cgroup driver,
- 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3
- 2. The BestEffort path is in the form: kubepods/besteffort/pod34152897-dbaf-11ea-8cb9-0653660051c3
- 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3
-
- When using systemd as cgroup driver:
- 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
- 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
- 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/
- */
-
- if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd {
- if qosClassPath == "" {
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
- path = filepath.Join(
- constant.KubepodsCgroup+".slice",
- constant.KubepodsCgroup+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice",
- )
- }
- } else {
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
- path = filepath.Join(
- constant.KubepodsCgroup+".slice",
- constant.KubepodsCgroup+"-"+qosClassPath+".slice",
- pod.Linux.CgroupParent,
- )
-
- }
- }
- } else {
- if qosClassPath == "" {
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula, constant.ContainerEngineCrio:
- path = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id)
- }
- } else {
-
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula, constant.ContainerEngineCrio:
- path = filepath.Join(constant.KubepodsCgroup, qosClassPath, constant.PodCgroupNamePrefix+id)
- default:
- path = ""
- }
- }
-
- }
- return path
+ return cgroup.ConcatPodCgroupPath(qosClassPath, id)
}
// get pod running state
@@ -437,7 +378,7 @@ func (container *NRIRawContainer) GetResourceMaps() (ResourceMap, ResourceMap) {
}
// get current container engine
-func FixContainerEngine() {
+func getEngineFromCgroup() {
file, err := os.OpenFile(procSelfCgroupFile, os.O_RDONLY, fileMode)
if err != nil {
return
diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go
index b67126a..b653c71 100644
--- a/pkg/core/typedef/rawpod.go
+++ b/pkg/core/typedef/rawpod.go
@@ -16,13 +16,11 @@ package typedef
import (
"fmt"
- "path/filepath"
"strings"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
- "isula.org/rubik/pkg/common/constant"
"isula.org/rubik/pkg/core/typedef/cgroup"
)
@@ -86,83 +84,30 @@ func (pod *RawPod) ID() string {
return string(pod.UID)
}
+// Kubernetes defines three different pods:
+// 1. Burstable: pod requests are less than the value of limits and not 0;
+// 2. BestEffort: pod requests and limits are both 0;
+// 3. Guaranteed: pod requests are equal to the value set by limits;
+var k8sQosClass = map[corev1.PodQOSClass]string{
+ corev1.PodQOSGuaranteed: "",
+ corev1.PodQOSBurstable: strings.ToLower(string(corev1.PodQOSBurstable)),
+ corev1.PodQOSBestEffort: strings.ToLower(string(corev1.PodQOSBestEffort)),
+}
+
// CgroupPath returns cgroup path of raw pod
// handle different combinations of cgroupdriver and pod qos and container runtime
-func (pod *RawPod) CgroupPath() string {
+func (pod *RawPod) CgroupPath() (res string) {
id := string(pod.UID)
if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" {
id = configHash
}
- qosClassPath := ""
- switch pod.Status.QOSClass {
- case corev1.PodQOSGuaranteed:
- case corev1.PodQOSBurstable:
- qosClassPath = strings.ToLower(string(corev1.PodQOSBurstable))
- case corev1.PodQOSBestEffort:
- qosClassPath = strings.ToLower(string(corev1.PodQOSBestEffort))
- default:
+ qosPrefix, existed := k8sQosClass[pod.Status.QOSClass]
+ if !existed {
+ fmt.Printf("unsupported qos class: %v", pod.Status.QOSClass)
return ""
}
-
- /*
- Kubernetes defines three different pods:
- 1. Burstable: pod requests are less than the value of limits and not 0;
- 2. BestEffort: pod requests and limits are both 0;
- 3. Guaranteed: pod requests are equal to the value set by limits;
-
- When using cgroupfs as cgroup driver:
- 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3
- 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3
- 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3
-
- When using systemd as cgroup driver:
- 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
- 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
- 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/
- */
-
- if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd {
- if qosClassPath == "" {
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
- return filepath.Join(
- constant.KubepodsCgroup+".slice",
- constant.KubepodsCgroup+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice",
- )
- default:
- return ""
- }
- } else {
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
- return filepath.Join(
- constant.KubepodsCgroup+".slice",
- constant.KubepodsCgroup+"-"+qosClassPath+".slice",
- constant.KubepodsCgroup+"-"+qosClassPath+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice",
- )
- default:
- return ""
- }
-
- }
- } else {
- if qosClassPath == "" {
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineDocker, constant.ContainerEngineContainerd, constant.ContainerEngineIsula, constant.ContainerEngineCrio:
- return filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id)
- default:
- return ""
- }
- } else {
- switch containerEngineScopes[currentContainerEngines] {
- case constant.ContainerEngineDocker, constant.ContainerEngineContainerd, constant.ContainerEngineIsula, constant.ContainerEngineCrio:
- return filepath.Join(constant.KubepodsCgroup, qosClassPath, constant.PodCgroupNamePrefix+id)
- default:
- return ""
- }
- }
- }
+ return cgroup.ConcatPodCgroupPath(qosPrefix, id)
}
// ListRawContainers returns all RawContainers in the RawPod
@@ -221,6 +166,13 @@ func (cont *RawContainer) GetRealContainerID() (string, error) {
So we don't consider the case of midway container engine changes
`fixContainerEngine` is only executed when `getRealContainerID` is called for the first time
*/
+ setContainerEnginesOnce.Do(func() {
+ getEngineFromCgroup()
+ _, exist := supportEnginesPrefixMap[currentContainerEngines]
+ if !exist {
+ getEngineFromContainerID(cont.status.ContainerID)
+ }
+ })
if !currentContainerEngines.Support(cont) {
return "", fmt.Errorf("unsupported container engine: %v", cont.status.ContainerID)
--
2.46.0