From fe585042518f7c1a1da09028fe56177fa573ebbc Mon Sep 17 00:00:00 2001
From: vegbir <yangjiaqi16@huawei.com>
Date: Wed, 4 Sep 2024 10:35:50 +0800
Subject: [PATCH 2/2] rubik: set container engine when getting container id

reason:
bugfix: `/proc/self/cgroup` is not a sure way to get container engine because of the different cgroup path format.
So we have two approaches to get container engines: `proc/self/cgroup` or containerdID passed by k8s.
reconstruct: We abstract the cgroup driver, i.e., systemd &cgroupfs, to simplify the code for splicing cgroup paths.
Currently, only the path to the apiserver has been simplified, and the nri method needs to be reconstructed.

Signed-off-by: vegbir <yangjiaqi16@huawei.com>
---
 pkg/core/typedef/cgroup/cgroupfs/driver.go | 37 +++++++++
 pkg/core/typedef/cgroup/common.go          |  5 --
 pkg/core/typedef/cgroup/driver.go          | 48 +++++++++++
 pkg/core/typedef/cgroup/systemd/driver.go  | 51 ++++++++++++
 pkg/core/typedef/containerinfo.go          | 34 ++++----
 pkg/core/typedef/nrirawpod.go              | 63 +--------------
 pkg/core/typedef/rawpod.go                 | 92 ++++++----------------
 7 files changed, 175 insertions(+), 155 deletions(-)
 create mode 100644 pkg/core/typedef/cgroup/cgroupfs/driver.go
 create mode 100644 pkg/core/typedef/cgroup/driver.go
 create mode 100644 pkg/core/typedef/cgroup/systemd/driver.go

diff --git a/pkg/core/typedef/cgroup/cgroupfs/driver.go b/pkg/core/typedef/cgroup/cgroupfs/driver.go
new file mode 100644
index 0000000..4256deb
--- /dev/null
+++ b/pkg/core/typedef/cgroup/cgroupfs/driver.go
@@ -0,0 +1,37 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
+// rubik licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+//     http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Jiaqi Yang
+// Date: 2024-09-03
+// Description: This file is used for cgroupfs driver
+
+package cgroupfs
+
+import (
+	"path/filepath"
+
+	"isula.org/rubik/pkg/common/constant"
+)
+
+const Name = "cgroupfs"
+
+type Driver struct{}
+
+func (d *Driver) Name() string {
+	return Name
+}
+
+func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string {
+	// When using cgroupfs as cgroup driver:
+	// 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3
+	// 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3
+	// 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3
+
+	return filepath.Join(constant.KubepodsCgroup, qosClass, constant.PodCgroupNamePrefix+id)
+}
diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go
index 668f951..5252e7d 100644
--- a/pkg/core/typedef/cgroup/common.go
+++ b/pkg/core/typedef/cgroup/common.go
@@ -27,11 +27,6 @@ import (
 var rootDir = constant.DefaultCgroupRoot
 var cgroupDriver = constant.CgroupDriverCgroupfs
 
-// SetCgroupDriver is the setter of global cgroup driver
-func SetCgroupDriver(driver string) {
-	cgroupDriver = driver
-}
-
 // GetCgroupDriver is the getter of global cgroup driver
 func GetCgroupDriver() string {
 	return cgroupDriver
diff --git a/pkg/core/typedef/cgroup/driver.go b/pkg/core/typedef/cgroup/driver.go
new file mode 100644
index 0000000..b8cd4d5
--- /dev/null
+++ b/pkg/core/typedef/cgroup/driver.go
@@ -0,0 +1,48 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
+// rubik licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+//     http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Jiaqi Yang
+// Date: 2024-09-03
+// Description: This file is used for cgroup driver
+
+package cgroup
+
+import (
+	"isula.org/rubik/pkg/core/typedef/cgroup/cgroupfs"
+	"isula.org/rubik/pkg/core/typedef/cgroup/systemd"
+)
+
+type Driver interface {
+	Name() string
+	ConcatPodCgroupPath(qosClass string, id string) string
+}
+
+var driver Driver = &cgroupfs.Driver{}
+
+// SetCgroupDriver is the setter of global cgroup driver
+func SetCgroupDriver(driverTyp string) {
+	cgroupDriver = driverTyp
+	switch driverTyp {
+	case systemd.Name:
+		driver = &systemd.Driver{}
+	case cgroupfs.Name:
+		driver = &cgroupfs.Driver{}
+	}
+}
+
+func Type() string {
+	return driver.Name()
+}
+func ConcatPodCgroupPath(qosClass, id string) string {
+	return driver.ConcatPodCgroupPath(qosClass, id)
+}
+
+func ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string {
+	return driver.ConcatPodCgroupPath(podCgroupPath, containerScope)
+}
diff --git a/pkg/core/typedef/cgroup/systemd/driver.go b/pkg/core/typedef/cgroup/systemd/driver.go
new file mode 100644
index 0000000..740b7ba
--- /dev/null
+++ b/pkg/core/typedef/cgroup/systemd/driver.go
@@ -0,0 +1,51 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
+// rubik licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+//     http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Jiaqi Yang
+// Date: 2024-09-03
+// Description: This file is used for system cgroup driver
+
+package systemd
+
+import (
+	"path/filepath"
+	"strings"
+
+	"isula.org/rubik/pkg/common/constant"
+)
+
+const Name = "systemd"
+
+type Driver struct{}
+
+func (d *Driver) Name() string {
+	return Name
+}
+
+func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string {
+	// When using systemd as cgroup driver:
+	// 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
+	// 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
+	// 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/
+	const suffix = ".slice"
+	var (
+		prefix  = constant.KubepodsCgroup
+		podPath = constant.KubepodsCgroup + suffix
+	)
+	if qosClass != "" {
+		podPath = filepath.Join(podPath, constant.KubepodsCgroup+"-"+qosClass+suffix)
+		prefix = strings.Join([]string{prefix, qosClass}, "-")
+	}
+	return filepath.Join(podPath,
+		strings.Join([]string{prefix, constant.PodCgroupNamePrefix + strings.Replace(id, "-", "_", -1) + suffix}, "-"))
+}
+
+func (d *Driver) ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string {
+	return filepath.Join(podCgroupPath, containerScope+".scope")
+}
diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go
index 841c800..ec04ed8 100644
--- a/pkg/core/typedef/containerinfo.go
+++ b/pkg/core/typedef/containerinfo.go
@@ -85,33 +85,30 @@ type ContainerInfo struct {
 	PodSandboxId string `json:"podisandid,omitempty"`
 }
 
+func containerPath(id, podCgroupPath string) string {
+	if cgroup.Type() == constant.CgroupDriverSystemd {
+		return filepath.Join(podCgroupPath, containerEngineScopes[currentContainerEngines]+"-"+id+".scope")
+	}
+	// In the case of cgroupfs, the path of crio contains a special prefix
+	if containerEngineScopes[currentContainerEngines] == constant.ContainerEngineCrio {
+		return filepath.Join(podCgroupPath, constant.ContainerEngineCrio+"-"+id)
+	}
+	return filepath.Join(podCgroupPath, id)
+}
+
 // NewContainerInfo creates a ContainerInfo instance
 func NewContainerInfo(id, podCgroupPath string, rawContainer *RawContainer) *ContainerInfo {
-	scopeName := containerEngineScopes[currentContainerEngines]
 	requests, limits := rawContainer.GetResourceMaps()
-	var path string
-	if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd {
-		switch containerEngineScopes[currentContainerEngines] {
-		case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
-			path = filepath.Join(podCgroupPath, scopeName+"-"+id+".scope")
-		}
-	} else {
-		switch containerEngineScopes[currentContainerEngines] {
-		case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
-			path = filepath.Join(podCgroupPath, id)
-		case constant.ContainerEngineCrio:
-			path = filepath.Join(podCgroupPath, scopeName+"-"+id)
-		}
-	}
 	return &ContainerInfo{
 		Name:             rawContainer.status.Name,
 		ID:               id,
-		Hierarchy:        cgroup.Hierarchy{Path: path},
+		Hierarchy:        cgroup.Hierarchy{Path: containerPath(id, podCgroupPath)},
 		RequestResources: requests,
-		LimitResources:   limits}
+		LimitResources:   limits,
+	}
 }
 
-func fixContainerEngine(containerID string) {
+func getEngineFromContainerID(containerID string) {
 	for engine, prefix := range supportEnginesPrefixMap {
 		if strings.HasPrefix(containerID, prefix) {
 			currentContainerEngines = engine
@@ -119,7 +116,6 @@ func fixContainerEngine(containerID string) {
 			return
 		}
 	}
-	currentContainerEngines = UNDEFINED
 }
 
 // DeepCopy returns deepcopy object.
diff --git a/pkg/core/typedef/nrirawpod.go b/pkg/core/typedef/nrirawpod.go
index 0749c8a..d060923 100644
--- a/pkg/core/typedef/nrirawpod.go
+++ b/pkg/core/typedef/nrirawpod.go
@@ -49,10 +49,6 @@ const (
 	fileMode      os.FileMode = 0666
 )
 
-func init() {
-	setContainerEnginesOnce.Do(FixContainerEngine)
-}
-
 // convert NRIRawPod structure to PodInfo structure
 func (pod *NRIRawPod) ConvertNRIRawPod2PodInfo() *PodInfo {
 	if pod == nil {
@@ -87,7 +83,6 @@ func (pod *NRIRawPod) GetQosClass() string {
 
 // get pod cgroupPath
 func (pod *NRIRawPod) CgroupPath() string {
-	var path string
 	id := pod.Uid
 
 	qosClassPath := ""
@@ -100,61 +95,7 @@ func (pod *NRIRawPod) CgroupPath() string {
 	default:
 		return ""
 	}
-	/*
-		Kubernetes defines three different pods:
-		1. Burstable: pod requests are less than the value of limits and not 0;
-		2. BestEffort: pod requests and limits are both 0;
-		3. Guaranteed: pod requests are equal to the value set by limits;
-
-		When using cgroupfs as cgroup driver,
-		1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3
-		2. The BestEffort path is in the form: kubepods/besteffort/pod34152897-dbaf-11ea-8cb9-0653660051c3
-		3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3
-
-		When using systemd as cgroup driver:
-		1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
-		2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
-		3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/
-	*/
-
-	if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd {
-		if qosClassPath == "" {
-			switch containerEngineScopes[currentContainerEngines] {
-			case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
-				path = filepath.Join(
-					constant.KubepodsCgroup+".slice",
-					constant.KubepodsCgroup+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice",
-				)
-			}
-		} else {
-			switch containerEngineScopes[currentContainerEngines] {
-			case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
-				path = filepath.Join(
-					constant.KubepodsCgroup+".slice",
-					constant.KubepodsCgroup+"-"+qosClassPath+".slice",
-					pod.Linux.CgroupParent,
-				)
-
-			}
-		}
-	} else {
-		if qosClassPath == "" {
-			switch containerEngineScopes[currentContainerEngines] {
-			case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula, constant.ContainerEngineCrio:
-				path = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id)
-			}
-		} else {
-
-			switch containerEngineScopes[currentContainerEngines] {
-			case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula, constant.ContainerEngineCrio:
-				path = filepath.Join(constant.KubepodsCgroup, qosClassPath, constant.PodCgroupNamePrefix+id)
-			default:
-				path = ""
-			}
-		}
-
-	}
-	return path
+	return cgroup.ConcatPodCgroupPath(qosClassPath, id)
 }
 
 // get pod running state
@@ -437,7 +378,7 @@ func (container *NRIRawContainer) GetResourceMaps() (ResourceMap, ResourceMap) {
 }
 
 // get current container engine
-func FixContainerEngine() {
+func getEngineFromCgroup() {
 	file, err := os.OpenFile(procSelfCgroupFile, os.O_RDONLY, fileMode)
 	if err != nil {
 		return
diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go
index b67126a..b653c71 100644
--- a/pkg/core/typedef/rawpod.go
+++ b/pkg/core/typedef/rawpod.go
@@ -16,13 +16,11 @@ package typedef
 
 import (
 	"fmt"
-	"path/filepath"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 
-	"isula.org/rubik/pkg/common/constant"
 	"isula.org/rubik/pkg/core/typedef/cgroup"
 )
 
@@ -86,83 +84,30 @@ func (pod *RawPod) ID() string {
 	return string(pod.UID)
 }
 
+// Kubernetes defines three different pods:
+// 1. Burstable: pod requests are less than the value of limits and not 0;
+// 2. BestEffort: pod requests and limits are both 0;
+// 3. Guaranteed: pod requests are equal to the value set by limits;
+var k8sQosClass = map[corev1.PodQOSClass]string{
+	corev1.PodQOSGuaranteed: "",
+	corev1.PodQOSBurstable:  strings.ToLower(string(corev1.PodQOSBurstable)),
+	corev1.PodQOSBestEffort: strings.ToLower(string(corev1.PodQOSBestEffort)),
+}
+
 // CgroupPath returns cgroup path of raw pod
 // handle different combinations of cgroupdriver and pod qos and container runtime
-func (pod *RawPod) CgroupPath() string {
+func (pod *RawPod) CgroupPath() (res string) {
 	id := string(pod.UID)
 	if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" {
 		id = configHash
 	}
 
-	qosClassPath := ""
-	switch pod.Status.QOSClass {
-	case corev1.PodQOSGuaranteed:
-	case corev1.PodQOSBurstable:
-		qosClassPath = strings.ToLower(string(corev1.PodQOSBurstable))
-	case corev1.PodQOSBestEffort:
-		qosClassPath = strings.ToLower(string(corev1.PodQOSBestEffort))
-	default:
+	qosPrefix, existed := k8sQosClass[pod.Status.QOSClass]
+	if !existed {
+		fmt.Printf("unsupported qos class: %v", pod.Status.QOSClass)
 		return ""
 	}
-
-	/*
-		Kubernetes defines three different pods:
-		1. Burstable: pod requests are less than the value of limits and not 0;
-		2. BestEffort: pod requests and limits are both 0;
-		3. Guaranteed: pod requests are equal to the value set by limits;
-
-		When using cgroupfs as cgroup driver:
-		1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3
-		2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3
-		3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3
-
-		When using systemd as cgroup driver:
-		1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
-		2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice
-		3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/
-	*/
-
-	if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd {
-		if qosClassPath == "" {
-			switch containerEngineScopes[currentContainerEngines] {
-			case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
-				return filepath.Join(
-					constant.KubepodsCgroup+".slice",
-					constant.KubepodsCgroup+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice",
-				)
-			default:
-				return ""
-			}
-		} else {
-			switch containerEngineScopes[currentContainerEngines] {
-			case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula:
-				return filepath.Join(
-					constant.KubepodsCgroup+".slice",
-					constant.KubepodsCgroup+"-"+qosClassPath+".slice",
-					constant.KubepodsCgroup+"-"+qosClassPath+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice",
-				)
-			default:
-				return ""
-			}
-
-		}
-	} else {
-		if qosClassPath == "" {
-			switch containerEngineScopes[currentContainerEngines] {
-			case constant.ContainerEngineDocker, constant.ContainerEngineContainerd, constant.ContainerEngineIsula, constant.ContainerEngineCrio:
-				return filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id)
-			default:
-				return ""
-			}
-		} else {
-			switch containerEngineScopes[currentContainerEngines] {
-			case constant.ContainerEngineDocker, constant.ContainerEngineContainerd, constant.ContainerEngineIsula, constant.ContainerEngineCrio:
-				return filepath.Join(constant.KubepodsCgroup, qosClassPath, constant.PodCgroupNamePrefix+id)
-			default:
-				return ""
-			}
-		}
-	}
+	return cgroup.ConcatPodCgroupPath(qosPrefix, id)
 }
 
 // ListRawContainers returns all RawContainers in the RawPod
@@ -221,6 +166,13 @@ func (cont *RawContainer) GetRealContainerID() (string, error) {
 		So we don't consider the case of midway container engine changes
 		`fixContainerEngine` is only executed when `getRealContainerID` is called for the first time
 	*/
+	setContainerEnginesOnce.Do(func() {
+		getEngineFromCgroup()
+		_, exist := supportEnginesPrefixMap[currentContainerEngines]
+		if !exist {
+			getEngineFromContainerID(cont.status.ContainerID)
+		}
+	})
 
 	if !currentContainerEngines.Support(cont) {
 		return "", fmt.Errorf("unsupported container engine: %v", cont.status.ContainerID)
-- 
2.46.0

