eggo/0001-remove-residual-objects.patch

From 989d35593ad625cafa6ce60a1dc57c8bd2a917c8 Mon Sep 17 00:00:00 2001
From: haozi007 <liuhao27@huawei.com>
Date: Thu, 2 Sep 2021 03:02:35 +0100
Subject: [PATCH 1/2] remove residual objects

Signed-off-by: haozi007 <liuhao27@huawei.com>
---
 pkg/clusterdeployment/binary/binary.go        | 30 +++++++++++++++++++
 .../binary/controlplane/controlplane.go       |  2 +-
 .../binary/infrastructure/infrastructure.go   | 11 ++++++-
 pkg/clusterdeployment/clusterdeploy.go        |  1 +
 pkg/clusterdeployment/runtime/runtime.go      |  4 +--
 pkg/constants/constants.go                    |  5 ++--
 pkg/utils/runner/runner.go                    | 15 +++++++---
 7 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/pkg/clusterdeployment/binary/binary.go b/pkg/clusterdeployment/binary/binary.go
index 246f547..4596047 100644
--- a/pkg/clusterdeployment/binary/binary.go
+++ b/pkg/clusterdeployment/binary/binary.go
@@ -444,11 +444,37 @@ func (bcp *BinaryClusterDeployment) PreDeleteClusterHooks() {
 	}
 }
 
+func clearResiduals(nodes []*api.HostConfig, confDir string) error {
+	if len(nodes) == 0 {
+		return nil
+	}
+
+	var strNodes []string
+	for _, n := range nodes {
+		strNodes = append(strNodes, n.Address)
+	}
+
+	shell := fmt.Sprintf("#!/bin/bash\nrm -rf %s\nexit 0", confDir)
+
+	checker := task.NewTaskIgnoreErrInstance(
+		&commontools.RunShellTask{
+			ShellName: "checkMaster",
+			Shell:     shell,
+		},
+	)
+
+	return nodemanager.RunTaskOnNodes(checker, strNodes)
+}
+
 func (bcp *BinaryClusterDeployment) PostDeleteClusterHooks() {
 	role := []uint16{api.Worker, api.Master, api.ETCD, api.LoadBalance}
 	if err := dependency.HookSchedule(bcp.config, bcp.config.Nodes, role, api.SchedulePostCleanup); err != nil {
 		logrus.Warnf("Ignore: Delete cluster PostHook failed: %v", err)
 	}
+	// clear all residual objects
+	if err := clearResiduals(bcp.config.Nodes, bcp.config.GetConfigDir()); err != nil {
+		logrus.Warnf("Ignore: Delete cluster PostHook failed: %v", err)
+	}
 }
 
 func (bcp *BinaryClusterDeployment) PreNodeJoinHooks(node *api.HostConfig) error {
@@ -562,4 +588,8 @@ func (bcp *BinaryClusterDeployment) PostNodeCleanupHooks(node *api.HostConfig) {
 	if err := dependency.HookSchedule(bcp.config, []*api.HostConfig{node}, role, api.SchedulePostCleanup); err != nil {
 		logrus.Warnf("Ignore: Delete Node PostHook failed: %v", err)
 	}
+
+	if err := clearResiduals([]*api.HostConfig{node}, bcp.config.GetConfigDir()); err != nil {
+		logrus.Warnf("Ignore: Delete cluster PostHook failed: %v", err)
+	}
 }
diff --git a/pkg/clusterdeployment/binary/controlplane/controlplane.go b/pkg/clusterdeployment/binary/controlplane/controlplane.go
index 38e138f..5e6a8a0 100644
--- a/pkg/clusterdeployment/binary/controlplane/controlplane.go
+++ b/pkg/clusterdeployment/binary/controlplane/controlplane.go
@@ -636,7 +636,7 @@ func (ct *PostControlPlaneTask) bootstrapClusterRoleBinding(r runner.Runner) err
 }
 
 func (ct *PostControlPlaneTask) kubeletServerCRB(r runner.Runner) error {
-	if !ct.cluster.WorkerConfig.KubeletConf.EnableServer {
+	if ct.cluster.WorkerConfig.KubeletConf == nil || !ct.cluster.WorkerConfig.KubeletConf.EnableServer {
 		return nil
 	}
 
diff --git a/pkg/clusterdeployment/binary/infrastructure/infrastructure.go b/pkg/clusterdeployment/binary/infrastructure/infrastructure.go
index 5a489dc..71ad610 100644
--- a/pkg/clusterdeployment/binary/infrastructure/infrastructure.go
+++ b/pkg/clusterdeployment/binary/infrastructure/infrastructure.go
@@ -27,6 +27,7 @@ import (
 
 	"isula.org/eggo/pkg/api"
 	"isula.org/eggo/pkg/clusterdeployment/binary/cleanupcluster"
+	"isula.org/eggo/pkg/constants"
 	"isula.org/eggo/pkg/utils"
 	"isula.org/eggo/pkg/utils/dependency"
 	"isula.org/eggo/pkg/utils/nodemanager"
@@ -319,6 +320,13 @@ func (it *DestroyInfraTask) Name() string {
 	return "DestroyInfraTask"
 }
 
+func getCopyDefaultDir(user string) string {
+	if user == "root" {
+		return constants.DefaultRootCopyTempDirHome
+	}
+	return fmt.Sprintf(constants.DefaultUserCopyTempHomeFormat, user)
+}
+
 func (it *DestroyInfraTask) Run(r runner.Runner, hcg *api.HostConfig) error {
 	if hcg == nil {
 		return fmt.Errorf("empty host config")
@@ -339,7 +347,8 @@ func (it *DestroyInfraTask) Run(r runner.Runner, hcg *api.HostConfig) error {
 		logrus.Errorf("path %s not in White List and cannot remove", dstDir)
 		return nil
 	}
-	if _, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"rm -rf %s\"", dstDir)); err != nil {
+	copyTempDir := getCopyDefaultDir(hcg.UserName)
+	if _, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"rm -rf %s %s\"", dstDir, copyTempDir)); err != nil {
 		return fmt.Errorf("rm dependency failed: %v", err)
 	}
 
diff --git a/pkg/clusterdeployment/clusterdeploy.go b/pkg/clusterdeployment/clusterdeploy.go
index ee57ec3..f31ac42 100644
--- a/pkg/clusterdeployment/clusterdeploy.go
+++ b/pkg/clusterdeployment/clusterdeploy.go
@@ -216,6 +216,7 @@ func rollbackFailedNoeds(handler api.ClusterDeploymentAPI, nodes []*api.HostConf
 	for _, n := range nodes {
 		// do best to cleanup, if error, just ignore
 		handler.ClusterNodeCleanup(n, n.Type)
+		handler.PostNodeCleanupHooks(n)
 		handler.MachineInfraDestroy(n)
 		rollIDs = append(rollIDs, n.Address)
 	}
diff --git a/pkg/clusterdeployment/runtime/runtime.go b/pkg/clusterdeployment/runtime/runtime.go
index 29fab7d..2e352b6 100644
--- a/pkg/clusterdeployment/runtime/runtime.go
+++ b/pkg/clusterdeployment/runtime/runtime.go
@@ -199,9 +199,9 @@ func (ct *DeployRuntimeTask) Run(r runner.Runner, hcg *api.HostConfig) error {
 	}
 
 	// start service
-	if _, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl restart %s\"",
+	if output, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl restart %s\"",
 		ct.runtime.GetRuntimeService())); err != nil {
-		logrus.Errorf("start %s failed: %v", ct.runtime.GetRuntimeService(), err)
+		logrus.Errorf("start %s failed: %v\nout: %s", ct.runtime.GetRuntimeService(), err, output)
 		return err
 	}
 
diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go
index 0935c36..ee02e24 100644
--- a/pkg/constants/constants.go
+++ b/pkg/constants/constants.go
@@ -21,8 +21,9 @@ const (
 	DefaultImagePath   = "/image"
 
 	// user home dir formats
-	UserHomeFormat               = "/home/%s"
-	DefaultUserCopyTempDirFormat = "/home/%s/.eggo/temp"
+	UserHomeFormat                = "/home/%s"
+	DefaultUserCopyTempHomeFormat = "/home/%s/.eggo"
+	DefaultRootCopyTempDirHome    = "/root/.eggo"
 
 	// network plugin arguments key
 	NetworkPluginArgKeyYamlPath = "NetworkYamlPath"
diff --git a/pkg/utils/runner/runner.go b/pkg/utils/runner/runner.go
index 7436f63..3b15a08 100644
--- a/pkg/utils/runner/runner.go
+++ b/pkg/utils/runner/runner.go
@@ -164,7 +164,7 @@ func (ssh *SSHRunner) Reconnect() error {
 func clearUserTempDir(conn ssh.Connection, host *kkv1alpha1.HostCfg) {
 	tmpShell := "/tmp/" + RunnerShellPrefix + "*"
 	// scp to tmp file
-	dir := fmt.Sprintf(constants.DefaultUserCopyTempDirFormat, host.User)
+	dir := getCopyDefaultDir(host.User)
 	_, err := conn.Exec(fmt.Sprintf("sudo -E /bin/sh -c \"rm -rf %s; rm -rf %s\"", dir, tmpShell), host)
 	if err != nil {
 		logrus.Warnf("[%s] remove temp dir: %s failed: %v", host.Name, dir, err)
@@ -175,7 +175,7 @@ func clearUserTempDir(conn ssh.Connection, host *kkv1alpha1.HostCfg) {
 
 func prepareUserTempDir(conn ssh.Connection, host *kkv1alpha1.HostCfg) error {
 	// scp to tmp file
-	dir := fmt.Sprintf(constants.DefaultUserCopyTempDirFormat, host.User)
+	dir := getCopyDefaultDir(host.User)
 	var sb strings.Builder
 	sb.WriteString("sudo -E /bin/sh -c \"")
 	sb.WriteString(fmt.Sprintf("mkdir -p %s", dir))
@@ -190,11 +190,18 @@ func prepareUserTempDir(conn ssh.Connection, host *kkv1alpha1.HostCfg) error {
 	return nil
 }
 
+func getCopyDefaultDir(user string) string {
+	if user == "root" {
+		return constants.DefaultRootCopyTempDirHome + "/temp"
+	}
+	return fmt.Sprintf(constants.DefaultUserCopyTempHomeFormat, user) + "/temp"
+}
+
 func (ssh *SSHRunner) copyFile(src, dst string) error {
 	if ssh.Conn == nil {
 		return fmt.Errorf("[%s] SSH runner is not connected", ssh.Host.Name)
 	}
-	tempDir := fmt.Sprintf(constants.DefaultUserCopyTempDirFormat, ssh.Host.User)
+	tempDir := getCopyDefaultDir(ssh.Host.User)
 	// scp to tmp file
 	tempCpyFile := filepath.Join(tempDir, filepath.Base(src))
 	err := ssh.Conn.Scp(src, tempCpyFile)
@@ -240,7 +247,7 @@ func (ssh *SSHRunner) copyDir(srcDir, dstDir string) error {
 		logrus.Errorf("[%s] create cert tmp tar failed: %v", ssh.Host.Name, err)
 		return err
 	}
-	tmpCpyDir := fmt.Sprintf(constants.DefaultUserCopyTempDirFormat, ssh.Host.User)
+	tmpCpyDir := getCopyDefaultDir(ssh.Host.User)
 	tmpPkiFile := filepath.Join(tmpCpyDir, "pkg.tar")
 	// scp to user home directory
 	err = ssh.Copy(tmpPkgFile, tmpPkiFile)
-- 
2.25.1