From 989d35593ad625cafa6ce60a1dc57c8bd2a917c8 Mon Sep 17 00:00:00 2001 From: haozi007 Date: Thu, 2 Sep 2021 03:02:35 +0100 Subject: [PATCH 1/2] remove residual objects Signed-off-by: haozi007 --- pkg/clusterdeployment/binary/binary.go | 30 +++++++++++++++++++ .../binary/controlplane/controlplane.go | 2 +- .../binary/infrastructure/infrastructure.go | 11 ++++++- pkg/clusterdeployment/clusterdeploy.go | 1 + pkg/clusterdeployment/runtime/runtime.go | 4 +-- pkg/constants/constants.go | 5 ++-- pkg/utils/runner/runner.go | 15 +++++++--- 7 files changed, 58 insertions(+), 10 deletions(-) diff --git a/pkg/clusterdeployment/binary/binary.go b/pkg/clusterdeployment/binary/binary.go index 246f547..4596047 100644 --- a/pkg/clusterdeployment/binary/binary.go +++ b/pkg/clusterdeployment/binary/binary.go @@ -444,11 +444,37 @@ func (bcp *BinaryClusterDeployment) PreDeleteClusterHooks() { } } +func clearResiduals(nodes []*api.HostConfig, confDir string) error { + if len(nodes) == 0 { + return nil + } + + var strNodes []string + for _, n := range nodes { + strNodes = append(strNodes, n.Address) + } + + shell := fmt.Sprintf("#!/bin/bash\nrm -rf %s\nexit 0", confDir) + + checker := task.NewTaskIgnoreErrInstance( + &commontools.RunShellTask{ + ShellName: "checkMaster", + Shell: shell, + }, + ) + + return nodemanager.RunTaskOnNodes(checker, strNodes) +} + func (bcp *BinaryClusterDeployment) PostDeleteClusterHooks() { role := []uint16{api.Worker, api.Master, api.ETCD, api.LoadBalance} if err := dependency.HookSchedule(bcp.config, bcp.config.Nodes, role, api.SchedulePostCleanup); err != nil { logrus.Warnf("Ignore: Delete cluster PostHook failed: %v", err) } + // clear all residual objects + if err := clearResiduals(bcp.config.Nodes, bcp.config.GetConfigDir()); err != nil { + logrus.Warnf("Ignore: Delete cluster PostHook failed: %v", err) + } } func (bcp *BinaryClusterDeployment) PreNodeJoinHooks(node *api.HostConfig) error { @@ -562,4 +588,8 @@ func (bcp *BinaryClusterDeployment) PostNodeCleanupHooks(node *api.HostConfig) { if err := dependency.HookSchedule(bcp.config, []*api.HostConfig{node}, role, api.SchedulePostCleanup); err != nil { logrus.Warnf("Ignore: Delete Node PostHook failed: %v", err) } + + if err := clearResiduals([]*api.HostConfig{node}, bcp.config.GetConfigDir()); err != nil { + logrus.Warnf("Ignore: Delete cluster PostHook failed: %v", err) + } } diff --git a/pkg/clusterdeployment/binary/controlplane/controlplane.go b/pkg/clusterdeployment/binary/controlplane/controlplane.go index 38e138f..5e6a8a0 100644 --- a/pkg/clusterdeployment/binary/controlplane/controlplane.go +++ b/pkg/clusterdeployment/binary/controlplane/controlplane.go @@ -636,7 +636,7 @@ func (ct *PostControlPlaneTask) bootstrapClusterRoleBinding(r runner.Runner) err } func (ct *PostControlPlaneTask) kubeletServerCRB(r runner.Runner) error { - if !ct.cluster.WorkerConfig.KubeletConf.EnableServer { + if ct.cluster.WorkerConfig.KubeletConf == nil || !ct.cluster.WorkerConfig.KubeletConf.EnableServer { return nil } diff --git a/pkg/clusterdeployment/binary/infrastructure/infrastructure.go b/pkg/clusterdeployment/binary/infrastructure/infrastructure.go index 5a489dc..71ad610 100644 --- a/pkg/clusterdeployment/binary/infrastructure/infrastructure.go +++ b/pkg/clusterdeployment/binary/infrastructure/infrastructure.go @@ -27,6 +27,7 @@ import ( "isula.org/eggo/pkg/api" "isula.org/eggo/pkg/clusterdeployment/binary/cleanupcluster" + "isula.org/eggo/pkg/constants" "isula.org/eggo/pkg/utils" "isula.org/eggo/pkg/utils/dependency" "isula.org/eggo/pkg/utils/nodemanager" @@ -319,6 +320,13 @@ func (it *DestroyInfraTask) Name() string { return "DestroyInfraTask" } +func getCopyDefaultDir(user string) string { + if user == "root" { + return constants.DefaultRootCopyTempDirHome + } + return fmt.Sprintf(constants.DefaultUserCopyTempHomeFormat, user) +} + func (it *DestroyInfraTask) Run(r runner.Runner, hcg *api.HostConfig) error { if hcg == nil { return fmt.Errorf("empty host config") @@ -339,7 +347,8 @@ func (it *DestroyInfraTask) Run(r runner.Runner, hcg *api.HostConfig) error { logrus.Errorf("path %s not in White List and cannot remove", dstDir) return nil } - if _, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"rm -rf %s\"", dstDir)); err != nil { + copyTempDir := getCopyDefaultDir(hcg.UserName) + if _, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"rm -rf %s %s\"", dstDir, copyTempDir)); err != nil { return fmt.Errorf("rm dependency failed: %v", err) } diff --git a/pkg/clusterdeployment/clusterdeploy.go b/pkg/clusterdeployment/clusterdeploy.go index ee57ec3..f31ac42 100644 --- a/pkg/clusterdeployment/clusterdeploy.go +++ b/pkg/clusterdeployment/clusterdeploy.go @@ -216,6 +216,7 @@ func rollbackFailedNoeds(handler api.ClusterDeploymentAPI, nodes []*api.HostConf for _, n := range nodes { // do best to cleanup, if error, just ignore handler.ClusterNodeCleanup(n, n.Type) + handler.PostNodeCleanupHooks(n) handler.MachineInfraDestroy(n) rollIDs = append(rollIDs, n.Address) } diff --git a/pkg/clusterdeployment/runtime/runtime.go b/pkg/clusterdeployment/runtime/runtime.go index 29fab7d..2e352b6 100644 --- a/pkg/clusterdeployment/runtime/runtime.go +++ b/pkg/clusterdeployment/runtime/runtime.go @@ -199,9 +199,9 @@ func (ct *DeployRuntimeTask) Run(r runner.Runner, hcg *api.HostConfig) error { } // start service - if _, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl restart %s\"", + if output, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"systemctl daemon-reload && systemctl restart %s\"", ct.runtime.GetRuntimeService())); err != nil { - logrus.Errorf("start %s failed: %v", ct.runtime.GetRuntimeService(), err) + logrus.Errorf("start %s failed: %v\nout: %s", ct.runtime.GetRuntimeService(), err, output) return err } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 0935c36..ee02e24 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -21,8 +21,9 @@ const ( DefaultImagePath = "/image" // user home dir formats - UserHomeFormat = "/home/%s" - DefaultUserCopyTempDirFormat = "/home/%s/.eggo/temp" + UserHomeFormat = "/home/%s" + DefaultUserCopyTempHomeFormat = "/home/%s/.eggo" + DefaultRootCopyTempDirHome = "/root/.eggo" // network plugin arguments key NetworkPluginArgKeyYamlPath = "NetworkYamlPath" diff --git a/pkg/utils/runner/runner.go b/pkg/utils/runner/runner.go index 7436f63..3b15a08 100644 --- a/pkg/utils/runner/runner.go +++ b/pkg/utils/runner/runner.go @@ -164,7 +164,7 @@ func (ssh *SSHRunner) Reconnect() error { func clearUserTempDir(conn ssh.Connection, host *kkv1alpha1.HostCfg) { tmpShell := "/tmp/" + RunnerShellPrefix + "*" // scp to tmp file - dir := fmt.Sprintf(constants.DefaultUserCopyTempDirFormat, host.User) + dir := getCopyDefaultDir(host.User) _, err := conn.Exec(fmt.Sprintf("sudo -E /bin/sh -c \"rm -rf %s; rm -rf %s\"", dir, tmpShell), host) if err != nil { logrus.Warnf("[%s] remove temp dir: %s failed: %v", host.Name, dir, err) @@ -175,7 +175,7 @@ func clearUserTempDir(conn ssh.Connection, host *kkv1alpha1.HostCfg) { func prepareUserTempDir(conn ssh.Connection, host *kkv1alpha1.HostCfg) error { // scp to tmp file - dir := fmt.Sprintf(constants.DefaultUserCopyTempDirFormat, host.User) + dir := getCopyDefaultDir(host.User) var sb strings.Builder sb.WriteString("sudo -E /bin/sh -c \"") sb.WriteString(fmt.Sprintf("mkdir -p %s", dir)) @@ -190,11 +190,18 @@ func prepareUserTempDir(conn ssh.Connection, host *kkv1alpha1.HostCfg) error { return nil } +func getCopyDefaultDir(user string) string { + if user == "root" { + return constants.DefaultRootCopyTempDirHome + "/temp" + } + return fmt.Sprintf(constants.DefaultUserCopyTempHomeFormat, user) + "/temp" +} + func (ssh *SSHRunner) copyFile(src, dst string) error { if ssh.Conn == nil { return fmt.Errorf("[%s] SSH runner is not connected", ssh.Host.Name) } - tempDir := fmt.Sprintf(constants.DefaultUserCopyTempDirFormat, ssh.Host.User) + tempDir := getCopyDefaultDir(ssh.Host.User) // scp to tmp file tempCpyFile := filepath.Join(tempDir, filepath.Base(src)) err := ssh.Conn.Scp(src, tempCpyFile) @@ -240,7 +247,7 @@ func (ssh *SSHRunner) copyDir(srcDir, dstDir string) error { logrus.Errorf("[%s] create cert tmp tar failed: %v", ssh.Host.Name, err) return err } - tmpCpyDir := fmt.Sprintf(constants.DefaultUserCopyTempDirFormat, ssh.Host.User) + tmpCpyDir := getCopyDefaultDir(ssh.Host.User) tmpPkiFile := filepath.Join(tmpCpyDir, "pkg.tar") // scp to user home directory err = ssh.Copy(tmpPkgFile, tmpPkiFile) -- 2.25.1