From 75f27f511e5a379bc29e2b9fd524a1bd960d0dae Mon Sep 17 00:00:00 2001 From: zhangxiaoyu Date: Wed, 23 Jun 2021 16:25:59 +0800 Subject: [PATCH 01/14] mkdir etcd work dir and manifest dir Signed-off-by: zhangxiaoyu --- pkg/clusterdeployment/binary/binary.go | 3 +-- .../binary/bootstrap/bootstrap.go | 3 ++- .../binary/commontools/token.go | 14 +++++------ .../binary/controlplane/controlplane.go | 18 +++++++------ .../binary/coredns/binarycoredns.go | 12 ++++++--- .../binary/etcdcluster/etcdcluster.go | 25 ++++++++++++++----- 6 files changed, 47 insertions(+), 28 deletions(-) diff --git a/pkg/clusterdeployment/binary/binary.go b/pkg/clusterdeployment/binary/binary.go index 70402bd..2e40c44 100644 --- a/pkg/clusterdeployment/binary/binary.go +++ b/pkg/clusterdeployment/binary/binary.go @@ -140,8 +140,7 @@ func (bcp *BinaryClusterDeployment) DeployLoadBalancer() error { func (bcp *BinaryClusterDeployment) InitControlPlane() error { logrus.Info("do init control plane...") - controlplane.Init(bcp.config) - return nil + return controlplane.Init(bcp.config) } func (bcp *BinaryClusterDeployment) JoinBootstrap() error { diff --git a/pkg/clusterdeployment/binary/bootstrap/bootstrap.go b/pkg/clusterdeployment/binary/bootstrap/bootstrap.go index 76423f8..148bdec 100644 --- a/pkg/clusterdeployment/binary/bootstrap/bootstrap.go +++ b/pkg/clusterdeployment/binary/bootstrap/bootstrap.go @@ -57,7 +57,8 @@ func (gt *GetTokenTask) Name() string { } func (gt *GetTokenTask) Run(r runner.Runner, hcg *api.HostConfig) error { - token, err := commontools.GetBootstrapToken(r, gt.tokenStr, filepath.Join(gt.cluster.GetConfigDir(), constants.KubeConfigFileNameAdmin)) + token, err := commontools.GetBootstrapToken(r, gt.tokenStr, + filepath.Join(gt.cluster.GetConfigDir(), constants.KubeConfigFileNameAdmin), gt.cluster.GetManifestDir()) if err != nil { return err } diff --git a/pkg/clusterdeployment/binary/commontools/token.go b/pkg/clusterdeployment/binary/commontools/token.go index c946a99..f1d4c5d 100644 --- a/pkg/clusterdeployment/binary/commontools/token.go +++ b/pkg/clusterdeployment/binary/commontools/token.go @@ -52,7 +52,7 @@ stringData: ` ) -func CreateBootstrapToken(r runner.Runner, bconf *api.BootstrapTokenConfig, kubeconfig string) error { +func CreateBootstrapToken(r runner.Runner, bconf *api.BootstrapTokenConfig, kubeconfig, manifestDir string) error { var sb strings.Builder var usages []string now := time.Now() @@ -80,10 +80,10 @@ func CreateBootstrapToken(r runner.Runner, bconf *api.BootstrapTokenConfig, kube return err } sb.WriteString("sudo -E /bin/sh -c \"") - sb.WriteString(fmt.Sprintf("mkdir -p %s", constants.DefaultK8SManifestsDir)) + sb.WriteString(fmt.Sprintf("mkdir -p %s", manifestDir)) tokenYamlBase64 := base64.StdEncoding.EncodeToString([]byte(coreConfig)) - sb.WriteString(fmt.Sprintf(" && echo %s | base64 -d > %s/bootstrap_token.yaml", tokenYamlBase64, constants.DefaultK8SManifestsDir)) - sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s kubectl apply -f %s/bootstrap_token.yaml", kubeconfig, constants.DefaultK8SManifestsDir)) + sb.WriteString(fmt.Sprintf(" && echo %s | base64 -d > %s/bootstrap_token.yaml", tokenYamlBase64, manifestDir)) + sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s kubectl apply -f %s/bootstrap_token.yaml", kubeconfig, manifestDir)) sb.WriteString("\"") _, err = r.RunCommand(sb.String()) @@ -96,7 +96,7 @@ func CreateBootstrapToken(r runner.Runner, bconf *api.BootstrapTokenConfig, kube func CreateBootstrapTokensForCluster(r runner.Runner, ccfg *api.ClusterConfig) error { for _, token := range ccfg.BootStrapTokens { - if err := CreateBootstrapToken(r, token, filepath.Join(ccfg.GetConfigDir(), constants.KubeConfigFileNameAdmin)); err != nil { + if err := CreateBootstrapToken(r, token, filepath.Join(ccfg.GetConfigDir(), constants.KubeConfigFileNameAdmin), ccfg.GetManifestDir()); err != nil { logrus.Errorf("create bootstrap token failed: %v", err) return err } @@ -104,7 +104,7 @@ func CreateBootstrapTokensForCluster(r runner.Runner, ccfg *api.ClusterConfig) e return nil } -func GetBootstrapToken(r runner.Runner, tokenStr string, kubeconfig string) (string, error) { +func GetBootstrapToken(r runner.Runner, tokenStr string, kubeconfig, manifestDir string) (string, error) { // TODO: check exist token first token, id, secret, err := ParseBootstrapTokenStr(tokenStr) if err != nil { @@ -117,7 +117,7 @@ func GetBootstrapToken(r runner.Runner, tokenStr string, kubeconfig string) (str Usages: []string{"authentication", "signing"}, AuthExtraGroups: []string{"system:bootstrappers:worker,system:bootstrappers:ingress"}, } - err = CreateBootstrapToken(r, bconf, kubeconfig) + err = CreateBootstrapToken(r, bconf, kubeconfig, manifestDir) return token, err } diff --git a/pkg/clusterdeployment/binary/controlplane/controlplane.go b/pkg/clusterdeployment/binary/controlplane/controlplane.go index 7d92e16..b8f1ba6 100644 --- a/pkg/clusterdeployment/binary/controlplane/controlplane.go +++ b/pkg/clusterdeployment/binary/controlplane/controlplane.go @@ -512,11 +512,13 @@ func (ct *PostControlPlaneTask) Name() string { } func (ct *PostControlPlaneTask) doAdminRole(r runner.Runner) error { + manifestDir := ct.cluster.GetManifestDir() var sb strings.Builder sb.WriteString("sudo -E /bin/sh -c \"") + sb.WriteString(fmt.Sprintf("mkdir -p %s", manifestDir)) roleBase64 := base64.StdEncoding.EncodeToString([]byte(AdminRoleConfig)) - sb.WriteString(fmt.Sprintf(" echo %s | base64 -d > %s/admin_cluster_role.yaml", roleBase64, ct.cluster.GetManifestDir())) - sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s/admin.conf kubectl apply -f %s/admin_cluster_role.yaml", ct.cluster.GetConfigDir(), ct.cluster.GetManifestDir())) + sb.WriteString(fmt.Sprintf(" && echo %s | base64 -d > %s/admin_cluster_role.yaml", roleBase64, manifestDir)) + sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s/admin.conf kubectl apply -f %s/admin_cluster_role.yaml", ct.cluster.GetConfigDir(), manifestDir)) sb.WriteString("\"") _, err := r.RunCommand(sb.String()) if err != nil { @@ -531,7 +533,7 @@ func (ct *PostControlPlaneTask) doAdminRole(r runner.Runner) error { RoleName: "system:kube-apiserver-to-kubelet", } - if err := ct.applyClusterRoleBinding(r, adminRoleBindConfig); err != nil { + if err := ct.applyClusterRoleBinding(r, adminRoleBindConfig, manifestDir); err != nil { logrus.Errorf("apply admin rolebind failed: %v", err) return err } @@ -562,7 +564,7 @@ func (ct *PostControlPlaneTask) createBootstrapCrb() []*api.ClusterRoleBindingCo return []*api.ClusterRoleBindingConfig{csr, approve, renew} } -func (ct *PostControlPlaneTask) applyClusterRoleBinding(r runner.Runner, crbc *api.ClusterRoleBindingConfig) error { +func (ct *PostControlPlaneTask) applyClusterRoleBinding(r runner.Runner, crbc *api.ClusterRoleBindingConfig, manifestDir string) error { datastore := map[string]interface{}{} datastore["Name"] = crbc.Name datastore["SubjectName"] = crbc.SubjectName @@ -575,10 +577,10 @@ func (ct *PostControlPlaneTask) applyClusterRoleBinding(r runner.Runner, crbc *a var sb strings.Builder sb.WriteString("sudo -E /bin/sh -c \"") - sb.WriteString(fmt.Sprintf("mkdir -p %s", constants.DefaultK8SManifestsDir)) + sb.WriteString(fmt.Sprintf("mkdir -p %s", manifestDir)) crbYamlBase64 := base64.StdEncoding.EncodeToString([]byte(crb)) - sb.WriteString(fmt.Sprintf(" && echo %s | base64 -d > %s/%s.yaml", crbYamlBase64, constants.DefaultK8SManifestsDir, crbc.Name)) - sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s/admin.conf kubectl apply -f %s/%s.yaml", ct.cluster.GetConfigDir(), constants.DefaultK8SManifestsDir, crbc.Name)) + sb.WriteString(fmt.Sprintf(" && echo %s | base64 -d > %s/%s.yaml", crbYamlBase64, manifestDir, crbc.Name)) + sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s/admin.conf kubectl apply -f %s/%s.yaml", ct.cluster.GetConfigDir(), manifestDir, crbc.Name)) sb.WriteString("\"") _, err = r.RunCommand(sb.String()) @@ -592,7 +594,7 @@ func (ct *PostControlPlaneTask) applyClusterRoleBinding(r runner.Runner, crbc *a func (ct *PostControlPlaneTask) bootstrapClusterRoleBinding(r runner.Runner) error { crbcs := ct.createBootstrapCrb() for _, crbc := range crbcs { - if err := ct.applyClusterRoleBinding(r, crbc); err != nil { + if err := ct.applyClusterRoleBinding(r, crbc, ct.cluster.GetManifestDir()); err != nil { logrus.Errorf("apply ClusterRoleBinding failed: %v", err) return err } diff --git a/pkg/clusterdeployment/binary/coredns/binarycoredns.go b/pkg/clusterdeployment/binary/coredns/binarycoredns.go index 9ed7d1c..63aa40d 100644 --- a/pkg/clusterdeployment/binary/coredns/binarycoredns.go +++ b/pkg/clusterdeployment/binary/coredns/binarycoredns.go @@ -144,10 +144,12 @@ func createCoreServerTemplate(cluster *api.ClusterConfig, r runner.Runner) error logrus.Errorf("rend core dns server failed: %v", err) return err } + manifestDir := cluster.GetManifestDir() sb.WriteString("sudo -E /bin/sh -c \"") + sb.WriteString(fmt.Sprintf("mkdir -p %s && ", manifestDir)) serverBase64 := base64.StdEncoding.EncodeToString([]byte(serverConfig)) - sb.WriteString(fmt.Sprintf("echo %s | base64 -d > %s/coredns_server.yaml", serverBase64, cluster.GetManifestDir())) - sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s kubectl apply -f %s/coredns_server.yaml", fmt.Sprintf("%s/%s", cluster.GetConfigDir(), constants.KubeConfigFileNameAdmin), cluster.GetManifestDir())) + sb.WriteString(fmt.Sprintf("echo %s | base64 -d > %s/coredns_server.yaml", serverBase64, manifestDir)) + sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s kubectl apply -f %s/coredns_server.yaml", fmt.Sprintf("%s/%s", cluster.GetConfigDir(), constants.KubeConfigFileNameAdmin), manifestDir)) sb.WriteString("\"") _, err = r.RunCommand(sb.String()) @@ -170,10 +172,12 @@ func createCoreEndpointTemplate(cluster *api.ClusterConfig, r runner.Runner, ips logrus.Errorf("rend core dns endpoint failed: %v", err) return err } + manifestDir := cluster.GetManifestDir() sb.WriteString("sudo -E /bin/sh -c \"") + sb.WriteString(fmt.Sprintf("mkdir -p %s && ", manifestDir)) epBase64 := base64.StdEncoding.EncodeToString([]byte(epConfig)) - sb.WriteString(fmt.Sprintf("echo %s | base64 -d > %s/coredns_ep.yaml", epBase64, cluster.GetManifestDir())) - sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s kubectl apply -f %s/coredns_ep.yaml", fmt.Sprintf("%s/%s", cluster.GetConfigDir(), constants.KubeConfigFileNameAdmin), cluster.GetManifestDir())) + sb.WriteString(fmt.Sprintf("echo %s | base64 -d > %s/coredns_ep.yaml", epBase64, manifestDir)) + sb.WriteString(fmt.Sprintf(" && KUBECONFIG=%s kubectl apply -f %s/coredns_ep.yaml", fmt.Sprintf("%s/%s", cluster.GetConfigDir(), constants.KubeConfigFileNameAdmin), manifestDir)) sb.WriteString("\"") _, err = r.RunCommand(sb.String()) diff --git a/pkg/clusterdeployment/binary/etcdcluster/etcdcluster.go b/pkg/clusterdeployment/binary/etcdcluster/etcdcluster.go index 7466c7a..6129d4f 100644 --- a/pkg/clusterdeployment/binary/etcdcluster/etcdcluster.go +++ b/pkg/clusterdeployment/binary/etcdcluster/etcdcluster.go @@ -19,6 +19,7 @@ import ( "encoding/base64" "fmt" "path/filepath" + "strings" "time" "gitee.com/openeuler/eggo/pkg/api" @@ -91,6 +92,12 @@ func (t *EtcdDeployEtcdsTask) Run(r runner.Runner, hostConfig *api.HostConfig) e return fmt.Errorf("empty host config") } + // prepare etcd dir + if err := prepareEtcdDir(r); err != nil { + logrus.Errorf("prepare etcd dir failed: %v", err) + return err + } + // prepare config if err := prepareEtcdConfigs(t.ccfg, r, hostConfig, EtcdConfFile, EtcdServiceFile); err != nil { return err @@ -105,12 +112,6 @@ func (t *EtcdDeployEtcdsTask) Run(r runner.Runner, hostConfig *api.HostConfig) e return err } - // create etcd service working dir - if _, err := r.RunCommand("sudo -E /bin/sh -c \"mkdir -p -m 700 /var/lib/etcd\""); err != nil { - logrus.Errorf("create etcd working dir failed: %v", err) - return err - } - shell, err := commontools.GetSystemdServiceShell("etcd", "", true) if err != nil { logrus.Errorf("get etcd systemd service shell failed: %v", err) @@ -152,6 +153,18 @@ func (t *EtcdPostDeployEtcdsTask) Run(r runner.Runner, hostConfig *api.HostConfi return nil } +func prepareEtcdDir(r runner.Runner) error { + dirs := []string{filepath.Dir(EtcdConfFile), filepath.Dir(DefaultEtcdDataDir)} + + // create etcd working dir + join := strings.Join(dirs, " ") + if _, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"mkdir -p %s\"", join)); err != nil { + return err + } + + return nil +} + func prepareEtcdConfigs(ccfg *api.ClusterConfig, r runner.Runner, hostConfig *api.HostConfig, confPath string, servicePath string) error { var peerAddresses string -- 2.25.1