From b3c544dea6932318c0adc422e0441dc48c03d8bb Mon Sep 17 00:00:00 2001 From: DCCooper <1866858@gmail.com> Date: Wed, 2 Jun 2021 14:45:19 +0800 Subject: [PATCH] isula-build: sync patches from upstream reason: updates are showing as follow: - fix data and run root not effective when setting configuration.toml - enhancement on go tests - set user's uid and gid for containers - make isula-build client side static so that which can be run in containers environment Signed-off-by: DCCooper <1866858@gmail.com> --- VERSION-openeuler | 2 +- git-commit | 2 +- isula-build.spec | 8 +- ...-root-not-effective-when-setting-con.patch | 120 ++ ...0050-data-and-run-root-set-unit-test.patch | 133 ++ ...et-user-s-uid-and-gid-for-containers.patch | 34 + ...-hack-make-isula-build-binary-static.patch | 1264 +++++++++++++++++ ...-from-new-flaw-of-run-and-data-root-.patch | 186 +++ series.conf | 5 + 9 files changed, 1751 insertions(+), 3 deletions(-) create mode 100644 patch/0049-fix-data-and-run-root-not-effective-when-setting-con.patch create mode 100644 patch/0050-data-and-run-root-set-unit-test.patch create mode 100644 patch/0051-bugfix-set-user-s-uid-and-gid-for-containers.patch create mode 100644 patch/0052-hack-make-isula-build-binary-static.patch create mode 100644 patch/0053-integration-test-from-new-flaw-of-run-and-data-root-.patch diff --git a/VERSION-openeuler b/VERSION-openeuler index ee93a79..b0650a0 100644 --- a/VERSION-openeuler +++ b/VERSION-openeuler @@ -1 +1 @@ -0.9.5-6 +0.9.5-7 diff --git a/git-commit b/git-commit index 0ab27e6..4c83c1f 100644 --- a/git-commit +++ b/git-commit @@ -1 +1 @@ -b82408f23540642f79ab000483086997321305bf +2a48f637ab271e57f8f1daf9e753766b7ed98bd7 diff --git a/isula-build.spec b/isula-build.spec index b41d432..b6f584e 100644 --- a/isula-build.spec +++ b/isula-build.spec @@ -2,7 +2,7 @@ Name: isula-build Version: 0.9.5 -Release: 6 +Release: 7 Summary: A tool to build container images License: Mulan PSL V2 URL: https://gitee.com/openeuler/isula-build @@ -85,6 +85,12 @@ fi /usr/share/bash-completion/completions/isula-build %changelog +* Wed Jun 02 2021 DCCooper <1866858@gmail.com> - 0.9.5-7 +- Type:enhancement +- CVE:NA +- SUG:restart +- DESC:sync patches from upstream + * Wed Mar 03 2021 lixiang - 0.9.5-6 - Type:enhancement - CVE:NA diff --git a/patch/0049-fix-data-and-run-root-not-effective-when-setting-con.patch b/patch/0049-fix-data-and-run-root-not-effective-when-setting-con.patch new file mode 100644 index 0000000..d12e2e6 --- /dev/null +++ b/patch/0049-fix-data-and-run-root-not-effective-when-setting-con.patch @@ -0,0 +1,120 @@ +From 022e5f3bfe5ec9731cf2d8808780a07d7408c820 Mon Sep 17 00:00:00 2001 +From: xingweizheng +Date: Thu, 20 May 2021 15:58:43 +0800 +Subject: [PATCH 1/5] fix data and run root not effective when setting + configuration.toml after upgrading containers/storage + +--- + cmd/daemon/main.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++---- + 1 file changed, 59 insertions(+), 4 deletions(-) + +diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go +index 41d2b60..4fd5356 100644 +--- a/cmd/daemon/main.go ++++ b/cmd/daemon/main.go +@@ -213,6 +213,35 @@ func loadConfig(path string) (config.TomlConfig, error) { + return conf, err + } + ++func checkRootSetInConfig(path string) (setRunRoot, setGraphRoot bool, err error) { ++ fi, err := os.Stat(path) ++ if err != nil { ++ return false, false, err ++ } ++ ++ if !fi.Mode().IsRegular() { ++ err = errors.New("config file must be a regular file") ++ return false, false, err ++ } ++ ++ if err = util.CheckFileSize(path, constant.MaxFileSize); err != nil { ++ return false, false, err ++ } ++ ++ configData, err := ioutil.ReadFile(filepath.Clean(path)) ++ if err != nil { ++ return false, false, err ++ } ++ conf := struct { ++ Storage struct { ++ RunRoot string `toml:"runroot"` ++ DataRoot string `toml:"graphroot"` ++ } `toml:"storage"` ++ }{} ++ _, err = toml.Decode(string(configData), &conf) ++ return conf.Storage.RunRoot != "", conf.Storage.DataRoot != "", err ++} ++ + func mergeStorageConfig(cmd *cobra.Command) error { + store.SetDefaultConfigFilePath(constant.StorageConfigPath) + option, err := store.GetDefaultStoreOptions(true) +@@ -226,13 +255,21 @@ func mergeStorageConfig(cmd *cobra.Command) error { + } + + var storeOpt store.DaemonStoreOptions +- if option.RunRoot == "" { ++ storeOpt.RunRoot = option.RunRoot ++ storeOpt.DataRoot = option.GraphRoot ++ ++ setRunRoot, setDataRoot, err := checkRootSetInConfig(constant.StorageConfigPath) ++ if err != nil { ++ return err ++ } ++ ++ if !setRunRoot { + storeOpt.RunRoot, err = securejoin.SecureJoin(daemonOpts.RunRoot, "storage") + if err != nil { + return err + } + } +- if option.GraphRoot == "" { ++ if !setDataRoot { + storeOpt.DataRoot, err = securejoin.SecureJoin(daemonOpts.DataRoot, "storage") + if err != nil { + return err +@@ -249,7 +286,7 @@ func mergeStorageConfig(cmd *cobra.Command) error { + return nil + } + +-func mergeConfig(conf config.TomlConfig, cmd *cobra.Command) { ++func mergeConfig(conf config.TomlConfig, cmd *cobra.Command) error { + if conf.Debug && !cmd.Flag("debug").Changed { + daemonOpts.Debug = true + } +@@ -271,6 +308,22 @@ func mergeConfig(conf config.TomlConfig, cmd *cobra.Command) { + if conf.DataRoot != "" && !cmd.Flag("dataroot").Changed { + daemonOpts.DataRoot = conf.DataRoot + } ++ ++ runRoot, err := securejoin.SecureJoin(daemonOpts.RunRoot, "storage") ++ if err != nil { ++ return err ++ } ++ ++ dataRoot, err := securejoin.SecureJoin(daemonOpts.DataRoot, "storage") ++ if err != nil { ++ return err ++ } ++ store.SetDefaultStoreOptions(store.DaemonStoreOptions{ ++ DataRoot: dataRoot, ++ RunRoot: runRoot, ++ }) ++ ++ return nil + } + + func setupWorkingDirectories() error { +@@ -319,7 +372,9 @@ func checkAndValidateConfig(cmd *cobra.Command) error { + os.Exit(constant.DefaultFailedCode) + } + +- mergeConfig(conf, cmd) ++ if err = mergeConfig(conf, cmd); err != nil { ++ return err ++ } + } + + // file policy.json must be exist +-- +1.8.3.1 + diff --git a/patch/0050-data-and-run-root-set-unit-test.patch b/patch/0050-data-and-run-root-set-unit-test.patch new file mode 100644 index 0000000..a55f790 --- /dev/null +++ b/patch/0050-data-and-run-root-set-unit-test.patch @@ -0,0 +1,133 @@ +From d6c6c205122386b66ef82adc4af16c3c2eb86b18 Mon Sep 17 00:00:00 2001 +From: xingweizheng +Date: Mon, 31 May 2021 00:46:16 +0800 +Subject: [PATCH 2/5] data and run root set unit test + +--- + cmd/daemon/main_test.go | 103 ++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 103 insertions(+) + +diff --git a/cmd/daemon/main_test.go b/cmd/daemon/main_test.go +index 790fdfc..d98ea83 100644 +--- a/cmd/daemon/main_test.go ++++ b/cmd/daemon/main_test.go +@@ -18,9 +18,12 @@ import ( + "os" + "testing" + ++ "gotest.tools/v3/assert" + "gotest.tools/v3/fs" + + constant "isula.org/isula-build" ++ "isula.org/isula-build/cmd/daemon/config" ++ "isula.org/isula-build/store" + ) + + func TestSetupWorkingDirectories(t *testing.T) { +@@ -104,3 +107,103 @@ func TestSetupWorkingDirectories(t *testing.T) { + }) + } + } ++ ++func TestRunAndDataRootSet(t *testing.T) { ++ dataRoot := fs.NewDir(t, t.Name()) ++ runRoot := fs.NewDir(t, t.Name()) ++ ++ conf := config.TomlConfig{ ++ Debug: true, ++ Group: "isula", ++ LogLevel: "debug", ++ Runtime: "", ++ RunRoot: "", ++ DataRoot: "", ++ } ++ cmd := newDaemonCommand() ++ ++ result := store.DaemonStoreOptions{ ++ DataRoot: dataRoot.Join("storage"), ++ RunRoot: runRoot.Join("storage"), ++ } ++ ++ setStorage := func(content string) func() { ++ return func() { ++ if err := mergeConfig(conf, cmd); err != nil { ++ t.Fatalf("mrege config failed with error: %v", err) ++ } ++ ++ fileName := "storage.toml" ++ tmpDir := fs.NewDir(t, t.Name(), fs.WithFile(fileName, content)) ++ defer tmpDir.Remove() ++ ++ filePath := tmpDir.Join(fileName) ++ store.SetDefaultConfigFilePath(filePath) ++ option, err := store.GetDefaultStoreOptions(true) ++ if err != nil { ++ t.Fatalf("get default store options failed with error: %v", err) ++ } ++ ++ var storeOpt store.DaemonStoreOptions ++ storeOpt.RunRoot = option.RunRoot ++ storeOpt.DataRoot = option.GraphRoot ++ store.SetDefaultStoreOptions(storeOpt) ++ } ++ ++ } ++ ++ testcases := []struct { ++ name string ++ setF func() ++ expectation store.DaemonStoreOptions ++ }{ ++ { ++ name: "TC1 - cmd set, configuration and storage not set", ++ setF: func() { ++ cmd.PersistentFlags().Set("runroot", runRoot.Path()) ++ cmd.PersistentFlags().Set("dataroot", dataRoot.Path()) ++ checkAndValidateConfig(cmd) ++ }, ++ expectation: result, ++ }, ++ { ++ name: "TC2 - cmd and storage not set, configuration set", ++ setF: func() { ++ conf.DataRoot = dataRoot.Path() ++ conf.RunRoot = runRoot.Path() ++ checkAndValidateConfig(cmd) ++ }, ++ expectation: result, ++ }, ++ { ++ name: "TC3 - all not set", ++ setF: setStorage("[storage]"), ++ expectation: store.DaemonStoreOptions{ ++ DataRoot: "/var/lib/containers/storage", ++ RunRoot: "/var/run/containers/storage", ++ }, ++ }, ++ { ++ name: "TC4 - cmd and configuration not set, storage set", ++ setF: func() { ++ config := "[storage]\nrunroot = \"" + runRoot.Join("storage") + "\"\ngraphroot = \"" + dataRoot.Join("storage") + "\"" ++ sT := setStorage(config) ++ sT() ++ }, ++ expectation: result, ++ }, ++ } ++ ++ for _, tc := range testcases { ++ t.Run(tc.name, func(t *testing.T) { ++ tc.setF() ++ storeOptions, err := store.GetDefaultStoreOptions(false) ++ if err != nil { ++ t.Fatalf("get default store options failed with error: %v", err) ++ } ++ assert.Equal(t, tc.expectation.DataRoot, storeOptions.GraphRoot) ++ assert.Equal(t, tc.expectation.RunRoot, storeOptions.RunRoot) ++ }) ++ ++ } ++} +-- +1.8.3.1 + diff --git a/patch/0051-bugfix-set-user-s-uid-and-gid-for-containers.patch b/patch/0051-bugfix-set-user-s-uid-and-gid-for-containers.patch new file mode 100644 index 0000000..5994c3e --- /dev/null +++ b/patch/0051-bugfix-set-user-s-uid-and-gid-for-containers.patch @@ -0,0 +1,34 @@ +From fbd95494e6e402fd123955fbaf337696cc22c750 Mon Sep 17 00:00:00 2001 +From: DCCooper <1866858@gmail.com> +Date: Mon, 31 May 2021 20:50:24 +0800 +Subject: [PATCH 3/5] bugfix: set user's uid and gid for containers + +Signed-off-by: DCCooper <1866858@gmail.com> +--- + builder/dockerfile/run.go | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/builder/dockerfile/run.go b/builder/dockerfile/run.go +index 6c38b55..828fe67 100644 +--- a/builder/dockerfile/run.go ++++ b/builder/dockerfile/run.go +@@ -95,6 +95,16 @@ func (c *cmdBuilder) setupRuntimeSpec(command []string) (*specs.Spec, error) { + } + + // set specific runtime spec config ++ user := c.stage.docker.Config.User ++ if user != "" { ++ pair, err := util.GetChownOptions(user, c.stage.mountpoint) ++ if err != nil { ++ return nil, err ++ } ++ g.SetProcessUID(uint32(pair.UID)) ++ g.SetProcessGID(uint32(pair.GID)) ++ g.SetProcessUsername(c.stage.docker.Config.User) ++ } + g.RemoveHostname() + g.SetProcessArgs(command) + g.SetProcessTerminal(false) +-- +1.8.3.1 + diff --git a/patch/0052-hack-make-isula-build-binary-static.patch b/patch/0052-hack-make-isula-build-binary-static.patch new file mode 100644 index 0000000..be007c0 --- /dev/null +++ b/patch/0052-hack-make-isula-build-binary-static.patch @@ -0,0 +1,1264 @@ +From b575a9ae9970bbdafe132f1ac73db1d0b7ee50ba Mon Sep 17 00:00:00 2001 +From: DCCooper <1866858@gmail.com> +Date: Wed, 19 May 2021 09:36:12 +0800 +Subject: [PATCH 4/5] hack: make isula-build binary static + +Signed-off-by: DCCooper <1866858@gmail.com> +--- + Makefile | 5 +- + builder/dockerfile/builder.go | 12 ++--- + cmd/cli/build.go | 17 +++---- + cmd/cli/build_test.go | 7 ++- + cmd/cli/import.go | 10 +--- + cmd/cli/pull.go | 8 +--- + cmd/cli/pull_test.go | 12 ----- + cmd/cli/push.go | 13 ++---- + cmd/cli/push_test.go | 14 ------ + cmd/cli/save.go | 10 ++-- + constant.go | 14 ++++++ + daemon/import.go | 7 +++ + daemon/load.go | 10 ++-- + daemon/pull.go | 5 ++ + daemon/push.go | 13 ++++-- + daemon/save.go | 8 ++-- + exporter/common.go | 30 ++---------- + exporter/common_test.go | 88 ++++------------------------------- + exporter/docker/archive/archive.go | 3 +- + exporter/docker/daemon/daemon.go | 3 +- + exporter/docker/docker.go | 3 +- + exporter/exporter.go | 23 ---------- + exporter/isulad/isulad.go | 2 +- + exporter/manifest/manifest.go | 3 +- + exporter/oci/archive/archive.go | 3 +- + exporter/oci/oci.go | 3 +- + image/image.go | 14 +++--- + image/image_test.go | 5 +- + pkg/manifest/list.go | 3 +- + util/util.go | 49 ++++++++++++++++++++ + util/util_test.go | 94 ++++++++++++++++++++++++++++++++++++++ + 31 files changed, 254 insertions(+), 237 deletions(-) + +diff --git a/Makefile b/Makefile +index cbace59..e03254c 100644 +--- a/Makefile ++++ b/Makefile +@@ -22,7 +22,8 @@ BUILDTAGS := seccomp + BUILDFLAGS := -tags "$(BUILDTAGS)" + TMPDIR := /tmp/isula_build_tmpdir + BEFLAG := -tmpdir=${TMPDIR} +-SAFEBUILDFLAGS := -buildid=IdByIsula -buildmode=pie -extldflags=-ftrapv -extldflags=-static -extldflags=-zrelro -extldflags=-znow $(LDFLAGS) $(BEFLAG) ++SAFEBUILDFLAGS := -buildid=IdByIsula -buildmode=pie -extldflags=-ftrapv -extldflags=-zrelro -extldflags=-znow $(BEFLAG) $(LDFLAGS) ++STATIC_LDFLAGS := -linkmode=external -extldflags=-static + + IMAGE_BUILDARGS := $(if $(http_proxy), --build-arg http_proxy=$(http_proxy)) + IMAGE_BUILDARGS += $(if $(https_proxy), --build-arg https_proxy=$(https_proxy)) +@@ -56,7 +57,7 @@ isula-builder: ./cmd/daemon + safe: + @echo "Safe building isula-build..." + mkdir -p ${TMPDIR} +- $(GO_BUILD) -ldflags '$(SAFEBUILDFLAGS)' -o bin/isula-build $(BUILDFLAGS) ./cmd/cli ++ $(GO_BUILD) -ldflags '$(SAFEBUILDFLAGS) $(STATIC_LDFLAGS)' -o bin/isula-build $(BUILDFLAGS) ./cmd/cli 2>/dev/null + $(GO_BUILD) -ldflags '$(SAFEBUILDFLAGS)' -o bin/isula-builder $(BUILDFLAGS) ./cmd/daemon + @echo "Safe build isula-build done!" + +diff --git a/builder/dockerfile/builder.go b/builder/dockerfile/builder.go +index 4222974..cbd1e58 100644 +--- a/builder/dockerfile/builder.go ++++ b/builder/dockerfile/builder.go +@@ -159,11 +159,11 @@ func NewBuilder(ctx context.Context, store *store.Store, req *pb.BuildRequest, r + } + + func (b *Builder) parseFormat(format string) error { +- if err := exporter.CheckImageFormat(format); err != nil { ++ if err := util.CheckImageFormat(format); err != nil { + return err + } + +- if format == exporter.OCITransport { ++ if format == constant.OCITransport { + b.manifestType = imgspecv1.MediaTypeImageManifest + } + +@@ -183,12 +183,12 @@ func (b *Builder) parseOutputManifest(output []string) error { + } + + transport := segments[0] +- if transport == exporter.OCITransport { ++ if transport == constant.OCITransport { + // When transport is oci, still, we need to set b.buildOpts.Output[i] starting with prefix "docker://". We only need to set the related b.outputManifestType. + // As a result, we can push oci format image into registry. When with prefix "oci://", image is exported to local dir, which is not what we expect. + // See github.com/containers/image package for more information. + b.outputManifestType = append(b.outputManifestType, imgspecv1.MediaTypeImageManifest) +- b.buildOpts.Output[i] = fmt.Sprintf("%s:%s", exporter.DockerTransport, segments[1]) ++ b.buildOpts.Output[i] = fmt.Sprintf("%s:%s", constant.DockerTransport, segments[1]) + } + b.outputManifestType = append(b.outputManifestType, manifest.DockerV2Schema2MediaType) + } +@@ -617,11 +617,11 @@ func parseOutputTag(output string) string { + + var tag string + switch { +- case (outputFields[0] == exporter.DockerDaemonTransport || outputFields[0] == exporter.IsuladTransport) && len(outputFields) > 1: ++ case (outputFields[0] == constant.DockerDaemonTransport || outputFields[0] == constant.IsuladTransport) && len(outputFields) > 1: + tag = strings.Join(outputFields[1:], ":") + case exporter.CheckArchiveFormat(outputFields[0]) == nil && len(outputFields) > archiveOutputWithoutTagLen: + tag = strings.Join(outputFields[archiveOutputWithoutTagLen:], ":") +- case exporter.CheckImageFormat(outputFields[0]) == nil && len(outputFields) > 1: ++ case util.CheckImageFormat(outputFields[0]) == nil && len(outputFields) > 1: + repoAndTag := strings.Join(outputFields[1:], ":") + // repo format regexp, "//registry.example.com/" for example + repo := regexp.MustCompile(`^\/\/[\w\.\-\:]+\/`).FindString(repoAndTag) +diff --git a/cmd/cli/build.go b/cmd/cli/build.go +index decf285..3d9f549 100644 +--- a/cmd/cli/build.go ++++ b/cmd/cli/build.go +@@ -25,7 +25,6 @@ import ( + "strings" + "time" + +- "github.com/containers/storage/pkg/stringid" + "github.com/gogo/protobuf/types" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +@@ -35,8 +34,6 @@ import ( + + constant "isula.org/isula-build" + pb "isula.org/isula-build/api/services" +- "isula.org/isula-build/exporter" +- _ "isula.org/isula-build/exporter/register" + "isula.org/isula-build/pkg/opts" + "isula.org/isula-build/util" + ) +@@ -107,7 +104,7 @@ func NewBuildCmd() *cobra.Command { + if util.CheckCliExperimentalEnabled() { + buildCmd.PersistentFlags().StringVar(&buildOpts.format, "format", "oci", "Image format of the built image") + } else { +- buildOpts.format = exporter.DockerTransport ++ buildOpts.format = constant.DockerTransport + } + buildCmd.PersistentFlags().StringVarP(&buildOpts.output, "output", "o", "", "Destination of output images") + buildCmd.PersistentFlags().BoolVar(&buildOpts.proxyFlag, "proxy", true, "Inherit proxy environment variables from host") +@@ -161,7 +158,7 @@ func buildCommand(c *cobra.Command, args []string) error { + + func newBuildOptions(args []string) error { + // unique buildID for each build progress +- buildOpts.buildID = stringid.GenerateNonCryptoID()[:constant.DefaultIDLen] ++ buildOpts.buildID = util.GenerateNonCryptoID()[:constant.DefaultIDLen] + + if len(args) < 1 { + // use current working directory as default context directory +@@ -219,7 +216,7 @@ func checkOutput(output string) ([]string, error) { + if len(transport) == 0 { + return nil, errors.New("transport should not be empty") + } +- if !exporter.IsSupport(transport) { ++ if !util.IsSupportExporter(transport) { + return nil, errors.Errorf("transport %q not support", transport) + } + +@@ -248,12 +245,12 @@ func checkAbsPath(path string) (string, error) { + func modifyLocalTransporter(transport string, absPath string, segments []string) error { + const validIsuladFieldsLen = 3 + switch transport { +- case exporter.DockerArchiveTransport, exporter.OCIArchiveTransport: ++ case constant.DockerArchiveTransport, constant.OCIArchiveTransport: + newSeg := util.CopyStrings(segments) + newSeg[1] = absPath + buildOpts.output = strings.Join(newSeg, ":") + return nil +- case exporter.IsuladTransport: ++ case constant.IsuladTransport: + if len(segments) != validIsuladFieldsLen { + return errors.Errorf("invalid isulad output format: %v", buildOpts.output) + } +@@ -275,7 +272,7 @@ func checkAndProcessOutput() error { + + transport := segments[0] + // just build, not need to export to any destination +- if !exporter.IsClientExporter(transport) { ++ if !util.IsClientExporter(transport) { + return nil + } + +@@ -325,7 +322,7 @@ func runBuild(ctx context.Context, cli Cli) (string, error) { + digest string + ) + +- if err = exporter.CheckImageFormat(buildOpts.format); err != nil { ++ if err = util.CheckImageFormat(buildOpts.format); err != nil { + return "", err + } + if err = checkAndProcessOutput(); err != nil { +diff --git a/cmd/cli/build_test.go b/cmd/cli/build_test.go +index fcdf6b4..a7fe64e 100644 +--- a/cmd/cli/build_test.go ++++ b/cmd/cli/build_test.go +@@ -25,7 +25,6 @@ import ( + "gotest.tools/v3/fs" + + constant "isula.org/isula-build" +- "isula.org/isula-build/exporter" + _ "isula.org/isula-build/exporter/register" + "isula.org/isula-build/util" + ) +@@ -175,7 +174,7 @@ func TestRunBuildWithNArchiveExporter(t *testing.T) { + format: "docker", + }, + { +- exporter: exporter.OCIArchiveTransport, ++ exporter: constant.OCIArchiveTransport, + descSpec: "oci-archive:isula:latest", + format: "oci", + }, +@@ -222,12 +221,12 @@ func TestRunBuildWithArchiveExporter(t *testing.T) { + + var testcases = []testcase{ + { +- exporter: exporter.DockerArchiveTransport, ++ exporter: constant.DockerArchiveTransport, + descSpec: "docker-archive:/tmp/image:isula:latest", + format: "docker", + }, + { +- exporter: exporter.OCIArchiveTransport, ++ exporter: constant.OCIArchiveTransport, + descSpec: "oci-archive:/tmp/image:isula:latest", + format: "oci", + }, +diff --git a/cmd/cli/import.go b/cmd/cli/import.go +index 320197b..96263db 100644 +--- a/cmd/cli/import.go ++++ b/cmd/cli/import.go +@@ -20,8 +20,6 @@ import ( + "os" + "path/filepath" + +- dockerref "github.com/containers/image/v5/docker/reference" +- "github.com/containers/storage/pkg/stringid" + "github.com/pkg/errors" + "github.com/spf13/cobra" + +@@ -76,12 +74,6 @@ func importCommand(c *cobra.Command, args []string) error { + } + + func runImport(ctx context.Context, cli Cli) error { +- if importOpts.reference != "" { +- if _, err := dockerref.Parse(importOpts.reference); err != nil { +- return err +- } +- } +- + if !filepath.IsAbs(importOpts.source) { + pwd, err := os.Getwd() + if err != nil { +@@ -90,7 +82,7 @@ func runImport(ctx context.Context, cli Cli) error { + importOpts.source = util.MakeAbsolute(importOpts.source, pwd) + } + +- importOpts.importID = stringid.GenerateNonCryptoID()[:constant.DefaultIDLen] ++ importOpts.importID = util.GenerateNonCryptoID()[:constant.DefaultIDLen] + + stream, err := cli.Client().Import(ctx, &pb.ImportRequest{ + Source: importOpts.source, +diff --git a/cmd/cli/pull.go b/cmd/cli/pull.go +index 316c548..02951ec 100644 +--- a/cmd/cli/pull.go ++++ b/cmd/cli/pull.go +@@ -18,13 +18,12 @@ import ( + "fmt" + "io" + +- dockerref "github.com/containers/image/v5/docker/reference" +- "github.com/containers/storage/pkg/stringid" + "github.com/pkg/errors" + "github.com/spf13/cobra" + + constant "isula.org/isula-build" + pb "isula.org/isula-build/api/services" ++ "isula.org/isula-build/util" + ) + + const ( +@@ -46,9 +45,6 @@ func pullCommand(c *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("pull requires exactly one argument") + } +- if _, err := dockerref.Parse(args[0]); err != nil { +- return err +- } + + ctx := context.TODO() + cli, err := NewClient(ctx) +@@ -60,7 +56,7 @@ func pullCommand(c *cobra.Command, args []string) error { + } + + func runPull(ctx context.Context, cli Cli, imageName string) error { +- pullID := stringid.GenerateNonCryptoID()[:constant.DefaultIDLen] ++ pullID := util.GenerateNonCryptoID()[:constant.DefaultIDLen] + + pullStream, err := cli.Client().Pull(ctx, &pb.PullRequest{ + PullID: pullID, +diff --git a/cmd/cli/pull_test.go b/cmd/cli/pull_test.go +index a4dbd04..0a11b0b 100644 +--- a/cmd/cli/pull_test.go ++++ b/cmd/cli/pull_test.go +@@ -40,18 +40,6 @@ func TestPullCommand(t *testing.T) { + wantErr: true, + errString: "pull requires exactly one argument", + }, +- { +- name: "abnormal case with empty args", +- args: []string{""}, +- wantErr: true, +- errString: "repository name must have at least one component", +- }, +- { +- name: "abnormal case with invalid args", +- args: []string{"busybox-:latest"}, +- wantErr: true, +- errString: "invalid reference format", +- }, + } + + for _, tc := range testcases { +diff --git a/cmd/cli/push.go b/cmd/cli/push.go +index 7502abd..061c17b 100644 +--- a/cmd/cli/push.go ++++ b/cmd/cli/push.go +@@ -18,14 +18,11 @@ import ( + "fmt" + "io" + +- dockerref "github.com/containers/image/v5/docker/reference" +- "github.com/containers/storage/pkg/stringid" + "github.com/pkg/errors" + "github.com/spf13/cobra" + + constant "isula.org/isula-build" + pb "isula.org/isula-build/api/services" +- "isula.org/isula-build/exporter" + "isula.org/isula-build/util" + ) + +@@ -50,7 +47,7 @@ func NewPushCmd() *cobra.Command { + if util.CheckCliExperimentalEnabled() { + pushCmd.PersistentFlags().StringVarP(&pushOpts.format, "format", "f", "oci", "Format for image pushing to a registry") + } else { +- pushOpts.format = exporter.DockerTransport ++ pushOpts.format = constant.DockerTransport + } + return pushCmd + } +@@ -59,10 +56,8 @@ func pushCommand(c *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("push requires exactly one argument") + } +- if _, err := dockerref.Parse(args[0]); err != nil { +- return err +- } +- if err := exporter.CheckImageFormat(pushOpts.format); err != nil { ++ ++ if err := util.CheckImageFormat(pushOpts.format); err != nil { + return err + } + +@@ -76,7 +71,7 @@ func pushCommand(c *cobra.Command, args []string) error { + } + + func runPush(ctx context.Context, cli Cli, imageName string) error { +- pushID := stringid.GenerateNonCryptoID()[:constant.DefaultIDLen] ++ pushID := util.GenerateNonCryptoID()[:constant.DefaultIDLen] + + pushStream, err := cli.Client().Push(ctx, &pb.PushRequest{ + PushID: pushID, +diff --git a/cmd/cli/push_test.go b/cmd/cli/push_test.go +index 0f8db2e..27caef3 100644 +--- a/cmd/cli/push_test.go ++++ b/cmd/cli/push_test.go +@@ -43,20 +43,6 @@ func TestPushCommand(t *testing.T) { + errString: "push requires exactly one argument", + }, + { +- name: "abnormal case with empty args", +- args: []string{""}, +- format: "docker", +- wantErr: true, +- errString: "repository name must have at least one component", +- }, +- { +- name: "abnormal case with invalid args", +- args: []string{"busybox-:latest"}, +- format: "oci", +- wantErr: true, +- errString: "invalid reference format", +- }, +- { + name: "normal case with image format oci", + args: []string{"openeuler:latest"}, + format: "oci", +diff --git a/cmd/cli/save.go b/cmd/cli/save.go +index fe67673..cb78ecf 100644 +--- a/cmd/cli/save.go ++++ b/cmd/cli/save.go +@@ -21,13 +21,11 @@ import ( + "path/filepath" + "strings" + +- "github.com/containers/storage/pkg/stringid" + "github.com/pkg/errors" + "github.com/spf13/cobra" + + constant "isula.org/isula-build" + pb "isula.org/isula-build/api/services" +- "isula.org/isula-build/exporter" + "isula.org/isula-build/util" + ) + +@@ -59,7 +57,7 @@ func NewSaveCmd() *cobra.Command { + if util.CheckCliExperimentalEnabled() { + saveCmd.PersistentFlags().StringVarP(&saveOpts.format, "format", "f", "oci", "Format of image saving to local tarball") + } else { +- saveOpts.format = exporter.DockerTransport ++ saveOpts.format = constant.DockerTransport + } + + return saveCmd +@@ -72,10 +70,10 @@ func saveCommand(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("save accepts at least one image") + } +- if saveOpts.format == exporter.OCITransport && len(args) >= 2 { ++ if saveOpts.format == constant.OCITransport && len(args) >= 2 { + return errors.New("oci image format now only supports saving single image") + } +- if err := exporter.CheckImageFormat(saveOpts.format); err != nil { ++ if err := util.CheckImageFormat(saveOpts.format); err != nil { + return err + } + if err := checkSavePath(); err != nil { +@@ -115,7 +113,7 @@ func checkSavePath() error { + } + + func runSave(ctx context.Context, cli Cli, args []string) error { +- saveOpts.saveID = stringid.GenerateNonCryptoID()[:constant.DefaultIDLen] ++ saveOpts.saveID = util.GenerateNonCryptoID()[:constant.DefaultIDLen] + saveOpts.images = args + + saveStream, err := cli.Client().Save(ctx, &pb.SaveRequest{ +diff --git a/constant.go b/constant.go +index 30c1653..9926728 100644 +--- a/constant.go ++++ b/constant.go +@@ -75,6 +75,20 @@ const ( + BuildContainerImageType = "ctr-img" + // BufferSize is default buffer size for file transportation + BufferSize = 32 * 1024 ++ // DockerTransport used to export docker image format images to registry ++ DockerTransport = "docker" ++ // DockerArchiveTransport used to export docker image format images to local tarball ++ DockerArchiveTransport = "docker-archive" ++ // DockerDaemonTransport used to export images to docker daemon ++ DockerDaemonTransport = "docker-daemon" ++ // OCITransport used to export oci image format images to registry ++ OCITransport = "oci" ++ // OCIArchiveTransport used to export oci image format images to local tarball ++ OCIArchiveTransport = "oci-archive" ++ // IsuladTransport use to export images to isulad ++ IsuladTransport = "isulad" ++ // ManifestTransport used to export manifest list ++ ManifestTransport = "manifest" + ) + + var ( +diff --git a/daemon/import.go b/daemon/import.go +index 3d7c0d0..40a0a92 100644 +--- a/daemon/import.go ++++ b/daemon/import.go +@@ -18,6 +18,7 @@ import ( + "path/filepath" + + cp "github.com/containers/image/v5/copy" ++ dockerref "github.com/containers/image/v5/docker/reference" + is "github.com/containers/image/v5/storage" + "github.com/containers/image/v5/tarball" + "github.com/containers/image/v5/transports" +@@ -49,6 +50,12 @@ func (b *Backend) Import(req *pb.ImportRequest, stream pb.Control_ImportServer) + logEntry := logrus.WithFields(logrus.Fields{"ImportID": importID}) + logEntry.Info("ImportRequest received") + ++ if reference != "" { ++ if _, err := dockerref.Parse(reference); err != nil { ++ return err ++ } ++ } ++ + tmpName := importID + "-import-tmp" + dstRef, err := is.Transport.ParseStoreReference(localStore, tmpName) + if err != nil { +diff --git a/daemon/load.go b/daemon/load.go +index b557d38..f2d818f 100644 +--- a/daemon/load.go ++++ b/daemon/load.go +@@ -127,16 +127,16 @@ func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]s + + allRepoTags, err = getDockerRepoTagFromImageTar(systemContext, opts.path) + if err == nil { +- logrus.Infof("Parse image successful with %q format", exporter.DockerTransport) +- opts.format = exporter.DockerArchiveTransport ++ logrus.Infof("Parse image successful with %q format", constant.DockerTransport) ++ opts.format = constant.DockerArchiveTransport + return allRepoTags, nil + } + logrus.Warnf("Try to Parse image of docker format failed with error: %v", err) + + allRepoTags, err = getOCIRepoTagFromImageTar(systemContext, opts.path) + if err == nil { +- logrus.Infof("Parse image successful with %q format", exporter.OCITransport) +- opts.format = exporter.OCIArchiveTransport ++ logrus.Infof("Parse image successful with %q format", constant.OCITransport) ++ opts.format = constant.OCIArchiveTransport + return allRepoTags, nil + } + logrus.Warnf("Try to parse image of oci format failed with error: %v", err) +@@ -170,7 +170,7 @@ func getOCIRepoTagFromImageTar(systemContext *types.SystemContext, path string) + err error + ) + +- srcRef, err := alltransports.ParseImageName(exporter.FormatTransport(exporter.OCIArchiveTransport, path)) ++ srcRef, err := alltransports.ParseImageName(exporter.FormatTransport(constant.OCIArchiveTransport, path)) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse image name of oci image format") + } +diff --git a/daemon/pull.go b/daemon/pull.go +index 56be755..6d2e33d 100644 +--- a/daemon/pull.go ++++ b/daemon/pull.go +@@ -16,6 +16,7 @@ package daemon + import ( + "context" + ++ dockerref "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/types" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +@@ -53,6 +54,10 @@ func (b *Backend) Pull(req *pb.PullRequest, stream pb.Control_PullServer) error + imageName: req.GetImageName(), + } + ++ if _, err := dockerref.Parse(opt.imageName); err != nil { ++ return err ++ } ++ + ctx := context.WithValue(stream.Context(), util.LogFieldKey(util.LogKeySessionID), req.GetPullID()) + eg, egCtx := errgroup.WithContext(ctx) + eg.Go(pullHandler(egCtx, opt)) +diff --git a/daemon/push.go b/daemon/push.go +index 4e3a6ed..e36198d 100644 +--- a/daemon/push.go ++++ b/daemon/push.go +@@ -16,6 +16,7 @@ package daemon + import ( + "context" + ++ dockerref "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/types" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +@@ -59,7 +60,11 @@ func (b *Backend) Push(req *pb.PushRequest, stream pb.Control_PushServer) error + format: req.GetFormat(), + } + +- if err := exporter.CheckImageFormat(opt.format); err != nil { ++ if err := util.CheckImageFormat(opt.format); err != nil { ++ return err ++ } ++ ++ if _, err := dockerref.Parse(opt.imageName); err != nil { + return err + } + +@@ -111,11 +116,11 @@ func pushHandler(ctx context.Context, options pushOptions) func() error { + ManifestType: options.manifestType, + } + +- if err := exporter.Export(options.imageName, exporter.FormatTransport(exporter.DockerTransport, options.imageName), ++ if err := exporter.Export(options.imageName, exporter.FormatTransport(constant.DockerTransport, options.imageName), + exOpts, options.localStore); err != nil { + logrus.WithField(util.LogKeySessionID, options.pushID). +- Errorf("Push image %q of format %q failed with %v", options.imageName, exporter.DockerTransport, err) +- return errors.Wrapf(err, "push image %q of format %q failed", options.imageName, exporter.DockerTransport) ++ Errorf("Push image %q of format %q failed with %v", options.imageName, constant.DockerTransport, err) ++ return errors.Wrapf(err, "push image %q of format %q failed", options.imageName, constant.DockerTransport) + } + + return nil +diff --git a/daemon/save.go b/daemon/save.go +index fd6174b..de644c3 100644 +--- a/daemon/save.go ++++ b/daemon/save.go +@@ -71,10 +71,10 @@ func (b *Backend) Save(req *pb.SaveRequest, stream pb.Control_SaveServer) error + opts := b.getSaveOptions(req) + + switch opts.format { +- case exporter.DockerTransport: +- opts.format = exporter.DockerArchiveTransport +- case exporter.OCITransport: +- opts.format = exporter.OCIArchiveTransport ++ case constant.DockerTransport: ++ opts.format = constant.DockerArchiveTransport ++ case constant.OCITransport: ++ opts.format = constant.OCIArchiveTransport + default: + return errors.New("wrong image format provided") + } +diff --git a/exporter/common.go b/exporter/common.go +index 8f390ac..bded6ec 100644 +--- a/exporter/common.go ++++ b/exporter/common.go +@@ -209,17 +209,7 @@ func NewPolicyContext(sc *types.SystemContext) (*signature.PolicyContext, error) + // CheckArchiveFormat used to check if save or load image format is either docker-archive or oci-archive + func CheckArchiveFormat(format string) error { + switch format { +- case DockerArchiveTransport, OCIArchiveTransport: +- return nil +- default: +- return errors.New("wrong image format provided") +- } +-} +- +-// CheckImageFormat used to check if the image format is either docker or oci +-func CheckImageFormat(format string) error { +- switch format { +- case DockerTransport, OCITransport: ++ case constant.DockerArchiveTransport, constant.OCIArchiveTransport: + return nil + default: + return errors.New("wrong image format provided") +@@ -228,7 +218,7 @@ func CheckImageFormat(format string) error { + + // FormatTransport for formatting transport with corresponding path + func FormatTransport(transport, path string) string { +- if transport == DockerTransport { ++ if transport == constant.DockerTransport { + return fmt.Sprintf("%s://%s", transport, path) + } + return fmt.Sprintf("%s:%s", transport, path) +@@ -238,23 +228,13 @@ func FormatTransport(transport, path string) string { + func GetManifestType(format string) (string, error) { + var manifestType string + switch format { +- case OCITransport: ++ case constant.OCITransport: + manifestType = imgspecv1.MediaTypeImageManifest +- case DockerTransport: ++ case constant.DockerTransport: + manifestType = manifest.DockerV2Schema2MediaType + default: +- return "", errors.Errorf("unknown format %q. Choose one of the supported formats: %s,%s", format, DockerTransport, OCITransport) ++ return "", errors.Errorf("unknown format %q. Choose one of the supported formats: %s,%s", format, constant.DockerTransport, constant.OCITransport) + } + return manifestType, nil + } + +-// IsClientExporter used to determinate exporter whether need to send the image to client +-func IsClientExporter(exporter string) bool { +- clientExporters := map[string]bool{ +- DockerArchiveTransport: true, +- OCIArchiveTransport: true, +- IsuladTransport: true, +- } +- _, ok := clientExporters[exporter] +- return ok +-} +diff --git a/exporter/common_test.go b/exporter/common_test.go +index ca29296..7434d3b 100644 +--- a/exporter/common_test.go ++++ b/exporter/common_test.go +@@ -21,6 +21,8 @@ import ( + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "gotest.tools/v3/assert" + "gotest.tools/v3/fs" ++ ++ "isula.org/isula-build" + ) + + func TestFormatTransport(t *testing.T) { +@@ -36,13 +38,13 @@ func TestFormatTransport(t *testing.T) { + }{ + { + name: "docker format transport", +- transport: DockerTransport, ++ transport: constant.DockerTransport, + path: "registry.example.com/library/image:test", + result: "docker://registry.example.com/library/image:test", + }, + { + name: "oci-archive format transport", +- transport: OCIArchiveTransport, ++ transport: constant.OCIArchiveTransport, + path: ociArchiveFilePath, + result: "oci-archive:", + }, +@@ -59,43 +61,6 @@ func TestFormatTransport(t *testing.T) { + } + } + +-func TestCheckImageFormat(t *testing.T) { +- testcases := []struct { +- name string +- format string +- wantErr bool +- errString string +- }{ +- { +- name: "docker image format", +- format: DockerTransport, +- wantErr: false, +- }, +- { +- name: "oci image format", +- format: OCITransport, +- wantErr: false, +- }, +- { +- name: "unknown image format", +- format: "you guess", +- wantErr: true, +- }, +- } +- for _, tc := range testcases { +- t.Run(tc.name, func(t *testing.T) { +- err := CheckImageFormat(tc.format) +- if tc.wantErr { +- assert.Error(t, err, "wrong image format provided") +- return +- } +- if !tc.wantErr { +- assert.NilError(t, err) +- } +- }) +- } +-} +- + func TestCheckArchiveFormat(t *testing.T) { + testcases := []struct { + name string +@@ -105,12 +70,12 @@ func TestCheckArchiveFormat(t *testing.T) { + }{ + { + name: "docker-archive image format", +- format: DockerArchiveTransport, ++ format: constant.DockerArchiveTransport, + wantErr: false, + }, + { + name: "oci-archive imagee format", +- format: OCIArchiveTransport, ++ format: constant.OCIArchiveTransport, + wantErr: false, + }, + { +@@ -143,13 +108,13 @@ func TestGetManifestType(t *testing.T) { + }{ + { + name: "docker format manifest type", +- format: DockerTransport, ++ format: constant.DockerTransport, + manifest: manifest.DockerV2Schema2MediaType, + wantErr: false, + }, + { + name: "oci format manifest type", +- format: OCITransport, ++ format: constant.OCITransport, + manifest: imgspecv1.MediaTypeImageManifest, + wantErr: false, + }, +@@ -173,40 +138,3 @@ func TestGetManifestType(t *testing.T) { + } + } + +-func TestIsClientExporter(t *testing.T) { +- testcases := []struct { +- name string +- exporter string +- wantResult bool +- }{ +- { +- name: "normal docker archive exporter", +- exporter: DockerArchiveTransport, +- wantResult: true, +- }, +- { +- name: "normal oci archive exporter", +- exporter: OCIArchiveTransport, +- wantResult: true, +- }, +- { +- name: "normal isulad exporter", +- exporter: IsuladTransport, +- wantResult: true, +- }, +- { +- name: "abnormal unkown", +- exporter: "unkown", +- wantResult: false, +- }, +- } +- +- for _, tc := range testcases { +- t.Run(tc.name, func(t *testing.T) { +- isExporter := IsClientExporter(tc.exporter) +- if isExporter != tc.wantResult { +- t.Fatal("test client exporter failed") +- } +- }) +- } +-} +diff --git a/exporter/docker/archive/archive.go b/exporter/docker/archive/archive.go +index 5da3f53..04654cf 100644 +--- a/exporter/docker/archive/archive.go ++++ b/exporter/docker/archive/archive.go +@@ -25,6 +25,7 @@ import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + ++ constant "isula.org/isula-build" + "isula.org/isula-build/exporter" + "isula.org/isula-build/image" + "isula.org/isula-build/store" +@@ -47,7 +48,7 @@ var DockerArchiveExporter = dockerArchiveExporter{ + } + + func (d *dockerArchiveExporter) Name() string { +- return exporter.DockerArchiveTransport ++ return constant.DockerArchiveTransport + } + + func (d *dockerArchiveExporter) Init(opts exporter.ExportOptions, src, destSpec string, localStore *store.Store) error { +diff --git a/exporter/docker/daemon/daemon.go b/exporter/docker/daemon/daemon.go +index c7e06b7..308495c 100644 +--- a/exporter/docker/daemon/daemon.go ++++ b/exporter/docker/daemon/daemon.go +@@ -21,6 +21,7 @@ import ( + "github.com/containers/image/v5/types" + "github.com/pkg/errors" + ++ constant "isula.org/isula-build" + "isula.org/isula-build/exporter" + "isula.org/isula-build/image" + "isula.org/isula-build/store" +@@ -40,7 +41,7 @@ var _dockerDaemonExporter = dockerDaemonExporter{ + } + + func (d *dockerDaemonExporter) Name() string { +- return exporter.DockerDaemonTransport ++ return constant.DockerDaemonTransport + } + + func (d *dockerDaemonExporter) Init(opts exporter.ExportOptions, src, destSpec string, localStore *store.Store) error { +diff --git a/exporter/docker/docker.go b/exporter/docker/docker.go +index e830811..987ee94 100644 +--- a/exporter/docker/docker.go ++++ b/exporter/docker/docker.go +@@ -21,6 +21,7 @@ import ( + "github.com/containers/image/v5/types" + "github.com/pkg/errors" + ++ constant "isula.org/isula-build" + "isula.org/isula-build/exporter" + "isula.org/isula-build/image" + "isula.org/isula-build/store" +@@ -41,7 +42,7 @@ var _dockerExporter = dockerExporter{ + } + + func (d *dockerExporter) Name() string { +- return exporter.DockerTransport ++ return constant.DockerTransport + } + + func (d *dockerExporter) Init(opts exporter.ExportOptions, src, destSpec string, localStore *store.Store) error { +diff --git a/exporter/exporter.go b/exporter/exporter.go +index af1148f..60af001 100644 +--- a/exporter/exporter.go ++++ b/exporter/exporter.go +@@ -21,29 +21,6 @@ import ( + "isula.org/isula-build/store" + ) + +-const ( +- // DockerTransport used to export docker image format images to registry +- DockerTransport = "docker" +- +- // DockerArchiveTransport used to export docker image format images to local tarball +- DockerArchiveTransport = "docker-archive" +- +- // DockerDaemonTransport used to export images to docker daemon +- DockerDaemonTransport = "docker-daemon" +- +- // OCITransport used to export oci image format images to registry +- OCITransport = "oci" +- +- // OCIArchiveTransport used to export oci image format images to local tarball +- OCIArchiveTransport = "oci-archive" +- +- // IsuladTransport use to export images to isulad +- IsuladTransport = "isulad" +- +- // ManifestTransport used to export manifest list +- ManifestTransport = "manifest" +-) +- + type exportHub struct { + items map[string]Exporter + sync.RWMutex +diff --git a/exporter/isulad/isulad.go b/exporter/isulad/isulad.go +index dd41d2d..73b0496 100644 +--- a/exporter/isulad/isulad.go ++++ b/exporter/isulad/isulad.go +@@ -46,7 +46,7 @@ var _isuladExporter = isuladExporter{ + } + + func (d *isuladExporter) Name() string { +- return exporter.IsuladTransport ++ return constant.IsuladTransport + } + + func (d *isuladExporter) Init(opts exporter.ExportOptions, src, destSpec string, localStore *store.Store) error { +diff --git a/exporter/manifest/manifest.go b/exporter/manifest/manifest.go +index b6b8d2a..1b06788 100644 +--- a/exporter/manifest/manifest.go ++++ b/exporter/manifest/manifest.go +@@ -22,6 +22,7 @@ import ( + "github.com/containers/image/v5/types" + "github.com/pkg/errors" + ++ constant "isula.org/isula-build" + "isula.org/isula-build/exporter" + "isula.org/isula-build/pkg/manifest" + "isula.org/isula-build/store" +@@ -42,7 +43,7 @@ var _manifestExporter = manifestExporter{ + } + + func (d *manifestExporter) Name() string { +- return exporter.ManifestTransport ++ return constant.ManifestTransport + } + + func (d *manifestExporter) Init(opts exporter.ExportOptions, src, destSpec string, localStore *store.Store) error { +diff --git a/exporter/oci/archive/archive.go b/exporter/oci/archive/archive.go +index 03f5e4f..9c39df3 100644 +--- a/exporter/oci/archive/archive.go ++++ b/exporter/oci/archive/archive.go +@@ -22,6 +22,7 @@ import ( + "github.com/containers/image/v5/types" + "github.com/pkg/errors" + ++ constant "isula.org/isula-build" + "isula.org/isula-build/exporter" + "isula.org/isula-build/image" + "isula.org/isula-build/store" +@@ -42,7 +43,7 @@ var _ociArchiveExporter = ociArchiveExporter{ + } + + func (o *ociArchiveExporter) Name() string { +- return exporter.OCIArchiveTransport ++ return constant.OCIArchiveTransport + } + + func (o *ociArchiveExporter) Init(opts exporter.ExportOptions, src, destSpec string, localStore *store.Store) error { +diff --git a/exporter/oci/oci.go b/exporter/oci/oci.go +index 2328a77..c0d703d 100644 +--- a/exporter/oci/oci.go ++++ b/exporter/oci/oci.go +@@ -20,6 +20,7 @@ import ( + "github.com/containers/image/v5/types" + "github.com/pkg/errors" + ++ constant "isula.org/isula-build" + "isula.org/isula-build/exporter" + "isula.org/isula-build/image" + "isula.org/isula-build/store" +@@ -39,7 +40,7 @@ var _ociExporter = ociExporter{ + } + + func (o *ociExporter) Name() string { +- return exporter.OCITransport ++ return constant.OCITransport + } + + func (o *ociExporter) Init(opts exporter.ExportOptions, src, destSpec string, localStore *store.Store) error { +diff --git a/image/image.go b/image/image.go +index 1e48039..91ab720 100644 +--- a/image/image.go ++++ b/image/image.go +@@ -152,7 +152,7 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora + ) + + imageName := exporter.FormatTransport(transport, strImage) +- if transport == exporter.DockerArchiveTransport { ++ if transport == constant.DockerArchiveTransport { + srcRef, pErr = alltransports.ParseImageName(imageName + ":@" + strconv.Itoa(opt.ManifestIndex)) + } else { + srcRef, pErr = alltransports.ParseImageName(imageName) +@@ -255,7 +255,7 @@ func getLocalImageNameFromRef(store storage.Store, srcRef types.ImageReference) + return stringid.GenerateRandomID() + ":" + stringid.GenerateRandomID(), nil + } + +- if srcRef.Transport().Name() != exporter.DockerTransport { ++ if srcRef.Transport().Name() != constant.DockerTransport { + return "", errors.Errorf("the %s transport is not supported yet", srcRef.Transport().Name()) + } + +@@ -612,11 +612,11 @@ func tryResolveNameWithTransport(name string) (string, string) { + if len(splits) == 2 { + if trans := transports.Get(splits[0]); trans != nil { + switch trans.Name() { +- case exporter.DockerTransport: ++ case constant.DockerTransport: + // trim prefix if dest like docker://registry.example.com format + dest := strings.TrimPrefix(splits[1], "//") + return dest, trans.Name() +- case exporter.DockerArchiveTransport, exporter.OCIArchiveTransport: ++ case constant.DockerArchiveTransport, constant.OCIArchiveTransport: + dest := strings.TrimSpace(splits[1]) + return dest, trans.Name() + } +@@ -632,7 +632,7 @@ func tryResolveNameWithDockerReference(name string) (string, string, error) { + return "", "", errors.Wrapf(err, "error parsing image name %q", name) + } + if named.String() == name { +- return name, exporter.DockerTransport, nil ++ return name, constant.DockerTransport, nil + } + + domain := reference.Domain(named) +@@ -648,7 +648,7 @@ func tryResolveNameWithDockerReference(name string) (string, string, error) { + } + defaultPrefix := pathPrefix + string(os.PathSeparator) + if strings.HasPrefix(repoPath, defaultPrefix) && path.Join(domain, repoPath[len(defaultPrefix):])+tag+digest == name { +- return name, exporter.DockerTransport, nil ++ return name, constant.DockerTransport, nil + } + } + +@@ -687,7 +687,7 @@ func tryResolveNameInRegistries(name string, sc *types.SystemContext) ([]string, + candidate := path.Join(registry, middle, name) + candidates = append(candidates, candidate) + } +- return candidates, exporter.DockerTransport ++ return candidates, constant.DockerTransport + } + + // CheckAndAddDefaultTag checks if src is format of repository[:tag], add default tag if src without tag +diff --git a/image/image_test.go b/image/image_test.go +index 17176d4..c698b4d 100644 +--- a/image/image_test.go ++++ b/image/image_test.go +@@ -25,7 +25,6 @@ import ( + "gotest.tools/v3/fs" + + constant "isula.org/isula-build" +- "isula.org/isula-build/exporter" + "isula.org/isula-build/store" + ) + +@@ -65,7 +64,7 @@ func TestTryResolveNameWithDockerReference(t *testing.T) { + var testcases = []testcase{ + { + name: "docker.io/library/busybox:latest", +- expectTrans: exporter.DockerTransport, ++ expectTrans: constant.DockerTransport, + errStr: "", + }, { + name: "busybox:latest", +@@ -122,5 +121,5 @@ registries = [] + name := "busybox:latest" + candidates, transport := tryResolveNameInRegistries(name, nil) + assert.Assert(t, cmp.Contains(candidates, "localhost/busybox:latest")) +- assert.Equal(t, transport, exporter.DockerTransport) ++ assert.Equal(t, transport, constant.DockerTransport) + } +diff --git a/pkg/manifest/list.go b/pkg/manifest/list.go +index 381746f..bc6037f 100644 +--- a/pkg/manifest/list.go ++++ b/pkg/manifest/list.go +@@ -24,6 +24,7 @@ import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + ++ constant "isula.org/isula-build" + pb "isula.org/isula-build/api/services" + "isula.org/isula-build/builder/dockerfile/container" + "isula.org/isula-build/exporter" +@@ -60,7 +61,7 @@ func NewManifestList() *List { + func (l *List) AddImage(ctx context.Context, store *store.Store, imageSpec string) (digest.Digest, error) { + img, _, err := image.ResolveFromImage(&image.PrepareImageOptions{ + Ctx: ctx, +- FromImage: exporter.FormatTransport(exporter.DockerTransport, imageSpec), ++ FromImage: exporter.FormatTransport(constant.DockerTransport, imageSpec), + SystemContext: image.GetSystemContext(), + Store: store, + }) +diff --git a/util/util.go b/util/util.go +index 3f46d79..f527608 100644 +--- a/util/util.go ++++ b/util/util.go +@@ -315,3 +315,52 @@ func ChangeGroup(path, g string) error { + } + return nil + } ++ ++// GenerateNonCryptoID generate none crypto id with length 32 ++func GenerateNonCryptoID() string { ++ b := make([]byte, 32) ++ _, err := rand.Read(b) ++ if err != nil { ++ panic(err) // This shouldn't happen ++ } ++ id := fmt.Sprintf("%x", b) ++ ++ return id ++} ++ ++// IsSupportExporter returns true when the specific exporter is supported ++func IsSupportExporter(name string) bool { ++ exporters := map[string]bool{ ++ constant.DockerTransport: true, ++ constant.DockerArchiveTransport: true, ++ constant.DockerDaemonTransport: true, ++ constant.OCITransport: true, ++ constant.OCIArchiveTransport: true, ++ constant.IsuladTransport: true, ++ constant.ManifestTransport: true, ++ } ++ _, ok := exporters[name] ++ ++ return ok ++} ++ ++// CheckImageFormat used to check if the image format is either docker or oci ++func CheckImageFormat(format string) error { ++ switch format { ++ case constant.DockerTransport, constant.OCITransport: ++ return nil ++ default: ++ return errors.New("wrong image format provided") ++ } ++} ++ ++// IsClientExporter used to determinate exporter whether need to send the image to client ++func IsClientExporter(exporter string) bool { ++ clientExporters := map[string]bool{ ++ constant.DockerArchiveTransport: true, ++ constant.OCIArchiveTransport: true, ++ constant.IsuladTransport: true, ++ } ++ _, ok := clientExporters[exporter] ++ return ok ++} +diff --git a/util/util_test.go b/util/util_test.go +index 722c2a3..db57393 100644 +--- a/util/util_test.go ++++ b/util/util_test.go +@@ -315,3 +315,97 @@ func TestSetDaemonLock(t *testing.T) { + _, err = SetDaemonLock(root, name) + assert.ErrorContains(t, err, "check if there is another daemon running") + } ++ ++func TestGenerateNonCryptoID(t *testing.T) { ++ tests := []struct { ++ name string ++ want int ++ }{ ++ { ++ name: "TC1 - generate id", ++ want:64, ++ }, ++ } ++ for _, tt := range tests { ++ t.Run(tt.name, func(t *testing.T) { ++ if got := GenerateNonCryptoID(); len(got) != tt.want { ++ t.Errorf("GenerateNonCryptoID() = %v, want %v", got, tt.want) ++ } ++ }) ++ } ++} ++ ++func TestCheckImageFormat(t *testing.T) { ++ testcases := []struct { ++ name string ++ format string ++ wantErr bool ++ errString string ++ }{ ++ { ++ name: "docker image format", ++ format: constant.DockerTransport, ++ wantErr: false, ++ }, ++ { ++ name: "oci image format", ++ format: constant.OCITransport, ++ wantErr: false, ++ }, ++ { ++ name: "unknown image format", ++ format: "you guess", ++ wantErr: true, ++ }, ++ } ++ for _, tc := range testcases { ++ t.Run(tc.name, func(t *testing.T) { ++ err := CheckImageFormat(tc.format) ++ if tc.wantErr { ++ assert.Error(t, err, "wrong image format provided") ++ return ++ } ++ if !tc.wantErr { ++ assert.NilError(t, err) ++ } ++ }) ++ } ++} ++ ++func TestIsClientExporter(t *testing.T) { ++ testcases := []struct { ++ name string ++ exporter string ++ wantResult bool ++ }{ ++ { ++ name: "normal docker archive exporter", ++ exporter: constant.DockerArchiveTransport, ++ wantResult: true, ++ }, ++ { ++ name: "normal oci archive exporter", ++ exporter: constant.OCIArchiveTransport, ++ wantResult: true, ++ }, ++ { ++ name: "normal isulad exporter", ++ exporter: constant.IsuladTransport, ++ wantResult: true, ++ }, ++ { ++ name: "abnormal unkown", ++ exporter: "unkown", ++ wantResult: false, ++ }, ++ } ++ ++ for _, tc := range testcases { ++ t.Run(tc.name, func(t *testing.T) { ++ isExporter := IsClientExporter(tc.exporter) ++ if isExporter != tc.wantResult { ++ t.Fatal("test client exporter failed") ++ } ++ }) ++ } ++} +-- +1.8.3.1 + diff --git a/patch/0053-integration-test-from-new-flaw-of-run-and-data-root-.patch b/patch/0053-integration-test-from-new-flaw-of-run-and-data-root-.patch new file mode 100644 index 0000000..1e19829 --- /dev/null +++ b/patch/0053-integration-test-from-new-flaw-of-run-and-data-root-.patch @@ -0,0 +1,186 @@ +From 78d5ee37ff4b2b3ef0a3e3031087d8cdb2e0c0cd Mon Sep 17 00:00:00 2001 +From: xingweizheng +Date: Sun, 30 May 2021 20:55:07 +0800 +Subject: [PATCH 5/5] integration test from new flaw of run and data root set + +--- + Makefile | 18 ++++++--- + README.zh.md | 2 +- + tests/src/test_integration_set_new_root.sh | 60 ++++++++++++++++++++++++++++++ + tests/test.sh | 29 +++++++++++++-- + 4 files changed, 98 insertions(+), 11 deletions(-) + create mode 100644 tests/src/test_integration_set_new_root.sh + +diff --git a/Makefile b/Makefile +index cbace59..f8578a4 100644 +--- a/Makefile ++++ b/Makefile +@@ -73,13 +73,13 @@ debug: + build-image: + isula-build ctr-img build -f Dockerfile.proto ${IMAGE_BUILDARGS} -o isulad:${IMAGE_NAME}:latest . + +-tests: test-integration test-unit ++tests: test-base test-unit test-integration + +-.PHONY: test-integration +-test-integration: +- @echo "Integration test starting..." +- @./tests/test.sh +- @echo "Integration test done!" ++.PHONY: test-base ++test-base: ++ @echo "Base test starting..." ++ @./tests/test.sh base ++ @echo "Base test done!" + + .PHONY: test-unit + test-unit: +@@ -87,6 +87,12 @@ test-unit: + @./hack/unit_test.sh + @echo "Unit test done!" + ++.PHONY: test-integration ++test-integration: ++ @echo "Integration test starting..." ++ @./tests/test.sh integration ++ @echo "Integration test done!" ++ + .PHONY: proto + proto: + @echo "Generating protobuf..." +diff --git a/README.zh.md b/README.zh.md +index 4b53ba3..15301c0 100644 +--- a/README.zh.md ++++ b/README.zh.md +@@ -106,7 +106,7 @@ sudo rpm -ivh isula-build-*.rpm + 如果需要使用`systemd`进行管理isula-build，请参考以下步骤： + + ```sh +-sudo install -p -m 640 ./isula-build.service /etc/systemd/system/isula-build. ++sudo install -p -m 640 ./isula-build.service /etc/systemd/system/isula-build.service + sudo systemctl enable isula-build + sudo systemctl start isula-build + ``` +diff --git a/tests/src/test_integration_set_new_root.sh b/tests/src/test_integration_set_new_root.sh +new file mode 100644 +index 0000000..85b724a +--- /dev/null ++++ b/tests/src/test_integration_set_new_root.sh +@@ -0,0 +1,60 @@ ++#!/bin/bash ++ ++# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++# isula-build licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# Author: Weizheng Xing ++# Create: 2021-05-29 ++# Description: test set new run and data root in configuration.toml ++ ++run_root="/var/run/new-isula-build" ++data_root="/var/lib/new-isula-build" ++config_file="/etc/isula-build/configuration.toml" ++base_image="hub.oepkgs.net/openeuler/openeuler:21.03" ++ ++function clean() ++{ ++ isula-build ctr-img rm $base_image >/dev/null 2>&1 ++ rm -f $config_file ++ mv "$config_file".bak $config_file ++ systemctl stop isula-build ++ rm -rf $run_root $data_root ++} ++ ++# change to new data and run root ++function pre_test() ++{ ++ cp $config_file "$config_file".bak ++ sed -i "/run_root/d;/data_root/d" $config_file ++ echo "run_root = \"${run_root}\"" >> $config_file ++ echo "data_root = \"${data_root}\"" >> $config_file ++ ++ systemctl restart isula-build ++} ++ ++# check if new resources are downloaded in new root ++function do_test() ++{ ++ tree_node_befor=$(tree -L 3 $data_root | wc -l) ++ isula-build ctr-img pull $base_image >/dev/null 2>&1 ++ tree_node_after=$(tree -L 3 $data_root | wc -l) ++ ++ if [ $(($tree_node_after - $tree_node_befor)) -eq 8 ]; then ++ echo "PASS" ++ else ++ echo "Sets of run and data root are not effective" ++ clean ++ exit 1 ++ fi ++} ++ ++# clean ++pre_test ++do_test ++clean +diff --git a/tests/test.sh b/tests/test.sh +index 79fde8a..e04cc96 100755 +--- a/tests/test.sh ++++ b/tests/test.sh +@@ -2,8 +2,8 @@ + + top_dir=$(git rev-parse --show-toplevel) + +-# normal test +-function normal() { ++# base test ++function base() { + source "$top_dir"/tests/lib/common.sh + pre_check + start_isula_builder +@@ -33,15 +33,36 @@ function fuzz() { + exit $failed + } + ++# base test ++function integration() { ++ source "$top_dir"/tests/lib/common.sh ++ pre_check ++ systemctl restart isula-build ++ ++ while IFS= read -r testfile; do ++ printf "%-45s" "test $(basename "$testfile"): " ++ if ! bash "$testfile"; then ++ exit 1 ++ fi ++ done < <(find "$top_dir"/tests/src -maxdepth 1 -name "test_integration*" -type f -print) ++} ++ + # main function to chose which kind of test + function main() { + case "$1" in + fuzz) + fuzz "$2" + ;; ++ base) ++ base ++ ;; ++ integration) ++ integration ++ ;; + *) +- normal +- ;; ++ echo "Unknow test type." ++ exit 1 ++ ;; + esac + } + +-- +1.8.3.1 + diff --git a/series.conf b/series.conf index 8dbc7bc..78ee5e7 100644 --- a/series.conf +++ b/series.conf @@ -12,3 +12,8 @@ patch/0045-fix-images-command-when-only-give-repository.patch patch/0046-check-if-add-default-tag-to-image-name-when-using-pu.patch patch/0047-checkAndExpandTag-return-empty-when-tag-is-empty.patch patch/0048-trim-space-when-counting-length-of-fields-to-avoid-p.patch +patch/0049-fix-data-and-run-root-not-effective-when-setting-con.patch +patch/0050-data-and-run-root-set-unit-test.patch +patch/0051-bugfix-set-user-s-uid-and-gid-for-containers.patch +patch/0052-hack-make-isula-build-binary-static.patch +patch/0053-integration-test-from-new-flaw-of-run-and-data-root-.patch