diff --git a/VERSION-openeuler b/VERSION-openeuler index 5cf2da7..3b6ec87 100644 --- a/VERSION-openeuler +++ b/VERSION-openeuler @@ -1 +1 @@ -0.9.5-15 +0.9.5-18 diff --git a/git-commit b/git-commit index dbc6a54..317f3e6 100644 --- a/git-commit +++ b/git-commit @@ -1 +1 @@ -42f0c2f0f09f978ca9282ae085e68fa5446f231b +f7609ebf9b086beeb7ef204cb89a70c4a34fde84 diff --git a/isula-build.spec b/isula-build.spec index 8bba570..8e6c3a3 100644 --- a/isula-build.spec +++ b/isula-build.spec @@ -2,7 +2,7 @@ Name: isula-build Version: 0.9.5 -Release: 15 +Release: 18 Summary: A tool to build container images License: Mulan PSL V2 URL: https://gitee.com/openeuler/isula-build @@ -85,6 +85,24 @@ fi /usr/share/bash-completion/completions/isula-build %changelog +* Wed Nov 03 2021 lixiang - 0.9.5-18 +- Type:bugfix +- CVE:NA +- SUG:restart +- DESC:fix loaded images cover existing images name and tag + +* Wed Nov 03 2021 DCCooper <1866858@gmail.com> - 0.9.5-17 +- Type:bugfix +- CVE:NA +- SUG:restart +- DESC:optimize function IsExist and add more testcase for filepath.go + +* Wed Nov 03 2021 DCCooper <1866858@gmail.com> - 0.9.5-16 +- Type:bugfix +- CVE:NA +- SUG:restart +- DESC:fix random sequence for saving separated image tarball + * Tue Nov 02 2021 lixiang - 0.9.5-15 - Type:requirement - CVE:NA diff --git a/patch/0086-bugfix-fix-random-sequence-for-saving-separated-imag.patch b/patch/0086-bugfix-fix-random-sequence-for-saving-separated-imag.patch new file mode 100644 index 0000000..b271e4e --- /dev/null +++ b/patch/0086-bugfix-fix-random-sequence-for-saving-separated-imag.patch @@ -0,0 +1,43 @@ +From d6c302a3d5563286614c59a442f4cd65a8351ce2 Mon Sep 17 00:00:00 2001 +From: DCCooper <1866858@gmail.com> +Date: Tue, 2 Nov 2021 20:54:24 +0800 +Subject: [PATCH 1/2] bugfix: fix random sequence for saving separated image + tarball + +reason: sort the map key and read the key in alohabetical orger + +Signed-off-by: DCCooper <1866858@gmail.com> +--- + daemon/save.go | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +diff --git a/daemon/save.go b/daemon/save.go +index ecac5b6..9ad4e03 100644 +--- a/daemon/save.go ++++ b/daemon/save.go +@@ -20,6 +20,7 @@ import ( + "io/ioutil" + "os" + "path/filepath" ++ "sort" + "strings" + + "github.com/containers/image/v5/docker/reference" +@@ -513,7 +514,13 @@ func (s *separatorSave) processImageLayers(imgInfos map[string]imageInfo) error + libImagesMap = make(imageLayersMap, 1) + appImagesMap = make(imageLayersMap, 1) + ) +- for _, info := range imgInfos { ++ var sortedKey []string ++ for k := range imgInfos { ++ sortedKey = append(sortedKey, k) ++ } ++ sort.Strings(sortedKey) ++ for _, k := range sortedKey { ++ info := imgInfos[k] + if err := s.clearDirs(true); err != nil { + return errors.Wrap(err, "clear tmp dirs failed") + } +-- +1.8.3.1 + diff --git a/patch/0087-bugfix-optimize-function-IsExist.patch b/patch/0087-bugfix-optimize-function-IsExist.patch new file mode 100644 index 0000000..5c15c45 --- /dev/null +++ b/patch/0087-bugfix-optimize-function-IsExist.patch @@ -0,0 +1,443 @@ +From 6866f2e7f80ac9d8decf0e34a34de31df17c25aa Mon Sep 17 00:00:00 2001 +From: DCCooper <1866858@gmail.com> +Date: Tue, 2 Nov 2021 20:59:35 +0800 +Subject: [PATCH 2/2] bugfix: optimize function IsExist + +reason: IsExist should return two value: +1. err: if err is not nil, which means the + input path is not valid, so the caller + should just return +2. true/false: this boolean value indicate the + path is exist or not, the value only valid + when no err occured + +also add testcase for filepath.go file + +Signed-off-by: DCCooper <1866858@gmail.com> +--- + cmd/cli/build.go | 4 +- + cmd/cli/load.go | 4 +- + cmd/cli/save.go | 8 +- + cmd/daemon/main.go | 20 +++- + util/cipher.go | 4 +- + util/filepath.go | 18 ++-- + util/filepath_test.go | 248 ++++++++++++++++++++++++++++++++++++++++++++++++++ + 7 files changed, 289 insertions(+), 17 deletions(-) + create mode 100644 util/filepath_test.go + +diff --git a/cmd/cli/build.go b/cmd/cli/build.go +index 3d9f549..b0f7765 100644 +--- a/cmd/cli/build.go ++++ b/cmd/cli/build.go +@@ -235,7 +235,9 @@ func checkAbsPath(path string) (string, error) { + } + path = util.MakeAbsolute(path, pwd) + } +- if util.IsExist(path) { ++ if exist, err := util.IsExist(path); err != nil { ++ return "", err ++ } else if exist { + return "", errors.Errorf("output file already exist: %q, try to remove existing tarball or rename output file", path) + } + +diff --git a/cmd/cli/load.go b/cmd/cli/load.go +index 44fefdd..90d189a 100644 +--- a/cmd/cli/load.go ++++ b/cmd/cli/load.go +@@ -202,7 +202,9 @@ func (sep *separatorLoadOption) check(pwd string) error { + } + + sep.dir = util.MakeAbsolute(sep.dir, pwd) +- if !util.IsExist(sep.dir) { ++ if exist, err := util.IsExist(sep.dir); err != nil { ++ return errors.Wrap(err, "resolve dir path failed") ++ } else if !exist { + return errors.Errorf("image tarball directory %q is not exist", sep.dir) + } + +diff --git a/cmd/cli/save.go b/cmd/cli/save.go +index 599d394..5a63e02 100644 +--- a/cmd/cli/save.go ++++ b/cmd/cli/save.go +@@ -112,7 +112,9 @@ func (sep *separatorSaveOption) check(pwd string) error { + sep.destPath = "Images" + } + sep.destPath = util.MakeAbsolute(sep.destPath, pwd) +- if util.IsExist(sep.destPath) { ++ if exist, err := util.IsExist(sep.destPath); err != nil { ++ return errors.Wrap(err, "check dest path failed") ++ } else if exist { + return errors.Errorf("dest path already exist: %q, try to remove or rename it", sep.destPath) + } + if len(sep.renameFile) != 0 { +@@ -162,7 +164,9 @@ func (opt *saveOptions) checkSaveOpts(args []string) error { + return err + } + opt.path = util.MakeAbsolute(opt.path, pwd) +- if util.IsExist(opt.path) { ++ if exist, err := util.IsExist(opt.path); err != nil { ++ return errors.Wrap(err, "check output path failed") ++ } else if exist { + return errors.Errorf("output file already exist: %q, try to remove existing tarball or rename output file", opt.path) + } + return nil +diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go +index 4fd5356..3665f6b 100644 +--- a/cmd/daemon/main.go ++++ b/cmd/daemon/main.go +@@ -341,7 +341,9 @@ func setupWorkingDirectories() error { + return errors.Errorf("%q not an absolute dir, the \"dataroot\" and \"runroot\" must be an absolute path", dir) + } + +- if !util.IsExist(dir) { ++ if exist, err := util.IsExist(dir); err != nil { ++ return err ++ } else if !exist { + if err := os.MkdirAll(dir, constant.DefaultRootDirMode); err != nil { + return errors.Wrapf(err, "create directory for %q failed", dir) + } +@@ -363,7 +365,9 @@ func setupWorkingDirectories() error { + + func checkAndValidateConfig(cmd *cobra.Command) error { + // check if configuration.toml file exists, merge config if exists +- if !util.IsExist(constant.ConfigurationPath) { ++ if exist, err := util.IsExist(constant.ConfigurationPath); err != nil { ++ return err ++ } else if !exist { + logrus.Warnf("Main config file missing, the default configuration is used") + } else { + conf, err := loadConfig(constant.ConfigurationPath) +@@ -378,14 +382,18 @@ func checkAndValidateConfig(cmd *cobra.Command) error { + } + + // file policy.json must be exist +- if !util.IsExist(constant.SignaturePolicyPath) { ++ if exist, err := util.IsExist(constant.SignaturePolicyPath); err != nil { ++ return err ++ } else if !exist { + return errors.Errorf("policy config file %v is not exist", constant.SignaturePolicyPath) + } + + // check all config files + confFiles := []string{constant.RegistryConfigPath, constant.SignaturePolicyPath, constant.StorageConfigPath} + for _, file := range confFiles { +- if util.IsExist(file) { ++ if exist, err := util.IsExist(file); err != nil { ++ return err ++ } else if exist { + fi, err := os.Stat(file) + if err != nil { + return errors.Wrapf(err, "stat file %q failed", file) +@@ -402,7 +410,9 @@ func checkAndValidateConfig(cmd *cobra.Command) error { + } + + // if storage config file exists, merge storage config +- if util.IsExist(constant.StorageConfigPath) { ++ if exist, err := util.IsExist(constant.StorageConfigPath); err != nil { ++ return err ++ } else if exist { + return mergeStorageConfig(cmd) + } + +diff --git a/util/cipher.go b/util/cipher.go +index ce47b71..ecbbc47 100644 +--- a/util/cipher.go ++++ b/util/cipher.go +@@ -185,7 +185,9 @@ func DecryptRSA(data string, key *rsa.PrivateKey, h crypto.Hash) (string, error) + + // GenRSAPublicKeyFile store public key from rsa key pair into local file + func GenRSAPublicKeyFile(key *rsa.PrivateKey, path string) error { +- if IsExist(path) { ++ if exist, err := IsExist(path); err != nil { ++ return err ++ } else if exist { + if err := os.Remove(path); err != nil { + return errors.Errorf("failed to delete the residual key file: %v", err) + } +diff --git a/util/filepath.go b/util/filepath.go +index 59b22da..a10ed85 100644 +--- a/util/filepath.go ++++ b/util/filepath.go +@@ -56,14 +56,18 @@ func IsDirectory(path string) bool { + return fi.IsDir() + } + +-// IsExist returns true if the path exists +-func IsExist(path string) bool { +- if _, err := os.Lstat(path); err != nil { +- if os.IsNotExist(err) { +- return false +- } ++// IsExist returns true if the path exists when err is nil ++// and return false if path not exists when err is nil ++// Caller should focus on whether the err is nil or not ++func IsExist(path string) (bool, error) { ++ _, err := os.Lstat(path) ++ if err == nil { ++ return true, nil ++ } ++ if os.IsNotExist(err) { ++ return false, nil + } +- return true ++ return false, err + } + + // IsSymbolFile returns true if the path file is a symbol file +diff --git a/util/filepath_test.go b/util/filepath_test.go +new file mode 100644 +index 0000000..add4545 +--- /dev/null ++++ b/util/filepath_test.go +@@ -0,0 +1,248 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++// isula-build licensed under the Mulan PSL v2. ++// You can use this software according to the terms and conditions of the Mulan PSL v2. ++// You may obtain a copy of Mulan PSL v2 at: ++// http://license.coscl.org.cn/MulanPSL2 ++// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++// PURPOSE. ++// See the Mulan PSL v2 for more details. ++// Author: Xiang Li ++// Create: 2021-11-02 ++// Description: testcase for filepath related common functions ++ ++package util ++ ++import ( ++ "os" ++ "path/filepath" ++ "strings" ++ "testing" ++ ++ "gotest.tools/v3/assert" ++ constant "isula.org/isula-build" ++) ++ ++func TestIsExist(t *testing.T) { ++ type args struct { ++ path string ++ workingDir string ++ } ++ tests := []struct { ++ name string ++ args args ++ want string ++ isExist bool ++ wantErr bool ++ preHook func(t *testing.T, path string) ++ postHook func(t *testing.T) ++ }{ ++ { ++ name: "TC-filename too long", ++ args: args{ ++ path: strings.Repeat("a", 256), ++ workingDir: "/tmp", ++ }, ++ want: filepath.Join("/tmp", strings.Repeat("a", 256)), ++ isExist: false, ++ wantErr: true, ++ }, ++ { ++ name: "TC-filename valid", ++ args: args{ ++ path: strings.Repeat("a", 255), ++ workingDir: "/tmp", ++ }, ++ want: filepath.Join("/tmp", strings.Repeat("a", 255)), ++ isExist: false, ++ wantErr: false, ++ }, ++ { ++ name: "TC-path too long", ++ args: args{ ++ path: strings.Repeat(strings.Repeat("a", 256)+"/", 16), ++ workingDir: "/tmp", ++ }, ++ want: filepath.Join("/tmp", strings.Repeat(strings.Repeat("a", 256)+"/", 16)) + "/", ++ isExist: false, ++ wantErr: true, ++ }, ++ { ++ name: "TC-path exist", ++ args: args{ ++ path: strings.Repeat(strings.Repeat("a", 255)+"/", 15), ++ workingDir: "/tmp", ++ }, ++ want: filepath.Join("/tmp", strings.Repeat(strings.Repeat("a", 255)+"/", 15)) + "/", ++ isExist: true, ++ wantErr: false, ++ preHook: func(t *testing.T, path string) { ++ err := os.MkdirAll(path, constant.DefaultRootDirMode) ++ assert.NilError(t, err) ++ }, ++ postHook: func(t *testing.T) { ++ err := os.RemoveAll(filepath.Join("/tmp", strings.Repeat("a", 255)+"/")) ++ assert.NilError(t, err) ++ }, ++ }, ++ { ++ name: "TC-path with dot exist", ++ args: args{ ++ path: ".", ++ workingDir: filepath.Join("/tmp", strings.Repeat("./"+strings.Repeat("a", 255)+"/", 15)), ++ }, ++ want: filepath.Join("/tmp", strings.Repeat(strings.Repeat("a", 255)+"/", 15)) + "/", ++ isExist: true, ++ wantErr: false, ++ preHook: func(t *testing.T, path string) { ++ err := os.MkdirAll(path, constant.DefaultRootDirMode) ++ assert.NilError(t, err) ++ }, ++ postHook: func(t *testing.T) { ++ err := os.RemoveAll(filepath.Join("/tmp", strings.Repeat("a", 255)+"/")) ++ assert.NilError(t, err) ++ }, ++ }, ++ } ++ for _, tt := range tests { ++ t.Run(tt.name, func(t *testing.T) { ++ got := MakeAbsolute(tt.args.path, tt.args.workingDir) ++ if got != tt.want { ++ t.Errorf("MakeAbsolute() = %v, want %v", got, tt.want) ++ t.Skip() ++ } ++ ++ if tt.preHook != nil { ++ tt.preHook(t, got) ++ } ++ exist, err := IsExist(got) ++ if exist != tt.isExist { ++ t.Errorf("IsExist() = %v, want %v", exist, tt.isExist) ++ } ++ if (err != nil) != tt.wantErr { ++ t.Errorf("IsExist() = %v, want %v", err, tt.wantErr) ++ } ++ if tt.postHook != nil { ++ tt.postHook(t) ++ } ++ }) ++ } ++} ++ ++func TestIsSymbolFile(t *testing.T) { ++ originFile := "/tmp/originFile" ++ symbolFile := "/tmp/symbolFile" ++ noneExistFile := "/tmp/none_exist_file" ++ type args struct { ++ path string ++ } ++ tests := []struct { ++ name string ++ args args ++ want bool ++ preHook func(t *testing.T) ++ postHook func(t *testing.T) ++ }{ ++ { ++ name: "TC-is symbol file", ++ args: args{path: "/tmp/symbolFile"}, ++ want: true, ++ preHook: func(t *testing.T) { ++ _, err := os.Create(originFile) ++ assert.NilError(t, err) ++ assert.NilError(t, os.Symlink(originFile, symbolFile)) ++ }, ++ postHook: func(t *testing.T) { ++ assert.NilError(t, os.RemoveAll(originFile)) ++ assert.NilError(t, os.RemoveAll(symbolFile)) ++ }, ++ }, ++ { ++ name: "TC-is normal file", ++ args: args{path: originFile}, ++ want: false, ++ preHook: func(t *testing.T) { ++ _, err := os.Create(originFile) ++ assert.NilError(t, err) ++ }, ++ postHook: func(t *testing.T) { ++ assert.NilError(t, os.RemoveAll(originFile)) ++ }, ++ }, ++ { ++ name: "TC-file not exist", ++ args: args{path: noneExistFile}, ++ want: false, ++ }, ++ } ++ for _, tt := range tests { ++ t.Run(tt.name, func(t *testing.T) { ++ if tt.preHook != nil { ++ tt.preHook(t) ++ } ++ if got := IsSymbolFile(tt.args.path); got != tt.want { ++ t.Errorf("IsSymbolFile() = %v, want %v", got, tt.want) ++ } ++ if tt.postHook != nil { ++ tt.postHook(t) ++ } ++ }) ++ } ++} ++ ++func TestIsDirectory(t *testing.T) { ++ dirPath := filepath.Join("/tmp", t.Name()) ++ filePath := filepath.Join("/tmp", t.Name()) ++ noneExistFile := "/tmp/none_exist_file" ++ ++ type args struct { ++ path string ++ } ++ tests := []struct { ++ name string ++ args args ++ want bool ++ preHook func(t *testing.T) ++ postHook func(t *testing.T) ++ }{ ++ { ++ name: "TC-is directory", ++ args: args{path: dirPath}, ++ preHook: func(t *testing.T) { ++ assert.NilError(t, os.MkdirAll(dirPath, constant.DefaultRootDirMode)) ++ }, ++ postHook: func(t *testing.T) { ++ assert.NilError(t, os.RemoveAll(dirPath)) ++ }, ++ want: true, ++ }, ++ { ++ name: "TC-is file", ++ args: args{path: dirPath}, ++ preHook: func(t *testing.T) { ++ _, err := os.Create(filePath) ++ assert.NilError(t, err) ++ }, ++ postHook: func(t *testing.T) { ++ assert.NilError(t, os.RemoveAll(filePath)) ++ }, ++ }, ++ { ++ name: "TC-path not exist", ++ args: args{path: noneExistFile}, ++ }, ++ } ++ for _, tt := range tests { ++ t.Run(tt.name, func(t *testing.T) { ++ if tt.preHook != nil { ++ tt.preHook(t) ++ } ++ if got := IsDirectory(tt.args.path); got != tt.want { ++ t.Errorf("IsDirectory() = %v, want %v", got, tt.want) ++ } ++ if tt.postHook != nil { ++ tt.postHook(t) ++ } ++ }) ++ } ++} +-- +1.8.3.1 + diff --git a/patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch b/patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch new file mode 100644 index 0000000..ad75be5 --- /dev/null +++ b/patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch @@ -0,0 +1,434 @@ +From 6efcb2e785e452505894b0e1e589e72487439e17 Mon Sep 17 00:00:00 2001 +From: xingweizheng +Date: Wed, 27 Oct 2021 18:27:11 +0800 +Subject: [PATCH] bugfix: loaded images cover existing images name and tag + +--- + daemon/load.go | 114 +++++++++++++++++++++++++++++++------------- + daemon/load_test.go | 26 +++++----- + image/image.go | 66 ++++++++++++------------- + 3 files changed, 129 insertions(+), 77 deletions(-) + +diff --git a/daemon/load.go b/daemon/load.go +index 41690ab..b6d675b 100644 +--- a/daemon/load.go ++++ b/daemon/load.go +@@ -17,6 +17,8 @@ import ( + "io/ioutil" + "os" + "path/filepath" ++ "context" ++ "strings" + + "github.com/containers/image/v5/docker/tarfile" + ociarchive "github.com/containers/image/v5/oci/archive" +@@ -25,6 +27,7 @@ import ( + "github.com/containers/storage" + "github.com/containers/storage/pkg/archive" + securejoin "github.com/cyphar/filepath-securejoin" ++ digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +@@ -72,6 +75,12 @@ type loadOptions struct { + sep separatorLoad + } + ++type singleImage struct { ++ index int ++ id string ++ nameTag []string ++} ++ + func (b *Backend) getLoadOptions(req *pb.LoadRequest) (loadOptions, error) { + var opt = loadOptions{ + path: req.GetPath(), +@@ -119,10 +128,8 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error + "LoadID": req.GetLoadID(), + }).Info("LoadRequest received") + +- var ( +- si *storage.Image +- repoTags [][]string +- ) ++ var si *storage.Image ++ + opts, err := b.getLoadOptions(req) + if err != nil { + return errors.Wrap(err, "process load options failed") +@@ -142,7 +149,7 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error + } + } + +- repoTags, err = tryToParseImageFormatFromTarball(b.daemon.opts.DataRoot, &opts) ++ imagesInTar, err := tryToParseImageFormatFromTarball(b.daemon.opts.DataRoot, &opts) + if err != nil { + return err + } +@@ -163,24 +170,30 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error + eg.Go(func() error { + defer log.CloseContent() + +- for index, nameAndTag := range repoTags { ++ for _, singleImage := range imagesInTar { + _, si, err = image.ResolveFromImage(&image.PrepareImageOptions{ + Ctx: ctx, + FromImage: exporter.FormatTransport(opts.format, opts.path), ++ ToImage: singleImage.id, + SystemContext: image.GetSystemContext(), + Store: b.daemon.localStore, + Reporter: log, +- ManifestIndex: index, ++ ManifestIndex: singleImage.index, + }) + if err != nil { + return err + } + +- if sErr := b.daemon.localStore.SetNames(si.ID, nameAndTag); sErr != nil { +- return sErr ++ originalNames, err := b.daemon.localStore.Names(si.ID) ++ if err != nil { ++ return err ++ } ++ if err = b.daemon.localStore.SetNames(si.ID, append(originalNames, singleImage.nameTag...)); err != nil { ++ return err + } + + log.Print("Loaded image as %s\n", si.ID) ++ logrus.Infof("Loaded image as %s", si.ID) + } + + return nil +@@ -189,17 +202,11 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error + if wErr := eg.Wait(); wErr != nil { + return wErr + } +- logrus.Infof("Loaded image as %s", si.ID) + + return nil + } + +-func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]string, error) { +- var ( +- allRepoTags [][]string +- err error +- ) +- ++func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([]singleImage, error) { + // tmp dir will be removed after NewSourceFromFileWithContext + tmpDir, err := securejoin.SecureJoin(dataRoot, dataRootTmpDirPrefix) + if err != nil { +@@ -208,19 +215,21 @@ func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]s + systemContext := image.GetSystemContext() + systemContext.BigFilesTemporaryDir = tmpDir + +- allRepoTags, err = getDockerRepoTagFromImageTar(systemContext, opts.path) ++ // try docker format loading ++ imagesInTar, err := getDockerRepoTagFromImageTar(systemContext, opts.path) + if err == nil { + logrus.Infof("Parse image successful with %q format", constant.DockerTransport) + opts.format = constant.DockerArchiveTransport +- return allRepoTags, nil ++ return imagesInTar, nil + } + logrus.Warnf("Try to Parse image of docker format failed with error: %v", err) + +- allRepoTags, err = getOCIRepoTagFromImageTar(systemContext, opts.path) ++ // try oci format loading ++ imagesInTar, err = getOCIRepoTagFromImageTar(systemContext, opts.path) + if err == nil { + logrus.Infof("Parse image successful with %q format", constant.OCITransport) + opts.format = constant.OCIArchiveTransport +- return allRepoTags, nil ++ return imagesInTar, nil + } + logrus.Warnf("Try to parse image of oci format failed with error: %v", err) + +@@ -228,7 +237,7 @@ func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]s + return nil, errors.Wrap(err, "wrong image format detected from local tarball") + } + +-func getDockerRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([][]string, error) { ++func getDockerRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([]singleImage, error) { + // tmp dir will be removed after NewSourceFromFileWithContext + tarfileSource, err := tarfile.NewSourceFromFileWithContext(systemContext, path) + if err != nil { +@@ -245,35 +254,74 @@ func getDockerRepoTagFromImageTar(systemContext *types.SystemContext, path strin + return nil, errors.Errorf("failed to get the top level image manifest: %v", err) + } + +- var allRepoTags [][]string +- for _, manifestItem := range topLevelImageManifest { +- allRepoTags = append(allRepoTags, manifestItem.RepoTags) ++ imagesInTar := make([]singleImage, 0, len(topLevelImageManifest)) ++ for i, manifestItem := range topLevelImageManifest { ++ imageID, err := parseConfigID(manifestItem.Config) ++ if err != nil { ++ return nil, err ++ } ++ imagesInTar = append(imagesInTar, singleImage{index: i, id: imageID, nameTag: manifestItem.RepoTags}) + } + +- return allRepoTags, nil ++ return imagesInTar, nil + } + +-func getOCIRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([][]string, error) { +- var ( +- err error +- ) +- ++func getOCIRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([]singleImage, error) { + srcRef, err := alltransports.ParseImageName(exporter.FormatTransport(constant.OCIArchiveTransport, path)) + if err != nil { + return nil, errors.Wrap(err, "failed to parse image name of oci image format") + } + ++ imageID, err := getLoadedImageID(srcRef, systemContext) ++ if err != nil { ++ return nil, err ++ } + tarManifest, err := ociarchive.LoadManifestDescriptorWithContext(systemContext, srcRef) + if err != nil { + return nil, errors.Wrap(err, "failed to load manifest descriptor of oci image format") + } + +- // For now, we only support load single image in archive file ++ // For now, we only support loading oci-archive file with one single image + if _, ok := tarManifest.Annotations[imgspecv1.AnnotationRefName]; ok { +- return [][]string{{tarManifest.Annotations[imgspecv1.AnnotationRefName]}}, nil ++ return []singleImage{{0, imageID, []string{tarManifest.Annotations[imgspecv1.AnnotationRefName]}}}, nil ++ } ++ return []singleImage{{0, imageID, []string{}}}, nil ++} ++ ++func parseConfigID(configID string) (string, error) { ++ parts := strings.SplitN(configID, ".", 2) ++ if len(parts) != 2 { ++ return "", errors.New("wrong config info of manifest.json") ++ } ++ ++ configDigest := "sha256:" + digest.Digest(parts[0]) ++ if err := configDigest.Validate(); err != nil { ++ return "", errors.Wrapf(err, "failed to get config info") ++ } ++ ++ return "@" + configDigest.Encoded(), nil ++} ++ ++func getLoadedImageID(imageRef types.ImageReference, systemContext *types.SystemContext) (string, error) { ++ if imageRef == nil || systemContext == nil { ++ return "", errors.New("nil image reference or system context when loading image") ++ } ++ ++ newImage, err := imageRef.NewImage(context.TODO(), systemContext) ++ if err != nil { ++ return "", err ++ } ++ defer func() { ++ if err = newImage.Close(); err != nil { ++ logrus.Errorf("failed to close image: %v", err) ++ } ++ }() ++ imageDigest := newImage.ConfigInfo().Digest ++ if err = imageDigest.Validate(); err != nil { ++ return "", errors.Wrap(err, "failed to get config info") + } + +- return [][]string{{}}, nil ++ return "@" + imageDigest.Encoded(), nil + } + + func loadSeparatedImage(opt *loadOptions) error { +diff --git a/daemon/load_test.go b/daemon/load_test.go +index cbcb5d8..860b897 100644 +--- a/daemon/load_test.go ++++ b/daemon/load_test.go +@@ -31,9 +31,9 @@ import ( + ) + + const ( +- loadedTarFile = "load.tar" ++ loadedTarFile = "load.tar" + manifestJSONFile = "manifest.json" +- indexJSONFile = "index.json" ++ indexJSONFile = "index.json" + ) + + var ( +@@ -167,8 +167,10 @@ func TestLoadSingleImage(t *testing.T) { + } + ] + }`, +- format: "oci", +- withTag: true, ++ format: "oci", ++ withTag: true, ++ wantErr: true, ++ errString: "no such file or directory", + }, + { + name: "TC3 normal case load docker tar with no RepoTags", +@@ -197,8 +199,10 @@ func TestLoadSingleImage(t *testing.T) { + } + ] + }`, +- format: "oci", +- withTag: false, ++ format: "oci", ++ withTag: false, ++ wantErr: true, ++ errString: "no such file or directory", + }, + { + name: "TC5 abnormal case load docker tar with wrong manifestJSON", +@@ -217,7 +221,7 @@ func TestLoadSingleImage(t *testing.T) { + format: "docker", + withTag: true, + wantErr: true, +- errString: "error loading index", ++ errString: "no such file or directory", + }, + { + name: "TC6 abnormal case with wrong tar path", +@@ -312,10 +316,10 @@ func TestLoadMultipleImages(t *testing.T) { + path := dir.Join(loadedTarFile) + repoTags, err := tryToParseImageFormatFromTarball(daemon.opts.DataRoot, &loadOptions{path: path}) + assert.NilError(t, err) +- assert.Equal(t, repoTags[0][0], "registry.example.com/sayhello:first") +- assert.Equal(t, repoTags[1][0], "registry.example.com/sayhello:second") +- assert.Equal(t, repoTags[1][1], "registry.example.com/sayhello:third") +- assert.Equal(t, len(repoTags[2]), 0) ++ assert.Equal(t, repoTags[0].nameTag[0], "registry.example.com/sayhello:first") ++ assert.Equal(t, repoTags[1].nameTag[0], "registry.example.com/sayhello:second") ++ assert.Equal(t, repoTags[1].nameTag[1], "registry.example.com/sayhello:third") ++ assert.Equal(t, len(repoTags[2].nameTag), 0) + + req := &pb.LoadRequest{Path: path} + stream := &controlLoadServer{} +diff --git a/image/image.go b/image/image.go +index 5dda185..b24cb41 100644 +--- a/image/image.go ++++ b/image/image.go +@@ -37,7 +37,6 @@ import ( + "github.com/containers/image/v5/transports/alltransports" + "github.com/containers/image/v5/types" + "github.com/containers/storage" +- "github.com/containers/storage/pkg/stringid" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +@@ -55,6 +54,7 @@ type PrepareImageOptions struct { + SystemContext *types.SystemContext + Ctx context.Context + FromImage string ++ ToImage string + Store *store.Store + Reporter io.Writer + ManifestIndex int +@@ -125,20 +125,17 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora + } + + if transport == "" { +- // if the image can be obtained from the local storage by image id, +- // then only one image can be obtained. ++ // if the image can be obtained from the local storage by image id, then only one image can be obtained. + if len(candidates) != 1 { + return nil, nil, errors.New("transport is empty and multi or no image be found") + } +- img, err := opt.Store.Image(candidates[0]) +- if err != nil { +- pLog.Errorf("Failed to find the image %q in local store: %v", candidates[0], err) +- return nil, nil, err +- } +- ref, err := is.Transport.ParseStoreReference(opt.Store, img.ID) +- if err != nil { +- return nil, nil, errors.Wrapf(err, "failed to get the ref in store by %q", candidates[0]) ++ ++ ref, img, fErr := FindImage(opt.Store, candidates[0]) ++ if fErr != nil { ++ pLog.Errorf("Failed to find the image %q in local store: %v", candidates[0], fErr) ++ return nil, nil, fErr + } ++ + pLog.Infof("Get image from local store first search by %q", opt.FromImage) + return ref, img, nil + } +@@ -147,25 +144,33 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora + var errPull error + for _, strImage := range candidates { + var ( +- srcRef types.ImageReference +- pErr error ++ srcRef types.ImageReference ++ destImage string + ) + + imageName := exporter.FormatTransport(transport, strImage) +- if transport == constant.DockerArchiveTransport { +- srcRef, pErr = alltransports.ParseImageName(imageName + ":@" + strconv.Itoa(opt.ManifestIndex)) +- } else { +- srcRef, pErr = alltransports.ParseImageName(imageName) +- } +- if pErr != nil { +- pLog.Debugf("Failed to parse the image %q: %v", imageName, pErr) +- continue +- } +- +- destImage, err := getLocalImageNameFromRef(opt.Store, srcRef) +- if err != nil { +- pLog.Debugf("Failed to get local image name for %q: %v", imageName, err) +- continue ++ switch transport { ++ case constant.DockerArchiveTransport: ++ if srcRef, err = alltransports.ParseImageName(imageName + ":@" + strconv.Itoa(opt.ManifestIndex)); err != nil { ++ pLog.Debugf("Failed to parse the image %q with %q transport: %v", imageName, constant.DockerArchiveTransport, err) ++ continue ++ } ++ destImage = opt.ToImage ++ case constant.OCIArchiveTransport: ++ if srcRef, err = alltransports.ParseImageName(imageName); err != nil { ++ pLog.Debugf("Failed to parse the image %q with %q transport: %v", imageName, constant.OCIArchiveTransport, err) ++ continue ++ } ++ destImage = opt.ToImage ++ default: ++ if srcRef, err = alltransports.ParseImageName(imageName); err != nil { ++ pLog.Debugf("Failed to get local image name for %q: %v", imageName, err) ++ continue ++ } ++ if destImage, err = getLocalImageNameFromRef(srcRef); err != nil { ++ pLog.Debugf("Failed to parse store reference for %q: %v", destImage, err) ++ continue ++ } + } + + destRef, err := is.Transport.ParseStoreReference(opt.Store, destImage) +@@ -173,7 +178,6 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora + pLog.Debugf("Failed to parse store reference for %q: %v", destImage, err) + continue + } +- + img, err := is.Transport.GetStoreImage(opt.Store, destRef) + if err == nil { + // find the unique image in local store by name or digest +@@ -246,14 +250,10 @@ func instantiatingImage(ctx context.Context, sc *types.SystemContext, ref types. + return baseImg, nil + } + +-func getLocalImageNameFromRef(store storage.Store, srcRef types.ImageReference) (string, error) { ++func getLocalImageNameFromRef(srcRef types.ImageReference) (string, error) { + if srcRef == nil { + return "", errors.Errorf("reference to image is empty") + } +- +- if err := exporter.CheckArchiveFormat(srcRef.Transport().Name()); err == nil { +- return stringid.GenerateRandomID() + ":" + stringid.GenerateRandomID(), nil +- } + if srcRef.Transport().Name() != constant.DockerTransport { + return "", errors.Errorf("the %s transport is not supported yet", srcRef.Transport().Name()) + } +-- +2.27.0 + diff --git a/series.conf b/series.conf index 451bf96..a66abfa 100644 --- a/series.conf +++ b/series.conf @@ -48,3 +48,6 @@ patch/0082-test-optimize-scripts-in-hack.patch patch/0083-test-add-common-function-for-testing-separated-image.patch patch/0084-test-add-integration-tests-for-saving-and-loading-se.patch patch/0085-util-add-unit-test-for-increment-util-functions.patch +patch/0086-bugfix-fix-random-sequence-for-saving-separated-imag.patch +patch/0087-bugfix-optimize-function-IsExist.patch +patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch