packages/isula-build
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!146 isula-build: sync upstream bugfixes

Browse Source 
From: @DCCooper
Reviewed-by: @jingxiaolu
Signed-off-by: @jingxiaolu
This commit is contained in:
openeuler-ci-bot 2021-11-04 03:50:47 +00:00 committed by Gitee
commit 67f4c2962e
7 changed files with 944 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VERSION-openeuler
View File

@ -1 +1 @@
0.9.5-15 0.9.5-18

2
git-commit
View File

@ -1 +1 @@
42f0c2f0f09f978ca9282ae085e68fa5446f231b f7609ebf9b086beeb7ef204cb89a70c4a34fde84

20
isula-build.spec
View File

@ -2,7 +2,7 @@
Name: isula-build Name: isula-build
Version: 0.9.5 Version: 0.9.5
Release: 15 Release: 18
Summary: A tool to build container images Summary: A tool to build container images
License: Mulan PSL V2 License: Mulan PSL V2
URL: https://gitee.com/openeuler/isula-build URL: https://gitee.com/openeuler/isula-build
@ -85,6 +85,24 @@ fi
/usr/share/bash-completion/completions/isula-build /usr/share/bash-completion/completions/isula-build
%changelog %changelog
* Wed Nov 03 2021 lixiang <lixiang172@huawei.com> - 0.9.5-18
- Type:bugfix
- CVE:NA
- SUG:restart
- DESC:fix loaded images cover existing images name and tag
* Wed Nov 03 2021 DCCooper <1866858@gmail.com> - 0.9.5-17
- Type:bugfix
- CVE:NA
- SUG:restart
- DESC:optimize function IsExist and add more testcase for filepath.go
* Wed Nov 03 2021 DCCooper <1866858@gmail.com> - 0.9.5-16
- Type:bugfix
- CVE:NA
- SUG:restart
- DESC:fix random sequence for saving separated image tarball
* Tue Nov 02 2021 lixiang <lixiang172@huawei.com> - 0.9.5-15 * Tue Nov 02 2021 lixiang <lixiang172@huawei.com> - 0.9.5-15
- Type:requirement - Type:requirement
- CVE:NA - CVE:NA

43
patch/0086-bugfix-fix-random-sequence-for-saving-separated-imag.patch Normal file
View File

@ -0,0 +1,43 @@
From d6c302a3d5563286614c59a442f4cd65a8351ce2 Mon Sep 17 00:00:00 2001
From: DCCooper <1866858@gmail.com>
Date: Tue, 2 Nov 2021 20:54:24 +0800
Subject: [PATCH 1/2] bugfix: fix random sequence for saving separated image
tarball
reason: sort the map key and read the key in alohabetical orger
Signed-off-by: DCCooper <1866858@gmail.com>
---
daemon/save.go | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/daemon/save.go b/daemon/save.go
index ecac5b6..9ad4e03 100644
--- a/daemon/save.go
+++ b/daemon/save.go
@@ -20,6 +20,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
+ "sort"
"strings"
"github.com/containers/image/v5/docker/reference"
@@ -513,7 +514,13 @@ func (s *separatorSave) processImageLayers(imgInfos map[string]imageInfo) error
libImagesMap = make(imageLayersMap, 1)
appImagesMap = make(imageLayersMap, 1)
)
- for _, info := range imgInfos {
+ var sortedKey []string
+ for k := range imgInfos {
+ sortedKey = append(sortedKey, k)
+ }
+ sort.Strings(sortedKey)
+ for _, k := range sortedKey {
+ info := imgInfos[k]
if err := s.clearDirs(true); err != nil {
return errors.Wrap(err, "clear tmp dirs failed")
}
--
1.8.3.1

443
patch/0087-bugfix-optimize-function-IsExist.patch Normal file
View File

@ -0,0 +1,443 @@
From 6866f2e7f80ac9d8decf0e34a34de31df17c25aa Mon Sep 17 00:00:00 2001
From: DCCooper <1866858@gmail.com>
Date: Tue, 2 Nov 2021 20:59:35 +0800
Subject: [PATCH 2/2] bugfix: optimize function IsExist
reason: IsExist should return two value:
1. err: if err is not nil, which means the
input path is not valid, so the caller
should just return
2. true/false: this boolean value indicate the
path is exist or not, the value only valid
when no err occured
also add testcase for filepath.go file
Signed-off-by: DCCooper <1866858@gmail.com>
---
cmd/cli/build.go | 4 +-
cmd/cli/load.go | 4 +-
cmd/cli/save.go | 8 +-
cmd/daemon/main.go | 20 +++-
util/cipher.go | 4 +-
util/filepath.go | 18 ++--
util/filepath_test.go | 248 ++++++++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 289 insertions(+), 17 deletions(-)
create mode 100644 util/filepath_test.go
diff --git a/cmd/cli/build.go b/cmd/cli/build.go
index 3d9f549..b0f7765 100644
--- a/cmd/cli/build.go
+++ b/cmd/cli/build.go
@@ -235,7 +235,9 @@ func checkAbsPath(path string) (string, error) {
}
path = util.MakeAbsolute(path, pwd)
}
- if util.IsExist(path) {
+ if exist, err := util.IsExist(path); err != nil {
+ return "", err
+ } else if exist {
return "", errors.Errorf("output file already exist: %q, try to remove existing tarball or rename output file", path)
}
diff --git a/cmd/cli/load.go b/cmd/cli/load.go
index 44fefdd..90d189a 100644
--- a/cmd/cli/load.go
+++ b/cmd/cli/load.go
@@ -202,7 +202,9 @@ func (sep *separatorLoadOption) check(pwd string) error {
}
sep.dir = util.MakeAbsolute(sep.dir, pwd)
- if !util.IsExist(sep.dir) {
+ if exist, err := util.IsExist(sep.dir); err != nil {
+ return errors.Wrap(err, "resolve dir path failed")
+ } else if !exist {
return errors.Errorf("image tarball directory %q is not exist", sep.dir)
}
diff --git a/cmd/cli/save.go b/cmd/cli/save.go
index 599d394..5a63e02 100644
--- a/cmd/cli/save.go
+++ b/cmd/cli/save.go
@@ -112,7 +112,9 @@ func (sep *separatorSaveOption) check(pwd string) error {
sep.destPath = "Images"
}
sep.destPath = util.MakeAbsolute(sep.destPath, pwd)
- if util.IsExist(sep.destPath) {
+ if exist, err := util.IsExist(sep.destPath); err != nil {
+ return errors.Wrap(err, "check dest path failed")
+ } else if exist {
return errors.Errorf("dest path already exist: %q, try to remove or rename it", sep.destPath)
}
if len(sep.renameFile) != 0 {
@@ -162,7 +164,9 @@ func (opt *saveOptions) checkSaveOpts(args []string) error {
return err
}
opt.path = util.MakeAbsolute(opt.path, pwd)
- if util.IsExist(opt.path) {
+ if exist, err := util.IsExist(opt.path); err != nil {
+ return errors.Wrap(err, "check output path failed")
+ } else if exist {
return errors.Errorf("output file already exist: %q, try to remove existing tarball or rename output file", opt.path)
}
return nil
diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go
index 4fd5356..3665f6b 100644
--- a/cmd/daemon/main.go
+++ b/cmd/daemon/main.go
@@ -341,7 +341,9 @@ func setupWorkingDirectories() error {
return errors.Errorf("%q not an absolute dir, the \"dataroot\" and \"runroot\" must be an absolute path", dir)
}
- if !util.IsExist(dir) {
+ if exist, err := util.IsExist(dir); err != nil {
+ return err
+ } else if !exist {
if err := os.MkdirAll(dir, constant.DefaultRootDirMode); err != nil {
return errors.Wrapf(err, "create directory for %q failed", dir)
}
@@ -363,7 +365,9 @@ func setupWorkingDirectories() error {
func checkAndValidateConfig(cmd *cobra.Command) error {
// check if configuration.toml file exists, merge config if exists
- if !util.IsExist(constant.ConfigurationPath) {
+ if exist, err := util.IsExist(constant.ConfigurationPath); err != nil {
+ return err
+ } else if !exist {
logrus.Warnf("Main config file missing, the default configuration is used")
} else {
conf, err := loadConfig(constant.ConfigurationPath)
@@ -378,14 +382,18 @@ func checkAndValidateConfig(cmd *cobra.Command) error {
}
// file policy.json must be exist
- if !util.IsExist(constant.SignaturePolicyPath) {
+ if exist, err := util.IsExist(constant.SignaturePolicyPath); err != nil {
+ return err
+ } else if !exist {
return errors.Errorf("policy config file %v is not exist", constant.SignaturePolicyPath)
}
// check all config files
confFiles := []string{constant.RegistryConfigPath, constant.SignaturePolicyPath, constant.StorageConfigPath}
for _, file := range confFiles {
- if util.IsExist(file) {
+ if exist, err := util.IsExist(file); err != nil {
+ return err
+ } else if exist {
fi, err := os.Stat(file)
if err != nil {
return errors.Wrapf(err, "stat file %q failed", file)
@@ -402,7 +410,9 @@ func checkAndValidateConfig(cmd *cobra.Command) error {
}
// if storage config file exists, merge storage config
- if util.IsExist(constant.StorageConfigPath) {
+ if exist, err := util.IsExist(constant.StorageConfigPath); err != nil {
+ return err
+ } else if exist {
return mergeStorageConfig(cmd)
}
diff --git a/util/cipher.go b/util/cipher.go
index ce47b71..ecbbc47 100644
--- a/util/cipher.go
+++ b/util/cipher.go
@@ -185,7 +185,9 @@ func DecryptRSA(data string, key *rsa.PrivateKey, h crypto.Hash) (string, error)
// GenRSAPublicKeyFile store public key from rsa key pair into local file
func GenRSAPublicKeyFile(key *rsa.PrivateKey, path string) error {
- if IsExist(path) {
+ if exist, err := IsExist(path); err != nil {
+ return err
+ } else if exist {
if err := os.Remove(path); err != nil {
return errors.Errorf("failed to delete the residual key file: %v", err)
}
diff --git a/util/filepath.go b/util/filepath.go
index 59b22da..a10ed85 100644
--- a/util/filepath.go
+++ b/util/filepath.go
@@ -56,14 +56,18 @@ func IsDirectory(path string) bool {
return fi.IsDir()
}
-// IsExist returns true if the path exists
-func IsExist(path string) bool {
- if _, err := os.Lstat(path); err != nil {
- if os.IsNotExist(err) {
- return false
- }
+// IsExist returns true if the path exists when err is nil
+// and return false if path not exists when err is nil
+// Caller should focus on whether the err is nil or not
+func IsExist(path string) (bool, error) {
+ _, err := os.Lstat(path)
+ if err == nil {
+ return true, nil
+ }
+ if os.IsNotExist(err) {
+ return false, nil
}
- return true
+ return false, err
}
// IsSymbolFile returns true if the path file is a symbol file
diff --git a/util/filepath_test.go b/util/filepath_test.go
new file mode 100644
index 0000000..add4545
--- /dev/null
+++ b/util/filepath_test.go
@@ -0,0 +1,248 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// isula-build licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+// http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Xiang Li
+// Create: 2021-11-02
+// Description: testcase for filepath related common functions
+
+package util
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "gotest.tools/v3/assert"
+ constant "isula.org/isula-build"
+)
+
+func TestIsExist(t *testing.T) {
+ type args struct {
+ path string
+ workingDir string
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ isExist bool
+ wantErr bool
+ preHook func(t *testing.T, path string)
+ postHook func(t *testing.T)
+ }{
+ {
+ name: "TC-filename too long",
+ args: args{
+ path: strings.Repeat("a", 256),
+ workingDir: "/tmp",
+ },
+ want: filepath.Join("/tmp", strings.Repeat("a", 256)),
+ isExist: false,
+ wantErr: true,
+ },
+ {
+ name: "TC-filename valid",
+ args: args{
+ path: strings.Repeat("a", 255),
+ workingDir: "/tmp",
+ },
+ want: filepath.Join("/tmp", strings.Repeat("a", 255)),
+ isExist: false,
+ wantErr: false,
+ },
+ {
+ name: "TC-path too long",
+ args: args{
+ path: strings.Repeat(strings.Repeat("a", 256)+"/", 16),
+ workingDir: "/tmp",
+ },
+ want: filepath.Join("/tmp", strings.Repeat(strings.Repeat("a", 256)+"/", 16)) + "/",
+ isExist: false,
+ wantErr: true,
+ },
+ {
+ name: "TC-path exist",
+ args: args{
+ path: strings.Repeat(strings.Repeat("a", 255)+"/", 15),
+ workingDir: "/tmp",
+ },
+ want: filepath.Join("/tmp", strings.Repeat(strings.Repeat("a", 255)+"/", 15)) + "/",
+ isExist: true,
+ wantErr: false,
+ preHook: func(t *testing.T, path string) {
+ err := os.MkdirAll(path, constant.DefaultRootDirMode)
+ assert.NilError(t, err)
+ },
+ postHook: func(t *testing.T) {
+ err := os.RemoveAll(filepath.Join("/tmp", strings.Repeat("a", 255)+"/"))
+ assert.NilError(t, err)
+ },
+ },
+ {
+ name: "TC-path with dot exist",
+ args: args{
+ path: ".",
+ workingDir: filepath.Join("/tmp", strings.Repeat("./"+strings.Repeat("a", 255)+"/", 15)),
+ },
+ want: filepath.Join("/tmp", strings.Repeat(strings.Repeat("a", 255)+"/", 15)) + "/",
+ isExist: true,
+ wantErr: false,
+ preHook: func(t *testing.T, path string) {
+ err := os.MkdirAll(path, constant.DefaultRootDirMode)
+ assert.NilError(t, err)
+ },
+ postHook: func(t *testing.T) {
+ err := os.RemoveAll(filepath.Join("/tmp", strings.Repeat("a", 255)+"/"))
+ assert.NilError(t, err)
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := MakeAbsolute(tt.args.path, tt.args.workingDir)
+ if got != tt.want {
+ t.Errorf("MakeAbsolute() = %v, want %v", got, tt.want)
+ t.Skip()
+ }
+
+ if tt.preHook != nil {
+ tt.preHook(t, got)
+ }
+ exist, err := IsExist(got)
+ if exist != tt.isExist {
+ t.Errorf("IsExist() = %v, want %v", exist, tt.isExist)
+ }
+ if (err != nil) != tt.wantErr {
+ t.Errorf("IsExist() = %v, want %v", err, tt.wantErr)
+ }
+ if tt.postHook != nil {
+ tt.postHook(t)
+ }
+ })
+ }
+}
+
+func TestIsSymbolFile(t *testing.T) {
+ originFile := "/tmp/originFile"
+ symbolFile := "/tmp/symbolFile"
+ noneExistFile := "/tmp/none_exist_file"
+ type args struct {
+ path string
+ }
+ tests := []struct {
+ name string
+ args args
+ want bool
+ preHook func(t *testing.T)
+ postHook func(t *testing.T)
+ }{
+ {
+ name: "TC-is symbol file",
+ args: args{path: "/tmp/symbolFile"},
+ want: true,
+ preHook: func(t *testing.T) {
+ _, err := os.Create(originFile)
+ assert.NilError(t, err)
+ assert.NilError(t, os.Symlink(originFile, symbolFile))
+ },
+ postHook: func(t *testing.T) {
+ assert.NilError(t, os.RemoveAll(originFile))
+ assert.NilError(t, os.RemoveAll(symbolFile))
+ },
+ },
+ {
+ name: "TC-is normal file",
+ args: args{path: originFile},
+ want: false,
+ preHook: func(t *testing.T) {
+ _, err := os.Create(originFile)
+ assert.NilError(t, err)
+ },
+ postHook: func(t *testing.T) {
+ assert.NilError(t, os.RemoveAll(originFile))
+ },
+ },
+ {
+ name: "TC-file not exist",
+ args: args{path: noneExistFile},
+ want: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if tt.preHook != nil {
+ tt.preHook(t)
+ }
+ if got := IsSymbolFile(tt.args.path); got != tt.want {
+ t.Errorf("IsSymbolFile() = %v, want %v", got, tt.want)
+ }
+ if tt.postHook != nil {
+ tt.postHook(t)
+ }
+ })
+ }
+}
+
+func TestIsDirectory(t *testing.T) {
+ dirPath := filepath.Join("/tmp", t.Name())
+ filePath := filepath.Join("/tmp", t.Name())
+ noneExistFile := "/tmp/none_exist_file"
+
+ type args struct {
+ path string
+ }
+ tests := []struct {
+ name string
+ args args
+ want bool
+ preHook func(t *testing.T)
+ postHook func(t *testing.T)
+ }{
+ {
+ name: "TC-is directory",
+ args: args{path: dirPath},
+ preHook: func(t *testing.T) {
+ assert.NilError(t, os.MkdirAll(dirPath, constant.DefaultRootDirMode))
+ },
+ postHook: func(t *testing.T) {
+ assert.NilError(t, os.RemoveAll(dirPath))
+ },
+ want: true,
+ },
+ {
+ name: "TC-is file",
+ args: args{path: dirPath},
+ preHook: func(t *testing.T) {
+ _, err := os.Create(filePath)
+ assert.NilError(t, err)
+ },
+ postHook: func(t *testing.T) {
+ assert.NilError(t, os.RemoveAll(filePath))
+ },
+ },
+ {
+ name: "TC-path not exist",
+ args: args{path: noneExistFile},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if tt.preHook != nil {
+ tt.preHook(t)
+ }
+ if got := IsDirectory(tt.args.path); got != tt.want {
+ t.Errorf("IsDirectory() = %v, want %v", got, tt.want)
+ }
+ if tt.postHook != nil {
+ tt.postHook(t)
+ }
+ })
+ }
+}
--
1.8.3.1

434
patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch Normal file
View File

@ -0,0 +1,434 @@
From 6efcb2e785e452505894b0e1e589e72487439e17 Mon Sep 17 00:00:00 2001
From: xingweizheng <xingweizheng@huawei.com>
Date: Wed, 27 Oct 2021 18:27:11 +0800
Subject: [PATCH] bugfix: loaded images cover existing images name and tag
---
daemon/load.go | 114 +++++++++++++++++++++++++++++++-------------
daemon/load_test.go | 26 +++++-----
image/image.go | 66 ++++++++++++-------------
3 files changed, 129 insertions(+), 77 deletions(-)
diff --git a/daemon/load.go b/daemon/load.go
index 41690ab..b6d675b 100644
--- a/daemon/load.go
+++ b/daemon/load.go
@@ -17,6 +17,8 @@ import (
"io/ioutil"
"os"
"path/filepath"
+ "context"
+ "strings"
"github.com/containers/image/v5/docker/tarfile"
ociarchive "github.com/containers/image/v5/oci/archive"
@@ -25,6 +27,7 @@ import (
"github.com/containers/storage"
"github.com/containers/storage/pkg/archive"
securejoin "github.com/cyphar/filepath-securejoin"
+ digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@@ -72,6 +75,12 @@ type loadOptions struct {
sep separatorLoad
}
+type singleImage struct {
+ index int
+ id string
+ nameTag []string
+}
+
func (b *Backend) getLoadOptions(req *pb.LoadRequest) (loadOptions, error) {
var opt = loadOptions{
path: req.GetPath(),
@@ -119,10 +128,8 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error
"LoadID": req.GetLoadID(),
}).Info("LoadRequest received")
- var (
- si *storage.Image
- repoTags [][]string
- )
+ var si *storage.Image
+
opts, err := b.getLoadOptions(req)
if err != nil {
return errors.Wrap(err, "process load options failed")
@@ -142,7 +149,7 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error
}
}
- repoTags, err = tryToParseImageFormatFromTarball(b.daemon.opts.DataRoot, &opts)
+ imagesInTar, err := tryToParseImageFormatFromTarball(b.daemon.opts.DataRoot, &opts)
if err != nil {
return err
}
@@ -163,24 +170,30 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error
eg.Go(func() error {
defer log.CloseContent()
- for index, nameAndTag := range repoTags {
+ for _, singleImage := range imagesInTar {
_, si, err = image.ResolveFromImage(&image.PrepareImageOptions{
Ctx: ctx,
FromImage: exporter.FormatTransport(opts.format, opts.path),
+ ToImage: singleImage.id,
SystemContext: image.GetSystemContext(),
Store: b.daemon.localStore,
Reporter: log,
- ManifestIndex: index,
+ ManifestIndex: singleImage.index,
})
if err != nil {
return err
}
- if sErr := b.daemon.localStore.SetNames(si.ID, nameAndTag); sErr != nil {
- return sErr
+ originalNames, err := b.daemon.localStore.Names(si.ID)
+ if err != nil {
+ return err
+ }
+ if err = b.daemon.localStore.SetNames(si.ID, append(originalNames, singleImage.nameTag...)); err != nil {
+ return err
}
log.Print("Loaded image as %s\n", si.ID)
+ logrus.Infof("Loaded image as %s", si.ID)
}
return nil
@@ -189,17 +202,11 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error
if wErr := eg.Wait(); wErr != nil {
return wErr
}
- logrus.Infof("Loaded image as %s", si.ID)
return nil
}
-func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]string, error) {
- var (
- allRepoTags [][]string
- err error
- )
-
+func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([]singleImage, error) {
// tmp dir will be removed after NewSourceFromFileWithContext
tmpDir, err := securejoin.SecureJoin(dataRoot, dataRootTmpDirPrefix)
if err != nil {
@@ -208,19 +215,21 @@ func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]s
systemContext := image.GetSystemContext()
systemContext.BigFilesTemporaryDir = tmpDir
- allRepoTags, err = getDockerRepoTagFromImageTar(systemContext, opts.path)
+ // try docker format loading
+ imagesInTar, err := getDockerRepoTagFromImageTar(systemContext, opts.path)
if err == nil {
logrus.Infof("Parse image successful with %q format", constant.DockerTransport)
opts.format = constant.DockerArchiveTransport
- return allRepoTags, nil
+ return imagesInTar, nil
}
logrus.Warnf("Try to Parse image of docker format failed with error: %v", err)
- allRepoTags, err = getOCIRepoTagFromImageTar(systemContext, opts.path)
+ // try oci format loading
+ imagesInTar, err = getOCIRepoTagFromImageTar(systemContext, opts.path)
if err == nil {
logrus.Infof("Parse image successful with %q format", constant.OCITransport)
opts.format = constant.OCIArchiveTransport
- return allRepoTags, nil
+ return imagesInTar, nil
}
logrus.Warnf("Try to parse image of oci format failed with error: %v", err)
@@ -228,7 +237,7 @@ func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([][]s
return nil, errors.Wrap(err, "wrong image format detected from local tarball")
}
-func getDockerRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([][]string, error) {
+func getDockerRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([]singleImage, error) {
// tmp dir will be removed after NewSourceFromFileWithContext
tarfileSource, err := tarfile.NewSourceFromFileWithContext(systemContext, path)
if err != nil {
@@ -245,35 +254,74 @@ func getDockerRepoTagFromImageTar(systemContext *types.SystemContext, path strin
return nil, errors.Errorf("failed to get the top level image manifest: %v", err)
}
- var allRepoTags [][]string
- for _, manifestItem := range topLevelImageManifest {
- allRepoTags = append(allRepoTags, manifestItem.RepoTags)
+ imagesInTar := make([]singleImage, 0, len(topLevelImageManifest))
+ for i, manifestItem := range topLevelImageManifest {
+ imageID, err := parseConfigID(manifestItem.Config)
+ if err != nil {
+ return nil, err
+ }
+ imagesInTar = append(imagesInTar, singleImage{index: i, id: imageID, nameTag: manifestItem.RepoTags})
}
- return allRepoTags, nil
+ return imagesInTar, nil
}
-func getOCIRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([][]string, error) {
- var (
- err error
- )
-
+func getOCIRepoTagFromImageTar(systemContext *types.SystemContext, path string) ([]singleImage, error) {
srcRef, err := alltransports.ParseImageName(exporter.FormatTransport(constant.OCIArchiveTransport, path))
if err != nil {
return nil, errors.Wrap(err, "failed to parse image name of oci image format")
}
+ imageID, err := getLoadedImageID(srcRef, systemContext)
+ if err != nil {
+ return nil, err
+ }
tarManifest, err := ociarchive.LoadManifestDescriptorWithContext(systemContext, srcRef)
if err != nil {
return nil, errors.Wrap(err, "failed to load manifest descriptor of oci image format")
}
- // For now, we only support load single image in archive file
+ // For now, we only support loading oci-archive file with one single image
if _, ok := tarManifest.Annotations[imgspecv1.AnnotationRefName]; ok {
- return [][]string{{tarManifest.Annotations[imgspecv1.AnnotationRefName]}}, nil
+ return []singleImage{{0, imageID, []string{tarManifest.Annotations[imgspecv1.AnnotationRefName]}}}, nil
+ }
+ return []singleImage{{0, imageID, []string{}}}, nil
+}
+
+func parseConfigID(configID string) (string, error) {
+ parts := strings.SplitN(configID, ".", 2)
+ if len(parts) != 2 {
+ return "", errors.New("wrong config info of manifest.json")
+ }
+
+ configDigest := "sha256:" + digest.Digest(parts[0])
+ if err := configDigest.Validate(); err != nil {
+ return "", errors.Wrapf(err, "failed to get config info")
+ }
+
+ return "@" + configDigest.Encoded(), nil
+}
+
+func getLoadedImageID(imageRef types.ImageReference, systemContext *types.SystemContext) (string, error) {
+ if imageRef == nil || systemContext == nil {
+ return "", errors.New("nil image reference or system context when loading image")
+ }
+
+ newImage, err := imageRef.NewImage(context.TODO(), systemContext)
+ if err != nil {
+ return "", err
+ }
+ defer func() {
+ if err = newImage.Close(); err != nil {
+ logrus.Errorf("failed to close image: %v", err)
+ }
+ }()
+ imageDigest := newImage.ConfigInfo().Digest
+ if err = imageDigest.Validate(); err != nil {
+ return "", errors.Wrap(err, "failed to get config info")
}
- return [][]string{{}}, nil
+ return "@" + imageDigest.Encoded(), nil
}
func loadSeparatedImage(opt *loadOptions) error {
diff --git a/daemon/load_test.go b/daemon/load_test.go
index cbcb5d8..860b897 100644
--- a/daemon/load_test.go
+++ b/daemon/load_test.go
@@ -31,9 +31,9 @@ import (
)
const (
- loadedTarFile = "load.tar"
+ loadedTarFile = "load.tar"
manifestJSONFile = "manifest.json"
- indexJSONFile = "index.json"
+ indexJSONFile = "index.json"
)
var (
@@ -167,8 +167,10 @@ func TestLoadSingleImage(t *testing.T) {
}
]
}`,
- format: "oci",
- withTag: true,
+ format: "oci",
+ withTag: true,
+ wantErr: true,
+ errString: "no such file or directory",
},
{
name: "TC3 normal case load docker tar with no RepoTags",
@@ -197,8 +199,10 @@ func TestLoadSingleImage(t *testing.T) {
}
]
}`,
- format: "oci",
- withTag: false,
+ format: "oci",
+ withTag: false,
+ wantErr: true,
+ errString: "no such file or directory",
},
{
name: "TC5 abnormal case load docker tar with wrong manifestJSON",
@@ -217,7 +221,7 @@ func TestLoadSingleImage(t *testing.T) {
format: "docker",
withTag: true,
wantErr: true,
- errString: "error loading index",
+ errString: "no such file or directory",
},
{
name: "TC6 abnormal case with wrong tar path",
@@ -312,10 +316,10 @@ func TestLoadMultipleImages(t *testing.T) {
path := dir.Join(loadedTarFile)
repoTags, err := tryToParseImageFormatFromTarball(daemon.opts.DataRoot, &loadOptions{path: path})
assert.NilError(t, err)
- assert.Equal(t, repoTags[0][0], "registry.example.com/sayhello:first")
- assert.Equal(t, repoTags[1][0], "registry.example.com/sayhello:second")
- assert.Equal(t, repoTags[1][1], "registry.example.com/sayhello:third")
- assert.Equal(t, len(repoTags[2]), 0)
+ assert.Equal(t, repoTags[0].nameTag[0], "registry.example.com/sayhello:first")
+ assert.Equal(t, repoTags[1].nameTag[0], "registry.example.com/sayhello:second")
+ assert.Equal(t, repoTags[1].nameTag[1], "registry.example.com/sayhello:third")
+ assert.Equal(t, len(repoTags[2].nameTag), 0)
req := &pb.LoadRequest{Path: path}
stream := &controlLoadServer{}
diff --git a/image/image.go b/image/image.go
index 5dda185..b24cb41 100644
--- a/image/image.go
+++ b/image/image.go
@@ -37,7 +37,6 @@ import (
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
- "github.com/containers/storage/pkg/stringid"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@@ -55,6 +54,7 @@ type PrepareImageOptions struct {
SystemContext *types.SystemContext
Ctx context.Context
FromImage string
+ ToImage string
Store *store.Store
Reporter io.Writer
ManifestIndex int
@@ -125,20 +125,17 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora
}
if transport == "" {
- // if the image can be obtained from the local storage by image id,
- // then only one image can be obtained.
+ // if the image can be obtained from the local storage by image id, then only one image can be obtained.
if len(candidates) != 1 {
return nil, nil, errors.New("transport is empty and multi or no image be found")
}
- img, err := opt.Store.Image(candidates[0])
- if err != nil {
- pLog.Errorf("Failed to find the image %q in local store: %v", candidates[0], err)
- return nil, nil, err
- }
- ref, err := is.Transport.ParseStoreReference(opt.Store, img.ID)
- if err != nil {
- return nil, nil, errors.Wrapf(err, "failed to get the ref in store by %q", candidates[0])
+
+ ref, img, fErr := FindImage(opt.Store, candidates[0])
+ if fErr != nil {
+ pLog.Errorf("Failed to find the image %q in local store: %v", candidates[0], fErr)
+ return nil, nil, fErr
}
+
pLog.Infof("Get image from local store first search by %q", opt.FromImage)
return ref, img, nil
}
@@ -147,25 +144,33 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora
var errPull error
for _, strImage := range candidates {
var (
- srcRef types.ImageReference
- pErr error
+ srcRef types.ImageReference
+ destImage string
)
imageName := exporter.FormatTransport(transport, strImage)
- if transport == constant.DockerArchiveTransport {
- srcRef, pErr = alltransports.ParseImageName(imageName + ":@" + strconv.Itoa(opt.ManifestIndex))
- } else {
- srcRef, pErr = alltransports.ParseImageName(imageName)
- }
- if pErr != nil {
- pLog.Debugf("Failed to parse the image %q: %v", imageName, pErr)
- continue
- }
-
- destImage, err := getLocalImageNameFromRef(opt.Store, srcRef)
- if err != nil {
- pLog.Debugf("Failed to get local image name for %q: %v", imageName, err)
- continue
+ switch transport {
+ case constant.DockerArchiveTransport:
+ if srcRef, err = alltransports.ParseImageName(imageName + ":@" + strconv.Itoa(opt.ManifestIndex)); err != nil {
+ pLog.Debugf("Failed to parse the image %q with %q transport: %v", imageName, constant.DockerArchiveTransport, err)
+ continue
+ }
+ destImage = opt.ToImage
+ case constant.OCIArchiveTransport:
+ if srcRef, err = alltransports.ParseImageName(imageName); err != nil {
+ pLog.Debugf("Failed to parse the image %q with %q transport: %v", imageName, constant.OCIArchiveTransport, err)
+ continue
+ }
+ destImage = opt.ToImage
+ default:
+ if srcRef, err = alltransports.ParseImageName(imageName); err != nil {
+ pLog.Debugf("Failed to get local image name for %q: %v", imageName, err)
+ continue
+ }
+ if destImage, err = getLocalImageNameFromRef(srcRef); err != nil {
+ pLog.Debugf("Failed to parse store reference for %q: %v", destImage, err)
+ continue
+ }
}
destRef, err := is.Transport.ParseStoreReference(opt.Store, destImage)
@@ -173,7 +178,6 @@ func PullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *stora
pLog.Debugf("Failed to parse store reference for %q: %v", destImage, err)
continue
}
-
img, err := is.Transport.GetStoreImage(opt.Store, destRef)
if err == nil {
// find the unique image in local store by name or digest
@@ -246,14 +250,10 @@ func instantiatingImage(ctx context.Context, sc *types.SystemContext, ref types.
return baseImg, nil
}
-func getLocalImageNameFromRef(store storage.Store, srcRef types.ImageReference) (string, error) {
+func getLocalImageNameFromRef(srcRef types.ImageReference) (string, error) {
if srcRef == nil {
return "", errors.Errorf("reference to image is empty")
}
-
- if err := exporter.CheckArchiveFormat(srcRef.Transport().Name()); err == nil {
- return stringid.GenerateRandomID() + ":" + stringid.GenerateRandomID(), nil
- }
if srcRef.Transport().Name() != constant.DockerTransport {
return "", errors.Errorf("the %s transport is not supported yet", srcRef.Transport().Name())
}
--
2.27.0

3
series.conf
View File

@ -48,3 +48,6 @@ patch/0082-test-optimize-scripts-in-hack.patch
patch/0083-test-add-common-function-for-testing-separated-image.patch patch/0083-test-add-common-function-for-testing-separated-image.patch
patch/0084-test-add-integration-tests-for-saving-and-loading-se.patch patch/0084-test-add-integration-tests-for-saving-and-loading-se.patch
patch/0085-util-add-unit-test-for-increment-util-functions.patch patch/0085-util-add-unit-test-for-increment-util-functions.patch
patch/0086-bugfix-fix-random-sequence-for-saving-separated-imag.patch
patch/0087-bugfix-optimize-function-IsExist.patch
patch/0088-bugfix-loaded-images-cover-existing-images-name-and-.patch