packages/isula-build
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
isula-build/patch/0099-Refactor-refactor-image-separator-related.patch

2344 lines
71 KiB
Diff
Raw Normal View History

isula-build: refactor image separator Signed-off-by: jingxiaolu <lujingxiao@huawei.com>
2021-12-31 17:46:43 +08:00
From c4f400ec1cfaf65d3e83dbd796f1f2b00574ba6e Mon Sep 17 00:00:00 2001
From: Lu Jingxiao <lujingxiao@huawei.com>
Date: Fri, 10 Dec 2021 13:36:26 +0800
Subject: [PATCH] Refactor: refactor image separator related
Signed-off-by: Lu Jingxiao <lujingxiao@huawei.com>
---
cmd/cli/save.go | 4 +-
cmd/daemon/main.go | 7 +-
constant.go | 26 +-
daemon/daemon.go | 5 +-
daemon/import.go | 4 +-
daemon/load.go | 295 ++-------------
daemon/load_test.go | 2 +-
daemon/save.go | 649 +--------------------------------
daemon/separator/image_info.go | 235 ++++++++++++
daemon/separator/load.go | 283 ++++++++++++++
daemon/separator/save.go | 407 +++++++++++++++++++++
daemon/separator/utils.go | 78 ++++
12 files changed, 1074 insertions(+), 921 deletions(-)
create mode 100644 daemon/separator/image_info.go
create mode 100644 daemon/separator/load.go
create mode 100644 daemon/separator/save.go
create mode 100644 daemon/separator/utils.go
diff --git a/cmd/cli/save.go b/cmd/cli/save.go
index 5a63e02..a30681f 100644
--- a/cmd/cli/save.go
+++ b/cmd/cli/save.go
@@ -144,8 +144,8 @@ func (opt *saveOptions) checkSaveOpts(args []string) error {
}
// separate image only support docker image spec
opt.format = constant.DockerTransport
- if err := opt.sep.check(pwd); err != nil {
- return err
+ if cerr := opt.sep.check(pwd); cerr != nil {
+ return cerr
}
opt.sep.enabled = true
diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go
index 3665f6b..3cecbf9 100644
--- a/cmd/daemon/main.go
+++ b/cmd/daemon/main.go
@@ -36,10 +36,7 @@ import (
"isula.org/isula-build/util"
)
-const (
- lockFileName = "isula-builder.lock"
- dataRootTmpDirPrefix = "tmp"
-)
+const lockFileName = "isula-builder.lock"
var daemonOpts daemon.Options
@@ -331,7 +328,7 @@ func setupWorkingDirectories() error {
return errors.Errorf("runroot(%q) and dataroot(%q) must be different paths", daemonOpts.RunRoot, daemonOpts.DataRoot)
}
- buildTmpDir, err := securejoin.SecureJoin(daemonOpts.DataRoot, dataRootTmpDirPrefix)
+ buildTmpDir, err := securejoin.SecureJoin(daemonOpts.DataRoot, constant.DataRootTmpDirPrefix)
if err != nil {
return err
}
diff --git a/constant.go b/constant.go
index 4d1596a..5af4fe2 100644
--- a/constant.go
+++ b/constant.go
@@ -31,14 +31,20 @@ const (
RegistryDirPath = ConfigRoot + "registries.d"
// AuthFilePath is authentication file used for registry connection
AuthFilePath = ConfigRoot + "auth.json"
- // DefaultGRPCAddress is the local unix socket used by isula-builder
- DefaultGRPCAddress = "unix:///var/run/isula_build.sock"
- // UnixPrefix is the prefix used on defined an unix sock
- UnixPrefix = "unix://"
+ // DefaultCertRoot is path of certification used for registry connection
+ DefaultCertRoot = ConfigRoot + "certs.d"
+
// DefaultDataRoot is the default persistent data root used by isula-builder
DefaultDataRoot = "/var/lib/isula-build"
// DefaultRunRoot is the default run root used by isula-builder
DefaultRunRoot = "/var/run/isula-build"
+ // UnixPrefix is the prefix used on defined an unix sock
+ UnixPrefix = "unix://"
+ // DefaultGRPCAddress is the local unix socket used by isula-builder
+ DefaultGRPCAddress = UnixPrefix + "/var/run/isula_build.sock"
+ // DataRootTmpDirPrefix is the dir for storing temporary items using during images building
+ DataRootTmpDirPrefix = "tmp"
+
// DefaultSharedDirMode is dir perm mode with higher permission
DefaultSharedDirMode = 0755
// DefaultSharedFileMode is file perm mode with higher permission
@@ -53,12 +59,14 @@ const (
DefaultReadOnlyFileMode = 0400
// DefaultUmask is the working umask of isula-builder as a process, not for users
DefaultUmask = 0022
- // CliLogBufferLen is log channel buffer size
- CliLogBufferLen = 8
+
// HostsFilePath is the path of file hosts
HostsFilePath = "/etc/hosts"
// ResolvFilePath is the path of file resolv.conf
ResolvFilePath = "/etc/resolv.conf"
+
+ // CliLogBufferLen is log channel buffer size
+ CliLogBufferLen = 8
// MaxFileSize is the maximum file size allowed, set 1M
MaxFileSize = 1024 * 1024
// DefaultHTTPTimeout includes the total time of dial, TLS handshake, request, resp headers and body
@@ -67,14 +75,12 @@ const (
DefaultFailedCode = 1
// DefaultIDLen is the ID length for image ID and build ID
DefaultIDLen = 12
- // DefaultCertRoot is path of certification used for registry connection
- DefaultCertRoot = ConfigRoot + "certs.d"
+
// LayoutTime is the time format used to parse time from a string
LayoutTime = "2006-01-02 15:04:05"
// BuildContainerImageType is the default build type
BuildContainerImageType = "ctr-img"
- // BufferSize is default buffer size for file transportation
- BufferSize = 32 * 1024
+
// DockerTransport used to export docker image format images to registry
DockerTransport = "docker"
// DockerArchiveTransport used to export docker image format images to local tarball
diff --git a/daemon/daemon.go b/daemon/daemon.go
index 4e0435a..1847dbb 100644
--- a/daemon/daemon.go
+++ b/daemon/daemon.go
@@ -28,6 +28,7 @@ import (
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
+ constant "isula.org/isula-build"
pb "isula.org/isula-build/api/services"
"isula.org/isula-build/builder"
"isula.org/isula-build/pkg/gc"
@@ -37,8 +38,6 @@ import (
"isula.org/isula-build/util"
)
-const dataRootTmpDirPrefix = "tmp"
-
// Options carries the options configured to daemon
type Options struct {
Debug bool
@@ -137,7 +136,7 @@ func (d *Daemon) NewBuilder(ctx context.Context, req *pb.BuildRequest) (b builde
runDir string
)
// buildDir is used to set directory which is used to store tmp data
- buildDir, err = securejoin.SecureJoin(d.opts.DataRoot, filepath.Join(dataRootTmpDirPrefix, req.BuildID))
+ buildDir, err = securejoin.SecureJoin(d.opts.DataRoot, filepath.Join(constant.DataRootTmpDirPrefix, req.BuildID))
if err != nil {
return nil, err
}
diff --git a/daemon/import.go b/daemon/import.go
index 2da36be..21ab729 100644
--- a/daemon/import.go
+++ b/daemon/import.go
@@ -87,7 +87,7 @@ func (b *Backend) Import(req *pb.ImportRequest, stream pb.Control_ImportServer)
log := logger.NewCliLogger(constant.CliLogBufferLen)
imageCopyOptions := image.NewImageCopyOptions(log)
- tmpDir, err = securejoin.SecureJoin(b.daemon.opts.DataRoot, filepath.Join(dataRootTmpDirPrefix, importID))
+ tmpDir, err = securejoin.SecureJoin(b.daemon.opts.DataRoot, filepath.Join(constant.DataRootTmpDirPrefix, importID))
if err != nil {
return err
}
@@ -97,7 +97,7 @@ func (b *Backend) Import(req *pb.ImportRequest, stream pb.Control_ImportServer)
}
defer func() {
if rErr := os.RemoveAll(tmpDir); rErr != nil {
- logEntry.Errorf("Failed to remove import temporary dir %q, err: %v", filepath.Join(dataRootTmpDirPrefix, importID), rErr)
+ logEntry.Errorf("Failed to remove import temporary dir %q, err: %v", filepath.Join(constant.DataRootTmpDirPrefix, importID), rErr)
}
}()
diff --git a/daemon/load.go b/daemon/load.go
index 894159b..1ee025b 100644
--- a/daemon/load.go
+++ b/daemon/load.go
@@ -15,9 +15,7 @@ package daemon
import (
"context"
- "io/ioutil"
"os"
- "path/filepath"
"strings"
"github.com/containers/image/v5/docker/tarfile"
@@ -25,7 +23,6 @@ import (
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
- "github.com/containers/storage/pkg/archive"
securejoin "github.com/cyphar/filepath-securejoin"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -35,89 +32,47 @@ import (
constant "isula.org/isula-build"
pb "isula.org/isula-build/api/services"
+ "isula.org/isula-build/daemon/separator"
"isula.org/isula-build/exporter"
"isula.org/isula-build/image"
"isula.org/isula-build/pkg/logger"
"isula.org/isula-build/util"
)
-const (
- tmpBaseDirName = "base"
- tmpAppDirName = "app"
- tmpLibDirName = "lib"
- unionCompressedTarName = "all.tar.gz"
-)
-
-type loadImageTmpDir struct {
- app string
- base string
- lib string
- root string
-}
-
-type separatorLoad struct {
- log *logrus.Entry
- tmpDir loadImageTmpDir
- info tarballInfo
- appName string
- basePath string
- appPath string
- libPath string
- dir string
- skipCheck bool
- enabled bool
-}
-
-type loadOptions struct {
- logEntry *logrus.Entry
- path string
- format string
- sep separatorLoad
-}
-
type singleImage struct {
index int
id string
nameTag []string
}
-func (b *Backend) getLoadOptions(req *pb.LoadRequest) (loadOptions, error) {
- var opt = loadOptions{
- path: req.GetPath(),
- sep: separatorLoad{
- appName: req.GetSep().GetApp(),
- basePath: req.GetSep().GetBase(),
- libPath: req.GetSep().GetLib(),
- dir: req.GetSep().GetDir(),
- skipCheck: req.GetSep().GetSkipCheck(),
- enabled: req.GetSep().GetEnabled(),
- },
- logEntry: logrus.WithFields(logrus.Fields{"LoadID": req.GetLoadID()}),
+// LoadOptions stores the options for image loading
+type LoadOptions struct {
+ LogEntry *logrus.Entry
+ path string
+ format string
+ sep separator.Loader
+}
+
+func (b *Backend) getLoadOptions(req *pb.LoadRequest) (LoadOptions, error) {
+ var err error
+ var opt = LoadOptions{
+ path: req.GetPath(),
+ LogEntry: logrus.WithFields(logrus.Fields{"LoadID": req.GetLoadID()}),
}
- // normal loadOptions
- if !opt.sep.enabled {
- if err := util.CheckLoadFile(opt.path); err != nil {
- return loadOptions{}, err
+ // normal image loading
+ if !req.GetSep().GetEnabled() {
+ if err = util.CheckLoadFile(opt.path); err != nil {
+ return LoadOptions{}, err
}
return opt, nil
}
- // load separated images
- // log is used for sep methods
- opt.sep.log = opt.logEntry
- tmpRoot := filepath.Join(b.daemon.opts.DataRoot, filepath.Join(dataRootTmpDirPrefix, req.GetLoadID()))
- opt.sep.tmpDir.root = tmpRoot
- opt.sep.tmpDir.base = filepath.Join(tmpRoot, tmpBaseDirName)
- opt.sep.tmpDir.app = filepath.Join(tmpRoot, tmpAppDirName)
- opt.sep.tmpDir.lib = filepath.Join(tmpRoot, tmpLibDirName)
-
- // check image name and add "latest" tag if not present
- _, appImgName, err := image.GetNamedTaggedReference(opt.sep.appName)
+ // separated images loading
+ opt.sep, err = separator.GetSepLoadOptions(req, opt.LogEntry, b.daemon.opts.DataRoot)
if err != nil {
- return loadOptions{}, err
+ return LoadOptions{}, err
}
- opt.sep.appName = appImgName
return opt, nil
}
@@ -136,15 +91,16 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error
}
defer func() {
- if tErr := os.RemoveAll(opts.sep.tmpDir.root); tErr != nil {
- opts.logEntry.Warnf("Removing load tmp directory %q failed: %v", opts.sep.tmpDir.root, tErr)
+ if tErr := os.RemoveAll(opts.sep.TmpDirRoot()); tErr != nil {
+ opts.LogEntry.Warnf("Removing load tmp directory %q failed: %v", opts.sep.TmpDirRoot(), tErr)
}
}()
// construct separated images
- if opts.sep.enabled {
- if lErr := loadSeparatedImage(&opts); lErr != nil {
- opts.logEntry.Errorf("Load separated image for %s failed: %v", opts.sep.appName, lErr)
+ if opts.sep.Enabled() {
+ var lErr error
+ if opts.path, lErr = opts.sep.LoadSeparatedImage(); lErr != nil {
+ opts.LogEntry.Errorf("Load separated image for %s failed: %v", opts.sep.AppName(), lErr)
return lErr
}
}
@@ -206,9 +162,9 @@ func (b *Backend) Load(req *pb.LoadRequest, stream pb.Control_LoadServer) error
return nil
}
-func tryToParseImageFormatFromTarball(dataRoot string, opts *loadOptions) ([]singleImage, error) {
+func tryToParseImageFormatFromTarball(dataRoot string, opts *LoadOptions) ([]singleImage, error) {
// tmp dir will be removed after NewSourceFromFileWithContext
- tmpDir, err := securejoin.SecureJoin(dataRoot, dataRootTmpDirPrefix)
+ tmpDir, err := securejoin.SecureJoin(dataRoot, constant.DataRootTmpDirPrefix)
if err != nil {
return nil, err
}
@@ -323,196 +279,3 @@ func getLoadedImageID(imageRef types.ImageReference, systemContext *types.System
return "@" + imageDigest.Encoded(), nil
}
-
-func loadSeparatedImage(opt *loadOptions) error {
- s := &opt.sep
- s.log.Infof("Starting load separated image %s", s.appName)
-
- // load manifest file to get tarball info
- if err := s.getTarballInfo(); err != nil {
- return errors.Wrap(err, "failed to get tarball info")
- }
- if err := s.constructTarballInfo(); err != nil {
- return err
- }
- // checksum for image tarballs
- if err := s.tarballCheckSum(); err != nil {
- return err
- }
- // process image tarballs and get final constructed image tarball
- tarPath, err := s.processTarballs()
- if err != nil {
- return err
- }
- opt.path = tarPath
-
- return nil
-}
-
-func (s *separatorLoad) getTarballInfo() error {
- manifest, err := securejoin.SecureJoin(s.dir, manifestFile)
- if err != nil {
- return errors.Wrap(err, "join manifest file path failed")
- }
-
- var t = make(map[string]tarballInfo, 1)
- if err = util.LoadJSONFile(manifest, &t); err != nil {
- return errors.Wrap(err, "load manifest file failed")
- }
-
- tarball, ok := t[s.appName]
- if !ok {
- return errors.Errorf("failed to find app image %s", s.appName)
- }
- s.info = tarball
-
- return nil
-}
-
-func (s *separatorLoad) constructTarballInfo() (err error) {
- s.log.Infof("Construct image tarball info for %s", s.appName)
- // fill up path for separator
- // this case should not happened since client side already check this flag
- if len(s.appName) == 0 {
- return errors.New("app image name should not be empty")
- }
- s.appPath, err = securejoin.SecureJoin(s.dir, s.info.AppTarName)
- if err != nil {
- return err
- }
-
- if len(s.basePath) == 0 {
- if len(s.info.BaseTarName) == 0 {
- return errors.Errorf("base image %s tarball can not be empty", s.info.BaseImageName)
- }
- s.log.Info("Base image path is empty, use path from manifest")
- s.basePath, err = securejoin.SecureJoin(s.dir, s.info.BaseTarName)
- if err != nil {
- return err
- }
- }
- if len(s.libPath) == 0 && len(s.info.LibTarName) != 0 {
- s.log.Info("Lib image path is empty, use path from manifest")
- s.libPath, err = securejoin.SecureJoin(s.dir, s.info.LibTarName)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func (s *separatorLoad) tarballCheckSum() error {
- if s.skipCheck {
- s.log.Info("Skip checksum for tarballs")
- return nil
- }
-
- type checkInfo struct {
- path string
- hash string
- str string
- canBeEmpty bool
- }
- checkLen := 3
- var checkList = make([]checkInfo, 0, checkLen)
- checkList = append(checkList, checkInfo{path: s.basePath, hash: s.info.BaseHash, canBeEmpty: false, str: "base image"})
- checkList = append(checkList, checkInfo{path: s.libPath, hash: s.info.LibHash, canBeEmpty: true, str: "lib image"})
- checkList = append(checkList, checkInfo{path: s.appPath, hash: s.info.AppHash, canBeEmpty: false, str: "app image"})
- for _, p := range checkList {
- if len(p.path) == 0 && !p.canBeEmpty {
- return errors.Errorf("%s tarball path can not be empty", p.str)
- }
- if len(p.path) != 0 {
- if err := util.CheckSum(p.path, p.hash); err != nil {
- return errors.Wrapf(err, "check sum for file %q failed", p.path)
- }
- }
- }
-
- return nil
-}
-
-func (s *separatorLoad) processTarballs() (string, error) {
- if err := s.unpackTarballs(); err != nil {
- return "", err
- }
-
- if err := s.reconstructImage(); err != nil {
- return "", err
- }
-
- // pack app image to tarball
- tarPath := filepath.Join(s.tmpDir.root, unionCompressedTarName)
- if err := util.PackFiles(s.tmpDir.base, tarPath, archive.Gzip, true); err != nil {
- return "", err
- }
-
- return tarPath, nil
-}
-
-func (s *separatorLoad) unpackTarballs() error {
- if err := s.makeTempDir(); err != nil {
- return errors.Wrap(err, "failed to make temporary directories")
- }
-
- type unpackInfo struct{ path, dir, str string }
- unpackLen := 3
- var unpackList = make([]unpackInfo, 0, unpackLen)
- unpackList = append(unpackList, unpackInfo{path: s.basePath, dir: s.tmpDir.base, str: "base image"})
- unpackList = append(unpackList, unpackInfo{path: s.appPath, dir: s.tmpDir.app, str: "app image"})
- unpackList = append(unpackList, unpackInfo{path: s.libPath, dir: s.tmpDir.lib, str: "lib image"})
-
- for _, p := range unpackList {
- if len(p.path) != 0 {
- if err := util.UnpackFile(p.path, p.dir, archive.Gzip, false); err != nil {
- return errors.Wrapf(err, "unpack %s tarball %q failed", p.str, p.path)
- }
- }
- }
-
- return nil
-}
-
-func (s *separatorLoad) reconstructImage() error {
- files, err := ioutil.ReadDir(s.tmpDir.app)
- if err != nil {
- return err
- }
-
- for _, f := range files {
- src := filepath.Join(s.tmpDir.app, f.Name())
- dest := filepath.Join(s.tmpDir.base, f.Name())
- if err := os.Rename(src, dest); err != nil {
- return errors.Wrapf(err, "reconstruct app file %q failed", s.info.AppTarName)
- }
- }
-
- if len(s.libPath) != 0 {
- files, err := ioutil.ReadDir(s.tmpDir.lib)
- if err != nil {
- return err
- }
-
- for _, f := range files {
- src := filepath.Join(s.tmpDir.lib, f.Name())
- dest := filepath.Join(s.tmpDir.base, f.Name())
- if err := os.Rename(src, dest); err != nil {
- return errors.Wrapf(err, "reconstruct lib file %q failed", s.info.LibTarName)
- }
- }
- }
-
- return nil
-}
-
-func (s *separatorLoad) makeTempDir() error {
- dirs := []string{s.tmpDir.root, s.tmpDir.app, s.tmpDir.base, s.tmpDir.lib}
- for _, dir := range dirs {
- if err := os.MkdirAll(dir, constant.DefaultRootDirMode); err != nil {
- return err
- }
- }
-
- return nil
-}
diff --git a/daemon/load_test.go b/daemon/load_test.go
index 860b897..5e1a42b 100644
--- a/daemon/load_test.go
+++ b/daemon/load_test.go
@@ -314,7 +314,7 @@ func TestLoadMultipleImages(t *testing.T) {
defer clean(dir)
path := dir.Join(loadedTarFile)
- repoTags, err := tryToParseImageFormatFromTarball(daemon.opts.DataRoot, &loadOptions{path: path})
+ repoTags, err := tryToParseImageFormatFromTarball(daemon.opts.DataRoot, &LoadOptions{path: path})
assert.NilError(t, err)
assert.Equal(t, repoTags[0].nameTag[0], "registry.example.com/sayhello:first")
assert.Equal(t, repoTags[1].nameTag[0], "registry.example.com/sayhello:second")
diff --git a/daemon/save.go b/daemon/save.go
index 7a110bd..708fab3 100644
--- a/daemon/save.go
+++ b/daemon/save.go
@@ -15,24 +15,19 @@ package daemon
import (
"context"
- "encoding/json"
- "fmt"
- "io/ioutil"
"os"
"path/filepath"
- "sort"
"strings"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/types"
- "github.com/containers/storage/pkg/archive"
- "github.com/docker/docker/pkg/ioutils"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
constant "isula.org/isula-build"
pb "isula.org/isula-build/api/services"
+ "isula.org/isula-build/daemon/separator"
"isula.org/isula-build/exporter"
savedocker "isula.org/isula-build/exporter/docker/archive"
"isula.org/isula-build/image"
@@ -41,28 +36,13 @@ import (
"isula.org/isula-build/util"
)
-const (
- manifestDataFile = "manifest.json"
- manifestFile = "manifest"
- repositoriesFile = "repositories"
- baseTarNameSuffix = "_base_image.tar.gz"
- appTarNameSuffix = "_app_image.tar.gz"
- libTarNameSuffix = "_lib_image.tar.gz"
- untarTempDirName = "untar"
- baseUntarTempDirName = "base_images"
- appUntarTempDirName = "app_images"
- libUntarTempDirName = "lib_images"
- unionTarName = "all.tar"
- layerTarName = "layer.tar"
- tarSuffix = ".tar"
-)
-
type savedImage struct {
exist bool
tags []reference.NamedTagged
}
-type saveOptions struct {
+// SaveOptions stores the options for saving images
+type SaveOptions struct {
sysCtx *types.SystemContext
localStore *store.Store
logger *logger.Logger
@@ -73,77 +53,11 @@ type saveOptions struct {
oriImgList []string
finalImageOrdered []string
finalImageSet map[string]*savedImage
- sep separatorSave
-}
-
-type separatorSave struct {
- log *logrus.Entry
- renameData []renames
- tmpDir imageTmpDir
- base string
- lib string
- dest string
- renameFile string
- enabled bool
-}
-
-type renames struct {
- Name string `json:"name"`
- Rename string `json:"rename"`
-}
-
-type imageTmpDir struct {
- app string
- base string
- lib string
- untar string
- root string
-}
-
-type layer struct {
- all []string
- base []string
- lib []string
- app []string
-}
-
-type imageInfo struct {
- layers layer
- repoTags []string
- config string
- name string
- tag string
- nameTag string
- topLayer string
-}
-
-// imageManifest return image's manifest info
-type imageManifest struct {
- Config string `json:"Config"`
- RepoTags []string `json:"RepoTags"`
- Layers []string `json:"Layers"`
- // Not shown in the json file
- HashMap map[string]string `json:"-"`
+ sep separator.Saver
}
-type imageLayersMap map[string]string
-
-type tarballInfo struct {
- AppTarName string `json:"app"`
- AppHash string `json:"appHash"`
- AppLayers []string `json:"appLayers"`
- LibTarName string `json:"lib"`
- LibHash string `json:"libHash"`
- LibImageName string `json:"libImageName"`
- LibLayers []string `json:"libLayers"`
- BaseTarName string `json:"base"`
- BaseHash string `json:"baseHash"`
- BaseImageName string `json:"baseImageName"`
- BaseLayer string `json:"baseLayer"`
-}
-
-func (b *Backend) getSaveOptions(req *pb.SaveRequest) saveOptions {
- var opt = saveOptions{
+func (b *Backend) getSaveOptions(req *pb.SaveRequest) SaveOptions {
+ var opt = SaveOptions{
sysCtx: image.GetSystemContext(),
localStore: b.daemon.localStore,
saveID: req.GetSaveID(),
@@ -160,24 +74,7 @@ func (b *Backend) getSaveOptions(req *pb.SaveRequest) saveOptions {
return opt
}
- opt.sep = separatorSave{
- base: req.GetSep().GetBase(),
- lib: req.GetSep().GetLib(),
- dest: req.GetSep().GetDest(),
- log: logrus.WithFields(logrus.Fields{"SaveID": req.GetSaveID()}),
- enabled: req.GetSep().GetEnabled(),
- renameFile: req.GetSep().GetRename(),
- }
-
- // save separated image
- tmpRoot := filepath.Join(b.daemon.opts.DataRoot, filepath.Join(dataRootTmpDirPrefix, req.GetSaveID()))
- untar := filepath.Join(tmpRoot, untarTempDirName)
- appDir := filepath.Join(tmpRoot, appUntarTempDirName)
- baseDir := filepath.Join(tmpRoot, baseUntarTempDirName)
- libDir := filepath.Join(tmpRoot, libUntarTempDirName)
-
- opt.sep.tmpDir = imageTmpDir{app: appDir, base: baseDir, lib: libDir, untar: untar, root: tmpRoot}
- opt.outputPath = filepath.Join(untar, unionTarName)
+ opt.sep, opt.outputPath = separator.GetSepSaveOptions(req, opt.logEntry, b.daemon.opts.DataRoot)
return opt
}
@@ -213,15 +110,14 @@ func (b *Backend) Save(req *pb.SaveRequest, stream pb.Control_SaveServer) (err e
return err
}
- // separatorSave found
- if opts.sep.enabled {
- return separateImage(opts)
+ if opts.sep.Enabled() {
+ return opts.sep.SeparateImage(opts.localStore, opts.oriImgList, opts.outputPath)
}
return nil
}
-func exportHandler(ctx context.Context, opts *saveOptions) func() error {
+func exportHandler(ctx context.Context, opts *SaveOptions) func() error {
return func() error {
defer func() {
opts.logger.CloseContent()
@@ -278,7 +174,7 @@ func messageHandler(stream pb.Control_SaveServer, cliLogger *logger.Logger) func
}
}
-func (opts *saveOptions) manage() error {
+func (opts *SaveOptions) manage() error {
if err := opts.checkImageNameIsID(); err != nil {
return err
}
@@ -288,29 +184,22 @@ func (opts *saveOptions) manage() error {
if err := opts.filterImageName(); err != nil {
return err
}
- if err := opts.loadRenameFile(); err != nil {
+ if err := opts.sep.LoadRenameFile(); err != nil {
return err
}
return nil
}
-func (opts *saveOptions) checkImageNameIsID() error {
+func (opts *SaveOptions) checkImageNameIsID() error {
imageNames := opts.oriImgList
- if opts.sep.enabled {
- if len(opts.sep.base) != 0 {
- imageNames = append(imageNames, opts.sep.base)
- }
- if len(opts.sep.lib) != 0 {
- imageNames = append(imageNames, opts.sep.lib)
- }
- }
+ imageNames = append(imageNames, opts.sep.ImageNames()...)
for _, name := range imageNames {
_, img, err := image.FindImage(opts.localStore, name)
if err != nil {
return errors.Wrapf(err, "check image name failed when finding image name %q", name)
}
- if strings.HasPrefix(img.ID, name) && opts.sep.enabled {
+ if strings.HasPrefix(img.ID, name) && opts.sep.Enabled() {
return errors.Errorf("using image ID %q as image name to save separated image is not allowed", name)
}
}
@@ -318,7 +207,7 @@ func (opts *saveOptions) checkImageNameIsID() error {
return nil
}
-func (opts *saveOptions) setFormat() error {
+func (opts *SaveOptions) setFormat() error {
switch opts.format {
case constant.DockerTransport:
opts.format = constant.DockerArchiveTransport
@@ -331,7 +220,7 @@ func (opts *saveOptions) setFormat() error {
return nil
}
-func (opts *saveOptions) filterImageName() error {
+func (opts *SaveOptions) filterImageName() error {
if opts.format == constant.OCIArchiveTransport {
opts.finalImageOrdered = opts.oriImgList
return nil
@@ -367,507 +256,3 @@ func (opts *saveOptions) filterImageName() error {
return nil
}
-
-func (opts *saveOptions) loadRenameFile() error {
- if len(opts.sep.renameFile) != 0 {
- var reName []renames
- if err := util.LoadJSONFile(opts.sep.renameFile, &reName); err != nil {
- return errors.Wrap(err, "check rename file failed")
- }
- opts.sep.renameData = reName
- }
-
- return nil
-}
-
-func (s *separatorSave) getLayerHashFromStorage(store *store.Store, name string) ([]string, error) {
- if len(name) == 0 {
- return nil, nil
- }
- _, img, err := image.FindImage(store, name)
- if err != nil {
- return nil, err
- }
-
- layer, err := store.Layer(img.TopLayer)
- if err != nil {
- return nil, errors.Wrapf(err, "failed to get top layer for image %s", name)
- }
-
- var layers []string
- // add each layer in the layers until reach the root layer
- for layer != nil {
- fields := strings.Split(layer.UncompressedDigest.String(), ":")
- if len(fields) != 2 {
- return nil, errors.Errorf("error format of layer of image %s", name)
- }
- layers = append(layers, fields[1])
- if layer.Parent == "" {
- break
- }
- layer, err = store.Layer(layer.Parent)
- if err != nil {
- return nil, errors.Wrapf(err, "unable to read layer %q", layer.Parent)
- }
- }
-
- return layers, nil
-}
-
-// process physic file
-func (s *separatorSave) constructLayerMap() (map[string]string, error) {
- path := s.tmpDir.untar
- files, rErr := ioutil.ReadDir(path)
- if rErr != nil {
- return nil, rErr
- }
-
- var layerMap = make(map[string]string, len(files))
- // process layer's file
- for _, file := range files {
- if file.IsDir() {
- layerFile := filepath.Join(path, file.Name(), layerTarName)
- oriFile, err := os.Readlink(layerFile)
- if err != nil {
- return nil, err
- }
- physicFile := filepath.Join(path, file.Name(), oriFile)
- layerMap[filepath.Base(physicFile)] = filepath.Join(file.Name(), layerTarName)
- if err := os.Rename(physicFile, layerFile); err != nil {
- return nil, err
- }
- }
- }
-
- return layerMap, nil
-}
-
-func getLayerHashFromTar(layerMap map[string]string, layer []string) map[string]string {
- hashMap := make(map[string]string, len(layer))
- // first reverse map since it's <k-v> is unique
- revMap := make(map[string]string, len(layerMap))
- for k, v := range layerMap {
- revMap[v] = k
- }
- for _, l := range layer {
- if v, ok := revMap[l]; ok {
- // format is like xxx(hash): xxx/layer.tar
- hashMap[strings.TrimSuffix(v, tarSuffix)] = l
- }
- }
-
- return hashMap
-}
-
-func (s *separatorSave) adjustLayers() ([]imageManifest, error) {
- s.log.Info("Adjusting layers for saving separated image")
-
- layerMap, err := s.constructLayerMap()
- if err != nil {
- s.log.Errorf("Process layers failed: %v", err)
- return nil, err
- }
-
- // process manifest file
- var man []imageManifest
- if lErr := util.LoadJSONFile(filepath.Join(s.tmpDir.untar, manifestDataFile), &man); lErr != nil {
- return nil, lErr
- }
-
- for i, img := range man {
- layers := make([]string, len(img.Layers))
- for i, layer := range img.Layers {
- layers[i] = layerMap[layer]
- }
- man[i].Layers = layers
- man[i].HashMap = getLayerHashFromTar(layerMap, layers)
- }
- buf, err := json.Marshal(&man)
- if err != nil {
- return nil, err
- }
- if err := ioutils.AtomicWriteFile(manifestFile, buf, constant.DefaultSharedFileMode); err != nil {
- return nil, err
- }
-
- return man, nil
-}
-
-func separateImage(opt saveOptions) (err error) {
- s := &opt.sep
- s.log.Infof("Start saving separated images %v", opt.oriImgList)
-
- if err = os.MkdirAll(s.dest, constant.DefaultRootDirMode); err != nil {
- return err
- }
-
- defer func() {
- if tErr := os.RemoveAll(s.tmpDir.root); tErr != nil && !os.IsNotExist(tErr) {
- s.log.Warnf("Removing save tmp directory %q failed: %v", s.tmpDir.root, tErr)
- }
- if err != nil {
- if rErr := os.RemoveAll(s.dest); rErr != nil && !os.IsNotExist(rErr) {
- s.log.Warnf("Removing save dest directory %q failed: %v", s.dest, rErr)
- }
- }
- }()
- if err = util.UnpackFile(opt.outputPath, s.tmpDir.untar, archive.Gzip, true); err != nil {
- return errors.Wrapf(err, "unpack %q failed", opt.outputPath)
- }
- manifest, aErr := s.adjustLayers()
- if aErr != nil {
- return errors.Wrap(aErr, "adjust layers failed")
- }
-
- imgInfos, cErr := s.constructImageInfos(manifest, opt.localStore)
- if cErr != nil {
- return errors.Wrap(cErr, "process image infos failed")
- }
-
- if err = s.processImageLayers(imgInfos); err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *separatorSave) processImageLayers(imgInfos map[string]imageInfo) error {
- s.log.Info("Processing image layers")
- var (
- tarballs = make(map[string]tarballInfo)
- baseImagesMap = make(imageLayersMap, 1)
- libImagesMap = make(imageLayersMap, 1)
- appImagesMap = make(imageLayersMap, 1)
- )
- var sortedKey []string
- for k := range imgInfos {
- sortedKey = append(sortedKey, k)
- }
- sort.Strings(sortedKey)
- for _, k := range sortedKey {
- info := imgInfos[k]
- if err := s.clearTempDirs(); err != nil {
- return errors.Wrap(err, "clear tmp dirs failed")
- }
- var t tarballInfo
- // process base
- if err := info.processBaseImg(s, baseImagesMap, &t); err != nil {
- return errors.Wrapf(err, "process base images %s failed", info.nameTag)
- }
- // process lib
- if err := info.processLibImg(s, libImagesMap, &t); err != nil {
- return errors.Wrapf(err, "process lib images %s failed", info.nameTag)
- }
- // process app
- if err := info.processAppImg(s, appImagesMap, &t); err != nil {
- return errors.Wrapf(err, "process app images %s failed", info.nameTag)
- }
- tarballs[info.nameTag] = t
- }
- buf, err := json.Marshal(&tarballs)
- if err != nil {
- return err
- }
- // manifest file
- manifestFile := filepath.Join(s.dest, manifestFile)
- if err := ioutils.AtomicWriteFile(manifestFile, buf, constant.DefaultRootFileMode); err != nil {
- return err
- }
-
- s.log.Info("Save separated image succeed")
- return nil
-}
-
-func (s *separatorSave) clearTempDirs() error {
- dirs := []string{s.tmpDir.base, s.tmpDir.app, s.tmpDir.lib}
- for _, dir := range dirs {
- if err := os.RemoveAll(dir); err != nil {
- return err
- }
- if err := os.MkdirAll(dir, constant.DefaultRootDirMode); err != nil {
- return err
- }
- }
- return nil
-}
-
-// processTarName will trim the prefix of image name like example.io/library/myapp:v1
-// after processed, the name will be myapp_v1_suffix
-// mind: suffix here should not contain path separator
-func (info imageInfo) processTarName(suffix string) string {
- originNames := strings.Split(info.name, string(os.PathSeparator))
- originTags := strings.Split(info.tag, string(os.PathSeparator))
- // get the last element of the list, which mast be the right name without prefix
- name := originNames[len(originNames)-1]
- tag := originTags[len(originTags)-1]
-
- return fmt.Sprintf("%s_%s%s", name, tag, suffix)
-}
-
-func (info *imageInfo) processBaseImg(sep *separatorSave, baseImagesMap map[string]string, tarball *tarballInfo) error {
- // process base
- tarball.BaseImageName = sep.base
- if len(info.layers.base) != 0 {
- sep.log.Infof("Base image %s has %d layers", sep.base, len(info.layers.base))
- tarball.BaseLayer = info.layers.base[0]
- }
- for _, layerID := range info.layers.base {
- if baseImg, ok := baseImagesMap[layerID]; !ok {
- srcLayerPath := filepath.Join(sep.tmpDir.untar, layerID)
- destLayerPath := filepath.Join(sep.tmpDir.base, layerID)
- if err := os.Rename(srcLayerPath, destLayerPath); err != nil {
- return err
- }
- baseTarName := info.processTarName(baseTarNameSuffix)
- baseTarName = sep.rename(baseTarName)
- baseTarPath := filepath.Join(sep.dest, baseTarName)
- if err := util.PackFiles(sep.tmpDir.base, baseTarPath, archive.Gzip, true); err != nil {
- return err
- }
- baseImagesMap[layerID] = baseTarPath
- tarball.BaseTarName = baseTarName
- digest, err := util.SHA256Sum(baseTarPath)
- if err != nil {
- return errors.Wrapf(err, "check sum for new base image %s failed", baseTarName)
- }
- tarball.BaseHash = digest
- } else {
- tarball.BaseTarName = filepath.Base(baseImg)
- digest, err := util.SHA256Sum(baseImg)
- if err != nil {
- return errors.Wrapf(err, "check sum for reuse base image %s failed", baseImg)
- }
- tarball.BaseHash = digest
- }
- }
-
- return nil
-}
-
-func (info *imageInfo) processLibImg(sep *separatorSave, libImagesMap map[string]string, tarball *tarballInfo) error {
- // process lib
- if info.layers.lib == nil {
- return nil
- }
-
- tarball.LibImageName = sep.lib
- sep.log.Infof("Lib image %s has %d layers", sep.lib, len(info.layers.lib))
- for _, layerID := range info.layers.lib {
- tarball.LibLayers = append(tarball.LibLayers, layerID)
- if libImg, ok := libImagesMap[layerID]; !ok {
- srcLayerPath := filepath.Join(sep.tmpDir.untar, layerID)
- destLayerPath := filepath.Join(sep.tmpDir.lib, layerID)
- if err := os.Rename(srcLayerPath, destLayerPath); err != nil {
- return err
- }
- libTarName := info.processTarName(libTarNameSuffix)
- libTarName = sep.rename(libTarName)
- libTarPath := filepath.Join(sep.dest, libTarName)
- if err := util.PackFiles(sep.tmpDir.lib, libTarPath, archive.Gzip, true); err != nil {
- return err
- }
- libImagesMap[layerID] = libTarPath
- tarball.LibTarName = libTarName
- digest, err := util.SHA256Sum(libTarPath)
- if err != nil {
- return errors.Wrapf(err, "check sum for lib image %s failed", sep.lib)
- }
- tarball.LibHash = digest
- } else {
- tarball.LibTarName = filepath.Base(libImg)
- digest, err := util.SHA256Sum(libImg)
- if err != nil {
- return errors.Wrapf(err, "check sum for lib image %s failed", sep.lib)
- }
- tarball.LibHash = digest
- }
- }
-
- return nil
-}
-
-func (info *imageInfo) processAppImg(sep *separatorSave, appImagesMap map[string]string, tarball *tarballInfo) error {
- // process app
- sep.log.Infof("App image %s has %d layers", info.nameTag, len(info.layers.app))
- appTarName := info.processTarName(appTarNameSuffix)
- appTarName = sep.rename(appTarName)
- appTarPath := filepath.Join(sep.dest, appTarName)
- for _, layerID := range info.layers.app {
- srcLayerPath := filepath.Join(sep.tmpDir.untar, layerID)
- destLayerPath := filepath.Join(sep.tmpDir.app, layerID)
- if err := os.Rename(srcLayerPath, destLayerPath); err != nil {
- if appImg, ok := appImagesMap[layerID]; ok {
- return errors.Errorf("lib layers %s already saved in %s for image %s",
- layerID, appImg, info.nameTag)
- }
- }
- appImagesMap[layerID] = appTarPath
- tarball.AppLayers = append(tarball.AppLayers, layerID)
- }
- // create config file
- if err := info.createManifestFile(sep); err != nil {
- return err
- }
- if err := info.createRepositoriesFile(sep); err != nil {
- return err
- }
-
- srcConfigPath := filepath.Join(sep.tmpDir.untar, info.config)
- destConfigPath := filepath.Join(sep.tmpDir.app, info.config)
- if err := os.Rename(srcConfigPath, destConfigPath); err != nil {
- return err
- }
-
- if err := util.PackFiles(sep.tmpDir.app, appTarPath, archive.Gzip, true); err != nil {
- return err
- }
- tarball.AppTarName = appTarName
- digest, err := util.SHA256Sum(appTarPath)
- if err != nil {
- return errors.Wrapf(err, "check sum for app image %s failed", info.nameTag)
- }
- tarball.AppHash = digest
-
- return nil
-}
-
-func (info imageInfo) createRepositoriesFile(sep *separatorSave) error {
- // create repositories
- type repoItem map[string]string
- repo := make(map[string]repoItem, 1)
- item := make(repoItem, 1)
- if _, ok := item[info.tag]; !ok {
- item[info.tag] = info.topLayer
- }
- repo[info.name] = item
- buf, err := json.Marshal(repo)
- if err != nil {
- return err
- }
- repositoryFile := filepath.Join(sep.tmpDir.app, repositoriesFile)
- if err := ioutils.AtomicWriteFile(repositoryFile, buf, constant.DefaultRootFileMode); err != nil {
- return err
- }
- return nil
-}
-
-func (info imageInfo) createManifestFile(sep *separatorSave) error {
- // create manifest.json
- var s = imageManifest{
- Config: info.config,
- Layers: info.layers.all,
- RepoTags: info.repoTags,
- }
- var m []imageManifest
- m = append(m, s)
- buf, err := json.Marshal(&m)
- if err != nil {
- return err
- }
- data := filepath.Join(sep.tmpDir.app, manifestDataFile)
- if err := ioutils.AtomicWriteFile(data, buf, constant.DefaultRootFileMode); err != nil {
- return err
- }
- return nil
-}
-
-func getLayersID(layer []string) []string {
- var after = make([]string, len(layer))
- for i, v := range layer {
- after[i] = strings.Split(v, "/")[0]
- }
- return after
-}
-
-func (s *separatorSave) constructSingleImgInfo(mani imageManifest, store *store.Store) (imageInfo, error) {
- var libLayers, appLayers []string
- // image name should not be empty here
- if len(mani.RepoTags) == 0 {
- return imageInfo{}, errors.New("image name and tag is empty")
- }
- // if there is more than one repoTag, will use first one as image name
- imageRepoFields := strings.Split(mani.RepoTags[0], ":")
- imageLayers := getLayersID(mani.Layers)
-
- libs, bases, err := s.checkLayersHash(mani.HashMap, store)
- if err != nil {
- return imageInfo{}, errors.Wrap(err, "compare layers failed")
- }
- baseLayers := imageLayers[0:len(bases)]
- if len(libs) != 0 {
- libLayers = imageLayers[len(bases):len(libs)]
- appLayers = imageLayers[len(libs):]
- } else {
- libLayers = nil
- appLayers = imageLayers[len(bases):]
- }
-
- return imageInfo{
- config: mani.Config,
- repoTags: mani.RepoTags,
- nameTag: mani.RepoTags[0],
- name: strings.Join(imageRepoFields[0:len(imageRepoFields)-1], ":"),
- tag: imageRepoFields[len(imageRepoFields)-1],
- layers: layer{app: appLayers, lib: libLayers, base: baseLayers, all: mani.Layers},
- topLayer: imageLayers[len(imageLayers)-1],
- }, nil
-}
-
-func (s *separatorSave) checkLayersHash(layerHashMap map[string]string, store *store.Store) ([]string, []string, error) {
- libHash, err := s.getLayerHashFromStorage(store, s.lib)
- if err != nil {
- return nil, nil, errors.Wrapf(err, "get lib image %s layers failed", s.lib)
- }
- baseHash, err := s.getLayerHashFromStorage(store, s.base)
- if err != nil {
- return nil, nil, errors.Wrapf(err, "get base image %s layers failed", s.base)
- }
- if len(baseHash) > 1 {
- return nil, nil, errors.Errorf("number of base layers %d more than one", len(baseHash))
- }
- if len(libHash) >= len(layerHashMap) || len(baseHash) >= len(layerHashMap) {
- return nil, nil, errors.Errorf("number of base or lib layers is equal or greater than saved app layers")
- }
-
- for _, l := range libHash {
- if _, ok := layerHashMap[l]; !ok {
- return nil, nil, errors.Errorf("dismatch checksum for lib image %s", s.lib)
- }
- }
- for _, b := range baseHash {
- if _, ok := layerHashMap[b]; !ok {
- return nil, nil, errors.Errorf("dismatch checksum for base image %s", s.base)
- }
- }
-
- return libHash, baseHash, nil
-}
-
-func (s *separatorSave) constructImageInfos(manifest []imageManifest, store *store.Store) (map[string]imageInfo, error) {
- s.log.Info("Constructing image info")
-
- var imgInfos = make(map[string]imageInfo, 1)
- for _, mani := range manifest {
- imgInfo, err := s.constructSingleImgInfo(mani, store)
- if err != nil {
- s.log.Errorf("Constructing image info failed: %v", err)
- return nil, errors.Wrap(err, "construct image info failed")
- }
- if _, ok := imgInfos[imgInfo.nameTag]; !ok {
- imgInfos[imgInfo.nameTag] = imgInfo
- }
- }
- return imgInfos, nil
-}
-
-func (s *separatorSave) rename(name string) string {
- if len(s.renameData) != 0 {
- s.log.Info("Renaming image tarballs")
- for _, item := range s.renameData {
- if item.Name == name {
- return item.Rename
- }
- }
- }
- return name
-}
diff --git a/daemon/separator/image_info.go b/daemon/separator/image_info.go
new file mode 100644
index 0000000..53bcd7a
--- /dev/null
+++ b/daemon/separator/image_info.go
@@ -0,0 +1,235 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// isula-build licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+// http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Xiang Li
+// Create: 2021-12-09
+// Description: This file is handling image info for image separator
+
+package separator
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/containers/storage/pkg/archive"
+ "github.com/docker/docker/pkg/ioutils"
+ "github.com/pkg/errors"
+
+ constant "isula.org/isula-build"
+ "isula.org/isula-build/util"
+)
+
+type layer struct {
+ all []string
+ base []string
+ lib []string
+ app []string
+}
+
+type imageInfo struct {
+ layers layer
+ repoTags []string
+ config string
+ name string
+ tag string
+ nameTag string
+ topLayer string
+}
+
+// processTarName will trim the prefix of image name like example.io/library/myapp:v1
+// after processed, the name will be myapp_v1_suffix
+// mind: suffix here should not contain path separator
+func (info *imageInfo) processTarName(suffix string) string {
+ originNames := strings.Split(info.name, string(os.PathSeparator))
+ originTags := strings.Split(info.tag, string(os.PathSeparator))
+ // get the last element of the list, which mast be the right name without prefix
+ name := originNames[len(originNames)-1]
+ tag := originTags[len(originTags)-1]
+
+ return fmt.Sprintf("%s_%s%s", name, tag, suffix)
+}
+
+func (info *imageInfo) processBaseImg(sep *Saver, baseImagesMap map[string]string, tarball *tarballInfo) error {
+ // process base
+ tarball.BaseImageName = sep.base
+ if len(info.layers.base) != 0 {
+ sep.log.Infof("Base image %s has %d layers", sep.base, len(info.layers.base))
+ tarball.BaseLayer = info.layers.base[0]
+ }
+ for _, layerID := range info.layers.base {
+ if baseImg, ok := baseImagesMap[layerID]; !ok {
+ srcLayerPath := filepath.Join(sep.tmpDir.untar, layerID)
+ destLayerPath := filepath.Join(sep.tmpDir.base, layerID)
+ if err := os.Rename(srcLayerPath, destLayerPath); err != nil {
+ return err
+ }
+ baseTarName := info.processTarName(baseTarNameSuffix)
+ baseTarName = sep.getRename(baseTarName)
+ baseTarPath := filepath.Join(sep.dest, baseTarName)
+ if err := util.PackFiles(sep.tmpDir.base, baseTarPath, archive.Gzip, true); err != nil {
+ return err
+ }
+ baseImagesMap[layerID] = baseTarPath
+ tarball.BaseTarName = baseTarName
+ digest, err := util.SHA256Sum(baseTarPath)
+ if err != nil {
+ return errors.Wrapf(err, "check sum for new base image %s failed", baseTarName)
+ }
+ tarball.BaseHash = digest
+ } else {
+ tarball.BaseTarName = filepath.Base(baseImg)
+ digest, err := util.SHA256Sum(baseImg)
+ if err != nil {
+ return errors.Wrapf(err, "check sum for reuse base image %s failed", baseImg)
+ }
+ tarball.BaseHash = digest
+ }
+ }
+
+ return nil
+}
+
+func (info *imageInfo) processLibImg(sep *Saver, libImagesMap map[string]string, tarball *tarballInfo) error {
+ // process lib
+ if info.layers.lib == nil {
+ return nil
+ }
+
+ tarball.LibImageName = sep.lib
+ sep.log.Infof("Lib image %s has %d layers", sep.lib, len(info.layers.lib))
+ for _, layerID := range info.layers.lib {
+ tarball.LibLayers = append(tarball.LibLayers, layerID)
+ if libImg, ok := libImagesMap[layerID]; !ok {
+ srcLayerPath := filepath.Join(sep.tmpDir.untar, layerID)
+ destLayerPath := filepath.Join(sep.tmpDir.lib, layerID)
+ if err := os.Rename(srcLayerPath, destLayerPath); err != nil {
+ return err
+ }
+ libTarName := info.processTarName(libTarNameSuffix)
+ libTarName = sep.getRename(libTarName)
+ libTarPath := filepath.Join(sep.dest, libTarName)
+ if err := util.PackFiles(sep.tmpDir.lib, libTarPath, archive.Gzip, true); err != nil {
+ return err
+ }
+ libImagesMap[layerID] = libTarPath
+ tarball.LibTarName = libTarName
+ digest, err := util.SHA256Sum(libTarPath)
+ if err != nil {
+ return errors.Wrapf(err, "check sum for lib image %s failed", sep.lib)
+ }
+ tarball.LibHash = digest
+ } else {
+ tarball.LibTarName = filepath.Base(libImg)
+ digest, err := util.SHA256Sum(libImg)
+ if err != nil {
+ return errors.Wrapf(err, "check sum for lib image %s failed", sep.lib)
+ }
+ tarball.LibHash = digest
+ }
+ }
+
+ return nil
+}
+
+func (info *imageInfo) processAppImg(sep *Saver, appImagesMap map[string]string, tarball *tarballInfo) error {
+ // process app
+ sep.log.Infof("App image %s has %d layers", info.nameTag, len(info.layers.app))
+ appTarName := info.processTarName(appTarNameSuffix)
+ appTarName = sep.getRename(appTarName)
+ appTarPath := filepath.Join(sep.dest, appTarName)
+ for _, layerID := range info.layers.app {
+ srcLayerPath := filepath.Join(sep.tmpDir.untar, layerID)
+ destLayerPath := filepath.Join(sep.tmpDir.app, layerID)
+ if err := os.Rename(srcLayerPath, destLayerPath); err != nil {
+ if appImg, ok := appImagesMap[layerID]; ok {
+ return errors.Errorf("lib layers %s already saved in %s for image %s",
+ layerID, appImg, info.nameTag)
+ }
+ }
+ appImagesMap[layerID] = appTarPath
+ tarball.AppLayers = append(tarball.AppLayers, layerID)
+ }
+ // create config file
+ if err := info.createManifestFile(sep); err != nil {
+ return err
+ }
+ if err := info.createRepositoriesFile(sep); err != nil {
+ return err
+ }
+
+ srcConfigPath := filepath.Join(sep.tmpDir.untar, info.config)
+ destConfigPath := filepath.Join(sep.tmpDir.app, info.config)
+ if err := os.Rename(srcConfigPath, destConfigPath); err != nil {
+ return err
+ }
+
+ if err := util.PackFiles(sep.tmpDir.app, appTarPath, archive.Gzip, true); err != nil {
+ return err
+ }
+ tarball.AppTarName = appTarName
+ digest, err := util.SHA256Sum(appTarPath)
+ if err != nil {
+ return errors.Wrapf(err, "check sum for app image %s failed", info.nameTag)
+ }
+ tarball.AppHash = digest
+
+ return nil
+}
+
+func (info *imageInfo) createRepositoriesFile(sep *Saver) error {
+ // create repositories
+ type repoItem map[string]string
+ repo := make(map[string]repoItem, 1)
+ item := make(repoItem, 1)
+ if _, ok := item[info.tag]; !ok {
+ item[info.tag] = info.topLayer
+ }
+ repo[info.name] = item
+ buf, err := json.Marshal(repo)
+ if err != nil {
+ return err
+ }
+ repositoryFile := filepath.Join(sep.tmpDir.app, repositoriesFile)
+ if err := ioutils.AtomicWriteFile(repositoryFile, buf, constant.DefaultRootFileMode); err != nil {
+ return err
+ }
+ return nil
+}
+
+// imageManifest return image's manifest info
+type imageManifest struct {
+ Config string `json:"Config"`
+ RepoTags []string `json:"RepoTags"`
+ Layers []string `json:"Layers"`
+ // Not shown in the json file
+ HashMap map[string]string `json:"-"`
+}
+
+func (info *imageInfo) createManifestFile(sep *Saver) error {
+ // create manifest.json
+ var s = imageManifest{
+ Config: info.config,
+ Layers: info.layers.all,
+ RepoTags: info.repoTags,
+ }
+ var m []imageManifest
+ m = append(m, s)
+ buf, err := json.Marshal(&m)
+ if err != nil {
+ return err
+ }
+ data := filepath.Join(sep.tmpDir.app, manifestDataFile)
+ if err := ioutils.AtomicWriteFile(data, buf, constant.DefaultRootFileMode); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/daemon/separator/load.go b/daemon/separator/load.go
new file mode 100644
index 0000000..d02dccf
--- /dev/null
+++ b/daemon/separator/load.go
@@ -0,0 +1,283 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// isula-build licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+// http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Xiang Li
+// Create: 2021-12-09
+// Description: This file is handling "load" part for image separator at server side
+
+package separator
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+
+ "github.com/containers/storage/pkg/archive"
+ filepath_securejoin "github.com/cyphar/filepath-securejoin"
+ "github.com/pkg/errors"
+ "github.com/sirupsen/logrus"
+
+ constant "isula.org/isula-build"
+ pb "isula.org/isula-build/api/services"
+ "isula.org/isula-build/image"
+ "isula.org/isula-build/util"
+)
+
+type loadImageTmpDir struct {
+ app string
+ base string
+ lib string
+ root string
+}
+
+// Loader the main instance for loading separated images
+type Loader struct {
+ log *logrus.Entry
+ tmpDir loadImageTmpDir
+ info tarballInfo
+ appName string
+ basePath string
+ appPath string
+ libPath string
+ dir string
+ skipCheck bool
+ enabled bool
+}
+
+// GetSepLoadOptions returns Loader instance from LoadRequest
+func GetSepLoadOptions(req *pb.LoadRequest, logEntry *logrus.Entry, dataRoot string) (Loader, error) {
+ var tmpRoot = filepath.Join(dataRoot, filepath.Join(constant.DataRootTmpDirPrefix, req.GetLoadID()))
+ var sep = Loader{
+ appName: req.GetSep().GetApp(),
+ basePath: req.GetSep().GetBase(),
+ libPath: req.GetSep().GetLib(),
+ dir: req.GetSep().GetDir(),
+ log: logEntry,
+ tmpDir: loadImageTmpDir{
+ root: tmpRoot,
+ base: filepath.Join(tmpRoot, tmpBaseDirName),
+ lib: filepath.Join(tmpRoot, tmpLibDirName),
+ app: filepath.Join(tmpRoot, tmpAppDirName),
+ },
+ skipCheck: req.GetSep().GetSkipCheck(),
+ enabled: req.GetSep().GetEnabled(),
+ }
+
+ // check image name and add "latest" tag if not present
+ _, appImgName, err := image.GetNamedTaggedReference(sep.appName)
+ if err != nil {
+ return Loader{}, err
+ }
+ if len(appImgName) == 0 {
+ return Loader{}, errors.New("app image name should not be empty")
+ }
+ sep.appName = appImgName
+ return sep, nil
+}
+
+// LoadSeparatedImage the main method of Loader, tries to load the separated images, and returns the path of the
+// reconstructed image tarball for later handling
+func (l *Loader) LoadSeparatedImage() (string, error) {
+ l.log.Infof("Starting load separated image %s", l.appName)
+
+ // load manifest file to get tarball info
+ if err := l.getTarballInfo(); err != nil {
+ return "", errors.Wrap(err, "failed to get tarball info")
+ }
+ if err := l.constructLayerPath(); err != nil {
+ return "", err
+ }
+ // checksum for image tarballs
+ if err := l.tarballCheckSum(); err != nil {
+ return "", err
+ }
+ // process image tarballs and get final constructed image tarball
+ return l.processTarballs()
+}
+
+func (l *Loader) getTarballInfo() error {
+ manifest, err := filepath_securejoin.SecureJoin(l.dir, manifestFile)
+ if err != nil {
+ return errors.Wrap(err, "join manifest file path failed")
+ }
+
+ var t = make(map[string]tarballInfo, 1)
+ if err = util.LoadJSONFile(manifest, &t); err != nil {
+ return errors.Wrap(err, "load manifest file failed")
+ }
+
+ tarball, ok := t[l.appName]
+ if !ok {
+ return errors.Errorf("failed to find app image %s", l.appName)
+ }
+ if len(tarball.AppTarName) == 0 {
+ return errors.Errorf("app image %s tarball can not be empty", tarball.AppTarName)
+ }
+ if len(tarball.BaseTarName) == 0 {
+ return errors.Errorf("base image %s tarball can not be empty", tarball.BaseImageName)
+ }
+ l.info = tarball
+
+ return nil
+}
+
+func (l *Loader) joinPath(path, tarName, str string, canBeEmpty bool) (string, error) {
+ if len(path) != 0 {
+ return path, nil
+ }
+ l.log.Infof("%s image path is empty, use path from manifest", str)
+ return filepath_securejoin.SecureJoin(l.dir, tarName)
+}
+
+func (l *Loader) constructLayerPath() error {
+ l.log.Infof("Construct image layer pathes for %s\n", l.appName)
+
+ var err error
+ if l.basePath, err = l.joinPath(l.basePath, l.info.BaseTarName, "Base", false); err != nil {
+ return err
+ }
+ if l.libPath, err = l.joinPath(l.libPath, l.info.LibTarName, "Lib", true); err != nil {
+ return err
+ }
+ if l.appPath, err = l.joinPath(l.appPath, l.info.AppTarName, "App", false); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (l *Loader) tarballCheckSum() error {
+ if l.skipCheck {
+ l.log.Info("Skip checksum for tarballs")
+ return nil
+ }
+
+ type checkInfo struct {
+ path string
+ hash string
+ str string
+ canBeEmpty bool
+ }
+ checkLen := 3
+ var checkList = make([]checkInfo, 0, checkLen)
+ checkList = append(checkList, checkInfo{path: l.basePath, hash: l.info.BaseHash, canBeEmpty: false, str: "base"})
+ checkList = append(checkList, checkInfo{path: l.libPath, hash: l.info.LibHash, canBeEmpty: true, str: "lib"})
+ checkList = append(checkList, checkInfo{path: l.appPath, hash: l.info.AppHash, canBeEmpty: false, str: "app"})
+ for _, p := range checkList {
+ if len(p.path) == 0 && !p.canBeEmpty {
+ return errors.Errorf("%s image tarball path can not be empty", p.str)
+ }
+ if len(p.path) != 0 {
+ if err := util.CheckSum(p.path, p.hash); err != nil {
+ return errors.Wrapf(err, "check sum for file %q failed", p.path)
+ }
+ }
+ }
+
+ return nil
+}
+
+func (l *Loader) processTarballs() (string, error) {
+ if err := l.unpackTarballs(); err != nil {
+ return "", err
+ }
+
+ if err := l.reconstructImage(); err != nil {
+ return "", err
+ }
+
+ // pack app image to tarball
+ tarPath := filepath.Join(l.tmpDir.root, unionCompressedTarName)
+ if err := util.PackFiles(l.tmpDir.base, tarPath, archive.Gzip, true); err != nil {
+ return "", err
+ }
+
+ return tarPath, nil
+}
+
+func (l *Loader) unpackTarballs() error {
+ if err := l.makeTempDir(); err != nil {
+ return errors.Wrap(err, "failed to make temporary directories")
+ }
+
+ type unpackInfo struct{ path, dir, str string }
+ unpackLen := 3
+ var unpackList = make([]unpackInfo, 0, unpackLen)
+ unpackList = append(unpackList, unpackInfo{path: l.basePath, dir: l.tmpDir.base, str: "base"})
+ unpackList = append(unpackList, unpackInfo{path: l.libPath, dir: l.tmpDir.lib, str: "lib"})
+ unpackList = append(unpackList, unpackInfo{path: l.appPath, dir: l.tmpDir.app, str: "app"})
+
+ for _, p := range unpackList {
+ if len(p.path) != 0 {
+ if err := util.UnpackFile(p.path, p.dir, archive.Gzip, false); err != nil {
+ return errors.Wrapf(err, "unpack %s image tarball %q failed", p.str, p.path)
+ }
+ }
+ }
+
+ return nil
+}
+
+func (l *Loader) reconstructImage() error {
+ files, err := ioutil.ReadDir(l.tmpDir.app)
+ if err != nil {
+ return err
+ }
+
+ for _, f := range files {
+ src := filepath.Join(l.tmpDir.app, f.Name())
+ dest := filepath.Join(l.tmpDir.base, f.Name())
+ if err := os.Rename(src, dest); err != nil {
+ return errors.Wrapf(err, "reconstruct app file %q failed", l.info.AppTarName)
+ }
+ }
+
+ if len(l.libPath) != 0 {
+ files, err := ioutil.ReadDir(l.tmpDir.lib)
+ if err != nil {
+ return err
+ }
+
+ for _, f := range files {
+ src := filepath.Join(l.tmpDir.lib, f.Name())
+ dest := filepath.Join(l.tmpDir.base, f.Name())
+ if err := os.Rename(src, dest); err != nil {
+ return errors.Wrapf(err, "reconstruct lib file %q failed", l.info.LibTarName)
+ }
+ }
+ }
+
+ return nil
+}
+
+func (l *Loader) makeTempDir() error {
+ dirs := []string{l.tmpDir.root, l.tmpDir.app, l.tmpDir.base, l.tmpDir.lib}
+ for _, dir := range dirs {
+ if err := os.MkdirAll(dir, constant.DefaultRootDirMode); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// AppName returns the AppName of Loader
+func (l *Loader) AppName() string {
+ return l.appName
+}
+
+// TmpDirRoot returns the tmpDir.root of Loader
+func (l *Loader) TmpDirRoot() string {
+ return l.tmpDir.root
+}
+
+// Enabled returns whether separated-image feature is enabled
+func (l *Loader) Enabled() bool {
+ return l.enabled
+}
diff --git a/daemon/separator/save.go b/daemon/separator/save.go
new file mode 100644
index 0000000..a455335
--- /dev/null
+++ b/daemon/separator/save.go
@@ -0,0 +1,407 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// isula-build licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+// http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Xiang Li
+// Create: 2021-12-09
+// Description: This file is handling "save" part for image separator at server side
+
+package separator
+
+import (
+ "encoding/json"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/containers/storage/pkg/archive"
+ "github.com/docker/docker/pkg/ioutils"
+ "github.com/pkg/errors"
+ "github.com/sirupsen/logrus"
+
+ constant "isula.org/isula-build"
+ pb "isula.org/isula-build/api/services"
+ "isula.org/isula-build/image"
+ "isula.org/isula-build/store"
+ "isula.org/isula-build/util"
+)
+
+// Saver the main instance for saving separated images
+type Saver struct {
+ log *logrus.Entry
+ renameData []renames
+ tmpDir imageTmpDir
+ base string
+ lib string
+ dest string
+ renameFile string
+ enabled bool
+}
+
+type renames struct {
+ OriName string `json:"name"`
+ NewName string `json:"rename"`
+}
+
+type imageTmpDir struct {
+ app string
+ base string
+ lib string
+ untar string
+ root string
+}
+
+type imageLayersMap map[string]string
+
+// GetSepSaveOptions returns Save instance from SaveRequest
+func GetSepSaveOptions(req *pb.SaveRequest, logEntry *logrus.Entry, dataRoot string) (Saver, string) {
+ var tmpRoot = filepath.Join(dataRoot, filepath.Join(constant.DataRootTmpDirPrefix, req.GetSaveID()))
+ var sep = Saver{
+ base: req.GetSep().GetBase(),
+ lib: req.GetSep().GetLib(),
+ dest: req.GetSep().GetDest(),
+ log: logEntry,
+ enabled: req.GetSep().GetEnabled(),
+ renameFile: req.GetSep().GetRename(),
+
+ tmpDir: imageTmpDir{
+ root: tmpRoot,
+ untar: filepath.Join(tmpRoot, untarTempDirName),
+ base: filepath.Join(tmpRoot, baseUntarTempDirName),
+ lib: filepath.Join(tmpRoot, libUntarTempDirName),
+ app: filepath.Join(tmpRoot, appUntarTempDirName),
+ },
+ }
+
+ return sep, filepath.Join(sep.tmpDir.untar, unionTarName)
+}
+
+// SeparateImage the main method of Saver, tries to separated the listed images to pieces
+func (s *Saver) SeparateImage(localStore *store.Store, oriImgList []string, outputPath string) (err error) {
+ s.log.Infof("Start saving separated images %v", oriImgList)
+
+ if err = os.MkdirAll(s.dest, constant.DefaultRootDirMode); err != nil {
+ return err
+ }
+
+ defer func() {
+ if tErr := os.RemoveAll(s.tmpDir.root); tErr != nil && !os.IsNotExist(tErr) {
+ s.log.Warnf("Removing save tmp directory %q failed: %v", s.tmpDir.root, tErr)
+ }
+ if err != nil {
+ if rErr := os.RemoveAll(s.dest); rErr != nil && !os.IsNotExist(rErr) {
+ s.log.Warnf("Removing save dest directory %q failed: %v", s.dest, rErr)
+ }
+ }
+ }()
+ if err = util.UnpackFile(outputPath, s.tmpDir.untar, archive.Gzip, true); err != nil {
+ return errors.Wrapf(err, "unpack %q failed", outputPath)
+ }
+ manifest, aErr := s.adjustLayers()
+ if aErr != nil {
+ return errors.Wrap(aErr, "adjust layers failed")
+ }
+
+ imgInfos, cErr := s.constructImageInfos(manifest, localStore)
+ if cErr != nil {
+ return errors.Wrap(cErr, "process image infos failed")
+ }
+
+ if err = s.processImageLayers(imgInfos); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Saver) getLayerHashFromStorage(store *store.Store, name string) ([]string, error) {
+ if len(name) == 0 {
+ return nil, nil
+ }
+ _, img, err := image.FindImage(store, name)
+ if err != nil {
+ return nil, err
+ }
+
+ layer, err := store.Layer(img.TopLayer)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to get top layer for image %s", name)
+ }
+
+ var layers []string
+ // add each layer in the layers until reach the root layer
+ for layer != nil {
+ fields := strings.Split(layer.UncompressedDigest.String(), ":")
+ if len(fields) != 2 {
+ return nil, errors.Errorf("error format of layer of image %s", name)
+ }
+ layers = append(layers, fields[1])
+ if layer.Parent == "" {
+ break
+ }
+ layer, err = store.Layer(layer.Parent)
+ if err != nil {
+ return nil, errors.Wrapf(err, "unable to read layer %q", layer.Parent)
+ }
+ }
+
+ return layers, nil
+}
+
+// process physic file
+func (s *Saver) constructLayerMap() (map[string]string, error) {
+ path := s.tmpDir.untar
+ files, rErr := ioutil.ReadDir(path)
+ if rErr != nil {
+ return nil, rErr
+ }
+
+ var layerMap = make(map[string]string, len(files))
+ // process layer's file
+ for _, file := range files {
+ if file.IsDir() {
+ layerFile := filepath.Join(path, file.Name(), layerTarName)
+ oriFile, err := os.Readlink(layerFile)
+ if err != nil {
+ return nil, err
+ }
+ physicFile := filepath.Join(path, file.Name(), oriFile)
+ layerMap[filepath.Base(physicFile)] = filepath.Join(file.Name(), layerTarName)
+ if err := os.Rename(physicFile, layerFile); err != nil {
+ return nil, err
+ }
+ }
+ }
+
+ return layerMap, nil
+}
+
+func (s *Saver) adjustLayers() ([]imageManifest, error) {
+ s.log.Info("Adjusting layers for saving separated image")
+
+ layerMap, err := s.constructLayerMap()
+ if err != nil {
+ s.log.Errorf("Process layers failed: %v", err)
+ return nil, err
+ }
+
+ // process manifest file
+ var man []imageManifest
+ if lErr := util.LoadJSONFile(filepath.Join(s.tmpDir.untar, manifestDataFile), &man); lErr != nil {
+ return nil, lErr
+ }
+
+ for i, img := range man {
+ layers := make([]string, len(img.Layers))
+ for i, layer := range img.Layers {
+ layers[i] = layerMap[layer]
+ }
+ man[i].Layers = layers
+ man[i].HashMap = getLayerHashFromTar(layerMap, layers)
+ }
+ buf, err := json.Marshal(&man)
+ if err != nil {
+ return nil, err
+ }
+ if err := ioutils.AtomicWriteFile(manifestFile, buf, constant.DefaultSharedFileMode); err != nil {
+ return nil, err
+ }
+
+ return man, nil
+}
+
+func (s *Saver) processImageLayers(imgInfos map[string]imageInfo) error {
+ s.log.Info("Processing image layers")
+ var (
+ tarballs = make(map[string]tarballInfo)
+ baseImagesMap = make(imageLayersMap, 1)
+ libImagesMap = make(imageLayersMap, 1)
+ appImagesMap = make(imageLayersMap, 1)
+ )
+ var sortedKey []string
+ for k := range imgInfos {
+ sortedKey = append(sortedKey, k)
+ }
+ sort.Strings(sortedKey)
+ for _, k := range sortedKey {
+ info := imgInfos[k]
+ if err := s.clearTempDirs(); err != nil {
+ return errors.Wrap(err, "clear tmp dirs failed")
+ }
+ var t tarballInfo
+ // process base
+ if err := info.processBaseImg(s, baseImagesMap, &t); err != nil {
+ return errors.Wrapf(err, "process base images %s failed", info.nameTag)
+ }
+ // process lib
+ if err := info.processLibImg(s, libImagesMap, &t); err != nil {
+ return errors.Wrapf(err, "process lib images %s failed", info.nameTag)
+ }
+ // process app
+ if err := info.processAppImg(s, appImagesMap, &t); err != nil {
+ return errors.Wrapf(err, "process app images %s failed", info.nameTag)
+ }
+ tarballs[info.nameTag] = t
+ }
+ buf, err := json.Marshal(&tarballs)
+ if err != nil {
+ return err
+ }
+ // manifest file
+ manifestFile := filepath.Join(s.dest, manifestFile)
+ if err := ioutils.AtomicWriteFile(manifestFile, buf, constant.DefaultRootFileMode); err != nil {
+ return err
+ }
+
+ s.log.Info("Save separated image succeed")
+ return nil
+}
+
+func (s *Saver) clearTempDirs() error {
+ dirs := []string{s.tmpDir.base, s.tmpDir.app, s.tmpDir.lib}
+ for _, dir := range dirs {
+ if err := os.RemoveAll(dir); err != nil {
+ return err
+ }
+ if err := os.MkdirAll(dir, constant.DefaultRootDirMode); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// ImageNames returns the images names of Saver
+func (s *Saver) ImageNames() []string {
+ var names = make([]string, 0, 2)
+ if !s.enabled {
+ return []string{}
+ }
+ if len(s.base) != 0 {
+ names = append(names, s.base)
+ }
+ if len(s.lib) != 0 {
+ names = append(names, s.lib)
+ }
+ return names
+}
+
+func (s *Saver) constructSingleImgInfo(mani imageManifest, store *store.Store) (imageInfo, error) {
+ var libLayers, appLayers []string
+ // image name should not be empty here
+ if len(mani.RepoTags) == 0 {
+ return imageInfo{}, errors.New("image name and tag is empty")
+ }
+ // if there is more than one repoTag, will use first one as image name
+ imageRepoFields := strings.Split(mani.RepoTags[0], ":")
+ imageLayers := getLayersID(mani.Layers)
+
+ libs, bases, err := s.checkLayersHash(mani.HashMap, store)
+ if err != nil {
+ return imageInfo{}, errors.Wrap(err, "compare layers failed")
+ }
+ baseLayers := imageLayers[0:len(bases)]
+ if len(libs) != 0 {
+ libLayers = imageLayers[len(bases):len(libs)]
+ appLayers = imageLayers[len(libs):]
+ } else {
+ libLayers = nil
+ appLayers = imageLayers[len(bases):]
+ }
+
+ return imageInfo{
+ config: mani.Config,
+ repoTags: mani.RepoTags,
+ nameTag: mani.RepoTags[0],
+ name: strings.Join(imageRepoFields[0:len(imageRepoFields)-1], ":"),
+ tag: imageRepoFields[len(imageRepoFields)-1],
+ layers: layer{app: appLayers, lib: libLayers, base: baseLayers, all: mani.Layers},
+ topLayer: imageLayers[len(imageLayers)-1],
+ }, nil
+}
+
+func (s *Saver) checkLayersHash(layerHashMap map[string]string, store *store.Store) ([]string, []string, error) {
+ libHash, err := s.getLayerHashFromStorage(store, s.lib)
+ if err != nil {
+ return nil, nil, errors.Wrapf(err, "get lib image %s layers failed", s.lib)
+ }
+ baseHash, err := s.getLayerHashFromStorage(store, s.base)
+ if err != nil {
+ return nil, nil, errors.Wrapf(err, "get base image %s layers failed", s.base)
+ }
+ if len(baseHash) > 1 {
+ return nil, nil, errors.Errorf("number of base layers %d more than one", len(baseHash))
+ }
+ if len(libHash) >= len(layerHashMap) || len(baseHash) >= len(layerHashMap) {
+ return nil, nil, errors.Errorf("number of base or lib layers is equal or greater than saved app layers")
+ }
+
+ for _, l := range libHash {
+ if _, ok := layerHashMap[l]; !ok {
+ return nil, nil, errors.Errorf("dismatch checksum for lib image %s", s.lib)
+ }
+ }
+ for _, b := range baseHash {
+ if _, ok := layerHashMap[b]; !ok {
+ return nil, nil, errors.Errorf("dismatch checksum for base image %s", s.base)
+ }
+ }
+
+ return libHash, baseHash, nil
+}
+
+func (s *Saver) constructImageInfos(manifest []imageManifest, store *store.Store) (map[string]imageInfo, error) {
+ s.log.Info("Constructing image info")
+
+ var imgInfos = make(map[string]imageInfo, 1)
+ for _, mani := range manifest {
+ imgInfo, err := s.constructSingleImgInfo(mani, store)
+ if err != nil {
+ s.log.Errorf("Constructing image info failed: %v", err)
+ return nil, errors.Wrap(err, "construct image info failed")
+ }
+ if _, ok := imgInfos[imgInfo.nameTag]; !ok {
+ imgInfos[imgInfo.nameTag] = imgInfo
+ }
+ }
+ return imgInfos, nil
+}
+
+// LoadRenameFile Saver tries to load the specified rename json
+func (s *Saver) LoadRenameFile() error {
+ if len(s.renameFile) == 0 {
+ return nil
+ }
+
+ var reName []renames
+ if err := util.LoadJSONFile(s.renameFile, &reName); err != nil {
+ return errors.Wrap(err, "check rename file failed")
+ }
+ s.renameData = reName
+ return nil
+}
+
+func (s *Saver) getRename(name string) string {
+ if len(s.renameData) == 0 {
+ return name
+ }
+
+ for _, item := range s.renameData {
+ if item.OriName == name {
+ s.log.Infof("Renaming image tarballs for %s to %s\n", name, item.NewName)
+ return item.NewName
+ }
+ }
+ return name
+}
+
+// Enabled returns whether separated-image feature is enabled
+func (s *Saver) Enabled() bool {
+ return s.enabled
+}
diff --git a/daemon/separator/utils.go b/daemon/separator/utils.go
new file mode 100644
index 0000000..fb05f58
--- /dev/null
+++ b/daemon/separator/utils.go
@@ -0,0 +1,78 @@
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// isula-build licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+// http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Xiang Li
+// Create: 2021-12-09
+// Description: This file is utils for image separator
+
+package separator
+
+import "strings"
+
+const (
+ manifestDataFile = "manifest.json"
+ manifestFile = "manifest"
+ repositoriesFile = "repositories"
+
+ tmpBaseDirName = "base"
+ tmpAppDirName = "app"
+ tmpLibDirName = "lib"
+ baseUntarTempDirName = "base_images"
+ appUntarTempDirName = "app_images"
+ libUntarTempDirName = "lib_images"
+ baseTarNameSuffix = "_base_image.tar.gz"
+ appTarNameSuffix = "_app_image.tar.gz"
+ libTarNameSuffix = "_lib_image.tar.gz"
+
+ unionTarName = "all.tar"
+ unionCompressedTarName = "all.tar.gz"
+
+ untarTempDirName = "untar"
+ layerTarName = "layer.tar"
+ tarSuffix = ".tar"
+)
+
+type tarballInfo struct {
+ AppTarName string `json:"app"`
+ AppHash string `json:"appHash"`
+ AppLayers []string `json:"appLayers"`
+ LibTarName string `json:"lib"`
+ LibHash string `json:"libHash"`
+ LibImageName string `json:"libImageName"`
+ LibLayers []string `json:"libLayers"`
+ BaseTarName string `json:"base"`
+ BaseHash string `json:"baseHash"`
+ BaseImageName string `json:"baseImageName"`
+ BaseLayer string `json:"baseLayer"`
+}
+
+func getLayerHashFromTar(layerMap map[string]string, layer []string) map[string]string {
+ hashMap := make(map[string]string, len(layer))
+ // first reverse map since it's <k-v> is unique
+ revMap := make(map[string]string, len(layerMap))
+ for k, v := range layerMap {
+ revMap[v] = k
+ }
+ for _, l := range layer {
+ if v, ok := revMap[l]; ok {
+ // format is like xxx(hash): xxx/layer.tar
+ hashMap[strings.TrimSuffix(v, tarSuffix)] = l
+ }
+ }
+
+ return hashMap
+}
+
+func getLayersID(layer []string) []string {
+ var after = make([]string, len(layer))
+ for i, v := range layer {
+ after[i] = strings.Split(v, "/")[0]
+ }
+ return after
+}
--
2.31.0.windows.1
Reference in New Issue Copy Permalink