isula-build: sync patches from upstream

reason: updates are showing as follow:
- fix data and run root not effective when setting configuration.toml
- enhancement on go tests
- set user's uid and gid for containers
- make isula-build client side static so that which can be run in containers environment

Signed-off-by: DCCooper <1866858@gmail.com>
This commit is contained in:
DCCooper 2021-06-02 14:45:19 +08:00
parent 9eae1d51d2
commit b3c544dea6
9 changed files with 1751 additions and 3 deletions

View File

@ -1 +1 @@
0.9.5-6 0.9.5-7

View File

@ -1 +1 @@
b82408f23540642f79ab000483086997321305bf 2a48f637ab271e57f8f1daf9e753766b7ed98bd7

View File

@ -2,7 +2,7 @@
Name: isula-build Name: isula-build
Version: 0.9.5 Version: 0.9.5
Release: 6 Release: 7
Summary: A tool to build container images Summary: A tool to build container images
License: Mulan PSL V2 License: Mulan PSL V2
URL: https://gitee.com/openeuler/isula-build URL: https://gitee.com/openeuler/isula-build
@ -85,6 +85,12 @@ fi
/usr/share/bash-completion/completions/isula-build /usr/share/bash-completion/completions/isula-build
%changelog %changelog
* Wed Jun 02 2021 DCCooper <1866858@gmail.com> - 0.9.5-7
- Type:enhancement
- CVE:NA
- SUG:restart
- DESC:sync patches from upstream
* Wed Mar 03 2021 lixiang <lixiang172@huawei.com> - 0.9.5-6 * Wed Mar 03 2021 lixiang <lixiang172@huawei.com> - 0.9.5-6
- Type:enhancement - Type:enhancement
- CVE:NA - CVE:NA

View File

@ -0,0 +1,120 @@
From 022e5f3bfe5ec9731cf2d8808780a07d7408c820 Mon Sep 17 00:00:00 2001
From: xingweizheng <xingweizheng@huawei.com>
Date: Thu, 20 May 2021 15:58:43 +0800
Subject: [PATCH 1/5] fix data and run root not effective when setting
configuration.toml after upgrading containers/storage
---
cmd/daemon/main.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 59 insertions(+), 4 deletions(-)
diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go
index 41d2b60..4fd5356 100644
--- a/cmd/daemon/main.go
+++ b/cmd/daemon/main.go
@@ -213,6 +213,35 @@ func loadConfig(path string) (config.TomlConfig, error) {
return conf, err
}
+func checkRootSetInConfig(path string) (setRunRoot, setGraphRoot bool, err error) {
+ fi, err := os.Stat(path)
+ if err != nil {
+ return false, false, err
+ }
+
+ if !fi.Mode().IsRegular() {
+ err = errors.New("config file must be a regular file")
+ return false, false, err
+ }
+
+ if err = util.CheckFileSize(path, constant.MaxFileSize); err != nil {
+ return false, false, err
+ }
+
+ configData, err := ioutil.ReadFile(filepath.Clean(path))
+ if err != nil {
+ return false, false, err
+ }
+ conf := struct {
+ Storage struct {
+ RunRoot string `toml:"runroot"`
+ DataRoot string `toml:"graphroot"`
+ } `toml:"storage"`
+ }{}
+ _, err = toml.Decode(string(configData), &conf)
+ return conf.Storage.RunRoot != "", conf.Storage.DataRoot != "", err
+}
+
func mergeStorageConfig(cmd *cobra.Command) error {
store.SetDefaultConfigFilePath(constant.StorageConfigPath)
option, err := store.GetDefaultStoreOptions(true)
@@ -226,13 +255,21 @@ func mergeStorageConfig(cmd *cobra.Command) error {
}
var storeOpt store.DaemonStoreOptions
- if option.RunRoot == "" {
+ storeOpt.RunRoot = option.RunRoot
+ storeOpt.DataRoot = option.GraphRoot
+
+ setRunRoot, setDataRoot, err := checkRootSetInConfig(constant.StorageConfigPath)
+ if err != nil {
+ return err
+ }
+
+ if !setRunRoot {
storeOpt.RunRoot, err = securejoin.SecureJoin(daemonOpts.RunRoot, "storage")
if err != nil {
return err
}
}
- if option.GraphRoot == "" {
+ if !setDataRoot {
storeOpt.DataRoot, err = securejoin.SecureJoin(daemonOpts.DataRoot, "storage")
if err != nil {
return err
@@ -249,7 +286,7 @@ func mergeStorageConfig(cmd *cobra.Command) error {
return nil
}
-func mergeConfig(conf config.TomlConfig, cmd *cobra.Command) {
+func mergeConfig(conf config.TomlConfig, cmd *cobra.Command) error {
if conf.Debug && !cmd.Flag("debug").Changed {
daemonOpts.Debug = true
}
@@ -271,6 +308,22 @@ func mergeConfig(conf config.TomlConfig, cmd *cobra.Command) {
if conf.DataRoot != "" && !cmd.Flag("dataroot").Changed {
daemonOpts.DataRoot = conf.DataRoot
}
+
+ runRoot, err := securejoin.SecureJoin(daemonOpts.RunRoot, "storage")
+ if err != nil {
+ return err
+ }
+
+ dataRoot, err := securejoin.SecureJoin(daemonOpts.DataRoot, "storage")
+ if err != nil {
+ return err
+ }
+ store.SetDefaultStoreOptions(store.DaemonStoreOptions{
+ DataRoot: dataRoot,
+ RunRoot: runRoot,
+ })
+
+ return nil
}
func setupWorkingDirectories() error {
@@ -319,7 +372,9 @@ func checkAndValidateConfig(cmd *cobra.Command) error {
os.Exit(constant.DefaultFailedCode)
}
- mergeConfig(conf, cmd)
+ if err = mergeConfig(conf, cmd); err != nil {
+ return err
+ }
}
// file policy.json must be exist
--
1.8.3.1

View File

@ -0,0 +1,133 @@
From d6c6c205122386b66ef82adc4af16c3c2eb86b18 Mon Sep 17 00:00:00 2001
From: xingweizheng <xingweizheng@huawei.com>
Date: Mon, 31 May 2021 00:46:16 +0800
Subject: [PATCH 2/5] data and run root set unit test
---
cmd/daemon/main_test.go | 103 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 103 insertions(+)
diff --git a/cmd/daemon/main_test.go b/cmd/daemon/main_test.go
index 790fdfc..d98ea83 100644
--- a/cmd/daemon/main_test.go
+++ b/cmd/daemon/main_test.go
@@ -18,9 +18,12 @@ import (
"os"
"testing"
+ "gotest.tools/v3/assert"
"gotest.tools/v3/fs"
constant "isula.org/isula-build"
+ "isula.org/isula-build/cmd/daemon/config"
+ "isula.org/isula-build/store"
)
func TestSetupWorkingDirectories(t *testing.T) {
@@ -104,3 +107,103 @@ func TestSetupWorkingDirectories(t *testing.T) {
})
}
}
+
+func TestRunAndDataRootSet(t *testing.T) {
+ dataRoot := fs.NewDir(t, t.Name())
+ runRoot := fs.NewDir(t, t.Name())
+
+ conf := config.TomlConfig{
+ Debug: true,
+ Group: "isula",
+ LogLevel: "debug",
+ Runtime: "",
+ RunRoot: "",
+ DataRoot: "",
+ }
+ cmd := newDaemonCommand()
+
+ result := store.DaemonStoreOptions{
+ DataRoot: dataRoot.Join("storage"),
+ RunRoot: runRoot.Join("storage"),
+ }
+
+ setStorage := func(content string) func() {
+ return func() {
+ if err := mergeConfig(conf, cmd); err != nil {
+ t.Fatalf("mrege config failed with error: %v", err)
+ }
+
+ fileName := "storage.toml"
+ tmpDir := fs.NewDir(t, t.Name(), fs.WithFile(fileName, content))
+ defer tmpDir.Remove()
+
+ filePath := tmpDir.Join(fileName)
+ store.SetDefaultConfigFilePath(filePath)
+ option, err := store.GetDefaultStoreOptions(true)
+ if err != nil {
+ t.Fatalf("get default store options failed with error: %v", err)
+ }
+
+ var storeOpt store.DaemonStoreOptions
+ storeOpt.RunRoot = option.RunRoot
+ storeOpt.DataRoot = option.GraphRoot
+ store.SetDefaultStoreOptions(storeOpt)
+ }
+
+ }
+
+ testcases := []struct {
+ name string
+ setF func()
+ expectation store.DaemonStoreOptions
+ }{
+ {
+ name: "TC1 - cmd set, configuration and storage not set",
+ setF: func() {
+ cmd.PersistentFlags().Set("runroot", runRoot.Path())
+ cmd.PersistentFlags().Set("dataroot", dataRoot.Path())
+ checkAndValidateConfig(cmd)
+ },
+ expectation: result,
+ },
+ {
+ name: "TC2 - cmd and storage not set, configuration set",
+ setF: func() {
+ conf.DataRoot = dataRoot.Path()
+ conf.RunRoot = runRoot.Path()
+ checkAndValidateConfig(cmd)
+ },
+ expectation: result,
+ },
+ {
+ name: "TC3 - all not set",
+ setF: setStorage("[storage]"),
+ expectation: store.DaemonStoreOptions{
+ DataRoot: "/var/lib/containers/storage",
+ RunRoot: "/var/run/containers/storage",
+ },
+ },
+ {
+ name: "TC4 - cmd and configuration not set, storage set",
+ setF: func() {
+ config := "[storage]\nrunroot = \"" + runRoot.Join("storage") + "\"\ngraphroot = \"" + dataRoot.Join("storage") + "\""
+ sT := setStorage(config)
+ sT()
+ },
+ expectation: result,
+ },
+ }
+
+ for _, tc := range testcases {
+ t.Run(tc.name, func(t *testing.T) {
+ tc.setF()
+ storeOptions, err := store.GetDefaultStoreOptions(false)
+ if err != nil {
+ t.Fatalf("get default store options failed with error: %v", err)
+ }
+ assert.Equal(t, tc.expectation.DataRoot, storeOptions.GraphRoot)
+ assert.Equal(t, tc.expectation.RunRoot, storeOptions.RunRoot)
+ })
+
+ }
+}
--
1.8.3.1

View File

@ -0,0 +1,34 @@
From fbd95494e6e402fd123955fbaf337696cc22c750 Mon Sep 17 00:00:00 2001
From: DCCooper <1866858@gmail.com>
Date: Mon, 31 May 2021 20:50:24 +0800
Subject: [PATCH 3/5] bugfix: set user's uid and gid for containers
Signed-off-by: DCCooper <1866858@gmail.com>
---
builder/dockerfile/run.go | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/builder/dockerfile/run.go b/builder/dockerfile/run.go
index 6c38b55..828fe67 100644
--- a/builder/dockerfile/run.go
+++ b/builder/dockerfile/run.go
@@ -95,6 +95,16 @@ func (c *cmdBuilder) setupRuntimeSpec(command []string) (*specs.Spec, error) {
}
// set specific runtime spec config
+ user := c.stage.docker.Config.User
+ if user != "" {
+ pair, err := util.GetChownOptions(user, c.stage.mountpoint)
+ if err != nil {
+ return nil, err
+ }
+ g.SetProcessUID(uint32(pair.UID))
+ g.SetProcessGID(uint32(pair.GID))
+ g.SetProcessUsername(c.stage.docker.Config.User)
+ }
g.RemoveHostname()
g.SetProcessArgs(command)
g.SetProcessTerminal(false)
--
1.8.3.1

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,186 @@
From 78d5ee37ff4b2b3ef0a3e3031087d8cdb2e0c0cd Mon Sep 17 00:00:00 2001
From: xingweizheng <xingweizheng@huawei.com>
Date: Sun, 30 May 2021 20:55:07 +0800
Subject: [PATCH 5/5] integration test from new flaw of run and data root set
---
Makefile | 18 ++++++---
README.zh.md | 2 +-
tests/src/test_integration_set_new_root.sh | 60 ++++++++++++++++++++++++++++++
tests/test.sh | 29 +++++++++++++--
4 files changed, 98 insertions(+), 11 deletions(-)
create mode 100644 tests/src/test_integration_set_new_root.sh
diff --git a/Makefile b/Makefile
index cbace59..f8578a4 100644
--- a/Makefile
+++ b/Makefile
@@ -73,13 +73,13 @@ debug:
build-image:
isula-build ctr-img build -f Dockerfile.proto ${IMAGE_BUILDARGS} -o isulad:${IMAGE_NAME}:latest .
-tests: test-integration test-unit
+tests: test-base test-unit test-integration
-.PHONY: test-integration
-test-integration:
- @echo "Integration test starting..."
- @./tests/test.sh
- @echo "Integration test done!"
+.PHONY: test-base
+test-base:
+ @echo "Base test starting..."
+ @./tests/test.sh base
+ @echo "Base test done!"
.PHONY: test-unit
test-unit:
@@ -87,6 +87,12 @@ test-unit:
@./hack/unit_test.sh
@echo "Unit test done!"
+.PHONY: test-integration
+test-integration:
+ @echo "Integration test starting..."
+ @./tests/test.sh integration
+ @echo "Integration test done!"
+
.PHONY: proto
proto:
@echo "Generating protobuf..."
diff --git a/README.zh.md b/README.zh.md
index 4b53ba3..15301c0 100644
--- a/README.zh.md
+++ b/README.zh.md
@@ -106,7 +106,7 @@ sudo rpm -ivh isula-build-*.rpm
如果需要使用`systemd`进行管理isula-build请参考以下步骤
```sh
-sudo install -p -m 640 ./isula-build.service /etc/systemd/system/isula-build.
+sudo install -p -m 640 ./isula-build.service /etc/systemd/system/isula-build.service
sudo systemctl enable isula-build
sudo systemctl start isula-build
```
diff --git a/tests/src/test_integration_set_new_root.sh b/tests/src/test_integration_set_new_root.sh
new file mode 100644
index 0000000..85b724a
--- /dev/null
+++ b/tests/src/test_integration_set_new_root.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+# isula-build licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Author: Weizheng Xing
+# Create: 2021-05-29
+# Description: test set new run and data root in configuration.toml
+
+run_root="/var/run/new-isula-build"
+data_root="/var/lib/new-isula-build"
+config_file="/etc/isula-build/configuration.toml"
+base_image="hub.oepkgs.net/openeuler/openeuler:21.03"
+
+function clean()
+{
+ isula-build ctr-img rm $base_image >/dev/null 2>&1
+ rm -f $config_file
+ mv "$config_file".bak $config_file
+ systemctl stop isula-build
+ rm -rf $run_root $data_root
+}
+
+# change to new data and run root
+function pre_test()
+{
+ cp $config_file "$config_file".bak
+ sed -i "/run_root/d;/data_root/d" $config_file
+ echo "run_root = \"${run_root}\"" >> $config_file
+ echo "data_root = \"${data_root}\"" >> $config_file
+
+ systemctl restart isula-build
+}
+
+# check if new resources are downloaded in new root
+function do_test()
+{
+ tree_node_befor=$(tree -L 3 $data_root | wc -l)
+ isula-build ctr-img pull $base_image >/dev/null 2>&1
+ tree_node_after=$(tree -L 3 $data_root | wc -l)
+
+ if [ $(($tree_node_after - $tree_node_befor)) -eq 8 ]; then
+ echo "PASS"
+ else
+ echo "Sets of run and data root are not effective"
+ clean
+ exit 1
+ fi
+}
+
+# clean
+pre_test
+do_test
+clean
diff --git a/tests/test.sh b/tests/test.sh
index 79fde8a..e04cc96 100755
--- a/tests/test.sh
+++ b/tests/test.sh
@@ -2,8 +2,8 @@
top_dir=$(git rev-parse --show-toplevel)
-# normal test
-function normal() {
+# base test
+function base() {
source "$top_dir"/tests/lib/common.sh
pre_check
start_isula_builder
@@ -33,15 +33,36 @@ function fuzz() {
exit $failed
}
+# base test
+function integration() {
+ source "$top_dir"/tests/lib/common.sh
+ pre_check
+ systemctl restart isula-build
+
+ while IFS= read -r testfile; do
+ printf "%-45s" "test $(basename "$testfile"): "
+ if ! bash "$testfile"; then
+ exit 1
+ fi
+ done < <(find "$top_dir"/tests/src -maxdepth 1 -name "test_integration*" -type f -print)
+}
+
# main function to chose which kind of test
function main() {
case "$1" in
fuzz)
fuzz "$2"
;;
+ base)
+ base
+ ;;
+ integration)
+ integration
+ ;;
*)
- normal
- ;;
+ echo "Unknow test type."
+ exit 1
+ ;;
esac
}
--
1.8.3.1

View File

@ -12,3 +12,8 @@ patch/0045-fix-images-command-when-only-give-repository.patch
patch/0046-check-if-add-default-tag-to-image-name-when-using-pu.patch patch/0046-check-if-add-default-tag-to-image-name-when-using-pu.patch
patch/0047-checkAndExpandTag-return-empty-when-tag-is-empty.patch patch/0047-checkAndExpandTag-return-empty-when-tag-is-empty.patch
patch/0048-trim-space-when-counting-length-of-fields-to-avoid-p.patch patch/0048-trim-space-when-counting-length-of-fields-to-avoid-p.patch
patch/0049-fix-data-and-run-root-not-effective-when-setting-con.patch
patch/0050-data-and-run-root-set-unit-test.patch
patch/0051-bugfix-set-user-s-uid-and-gid-for-containers.patch
patch/0052-hack-make-isula-build-binary-static.patch
patch/0053-integration-test-from-new-flaw-of-run-and-data-root-.patch