packages/A-Tune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!100 bugfix set collection/transfer/train dir to specific dir

Browse Source 
From: @gaoruoshu 
Reviewed-by: @hubin95 
Signed-off-by: @hubin95
This commit is contained in:
openeuler-ci-bot 2023-08-10 09:09:39 +00:00 committed by Gitee
commit 0769daba41
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
4 changed files with 560 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
0001-bugfix-transfer-can-only-save-file-to-specified-dir.patch Normal file
View File

@ -0,0 +1,47 @@
From 34007d0d2fba94e43fbaf294d18cb2fc68857116 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Wed, 9 Aug 2023 15:07:04 +0800
Subject: [PATCH 1/3] bugfix: transfer can only save file to specified dir
---
analysis/engine/transfer.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/analysis/engine/transfer.py b/analysis/engine/transfer.py
index 7fa5777..154f3df 100644
--- a/analysis/engine/transfer.py
+++ b/analysis/engine/transfer.py
@@ -31,7 +31,7 @@ LOGGER = logging.getLogger(__name__)
class Transfer(Resource):
"""restful api for transfer"""
- file_path = "/etc/atuned/"
+ file_path = "/etc/atuned/{service}"
def post(self):
"""provide the method of post"""
@@ -40,15 +40,19 @@ class Transfer(Resource):
file_obj = request.files.get("file")
service = request.form.get("service")
+ target_path = self.file_path.format(service=service)
+ dir_name, _ = os.path.split(os.path.abspath(save_path))
+ if not dir_name == target_path:
+ return "illegal path to save file", 400
+
if service == "classification":
os.makedirs(ANALYSIS_DATA_PATH, exist_ok=True)
- file_name = ANALYSIS_DATA_PATH + save_path.split(self.file_path + service)[1][1:]
+ file_name = ANALYSIS_DATA_PATH + save_path.split(target_path)[1][1:]
current_app.logger.info(file_name)
file_obj.save(file_name)
return file_name, 200
file_obj.save(save_path)
- target_path = self.file_path + service
res = utils.extract_file(save_path, target_path)
os.remove(save_path)
return res, 200
--
2.27.0

290
0002-bugfix-training-model-can-only-save-file-to-specifie.patch Normal file
View File

@ -0,0 +1,290 @@
From 65d2dc509e851d138154ee6a8a3ff3acb3780a30 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Wed, 9 Aug 2023 19:15:07 +0800
Subject: [PATCH 2/3] bugfix: training model can only save file to specified
dir
---
analysis/default_config.py | 1 +
analysis/engine/parser.py | 4 +--
analysis/engine/train.py | 38 ++++++++++++++++++----
api/profile/profile.pb.go | 6 ++--
api/profile/profile.proto | 2 +-
common/models/training.go | 2 +-
modules/client/profile/profile_train.go | 42 +++++++++----------------
modules/server/profile/profile.go | 4 +--
8 files changed, 56 insertions(+), 43 deletions(-)
diff --git a/analysis/default_config.py b/analysis/default_config.py
index cf56ac2..7c921cc 100644
--- a/analysis/default_config.py
+++ b/analysis/default_config.py
@@ -23,6 +23,7 @@ GRPC_CERT_PATH = '/etc/atuned/grpc_certs'
ANALYSIS_DATA_PATH = '/var/atune_data/analysis/'
TUNING_DATA_PATH = '/var/atune_data/tuning/'
TUNING_DATA_DIRS = ['running', 'finished', 'error']
+TRAINING_MODEL_PATH = '/usr/libexec/atuned/analysis/models/'
def get_or_default(config, section, key, value):
diff --git a/analysis/engine/parser.py b/analysis/engine/parser.py
index c16089f..c36c74d 100644
--- a/analysis/engine/parser.py
+++ b/analysis/engine/parser.py
@@ -69,8 +69,8 @@ CLASSIFICATION_POST_PARSER.add_argument('model',
TRAIN_POST_PARSER = reqparse.RequestParser()
TRAIN_POST_PARSER.add_argument('datapath', required=True,
help="The datapath can not be null")
-TRAIN_POST_PARSER.add_argument('outputpath', required=True,
- help="The output path can not be null")
+TRAIN_POST_PARSER.add_argument('modelname', required=True,
+ help="The model name can not be null")
TRAIN_POST_PARSER.add_argument('modelpath', required=True,
help="The model path can not be null")
diff --git a/analysis/engine/train.py b/analysis/engine/train.py
index 9fdca46..7608660 100644
--- a/analysis/engine/train.py
+++ b/analysis/engine/train.py
@@ -22,6 +22,7 @@ from flask_restful import Resource
from analysis.optimizer.workload_characterization import WorkloadCharacterization
from analysis.engine.parser import TRAIN_POST_PARSER
+from analysis.default_config import TRAINING_MODEL_PATH
LOGGER = logging.getLogger(__name__)
@@ -29,7 +30,7 @@ LOGGER = logging.getLogger(__name__)
class Training(Resource):
"""provide the method of post for training"""
model_path = "modelpath"
- output_path = "outputpath"
+ model_name = "modelname"
data_path = "datapath"
def post(self):
@@ -40,18 +41,43 @@ class Training(Resource):
LOGGER.info(args)
model_path = args.get(self.model_path)
- output_path = args.get(self.output_path)
+ model_name = args.get(self.model_name)
data_path = args.get(self.data_path)
+ valid, err = valid_model_name(model_name)
+ if not valid:
+ return "Illegal model name provide: {}".format(err), 400
+
characterization = WorkloadCharacterization(model_path)
try:
+ output_path = TRAINING_MODEL_PATH + model_name
characterization.retrain(data_path, output_path)
except Exception as err:
LOGGER.error(err)
abort(500)
- if os.path.isdir(data_path):
- shutil.rmtree(data_path)
- else:
- os.remove(data_path)
return {}, 200
+
+
+def valid_model_name(name):
+ file_name, file_ext = os.path.splitext(name)
+
+ if file_ext != ".m":
+ return False, "the ext of model name should be .m"
+
+ if file_name in ['scaler', 'aencoder', 'tencoder', 'default_clf', 'total_clf', 'throughput_performance_clf']:
+ return False, "model name cannot be set as default_clf/scaler/aencoder/tencoder/throughput_performance_clf/total_clf"
+
+ for ind, char in enumerate(file_name):
+ if 'a' <= char <= 'z':
+ continue
+ if 'A' <= char <= 'Z':
+ continue
+ if '0' <= char <= '9':
+ continue
+ if ind != 0 and ind != len(file_name) - 1 and char == '_':
+ continue
+ return False, "model name cannot contains special character"
+
+ return True, None
+
diff --git a/api/profile/profile.pb.go b/api/profile/profile.pb.go
index dba166d..81a6757 100644
--- a/api/profile/profile.pb.go
+++ b/api/profile/profile.pb.go
@@ -493,7 +493,7 @@ func (m *CollectFlag) GetType() string {
type TrainMessage struct {
DataPath string `protobuf:"bytes,1,opt,name=DataPath,proto3" json:"DataPath,omitempty"`
- OutputPath string `protobuf:"bytes,2,opt,name=OutputPath,proto3" json:"OutputPath,omitempty"`
+ ModelName string `protobuf:"bytes,2,opt,name=ModelName,proto3" json:"ModelName,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@@ -531,9 +531,9 @@ func (m *TrainMessage) GetDataPath() string {
return ""
}
-func (m *TrainMessage) GetOutputPath() string {
+func (m *TrainMessage) GetModelName() string {
if m != nil {
- return m.OutputPath
+ return m.ModelName
}
return ""
}
diff --git a/api/profile/profile.proto b/api/profile/profile.proto
index 2a6e751..29cf91e 100755
--- a/api/profile/profile.proto
+++ b/api/profile/profile.proto
@@ -84,7 +84,7 @@ message CollectFlag {
message TrainMessage {
string DataPath = 1;
- string OutputPath = 2;
+ string ModelName = 2;
}
message DetectMessage {
diff --git a/common/models/training.go b/common/models/training.go
index 3cc9d60..9497261 100644
--- a/common/models/training.go
+++ b/common/models/training.go
@@ -24,7 +24,7 @@ import (
type Training struct {
DataPath string `json:"datapath"`
ModelPath string `json:"modelpath"`
- OutputPath string `json:"outputpath"`
+ ModelName string `json:"modelname"`
}
// Post method call training service
diff --git a/modules/client/profile/profile_train.go b/modules/client/profile/profile_train.go
index f4e68cb..a645bc3 100644
--- a/modules/client/profile/profile_train.go
+++ b/modules/client/profile/profile_train.go
@@ -18,9 +18,9 @@ import (
"gitee.com/openeuler/A-Tune/common/client"
SVC "gitee.com/openeuler/A-Tune/common/service"
"gitee.com/openeuler/A-Tune/common/utils"
+ "gitee.com/openeuler/A-Tune/common/config"
"fmt"
"io"
- "os"
"path/filepath"
"github.com/urfave/cli"
@@ -39,8 +39,8 @@ var trainCommand = cli.Command{
Value: "",
},
cli.StringFlag{
- Name: "output_file,o",
- Usage: "the model to be generated",
+ Name: "model_name,m",
+ Usage: "the model name of generate model",
Value: "",
},
},
@@ -48,9 +48,9 @@ var trainCommand = cli.Command{
desc := `
training a new model with the self collected data, data_path option specified
the path that storage the collected data, the collected data must have more
- than two workload type. output_file specified the file path where to store
+ than two workload type. model_name specified the name of model to be generated
the trained model, which must be end with .m.
- example: atune-adm train --data_path=./data --output_file=./model/trained.m`
+ example: atune-adm train --data_path=/home/data --model_name=trained.m`
return desc
}(),
Action: train,
@@ -82,13 +82,13 @@ func checkTrainCtx(ctx *cli.Context) error {
return fmt.Errorf("input:%s is invalid", dataPath)
}
- outputPath := ctx.String("output_file")
- if outputPath == "" {
+ modelName := ctx.String("model_name")
+ if modelName == "" {
_ = cli.ShowCommandHelp(ctx, "train")
- return fmt.Errorf("error: output_file must be specified")
+ return fmt.Errorf("error: model_name must be specified")
}
- if !utils.IsInputStringValid(outputPath) {
- return fmt.Errorf("input:%s is invalid", outputPath)
+ if !utils.IsInputStringValid(modelName) {
+ return fmt.Errorf("input:%s is invalid", modelName)
}
return nil
@@ -103,23 +103,8 @@ func train(ctx *cli.Context) error {
if err != nil {
return err
}
- outputPath, err := filepath.Abs(ctx.String("output_file"))
- if err != nil {
- return err
- }
-
- dir := filepath.Dir(outputPath)
- exist, err := utils.PathExist(dir)
- if err != nil {
- return err
- }
- if !exist {
- err = os.MkdirAll(dir, utils.FilePerm)
- if err != nil {
- return err
- }
- }
+ modelName := ctx.String("model_name")
c, err := client.NewClientFromContext(ctx)
if err != nil {
@@ -128,7 +113,7 @@ func train(ctx *cli.Context) error {
defer c.Close()
svc := PB.NewProfileMgrClient(c.Connection())
- stream, err := svc.Training(CTX.Background(), &PB.TrainMessage{DataPath: dataPath, OutputPath: outputPath})
+ stream, err := svc.Training(CTX.Background(), &PB.TrainMessage{DataPath: dataPath, ModelName: modelName})
if err != nil {
return err
}
@@ -145,6 +130,7 @@ func train(ctx *cli.Context) error {
}
utils.Print(reply)
}
- fmt.Println("the model generate path:", outputPath)
+ modelPath := fmt.Sprintf("%s/models/%s", config.DefaultAnalysisPath, modelName)
+ fmt.Println("the model generate path:", modelPath)
return nil
}
diff --git a/modules/server/profile/profile.go b/modules/server/profile/profile.go
index 3264167..70047ca 100644
--- a/modules/server/profile/profile.go
+++ b/modules/server/profile/profile.go
@@ -1137,7 +1137,7 @@ func (s *ProfileServer) Training(message *PB.TrainMessage, stream PB.ProfileMgr_
}
DataPath := message.GetDataPath()
- OutputPath := message.GetOutputPath()
+ ModelName := message.GetModelName()
compressPath, err := utils.CreateCompressFile(DataPath)
if err != nil {
@@ -1156,7 +1156,7 @@ func (s *ProfileServer) Training(message *PB.TrainMessage, stream PB.ProfileMgr_
trainBody := new(models.Training)
trainBody.DataPath = trainPath
- trainBody.OutputPath = OutputPath
+ trainBody.ModelName = ModelName
trainBody.ModelPath = path.Join(config.DefaultAnalysisPath, "models")
success, err := trainBody.Post()
--
2.27.0

211
0003-bugfix-collection-res-can-only-save-file-to-specifie.patch Normal file
View File

@ -0,0 +1,211 @@
From 4b357a57599ed623abbe094838d22895dc22fb5a Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Wed, 9 Aug 2023 21:41:10 +0800
Subject: [PATCH 3/3] bugfix: collection res can only save file to specified
dir
---
Makefile | 3 +++
api/profile/profile.pb.go | 6 ++---
api/profile/profile.proto | 2 +-
common/config/config.go | 1 +
modules/client/profile/profile_collection.go | 24 ++++++--------------
modules/server/profile/profile.go | 11 +++++----
6 files changed, 21 insertions(+), 26 deletions(-)
diff --git a/Makefile b/Makefile
index 5d9d317..bcac447 100755
--- a/Makefile
+++ b/Makefile
@@ -53,6 +53,7 @@ cleanall: clean
rm -rf $(DESTDIR)/var/lib/atuned/
rm -rf $(DESTDIR)/var/run/atuned/
rm -rf $(DESTDIR)/var/atuned/
+ rm -rf $(DESTDIR)/var/atune_data/
db:
sqlite3 database/atuned.db ".read database/init.sql"
@@ -78,6 +79,7 @@ libinstall:
rm -rf $(DESTDIR)/var/lib/atuned/
rm -rf $(DESTDIR)/var/run/atuned/
rm -rf $(DESTDIR)/var/atuned/
+ rm -rf $(DESTDIR)/var/atune_data/
mkdir -p $(DESTDIR)/etc/atuned/tuning
mkdir -p $(DESTDIR)/etc/atuned/rules
mkdir -p $(DESTDIR)/etc/atuned/training
@@ -90,6 +92,7 @@ libinstall:
mkdir -p $(DESTDIR)/var/lib/atuned
mkdir -p $(DESTDIR)/var/run/atuned
mkdir -p $(DESTDIR)/var/atuned
+ mkdir -p $(DESTDIR)/var/atune_data/collection
mkdir -p $(DESTDIR)$(PREFIX)/share/bash-completion/completions
install -m 640 pkg/daemon_profile_server.so $(DESTDIR)$(PREFIX)/lib/atuned/modules
install -m 750 pkg/atune-adm $(BINDIR)
diff --git a/api/profile/profile.pb.go b/api/profile/profile.pb.go
index dba166d..4e70c41 100644
--- a/api/profile/profile.pb.go
+++ b/api/profile/profile.pb.go
@@ -408,7 +408,7 @@ type CollectFlag struct {
Interval int64 `protobuf:"varint,1,opt,name=Interval,proto3" json:"Interval,omitempty"`
Duration int64 `protobuf:"varint,2,opt,name=Duration,proto3" json:"Duration,omitempty"`
Workload string `protobuf:"bytes,3,opt,name=Workload,proto3" json:"Workload,omitempty"`
- OutputPath string `protobuf:"bytes,4,opt,name=OutputPath,proto3" json:"OutputPath,omitempty"`
+ OutputDir string `protobuf:"bytes,4,opt,name=OutputDir,proto3" json:"OutputDir,omitempty"`
Block string `protobuf:"bytes,5,opt,name=Block,proto3" json:"Block,omitempty"`
Network string `protobuf:"bytes,6,opt,name=Network,proto3" json:"Network,omitempty"`
Type string `protobuf:"bytes,7,opt,name=Type,proto3" json:"Type,omitempty"`
@@ -463,9 +463,9 @@ func (m *CollectFlag) GetWorkload() string {
return ""
}
-func (m *CollectFlag) GetOutputPath() string {
+func (m *CollectFlag) GetOutputDir() string {
if m != nil {
- return m.OutputPath
+ return m.OutputDir
}
return ""
}
diff --git a/api/profile/profile.proto b/api/profile/profile.proto
index 2a6e751..71d8ceb 100755
--- a/api/profile/profile.proto
+++ b/api/profile/profile.proto
@@ -76,7 +76,7 @@ message CollectFlag {
int64 Interval = 1;
int64 Duration = 2;
string Workload = 3;
- string OutputPath = 4;
+ string OutputDir = 4;
string Block = 5;
string Network = 6;
string Type = 7;
diff --git a/common/config/config.go b/common/config/config.go
index 4bd70a6..b9d2b66 100644
--- a/common/config/config.go
+++ b/common/config/config.go
@@ -55,6 +55,7 @@ const (
DefaultCheckerPath = "/usr/share/atuned/checker/"
DefaultBackupPath = "/usr/share/atuned/backup/"
DefaultTuningLogPath = "/var/atuned"
+ DefaultCollectionPath = "/var/atune_data/collection/"
)
// log config
diff --git a/modules/client/profile/profile_collection.go b/modules/client/profile/profile_collection.go
index 26b8d65..2873dac 100644
--- a/modules/client/profile/profile_collection.go
+++ b/modules/client/profile/profile_collection.go
@@ -20,7 +20,6 @@ import (
"gitee.com/openeuler/A-Tune/common/utils"
"fmt"
"io"
- "path/filepath"
"github.com/urfave/cli"
CTX "golang.org/x/net/context"
@@ -48,8 +47,8 @@ var collectionCommand = cli.Command{
Value: 1200,
},
cli.StringFlag{
- Name: "output_path,o",
- Usage: "the output path of the collecting data",
+ Name: "output_dir,o",
+ Usage: "the output dir name of the collecting data, full dir path would be /var/atune_data/collection/{output_dir}",
Value: "",
},
cli.StringFlag{
@@ -71,8 +70,8 @@ var collectionCommand = cli.Command{
Description: func() string {
desc := `
collect data for train machine learning model, you must set the command options
- which has no default value, the output_path must be a absolute path.
- example: atune-adm collection -f mysql -i 5 -d 1200 -o /home -b sda -n eth0 -t mysql`
+ which has no default value.
+ example: atune-adm collection -f mysql -i 5 -d 1200 -o tmp -b sda -n eth0 -t mysql`
return desc
}(),
Action: collection,
@@ -118,9 +117,9 @@ func checkCollectionCtx(ctx *cli.Context) error {
return fmt.Errorf("error: app type must be specified")
}
- if ctx.String("output_path") == "" {
+ if ctx.String("output_dir") == "" {
_ = cli.ShowCommandHelp(ctx, "collection")
- return fmt.Errorf("error: output_path must be specified")
+ return fmt.Errorf("error: output_dir must be specified")
}
if ctx.Int64("interval") < 1 || ctx.Int64("interval") > 60 {
@@ -131,10 +130,6 @@ func checkCollectionCtx(ctx *cli.Context) error {
return fmt.Errorf("error: collection duration value must be bigger than interval*10")
}
- if !filepath.IsAbs(ctx.String("output_path")) {
- return fmt.Errorf("error: output path must be absolute path")
- }
-
return nil
}
@@ -149,16 +144,11 @@ func collection(ctx *cli.Context) error {
}
defer c.Close()
- outputPath := ctx.String("output_path")
- outputPath, err = filepath.Abs(outputPath)
- if err != nil {
- return err
- }
message := PB.CollectFlag{
Interval: ctx.Int64("interval"),
Duration: ctx.Int64("duration"),
Workload: ctx.String("filename"),
- OutputPath: outputPath,
+ OutputDir: ctx.String("output_dir"),
Block: ctx.String("disk"),
Network: ctx.String("network"),
Type: ctx.String("app_type"),
diff --git a/modules/server/profile/profile.go b/modules/server/profile/profile.go
index 3264167..ec1e739 100644
--- a/modules/server/profile/profile.go
+++ b/modules/server/profile/profile.go
@@ -1019,8 +1019,8 @@ func (s *ProfileServer) Collection(message *PB.CollectFlag, stream PB.ProfileMgr
return fmt.Errorf("input:%s is invalid", message.GetWorkload())
}
- if valid := utils.IsInputStringValid(message.GetOutputPath()); !valid {
- return fmt.Errorf("input:%s is invalid", message.GetOutputPath())
+ if valid, _ := regexp.MatchString("^[a-zA-Z]+(_[a-zA-Z0-9]+)*$", message.GetOutputDir()); !valid {
+ return fmt.Errorf("output dir:%s is invalid", message.GetOutputDir())
}
if valid := utils.IsInputStringValid(message.GetType()); !valid {
@@ -1049,12 +1049,13 @@ func (s *ProfileServer) Collection(message *PB.CollectFlag, stream PB.ProfileMgr
return err
}
- exist, err := utils.PathExist(message.GetOutputPath())
+ outputDir := path.Join(config.DefaultCollectionPath, message.GetOutputDir())
+ exist, err := utils.PathExist(message.GetOutputDir())
if err != nil {
return err
}
if !exist {
- return fmt.Errorf("output_path %s is not exist", message.GetOutputPath())
+ _ = os.MkdirAll(outputDir, 0755)
}
if err = utils.InterfaceByName(message.GetNetwork()); err != nil {
@@ -1106,7 +1107,7 @@ func (s *ProfileServer) Collection(message *PB.CollectFlag, stream PB.ProfileMgr
collectorBody.Monitors = monitors
nowTime := time.Now().Format("20060702-150405")
fileName := fmt.Sprintf("%s-%s.csv", message.GetWorkload(), nowTime)
- collectorBody.File = path.Join(message.GetOutputPath(), fileName)
+ collectorBody.File = path.Join(outputDir, fileName)
if include == "" {
include = "default"
}
--
2.27.0

13
atune.spec
View File

@ -3,7 +3,7 @@
Summary: AI auto tuning system
Name: atune
Version: 1.0.0
Release: 11
Release: 12
License: MulanPSL-2.0
URL: https://gitee.com/openeuler/A-Tune
Source: https://gitee.com/openeuler/A-Tune/repository/archive/v%{version}.tar.gz
@ -14,6 +14,11 @@ Patch9002: change-Makefile-A-Tune-version-to-1.0.0.patch
Patch9003: fix-start-failed-of-atuned-service.patch
Patch9004: A-Tune-Add-sw64-architecture.patch
Patch9005: add-riscv64-support.patch
Patch9006: 0001-bugfix-transfer-can-only-save-file-to-specified-dir.patch
Patch9007: 0002-bugfix-training-model-can-only-save-file-to-specifie.patch
Patch9008: 0003-bugfix-collection-res-can-only-save-file-to-specifie.patch
BuildRequires: rpm-build golang-bin procps-ng
BuildRequires: sqlite >= 3.24.0 openssl
BuildRequires: python3-scikit-optimize python3-pandas python3-xgboost
@ -80,6 +85,9 @@ atune engine tool for manage atuned AI tuning system.
%ifarch riscv64
%patch9005 -p1
%endif
%patch9006 -p1
%patch9007 -p1
%patch9008 -p1
%build
%make_build
@ -169,6 +177,9 @@ atune engine tool for manage atuned AI tuning system.
%exclude /etc/atuned/rest_certs
%changelog
* Thu Aug 10 2023 gaoruoshu <gaoruoshu@huawei.com> - 1.0.0-12
- bugfix set collection/transfer/train dir to specific dir
* Wed Aug 2 2023 zhangxiang <zhangxiang@iscas.ac.cn> - 1.0.0-11
- add riscv64 support