!100 bugfix set collection/transfer/train dir to specific dir
From: @gaoruoshu Reviewed-by: @hubin95 Signed-off-by: @hubin95
This commit is contained in:
commit
0769daba41
@ -0,0 +1,47 @@
|
|||||||
|
From 34007d0d2fba94e43fbaf294d18cb2fc68857116 Mon Sep 17 00:00:00 2001
|
||||||
|
From: gaoruoshu <gaoruoshu@huawei.com>
|
||||||
|
Date: Wed, 9 Aug 2023 15:07:04 +0800
|
||||||
|
Subject: [PATCH 1/3] bugfix: transfer can only save file to specified dir
|
||||||
|
|
||||||
|
---
|
||||||
|
analysis/engine/transfer.py | 10 +++++++---
|
||||||
|
1 file changed, 7 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/analysis/engine/transfer.py b/analysis/engine/transfer.py
|
||||||
|
index 7fa5777..154f3df 100644
|
||||||
|
--- a/analysis/engine/transfer.py
|
||||||
|
+++ b/analysis/engine/transfer.py
|
||||||
|
@@ -31,7 +31,7 @@ LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class Transfer(Resource):
|
||||||
|
"""restful api for transfer"""
|
||||||
|
- file_path = "/etc/atuned/"
|
||||||
|
+ file_path = "/etc/atuned/{service}"
|
||||||
|
|
||||||
|
def post(self):
|
||||||
|
"""provide the method of post"""
|
||||||
|
@@ -40,15 +40,19 @@ class Transfer(Resource):
|
||||||
|
file_obj = request.files.get("file")
|
||||||
|
service = request.form.get("service")
|
||||||
|
|
||||||
|
+ target_path = self.file_path.format(service=service)
|
||||||
|
+ dir_name, _ = os.path.split(os.path.abspath(save_path))
|
||||||
|
+ if not dir_name == target_path:
|
||||||
|
+ return "illegal path to save file", 400
|
||||||
|
+
|
||||||
|
if service == "classification":
|
||||||
|
os.makedirs(ANALYSIS_DATA_PATH, exist_ok=True)
|
||||||
|
- file_name = ANALYSIS_DATA_PATH + save_path.split(self.file_path + service)[1][1:]
|
||||||
|
+ file_name = ANALYSIS_DATA_PATH + save_path.split(target_path)[1][1:]
|
||||||
|
current_app.logger.info(file_name)
|
||||||
|
file_obj.save(file_name)
|
||||||
|
return file_name, 200
|
||||||
|
|
||||||
|
file_obj.save(save_path)
|
||||||
|
- target_path = self.file_path + service
|
||||||
|
res = utils.extract_file(save_path, target_path)
|
||||||
|
os.remove(save_path)
|
||||||
|
return res, 200
|
||||||
|
--
|
||||||
|
2.27.0
|
||||||
|
|
||||||
290
0002-bugfix-training-model-can-only-save-file-to-specifie.patch
Normal file
290
0002-bugfix-training-model-can-only-save-file-to-specifie.patch
Normal file
@ -0,0 +1,290 @@
|
|||||||
|
From 65d2dc509e851d138154ee6a8a3ff3acb3780a30 Mon Sep 17 00:00:00 2001
|
||||||
|
From: gaoruoshu <gaoruoshu@huawei.com>
|
||||||
|
Date: Wed, 9 Aug 2023 19:15:07 +0800
|
||||||
|
Subject: [PATCH 2/3] bugfix: training model can only save file to specified
|
||||||
|
dir
|
||||||
|
|
||||||
|
---
|
||||||
|
analysis/default_config.py | 1 +
|
||||||
|
analysis/engine/parser.py | 4 +--
|
||||||
|
analysis/engine/train.py | 38 ++++++++++++++++++----
|
||||||
|
api/profile/profile.pb.go | 6 ++--
|
||||||
|
api/profile/profile.proto | 2 +-
|
||||||
|
common/models/training.go | 2 +-
|
||||||
|
modules/client/profile/profile_train.go | 42 +++++++++----------------
|
||||||
|
modules/server/profile/profile.go | 4 +--
|
||||||
|
8 files changed, 56 insertions(+), 43 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/analysis/default_config.py b/analysis/default_config.py
|
||||||
|
index cf56ac2..7c921cc 100644
|
||||||
|
--- a/analysis/default_config.py
|
||||||
|
+++ b/analysis/default_config.py
|
||||||
|
@@ -23,6 +23,7 @@ GRPC_CERT_PATH = '/etc/atuned/grpc_certs'
|
||||||
|
ANALYSIS_DATA_PATH = '/var/atune_data/analysis/'
|
||||||
|
TUNING_DATA_PATH = '/var/atune_data/tuning/'
|
||||||
|
TUNING_DATA_DIRS = ['running', 'finished', 'error']
|
||||||
|
+TRAINING_MODEL_PATH = '/usr/libexec/atuned/analysis/models/'
|
||||||
|
|
||||||
|
|
||||||
|
def get_or_default(config, section, key, value):
|
||||||
|
diff --git a/analysis/engine/parser.py b/analysis/engine/parser.py
|
||||||
|
index c16089f..c36c74d 100644
|
||||||
|
--- a/analysis/engine/parser.py
|
||||||
|
+++ b/analysis/engine/parser.py
|
||||||
|
@@ -69,8 +69,8 @@ CLASSIFICATION_POST_PARSER.add_argument('model',
|
||||||
|
TRAIN_POST_PARSER = reqparse.RequestParser()
|
||||||
|
TRAIN_POST_PARSER.add_argument('datapath', required=True,
|
||||||
|
help="The datapath can not be null")
|
||||||
|
-TRAIN_POST_PARSER.add_argument('outputpath', required=True,
|
||||||
|
- help="The output path can not be null")
|
||||||
|
+TRAIN_POST_PARSER.add_argument('modelname', required=True,
|
||||||
|
+ help="The model name can not be null")
|
||||||
|
TRAIN_POST_PARSER.add_argument('modelpath', required=True,
|
||||||
|
help="The model path can not be null")
|
||||||
|
|
||||||
|
diff --git a/analysis/engine/train.py b/analysis/engine/train.py
|
||||||
|
index 9fdca46..7608660 100644
|
||||||
|
--- a/analysis/engine/train.py
|
||||||
|
+++ b/analysis/engine/train.py
|
||||||
|
@@ -22,6 +22,7 @@ from flask_restful import Resource
|
||||||
|
|
||||||
|
from analysis.optimizer.workload_characterization import WorkloadCharacterization
|
||||||
|
from analysis.engine.parser import TRAIN_POST_PARSER
|
||||||
|
+from analysis.default_config import TRAINING_MODEL_PATH
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
@@ -29,7 +30,7 @@ LOGGER = logging.getLogger(__name__)
|
||||||
|
class Training(Resource):
|
||||||
|
"""provide the method of post for training"""
|
||||||
|
model_path = "modelpath"
|
||||||
|
- output_path = "outputpath"
|
||||||
|
+ model_name = "modelname"
|
||||||
|
data_path = "datapath"
|
||||||
|
|
||||||
|
def post(self):
|
||||||
|
@@ -40,18 +41,43 @@ class Training(Resource):
|
||||||
|
LOGGER.info(args)
|
||||||
|
|
||||||
|
model_path = args.get(self.model_path)
|
||||||
|
- output_path = args.get(self.output_path)
|
||||||
|
+ model_name = args.get(self.model_name)
|
||||||
|
data_path = args.get(self.data_path)
|
||||||
|
|
||||||
|
+ valid, err = valid_model_name(model_name)
|
||||||
|
+ if not valid:
|
||||||
|
+ return "Illegal model name provide: {}".format(err), 400
|
||||||
|
+
|
||||||
|
characterization = WorkloadCharacterization(model_path)
|
||||||
|
try:
|
||||||
|
+ output_path = TRAINING_MODEL_PATH + model_name
|
||||||
|
characterization.retrain(data_path, output_path)
|
||||||
|
except Exception as err:
|
||||||
|
LOGGER.error(err)
|
||||||
|
abort(500)
|
||||||
|
|
||||||
|
- if os.path.isdir(data_path):
|
||||||
|
- shutil.rmtree(data_path)
|
||||||
|
- else:
|
||||||
|
- os.remove(data_path)
|
||||||
|
return {}, 200
|
||||||
|
+
|
||||||
|
+
|
||||||
|
+def valid_model_name(name):
|
||||||
|
+ file_name, file_ext = os.path.splitext(name)
|
||||||
|
+
|
||||||
|
+ if file_ext != ".m":
|
||||||
|
+ return False, "the ext of model name should be .m"
|
||||||
|
+
|
||||||
|
+ if file_name in ['scaler', 'aencoder', 'tencoder', 'default_clf', 'total_clf', 'throughput_performance_clf']:
|
||||||
|
+ return False, "model name cannot be set as default_clf/scaler/aencoder/tencoder/throughput_performance_clf/total_clf"
|
||||||
|
+
|
||||||
|
+ for ind, char in enumerate(file_name):
|
||||||
|
+ if 'a' <= char <= 'z':
|
||||||
|
+ continue
|
||||||
|
+ if 'A' <= char <= 'Z':
|
||||||
|
+ continue
|
||||||
|
+ if '0' <= char <= '9':
|
||||||
|
+ continue
|
||||||
|
+ if ind != 0 and ind != len(file_name) - 1 and char == '_':
|
||||||
|
+ continue
|
||||||
|
+ return False, "model name cannot contains special character"
|
||||||
|
+
|
||||||
|
+ return True, None
|
||||||
|
+
|
||||||
|
diff --git a/api/profile/profile.pb.go b/api/profile/profile.pb.go
|
||||||
|
index dba166d..81a6757 100644
|
||||||
|
--- a/api/profile/profile.pb.go
|
||||||
|
+++ b/api/profile/profile.pb.go
|
||||||
|
@@ -493,7 +493,7 @@ func (m *CollectFlag) GetType() string {
|
||||||
|
|
||||||
|
type TrainMessage struct {
|
||||||
|
DataPath string `protobuf:"bytes,1,opt,name=DataPath,proto3" json:"DataPath,omitempty"`
|
||||||
|
- OutputPath string `protobuf:"bytes,2,opt,name=OutputPath,proto3" json:"OutputPath,omitempty"`
|
||||||
|
+ ModelName string `protobuf:"bytes,2,opt,name=ModelName,proto3" json:"ModelName,omitempty"`
|
||||||
|
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
XXX_sizecache int32 `json:"-"`
|
||||||
|
@@ -531,9 +531,9 @@ func (m *TrainMessage) GetDataPath() string {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (m *TrainMessage) GetOutputPath() string {
|
||||||
|
+func (m *TrainMessage) GetModelName() string {
|
||||||
|
if m != nil {
|
||||||
|
- return m.OutputPath
|
||||||
|
+ return m.ModelName
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
diff --git a/api/profile/profile.proto b/api/profile/profile.proto
|
||||||
|
index 2a6e751..29cf91e 100755
|
||||||
|
--- a/api/profile/profile.proto
|
||||||
|
+++ b/api/profile/profile.proto
|
||||||
|
@@ -84,7 +84,7 @@ message CollectFlag {
|
||||||
|
|
||||||
|
message TrainMessage {
|
||||||
|
string DataPath = 1;
|
||||||
|
- string OutputPath = 2;
|
||||||
|
+ string ModelName = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DetectMessage {
|
||||||
|
diff --git a/common/models/training.go b/common/models/training.go
|
||||||
|
index 3cc9d60..9497261 100644
|
||||||
|
--- a/common/models/training.go
|
||||||
|
+++ b/common/models/training.go
|
||||||
|
@@ -24,7 +24,7 @@ import (
|
||||||
|
type Training struct {
|
||||||
|
DataPath string `json:"datapath"`
|
||||||
|
ModelPath string `json:"modelpath"`
|
||||||
|
- OutputPath string `json:"outputpath"`
|
||||||
|
+ ModelName string `json:"modelname"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Post method call training service
|
||||||
|
diff --git a/modules/client/profile/profile_train.go b/modules/client/profile/profile_train.go
|
||||||
|
index f4e68cb..a645bc3 100644
|
||||||
|
--- a/modules/client/profile/profile_train.go
|
||||||
|
+++ b/modules/client/profile/profile_train.go
|
||||||
|
@@ -18,9 +18,9 @@ import (
|
||||||
|
"gitee.com/openeuler/A-Tune/common/client"
|
||||||
|
SVC "gitee.com/openeuler/A-Tune/common/service"
|
||||||
|
"gitee.com/openeuler/A-Tune/common/utils"
|
||||||
|
+ "gitee.com/openeuler/A-Tune/common/config"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
- "os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/urfave/cli"
|
||||||
|
@@ -39,8 +39,8 @@ var trainCommand = cli.Command{
|
||||||
|
Value: "",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
- Name: "output_file,o",
|
||||||
|
- Usage: "the model to be generated",
|
||||||
|
+ Name: "model_name,m",
|
||||||
|
+ Usage: "the model name of generate model",
|
||||||
|
Value: "",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
@@ -48,9 +48,9 @@ var trainCommand = cli.Command{
|
||||||
|
desc := `
|
||||||
|
training a new model with the self collected data, data_path option specified
|
||||||
|
the path that storage the collected data, the collected data must have more
|
||||||
|
- than two workload type. output_file specified the file path where to store
|
||||||
|
+ than two workload type. model_name specified the name of model to be generated
|
||||||
|
the trained model, which must be end with .m.
|
||||||
|
- example: atune-adm train --data_path=./data --output_file=./model/trained.m`
|
||||||
|
+ example: atune-adm train --data_path=/home/data --model_name=trained.m`
|
||||||
|
return desc
|
||||||
|
}(),
|
||||||
|
Action: train,
|
||||||
|
@@ -82,13 +82,13 @@ func checkTrainCtx(ctx *cli.Context) error {
|
||||||
|
return fmt.Errorf("input:%s is invalid", dataPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
- outputPath := ctx.String("output_file")
|
||||||
|
- if outputPath == "" {
|
||||||
|
+ modelName := ctx.String("model_name")
|
||||||
|
+ if modelName == "" {
|
||||||
|
_ = cli.ShowCommandHelp(ctx, "train")
|
||||||
|
- return fmt.Errorf("error: output_file must be specified")
|
||||||
|
+ return fmt.Errorf("error: model_name must be specified")
|
||||||
|
}
|
||||||
|
- if !utils.IsInputStringValid(outputPath) {
|
||||||
|
- return fmt.Errorf("input:%s is invalid", outputPath)
|
||||||
|
+ if !utils.IsInputStringValid(modelName) {
|
||||||
|
+ return fmt.Errorf("input:%s is invalid", modelName)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
@@ -103,23 +103,8 @@ func train(ctx *cli.Context) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
- outputPath, err := filepath.Abs(ctx.String("output_file"))
|
||||||
|
- if err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
- dir := filepath.Dir(outputPath)
|
||||||
|
|
||||||
|
- exist, err := utils.PathExist(dir)
|
||||||
|
- if err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
- if !exist {
|
||||||
|
- err = os.MkdirAll(dir, utils.FilePerm)
|
||||||
|
- if err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
- }
|
||||||
|
+ modelName := ctx.String("model_name")
|
||||||
|
|
||||||
|
c, err := client.NewClientFromContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
@@ -128,7 +113,7 @@ func train(ctx *cli.Context) error {
|
||||||
|
defer c.Close()
|
||||||
|
|
||||||
|
svc := PB.NewProfileMgrClient(c.Connection())
|
||||||
|
- stream, err := svc.Training(CTX.Background(), &PB.TrainMessage{DataPath: dataPath, OutputPath: outputPath})
|
||||||
|
+ stream, err := svc.Training(CTX.Background(), &PB.TrainMessage{DataPath: dataPath, ModelName: modelName})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -145,6 +130,7 @@ func train(ctx *cli.Context) error {
|
||||||
|
}
|
||||||
|
utils.Print(reply)
|
||||||
|
}
|
||||||
|
- fmt.Println("the model generate path:", outputPath)
|
||||||
|
+ modelPath := fmt.Sprintf("%s/models/%s", config.DefaultAnalysisPath, modelName)
|
||||||
|
+ fmt.Println("the model generate path:", modelPath)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
diff --git a/modules/server/profile/profile.go b/modules/server/profile/profile.go
|
||||||
|
index 3264167..70047ca 100644
|
||||||
|
--- a/modules/server/profile/profile.go
|
||||||
|
+++ b/modules/server/profile/profile.go
|
||||||
|
@@ -1137,7 +1137,7 @@ func (s *ProfileServer) Training(message *PB.TrainMessage, stream PB.ProfileMgr_
|
||||||
|
}
|
||||||
|
|
||||||
|
DataPath := message.GetDataPath()
|
||||||
|
- OutputPath := message.GetOutputPath()
|
||||||
|
+ ModelName := message.GetModelName()
|
||||||
|
|
||||||
|
compressPath, err := utils.CreateCompressFile(DataPath)
|
||||||
|
if err != nil {
|
||||||
|
@@ -1156,7 +1156,7 @@ func (s *ProfileServer) Training(message *PB.TrainMessage, stream PB.ProfileMgr_
|
||||||
|
|
||||||
|
trainBody := new(models.Training)
|
||||||
|
trainBody.DataPath = trainPath
|
||||||
|
- trainBody.OutputPath = OutputPath
|
||||||
|
+ trainBody.ModelName = ModelName
|
||||||
|
trainBody.ModelPath = path.Join(config.DefaultAnalysisPath, "models")
|
||||||
|
|
||||||
|
success, err := trainBody.Post()
|
||||||
|
--
|
||||||
|
2.27.0
|
||||||
|
|
||||||
211
0003-bugfix-collection-res-can-only-save-file-to-specifie.patch
Normal file
211
0003-bugfix-collection-res-can-only-save-file-to-specifie.patch
Normal file
@ -0,0 +1,211 @@
|
|||||||
|
From 4b357a57599ed623abbe094838d22895dc22fb5a Mon Sep 17 00:00:00 2001
|
||||||
|
From: gaoruoshu <gaoruoshu@huawei.com>
|
||||||
|
Date: Wed, 9 Aug 2023 21:41:10 +0800
|
||||||
|
Subject: [PATCH 3/3] bugfix: collection res can only save file to specified
|
||||||
|
dir
|
||||||
|
|
||||||
|
---
|
||||||
|
Makefile | 3 +++
|
||||||
|
api/profile/profile.pb.go | 6 ++---
|
||||||
|
api/profile/profile.proto | 2 +-
|
||||||
|
common/config/config.go | 1 +
|
||||||
|
modules/client/profile/profile_collection.go | 24 ++++++--------------
|
||||||
|
modules/server/profile/profile.go | 11 +++++----
|
||||||
|
6 files changed, 21 insertions(+), 26 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/Makefile b/Makefile
|
||||||
|
index 5d9d317..bcac447 100755
|
||||||
|
--- a/Makefile
|
||||||
|
+++ b/Makefile
|
||||||
|
@@ -53,6 +53,7 @@ cleanall: clean
|
||||||
|
rm -rf $(DESTDIR)/var/lib/atuned/
|
||||||
|
rm -rf $(DESTDIR)/var/run/atuned/
|
||||||
|
rm -rf $(DESTDIR)/var/atuned/
|
||||||
|
+ rm -rf $(DESTDIR)/var/atune_data/
|
||||||
|
|
||||||
|
db:
|
||||||
|
sqlite3 database/atuned.db ".read database/init.sql"
|
||||||
|
@@ -78,6 +79,7 @@ libinstall:
|
||||||
|
rm -rf $(DESTDIR)/var/lib/atuned/
|
||||||
|
rm -rf $(DESTDIR)/var/run/atuned/
|
||||||
|
rm -rf $(DESTDIR)/var/atuned/
|
||||||
|
+ rm -rf $(DESTDIR)/var/atune_data/
|
||||||
|
mkdir -p $(DESTDIR)/etc/atuned/tuning
|
||||||
|
mkdir -p $(DESTDIR)/etc/atuned/rules
|
||||||
|
mkdir -p $(DESTDIR)/etc/atuned/training
|
||||||
|
@@ -90,6 +92,7 @@ libinstall:
|
||||||
|
mkdir -p $(DESTDIR)/var/lib/atuned
|
||||||
|
mkdir -p $(DESTDIR)/var/run/atuned
|
||||||
|
mkdir -p $(DESTDIR)/var/atuned
|
||||||
|
+ mkdir -p $(DESTDIR)/var/atune_data/collection
|
||||||
|
mkdir -p $(DESTDIR)$(PREFIX)/share/bash-completion/completions
|
||||||
|
install -m 640 pkg/daemon_profile_server.so $(DESTDIR)$(PREFIX)/lib/atuned/modules
|
||||||
|
install -m 750 pkg/atune-adm $(BINDIR)
|
||||||
|
diff --git a/api/profile/profile.pb.go b/api/profile/profile.pb.go
|
||||||
|
index dba166d..4e70c41 100644
|
||||||
|
--- a/api/profile/profile.pb.go
|
||||||
|
+++ b/api/profile/profile.pb.go
|
||||||
|
@@ -408,7 +408,7 @@ type CollectFlag struct {
|
||||||
|
Interval int64 `protobuf:"varint,1,opt,name=Interval,proto3" json:"Interval,omitempty"`
|
||||||
|
Duration int64 `protobuf:"varint,2,opt,name=Duration,proto3" json:"Duration,omitempty"`
|
||||||
|
Workload string `protobuf:"bytes,3,opt,name=Workload,proto3" json:"Workload,omitempty"`
|
||||||
|
- OutputPath string `protobuf:"bytes,4,opt,name=OutputPath,proto3" json:"OutputPath,omitempty"`
|
||||||
|
+ OutputDir string `protobuf:"bytes,4,opt,name=OutputDir,proto3" json:"OutputDir,omitempty"`
|
||||||
|
Block string `protobuf:"bytes,5,opt,name=Block,proto3" json:"Block,omitempty"`
|
||||||
|
Network string `protobuf:"bytes,6,opt,name=Network,proto3" json:"Network,omitempty"`
|
||||||
|
Type string `protobuf:"bytes,7,opt,name=Type,proto3" json:"Type,omitempty"`
|
||||||
|
@@ -463,9 +463,9 @@ func (m *CollectFlag) GetWorkload() string {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (m *CollectFlag) GetOutputPath() string {
|
||||||
|
+func (m *CollectFlag) GetOutputDir() string {
|
||||||
|
if m != nil {
|
||||||
|
- return m.OutputPath
|
||||||
|
+ return m.OutputDir
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
diff --git a/api/profile/profile.proto b/api/profile/profile.proto
|
||||||
|
index 2a6e751..71d8ceb 100755
|
||||||
|
--- a/api/profile/profile.proto
|
||||||
|
+++ b/api/profile/profile.proto
|
||||||
|
@@ -76,7 +76,7 @@ message CollectFlag {
|
||||||
|
int64 Interval = 1;
|
||||||
|
int64 Duration = 2;
|
||||||
|
string Workload = 3;
|
||||||
|
- string OutputPath = 4;
|
||||||
|
+ string OutputDir = 4;
|
||||||
|
string Block = 5;
|
||||||
|
string Network = 6;
|
||||||
|
string Type = 7;
|
||||||
|
diff --git a/common/config/config.go b/common/config/config.go
|
||||||
|
index 4bd70a6..b9d2b66 100644
|
||||||
|
--- a/common/config/config.go
|
||||||
|
+++ b/common/config/config.go
|
||||||
|
@@ -55,6 +55,7 @@ const (
|
||||||
|
DefaultCheckerPath = "/usr/share/atuned/checker/"
|
||||||
|
DefaultBackupPath = "/usr/share/atuned/backup/"
|
||||||
|
DefaultTuningLogPath = "/var/atuned"
|
||||||
|
+ DefaultCollectionPath = "/var/atune_data/collection/"
|
||||||
|
)
|
||||||
|
|
||||||
|
// log config
|
||||||
|
diff --git a/modules/client/profile/profile_collection.go b/modules/client/profile/profile_collection.go
|
||||||
|
index 26b8d65..2873dac 100644
|
||||||
|
--- a/modules/client/profile/profile_collection.go
|
||||||
|
+++ b/modules/client/profile/profile_collection.go
|
||||||
|
@@ -20,7 +20,6 @@ import (
|
||||||
|
"gitee.com/openeuler/A-Tune/common/utils"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
- "path/filepath"
|
||||||
|
|
||||||
|
"github.com/urfave/cli"
|
||||||
|
CTX "golang.org/x/net/context"
|
||||||
|
@@ -48,8 +47,8 @@ var collectionCommand = cli.Command{
|
||||||
|
Value: 1200,
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
- Name: "output_path,o",
|
||||||
|
- Usage: "the output path of the collecting data",
|
||||||
|
+ Name: "output_dir,o",
|
||||||
|
+ Usage: "the output dir name of the collecting data, full dir path would be /var/atune_data/collection/{output_dir}",
|
||||||
|
Value: "",
|
||||||
|
},
|
||||||
|
cli.StringFlag{
|
||||||
|
@@ -71,8 +70,8 @@ var collectionCommand = cli.Command{
|
||||||
|
Description: func() string {
|
||||||
|
desc := `
|
||||||
|
collect data for train machine learning model, you must set the command options
|
||||||
|
- which has no default value, the output_path must be a absolute path.
|
||||||
|
- example: atune-adm collection -f mysql -i 5 -d 1200 -o /home -b sda -n eth0 -t mysql`
|
||||||
|
+ which has no default value.
|
||||||
|
+ example: atune-adm collection -f mysql -i 5 -d 1200 -o tmp -b sda -n eth0 -t mysql`
|
||||||
|
return desc
|
||||||
|
}(),
|
||||||
|
Action: collection,
|
||||||
|
@@ -118,9 +117,9 @@ func checkCollectionCtx(ctx *cli.Context) error {
|
||||||
|
return fmt.Errorf("error: app type must be specified")
|
||||||
|
}
|
||||||
|
|
||||||
|
- if ctx.String("output_path") == "" {
|
||||||
|
+ if ctx.String("output_dir") == "" {
|
||||||
|
_ = cli.ShowCommandHelp(ctx, "collection")
|
||||||
|
- return fmt.Errorf("error: output_path must be specified")
|
||||||
|
+ return fmt.Errorf("error: output_dir must be specified")
|
||||||
|
}
|
||||||
|
|
||||||
|
if ctx.Int64("interval") < 1 || ctx.Int64("interval") > 60 {
|
||||||
|
@@ -131,10 +130,6 @@ func checkCollectionCtx(ctx *cli.Context) error {
|
||||||
|
return fmt.Errorf("error: collection duration value must be bigger than interval*10")
|
||||||
|
}
|
||||||
|
|
||||||
|
- if !filepath.IsAbs(ctx.String("output_path")) {
|
||||||
|
- return fmt.Errorf("error: output path must be absolute path")
|
||||||
|
- }
|
||||||
|
-
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -149,16 +144,11 @@ func collection(ctx *cli.Context) error {
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
|
||||||
|
- outputPath := ctx.String("output_path")
|
||||||
|
- outputPath, err = filepath.Abs(outputPath)
|
||||||
|
- if err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
message := PB.CollectFlag{
|
||||||
|
Interval: ctx.Int64("interval"),
|
||||||
|
Duration: ctx.Int64("duration"),
|
||||||
|
Workload: ctx.String("filename"),
|
||||||
|
- OutputPath: outputPath,
|
||||||
|
+ OutputDir: ctx.String("output_dir"),
|
||||||
|
Block: ctx.String("disk"),
|
||||||
|
Network: ctx.String("network"),
|
||||||
|
Type: ctx.String("app_type"),
|
||||||
|
diff --git a/modules/server/profile/profile.go b/modules/server/profile/profile.go
|
||||||
|
index 3264167..ec1e739 100644
|
||||||
|
--- a/modules/server/profile/profile.go
|
||||||
|
+++ b/modules/server/profile/profile.go
|
||||||
|
@@ -1019,8 +1019,8 @@ func (s *ProfileServer) Collection(message *PB.CollectFlag, stream PB.ProfileMgr
|
||||||
|
return fmt.Errorf("input:%s is invalid", message.GetWorkload())
|
||||||
|
}
|
||||||
|
|
||||||
|
- if valid := utils.IsInputStringValid(message.GetOutputPath()); !valid {
|
||||||
|
- return fmt.Errorf("input:%s is invalid", message.GetOutputPath())
|
||||||
|
+ if valid, _ := regexp.MatchString("^[a-zA-Z]+(_[a-zA-Z0-9]+)*$", message.GetOutputDir()); !valid {
|
||||||
|
+ return fmt.Errorf("output dir:%s is invalid", message.GetOutputDir())
|
||||||
|
}
|
||||||
|
|
||||||
|
if valid := utils.IsInputStringValid(message.GetType()); !valid {
|
||||||
|
@@ -1049,12 +1049,13 @@ func (s *ProfileServer) Collection(message *PB.CollectFlag, stream PB.ProfileMgr
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
- exist, err := utils.PathExist(message.GetOutputPath())
|
||||||
|
+ outputDir := path.Join(config.DefaultCollectionPath, message.GetOutputDir())
|
||||||
|
+ exist, err := utils.PathExist(message.GetOutputDir())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !exist {
|
||||||
|
- return fmt.Errorf("output_path %s is not exist", message.GetOutputPath())
|
||||||
|
+ _ = os.MkdirAll(outputDir, 0755)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = utils.InterfaceByName(message.GetNetwork()); err != nil {
|
||||||
|
@@ -1106,7 +1107,7 @@ func (s *ProfileServer) Collection(message *PB.CollectFlag, stream PB.ProfileMgr
|
||||||
|
collectorBody.Monitors = monitors
|
||||||
|
nowTime := time.Now().Format("20060702-150405")
|
||||||
|
fileName := fmt.Sprintf("%s-%s.csv", message.GetWorkload(), nowTime)
|
||||||
|
- collectorBody.File = path.Join(message.GetOutputPath(), fileName)
|
||||||
|
+ collectorBody.File = path.Join(outputDir, fileName)
|
||||||
|
if include == "" {
|
||||||
|
include = "default"
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.27.0
|
||||||
|
|
||||||
13
atune.spec
13
atune.spec
@ -3,7 +3,7 @@
|
|||||||
Summary: AI auto tuning system
|
Summary: AI auto tuning system
|
||||||
Name: atune
|
Name: atune
|
||||||
Version: 1.0.0
|
Version: 1.0.0
|
||||||
Release: 11
|
Release: 12
|
||||||
License: MulanPSL-2.0
|
License: MulanPSL-2.0
|
||||||
URL: https://gitee.com/openeuler/A-Tune
|
URL: https://gitee.com/openeuler/A-Tune
|
||||||
Source: https://gitee.com/openeuler/A-Tune/repository/archive/v%{version}.tar.gz
|
Source: https://gitee.com/openeuler/A-Tune/repository/archive/v%{version}.tar.gz
|
||||||
@ -14,6 +14,11 @@ Patch9002: change-Makefile-A-Tune-version-to-1.0.0.patch
|
|||||||
Patch9003: fix-start-failed-of-atuned-service.patch
|
Patch9003: fix-start-failed-of-atuned-service.patch
|
||||||
Patch9004: A-Tune-Add-sw64-architecture.patch
|
Patch9004: A-Tune-Add-sw64-architecture.patch
|
||||||
Patch9005: add-riscv64-support.patch
|
Patch9005: add-riscv64-support.patch
|
||||||
|
Patch9006: 0001-bugfix-transfer-can-only-save-file-to-specified-dir.patch
|
||||||
|
Patch9007: 0002-bugfix-training-model-can-only-save-file-to-specifie.patch
|
||||||
|
Patch9008: 0003-bugfix-collection-res-can-only-save-file-to-specifie.patch
|
||||||
|
|
||||||
|
|
||||||
BuildRequires: rpm-build golang-bin procps-ng
|
BuildRequires: rpm-build golang-bin procps-ng
|
||||||
BuildRequires: sqlite >= 3.24.0 openssl
|
BuildRequires: sqlite >= 3.24.0 openssl
|
||||||
BuildRequires: python3-scikit-optimize python3-pandas python3-xgboost
|
BuildRequires: python3-scikit-optimize python3-pandas python3-xgboost
|
||||||
@ -80,6 +85,9 @@ atune engine tool for manage atuned AI tuning system.
|
|||||||
%ifarch riscv64
|
%ifarch riscv64
|
||||||
%patch9005 -p1
|
%patch9005 -p1
|
||||||
%endif
|
%endif
|
||||||
|
%patch9006 -p1
|
||||||
|
%patch9007 -p1
|
||||||
|
%patch9008 -p1
|
||||||
|
|
||||||
%build
|
%build
|
||||||
%make_build
|
%make_build
|
||||||
@ -169,6 +177,9 @@ atune engine tool for manage atuned AI tuning system.
|
|||||||
%exclude /etc/atuned/rest_certs
|
%exclude /etc/atuned/rest_certs
|
||||||
|
|
||||||
%changelog
|
%changelog
|
||||||
|
* Thu Aug 10 2023 gaoruoshu <gaoruoshu@huawei.com> - 1.0.0-12
|
||||||
|
- bugfix set collection/transfer/train dir to specific dir
|
||||||
|
|
||||||
* Wed Aug 2 2023 zhangxiang <zhangxiang@iscas.ac.cn> - 1.0.0-11
|
* Wed Aug 2 2023 zhangxiang <zhangxiang@iscas.ac.cn> - 1.0.0-11
|
||||||
- add riscv64 support
|
- add riscv64 support
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user