3071 lines
116 KiB
Diff
3071 lines
116 KiB
Diff
|
|
diff -Naru a/packageship/application/apps/lifecycle/function/concurrent.py b/packageship/application/apps/lifecycle/function/concurrent.py
|
|||
|
|
--- a/packageship/application/apps/lifecycle/function/concurrent.py 2020-09-22 23:34:04.037937224 +0800
|
|||
|
|
+++ b/packageship/application/apps/lifecycle/function/concurrent.py 2020-09-22 23:48:39.938515522 +0800
|
|||
|
|
@@ -1,65 +1,80 @@
|
|||
|
|
-#!/usr/bin/python3
|
|||
|
|
-"""
|
|||
|
|
-Use queues to implement the producer and consumer model
|
|||
|
|
-to solve the database lock introduced by high concurrency issues
|
|||
|
|
-"""
|
|||
|
|
-import threading
|
|||
|
|
-from queue import Queue
|
|||
|
|
-from sqlalchemy.exc import SQLAlchemyError
|
|||
|
|
-from packageship.libs.dbutils import DBHelper
|
|||
|
|
-from packageship.libs.exception import Error, ContentNoneException
|
|||
|
|
-from packageship.libs.log import Log
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class ProducerConsumer():
|
|||
|
|
- """
|
|||
|
|
- The data written in the database is added to the high
|
|||
|
|
- concurrency queue, and the high concurrency is solved
|
|||
|
|
- by the form of the queue
|
|||
|
|
- """
|
|||
|
|
- _queue = Queue(maxsize=0)
|
|||
|
|
- _instance_lock = threading.Lock()
|
|||
|
|
- _log = Log(__name__)
|
|||
|
|
-
|
|||
|
|
- def __init__(self):
|
|||
|
|
- self.thread_queue = threading.Thread(target=self.__queue_process)
|
|||
|
|
- if not self.thread_queue.isAlive():
|
|||
|
|
- self.thread_queue.start()
|
|||
|
|
-
|
|||
|
|
- def start_thread(self):
|
|||
|
|
- """
|
|||
|
|
- Judge a thread, if the thread is terminated, restart
|
|||
|
|
- """
|
|||
|
|
- if not self.thread_queue.isAlive():
|
|||
|
|
- self.thread_queue = threading.Thread(target=self.__queue_process)
|
|||
|
|
- self.thread_queue.start()
|
|||
|
|
-
|
|||
|
|
- def __new__(cls, *args, **kwargs): # pylint: disable=unused-argument
|
|||
|
|
- """
|
|||
|
|
- Use the singleton pattern to create a thread-safe producer pattern
|
|||
|
|
- """
|
|||
|
|
- if not hasattr(cls, "_instance"):
|
|||
|
|
- with cls._instance_lock:
|
|||
|
|
- if not hasattr(cls, "_instance"):
|
|||
|
|
- cls._instance = object.__new__(cls)
|
|||
|
|
- return cls._instance
|
|||
|
|
-
|
|||
|
|
- def __queue_process(self):
|
|||
|
|
- """
|
|||
|
|
- Read the content in the queue and save and update
|
|||
|
|
- """
|
|||
|
|
- while not self._queue.empty():
|
|||
|
|
- _queue_value = self._queue.get()
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name="lifecycle") as database:
|
|||
|
|
- database.add(_queue_value)
|
|||
|
|
- except (Error, ContentNoneException, SQLAlchemyError) as error:
|
|||
|
|
- self._log.logger.error(error)
|
|||
|
|
-
|
|||
|
|
- def put(self, pending_content):
|
|||
|
|
- """
|
|||
|
|
- The content of the operation is added to the queue
|
|||
|
|
- """
|
|||
|
|
- if pending_content:
|
|||
|
|
- self._queue.put(pending_content)
|
|||
|
|
- self.start_thread()
|
|||
|
|
+#!/usr/bin/python3
|
|||
|
|
+"""
|
|||
|
|
+Use queues to implement the producer and consumer model
|
|||
|
|
+to solve the database lock introduced by high concurrency issues
|
|||
|
|
+"""
|
|||
|
|
+import threading
|
|||
|
|
+import time
|
|||
|
|
+from queue import Queue
|
|||
|
|
+from sqlalchemy.exc import SQLAlchemyError
|
|||
|
|
+from sqlalchemy.exc import OperationalError
|
|||
|
|
+from packageship.libs.exception import Error, ContentNoneException
|
|||
|
|
+from packageship.libs.log import Log
|
|||
|
|
+from packageship.libs.configutils.readconfig import ReadConfig
|
|||
|
|
+from packageship import system_config
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class ProducerConsumer():
|
|||
|
|
+ """
|
|||
|
|
+ The data written in the database is added to the high
|
|||
|
|
+ concurrency queue, and the high concurrency is solved
|
|||
|
|
+ by the form of the queue
|
|||
|
|
+ """
|
|||
|
|
+ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
+ queue_maxsize = int(_readconfig.get_config('LIFECYCLE', 'queue_maxsize'))
|
|||
|
|
+ if not isinstance(queue_maxsize, int):
|
|||
|
|
+ queue_maxsize = 1000
|
|||
|
|
+ _queue = Queue(maxsize=queue_maxsize)
|
|||
|
|
+ _instance_lock = threading.Lock()
|
|||
|
|
+ _log = Log(__name__)
|
|||
|
|
+
|
|||
|
|
+ def __init__(self):
|
|||
|
|
+ self.thread_queue = threading.Thread(target=self.__queue_process)
|
|||
|
|
+ self._instance_lock.acquire()
|
|||
|
|
+ if not self.thread_queue.isAlive():
|
|||
|
|
+ self.thread_queue = threading.Thread(target=self.__queue_process)
|
|||
|
|
+ self.thread_queue.start()
|
|||
|
|
+ self._instance_lock.release()
|
|||
|
|
+
|
|||
|
|
+ def start_thread(self):
|
|||
|
|
+ """
|
|||
|
|
+ Judge a thread, if the thread is terminated, restart
|
|||
|
|
+ """
|
|||
|
|
+ self._instance_lock.acquire()
|
|||
|
|
+ if not self.thread_queue.isAlive():
|
|||
|
|
+ self.thread_queue = threading.Thread(target=self.__queue_process)
|
|||
|
|
+ self.thread_queue.start()
|
|||
|
|
+ self._instance_lock.release()
|
|||
|
|
+
|
|||
|
|
+ def __new__(cls, *args, **kwargs): # pylint: disable=unused-argument
|
|||
|
|
+ """
|
|||
|
|
+ Use the singleton pattern to create a thread-safe producer pattern
|
|||
|
|
+ """
|
|||
|
|
+ if not hasattr(cls, "_instance"):
|
|||
|
|
+ with cls._instance_lock:
|
|||
|
|
+ if not hasattr(cls, "_instance"):
|
|||
|
|
+ cls._instance = object.__new__(cls)
|
|||
|
|
+ return cls._instance
|
|||
|
|
+
|
|||
|
|
+ def __queue_process(self):
|
|||
|
|
+ """
|
|||
|
|
+ Read the content in the queue and save and update
|
|||
|
|
+ """
|
|||
|
|
+ while not self._queue.empty():
|
|||
|
|
+ _queue_value, method = self._queue.get()
|
|||
|
|
+ try:
|
|||
|
|
+ method(_queue_value)
|
|||
|
|
+ except OperationalError as error:
|
|||
|
|
+ self._log.logger.warning(error)
|
|||
|
|
+ time.sleep(0.2)
|
|||
|
|
+ self._queue.put((_queue_value, method))
|
|||
|
|
+ except (Error, ContentNoneException, SQLAlchemyError) as error:
|
|||
|
|
+ self._log.logger.error(error)
|
|||
|
|
+
|
|||
|
|
+ def put(self, pending_content):
|
|||
|
|
+ """
|
|||
|
|
+ The content of the operation is added to the queue
|
|||
|
|
+ """
|
|||
|
|
+ if pending_content:
|
|||
|
|
+ self._queue.put(pending_content)
|
|||
|
|
+ self.start_thread()
|
|||
|
|
diff -Naru a/packageship/application/apps/lifecycle/function/download_yaml.py b/packageship/application/apps/lifecycle/function/download_yaml.py
|
|||
|
|
--- a/packageship/application/apps/lifecycle/function/download_yaml.py 2020-09-22 23:34:04.037937224 +0800
|
|||
|
|
+++ b/packageship/application/apps/lifecycle/function/download_yaml.py 2020-09-22 23:48:46.478549707 +0800
|
|||
|
|
@@ -1,222 +1,224 @@
|
|||
|
|
-#!/usr/bin/python3
|
|||
|
|
-"""
|
|||
|
|
-Dynamically obtain the content of the yaml file \
|
|||
|
|
-that saves the package information, periodically \
|
|||
|
|
-obtain the content and save it in the database
|
|||
|
|
-"""
|
|||
|
|
-import copy
|
|||
|
|
-from concurrent.futures import ThreadPoolExecutor
|
|||
|
|
-import datetime as date
|
|||
|
|
-import requests
|
|||
|
|
-import yaml
|
|||
|
|
-from retrying import retry
|
|||
|
|
-from sqlalchemy.exc import SQLAlchemyError
|
|||
|
|
-from requests.exceptions import HTTPError
|
|||
|
|
-from packageship import system_config
|
|||
|
|
-from packageship.application.models.package import Packages
|
|||
|
|
-from packageship.application.models.package import PackagesMaintainer
|
|||
|
|
-from packageship.libs.dbutils import DBHelper
|
|||
|
|
-from packageship.libs.exception import Error, ContentNoneException
|
|||
|
|
-from packageship.libs.configutils.readconfig import ReadConfig
|
|||
|
|
-from .base import Base
|
|||
|
|
-from .gitee import Gitee
|
|||
|
|
-from .concurrent import ProducerConsumer
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class ParseYaml():
|
|||
|
|
- """
|
|||
|
|
- Description: Analyze the downloaded remote yaml file, obtain the tags
|
|||
|
|
- and maintainer information in the yaml file, and save the obtained
|
|||
|
|
- relevant information into the database
|
|||
|
|
-
|
|||
|
|
- Attributes:
|
|||
|
|
- base: base class instance
|
|||
|
|
- pkg: Specific package data
|
|||
|
|
- _table_name: The name of the data table to be operated
|
|||
|
|
- openeuler_advisor_url: Get the warehouse address of the yaml file
|
|||
|
|
- _yaml_content: The content of the yaml file
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def __init__(self, pkg_info, base, table_name):
|
|||
|
|
- self.base = base
|
|||
|
|
- self.pkg = pkg_info
|
|||
|
|
- self._table_name = table_name
|
|||
|
|
- self.openeuler_advisor_url = self._path_stitching(pkg_info.name)
|
|||
|
|
- self._yaml_content = None
|
|||
|
|
- self.timed_task_open = self._timed_task_status()
|
|||
|
|
- self.producer_consumer = ProducerConsumer()
|
|||
|
|
-
|
|||
|
|
- def _timed_task_status(self):
|
|||
|
|
- """
|
|||
|
|
- The open state of information such as the maintainer in the scheduled task
|
|||
|
|
- """
|
|||
|
|
- _timed_task_status = True
|
|||
|
|
- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
- open_status = _readconfig.get_config('TIMEDTASK', 'open')
|
|||
|
|
- if open_status not in ('True', 'False'):
|
|||
|
|
- self.base.log.logger.error(
|
|||
|
|
- 'Wrong setting of the open state value of the scheduled task')
|
|||
|
|
- if open_status == 'False':
|
|||
|
|
- self.timed_task_open = False
|
|||
|
|
- return _timed_task_status
|
|||
|
|
-
|
|||
|
|
- def _path_stitching(self, pkg_name):
|
|||
|
|
- """
|
|||
|
|
- The path of the remote service call
|
|||
|
|
- """
|
|||
|
|
- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
- _remote_url = _readconfig.get_config('LIFECYCLE', 'warehouse_remote')
|
|||
|
|
- if _remote_url is None:
|
|||
|
|
- _remote_url = 'https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/'
|
|||
|
|
- return _remote_url + '{pkg_name}.yaml'.format(pkg_name=pkg_name)
|
|||
|
|
-
|
|||
|
|
- def update_database(self):
|
|||
|
|
- """
|
|||
|
|
- For the current package, determine whether the specific yaml file exists, parse
|
|||
|
|
- the data in it and save it in the database if it exists, and record the relevant
|
|||
|
|
- log if it does not exist
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- if self._openeuler_advisor_exists_yaml():
|
|||
|
|
- self._save_to_database()
|
|||
|
|
- else:
|
|||
|
|
- msg = "The yaml information of the [%s] package has not been" \
|
|||
|
|
- "obtained yet" % self.pkg.name
|
|||
|
|
- self.base.log.logger.warning(msg)
|
|||
|
|
-
|
|||
|
|
- def _get_yaml_content(self, url):
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- response = requests.get(
|
|||
|
|
- url, headers=self.base.headers)
|
|||
|
|
- if response.status_code == 200:
|
|||
|
|
- self._yaml_content = yaml.safe_load(response.content)
|
|||
|
|
-
|
|||
|
|
- except HTTPError as error:
|
|||
|
|
- self.base.log.logger.error(error)
|
|||
|
|
-
|
|||
|
|
- def _openeuler_advisor_exists_yaml(self):
|
|||
|
|
- """
|
|||
|
|
- Determine whether there is a yaml file with the current \
|
|||
|
|
- package name under the openeuler-advisor project
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- self._get_yaml_content(self.openeuler_advisor_url)
|
|||
|
|
- if self._yaml_content:
|
|||
|
|
- return True
|
|||
|
|
- return False
|
|||
|
|
-
|
|||
|
|
- def _save_to_database(self):
|
|||
|
|
- """
|
|||
|
|
- Save the acquired yaml file information to the database
|
|||
|
|
-
|
|||
|
|
- Raises:
|
|||
|
|
- ContentNoneException: The added entity content is empty
|
|||
|
|
- Error: An error occurred during data addition
|
|||
|
|
- """
|
|||
|
|
- self._parse_warehouse_info()
|
|||
|
|
- tags = self._yaml_content.get('git_tag', None)
|
|||
|
|
- if tags:
|
|||
|
|
- self._parse_tags_content(tags)
|
|||
|
|
- self.producer_consumer.put(copy.deepcopy(self.pkg))
|
|||
|
|
- if self.timed_task_open:
|
|||
|
|
- _maintainer = self._yaml_content.get('maintainers')
|
|||
|
|
- if _maintainer and isinstance(_maintainer, list):
|
|||
|
|
- self.pkg.maintainer = _maintainer[0]
|
|||
|
|
- self.pkg.maintainlevel = self._yaml_content.get('maintainlevel')
|
|||
|
|
- try:
|
|||
|
|
- if self.timed_task_open:
|
|||
|
|
- @retry(stop_max_attempt_number=3, stop_max_delay=500)
|
|||
|
|
- def _save_maintainer_info():
|
|||
|
|
- with DBHelper(db_name="lifecycle") as database:
|
|||
|
|
- _packages_maintainer = database.session.query(
|
|||
|
|
- PackagesMaintainer).filter(
|
|||
|
|
- PackagesMaintainer.name == self.pkg.name).first()
|
|||
|
|
- if _packages_maintainer:
|
|||
|
|
- _packages_maintainer.name = self.pkg.name
|
|||
|
|
- _packages_maintainer.maintainer = self.pkg.maintainer
|
|||
|
|
- _packages_maintainer.maintainlevel = self.pkg.maintainlevel
|
|||
|
|
- else:
|
|||
|
|
- _packages_maintainer = PackagesMaintainer(
|
|||
|
|
- name=self.pkg.name, maintainer=self.pkg.maintainer,
|
|||
|
|
- maintainlevel=self.pkg.maintainlevel)
|
|||
|
|
- self.producer_consumer.put(
|
|||
|
|
- copy.deepcopy(_packages_maintainer))
|
|||
|
|
- _save_maintainer_info()
|
|||
|
|
- except (Error, ContentNoneException, SQLAlchemyError) as error:
|
|||
|
|
- self.base.log.logger.error(error)
|
|||
|
|
-
|
|||
|
|
- def _parse_warehouse_info(self):
|
|||
|
|
- """
|
|||
|
|
- Parse the warehouse information in the yaml file
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- if self._yaml_content:
|
|||
|
|
- self.pkg.version_control = self._yaml_content.get(
|
|||
|
|
- 'version_control')
|
|||
|
|
- self.pkg.src_repo = self._yaml_content.get('src_repo')
|
|||
|
|
- self.pkg.tag_prefix = self._yaml_content.get('tag_prefix')
|
|||
|
|
-
|
|||
|
|
- def _parse_tags_content(self, tags):
|
|||
|
|
- """
|
|||
|
|
- Parse the obtained tags content
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- # Integrate tags information into key-value pairs
|
|||
|
|
- _tags = [(tag.split()[0], tag.split()[1]) for tag in tags]
|
|||
|
|
- _tags = sorted(_tags, key=lambda x: x[0], reverse=True)
|
|||
|
|
- self.pkg.latest_version = _tags[0][1]
|
|||
|
|
- self.pkg.latest_version_time = _tags[0][0]
|
|||
|
|
- _end_time = date.datetime.strptime(
|
|||
|
|
- self.pkg.latest_version_time, '%Y-%m-%d')
|
|||
|
|
- if self.pkg.latest_version != self.pkg.version:
|
|||
|
|
- for _version in _tags:
|
|||
|
|
- if _version[1] == self.pkg.version:
|
|||
|
|
- _end_time = date.datetime.strptime(
|
|||
|
|
- _version[0], '%Y-%m-%d')
|
|||
|
|
- self.pkg.used_time = (date.datetime.now() - _end_time).days
|
|||
|
|
-
|
|||
|
|
- except (IndexError, Error) as index_error:
|
|||
|
|
- self.base.log.logger.error(index_error)
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-def update_pkg_info(pkg_info_update=True):
|
|||
|
|
- """
|
|||
|
|
- Update the information of the upstream warehouse in the source package
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- base_control = Base()
|
|||
|
|
- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
- pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers')
|
|||
|
|
- _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse')
|
|||
|
|
- if _warehouse is None:
|
|||
|
|
- _warehouse = 'src-openeuler'
|
|||
|
|
- if not isinstance(pool_workers, int):
|
|||
|
|
- pool_workers = 10
|
|||
|
|
- # Open thread pool
|
|||
|
|
- pool = ThreadPoolExecutor(max_workers=pool_workers)
|
|||
|
|
- with DBHelper(db_name="lifecycle") as database:
|
|||
|
|
- for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer'],
|
|||
|
|
- database.engine.table_names()):
|
|||
|
|
-
|
|||
|
|
- cls_model = Packages.package_meta(table_name)
|
|||
|
|
- # Query a specific table
|
|||
|
|
- for package_item in database.session.query(cls_model).all():
|
|||
|
|
- if pkg_info_update:
|
|||
|
|
- parse_yaml = ParseYaml(
|
|||
|
|
- pkg_info=copy.deepcopy(package_item),
|
|||
|
|
- base=base_control,
|
|||
|
|
- table_name=table_name)
|
|||
|
|
- pool.submit(parse_yaml.update_database)
|
|||
|
|
- else:
|
|||
|
|
- # Get the issue of each warehouse and save it
|
|||
|
|
- gitee_issue = Gitee(
|
|||
|
|
- package_item, _warehouse, package_item.name, table_name)
|
|||
|
|
- pool.submit(gitee_issue.query_issues_info)
|
|||
|
|
- pool.shutdown()
|
|||
|
|
- except SQLAlchemyError as error_msg:
|
|||
|
|
- base_control.log.logger.error(error_msg)
|
|||
|
|
+#!/usr/bin/python3
|
|||
|
|
+"""
|
|||
|
|
+Dynamically obtain the content of the yaml file \
|
|||
|
|
+that saves the package information, periodically \
|
|||
|
|
+obtain the content and save it in the database
|
|||
|
|
+"""
|
|||
|
|
+import copy
|
|||
|
|
+from concurrent.futures import ThreadPoolExecutor
|
|||
|
|
+import datetime as date
|
|||
|
|
+import requests
|
|||
|
|
+import yaml
|
|||
|
|
+from retrying import retry
|
|||
|
|
+from sqlalchemy.exc import SQLAlchemyError
|
|||
|
|
+from requests.exceptions import HTTPError
|
|||
|
|
+from packageship import system_config
|
|||
|
|
+from packageship.application.models.package import Packages
|
|||
|
|
+from packageship.application.models.package import PackagesMaintainer
|
|||
|
|
+from packageship.libs.dbutils import DBHelper
|
|||
|
|
+from packageship.libs.exception import Error, ContentNoneException
|
|||
|
|
+from packageship.libs.configutils.readconfig import ReadConfig
|
|||
|
|
+from .base import Base
|
|||
|
|
+from .gitee import Gitee
|
|||
|
|
+from .concurrent import ProducerConsumer
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class ParseYaml():
|
|||
|
|
+ """
|
|||
|
|
+ Description: Analyze the downloaded remote yaml file, obtain the tags
|
|||
|
|
+ and maintainer information in the yaml file, and save the obtained
|
|||
|
|
+ relevant information into the database
|
|||
|
|
+
|
|||
|
|
+ Attributes:
|
|||
|
|
+ base: base class instance
|
|||
|
|
+ pkg: Specific package data
|
|||
|
|
+ _table_name: The name of the data table to be operated
|
|||
|
|
+ openeuler_advisor_url: Get the warehouse address of the yaml file
|
|||
|
|
+ _yaml_content: The content of the yaml file
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def __init__(self, pkg_info, base, table_name):
|
|||
|
|
+ self.base = base
|
|||
|
|
+ self.pkg = pkg_info
|
|||
|
|
+ self._table_name = table_name
|
|||
|
|
+ self.openeuler_advisor_url = self._path_stitching(pkg_info.name)
|
|||
|
|
+ self._yaml_content = None
|
|||
|
|
+ self.timed_task_open = self._timed_task_status()
|
|||
|
|
+ self.producer_consumer = ProducerConsumer()
|
|||
|
|
+
|
|||
|
|
+ def _timed_task_status(self):
|
|||
|
|
+ """
|
|||
|
|
+ The open state of information such as the maintainer in the scheduled task
|
|||
|
|
+ """
|
|||
|
|
+ _timed_task_status = True
|
|||
|
|
+ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
+ open_status = _readconfig.get_config('TIMEDTASK', 'open')
|
|||
|
|
+ if open_status not in ('True', 'False'):
|
|||
|
|
+ self.base.log.logger.error(
|
|||
|
|
+ 'Wrong setting of the open state value of the scheduled task')
|
|||
|
|
+ if open_status == 'False':
|
|||
|
|
+ self.timed_task_open = False
|
|||
|
|
+ return _timed_task_status
|
|||
|
|
+
|
|||
|
|
+ def _path_stitching(self, pkg_name):
|
|||
|
|
+ """
|
|||
|
|
+ The path of the remote service call
|
|||
|
|
+ """
|
|||
|
|
+ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
+ _remote_url = _readconfig.get_config('LIFECYCLE', 'warehouse_remote')
|
|||
|
|
+ if _remote_url is None:
|
|||
|
|
+ _remote_url = 'https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/'
|
|||
|
|
+ return _remote_url + '{pkg_name}.yaml'.format(pkg_name=pkg_name)
|
|||
|
|
+
|
|||
|
|
+ def update_database(self):
|
|||
|
|
+ """
|
|||
|
|
+ For the current package, determine whether the specific yaml file exists, parse
|
|||
|
|
+ the data in it and save it in the database if it exists, and record the relevant
|
|||
|
|
+ log if it does not exist
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ if self._openeuler_advisor_exists_yaml():
|
|||
|
|
+ self._save_to_database()
|
|||
|
|
+ else:
|
|||
|
|
+ msg = "The yaml information of the [%s] package has not been" \
|
|||
|
|
+ "obtained yet" % self.pkg.name
|
|||
|
|
+ self.base.log.logger.warning(msg)
|
|||
|
|
+
|
|||
|
|
+ def _get_yaml_content(self, url):
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ response = requests.get(
|
|||
|
|
+ url, headers=self.base.headers)
|
|||
|
|
+ if response.status_code == 200:
|
|||
|
|
+ self._yaml_content = yaml.safe_load(response.content)
|
|||
|
|
+
|
|||
|
|
+ except HTTPError as error:
|
|||
|
|
+ self.base.log.logger.error(error)
|
|||
|
|
+
|
|||
|
|
+ def _openeuler_advisor_exists_yaml(self):
|
|||
|
|
+ """
|
|||
|
|
+ Determine whether there is a yaml file with the current \
|
|||
|
|
+ package name under the openeuler-advisor project
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ self._get_yaml_content(self.openeuler_advisor_url)
|
|||
|
|
+ if self._yaml_content:
|
|||
|
|
+ return True
|
|||
|
|
+ return False
|
|||
|
|
+
|
|||
|
|
+ def _save_to_database(self):
|
|||
|
|
+ """
|
|||
|
|
+ Save the acquired yaml file information to the database
|
|||
|
|
+
|
|||
|
|
+ Raises:
|
|||
|
|
+ ContentNoneException: The added entity content is empty
|
|||
|
|
+ Error: An error occurred during data addition
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def _save_package(package_module):
|
|||
|
|
+ with DBHelper(db_name="lifecycle") as database:
|
|||
|
|
+ database.add(package_module)
|
|||
|
|
+
|
|||
|
|
+ def _save_maintainer_info(maintainer_module):
|
|||
|
|
+ with DBHelper(db_name="lifecycle") as database:
|
|||
|
|
+ _packages_maintainer = database.session.query(
|
|||
|
|
+ PackagesMaintainer).filter(
|
|||
|
|
+ PackagesMaintainer.name == maintainer_module['name']).first()
|
|||
|
|
+ if _packages_maintainer:
|
|||
|
|
+ for key, val in maintainer_module.items():
|
|||
|
|
+ setattr(_packages_maintainer, key, val)
|
|||
|
|
+ else:
|
|||
|
|
+ _packages_maintainer = PackagesMaintainer(
|
|||
|
|
+ **maintainer_module)
|
|||
|
|
+ database.add(_packages_maintainer)
|
|||
|
|
+
|
|||
|
|
+ self._parse_warehouse_info()
|
|||
|
|
+ tags = self._yaml_content.get('git_tag', None)
|
|||
|
|
+ if tags:
|
|||
|
|
+ self._parse_tags_content(tags)
|
|||
|
|
+ self.producer_consumer.put(
|
|||
|
|
+ (copy.deepcopy(self.pkg), _save_package))
|
|||
|
|
+ if self.timed_task_open:
|
|||
|
|
+ maintainer = {'name': self.pkg.name}
|
|||
|
|
+ _maintainer = self._yaml_content.get('maintainers')
|
|||
|
|
+ if _maintainer and isinstance(_maintainer, list):
|
|||
|
|
+ maintainer['maintainer'] = _maintainer[0]
|
|||
|
|
+ maintainer['maintainlevel'] = self._yaml_content.get(
|
|||
|
|
+ 'maintainlevel')
|
|||
|
|
+
|
|||
|
|
+ self.producer_consumer.put((maintainer, _save_maintainer_info))
|
|||
|
|
+
|
|||
|
|
+ def _parse_warehouse_info(self):
|
|||
|
|
+ """
|
|||
|
|
+ Parse the warehouse information in the yaml file
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ if self._yaml_content:
|
|||
|
|
+ self.pkg.version_control = self._yaml_content.get(
|
|||
|
|
+ 'version_control')
|
|||
|
|
+ self.pkg.src_repo = self._yaml_content.get('src_repo')
|
|||
|
|
+ self.pkg.tag_prefix = self._yaml_content.get('tag_prefix')
|
|||
|
|
+
|
|||
|
|
+ def _parse_tags_content(self, tags):
|
|||
|
|
+ """
|
|||
|
|
+ Parse the obtained tags content
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ # Integrate tags information into key-value pairs
|
|||
|
|
+ _tags = [(tag.split()[0], tag.split()[1]) for tag in tags]
|
|||
|
|
+ _tags = sorted(_tags, key=lambda x: x[0], reverse=True)
|
|||
|
|
+ self.pkg.latest_version = _tags[0][1]
|
|||
|
|
+ self.pkg.latest_version_time = _tags[0][0]
|
|||
|
|
+ _end_time = date.datetime.strptime(
|
|||
|
|
+ self.pkg.latest_version_time, '%Y-%m-%d')
|
|||
|
|
+ if self.pkg.latest_version != self.pkg.version:
|
|||
|
|
+ for _version in _tags:
|
|||
|
|
+ if _version[1] == self.pkg.version:
|
|||
|
|
+ _end_time = date.datetime.strptime(
|
|||
|
|
+ _version[0], '%Y-%m-%d')
|
|||
|
|
+ self.pkg.used_time = (date.datetime.now() - _end_time).days
|
|||
|
|
+
|
|||
|
|
+ except (IndexError, Error) as index_error:
|
|||
|
|
+ self.base.log.logger.error(index_error)
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+def update_pkg_info(pkg_info_update=True):
|
|||
|
|
+ """
|
|||
|
|
+ Update the information of the upstream warehouse in the source package
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ base_control = Base()
|
|||
|
|
+ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
+ pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers')
|
|||
|
|
+ _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse')
|
|||
|
|
+ if _warehouse is None:
|
|||
|
|
+ _warehouse = 'src-openeuler'
|
|||
|
|
+ if not isinstance(pool_workers, int):
|
|||
|
|
+ pool_workers = 10
|
|||
|
|
+ # Open thread pool
|
|||
|
|
+ pool = ThreadPoolExecutor(max_workers=pool_workers)
|
|||
|
|
+ with DBHelper(db_name="lifecycle") as database:
|
|||
|
|
+ for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer', 'database_info'],
|
|||
|
|
+ database.engine.table_names()):
|
|||
|
|
+
|
|||
|
|
+ cls_model = Packages.package_meta(table_name)
|
|||
|
|
+ # Query a specific table
|
|||
|
|
+ for package_item in database.session.query(cls_model).all():
|
|||
|
|
+ if pkg_info_update:
|
|||
|
|
+ parse_yaml = ParseYaml(
|
|||
|
|
+ pkg_info=copy.deepcopy(package_item),
|
|||
|
|
+ base=base_control,
|
|||
|
|
+ table_name=table_name)
|
|||
|
|
+ pool.submit(parse_yaml.update_database)
|
|||
|
|
+ else:
|
|||
|
|
+ # Get the issue of each warehouse and save it
|
|||
|
|
+ gitee_issue = Gitee(
|
|||
|
|
+ copy.deepcopy(package_item), _warehouse, package_item.name, table_name)
|
|||
|
|
+ pool.submit(gitee_issue.query_issues_info)
|
|||
|
|
+ pool.shutdown()
|
|||
|
|
+ except SQLAlchemyError as error_msg:
|
|||
|
|
+ base_control.log.logger.error(error_msg)
|
|||
|
|
diff -Naru a/packageship/application/apps/lifecycle/function/gitee.py b/packageship/application/apps/lifecycle/function/gitee.py
|
|||
|
|
--- a/packageship/application/apps/lifecycle/function/gitee.py 2020-09-22 23:34:04.037937224 +0800
|
|||
|
|
+++ b/packageship/application/apps/lifecycle/function/gitee.py 2020-09-22 23:48:52.698582219 +0800
|
|||
|
|
@@ -1,224 +1,223 @@
|
|||
|
|
-#!/usr/bin/python3
|
|||
|
|
-"""
|
|||
|
|
-Description:Get issue info from gitee
|
|||
|
|
-Class: Gitee
|
|||
|
|
-"""
|
|||
|
|
-import copy
|
|||
|
|
-from json import JSONDecodeError
|
|||
|
|
-from retrying import retry
|
|||
|
|
-import requests
|
|||
|
|
-from requests.exceptions import HTTPError
|
|||
|
|
-from sqlalchemy.exc import SQLAlchemyError
|
|||
|
|
-from packageship.libs.dbutils import DBHelper
|
|||
|
|
-from packageship.libs.configutils.readconfig import ReadConfig
|
|||
|
|
-from packageship.libs.exception import Error, ContentNoneException
|
|||
|
|
-from packageship.application.models.package import PackagesIssue
|
|||
|
|
-from packageship import system_config
|
|||
|
|
-from packageship.libs.log import Log
|
|||
|
|
-from .concurrent import ProducerConsumer
|
|||
|
|
-
|
|||
|
|
-LOGGER = Log(__name__)
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class Gitee():
|
|||
|
|
- """
|
|||
|
|
- gitee version management tool related information acquisition
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def __init__(self, pkg_info, owner, repo, table_name):
|
|||
|
|
- self.pkg_info = pkg_info
|
|||
|
|
- self.owner = owner
|
|||
|
|
- self.repo = repo
|
|||
|
|
- self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
- self.url = "https://gitee.com/"
|
|||
|
|
- self.api_url = "https://gitee.com/api/v5/repos"
|
|||
|
|
- self.pool = None
|
|||
|
|
- self.issue_id = None
|
|||
|
|
- self.defect = 0
|
|||
|
|
- self.feature = 0
|
|||
|
|
- self.cve = 0
|
|||
|
|
- self.patch_files_path = self._read_config.get_system(
|
|||
|
|
- "patch_files_path")
|
|||
|
|
- self.table_name = table_name
|
|||
|
|
- self.producer_consumer = ProducerConsumer()
|
|||
|
|
-
|
|||
|
|
- def query_issues_info(self, issue_id=""):
|
|||
|
|
- """
|
|||
|
|
- Description: View the issue details of the specified package
|
|||
|
|
- Args:
|
|||
|
|
- issue_id: Issue id
|
|||
|
|
- Returns:
|
|||
|
|
- issue_content_list: The issue details of the specified package list
|
|||
|
|
- Raises:
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- issue_url = self.api_url + \
|
|||
|
|
- "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id)
|
|||
|
|
- try:
|
|||
|
|
- response = requests.get(
|
|||
|
|
- issue_url, params={"state": "all", "per_page": 100})
|
|||
|
|
- except Error as error:
|
|||
|
|
- LOGGER.logger.error(error)
|
|||
|
|
- return None
|
|||
|
|
- if response.status_code != 200:
|
|||
|
|
- return None
|
|||
|
|
- total_page = 1 if issue_id else int(response.headers['total_page'])
|
|||
|
|
- total_count = int(response.headers['total_count'])
|
|||
|
|
- if total_count > 0:
|
|||
|
|
- issue_list = self._query_per_page_issue_info(total_page, issue_url)
|
|||
|
|
- if not issue_list:
|
|||
|
|
- LOGGER.logger.error(
|
|||
|
|
- "An error occurred while querying {}".format(self.repo))
|
|||
|
|
- return None
|
|||
|
|
- self._save_issues(issue_list)
|
|||
|
|
-
|
|||
|
|
- def _query_per_page_issue_info(self, total_page, issue_url):
|
|||
|
|
- """
|
|||
|
|
- Description: View the issue details
|
|||
|
|
- Args:
|
|||
|
|
- total_page: total page
|
|||
|
|
- issue_url: issue url
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- issue_content_list = []
|
|||
|
|
- for i in range(1, total_page + 1):
|
|||
|
|
-
|
|||
|
|
- @retry(stop_max_attempt_number=3, stop_max_delay=1000)
|
|||
|
|
- def request_issue(page, issue_url):
|
|||
|
|
- try:
|
|||
|
|
- response = requests.get(issue_url,
|
|||
|
|
- params={"state": "all", "per_page": 100, "page": page})
|
|||
|
|
- except HTTPError:
|
|||
|
|
- raise HTTPError('Network request error')
|
|||
|
|
- return response
|
|||
|
|
-
|
|||
|
|
- try:
|
|||
|
|
- response = request_issue(i, issue_url)
|
|||
|
|
- if response.status_code != 200:
|
|||
|
|
- LOGGER.logger.warning(response.content.decode("utf-8"))
|
|||
|
|
- continue
|
|||
|
|
- issue_content_list.extend(
|
|||
|
|
- self.parse_issues_content(response.json()))
|
|||
|
|
- except (JSONDecodeError, Error) as error:
|
|||
|
|
- LOGGER.logger.error(error)
|
|||
|
|
- return issue_content_list
|
|||
|
|
-
|
|||
|
|
- def _save_issues(self, issue_list):
|
|||
|
|
- """
|
|||
|
|
- Save the obtained issue information
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- issue_ids = [issue['issue_id'] for issue in issue_list]
|
|||
|
|
- with DBHelper(db_name="lifecycle") as database:
|
|||
|
|
-
|
|||
|
|
- @retry(stop_max_attempt_number=3, stop_max_delay=500)
|
|||
|
|
- def _query_pkgissues():
|
|||
|
|
- exist_issues = database.session.query(PackagesIssue).filter(
|
|||
|
|
- PackagesIssue.issue_id.in_(issue_ids)).all() # pylint: disable=protected-access
|
|||
|
|
- return exist_issues
|
|||
|
|
-
|
|||
|
|
- exist_issues = _query_pkgissues()
|
|||
|
|
- # Save the issue
|
|||
|
|
- for issue_item in issue_list:
|
|||
|
|
- issue_model = [
|
|||
|
|
- issue for issue in exist_issues if issue.issue_id == issue_item['issue_id']]
|
|||
|
|
- if issue_model:
|
|||
|
|
- for key, val in issue_item.items():
|
|||
|
|
- setattr(issue_model[0], key, val)
|
|||
|
|
- self.producer_consumer.put(
|
|||
|
|
- copy.deepcopy(issue_model[0]))
|
|||
|
|
- else:
|
|||
|
|
- self.producer_consumer.put(
|
|||
|
|
- PackagesIssue(**issue_item))
|
|||
|
|
-
|
|||
|
|
- # The number of various issues in the update package
|
|||
|
|
- self.pkg_info.defect = self.defect
|
|||
|
|
- self.pkg_info.feature = self.feature
|
|||
|
|
- self.pkg_info.cve = self.cve
|
|||
|
|
- self.producer_consumer.put(copy.deepcopy(self.pkg_info))
|
|||
|
|
-
|
|||
|
|
- except (Error, ContentNoneException, SQLAlchemyError) as error:
|
|||
|
|
- LOGGER.logger.error(
|
|||
|
|
- 'An abnormal error occurred while saving related issues:%s' % error if error else '')
|
|||
|
|
-
|
|||
|
|
- def parse_issues_content(self, sources):
|
|||
|
|
- """
|
|||
|
|
- Description: Parse the response content and get issue content
|
|||
|
|
- Args:Issue list
|
|||
|
|
-
|
|||
|
|
- Returns:list:issue_id, issue_url, issue_content, issue_status, issue_download
|
|||
|
|
- Raises:
|
|||
|
|
- """
|
|||
|
|
- result_list = []
|
|||
|
|
- if isinstance(sources, list):
|
|||
|
|
- for source in sources:
|
|||
|
|
- issue_content = self.parse_issue_content(source)
|
|||
|
|
- if issue_content:
|
|||
|
|
- result_list.append(issue_content)
|
|||
|
|
- else:
|
|||
|
|
- issue_content = self.parse_issue_content(sources)
|
|||
|
|
- if issue_content:
|
|||
|
|
- result_list.append(issue_content)
|
|||
|
|
- return result_list
|
|||
|
|
-
|
|||
|
|
- def parse_issue_content(self, source):
|
|||
|
|
- """
|
|||
|
|
- Description: Parse the response content and get issue content
|
|||
|
|
- Args: Source of issue content
|
|||
|
|
-
|
|||
|
|
- Returns:list:issue_id, issue_url, issue_content, issue_status, issue_download, issue_status
|
|||
|
|
- issue_type, related_release
|
|||
|
|
- Raises:KeyError
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- result_dict = {"issue_id": source['number'], "issue_url": source['html_url'],
|
|||
|
|
- "issue_title": source['title'].strip(),
|
|||
|
|
- "issue_content": source['body'].strip(),
|
|||
|
|
- "issue_status": source['state'], "issue_download": "",
|
|||
|
|
- "issue_type": source["issue_type"],
|
|||
|
|
- "pkg_name": self.repo,
|
|||
|
|
- "related_release": source["labels"][0]['name'] if source["labels"] else ''}
|
|||
|
|
- if source["issue_type"] == "缺陷":
|
|||
|
|
- self.defect += 1
|
|||
|
|
- elif source["issue_type"] == "需求":
|
|||
|
|
- self.feature += 1
|
|||
|
|
- elif source["issue_type"] == "CVE和安全问题":
|
|||
|
|
- self.cve += 1
|
|||
|
|
- else:
|
|||
|
|
- pass
|
|||
|
|
- except KeyError as error:
|
|||
|
|
- LOGGER.logger.error(error)
|
|||
|
|
- return None
|
|||
|
|
- return result_dict
|
|||
|
|
-
|
|||
|
|
- def issue_hooks(self, issue_hook_info):
|
|||
|
|
- """
|
|||
|
|
- Description: Hook data triggered by a new task operation
|
|||
|
|
- Args:
|
|||
|
|
- issue_hook_info: Issue info
|
|||
|
|
- Returns:
|
|||
|
|
-
|
|||
|
|
- Raises:
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- if issue_hook_info is None:
|
|||
|
|
- raise ContentNoneException(
|
|||
|
|
- 'The content cannot be empty')
|
|||
|
|
- issue_info_list = []
|
|||
|
|
- issue_info = issue_hook_info["issue"]
|
|||
|
|
- issue_content = self.parse_issue_content(issue_info)
|
|||
|
|
- if issue_content:
|
|||
|
|
- issue_info_list.append(issue_content)
|
|||
|
|
- if self.feature != 0:
|
|||
|
|
- self.defect, self.feature, self.cve = self.pkg_info.defect, self.pkg_info.feature + \
|
|||
|
|
- 1, self.pkg_info.cve
|
|||
|
|
- if self.defect != 0:
|
|||
|
|
- self.defect, self.feature, self.cve = self.pkg_info.defect + \
|
|||
|
|
- 1, self.pkg_info.feature, self.pkg_info.cve
|
|||
|
|
- if self.cve != 0:
|
|||
|
|
- self.defect, self.feature, self.cve = self.pkg_info.defect, self.pkg_info.feature, self.pkg_info.cve + 1
|
|||
|
|
- self._save_issues(issue_info_list)
|
|||
|
|
+#!/usr/bin/python3
|
|||
|
|
+"""
|
|||
|
|
+Description:Get issue info from gitee
|
|||
|
|
+Class: Gitee
|
|||
|
|
+"""
|
|||
|
|
+import copy
|
|||
|
|
+from json import JSONDecodeError
|
|||
|
|
+from retrying import retry
|
|||
|
|
+import requests
|
|||
|
|
+from requests.exceptions import HTTPError
|
|||
|
|
+from sqlalchemy.exc import SQLAlchemyError
|
|||
|
|
+from packageship.libs.dbutils import DBHelper
|
|||
|
|
+from packageship.libs.configutils.readconfig import ReadConfig
|
|||
|
|
+from packageship.libs.exception import Error, ContentNoneException
|
|||
|
|
+from packageship.application.models.package import PackagesIssue
|
|||
|
|
+from packageship import system_config
|
|||
|
|
+from packageship.libs.log import Log
|
|||
|
|
+from .concurrent import ProducerConsumer
|
|||
|
|
+
|
|||
|
|
+LOGGER = Log(__name__)
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class Gitee():
|
|||
|
|
+ """
|
|||
|
|
+ gitee version management tool related information acquisition
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def __init__(self, pkg_info, owner, repo, table_name):
|
|||
|
|
+ self.pkg_info = pkg_info
|
|||
|
|
+ self.owner = owner
|
|||
|
|
+ self.repo = repo
|
|||
|
|
+ self._read_config = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
+ self.url = "https://gitee.com/"
|
|||
|
|
+ self.api_url = "https://gitee.com/api/v5/repos"
|
|||
|
|
+ self.pool = None
|
|||
|
|
+ self.issue_id = None
|
|||
|
|
+ self.defect = 0
|
|||
|
|
+ self.feature = 0
|
|||
|
|
+ self.cve = 0
|
|||
|
|
+ self.patch_files_path = self._read_config.get_system(
|
|||
|
|
+ "patch_files_path")
|
|||
|
|
+ self.table_name = table_name
|
|||
|
|
+ self.producer_consumer = ProducerConsumer()
|
|||
|
|
+
|
|||
|
|
+ def query_issues_info(self, issue_id=""):
|
|||
|
|
+ """
|
|||
|
|
+ Description: View the issue details of the specified package
|
|||
|
|
+ Args:
|
|||
|
|
+ issue_id: Issue id
|
|||
|
|
+ Returns:
|
|||
|
|
+ issue_content_list: The issue details of the specified package list
|
|||
|
|
+ Raises:
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ issue_url = self.api_url + \
|
|||
|
|
+ "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id)
|
|||
|
|
+ try:
|
|||
|
|
+ response = requests.get(
|
|||
|
|
+ issue_url, params={"state": "all", "per_page": 100})
|
|||
|
|
+ except Error as error:
|
|||
|
|
+ LOGGER.logger.error(error)
|
|||
|
|
+ return None
|
|||
|
|
+ if response.status_code != 200:
|
|||
|
|
+ return None
|
|||
|
|
+ total_page = 1 if issue_id else int(response.headers['total_page'])
|
|||
|
|
+ total_count = int(response.headers['total_count'])
|
|||
|
|
+ if total_count > 0:
|
|||
|
|
+ issue_list = self._query_per_page_issue_info(total_page, issue_url)
|
|||
|
|
+ if not issue_list:
|
|||
|
|
+ LOGGER.logger.error(
|
|||
|
|
+ "An error occurred while querying {}".format(self.repo))
|
|||
|
|
+ return None
|
|||
|
|
+ self._save_issues(issue_list)
|
|||
|
|
+
|
|||
|
|
+ def _query_per_page_issue_info(self, total_page, issue_url):
|
|||
|
|
+ """
|
|||
|
|
+ Description: View the issue details
|
|||
|
|
+ Args:
|
|||
|
|
+ total_page: total page
|
|||
|
|
+ issue_url: issue url
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ issue_content_list = []
|
|||
|
|
+ for i in range(1, total_page + 1):
|
|||
|
|
+
|
|||
|
|
+ @retry(stop_max_attempt_number=3, stop_max_delay=1000)
|
|||
|
|
+ def request_issue(page, issue_url):
|
|||
|
|
+ try:
|
|||
|
|
+ response = requests.get(issue_url,
|
|||
|
|
+ params={"state": "all", "per_page": 100, "page": page})
|
|||
|
|
+ except HTTPError:
|
|||
|
|
+ raise HTTPError('Network request error')
|
|||
|
|
+ return response
|
|||
|
|
+
|
|||
|
|
+ try:
|
|||
|
|
+ response = request_issue(i, issue_url)
|
|||
|
|
+ if response.status_code != 200:
|
|||
|
|
+ LOGGER.logger.warning(response.content.decode("utf-8"))
|
|||
|
|
+ continue
|
|||
|
|
+ issue_content_list.extend(
|
|||
|
|
+ self.parse_issues_content(response.json()))
|
|||
|
|
+ except (JSONDecodeError, Error) as error:
|
|||
|
|
+ LOGGER.logger.error(error)
|
|||
|
|
+ return issue_content_list
|
|||
|
|
+
|
|||
|
|
+ def _save_issues(self, issue_list):
|
|||
|
|
+ """
|
|||
|
|
+ Save the obtained issue information
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ def _save(issue_module):
|
|||
|
|
+ with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
+
|
|||
|
|
+ exist_issues = database.session.query(PackagesIssue).filter(
|
|||
|
|
+ PackagesIssue.issue_id == issue_module['issue_id']).first()
|
|||
|
|
+ if exist_issues:
|
|||
|
|
+
|
|||
|
|
+ # Save the issue
|
|||
|
|
+ for key, val in issue_module.items():
|
|||
|
|
+ setattr(exist_issues, key, val)
|
|||
|
|
+ else:
|
|||
|
|
+ exist_issues = PackagesIssue(**issue_module)
|
|||
|
|
+ database.add(exist_issues)
|
|||
|
|
+
|
|||
|
|
+ def _save_package(package_module):
|
|||
|
|
+ with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
+ database.add(package_module)
|
|||
|
|
+
|
|||
|
|
+ for issue_item in issue_list:
|
|||
|
|
+ self.producer_consumer.put(
|
|||
|
|
+ (copy.deepcopy(issue_item), _save))
|
|||
|
|
+
|
|||
|
|
+ # The number of various issues in the update package
|
|||
|
|
+ self.pkg_info.defect = self.defect
|
|||
|
|
+ self.pkg_info.feature = self.feature
|
|||
|
|
+ self.pkg_info.cve = self.cve
|
|||
|
|
+ self.producer_consumer.put((copy.deepcopy(self.pkg_info), _save_package))
|
|||
|
|
+
|
|||
|
|
+ except (Error, ContentNoneException, SQLAlchemyError) as error:
|
|||
|
|
+ LOGGER.logger.error(
|
|||
|
|
+ 'An abnormal error occurred while saving related issues:%s' % error if error else '')
|
|||
|
|
+
|
|||
|
|
+ def parse_issues_content(self, sources):
|
|||
|
|
+ """
|
|||
|
|
+ Description: Parse the response content and get issue content
|
|||
|
|
+ Args:Issue list
|
|||
|
|
+
|
|||
|
|
+ Returns:list:issue_id, issue_url, issue_content, issue_status, issue_download
|
|||
|
|
+ Raises:
|
|||
|
|
+ """
|
|||
|
|
+ result_list = []
|
|||
|
|
+ if isinstance(sources, list):
|
|||
|
|
+ for source in sources:
|
|||
|
|
+ issue_content = self.parse_issue_content(source)
|
|||
|
|
+ if issue_content:
|
|||
|
|
+ result_list.append(issue_content)
|
|||
|
|
+ else:
|
|||
|
|
+ issue_content = self.parse_issue_content(sources)
|
|||
|
|
+ if issue_content:
|
|||
|
|
+ result_list.append(issue_content)
|
|||
|
|
+ return result_list
|
|||
|
|
+
|
|||
|
|
+ def parse_issue_content(self, source):
|
|||
|
|
+ """
|
|||
|
|
+ Description: Parse the response content and get issue content
|
|||
|
|
+ Args: Source of issue content
|
|||
|
|
+
|
|||
|
|
+ Returns:list:issue_id, issue_url, issue_content, issue_status, issue_download, issue_status
|
|||
|
|
+ issue_type, related_release
|
|||
|
|
+ Raises:KeyError
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ result_dict = {"issue_id": source['number'], "issue_url": source['html_url'],
|
|||
|
|
+ "issue_title": source['title'].strip(),
|
|||
|
|
+ "issue_content": source['body'].strip(),
|
|||
|
|
+ "issue_status": source['state'], "issue_download": "",
|
|||
|
|
+ "issue_type": source["issue_type"],
|
|||
|
|
+ "pkg_name": self.repo,
|
|||
|
|
+ "related_release": source["labels"][0]['name'] if source["labels"] else ''}
|
|||
|
|
+ if source["issue_type"] == "缺陷":
|
|||
|
|
+ self.defect += 1
|
|||
|
|
+ elif source["issue_type"] == "需求":
|
|||
|
|
+ self.feature += 1
|
|||
|
|
+ elif source["issue_type"] == "CVE和安全问题":
|
|||
|
|
+ self.cve += 1
|
|||
|
|
+ else:
|
|||
|
|
+ pass
|
|||
|
|
+ except KeyError as error:
|
|||
|
|
+ LOGGER.logger.error(error)
|
|||
|
|
+ return None
|
|||
|
|
+ return result_dict
|
|||
|
|
+
|
|||
|
|
+ def issue_hooks(self, issue_hook_info):
|
|||
|
|
+ """
|
|||
|
|
+ Description: Hook data triggered by a new task operation
|
|||
|
|
+ Args:
|
|||
|
|
+ issue_hook_info: Issue info
|
|||
|
|
+ Returns:
|
|||
|
|
+
|
|||
|
|
+ Raises:
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ if issue_hook_info is None:
|
|||
|
|
+ raise ContentNoneException(
|
|||
|
|
+ 'The content cannot be empty')
|
|||
|
|
+ issue_info_list = []
|
|||
|
|
+ issue_info = issue_hook_info["issue"]
|
|||
|
|
+ issue_content = self.parse_issue_content(issue_info)
|
|||
|
|
+ if issue_content:
|
|||
|
|
+ issue_info_list.append(issue_content)
|
|||
|
|
+ if self.feature != 0:
|
|||
|
|
+ self.defect, self.feature, self.cve = self.pkg_info.defect, self.pkg_info.feature + \
|
|||
|
|
+ 1, self.pkg_info.cve
|
|||
|
|
+ if self.defect != 0:
|
|||
|
|
+ self.defect, self.feature, self.cve = self.pkg_info.defect + \
|
|||
|
|
+ 1, self.pkg_info.feature, self.pkg_info.cve
|
|||
|
|
+ if self.cve != 0:
|
|||
|
|
+ self.defect, self.feature, self.cve = self.pkg_info.defect, self.pkg_info.feature, self.pkg_info.cve + 1
|
|||
|
|
+ self._save_issues(issue_info_list)
|
|||
|
|
diff -Naru a/packageship/application/apps/lifecycle/view.py b/packageship/application/apps/lifecycle/view.py
|
|||
|
|
--- a/packageship/application/apps/lifecycle/view.py 2020-09-22 23:34:04.037937224 +0800
|
|||
|
|
+++ b/packageship/application/apps/lifecycle/view.py 2020-09-22 23:52:49.731821183 +0800
|
|||
|
|
@@ -1,760 +1,760 @@
|
|||
|
|
-#!/usr/bin/python3
|
|||
|
|
-"""
|
|||
|
|
-Life cycle related api interface
|
|||
|
|
-"""
|
|||
|
|
-import io
|
|||
|
|
-import json
|
|||
|
|
-import math
|
|||
|
|
-import os
|
|||
|
|
-from concurrent.futures import ThreadPoolExecutor
|
|||
|
|
-
|
|||
|
|
-import pandas as pd
|
|||
|
|
-import yaml
|
|||
|
|
-
|
|||
|
|
-from flask import request
|
|||
|
|
-from flask import jsonify, make_response
|
|||
|
|
-from flask import current_app
|
|||
|
|
-from flask_restful import Resource
|
|||
|
|
-from marshmallow import ValidationError
|
|||
|
|
-
|
|||
|
|
-from sqlalchemy.exc import DisconnectionError, SQLAlchemyError
|
|||
|
|
-
|
|||
|
|
-from packageship import system_config
|
|||
|
|
-from packageship.libs.configutils.readconfig import ReadConfig
|
|||
|
|
-from packageship.libs.exception import Error
|
|||
|
|
-from packageship.application.apps.package.function.constants import ResponseCode
|
|||
|
|
-from packageship.libs.dbutils.sqlalchemy_helper import DBHelper
|
|||
|
|
-from packageship.application.models.package import PackagesIssue
|
|||
|
|
-from packageship.application.models.package import Packages
|
|||
|
|
-from packageship.application.models.package import PackagesMaintainer
|
|||
|
|
-from packageship.libs.log import Log
|
|||
|
|
-from .serialize import IssueDownloadSchema, PackagesDownloadSchema, IssuePageSchema, IssueSchema
|
|||
|
|
-from ..package.serialize import DataFormatVerfi, UpdatePackagesSchema
|
|||
|
|
-from .function.gitee import Gitee as gitee
|
|||
|
|
-
|
|||
|
|
-LOGGER = Log(__name__)
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-# pylint: disable = no-self-use
|
|||
|
|
-
|
|||
|
|
-class DownloadFile(Resource):
|
|||
|
|
- """
|
|||
|
|
- Download the content of the issue or the excel file of the package content
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def _download_excel(self, file_type, table_name=None):
|
|||
|
|
- """
|
|||
|
|
- Download excel file
|
|||
|
|
- """
|
|||
|
|
- file_name = 'packages.xlsx'
|
|||
|
|
- if file_type == 'packages':
|
|||
|
|
- download_content = self.__get_packages_content(table_name)
|
|||
|
|
- else:
|
|||
|
|
- file_name = 'issues.xlsx'
|
|||
|
|
- download_content = self.__get_issues_content()
|
|||
|
|
- if download_content is None:
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SERVICE_ERROR))
|
|||
|
|
- pd_dataframe = self.__to_dataframe(download_content)
|
|||
|
|
-
|
|||
|
|
- _response = self.__bytes_save(pd_dataframe)
|
|||
|
|
- return self.__set_response_header(_response, file_name)
|
|||
|
|
-
|
|||
|
|
- def __bytes_save(self, data_frame):
|
|||
|
|
- """
|
|||
|
|
- Save the file content in the form of a binary file stream
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- bytes_io = io.BytesIO()
|
|||
|
|
- writer = pd.ExcelWriter( # pylint: disable=abstract-class-instantiated
|
|||
|
|
- bytes_io, engine='xlsxwriter')
|
|||
|
|
- data_frame.to_excel(writer, sheet_name='Summary', index=False)
|
|||
|
|
- writer.save()
|
|||
|
|
- writer.close()
|
|||
|
|
- bytes_io.seek(0)
|
|||
|
|
- _response = make_response(bytes_io.getvalue())
|
|||
|
|
- bytes_io.close()
|
|||
|
|
- return _response
|
|||
|
|
- except (IOError, Error) as io_error:
|
|||
|
|
- current_app.logger.error(io_error)
|
|||
|
|
- return make_response()
|
|||
|
|
-
|
|||
|
|
- def __set_response_header(self, response, file_name):
|
|||
|
|
- """
|
|||
|
|
- Set http response header information
|
|||
|
|
- """
|
|||
|
|
- response.headers['Content-Type'] = \
|
|||
|
|
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|||
|
|
- response.headers["Cache-Control"] = "no-cache"
|
|||
|
|
- response.headers['Content-Disposition'] = 'attachment; filename={file_name}'.format(
|
|||
|
|
- file_name=file_name)
|
|||
|
|
- return response
|
|||
|
|
-
|
|||
|
|
- def __get_packages_content(self, table_name):
|
|||
|
|
- """
|
|||
|
|
- Get package list information
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
- # Query all package data in the specified table
|
|||
|
|
- _model = Packages.package_meta(table_name)
|
|||
|
|
- _packageinfos = database.session.query(_model).all()
|
|||
|
|
- packages_dicts = PackagesDownloadSchema(
|
|||
|
|
- many=True).dump(_packageinfos)
|
|||
|
|
- return packages_dicts
|
|||
|
|
-
|
|||
|
|
- except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return None
|
|||
|
|
-
|
|||
|
|
- def __get_issues_content(self):
|
|||
|
|
- """
|
|||
|
|
- Get the list of issues
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
- _issues = database.session.query(PackagesIssue).all()
|
|||
|
|
- issues_dicts = IssueDownloadSchema(many=True).dump(_issues)
|
|||
|
|
- return issues_dicts
|
|||
|
|
- except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return None
|
|||
|
|
-
|
|||
|
|
- def __to_dataframe(self, datas):
|
|||
|
|
- """
|
|||
|
|
- Convert the obtained information into pandas content format
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- data_frame = pd.DataFrame(datas)
|
|||
|
|
- return data_frame
|
|||
|
|
-
|
|||
|
|
- def get(self, file_type):
|
|||
|
|
- """
|
|||
|
|
- Download package collection information and isse list information
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- if file_type not in ['packages', 'issues']:
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(
|
|||
|
|
- ResponseCode.PARAM_ERROR))
|
|||
|
|
-
|
|||
|
|
- table_name = request.args.get('table_name', None)
|
|||
|
|
- response = self._download_excel(file_type, table_name)
|
|||
|
|
- return response
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class MaintainerView(Resource):
|
|||
|
|
- """
|
|||
|
|
- Maintainer name collection
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def __query_maintainers(self):
|
|||
|
|
- """
|
|||
|
|
- Query the names of all maintainers in the specified table
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
- maintainers = database.session.query(
|
|||
|
|
- PackagesMaintainer.maintainer).group_by(PackagesMaintainer.maintainer).all()
|
|||
|
|
- return [maintainer_item[0] for maintainer_item in maintainers
|
|||
|
|
- if maintainer_item[0]]
|
|||
|
|
- except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return []
|
|||
|
|
-
|
|||
|
|
- def get(self):
|
|||
|
|
- """
|
|||
|
|
- Get the list of maintainers
|
|||
|
|
- """
|
|||
|
|
- # Group query of the names of all maintainers in the current table
|
|||
|
|
- maintainers = self.__query_maintainers()
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SUCCESS,
|
|||
|
|
- maintainers))
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class TableColView(Resource):
|
|||
|
|
- """
|
|||
|
|
- The default column of the package shows the interface
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def __columns_names(self):
|
|||
|
|
- """
|
|||
|
|
- Mapping of column name and title
|
|||
|
|
- """
|
|||
|
|
- columns = [
|
|||
|
|
- ('name', 'Name', True),
|
|||
|
|
- ('version', 'Version', True),
|
|||
|
|
- ('release', 'Release', True),
|
|||
|
|
- ('url', 'Url', True),
|
|||
|
|
- ('rpm_license', 'License', False),
|
|||
|
|
- ('feature', 'Feature', False),
|
|||
|
|
- ('maintainer', 'Maintainer', True),
|
|||
|
|
- ('maintainlevel', 'Maintenance Level', True),
|
|||
|
|
- ('release_time', 'Release Time', False),
|
|||
|
|
- ('used_time', 'Used Time', True),
|
|||
|
|
- ('maintainer_status', 'Maintain Status', True),
|
|||
|
|
- ('latest_version', 'Latest Version', False),
|
|||
|
|
- ('latest_version_time', 'Latest Version Release Time', False),
|
|||
|
|
- ('issue', 'Issue', True)]
|
|||
|
|
- return columns
|
|||
|
|
-
|
|||
|
|
- def __columns_mapping(self):
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- columns = list()
|
|||
|
|
- for column in self.__columns_names():
|
|||
|
|
- columns.append({
|
|||
|
|
- 'column_name': column[0],
|
|||
|
|
- 'label': column[1],
|
|||
|
|
- 'default_selected': column[2]
|
|||
|
|
- })
|
|||
|
|
- return columns
|
|||
|
|
-
|
|||
|
|
- def get(self):
|
|||
|
|
- """
|
|||
|
|
- Get the default display column of the package
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- table_mapping_columns = self.__columns_mapping()
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SUCCESS,
|
|||
|
|
- table_mapping_columns))
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class LifeTables(Resource):
|
|||
|
|
- """
|
|||
|
|
- description: LifeTables
|
|||
|
|
- Restful API: get
|
|||
|
|
- ChangeLog:
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def get(self):
|
|||
|
|
- """
|
|||
|
|
- return all table names in the database
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
- Return the table names in the database as a list
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name="lifecycle") as database_name:
|
|||
|
|
- # View all table names in the package-info database
|
|||
|
|
- all_table_names = database_name.engine.table_names()
|
|||
|
|
- all_table_names.remove("packages_issue")
|
|||
|
|
- all_table_names.remove("packages_maintainer")
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SUCCESS, data=all_table_names)
|
|||
|
|
- )
|
|||
|
|
- except (SQLAlchemyError, DisconnectionError, Error, ValueError) as sql_error:
|
|||
|
|
- LOGGER.logger.error(sql_error)
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(ResponseCode.DATABASE_NOT_FOUND)
|
|||
|
|
- )
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class IssueView(Resource):
|
|||
|
|
- """
|
|||
|
|
- Issue content collection
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def _query_issues(self, request_data):
|
|||
|
|
- """
|
|||
|
|
- Args:
|
|||
|
|
- request_data:
|
|||
|
|
- Returns:
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
- issues_query = database.session.query(PackagesIssue.issue_id,
|
|||
|
|
- PackagesIssue.issue_url,
|
|||
|
|
- PackagesIssue.issue_title,
|
|||
|
|
- PackagesIssue.issue_status,
|
|||
|
|
- PackagesIssue.pkg_name,
|
|||
|
|
- PackagesIssue.issue_type,
|
|||
|
|
- PackagesMaintainer.maintainer). \
|
|||
|
|
- outerjoin(PackagesMaintainer,
|
|||
|
|
- PackagesMaintainer.name == PackagesIssue.pkg_name)
|
|||
|
|
- if request_data.get("pkg_name"):
|
|||
|
|
- issues_query = issues_query.filter(
|
|||
|
|
- PackagesIssue.pkg_name == request_data.get("pkg_name"))
|
|||
|
|
- if request_data.get("issue_type"):
|
|||
|
|
- issues_query = issues_query.filter(
|
|||
|
|
- PackagesIssue.issue_type == request_data.get("issue_type"))
|
|||
|
|
- if request_data.get("issue_status"):
|
|||
|
|
- issues_query = issues_query.filter(
|
|||
|
|
- PackagesIssue.issue_status == request_data.get("issue_status"))
|
|||
|
|
- if request_data.get("maintainer"):
|
|||
|
|
- issues_query = issues_query.filter(
|
|||
|
|
- PackagesMaintainer.maintainer == request_data.get("maintainer"))
|
|||
|
|
- total_count = issues_query.count()
|
|||
|
|
- total_page = math.ceil(
|
|||
|
|
- total_count / int(request_data.get("page_size")))
|
|||
|
|
- issues_query = issues_query.limit(request_data.get("page_size")).offset(
|
|||
|
|
- (int(request_data.get("page_num")) - 1) * int(request_data.get("page_size")))
|
|||
|
|
- issue_dicts = IssuePageSchema(
|
|||
|
|
- many=True).dump(issues_query.all())
|
|||
|
|
- issue_data = ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SUCCESS, issue_dicts)
|
|||
|
|
- issue_data['total_count'] = total_count
|
|||
|
|
- issue_data['total_page'] = total_page
|
|||
|
|
- return issue_data
|
|||
|
|
- except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return ResponseCode.response_json(ResponseCode.DATABASE_NOT_FOUND)
|
|||
|
|
-
|
|||
|
|
- def get(self):
|
|||
|
|
- """
|
|||
|
|
- Description: Get all issues info or one specific issue
|
|||
|
|
- Args:
|
|||
|
|
- Returns:
|
|||
|
|
- [
|
|||
|
|
- {
|
|||
|
|
- "issue_id": "",
|
|||
|
|
- "issue_url": "",
|
|||
|
|
- "issue_title": "",
|
|||
|
|
- "issue_content": "",
|
|||
|
|
- "issue_status": "",
|
|||
|
|
- "issue_type": ""
|
|||
|
|
- },
|
|||
|
|
- ]
|
|||
|
|
- Raises:
|
|||
|
|
- DisconnectionError: Unable to connect to database exception
|
|||
|
|
- AttributeError: Object does not have this property
|
|||
|
|
- TypeError: Exception of type
|
|||
|
|
- Error: Abnormal error
|
|||
|
|
- """
|
|||
|
|
- schema = IssueSchema()
|
|||
|
|
- if schema.validate(request.args):
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(ResponseCode.PARAM_ERROR)
|
|||
|
|
- )
|
|||
|
|
- issue_dict = self._query_issues(request.args)
|
|||
|
|
- return issue_dict
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class IssueType(Resource):
|
|||
|
|
- """
|
|||
|
|
- Issue type collection
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def _get_issue_type(self):
|
|||
|
|
- """
|
|||
|
|
- Description: Query issue type
|
|||
|
|
- Returns:
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
- issues_query = database.session.query(PackagesIssue.issue_type).group_by(
|
|||
|
|
- PackagesIssue.issue_type).all()
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SUCCESS, [issue_query[0] for issue_query in issues_query]))
|
|||
|
|
- except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.PARAM_ERROR))
|
|||
|
|
-
|
|||
|
|
- def get(self):
|
|||
|
|
- """
|
|||
|
|
- Description: Get all issues info or one specific issue
|
|||
|
|
- Args:
|
|||
|
|
- Returns:
|
|||
|
|
- [
|
|||
|
|
- "issue_type",
|
|||
|
|
- "issue_type"
|
|||
|
|
- ]
|
|||
|
|
- Raises:
|
|||
|
|
- DisconnectionError: Unable to connect to database exception
|
|||
|
|
- AttributeError: Object does not have this property
|
|||
|
|
- TypeError: Exception of type
|
|||
|
|
- Error: Abnormal error
|
|||
|
|
- """
|
|||
|
|
- return self._get_issue_type()
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class IssueStatus(Resource):
|
|||
|
|
- """
|
|||
|
|
- Issue status collection
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def _get_issue_status(self):
|
|||
|
|
- """
|
|||
|
|
- Description: Query issue status
|
|||
|
|
- Returns:
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
- issues_query = database.session.query(PackagesIssue.issue_status).group_by(
|
|||
|
|
- PackagesIssue.issue_status).all()
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SUCCESS, [issue_query[0] for issue_query in issues_query]))
|
|||
|
|
- except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.PARAM_ERROR))
|
|||
|
|
-
|
|||
|
|
- def get(self):
|
|||
|
|
- """
|
|||
|
|
- Description: Get all issues info or one specific issue
|
|||
|
|
- Args:
|
|||
|
|
- Returns:
|
|||
|
|
- [
|
|||
|
|
- "issue_status",
|
|||
|
|
- "issue_status"
|
|||
|
|
- ]
|
|||
|
|
- Raises:
|
|||
|
|
- DisconnectionError: Unable to connect to database exception
|
|||
|
|
- AttributeError: Object does not have this property
|
|||
|
|
- TypeError: Exception of type
|
|||
|
|
- Error: Abnormal error
|
|||
|
|
- """
|
|||
|
|
- return self._get_issue_status()
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class IssueCatch(Resource):
|
|||
|
|
- """
|
|||
|
|
- description: Catch issue content
|
|||
|
|
- Restful API: put
|
|||
|
|
- ChangeLog:
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def post(self):
|
|||
|
|
- """
|
|||
|
|
- Searching issue content
|
|||
|
|
- Args:
|
|||
|
|
- Returns:
|
|||
|
|
- for examples:
|
|||
|
|
- [
|
|||
|
|
- {
|
|||
|
|
- "issue_id": "",
|
|||
|
|
- "issue_url": "",
|
|||
|
|
- "issue_title": "",
|
|||
|
|
- "issue_content": "",
|
|||
|
|
- "issue_status": "",
|
|||
|
|
- "issue_type": ""
|
|||
|
|
- },
|
|||
|
|
- ]
|
|||
|
|
- Raises:
|
|||
|
|
- DisconnectionError: Unable to connect to database exception
|
|||
|
|
- AttributeError: Object does not have this property
|
|||
|
|
- TypeError: Exception of type
|
|||
|
|
- Error: Abnormal error
|
|||
|
|
- """
|
|||
|
|
- data = json.loads(request.get_data())
|
|||
|
|
- if not isinstance(data, dict):
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(ResponseCode.PARAM_ERROR))
|
|||
|
|
- pkg_name = data["repository"]["path"]
|
|||
|
|
- try:
|
|||
|
|
- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
- pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers')
|
|||
|
|
- _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse')
|
|||
|
|
- if _warehouse is None:
|
|||
|
|
- _warehouse = 'src-openeuler'
|
|||
|
|
- if not isinstance(pool_workers, int):
|
|||
|
|
- pool_workers = 10
|
|||
|
|
- pool = ThreadPoolExecutor(max_workers=pool_workers)
|
|||
|
|
- with DBHelper(db_name="lifecycle") as database:
|
|||
|
|
- for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer'],
|
|||
|
|
- database.engine.table_names()):
|
|||
|
|
- cls_model = Packages.package_meta(table_name)
|
|||
|
|
- for package_item in database.session.query(cls_model).filter(
|
|||
|
|
- cls_model.name == pkg_name).all():
|
|||
|
|
- gitee_issue = gitee(
|
|||
|
|
- package_item, _warehouse, package_item.name, table_name)
|
|||
|
|
- pool.submit(gitee_issue.issue_hooks, data)
|
|||
|
|
- pool.shutdown()
|
|||
|
|
- return jsonify(ResponseCode.response_json(ResponseCode.SUCCESS))
|
|||
|
|
- except SQLAlchemyError as error_msg:
|
|||
|
|
- current_app.logger.error(error_msg)
|
|||
|
|
-
|
|||
|
|
-
|
|||
|
|
-class UpdatePackages(Resource):
|
|||
|
|
- """
|
|||
|
|
- description:Life cycle update information of a single package
|
|||
|
|
- Restful API: post
|
|||
|
|
- ChangeLog:
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
- def _get_all_yaml_name(self, filepath):
|
|||
|
|
- """
|
|||
|
|
- List of all yaml file names in the folder
|
|||
|
|
-
|
|||
|
|
- Args:
|
|||
|
|
- filepath: file path
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
- yaml_file_list:List of all yaml file names in the folder
|
|||
|
|
-
|
|||
|
|
- Attributes:
|
|||
|
|
- Error:Error
|
|||
|
|
- NotADirectoryError:Invalid directory name
|
|||
|
|
- FileNotFoundError:File not found error
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- yaml_file_list = os.listdir(filepath)
|
|||
|
|
- return yaml_file_list
|
|||
|
|
- except (Error, NotADirectoryError, FileNotFoundError) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return None
|
|||
|
|
-
|
|||
|
|
- def _get_yaml_content(self, yaml_file, filepath):
|
|||
|
|
- """
|
|||
|
|
- Read the content of the yaml file
|
|||
|
|
-
|
|||
|
|
- Args:
|
|||
|
|
- yaml_file: yaml file
|
|||
|
|
- filepath: file path
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
- Return a dictionary containing name, maintainer and maintainlevel
|
|||
|
|
- """
|
|||
|
|
- yaml_data_dict = dict()
|
|||
|
|
- if not yaml_file.endswith(".yaml"):
|
|||
|
|
- return None
|
|||
|
|
- pkg_name = yaml_file.rsplit('.yaml')[0]
|
|||
|
|
- single_yaml_path = os.path.join(filepath, yaml_file)
|
|||
|
|
- with open(single_yaml_path, 'r', encoding='utf-8') as file_context:
|
|||
|
|
- yaml_flie_data = yaml.load(
|
|||
|
|
- file_context.read(), Loader=yaml.FullLoader)
|
|||
|
|
- if yaml_flie_data is None or not isinstance(yaml_flie_data, dict):
|
|||
|
|
- return None
|
|||
|
|
- maintainer = yaml_flie_data.get("maintainer")
|
|||
|
|
- maintainlevel = yaml_flie_data.get("maintainlevel")
|
|||
|
|
- yaml_data_dict['name'] = pkg_name
|
|||
|
|
- if maintainer:
|
|||
|
|
- yaml_data_dict['maintainer'] = maintainer
|
|||
|
|
- if maintainlevel:
|
|||
|
|
- yaml_data_dict['maintainlevel'] = maintainlevel
|
|||
|
|
- return yaml_data_dict
|
|||
|
|
-
|
|||
|
|
- def _read_yaml_file(self, filepath):
|
|||
|
|
- """
|
|||
|
|
- Read the yaml file and combine the data of the nested dictionary of the list
|
|||
|
|
-
|
|||
|
|
- Args:
|
|||
|
|
- filepath: file path
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
- yaml.YAMLError:yaml file error
|
|||
|
|
- SQLAlchemyError:SQLAlchemy Error
|
|||
|
|
- DisconnectionError:Connect to database error
|
|||
|
|
- Error:Error
|
|||
|
|
- """
|
|||
|
|
- yaml_file_list = self._get_all_yaml_name(filepath)
|
|||
|
|
- if not yaml_file_list:
|
|||
|
|
- return None
|
|||
|
|
- try:
|
|||
|
|
- yaml_data_list = list()
|
|||
|
|
- _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
- pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers')
|
|||
|
|
- if not isinstance(pool_workers, int):
|
|||
|
|
- pool_workers = 10
|
|||
|
|
- with ThreadPoolExecutor(max_workers=pool_workers) as pool:
|
|||
|
|
- for yaml_file in yaml_file_list:
|
|||
|
|
- pool_result = pool.submit(
|
|||
|
|
- self._get_yaml_content, yaml_file, filepath)
|
|||
|
|
- yaml_data_dict = pool_result.result()
|
|||
|
|
- yaml_data_list.append(yaml_data_dict)
|
|||
|
|
- return yaml_data_list
|
|||
|
|
- except (yaml.YAMLError, SQLAlchemyError, DisconnectionError, Error) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return None
|
|||
|
|
-
|
|||
|
|
- def _verification_yaml_data_list(self, yaml_data_list):
|
|||
|
|
- """
|
|||
|
|
- Verify the data obtained in the yaml file
|
|||
|
|
-
|
|||
|
|
- Args:
|
|||
|
|
- yaml_data_list: yaml data list
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
- yaml_data_list: After verification yaml data list
|
|||
|
|
-
|
|||
|
|
- Attributes:
|
|||
|
|
- ValidationError: Validation error
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- DataFormatVerfi(many=True).load(yaml_data_list)
|
|||
|
|
- return yaml_data_list
|
|||
|
|
- except ValidationError as error:
|
|||
|
|
- current_app.logger.error(error.messages)
|
|||
|
|
- return None
|
|||
|
|
-
|
|||
|
|
- def _save_in_database(self, yaml_data_list):
|
|||
|
|
- """
|
|||
|
|
- Save the data to the database
|
|||
|
|
-
|
|||
|
|
- Args:
|
|||
|
|
- tbname: Table Name
|
|||
|
|
- name_separate_list: Split name list
|
|||
|
|
- _update_pack_data: Split new list of combined data
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
- SUCCESS or UPDATA_DATA_FAILED
|
|||
|
|
-
|
|||
|
|
- Attributes
|
|||
|
|
- DisconnectionError: Connect to database error
|
|||
|
|
- SQLAlchemyError: SQLAlchemy Error
|
|||
|
|
- Error: Error
|
|||
|
|
-
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name="lifecycle") as database_name:
|
|||
|
|
- if 'packages_maintainer' not in database_name.engine.table_names():
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.TABLE_NAME_NOT_EXIST))
|
|||
|
|
- database_name.session.begin(subtransactions=True)
|
|||
|
|
- for yaml_data in yaml_data_list:
|
|||
|
|
- name = yaml_data.get("name")
|
|||
|
|
- maintainer = yaml_data.get("maintainer")
|
|||
|
|
- maintainlevel = yaml_data.get("maintainlevel")
|
|||
|
|
- packages_maintainer_obj = database_name.session.query(
|
|||
|
|
- PackagesMaintainer).filter_by(name=name).first()
|
|||
|
|
- if packages_maintainer_obj:
|
|||
|
|
- if maintainer:
|
|||
|
|
- packages_maintainer_obj.maintainer = maintainer
|
|||
|
|
- if maintainlevel:
|
|||
|
|
- packages_maintainer_obj.maintainlevel = maintainlevel
|
|||
|
|
- else:
|
|||
|
|
- database_name.add(PackagesMaintainer(
|
|||
|
|
- name=name, maintainer=maintainer, maintainlevel=maintainlevel
|
|||
|
|
- ))
|
|||
|
|
- database_name.session.commit()
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SUCCESS))
|
|||
|
|
- except (DisconnectionError, SQLAlchemyError, Error, AttributeError) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.UPDATA_DATA_FAILED))
|
|||
|
|
-
|
|||
|
|
- def _overall_process(
|
|||
|
|
- self,
|
|||
|
|
- filepath):
|
|||
|
|
- """
|
|||
|
|
- Call each method to complete the entire function
|
|||
|
|
-
|
|||
|
|
- Args:
|
|||
|
|
- filepath: file path
|
|||
|
|
- tbname: table name
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
- SUCCESS or UPDATA_DATA_FAILED
|
|||
|
|
-
|
|||
|
|
- Attributes
|
|||
|
|
- DisconnectionError: Connect to database error
|
|||
|
|
- SQLAlchemyError: SQLAlchemy Error
|
|||
|
|
- Error: Error
|
|||
|
|
- """
|
|||
|
|
- try:
|
|||
|
|
- if filepath is None or not os.path.exists(filepath):
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SPECIFIED_FILE_NOT_EXIST))
|
|||
|
|
- yaml_file_list = self._get_all_yaml_name(filepath)
|
|||
|
|
- if not yaml_file_list:
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.EMPTY_FOLDER))
|
|||
|
|
- yaml_data_list_result = self._read_yaml_file(filepath)
|
|||
|
|
- yaml_data_list = self._verification_yaml_data_list(
|
|||
|
|
- yaml_data_list_result)
|
|||
|
|
- if yaml_data_list is None:
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.YAML_FILE_ERROR))
|
|||
|
|
- result = self._save_in_database(
|
|||
|
|
- yaml_data_list)
|
|||
|
|
- return result
|
|||
|
|
- except (DisconnectionError, SQLAlchemyError, Error) as error:
|
|||
|
|
- current_app.logger.error(error)
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.UPDATA_DATA_FAILED))
|
|||
|
|
-
|
|||
|
|
- def _update_single_package_info(
|
|||
|
|
- self, srcname, maintainer, maintainlevel):
|
|||
|
|
- """
|
|||
|
|
- Update the maintainer field and maintainlevel
|
|||
|
|
- field of a single package
|
|||
|
|
-
|
|||
|
|
- Args:
|
|||
|
|
- srcname: The name of the source package
|
|||
|
|
- maintainer: Package maintainer
|
|||
|
|
- maintainlevel: Package maintenance level
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
- success or failed
|
|||
|
|
-
|
|||
|
|
- Attributes
|
|||
|
|
- SQLAlchemyError: sqlalchemy error
|
|||
|
|
- DisconnectionError: Cannot connect to database error
|
|||
|
|
- Error: Error
|
|||
|
|
- """
|
|||
|
|
- if not srcname:
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(ResponseCode.PACK_NAME_NOT_FOUND)
|
|||
|
|
- )
|
|||
|
|
- if not maintainer and not maintainlevel:
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(ResponseCode.PARAM_ERROR)
|
|||
|
|
- )
|
|||
|
|
- try:
|
|||
|
|
- with DBHelper(db_name='lifecycle') as database_name:
|
|||
|
|
- if 'packages_maintainer' not in database_name.engine.table_names():
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.TABLE_NAME_NOT_EXIST))
|
|||
|
|
- update_obj = database_name.session.query(
|
|||
|
|
- PackagesMaintainer).filter_by(name=srcname).first()
|
|||
|
|
- if update_obj:
|
|||
|
|
- if maintainer:
|
|||
|
|
- update_obj.maintainer = maintainer
|
|||
|
|
- if maintainlevel:
|
|||
|
|
- update_obj.maintainlevel = maintainlevel
|
|||
|
|
- else:
|
|||
|
|
- database_name.add(PackagesMaintainer(
|
|||
|
|
- name=srcname, maintainer=maintainer, maintainlevel=maintainlevel
|
|||
|
|
- ))
|
|||
|
|
- database_name.session.commit()
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(
|
|||
|
|
- ResponseCode.SUCCESS))
|
|||
|
|
- except (SQLAlchemyError, DisconnectionError, Error) as sql_error:
|
|||
|
|
- current_app.logger.error(sql_error)
|
|||
|
|
- database_name.session.rollback()
|
|||
|
|
- return jsonify(ResponseCode.response_json(
|
|||
|
|
- ResponseCode.UPDATA_DATA_FAILED
|
|||
|
|
- ))
|
|||
|
|
-
|
|||
|
|
- def put(self):
|
|||
|
|
- """
|
|||
|
|
- Life cycle update information of a single package or
|
|||
|
|
- All packages
|
|||
|
|
-
|
|||
|
|
- Returns:
|
|||
|
|
- for example::
|
|||
|
|
- {
|
|||
|
|
- "code": "",
|
|||
|
|
- "data": "",
|
|||
|
|
- "msg": ""
|
|||
|
|
- }
|
|||
|
|
- """
|
|||
|
|
- schema = UpdatePackagesSchema()
|
|||
|
|
- data = request.get_json()
|
|||
|
|
- if schema.validate(data):
|
|||
|
|
- return jsonify(
|
|||
|
|
- ResponseCode.response_json(ResponseCode.PARAM_ERROR)
|
|||
|
|
- )
|
|||
|
|
- srcname = data.get('pkg_name', None)
|
|||
|
|
- maintainer = data.get('maintainer', None)
|
|||
|
|
- maintainlevel = data.get('maintainlevel', None)
|
|||
|
|
- batch = data.get('batch')
|
|||
|
|
- filepath = data.get('filepath', None)
|
|||
|
|
-
|
|||
|
|
- if batch:
|
|||
|
|
- result = self._overall_process(filepath)
|
|||
|
|
- else:
|
|||
|
|
- result = self._update_single_package_info(
|
|||
|
|
- srcname, maintainer, maintainlevel)
|
|||
|
|
- return result
|
|||
|
|
+#!/usr/bin/python3
|
|||
|
|
+"""
|
|||
|
|
+Life cycle related api interface
|
|||
|
|
+"""
|
|||
|
|
+import io
|
|||
|
|
+import json
|
|||
|
|
+import math
|
|||
|
|
+import os
|
|||
|
|
+from concurrent.futures import ThreadPoolExecutor
|
|||
|
|
+
|
|||
|
|
+import pandas as pd
|
|||
|
|
+import yaml
|
|||
|
|
+
|
|||
|
|
+from flask import request
|
|||
|
|
+from flask import jsonify, make_response
|
|||
|
|
+from flask import current_app
|
|||
|
|
+from flask_restful import Resource
|
|||
|
|
+from marshmallow import ValidationError
|
|||
|
|
+
|
|||
|
|
+from sqlalchemy.exc import DisconnectionError, SQLAlchemyError
|
|||
|
|
+
|
|||
|
|
+from packageship import system_config
|
|||
|
|
+from packageship.libs.configutils.readconfig import ReadConfig
|
|||
|
|
+from packageship.libs.exception import Error
|
|||
|
|
+from packageship.application.apps.package.function.constants import ResponseCode
|
|||
|
|
+from packageship.libs.dbutils.sqlalchemy_helper import DBHelper
|
|||
|
|
+from packageship.application.models.package import PackagesIssue
|
|||
|
|
+from packageship.application.models.package import Packages
|
|||
|
|
+from packageship.application.models.package import PackagesMaintainer
|
|||
|
|
+from packageship.libs.log import Log
|
|||
|
|
+from .serialize import IssueDownloadSchema, PackagesDownloadSchema, IssuePageSchema, IssueSchema
|
|||
|
|
+from ..package.serialize import DataFormatVerfi, UpdatePackagesSchema
|
|||
|
|
+from .function.gitee import Gitee as gitee
|
|||
|
|
+
|
|||
|
|
+LOGGER = Log(__name__)
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+# pylint: disable = no-self-use
|
|||
|
|
+
|
|||
|
|
+class DownloadFile(Resource):
|
|||
|
|
+ """
|
|||
|
|
+ Download the content of the issue or the excel file of the package content
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def _download_excel(self, file_type, table_name=None):
|
|||
|
|
+ """
|
|||
|
|
+ Download excel file
|
|||
|
|
+ """
|
|||
|
|
+ file_name = 'packages.xlsx'
|
|||
|
|
+ if file_type == 'packages':
|
|||
|
|
+ download_content = self.__get_packages_content(table_name)
|
|||
|
|
+ else:
|
|||
|
|
+ file_name = 'issues.xlsx'
|
|||
|
|
+ download_content = self.__get_issues_content()
|
|||
|
|
+ if download_content is None:
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SERVICE_ERROR))
|
|||
|
|
+ pd_dataframe = self.__to_dataframe(download_content)
|
|||
|
|
+
|
|||
|
|
+ _response = self.__bytes_save(pd_dataframe)
|
|||
|
|
+ return self.__set_response_header(_response, file_name)
|
|||
|
|
+
|
|||
|
|
+ def __bytes_save(self, data_frame):
|
|||
|
|
+ """
|
|||
|
|
+ Save the file content in the form of a binary file stream
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ bytes_io = io.BytesIO()
|
|||
|
|
+ writer = pd.ExcelWriter( # pylint: disable=abstract-class-instantiated
|
|||
|
|
+ bytes_io, engine='xlsxwriter')
|
|||
|
|
+ data_frame.to_excel(writer, sheet_name='Summary', index=False)
|
|||
|
|
+ writer.save()
|
|||
|
|
+ writer.close()
|
|||
|
|
+ bytes_io.seek(0)
|
|||
|
|
+ _response = make_response(bytes_io.getvalue())
|
|||
|
|
+ bytes_io.close()
|
|||
|
|
+ return _response
|
|||
|
|
+ except (IOError, Error) as io_error:
|
|||
|
|
+ current_app.logger.error(io_error)
|
|||
|
|
+ return make_response()
|
|||
|
|
+
|
|||
|
|
+ def __set_response_header(self, response, file_name):
|
|||
|
|
+ """
|
|||
|
|
+ Set http response header information
|
|||
|
|
+ """
|
|||
|
|
+ response.headers['Content-Type'] = \
|
|||
|
|
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|||
|
|
+ response.headers["Cache-Control"] = "no-cache"
|
|||
|
|
+ response.headers['Content-Disposition'] = 'attachment; filename={file_name}'.format(
|
|||
|
|
+ file_name=file_name)
|
|||
|
|
+ return response
|
|||
|
|
+
|
|||
|
|
+ def __get_packages_content(self, table_name):
|
|||
|
|
+ """
|
|||
|
|
+ Get package list information
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
+ # Query all package data in the specified table
|
|||
|
|
+ _model = Packages.package_meta(table_name)
|
|||
|
|
+ _packageinfos = database.session.query(_model).all()
|
|||
|
|
+ packages_dicts = PackagesDownloadSchema(
|
|||
|
|
+ many=True).dump(_packageinfos)
|
|||
|
|
+ return packages_dicts
|
|||
|
|
+
|
|||
|
|
+ except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return None
|
|||
|
|
+
|
|||
|
|
+ def __get_issues_content(self):
|
|||
|
|
+ """
|
|||
|
|
+ Get the list of issues
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
+ _issues = database.session.query(PackagesIssue).all()
|
|||
|
|
+ issues_dicts = IssueDownloadSchema(many=True).dump(_issues)
|
|||
|
|
+ return issues_dicts
|
|||
|
|
+ except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return None
|
|||
|
|
+
|
|||
|
|
+ def __to_dataframe(self, datas):
|
|||
|
|
+ """
|
|||
|
|
+ Convert the obtained information into pandas content format
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ data_frame = pd.DataFrame(datas)
|
|||
|
|
+ return data_frame
|
|||
|
|
+
|
|||
|
|
+ def get(self, file_type):
|
|||
|
|
+ """
|
|||
|
|
+ Download package collection information and isse list information
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ if file_type not in ['packages', 'issues']:
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.PARAM_ERROR))
|
|||
|
|
+
|
|||
|
|
+ table_name = request.args.get('table_name', None)
|
|||
|
|
+ response = self._download_excel(file_type, table_name)
|
|||
|
|
+ return response
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class MaintainerView(Resource):
|
|||
|
|
+ """
|
|||
|
|
+ Maintainer name collection
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def __query_maintainers(self):
|
|||
|
|
+ """
|
|||
|
|
+ Query the names of all maintainers in the specified table
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
+ maintainers = database.session.query(
|
|||
|
|
+ PackagesMaintainer.maintainer).group_by(PackagesMaintainer.maintainer).all()
|
|||
|
|
+ return [maintainer_item[0] for maintainer_item in maintainers
|
|||
|
|
+ if maintainer_item[0]]
|
|||
|
|
+ except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return []
|
|||
|
|
+
|
|||
|
|
+ def get(self):
|
|||
|
|
+ """
|
|||
|
|
+ Get the list of maintainers
|
|||
|
|
+ """
|
|||
|
|
+ # Group query of the names of all maintainers in the current table
|
|||
|
|
+ maintainers = self.__query_maintainers()
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SUCCESS,
|
|||
|
|
+ maintainers))
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class TableColView(Resource):
|
|||
|
|
+ """
|
|||
|
|
+ The default column of the package shows the interface
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def __columns_names(self):
|
|||
|
|
+ """
|
|||
|
|
+ Mapping of column name and title
|
|||
|
|
+ """
|
|||
|
|
+ columns = [
|
|||
|
|
+ ('name', 'Name', True),
|
|||
|
|
+ ('version', 'Version', True),
|
|||
|
|
+ ('release', 'Release', True),
|
|||
|
|
+ ('url', 'Url', True),
|
|||
|
|
+ ('rpm_license', 'License', False),
|
|||
|
|
+ ('feature', 'Feature', False),
|
|||
|
|
+ ('maintainer', 'Maintainer', True),
|
|||
|
|
+ ('maintainlevel', 'Maintenance Level', True),
|
|||
|
|
+ ('release_time', 'Release Time', False),
|
|||
|
|
+ ('used_time', 'Used Time', True),
|
|||
|
|
+ ('maintainer_status', 'Maintain Status', True),
|
|||
|
|
+ ('latest_version', 'Latest Version', False),
|
|||
|
|
+ ('latest_version_time', 'Latest Version Release Time', False),
|
|||
|
|
+ ('issue', 'Issue', True)]
|
|||
|
|
+ return columns
|
|||
|
|
+
|
|||
|
|
+ def __columns_mapping(self):
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ columns = list()
|
|||
|
|
+ for column in self.__columns_names():
|
|||
|
|
+ columns.append({
|
|||
|
|
+ 'column_name': column[0],
|
|||
|
|
+ 'label': column[1],
|
|||
|
|
+ 'default_selected': column[2]
|
|||
|
|
+ })
|
|||
|
|
+ return columns
|
|||
|
|
+
|
|||
|
|
+ def get(self):
|
|||
|
|
+ """
|
|||
|
|
+ Get the default display column of the package
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ table_mapping_columns = self.__columns_mapping()
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SUCCESS,
|
|||
|
|
+ table_mapping_columns))
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class LifeTables(Resource):
|
|||
|
|
+ """
|
|||
|
|
+ description: LifeTables
|
|||
|
|
+ Restful API: get
|
|||
|
|
+ ChangeLog:
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def get(self):
|
|||
|
|
+ """
|
|||
|
|
+ return all table names in the database
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+ Return the table names in the database as a list
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ with DBHelper(db_name="lifecycle") as database_name:
|
|||
|
|
+ # View all table names in the package-info database
|
|||
|
|
+ all_table_names = database_name.engine.table_names()
|
|||
|
|
+ all_table_names.remove("packages_issue")
|
|||
|
|
+ all_table_names.remove("packages_maintainer")
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SUCCESS, data=all_table_names)
|
|||
|
|
+ )
|
|||
|
|
+ except (SQLAlchemyError, DisconnectionError, Error, ValueError) as sql_error:
|
|||
|
|
+ LOGGER.logger.error(sql_error)
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(ResponseCode.DATABASE_NOT_FOUND)
|
|||
|
|
+ )
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class IssueView(Resource):
|
|||
|
|
+ """
|
|||
|
|
+ Issue content collection
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def _query_issues(self, request_data):
|
|||
|
|
+ """
|
|||
|
|
+ Args:
|
|||
|
|
+ request_data:
|
|||
|
|
+ Returns:
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
+ issues_query = database.session.query(PackagesIssue.issue_id,
|
|||
|
|
+ PackagesIssue.issue_url,
|
|||
|
|
+ PackagesIssue.issue_title,
|
|||
|
|
+ PackagesIssue.issue_status,
|
|||
|
|
+ PackagesIssue.pkg_name,
|
|||
|
|
+ PackagesIssue.issue_type,
|
|||
|
|
+ PackagesMaintainer.maintainer). \
|
|||
|
|
+ outerjoin(PackagesMaintainer,
|
|||
|
|
+ PackagesMaintainer.name == PackagesIssue.pkg_name)
|
|||
|
|
+ if request_data.get("pkg_name"):
|
|||
|
|
+ issues_query = issues_query.filter(
|
|||
|
|
+ PackagesIssue.pkg_name == request_data.get("pkg_name"))
|
|||
|
|
+ if request_data.get("issue_type"):
|
|||
|
|
+ issues_query = issues_query.filter(
|
|||
|
|
+ PackagesIssue.issue_type == request_data.get("issue_type"))
|
|||
|
|
+ if request_data.get("issue_status"):
|
|||
|
|
+ issues_query = issues_query.filter(
|
|||
|
|
+ PackagesIssue.issue_status == request_data.get("issue_status"))
|
|||
|
|
+ if request_data.get("maintainer"):
|
|||
|
|
+ issues_query = issues_query.filter(
|
|||
|
|
+ PackagesMaintainer.maintainer == request_data.get("maintainer"))
|
|||
|
|
+ total_count = issues_query.count()
|
|||
|
|
+ total_page = math.ceil(
|
|||
|
|
+ total_count / int(request_data.get("page_size")))
|
|||
|
|
+ issues_query = issues_query.limit(request_data.get("page_size")).offset(
|
|||
|
|
+ (int(request_data.get("page_num")) - 1) * int(request_data.get("page_size")))
|
|||
|
|
+ issue_dicts = IssuePageSchema(
|
|||
|
|
+ many=True).dump(issues_query.all())
|
|||
|
|
+ issue_data = ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SUCCESS, issue_dicts)
|
|||
|
|
+ issue_data['total_count'] = total_count
|
|||
|
|
+ issue_data['total_page'] = total_page
|
|||
|
|
+ return issue_data
|
|||
|
|
+ except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return ResponseCode.response_json(ResponseCode.DATABASE_NOT_FOUND)
|
|||
|
|
+
|
|||
|
|
+ def get(self):
|
|||
|
|
+ """
|
|||
|
|
+ Description: Get all issues info or one specific issue
|
|||
|
|
+ Args:
|
|||
|
|
+ Returns:
|
|||
|
|
+ [
|
|||
|
|
+ {
|
|||
|
|
+ "issue_id": "",
|
|||
|
|
+ "issue_url": "",
|
|||
|
|
+ "issue_title": "",
|
|||
|
|
+ "issue_content": "",
|
|||
|
|
+ "issue_status": "",
|
|||
|
|
+ "issue_type": ""
|
|||
|
|
+ },
|
|||
|
|
+ ]
|
|||
|
|
+ Raises:
|
|||
|
|
+ DisconnectionError: Unable to connect to database exception
|
|||
|
|
+ AttributeError: Object does not have this property
|
|||
|
|
+ TypeError: Exception of type
|
|||
|
|
+ Error: Abnormal error
|
|||
|
|
+ """
|
|||
|
|
+ schema = IssueSchema()
|
|||
|
|
+ if schema.validate(request.args):
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(ResponseCode.PARAM_ERROR)
|
|||
|
|
+ )
|
|||
|
|
+ issue_dict = self._query_issues(request.args)
|
|||
|
|
+ return issue_dict
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class IssueType(Resource):
|
|||
|
|
+ """
|
|||
|
|
+ Issue type collection
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def _get_issue_type(self):
|
|||
|
|
+ """
|
|||
|
|
+ Description: Query issue type
|
|||
|
|
+ Returns:
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
+ issues_query = database.session.query(PackagesIssue.issue_type).group_by(
|
|||
|
|
+ PackagesIssue.issue_type).all()
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SUCCESS, [issue_query[0] for issue_query in issues_query]))
|
|||
|
|
+ except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.PARAM_ERROR))
|
|||
|
|
+
|
|||
|
|
+ def get(self):
|
|||
|
|
+ """
|
|||
|
|
+ Description: Get all issues info or one specific issue
|
|||
|
|
+ Args:
|
|||
|
|
+ Returns:
|
|||
|
|
+ [
|
|||
|
|
+ "issue_type",
|
|||
|
|
+ "issue_type"
|
|||
|
|
+ ]
|
|||
|
|
+ Raises:
|
|||
|
|
+ DisconnectionError: Unable to connect to database exception
|
|||
|
|
+ AttributeError: Object does not have this property
|
|||
|
|
+ TypeError: Exception of type
|
|||
|
|
+ Error: Abnormal error
|
|||
|
|
+ """
|
|||
|
|
+ return self._get_issue_type()
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class IssueStatus(Resource):
|
|||
|
|
+ """
|
|||
|
|
+ Issue status collection
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def _get_issue_status(self):
|
|||
|
|
+ """
|
|||
|
|
+ Description: Query issue status
|
|||
|
|
+ Returns:
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ with DBHelper(db_name='lifecycle') as database:
|
|||
|
|
+ issues_query = database.session.query(PackagesIssue.issue_status).group_by(
|
|||
|
|
+ PackagesIssue.issue_status).all()
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SUCCESS, [issue_query[0] for issue_query in issues_query]))
|
|||
|
|
+ except (SQLAlchemyError, DisconnectionError) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.PARAM_ERROR))
|
|||
|
|
+
|
|||
|
|
+ def get(self):
|
|||
|
|
+ """
|
|||
|
|
+ Description: Get all issues info or one specific issue
|
|||
|
|
+ Args:
|
|||
|
|
+ Returns:
|
|||
|
|
+ [
|
|||
|
|
+ "issue_status",
|
|||
|
|
+ "issue_status"
|
|||
|
|
+ ]
|
|||
|
|
+ Raises:
|
|||
|
|
+ DisconnectionError: Unable to connect to database exception
|
|||
|
|
+ AttributeError: Object does not have this property
|
|||
|
|
+ TypeError: Exception of type
|
|||
|
|
+ Error: Abnormal error
|
|||
|
|
+ """
|
|||
|
|
+ return self._get_issue_status()
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class IssueCatch(Resource):
|
|||
|
|
+ """
|
|||
|
|
+ description: Catch issue content
|
|||
|
|
+ Restful API: put
|
|||
|
|
+ ChangeLog:
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def post(self):
|
|||
|
|
+ """
|
|||
|
|
+ Searching issue content
|
|||
|
|
+ Args:
|
|||
|
|
+ Returns:
|
|||
|
|
+ for examples:
|
|||
|
|
+ [
|
|||
|
|
+ {
|
|||
|
|
+ "issue_id": "",
|
|||
|
|
+ "issue_url": "",
|
|||
|
|
+ "issue_title": "",
|
|||
|
|
+ "issue_content": "",
|
|||
|
|
+ "issue_status": "",
|
|||
|
|
+ "issue_type": ""
|
|||
|
|
+ },
|
|||
|
|
+ ]
|
|||
|
|
+ Raises:
|
|||
|
|
+ DisconnectionError: Unable to connect to database exception
|
|||
|
|
+ AttributeError: Object does not have this property
|
|||
|
|
+ TypeError: Exception of type
|
|||
|
|
+ Error: Abnormal error
|
|||
|
|
+ """
|
|||
|
|
+ data = json.loads(request.get_data())
|
|||
|
|
+ if not isinstance(data, dict):
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(ResponseCode.PARAM_ERROR))
|
|||
|
|
+ pkg_name = data["repository"]["path"]
|
|||
|
|
+ try:
|
|||
|
|
+ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
+ pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers')
|
|||
|
|
+ _warehouse = _readconfig.get_config('LIFECYCLE', 'warehouse')
|
|||
|
|
+ if _warehouse is None:
|
|||
|
|
+ _warehouse = 'src-openeuler'
|
|||
|
|
+ if not isinstance(pool_workers, int):
|
|||
|
|
+ pool_workers = 10
|
|||
|
|
+ pool = ThreadPoolExecutor(max_workers=pool_workers)
|
|||
|
|
+ with DBHelper(db_name="lifecycle") as database:
|
|||
|
|
+ for table_name in filter(lambda x: x not in ['packages_issue', 'packages_maintainer', 'database_info'],
|
|||
|
|
+ database.engine.table_names()):
|
|||
|
|
+ cls_model = Packages.package_meta(table_name)
|
|||
|
|
+ for package_item in database.session.query(cls_model).filter(
|
|||
|
|
+ cls_model.name == pkg_name).all():
|
|||
|
|
+ gitee_issue = gitee(
|
|||
|
|
+ package_item, _warehouse, package_item.name, table_name)
|
|||
|
|
+ pool.submit(gitee_issue.issue_hooks, data)
|
|||
|
|
+ pool.shutdown()
|
|||
|
|
+ return jsonify(ResponseCode.response_json(ResponseCode.SUCCESS))
|
|||
|
|
+ except SQLAlchemyError as error_msg:
|
|||
|
|
+ current_app.logger.error(error_msg)
|
|||
|
|
+
|
|||
|
|
+
|
|||
|
|
+class UpdatePackages(Resource):
|
|||
|
|
+ """
|
|||
|
|
+ description:Life cycle update information of a single package
|
|||
|
|
+ Restful API: post
|
|||
|
|
+ ChangeLog:
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ def _get_all_yaml_name(self, filepath):
|
|||
|
|
+ """
|
|||
|
|
+ List of all yaml file names in the folder
|
|||
|
|
+
|
|||
|
|
+ Args:
|
|||
|
|
+ filepath: file path
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+ yaml_file_list:List of all yaml file names in the folder
|
|||
|
|
+
|
|||
|
|
+ Attributes:
|
|||
|
|
+ Error:Error
|
|||
|
|
+ NotADirectoryError:Invalid directory name
|
|||
|
|
+ FileNotFoundError:File not found error
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ yaml_file_list = os.listdir(filepath)
|
|||
|
|
+ return yaml_file_list
|
|||
|
|
+ except (Error, NotADirectoryError, FileNotFoundError) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return None
|
|||
|
|
+
|
|||
|
|
+ def _get_yaml_content(self, yaml_file, filepath):
|
|||
|
|
+ """
|
|||
|
|
+ Read the content of the yaml file
|
|||
|
|
+
|
|||
|
|
+ Args:
|
|||
|
|
+ yaml_file: yaml file
|
|||
|
|
+ filepath: file path
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+ Return a dictionary containing name, maintainer and maintainlevel
|
|||
|
|
+ """
|
|||
|
|
+ yaml_data_dict = dict()
|
|||
|
|
+ if not yaml_file.endswith(".yaml"):
|
|||
|
|
+ return None
|
|||
|
|
+ pkg_name = yaml_file.rsplit('.yaml')[0]
|
|||
|
|
+ single_yaml_path = os.path.join(filepath, yaml_file)
|
|||
|
|
+ with open(single_yaml_path, 'r', encoding='utf-8') as file_context:
|
|||
|
|
+ yaml_flie_data = yaml.load(
|
|||
|
|
+ file_context.read(), Loader=yaml.FullLoader)
|
|||
|
|
+ if yaml_flie_data is None or not isinstance(yaml_flie_data, dict):
|
|||
|
|
+ return None
|
|||
|
|
+ maintainer = yaml_flie_data.get("maintainer")
|
|||
|
|
+ maintainlevel = yaml_flie_data.get("maintainlevel")
|
|||
|
|
+ yaml_data_dict['name'] = pkg_name
|
|||
|
|
+ if maintainer:
|
|||
|
|
+ yaml_data_dict['maintainer'] = maintainer
|
|||
|
|
+ if maintainlevel:
|
|||
|
|
+ yaml_data_dict['maintainlevel'] = maintainlevel
|
|||
|
|
+ return yaml_data_dict
|
|||
|
|
+
|
|||
|
|
+ def _read_yaml_file(self, filepath):
|
|||
|
|
+ """
|
|||
|
|
+ Read the yaml file and combine the data of the nested dictionary of the list
|
|||
|
|
+
|
|||
|
|
+ Args:
|
|||
|
|
+ filepath: file path
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+ yaml.YAMLError:yaml file error
|
|||
|
|
+ SQLAlchemyError:SQLAlchemy Error
|
|||
|
|
+ DisconnectionError:Connect to database error
|
|||
|
|
+ Error:Error
|
|||
|
|
+ """
|
|||
|
|
+ yaml_file_list = self._get_all_yaml_name(filepath)
|
|||
|
|
+ if not yaml_file_list:
|
|||
|
|
+ return None
|
|||
|
|
+ try:
|
|||
|
|
+ yaml_data_list = list()
|
|||
|
|
+ _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH)
|
|||
|
|
+ pool_workers = _readconfig.get_config('LIFECYCLE', 'pool_workers')
|
|||
|
|
+ if not isinstance(pool_workers, int):
|
|||
|
|
+ pool_workers = 10
|
|||
|
|
+ with ThreadPoolExecutor(max_workers=pool_workers) as pool:
|
|||
|
|
+ for yaml_file in yaml_file_list:
|
|||
|
|
+ pool_result = pool.submit(
|
|||
|
|
+ self._get_yaml_content, yaml_file, filepath)
|
|||
|
|
+ yaml_data_dict = pool_result.result()
|
|||
|
|
+ yaml_data_list.append(yaml_data_dict)
|
|||
|
|
+ return yaml_data_list
|
|||
|
|
+ except (yaml.YAMLError, SQLAlchemyError, DisconnectionError, Error) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return None
|
|||
|
|
+
|
|||
|
|
+ def _verification_yaml_data_list(self, yaml_data_list):
|
|||
|
|
+ """
|
|||
|
|
+ Verify the data obtained in the yaml file
|
|||
|
|
+
|
|||
|
|
+ Args:
|
|||
|
|
+ yaml_data_list: yaml data list
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+ yaml_data_list: After verification yaml data list
|
|||
|
|
+
|
|||
|
|
+ Attributes:
|
|||
|
|
+ ValidationError: Validation error
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ DataFormatVerfi(many=True).load(yaml_data_list)
|
|||
|
|
+ return yaml_data_list
|
|||
|
|
+ except ValidationError as error:
|
|||
|
|
+ current_app.logger.error(error.messages)
|
|||
|
|
+ return None
|
|||
|
|
+
|
|||
|
|
+ def _save_in_database(self, yaml_data_list):
|
|||
|
|
+ """
|
|||
|
|
+ Save the data to the database
|
|||
|
|
+
|
|||
|
|
+ Args:
|
|||
|
|
+ tbname: Table Name
|
|||
|
|
+ name_separate_list: Split name list
|
|||
|
|
+ _update_pack_data: Split new list of combined data
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+ SUCCESS or UPDATA_DATA_FAILED
|
|||
|
|
+
|
|||
|
|
+ Attributes
|
|||
|
|
+ DisconnectionError: Connect to database error
|
|||
|
|
+ SQLAlchemyError: SQLAlchemy Error
|
|||
|
|
+ Error: Error
|
|||
|
|
+
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ with DBHelper(db_name="lifecycle") as database_name:
|
|||
|
|
+ if 'packages_maintainer' not in database_name.engine.table_names():
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.TABLE_NAME_NOT_EXIST))
|
|||
|
|
+ database_name.session.begin(subtransactions=True)
|
|||
|
|
+ for yaml_data in yaml_data_list:
|
|||
|
|
+ name = yaml_data.get("name")
|
|||
|
|
+ maintainer = yaml_data.get("maintainer")
|
|||
|
|
+ maintainlevel = yaml_data.get("maintainlevel")
|
|||
|
|
+ packages_maintainer_obj = database_name.session.query(
|
|||
|
|
+ PackagesMaintainer).filter_by(name=name).first()
|
|||
|
|
+ if packages_maintainer_obj:
|
|||
|
|
+ if maintainer:
|
|||
|
|
+ packages_maintainer_obj.maintainer = maintainer
|
|||
|
|
+ if maintainlevel:
|
|||
|
|
+ packages_maintainer_obj.maintainlevel = maintainlevel
|
|||
|
|
+ else:
|
|||
|
|
+ database_name.add(PackagesMaintainer(
|
|||
|
|
+ name=name, maintainer=maintainer, maintainlevel=maintainlevel
|
|||
|
|
+ ))
|
|||
|
|
+ database_name.session.commit()
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SUCCESS))
|
|||
|
|
+ except (DisconnectionError, SQLAlchemyError, Error, AttributeError) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.UPDATA_DATA_FAILED))
|
|||
|
|
+
|
|||
|
|
+ def _overall_process(
|
|||
|
|
+ self,
|
|||
|
|
+ filepath):
|
|||
|
|
+ """
|
|||
|
|
+ Call each method to complete the entire function
|
|||
|
|
+
|
|||
|
|
+ Args:
|
|||
|
|
+ filepath: file path
|
|||
|
|
+ tbname: table name
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+ SUCCESS or UPDATA_DATA_FAILED
|
|||
|
|
+
|
|||
|
|
+ Attributes
|
|||
|
|
+ DisconnectionError: Connect to database error
|
|||
|
|
+ SQLAlchemyError: SQLAlchemy Error
|
|||
|
|
+ Error: Error
|
|||
|
|
+ """
|
|||
|
|
+ try:
|
|||
|
|
+ if filepath is None or not os.path.exists(filepath):
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SPECIFIED_FILE_NOT_EXIST))
|
|||
|
|
+ yaml_file_list = self._get_all_yaml_name(filepath)
|
|||
|
|
+ if not yaml_file_list:
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.EMPTY_FOLDER))
|
|||
|
|
+ yaml_data_list_result = self._read_yaml_file(filepath)
|
|||
|
|
+ yaml_data_list = self._verification_yaml_data_list(
|
|||
|
|
+ yaml_data_list_result)
|
|||
|
|
+ if yaml_data_list is None:
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.YAML_FILE_ERROR))
|
|||
|
|
+ result = self._save_in_database(
|
|||
|
|
+ yaml_data_list)
|
|||
|
|
+ return result
|
|||
|
|
+ except (DisconnectionError, SQLAlchemyError, Error) as error:
|
|||
|
|
+ current_app.logger.error(error)
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.UPDATA_DATA_FAILED))
|
|||
|
|
+
|
|||
|
|
+ def _update_single_package_info(
|
|||
|
|
+ self, srcname, maintainer, maintainlevel):
|
|||
|
|
+ """
|
|||
|
|
+ Update the maintainer field and maintainlevel
|
|||
|
|
+ field of a single package
|
|||
|
|
+
|
|||
|
|
+ Args:
|
|||
|
|
+ srcname: The name of the source package
|
|||
|
|
+ maintainer: Package maintainer
|
|||
|
|
+ maintainlevel: Package maintenance level
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+ success or failed
|
|||
|
|
+
|
|||
|
|
+ Attributes
|
|||
|
|
+ SQLAlchemyError: sqlalchemy error
|
|||
|
|
+ DisconnectionError: Cannot connect to database error
|
|||
|
|
+ Error: Error
|
|||
|
|
+ """
|
|||
|
|
+ if not srcname:
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(ResponseCode.PACK_NAME_NOT_FOUND)
|
|||
|
|
+ )
|
|||
|
|
+ if not maintainer and not maintainlevel:
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(ResponseCode.PARAM_ERROR)
|
|||
|
|
+ )
|
|||
|
|
+ try:
|
|||
|
|
+ with DBHelper(db_name='lifecycle') as database_name:
|
|||
|
|
+ if 'packages_maintainer' not in database_name.engine.table_names():
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.TABLE_NAME_NOT_EXIST))
|
|||
|
|
+ update_obj = database_name.session.query(
|
|||
|
|
+ PackagesMaintainer).filter_by(name=srcname).first()
|
|||
|
|
+ if update_obj:
|
|||
|
|
+ if maintainer:
|
|||
|
|
+ update_obj.maintainer = maintainer
|
|||
|
|
+ if maintainlevel:
|
|||
|
|
+ update_obj.maintainlevel = maintainlevel
|
|||
|
|
+ else:
|
|||
|
|
+ database_name.add(PackagesMaintainer(
|
|||
|
|
+ name=srcname, maintainer=maintainer, maintainlevel=maintainlevel
|
|||
|
|
+ ))
|
|||
|
|
+ database_name.session.commit()
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.SUCCESS))
|
|||
|
|
+ except (SQLAlchemyError, DisconnectionError, Error) as sql_error:
|
|||
|
|
+ current_app.logger.error(sql_error)
|
|||
|
|
+ database_name.session.rollback()
|
|||
|
|
+ return jsonify(ResponseCode.response_json(
|
|||
|
|
+ ResponseCode.UPDATA_DATA_FAILED
|
|||
|
|
+ ))
|
|||
|
|
+
|
|||
|
|
+ def put(self):
|
|||
|
|
+ """
|
|||
|
|
+ Life cycle update information of a single package or
|
|||
|
|
+ All packages
|
|||
|
|
+
|
|||
|
|
+ Returns:
|
|||
|
|
+ for example::
|
|||
|
|
+ {
|
|||
|
|
+ "code": "",
|
|||
|
|
+ "data": "",
|
|||
|
|
+ "msg": ""
|
|||
|
|
+ }
|
|||
|
|
+ """
|
|||
|
|
+ schema = UpdatePackagesSchema()
|
|||
|
|
+ data = request.get_json()
|
|||
|
|
+ if schema.validate(data):
|
|||
|
|
+ return jsonify(
|
|||
|
|
+ ResponseCode.response_json(ResponseCode.PARAM_ERROR)
|
|||
|
|
+ )
|
|||
|
|
+ srcname = data.get('pkg_name', None)
|
|||
|
|
+ maintainer = data.get('maintainer', None)
|
|||
|
|
+ maintainlevel = data.get('maintainlevel', None)
|
|||
|
|
+ batch = data.get('batch')
|
|||
|
|
+ filepath = data.get('filepath', None)
|
|||
|
|
+
|
|||
|
|
+ if batch:
|
|||
|
|
+ result = self._overall_process(filepath)
|
|||
|
|
+ else:
|
|||
|
|
+ result = self._update_single_package_info(
|
|||
|
|
+ srcname, maintainer, maintainlevel)
|
|||
|
|
+ return result
|
|||
|
|
diff -Naru a/packageship/application/apps/package/function/be_depend.py b/packageship/application/apps/package/function/be_depend.py
|
|||
|
|
--- a/packageship/application/apps/package/function/be_depend.py 2020-09-22 23:34:04.037937224 +0800
|
|||
|
|
+++ b/packageship/application/apps/package/function/be_depend.py 2020-09-22 23:48:32.402476132 +0800
|
|||
|
|
@@ -5,11 +5,12 @@
|
|||
|
|
This includes both install and build dependencies
|
|||
|
|
Class: BeDepend
|
|||
|
|
"""
|
|||
|
|
+import copy
|
|||
|
|
+from collections import namedtuple, defaultdict
|
|||
|
|
from flask import current_app
|
|||
|
|
from sqlalchemy import text
|
|||
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|||
|
|
from sqlalchemy.sql import literal_column
|
|||
|
|
-from packageship.application.apps.package.function.constants import ResponseCode
|
|||
|
|
from packageship.application.models.package import SrcPack
|
|||
|
|
from packageship.libs.dbutils import DBHelper
|
|||
|
|
|
|||
|
|
@@ -36,6 +37,8 @@
|
|||
|
|
self.source_name_set = set()
|
|||
|
|
self.bin_name_set = set()
|
|||
|
|
self.result_dict = dict()
|
|||
|
|
+ self.comm_install_builds = defaultdict(set)
|
|||
|
|
+ self.provides_name = set()
|
|||
|
|
|
|||
|
|
def main(self):
|
|||
|
|
"""
|
|||
|
|
@@ -69,14 +72,16 @@
|
|||
|
|
[["root", None]]
|
|||
|
|
]
|
|||
|
|
self.source_name_set.add(self.source_name)
|
|||
|
|
- self.package_bedepend(
|
|||
|
|
+ self._provides_bedepend(
|
|||
|
|
[self.source_name], data_base, package_type='src')
|
|||
|
|
|
|||
|
|
+ for _, value in self.result_dict.items():
|
|||
|
|
+ value[-1] = list(value[-1])
|
|||
|
|
return self.result_dict
|
|||
|
|
|
|||
|
|
- def package_bedepend(self, pkg_name_list, data_base, package_type):
|
|||
|
|
+ def _get_provides(self, pkg_name_list, data_base, package_type):
|
|||
|
|
"""
|
|||
|
|
- Description: Query the dependent function
|
|||
|
|
+ Description: Query the components provided by the required package
|
|||
|
|
Args:
|
|||
|
|
pkg_name_list:source or binary packages name
|
|||
|
|
data_base: database
|
|||
|
|
@@ -84,35 +89,31 @@
|
|||
|
|
Returns:
|
|||
|
|
Raises:
|
|||
|
|
SQLAlchemyError: Database connection exception
|
|||
|
|
- """
|
|||
|
|
-
|
|||
|
|
+ """
|
|||
|
|
+ res = namedtuple(
|
|||
|
|
+ 'restuple', [
|
|||
|
|
+ 'search_bin_name', 'search_bin_version', 'source_name'])
|
|||
|
|
sql_com = """
|
|||
|
|
- SELECT DISTINCT b1.name AS search_bin_name,
|
|||
|
|
+ SELECT DISTINCT b1.name AS search_bin_name,
|
|||
|
|
b1.version AS search_bin_version,
|
|||
|
|
b1.src_name AS source_name,
|
|||
|
|
- b2.name AS bin_name,
|
|||
|
|
- s1.name AS bebuild_src_name,
|
|||
|
|
- b2.src_name AS install_depend_src_name
|
|||
|
|
+ bin_provides.name As pro_name
|
|||
|
|
FROM ( SELECT pkgKey,src_name,name,version FROM bin_pack WHERE {} ) b1
|
|||
|
|
- LEFT JOIN bin_provides ON bin_provides.pkgKey = b1.pkgKey
|
|||
|
|
- LEFT JOIN bin_requires br ON br.name = bin_provides.name
|
|||
|
|
- LEFT JOIN src_requires sr ON sr.name = bin_provides.name
|
|||
|
|
- LEFT JOIN src_pack s1 ON s1.pkgKey = sr.pkgKey
|
|||
|
|
- LEFT JOIN bin_pack b2 ON b2.pkgKey = br.pkgKey
|
|||
|
|
- """
|
|||
|
|
+ LEFT JOIN bin_provides ON bin_provides.pkgKey = b1.pkgKey;"""
|
|||
|
|
|
|||
|
|
+ # package_type
|
|||
|
|
if package_type == 'src':
|
|||
|
|
literal_name = 'src_name'
|
|||
|
|
-
|
|||
|
|
elif package_type == 'bin':
|
|||
|
|
literal_name = 'name'
|
|||
|
|
|
|||
|
|
- else:
|
|||
|
|
- return
|
|||
|
|
-
|
|||
|
|
+ # Query database
|
|||
|
|
+ # The lower version of SQLite can look up up to 999 parameters
|
|||
|
|
+ # simultaneously, so use 900 sharding queries
|
|||
|
|
try:
|
|||
|
|
result = []
|
|||
|
|
- for input_name in (pkg_name_list[i:i+900] for i in range(0, len(pkg_name_list), 900)):
|
|||
|
|
+ for input_name in (pkg_name_list[i:i + 900]
|
|||
|
|
+ for i in range(0, len(pkg_name_list), 900)):
|
|||
|
|
name_in = literal_column(literal_name).in_(input_name)
|
|||
|
|
sql_str = text(sql_com.format(name_in))
|
|||
|
|
result.extend(data_base.session.execute(
|
|||
|
|
@@ -124,74 +125,176 @@
|
|||
|
|
).fetchall())
|
|||
|
|
except SQLAlchemyError as sql_err:
|
|||
|
|
current_app.logger.error(sql_err)
|
|||
|
|
- return ResponseCode.response_json(ResponseCode.CONNECT_DB_ERROR)
|
|||
|
|
+ return
|
|||
|
|
|
|||
|
|
if not result:
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
- # Source and binary packages that were found to be dependent
|
|||
|
|
- source_name_list = []
|
|||
|
|
- bin_name_list = []
|
|||
|
|
+ # Process the result of the component
|
|||
|
|
+ pro_name_dict = dict()
|
|||
|
|
+
|
|||
|
|
+ _components = set()
|
|||
|
|
for obj in result:
|
|||
|
|
- if obj.source_name is None:
|
|||
|
|
- source_name = 'NOT FOUND'
|
|||
|
|
- else:
|
|||
|
|
- source_name = obj.source_name
|
|||
|
|
- if obj.bebuild_src_name:
|
|||
|
|
- # Determine if the source package has been checked
|
|||
|
|
- parent_node = obj.bebuild_src_name
|
|||
|
|
- be_type = "build"
|
|||
|
|
- # Call the spell dictionary function
|
|||
|
|
- self.make_dicts(
|
|||
|
|
- obj.search_bin_name,
|
|||
|
|
- source_name,
|
|||
|
|
+ if not obj.pro_name:
|
|||
|
|
+ continue
|
|||
|
|
+ # De-weight components
|
|||
|
|
+ if obj.pro_name not in self.comm_install_builds:
|
|||
|
|
+ pro_name_dict[obj.pro_name] = res(
|
|||
|
|
+ obj.search_bin_name, obj.search_bin_version, obj.source_name)
|
|||
|
|
+
|
|||
|
|
+ if obj.search_bin_name not in self.result_dict:
|
|||
|
|
+ self.result_dict[obj.search_bin_name] = [
|
|||
|
|
+ obj.source_name,
|
|||
|
|
obj.search_bin_version,
|
|||
|
|
- parent_node,
|
|||
|
|
- be_type)
|
|||
|
|
+ self.db_name,
|
|||
|
|
+ self.comm_install_builds[obj.pro_name]
|
|||
|
|
+ if self.comm_install_builds[obj.pro_name] else {(None, None)}
|
|||
|
|
+ ]
|
|||
|
|
+ tmp_ = copy.deepcopy(self.comm_install_builds[obj.pro_name])
|
|||
|
|
|
|||
|
|
- if obj.bebuild_src_name not in self.source_name_set:
|
|||
|
|
- self.source_name_set.add(obj.bebuild_src_name)
|
|||
|
|
- source_name_list.append(obj.bebuild_src_name)
|
|||
|
|
-
|
|||
|
|
- if obj.bin_name:
|
|||
|
|
- # Determine if the bin package has been checked
|
|||
|
|
- parent_node = obj.bin_name
|
|||
|
|
- be_type = "install"
|
|||
|
|
- # Call the spell dictionary function
|
|||
|
|
- self.make_dicts(
|
|||
|
|
- obj.search_bin_name,
|
|||
|
|
- source_name,
|
|||
|
|
- obj.search_bin_version,
|
|||
|
|
- parent_node,
|
|||
|
|
- be_type)
|
|||
|
|
+ tmp_.discard((obj.search_bin_name, 'install'))
|
|||
|
|
+ tmp_.discard((obj.search_bin_name, 'build'))
|
|||
|
|
|
|||
|
|
- if obj.bin_name not in self.bin_name_set:
|
|||
|
|
- self.bin_name_set.add(obj.bin_name)
|
|||
|
|
- bin_name_list.append(obj.bin_name)
|
|||
|
|
-
|
|||
|
|
- # With_sub_pack=1
|
|||
|
|
- if self.with_sub_pack == "1":
|
|||
|
|
- if obj.install_depend_src_name not in self.source_name_set:
|
|||
|
|
- self.source_name_set.add(
|
|||
|
|
- obj.install_depend_src_name)
|
|||
|
|
- source_name_list.append(
|
|||
|
|
- obj.install_depend_src_name)
|
|||
|
|
-
|
|||
|
|
- if obj.bebuild_src_name is None and obj.bin_name is None:
|
|||
|
|
- parent_node = None
|
|||
|
|
- be_type = None
|
|||
|
|
- self.make_dicts(
|
|||
|
|
- obj.search_bin_name,
|
|||
|
|
- source_name,
|
|||
|
|
- obj.search_bin_version,
|
|||
|
|
- parent_node,
|
|||
|
|
- be_type)
|
|||
|
|
+ if (None, None) in self.result_dict[obj.search_bin_name][-1] \
|
|||
|
|
+ and self.comm_install_builds[obj.pro_name]:
|
|||
|
|
+ self.result_dict[obj.search_bin_name][-1] = tmp_
|
|||
|
|
+ else:
|
|||
|
|
+ self.result_dict[obj.search_bin_name][-1].update(tmp_)
|
|||
|
|
+ return pro_name_dict
|
|||
|
|
+
|
|||
|
|
+ def _provides_bedepend(self, pkg_name_list, data_base, package_type):
|
|||
|
|
+ """
|
|||
|
|
+ Description: Query the dependent function
|
|||
|
|
+ Args:
|
|||
|
|
+ pkg_name_list:source or binary packages name
|
|||
|
|
+ data_base: database
|
|||
|
|
+ package_type: package type
|
|||
|
|
+ Returns:
|
|||
|
|
+ Raises:
|
|||
|
|
+ SQLAlchemyError: Database connection exception
|
|||
|
|
+ """
|
|||
|
|
+ # Query component
|
|||
|
|
+ pro_names = self._get_provides(pkg_name_list, data_base, package_type)
|
|||
|
|
|
|||
|
|
- if len(source_name_list) != 0:
|
|||
|
|
- self.package_bedepend(
|
|||
|
|
+ if not pro_names:
|
|||
|
|
+ return
|
|||
|
|
+
|
|||
|
|
+ sql_2_bin = """
|
|||
|
|
+ SELECT DISTINCT
|
|||
|
|
+ b2.name AS bin_name,
|
|||
|
|
+ b2.src_name AS install_depend_src_name,
|
|||
|
|
+ br.name AS pro_name
|
|||
|
|
+ FROM
|
|||
|
|
+ ( SELECT name, pkgKey FROM bin_requires WHERE {}) br
|
|||
|
|
+ LEFT JOIN bin_pack b2 ON b2.pkgKey = br.pkgKey;
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ sql_2_src = """
|
|||
|
|
+ SELECT DISTINCT
|
|||
|
|
+ s1.name AS bebuild_src_name,
|
|||
|
|
+ sr.name AS pro_name
|
|||
|
|
+ FROM
|
|||
|
|
+ ( SELECT name, pkgKey FROM src_requires WHERE {} ) sr
|
|||
|
|
+ LEFT JOIN src_pack s1 ON s1.pkgKey = sr.pkgKey;
|
|||
|
|
+ """
|
|||
|
|
+
|
|||
|
|
+ provides_name_list = [pro for pro, _ in pro_names.items()]
|
|||
|
|
+
|
|||
|
|
+ result_2_bin = []
|
|||
|
|
+ result_2_src = []
|
|||
|
|
+ # Query database
|
|||
|
|
+ try:
|
|||
|
|
+ for input_name in (
|
|||
|
|
+ provides_name_list[i:i + 900] for i in range(0, len(provides_name_list), 900)):
|
|||
|
|
+ name_in = literal_column('name').in_(input_name)
|
|||
|
|
+ sql_str_2_bin = text(sql_2_bin.format(name_in))
|
|||
|
|
+ result_2_bin.extend(data_base.session.execute(
|
|||
|
|
+ sql_str_2_bin,
|
|||
|
|
+ {
|
|||
|
|
+ 'name_{}'.format(i): v
|
|||
|
|
+ for i, v in enumerate(input_name, 1)
|
|||
|
|
+ }
|
|||
|
|
+ ).fetchall())
|
|||
|
|
+ sql_str_2src = text(sql_2_src.format(name_in))
|
|||
|
|
+ result_2_src.extend(data_base.session.execute(
|
|||
|
|
+ sql_str_2src,
|
|||
|
|
+ {
|
|||
|
|
+ 'name_{}'.format(i): v
|
|||
|
|
+ for i, v in enumerate(input_name, 1)
|
|||
|
|
+ }
|
|||
|
|
+ ).fetchall())
|
|||
|
|
+ except SQLAlchemyError as sql_err:
|
|||
|
|
+ current_app.logger.error(sql_err)
|
|||
|
|
+ return
|
|||
|
|
+
|
|||
|
|
+ source_name_list = []
|
|||
|
|
+ bin_name_list = []
|
|||
|
|
+
|
|||
|
|
+ # Process the data that the installation depends on
|
|||
|
|
+ for bin_info in result_2_bin:
|
|||
|
|
+ temp_bin_pkg = bin_info.bin_name
|
|||
|
|
+ temp_sub_src_pkg = bin_info.install_depend_src_name
|
|||
|
|
+
|
|||
|
|
+ #withsubpick ==1
|
|||
|
|
+ if self.with_sub_pack == '1' and temp_sub_src_pkg not in self.source_name_set:
|
|||
|
|
+ self.source_name_set.add(temp_sub_src_pkg)
|
|||
|
|
+ source_name_list.append(temp_sub_src_pkg)
|
|||
|
|
+
|
|||
|
|
+ if temp_bin_pkg not in self.bin_name_set:
|
|||
|
|
+ self.bin_name_set.add(temp_bin_pkg)
|
|||
|
|
+ bin_name_list.append(temp_bin_pkg)
|
|||
|
|
+
|
|||
|
|
+ if bin_info.pro_name not in self.comm_install_builds:
|
|||
|
|
+ self.comm_install_builds[bin_info.pro_name] = {
|
|||
|
|
+ (bin_info.bin_name, 'install')
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
+ elif (bin_info.bin_name, 'install') not in \
|
|||
|
|
+ self.comm_install_builds[bin_info.pro_name]:
|
|||
|
|
+
|
|||
|
|
+ self.comm_install_builds[bin_info.pro_name].add(
|
|||
|
|
+ (bin_info.bin_name, 'install')
|
|||
|
|
+ )
|
|||
|
|
+
|
|||
|
|
+ self.make_dicts(
|
|||
|
|
+ pro_names.get(bin_info.pro_name).search_bin_name,
|
|||
|
|
+ pro_names.get(bin_info.pro_name).source_name,
|
|||
|
|
+ pro_names.get(bin_info.pro_name).search_bin_version,
|
|||
|
|
+ bin_info.bin_name,
|
|||
|
|
+ 'install'
|
|||
|
|
+ )
|
|||
|
|
+ # Process data that is compile-dependent
|
|||
|
|
+ for src_info in result_2_src:
|
|||
|
|
+ if src_info.bebuild_src_name not in self.source_name_set:
|
|||
|
|
+ self.source_name_set.add(src_info.bebuild_src_name)
|
|||
|
|
+ source_name_list.append(src_info.bebuild_src_name)
|
|||
|
|
+
|
|||
|
|
+ if src_info.pro_name not in self.comm_install_builds:
|
|||
|
|
+ self.comm_install_builds[src_info.pro_name] = {
|
|||
|
|
+ (src_info.bebuild_src_name, 'build')
|
|||
|
|
+ }
|
|||
|
|
+ elif (src_info.bebuild_src_name, 'build') not in \
|
|||
|
|
+ self.comm_install_builds[src_info.pro_name]:
|
|||
|
|
+
|
|||
|
|
+ self.comm_install_builds[src_info.pro_name].add(
|
|||
|
|
+ (src_info.bebuild_src_name, 'build')
|
|||
|
|
+ )
|
|||
|
|
+
|
|||
|
|
+ self.make_dicts(
|
|||
|
|
+ pro_names.get(src_info.pro_name).search_bin_name,
|
|||
|
|
+ pro_names.get(src_info.pro_name).source_name,
|
|||
|
|
+ pro_names.get(src_info.pro_name).search_bin_version,
|
|||
|
|
+ src_info.bebuild_src_name,
|
|||
|
|
+ 'build'
|
|||
|
|
+ )
|
|||
|
|
+ # Recursively query all source packages that need to be looked up
|
|||
|
|
+ if source_name_list:
|
|||
|
|
+ self._provides_bedepend(
|
|||
|
|
source_name_list, data_base, package_type="src")
|
|||
|
|
- if len(bin_name_list) != 0:
|
|||
|
|
- self.package_bedepend(bin_name_list, data_base, package_type="bin")
|
|||
|
|
+ # Recursively query all binary packages that need to be looked up
|
|||
|
|
+ if bin_name_list:
|
|||
|
|
+ self._provides_bedepend(
|
|||
|
|
+ bin_name_list, data_base, package_type="bin")
|
|||
|
|
|
|||
|
|
def make_dicts(self, key, source_name, version, parent_node, be_type):
|
|||
|
|
"""
|
|||
|
|
@@ -210,29 +313,27 @@
|
|||
|
|
source_name,
|
|||
|
|
version,
|
|||
|
|
self.db_name,
|
|||
|
|
- [
|
|||
|
|
- [parent_node,
|
|||
|
|
+ {
|
|||
|
|
+ (parent_node,
|
|||
|
|
be_type
|
|||
|
|
- ]
|
|||
|
|
- ]
|
|||
|
|
+ )
|
|||
|
|
+ }
|
|||
|
|
+
|
|||
|
|
]
|
|||
|
|
else:
|
|||
|
|
if be_type and parent_node:
|
|||
|
|
- if [None, None] in self.result_dict[key][-1]:
|
|||
|
|
- self.result_dict.pop(key)
|
|||
|
|
- self.result_dict[key] = [
|
|||
|
|
- source_name,
|
|||
|
|
- version,
|
|||
|
|
- self.db_name,
|
|||
|
|
- [
|
|||
|
|
- [parent_node,
|
|||
|
|
- be_type
|
|||
|
|
- ]
|
|||
|
|
- ]
|
|||
|
|
- ]
|
|||
|
|
+ if (None, None) in self.result_dict[key][-1]:
|
|||
|
|
+ self.result_dict[key][-1] = {
|
|||
|
|
+ (
|
|||
|
|
+ parent_node,
|
|||
|
|
+ be_type
|
|||
|
|
+ )
|
|||
|
|
+ }
|
|||
|
|
|
|||
|
|
- elif [parent_node, be_type] not in self.result_dict[key][-1]:
|
|||
|
|
- self.result_dict[key][-1].append([
|
|||
|
|
- parent_node,
|
|||
|
|
- be_type
|
|||
|
|
- ])
|
|||
|
|
+ elif (parent_node, be_type) not in self.result_dict[key][-1]:
|
|||
|
|
+ self.result_dict[key][-1].add(
|
|||
|
|
+ (
|
|||
|
|
+ parent_node,
|
|||
|
|
+ be_type
|
|||
|
|
+ )
|
|||
|
|
+ )
|
|||
|
|
diff -Naru a/packageship/libs/dbutils/sqlalchemy_helper.py b/packageship/libs/dbutils/sqlalchemy_helper.py
|
|||
|
|
--- a/packageship/libs/dbutils/sqlalchemy_helper.py 2020-09-22 23:34:04.037937224 +0800
|
|||
|
|
+++ b/packageship/libs/dbutils/sqlalchemy_helper.py 2020-09-22 23:52:23.031681622 +0800
|
|||
|
|
@@ -9,6 +9,7 @@
|
|||
|
|
from sqlalchemy.orm import sessionmaker
|
|||
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|||
|
|
from sqlalchemy.exc import DisconnectionError
|
|||
|
|
+from sqlalchemy.exc import OperationalError
|
|||
|
|
from sqlalchemy.ext.declarative import declarative_base
|
|||
|
|
from sqlalchemy.engine.url import URL
|
|||
|
|
from packageship.libs.exception.ext import Error
|
|||
|
|
@@ -252,6 +253,8 @@
|
|||
|
|
|
|||
|
|
except SQLAlchemyError as sql_error:
|
|||
|
|
self.session.rollback()
|
|||
|
|
+ if isinstance(sql_error, OperationalError):
|
|||
|
|
+ raise OperationalError
|
|||
|
|
raise Error(sql_error)
|
|||
|
|
else:
|
|||
|
|
self.session.commit()
|
|||
|
|
diff -Naru a/packageship/package.ini b/packageship/package.ini
|
|||
|
|
--- a/packageship/package.ini 2020-09-22 23:34:04.037937224 +0800
|
|||
|
|
+++ b/packageship/package.ini 2020-09-22 23:49:12.154683915 +0800
|
|||
|
|
@@ -93,3 +93,5 @@
|
|||
|
|
; When this value is not set, the system will default to src-openeuler
|
|||
|
|
warehouse=src-openeuler
|
|||
|
|
|
|||
|
|
+; Maximum queue length
|
|||
|
|
+queue_maxsize = 1000
|
|||
|
|
\ No newline at end of file
|
|||
|
|
diff -Naru a/packageship/pkgshipd b/packageship/pkgshipd
|
|||
|
|
--- a/packageship/pkgshipd 2020-09-22 23:34:04.037937224 +0800
|
|||
|
|
+++ b/packageship/pkgshipd 2020-09-22 23:51:57.323547247 +0800
|
|||
|
|
@@ -1,6 +1,18 @@
|
|||
|
|
#!/bin/bash
|
|||
|
|
SYS_PATH=/etc/pkgship
|
|||
|
|
OUT_PATH=/var/run/pkgship_uwsgi
|
|||
|
|
+
|
|||
|
|
+MEM_THRESHOLD='700'
|
|||
|
|
+MEM_FREE=`free -m | grep "Mem" | awk '{print $7}'`
|
|||
|
|
+
|
|||
|
|
+if [ $1 = "start" ]
|
|||
|
|
+then
|
|||
|
|
+ if [ $MEM_FREE -lt $MEM_THRESHOLD ]; then
|
|||
|
|
+ echo "[ERROR] pkgship tool does not support memory less than ${MEM_THRESHOLD} MB."
|
|||
|
|
+ exit 0
|
|||
|
|
+ fi
|
|||
|
|
+fi
|
|||
|
|
+
|
|||
|
|
if [ ! -d "$OUT_PATH" ]; then
|
|||
|
|
mkdir $OUT_PATH
|
|||
|
|
fi
|
|||
|
|
diff -Naru a/test/common_files/package.ini b/test/common_files/package.ini
|
|||
|
|
--- a/test/common_files/package.ini 2020-09-22 23:34:04.041937245 +0800
|
|||
|
|
+++ b/test/common_files/package.ini 2020-09-22 23:50:56.559229634 +0800
|
|||
|
|
@@ -1,30 +1,31 @@
|
|||
|
|
-[SYSTEM]
|
|||
|
|
-init_conf_path = C:\Users\TAO\Desktop\pkgship-1.1.0\test\common_files\conf.yaml
|
|||
|
|
-write_port = 8080
|
|||
|
|
-query_port = 8090
|
|||
|
|
-write_ip_addr = 127.0.0.1
|
|||
|
|
-query_ip_addr = 127.0.0.1
|
|||
|
|
-remote_host = https://api.openeuler.org/pkgmanage
|
|||
|
|
-
|
|||
|
|
-[LOG]
|
|||
|
|
-log_level = INFO
|
|||
|
|
-log_name = log_info.log
|
|||
|
|
-backup_count = 10
|
|||
|
|
-max_bytes = 314572800
|
|||
|
|
-
|
|||
|
|
-[UWSGI]
|
|||
|
|
-daemonize = /var/log/uwsgi.log
|
|||
|
|
-buffer-size = 65536
|
|||
|
|
-http-timeout = 600
|
|||
|
|
-harakiri = 600
|
|||
|
|
-
|
|||
|
|
-[TIMEDTASK]
|
|||
|
|
-open = True
|
|||
|
|
-hour = 3
|
|||
|
|
-minute = 0
|
|||
|
|
-
|
|||
|
|
-[LIFECYCLE]
|
|||
|
|
-warehouse_remote = https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/
|
|||
|
|
-pool_workers = 10
|
|||
|
|
-warehouse = src-openeuler
|
|||
|
|
-
|
|||
|
|
+[SYSTEM]
|
|||
|
|
+init_conf_path =
|
|||
|
|
+write_port = 8080
|
|||
|
|
+query_port = 8090
|
|||
|
|
+write_ip_addr = 127.0.0.1
|
|||
|
|
+query_ip_addr = 127.0.0.1
|
|||
|
|
+remote_host = https://api.openeuler.org/pkgmanage
|
|||
|
|
+
|
|||
|
|
+[LOG]
|
|||
|
|
+log_level = INFO
|
|||
|
|
+log_name = log_info.log
|
|||
|
|
+backup_count = 10
|
|||
|
|
+max_bytes = 314572800
|
|||
|
|
+
|
|||
|
|
+[UWSGI]
|
|||
|
|
+daemonize = /var/log/uwsgi.log
|
|||
|
|
+buffer-size = 65536
|
|||
|
|
+http-timeout = 600
|
|||
|
|
+harakiri = 600
|
|||
|
|
+
|
|||
|
|
+[TIMEDTASK]
|
|||
|
|
+open = True
|
|||
|
|
+hour = 3
|
|||
|
|
+minute = 0
|
|||
|
|
+
|
|||
|
|
+[LIFECYCLE]
|
|||
|
|
+warehouse_remote = https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/
|
|||
|
|
+pool_workers = 10
|
|||
|
|
+warehouse = src-openeuler
|
|||
|
|
+queue_maxsize = 1000
|
|||
|
|
+
|