packages/pkgship
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: packages:491d433c34ffede18fe1af61df2583804a31c7e1
Branches Tags
packages:main
..
compare: packages:27a85b69f999115442b26cb7b63cd4be758bb5a2
Branches Tags
packages:main

No commits in common. "491d433c34ffede18fe1af61df2583804a31c7e1" and "27a85b69f999115442b26cb7b63cd4be758bb5a2" have entirely different histories.
491d433c34 ... 27a85b69f9

10 changed files with 3098 additions and 260 deletions
Show Stats Expand all files Collapse all files

21
0001-bugfix-pkginfo.patch Executable file
View File

@ -0,0 +1,21 @@
diff -Nur pkgship-2.2.0/packageship/application/core/pkginfo/pkg.py pkgship-2.2.0-new/packageship/application/core/pkginfo/pkg.py
--- pkgship-2.2.0/packageship/application/core/pkginfo/pkg.py 2021-08-14 22:23:58.000000000 +0800
+++ pkgship-2.2.0-new/packageship/application/core/pkginfo/pkg.py 2021-08-25 15:34:41.691761866 +0800
@@ -180,13 +180,15 @@
install_require_info = component_info.get("install_require")
required_by_bin = []
for req_info in install_require_info:
- required_by_bin.append(req_info.get("req_bin_name"))
+ if req_info.get("req_bin_name"):
+ required_by_bin.append(req_info.get("req_bin_name"))
# getting required_by_src
build_require_info = component_info.get("build_require")
required_by_src = []
for req_info in build_require_info:
- required_by_src.append(req_info.get("req_src_name"))
+ if req_info.get("req_src_name"):
+ required_by_src.append(req_info.get("req_src_name"))
component_dict = {"component": component_info.get("component"),
"required_by_bin": required_by_bin,

27
0002-fix-install-and-start-script-info.patch Normal file
View File

@ -0,0 +1,27 @@
diff --git a/packageship/pkgshipd b/packageship/pkgshipd
index 9cb3dc694e39f188a1e8775b37087c16059317b9..e1e843019d459ecde4aeaa6c18feb1e9d2d4fda1 100644
--- a/packageship/pkgshipd
+++ b/packageship/pkgshipd
@@ -357,7 +357,7 @@ try:
response = redis.Redis(host='$redis_ip',
port="$redis_port")
print(response.ping())
-except redis.ConnectionError:
+except Exception:
print(False)"
python3_version=$(python3 --version)
if [[ "${python3_version}" =~ "command not found" ]]; then
diff --git a/packageship/auto_install_pkgship_requires.sh b/packageship/auto_install_pkgship_requires.sh
index 4a914034170cd11e3996c34814806e8448862a32..5ecce235dae835050960046fa28a148b7326f1c4 100644
--- a/packageship/auto_install_pkgship_requires.sh
+++ b/packageship/auto_install_pkgship_requires.sh
@@ -128,7 +128,7 @@ function main() {
elif [ "${INSTALL_SOFTWARE}" = "redis" ]; then
download_install_redis
else
- echo "Failed to parse parameters, please use 'bash auto_install_es.sh elasticsearch/redis' "
+ echo "Failed to parse parameters, please use '/bin/bash auto_install_pkgship_requires.sh elasticsearch/redis' "
exit 1
fi
}

848
0003-bugfix-binary-name-mapping-source.patch Normal file
View File

@ -0,0 +1,848 @@
diff --git a/packageship/application/initialize/binary.json b/packageship/application/initialize/binary.json
index ebaa681a890a370745fc8a1282b8519fa1d3c734..a3abb5afd41085a83839ef3b2231f7efc4f9658f 100644
--- a/packageship/application/initialize/binary.json
+++ b/packageship/application/initialize/binary.json
@@ -135,18 +135,6 @@
"ignore_above": 256,
"index": false
},
- "relation": {
- "properties": {
- "bin_name": {
- "type": "keyword",
- "ignore_above": 256
- },
- "src_name": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
"release": {
"type": "keyword",
"ignore_above": 256
diff --git a/packageship/application/initialize/integration.py b/packageship/application/initialize/integration.py
index f7cd38f6258c3a0f0e05a2d2fea8317e269e2833..788b2f79edcab39a08e4d1094c48027db145c986 100644
--- a/packageship/application/initialize/integration.py
+++ b/packageship/application/initialize/integration.py
@@ -201,20 +201,21 @@ class InitializeService:
"""
sources = []
- for src_pack_name, src_pack in self._src_pack.items():
+ for _, src_pack in self._src_pack.items():
es_json = ESJson()
es_json.update(src_pack)
es_json["requires"] = self._build_requires(
src_pack['pkgKey'])
try:
- for bin_pack in self._bin_pack['sources'].pop(src_pack_name):
+ location_href = src_pack["location_href"].split('/')[-1]
+ for bin_pack in self._bin_pack['sources'].get(location_href):
_subpacks = dict(
name=bin_pack["name"], version=bin_pack["version"])
try:
es_json["subpacks"].append(_subpacks)
except TypeError:
es_json["subpacks"] = [_subpacks]
- except KeyError:
+ except TypeError:
es_json["subpacks"] = None
sources.append(self._es_json("-source", es_json))
helpers.bulk(self._session.client, sources)
@@ -227,6 +228,10 @@ class InitializeService:
binarys = []
be_depends = []
for _, bin_pack in self._bin_pack["packages"].items():
+ bin_pack["src_name"] = self._src_location[bin_pack["src_name"]]["name"]
+ if isinstance(bin_pack["src_name"], ESJson):
+ bin_pack["src_name"] = None
+
be_depends.append(self._be_depend(bin_pack))
es_json = ESJson()
es_json.update(bin_pack)
@@ -275,7 +280,9 @@ class InitializeService:
try:
for require in self._bin_requires.get(provide["name"]):
_bin_pack = self._bin_pack["pkg_key"][require["pkgKey"]]
- _src_pack = self._src_pack[_bin_pack["src_name"]]
+ _src_pack = self._src_location[_bin_pack["src_name"]]
+ if not _src_pack:
+ _src_pack = self._src_pack[_bin_pack["src_name"]]
component_json["install_require"].append({
"req_bin_name": _bin_pack["name"],
"req_bin_version": _bin_pack["version"],
@@ -373,7 +380,7 @@ class InitializeService:
return {
"_index": str(self._repo["dbname"] + index).lower(),
"_type": _type,
- "_source": source
+ "_source": dict(source)
}
def _relation(self, key):
@@ -407,7 +414,6 @@ class InitializeService:
except (KeyError, TypeError):
return build_requires
for src_require in requires:
- src_require["relation"] = self._relation(src_require['name'])
src_require["requires_type"] = "build"
build_requires.append(src_require)
return build_requires
@@ -422,7 +428,6 @@ class InitializeService:
install_requires = list()
try:
for bin_require in self._bin_requires.get(bin_pack['pkgKey']):
- bin_require["relation"] = self._relation(bin_require['name'])
bin_require["requires_type"] = "install"
install_requires.append(bin_require)
@@ -465,6 +470,14 @@ class InitializeService:
self._data.src_pkgkeys[src_pack["pkgKey"]] = src_pack
return self._data.src_pkgkeys
+ @property
+ def _src_location(self):
+ if not self._data.src_location:
+ for _, package in self._src_pack.items():
+ key = package["location_href"].split('/')[-1]
+ self._data["src_location"][key] = package
+ return self._data.src_location
+
@property
def _src_requires(self):
"""
@@ -549,22 +562,16 @@ class InitializeService:
"""
def combination_binary(row_data):
- try:
- src_package_name = None
- _pkgs = row_data.get('rpm_sourcerpm').split(
- '-' + row_data.get('version'))
- if len(_pkgs) != 1:
- src_package_name = ''.join(_pkgs[0:-1])
- except AttributeError:
- src_package_name = None
- row_data["src_name"] = src_package_name
+
+ rpm_sourcerpm = row_data.get('rpm_sourcerpm')
+ row_data["src_name"] = rpm_sourcerpm
self._data[table]['packages'][row_data[key]] = row_data
self._data[table]["pkg_key"][row_data["pkgKey"]] = row_data
- if src_package_name:
- if isinstance(self._data[table]["sources"][src_package_name], ESJson):
- self._data[table]["sources"][src_package_name] = [row_data]
+ if rpm_sourcerpm:
+ if isinstance(self._data[table]["sources"][rpm_sourcerpm], ESJson):
+ self._data[table]["sources"][rpm_sourcerpm] = [row_data]
else:
- self._data[table]["sources"][src_package_name].append(
+ self._data[table]["sources"][rpm_sourcerpm].append(
row_data)
def others(row_data, dict_key):
diff --git a/packageship/application/initialize/source.json b/packageship/application/initialize/source.json
index 5ee3544dd897b0469eee9c109c934221f6e3ba0c..12049bfd110baf885a06ead363ff4609f05ed21c 100644
--- a/packageship/application/initialize/source.json
+++ b/packageship/application/initialize/source.json
@@ -26,18 +26,6 @@
"type": "keyword",
"ignore_above": 256
},
- "relation": {
- "properties": {
- "bin_name": {
- "type": "keyword",
- "ignore_above": 256
- },
- "src_name": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
"release": {
"type": "keyword",
"ignore_above": 256
diff --git a/test/cli/init_command/comparison_relation_case/relationship/binary.json b/test/cli/init_command/comparison_relation_case/relationship/binary.json
index 5e202ba4819ad842d8ba5b344ebbf15f851da563..09a87898e08e787b571cfc35122cb0eca4f5251c 100644
--- a/test/cli/init_command/comparison_relation_case/relationship/binary.json
+++ b/test/cli/init_command/comparison_relation_case/relationship/binary.json
@@ -1,230 +1,212 @@
{
- "pkgKey": 35,
- "pkgId": "df487e3623660d477f556899fce77ac650322c4362d82e047c6efc013e07ae5b",
- "name": "Judy",
- "arch": "aarch64",
- "version": "1.0.5",
- "epoch": "0",
- "release": "19.oe1",
- "summary": "C library array",
- "description": "The package provides the most advanced core technology, the main\nadvantages are scalability, high performance and memory efficiency.",
- "url": "http://sourceforge.net/projects/judy/",
- "time_file": 1601219934,
- "time_build": 1601219910,
- "rpm_license": "LGPLv2+",
- "rpm_vendor": "http://openeuler.org",
- "rpm_group": "Unspecified",
- "rpm_buildhost": "obs-worker-0016",
- "rpm_sourcerpm": "Judy-1.0.5-19.oe1.src.rpm",
- "rpm_header_start": 5096,
- "rpm_header_end": 8772,
- "rpm_packager": "http://openeuler.org",
- "size_package": 114276,
- "size_installed": 356971,
- "size_archive": 358048,
- "location_href": "Packages/Judy-1.0.5-19.oe1.aarch64.rpm",
- "location_base": null,
- "checksum_type": "sha256",
- "src_name": "Judy",
- "provides": [
- {
- "name": "Judy",
- "flags": "EQ",
- "epoch": "0",
- "version": "1.0.5",
- "release": "19.oe1",
- "pkgKey": 35
- },
- {
- "name": "Judy(aarch-64)",
- "flags": "EQ",
- "epoch": "0",
- "version": "1.0.5",
- "release": "19.oe1",
- "pkgKey": 35
- },
- {
- "name": "libJudy.so.1()(64bit)",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 35
- }
- ],
- "files": [
- {
- "name": "/etc/ima/digest_lists/0-metadata_list-compact-Judy-1.0.5-19.oe1.aarch64",
- "type": "file",
- "pkgKey": 35
- },
- {
- "name": "/etc/ima/digest_lists.tlv/0-metadata_list-compact_tlv-Judy-1.0.5-19.oe1.aarch64",
- "type": "file",
- "pkgKey": 35
- }
- ],
- "filelists": [
- {
- "pkgKey": 35,
- "dirname": "/usr/share/licenses",
- "filenames": "Judy",
- "filetypes": "d"
- },
- {
- "pkgKey": 35,
- "dirname": "/usr/share/licenses/Judy",
- "filenames": "COPYING",
- "filetypes": "f"
- },
- {
- "pkgKey": 35,
- "dirname": "/etc/ima/digest_lists.tlv",
- "filenames": "0-metadata_list-compact_tlv-Judy-1.0.5-19.oe1.aarch64",
- "filetypes": "f"
- },
- {
- "pkgKey": 35,
- "dirname": "/usr/lib64",
- "filenames": "libJudy.so.1/libJudy.so.1.0.3",
- "filetypes": "ff"
- },
- {
- "pkgKey": 35,
- "dirname": "/etc/ima/digest_lists",
- "filenames": "0-metadata_list-compact-Judy-1.0.5-19.oe1.aarch64",
- "filetypes": "f"
- }
- ],
- "src_version": "1.0.5",
- "requires": [
- {
- "name": "coreutils",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 14,
- "pre": "FALSE",
- "relation": [{ "bin_name": "coreutils", "src_name": "coreutils" }],
- "requires_type": "build"
- },
- {
- "name": "gawk",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 14,
- "pre": "FALSE",
- "relation": [{ "bin_name": "gawk", "src_name": "gawk" }],
- "requires_type": "build"
- },
- {
- "name": "gcc",
- "flags": "GE",
- "epoch": "0",
- "version": "4.1",
- "release": null,
- "pkgKey": 14,
- "pre": "FALSE",
- "relation": [{ "bin_name": "gcc", "src_name": "gcc" }],
- "requires_type": "build"
- },
- {
- "name": "make",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 14,
- "pre": "FALSE",
- "relation": [{ "bin_name": "make", "src_name": "make" }],
- "requires_type": "build"
- },
- {
- "name": "sed",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 14,
- "pre": "FALSE",
- "relation": [{ "bin_name": "sed", "src_name": "sed" }],
- "requires_type": "build"
- },
- {
- "name": "/bin/sh",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 35,
- "pre": "TRUE",
- "relation": [
- { "bin_name": "bash", "src_name": "bash" },
- { "bin_name": "coreutils", "src_name": "coreutils" }
- ],
- "requires_type": "install"
- },
- {
- "name": "/bin/sh",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 35,
- "pre": "FALSE",
- "relation": [
- { "bin_name": "bash", "src_name": "bash" },
- { "bin_name": "coreutils", "src_name": "coreutils" }
- ],
- "requires_type": "install"
- },
- {
- "name": "ld-linux-aarch64.so.1()(64bit)",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 35,
- "pre": "FALSE",
- "relation": [{ "bin_name": "glibc", "src_name": "glibc" }],
- "requires_type": "install"
- },
- {
- "name": "ld-linux-aarch64.so.1(GLIBC_2.17)(64bit)",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 35,
- "pre": "FALSE",
- "relation": [{ "bin_name": "glibc", "src_name": "glibc" }],
- "requires_type": "install"
- },
- {
- "name": "rtld(GNU_HASH)",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 35,
- "pre": "FALSE",
- "relation": [{ "bin_name": "glibc", "src_name": "glibc" }],
- "requires_type": "install"
- },
- {
- "name": "libc.so.6(GLIBC_2.17)(64bit)",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 35,
- "pre": "FALSE",
- "relation": [{ "bin_name": "glibc", "src_name": "glibc" }],
- "requires_type": "install"
- }
- ]
- }
-
\ No newline at end of file
+ "pkgKey": 35,
+ "pkgId": "df487e3623660d477f556899fce77ac650322c4362d82e047c6efc013e07ae5b",
+ "name": "Judy",
+ "arch": "aarch64",
+ "version": "1.0.5",
+ "epoch": "0",
+ "release": "19.oe1",
+ "summary": "C library array",
+ "description": "The package provides the most advanced core technology, the main\nadvantages are scalability, high performance and memory efficiency.",
+ "url": "http://sourceforge.net/projects/judy/",
+ "time_file": 1601219934,
+ "time_build": 1601219910,
+ "rpm_license": "LGPLv2+",
+ "rpm_vendor": "http://openeuler.org",
+ "rpm_group": "Unspecified",
+ "rpm_buildhost": "obs-worker-0016",
+ "rpm_sourcerpm": "Judy-1.0.5-19.oe1.src.rpm",
+ "rpm_header_start": 5096,
+ "rpm_header_end": 8772,
+ "rpm_packager": "http://openeuler.org",
+ "size_package": 114276,
+ "size_installed": 356971,
+ "size_archive": 358048,
+ "location_href": "Packages/Judy-1.0.5-19.oe1.aarch64.rpm",
+ "location_base": null,
+ "checksum_type": "sha256",
+ "src_name": "Judy",
+ "provides": [
+ {
+ "name": "Judy",
+ "flags": "EQ",
+ "epoch": "0",
+ "version": "1.0.5",
+ "release": "19.oe1",
+ "pkgKey": 35
+ },
+ {
+ "name": "Judy(aarch-64)",
+ "flags": "EQ",
+ "epoch": "0",
+ "version": "1.0.5",
+ "release": "19.oe1",
+ "pkgKey": 35
+ },
+ {
+ "name": "libJudy.so.1()(64bit)",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 35
+ }
+ ],
+ "files": [
+ {
+ "name": "/etc/ima/digest_lists/0-metadata_list-compact-Judy-1.0.5-19.oe1.aarch64",
+ "type": "file",
+ "pkgKey": 35
+ },
+ {
+ "name": "/etc/ima/digest_lists.tlv/0-metadata_list-compact_tlv-Judy-1.0.5-19.oe1.aarch64",
+ "type": "file",
+ "pkgKey": 35
+ }
+ ],
+ "filelists": [
+ {
+ "pkgKey": 35,
+ "dirname": "/usr/share/licenses",
+ "filenames": "Judy",
+ "filetypes": "d"
+ },
+ {
+ "pkgKey": 35,
+ "dirname": "/usr/share/licenses/Judy",
+ "filenames": "COPYING",
+ "filetypes": "f"
+ },
+ {
+ "pkgKey": 35,
+ "dirname": "/etc/ima/digest_lists.tlv",
+ "filenames": "0-metadata_list-compact_tlv-Judy-1.0.5-19.oe1.aarch64",
+ "filetypes": "f"
+ },
+ {
+ "pkgKey": 35,
+ "dirname": "/usr/lib64",
+ "filenames": "libJudy.so.1/libJudy.so.1.0.3",
+ "filetypes": "ff"
+ },
+ {
+ "pkgKey": 35,
+ "dirname": "/etc/ima/digest_lists",
+ "filenames": "0-metadata_list-compact-Judy-1.0.5-19.oe1.aarch64",
+ "filetypes": "f"
+ }
+ ],
+ "src_version": "1.0.5",
+ "requires": [
+ {
+ "name": "coreutils",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 14,
+ "pre": "FALSE",
+ "requires_type": "build"
+ },
+ {
+ "name": "gawk",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 14,
+ "pre": "FALSE",
+ "requires_type": "build"
+ },
+ {
+ "name": "gcc",
+ "flags": "GE",
+ "epoch": "0",
+ "version": "4.1",
+ "release": null,
+ "pkgKey": 14,
+ "pre": "FALSE",
+ "requires_type": "build"
+ },
+ {
+ "name": "make",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 14,
+ "pre": "FALSE",
+ "requires_type": "build"
+ },
+ {
+ "name": "sed",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 14,
+ "pre": "FALSE",
+ "requires_type": "build"
+ },
+ {
+ "name": "/bin/sh",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 35,
+ "pre": "TRUE",
+ "requires_type": "install"
+ },
+ {
+ "name": "/bin/sh",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 35,
+ "pre": "FALSE",
+ "requires_type": "install"
+ },
+ {
+ "name": "ld-linux-aarch64.so.1()(64bit)",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 35,
+ "pre": "FALSE",
+ "requires_type": "install"
+ },
+ {
+ "name": "ld-linux-aarch64.so.1(GLIBC_2.17)(64bit)",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 35,
+ "pre": "FALSE",
+ "requires_type": "install"
+ },
+ {
+ "name": "rtld(GNU_HASH)",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 35,
+ "pre": "FALSE",
+ "requires_type": "install"
+ },
+ {
+ "name": "libc.so.6(GLIBC_2.17)(64bit)",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 35,
+ "pre": "FALSE",
+ "requires_type": "install"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/test/cli/init_command/comparison_relation_case/relationship/source.json b/test/cli/init_command/comparison_relation_case/relationship/source.json
index 664add83ce8467e7d40d0f0f36676579e68775c6..e09f3cb33648a139e8f0a81ff831a55c83f1270b 100644
--- a/test/cli/init_command/comparison_relation_case/relationship/source.json
+++ b/test/cli/init_command/comparison_relation_case/relationship/source.json
@@ -1,72 +1,74 @@
{
- "pkgKey": 3,
- "pkgId": "a2c8963c6d5bdea8d3c9dc1bfa609db643c5a17077eff4eca21aa6e33ff9d4e5",
- "name": "CUnit",
- "arch": "src",
- "version": "2.1.3",
- "epoch": "0",
- "release": "21.oe1",
- "summary": "A Unit Testing Framework for C",
- "description": "CUnit is a lightweight system for writing, administering, and running unit tests in C\nIt provides C programmers a basic testing functionality with a flexible variety of user\ninterfaces.\n\nCUnit is built as a static library which is linked with the user's testing code. It\nuses a simple framework for building test structures, and provides a rich set of\nassertions for testing common data types. In addition, several different interfaces are\nprovided for running tests and reporting results.",
- "url": "http://cunit.sourceforge.net/",
- "time_file": 1601215538,
- "time_build": 1601215498,
- "rpm_license": "LGPLv2+",
- "rpm_vendor": "http://openeuler.org",
- "rpm_group": "Unspecified",
- "rpm_buildhost": "obs-worker-009",
- "rpm_sourcerpm": "",
- "rpm_header_start": 5096,
- "rpm_header_end": 7552,
- "rpm_packager": "http://openeuler.org",
- "size_package": 523353,
- "size_installed": 516791,
- "size_archive": 517172,
- "location_href": "Packages/CUnit-2.1.3-21.oe1.src.rpm",
- "location_base": null,
- "checksum_type": "sha256",
- "requires": [
- {
- "name": "automake",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 3,
- "pre": "FALSE",
- "relation": [{ "bin_name": "automake", "src_name": "automake" }],
- "requires_type": "build"
- },
- {
- "name": "git",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 3,
- "pre": "FALSE",
- "relation": [
- { "bin_name": "git", "src_name": "git" },
- { "bin_name": "git", "src_name": "git" }
- ],
- "requires_type": "build"
- },
- {
- "name": "libtool",
- "flags": null,
- "epoch": null,
- "version": null,
- "release": null,
- "pkgKey": 3,
- "pre": "FALSE",
- "relation": [{ "bin_name": "libtool", "src_name": "libtool" }],
- "requires_type": "build"
- }
- ],
- "subpacks": [
- { "name": "CUnit", "version": "2.1.3" },
- { "name": "CUnit-devel", "version": "2.1.3" },
- { "name": "CUnit-help", "version": "2.1.3" }
- ]
- }
-
\ No newline at end of file
+ "pkgKey": 3,
+ "pkgId": "a2c8963c6d5bdea8d3c9dc1bfa609db643c5a17077eff4eca21aa6e33ff9d4e5",
+ "name": "CUnit",
+ "arch": "src",
+ "version": "2.1.3",
+ "epoch": "0",
+ "release": "21.oe1",
+ "summary": "A Unit Testing Framework for C",
+ "description": "CUnit is a lightweight system for writing, administering, and running unit tests in C\nIt provides C programmers a basic testing functionality with a flexible variety of user\ninterfaces.\n\nCUnit is built as a static library which is linked with the user's testing code. It\nuses a simple framework for building test structures, and provides a rich set of\nassertions for testing common data types. In addition, several different interfaces are\nprovided for running tests and reporting results.",
+ "url": "http://cunit.sourceforge.net/",
+ "time_file": 1601215538,
+ "time_build": 1601215498,
+ "rpm_license": "LGPLv2+",
+ "rpm_vendor": "http://openeuler.org",
+ "rpm_group": "Unspecified",
+ "rpm_buildhost": "obs-worker-009",
+ "rpm_sourcerpm": "",
+ "rpm_header_start": 5096,
+ "rpm_header_end": 7552,
+ "rpm_packager": "http://openeuler.org",
+ "size_package": 523353,
+ "size_installed": 516791,
+ "size_archive": 517172,
+ "location_href": "Packages/CUnit-2.1.3-21.oe1.src.rpm",
+ "location_base": null,
+ "checksum_type": "sha256",
+ "requires": [
+ {
+ "name": "automake",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 3,
+ "pre": "FALSE",
+ "requires_type": "build"
+ },
+ {
+ "name": "git",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 3,
+ "pre": "FALSE",
+ "requires_type": "build"
+ },
+ {
+ "name": "libtool",
+ "flags": null,
+ "epoch": null,
+ "version": null,
+ "release": null,
+ "pkgKey": 3,
+ "pre": "FALSE",
+ "requires_type": "build"
+ }
+ ],
+ "subpacks": [
+ {
+ "name": "CUnit",
+ "version": "2.1.3"
+ },
+ {
+ "name": "CUnit-devel",
+ "version": "2.1.3"
+ },
+ {
+ "name": "CUnit-help",
+ "version": "2.1.3"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/test/cli/init_command/comparison_relation_case/test_relationship.py b/test/cli/init_command/comparison_relation_case/test_relationship.py
index 7e389297339217f58438053e472b65030fd76fe2..147bb9823cadd0bfb3aec1dae8c62bf443a9042c 100644
--- a/test/cli/init_command/comparison_relation_case/test_relationship.py
+++ b/test/cli/init_command/comparison_relation_case/test_relationship.py
@@ -11,6 +11,7 @@
# See the Mulan PSL v2 for more details.
# ******************************************************************************/
# -*- coding:utf-8 -*-
+import json
import os
import shutil
from pathlib import Path
diff --git a/test/unpack/compress_files/gz_error.txt b/test/unpack/compress_files/gz_error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9ea415ee96ce4459710a2c5cab997053cb5c207f
--- /dev/null
+++ b/test/unpack/compress_files/gz_error.txt
@@ -0,0 +1 @@
+gz error file
\ No newline at end of file
diff --git a/test/unpack/compress_files/tar_error.txt b/test/unpack/compress_files/tar_error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..beb36750c7e906599a4765ec8debf914f4c0d035
--- /dev/null
+++ b/test/unpack/compress_files/tar_error.txt
@@ -0,0 +1 @@
+tar file error
\ No newline at end of file
diff --git a/test/unpack/compress_files/xz_error.txt b/test/unpack/compress_files/xz_error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3fd014f7e5ccf3864da047743a6af767355c4937
--- /dev/null
+++ b/test/unpack/compress_files/xz_error.txt
@@ -0,0 +1 @@
+xz file error
\ No newline at end of file
diff --git a/test/unpack/test_unpack.py b/test/unpack/test_unpack.py
index 64ad3381e58867c83e1c10a9f125341771e2c454..90bcae8dcba6e27375467c292e675e84a64d1e9b 100644
--- a/test/unpack/test_unpack.py
+++ b/test/unpack/test_unpack.py
@@ -34,13 +34,13 @@ class TestUnpack(TestCase):
def test_gz_unpack(self):
"""unpack gz file"""
with self.assertRaises(UnpackError):
- Unpack.dispatch(".gz", file_path=os.path.join(self.path, "bz2_error.txt"),
+ Unpack.dispatch(".gz", file_path=os.path.join(self.path, "gz_error.txt"),
save_file=os.path.join(self.path, "bz2_unpack.sqlite"))
def test_error_tar(self):
"""error tar file"""
with self.assertRaises(IOError):
- Unpack.dispatch(".tar", file_path=os.path.join(self.path, "bz2_error.txt"),
+ Unpack.dispatch(".tar", file_path=os.path.join(self.path, "tar_error.txt"),
save_file=os.path.join(self.path, "bz2_unpack.sqlite"))
def test_tar_unpack(self):
@@ -50,9 +50,14 @@ class TestUnpack(TestCase):
def test_error_zip_unpack(self):
"""unpack error zip file"""
+ file = os.path.join(self.path, "zip_error.txt")
+ with open(file, "w", encoding="utf-8") as f:
+ f.write("error zip file")
with self.assertRaises(IOError):
- Unpack.dispatch(".zip", file_path=os.path.join(self.path, "bz2_error.txt"),
+ Unpack.dispatch(".zip", file_path=file,
save_file=os.path.join(self.path, "zip_file.sqlite"))
+ if os.path.exists(file):
+ os.remove(file)
def test_zip_unpack(self):
"""unpack zip file"""
@@ -64,7 +69,7 @@ class TestUnpack(TestCase):
def test_xz_unpack(self):
"""unpack xz file"""
with self.assertRaises(UnpackError):
- Unpack.dispatch(".xz", file_path=os.path.join(self.path, "bz2_error.txt"),
+ Unpack.dispatch(".xz", file_path=os.path.join(self.path, "xz_error.txt"),
save_file=os.path.join(self.path, "xz.sqlite"))
def test_file_path_none(self):

212
0004-fix-depend-query.patch Normal file
View File

@ -0,0 +1,212 @@
diff --git a/packageship/application/apps/package/view.py b/packageship/application/apps/package/view.py
index 86072fc61f78ddcae87a53728d1490821ccc41a7..3ca7ffa261d93b8868571e9bedf13f1232e4768e 100644
--- a/packageship/application/apps/package/view.py
+++ b/packageship/application/apps/package/view.py
@@ -108,14 +108,16 @@ class SourcePackages(ParsePackageMethod):
page_num = result.get("page_num")
page_size = result.get("page_size")
query_pkg_name = [result.get("query_pkg_name")] if result.get(
- "query_pkg_name") else []
+ "query_pkg_name") else None
find_package = Package()
try:
result_all = find_package.all_src_packages(
result.get("database_name"), page_num=page_num, page_size=page_size, package_list=query_pkg_name,
command_line=result.get("command_line"))
- except (ElasticSearchQueryException, DatabaseConfigException, PackageInfoGettingError) as e:
+ except (ElasticSearchQueryException, DatabaseConfigException):
return jsonify(self.rspmsg.body('connect_db_error'))
+ except PackageInfoGettingError:
+ return jsonify(self.rspmsg.body('pack_name_not_found'))
if result_all:
return jsonify(self.parse_package(result_all, page_size))
return jsonify(self.rspmsg.body("table_name_not_exist"))
@@ -167,14 +169,16 @@ class BinaryPackages(ParsePackageMethod):
page_num = result.get("page_num")
page_size = result.get("page_size")
query_pkg_name = [result.get("query_pkg_name")] if result.get(
- "query_pkg_name") else []
+ "query_pkg_name") else None
find_package = Package()
try:
result_all = find_package.all_bin_packages(
result.get("database_name"), page_num=page_num, page_size=page_size, package_list=query_pkg_name,
command_line=result.get("command_line"))
- except (ElasticSearchQueryException, DatabaseConfigException, PackageInfoGettingError) as e:
+ except (ElasticSearchQueryException, DatabaseConfigException):
return jsonify(self.rspmsg.body('connect_db_error'))
+ except PackageInfoGettingError:
+ return jsonify(self.rspmsg.body('pack_name_not_found'))
if result_all:
return jsonify(self.parse_package(result_all, page_size))
return jsonify(self.rspmsg.body("table_name_not_exist"))
diff --git a/packageship/application/query/pkg.py b/packageship/application/query/pkg.py
index 6c4e1f253ecf0cef68ab6319a9fae1731e826a8b..67e5c2e3a2cab3107c1d1fcb1edc618ed6f2a370 100644
--- a/packageship/application/query/pkg.py
+++ b/packageship/application/query/pkg.py
@@ -82,7 +82,7 @@ class QueryPackage(object):
Raises: DatabaseConfigException ElasticSearchQueryException
"""
self.rpm_type = SOURCE_DB_TYPE
- response = self._get_rpm_info(src_list, database, page_num, page_size, command_line)
+ response = self._get_rpm_info(database, page_num, page_size, command_line, rpm_list=src_list)
return response
def get_bin_info(self, binary_list, database, page_num, page_size, command_line=False):
@@ -98,7 +98,7 @@ class QueryPackage(object):
Raises: DatabaseConfigException ElasticSearchQueryException
"""
self.rpm_type = BINARY_DB_TYPE
- response = self._get_rpm_info(binary_list, database, page_num, page_size, command_line)
+ response = self._get_rpm_info(database, page_num, page_size, command_line, rpm_list=binary_list)
return response
def _query_src_bin_rpm(self, rpm_list, query_db_type, specify_db):
@@ -150,7 +150,7 @@ class QueryPackage(object):
return rpm_info
- def _get_rpm_info(self, rpm_list, database, page_num, page_size, command_line):
+ def _get_rpm_info(self, database, page_num, page_size, command_line, rpm_list=None):
"""
General method for obtaining package details
Args:
@@ -162,10 +162,16 @@ class QueryPackage(object):
Returns: result of query package details
"""
- self.index = UNDERLINE.join((database, self.rpm_type))
response = dict(total=0, data=[])
+ # If the query list is empty, return directly; if the query list is None, it means query all packages.
+ if isinstance(rpm_list, list):
+ rpm_list = [rpm for rpm in rpm_list if rpm]
+ if not rpm_list:
+ return response
+
+ self.index = UNDERLINE.join((database, self.rpm_type))
# Used for Command Line,query all data and no Pagination
- if command_line and not rpm_list:
+ if command_line and rpm_list is None:
query_result = self._db_session.scan(index=self.index, body=QueryBody.QUERY_ALL)
total_num = self._db_session.count(index=self.index, body=QueryBody.QUERY_ALL)
response['total'] = total_num.get('count')
@@ -190,6 +196,7 @@ class QueryPackage(object):
if query_all else query_result['hits']['total']['value']
response['total'] = total_num
except KeyError:
+ response = dict(total=0, data=[])
return response
return response
@@ -211,7 +218,7 @@ class QueryPackage(object):
# Query specify rpm_list of Command line mode
query_body = self._process_query_terms(DEFAULT_PAGE_NUM, MAX_PAGE_SIZE, rpm_list)
else:
- if not rpm_list:
+ if rpm_list is None:
# Query all data and Pagination of UI mode
query_body = self._format_paging_query_all(page_num, page_size)
query_all = True
diff --git a/test/cli/depend_commands/__init__.py b/test/cli/depend_commands/__init__.py
index 1aab4d66def3400065e077e884258ab446e0f2da..13fbaa063d39e3a19c9dec58fc480fe05c35aa3b 100755
--- a/test/cli/depend_commands/__init__.py
+++ b/test/cli/depend_commands/__init__.py
@@ -130,6 +130,7 @@ class DependTestBase(ClientTest):
if not pkg_info:
continue
ret_dict_p["hits"]["hits"].append(pkg_info)
+ ret_dict_p['hits']['total']['value'] = len(pkg_lst)
@staticmethod
def make_newline_split_res(ln, lines, idx):
@@ -244,12 +245,12 @@ class DependTestBase(ClientTest):
if handle_tolong_name_in_newline(ln, source_lines, idx):
continue
source_lines.append(split_ln)
-
+
if not all(normal_query_judgment_lst):
raise ValueError(
"check Your expected str please, must be a normal query result"
)
-
+
return binary_lines, source_lines
def assert_result(self):
@@ -264,12 +265,12 @@ class DependTestBase(ClientTest):
current_bin, current_src = self._process_depend_command_value(self.print_result)
self.assertListEqual(sorted(excepted_bin), sorted(current_bin))
self.assertListEqual(sorted(excepted_src), sorted(current_src))
-
+
def assert_exc_result(self):
"""assert_exc_result
"""
- super(DependTestBase,self).assert_result()
-
+ super(DependTestBase, self).assert_result()
+
def _es_search_result(self, index: str, body: dict):
"""
Get different return values through different call parameters
@@ -281,7 +282,7 @@ class DependTestBase(ClientTest):
ret_dict(dict):Parsable es-like data for the project
"""
- ret_dict = {"hits": {"hits": []}}
+ ret_dict = {"hits": {"hits": [], "total": {"value": 0}}}
if index == "databaseinfo":
return DATA_BASE_INFO
elif "binary" in index:
diff --git a/test/cli/package_command/pkg_list/test_pkg_list.py b/test/cli/package_command/pkg_list/test_pkg_list.py
index 3c7f9930a773c9aea1eafeb6e0739a10d6c5b7db..da5300b1ca5935d116700f0d20ca8352e16bf464 100644
--- a/test/cli/package_command/pkg_list/test_pkg_list.py
+++ b/test/cli/package_command/pkg_list/test_pkg_list.py
@@ -77,8 +77,8 @@ HINT :Make sure the table is valid"""
test pkgInfoGetingError for all bin packages
"""
self.excepted_str = """
-ERROR_CONTENT :Failed to Connect the database
-HINT :Check the connection"""
+ERROR_CONTENT :The querying package does not exist in the databases
+HINT :Use the correct package name and try again"""
self.command_params = ["os-version"]
self.mock_es_search(return_value=DATA_BASE_INFO)
self.mock_es_scan(return_value=[])
diff --git a/test/cli/package_command/pkg_list/test_pkg_list_s.py b/test/cli/package_command/pkg_list/test_pkg_list_s.py
index 3044c93e7ac5b64fa1de8195c766bd10fd554e0b..45c949dcbe007e719a2004b22a83b72aa8c59986 100644
--- a/test/cli/package_command/pkg_list/test_pkg_list_s.py
+++ b/test/cli/package_command/pkg_list/test_pkg_list_s.py
@@ -77,8 +77,8 @@ HINT :Make sure the table is valid"""
test pkgInfoGettingError
"""
self.excepted_str = """
-ERROR_CONTENT :Failed to Connect the database
-HINT :Check the connection"""
+ERROR_CONTENT :The querying package does not exist in the databases
+HINT :Use the correct package name and try again"""
self.command_params = ["os-version", "-s"]
self.mock_es_search(return_value=DATA_BASE_INFO)
self.mock_es_scan(return_value=[])
diff --git a/test/test_module/test_database_query/test_database_query_package/test_query_bin_info.py b/test/test_module/test_database_query/test_database_query_package/test_query_bin_info.py
index e59e82e5768015a328ce2ceecf4d60e5a655b9b8..3254eb8b2089bc8a76ba1844d60d92938ae888d9 100644
--- a/test/test_module/test_database_query/test_database_query_package/test_query_bin_info.py
+++ b/test/test_module/test_database_query/test_database_query_package/test_query_bin_info.py
@@ -41,7 +41,7 @@ class TestQueryBinaryPkgInfo(TestCase):
"""
self.session.scan = MagicMock(return_value=self.ALL_BINARY_RPM_INFO)
self.session.count = MagicMock(return_value={"count": 5})
- query_result = self.query_package.get_bin_info(binary_list=[], database='os_version_1', page_num=1, page_size=20,
+ query_result = self.query_package.get_bin_info(binary_list=None, database='os_version_1', page_num=1, page_size=20,
command_line=True)
self.assertIsNotNone(query_result['data'])
@@ -52,7 +52,7 @@ class TestQueryBinaryPkgInfo(TestCase):
"""
self.session.query = MagicMock(return_value=self.ALL_BINARY_INFO_PAGING)
self.session.count = MagicMock(return_value={"count": 5})
- query_result = self.query_package.get_bin_info(binary_list=[], database='os_version_1', page_num=1, page_size=5,
+ query_result = self.query_package.get_bin_info(binary_list=None, database='os_version_1', page_num=1, page_size=5,
command_line=False)
self.assertEqual(len(query_result['data']), 5)

1864
0005-update-copyright.patch Normal file
View File

File diff suppressed because it is too large Load Diff

36
README.en.md Normal file
View File

@ -0,0 +1,36 @@
# pkgship
#### Description
a package depend query tool
#### Software Architecture
Software architecture description
#### Installation
1. xxxx
2. xxxx
3. xxxx
#### Instructions
1. xxxx
2. xxxx
3. xxxx
#### Contribution
1. Fork the repository
2. Create Feat_xxx branch
3. Commit your code
4. Create Pull Request
#### Gitee Feature
1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md
2. Gitee blog [blog.gitee.com](https://blog.gitee.com)
3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore)
4. The most valuable open source project [GVP](https://gitee.com/gvp)
5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help)
6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)

193
README.md
View File

@ -11,8 +11,6 @@
- [配置参数](#配置参数) - [配置参数](#配置参数)
- [服务启动和停止](#服务启动和停止) - [服务启动和停止](#服务启动和停止)
- [工具使用](#工具使用) - [工具使用](#工具使用)
- [日志查看和转储](#日志查看和转储)
- [扩展工具pkgship-panel](#扩展工具pkgship-panel)
<!-- /TOC --> <!-- /TOC -->
@ -26,36 +24,22 @@ pkgship是一款管理OS软件包依赖关系，提供依赖和被依赖关系
系统采用flask-restful开发 系统采用flask-restful开发
![输入图片说明](https://gitee.com/openeuler/pkgship/raw/master/doc/design/pkgimg/packagemanagement.JPG) ![avatar](./doc/design/pkgimg/packagemanagement.JPG)
## 软件下载 ## 软件下载
* Repo源挂载正式发布地址：<https://repo.openeuler.org/> * Repo源挂载正式发布地址：<https://repo.openeuler.org/>
* 源码获取地址：<https://gitee.com/openeuler/pkgship> * 源码获取地址：<https://gitee.com/openeuler/pkgship>
* rpm包版本获取地址：<https://build.openeuler.org/package/show/openEuler:22.09/pkgship> * rpm包版本获取地址：<https://117.78.1.88/project/show/openEuler:Mainline>
## 运行环境 ## 运行环境
- 硬件配置： * 可用内存700M以上
* python版本 3.8及以上
| 配置项 | 推荐规格 | * Elasticsearch 版本7.10.1
| -------- | ----------- | * Redis
| CPU | 8核 |
| 内存 | 32G，最小4G |
| 网络带宽 | 300M |
| I/O | 375MB/sec |
- 软件配置：
| 软件名 | 版本和规格 |
| ------------- | ------------------------------------------ |
| Elasticsearch | 版本7.10.1；单机部署可用；有能力可部署集群 |
| Redis | 建议5.0.4及以上；建议大小配置为内存的3/4 |
| Python | 版本 3.8及以上 |
## 安装工具 ## 安装工具
> 说明：该软件支持在docker下运行。目前在openEuler21.09版本下，由于环境条件限制，创建docker时请使用--privileged参数，不使用--privileged参数将会导致软件启动失败，后续适配后将更新该文档。
**1、pkgship工具安装** **1、pkgship工具安装**
工具安装可通过以下两种方式中的任意一种实现。 工具安装可通过以下两种方式中的任意一种实现。
@ -95,18 +79,12 @@ pkgship是一款管理OS软件包依赖关系，提供依赖和被依赖关系
/bin/bash auto_install_pkgship_requires.sh elasticsearch /bin/bash auto_install_pkgship_requires.sh elasticsearch
``` ```
> 目前由于rpm包方式安装Elasticsearch默认为无密码模式，且pkgship需使用无密码设置的Elasticsearch，所以建议Elasticsearch和pkgship安装在同一服务器，以通过网络隔离提高安全性。后续将支持Elasticsearch设置用户名密码。 或者
或者
``` ```
/bin/bash auto_install_pkgship_requires.sh redis /bin/bash auto_install_pkgship_requires.sh redis
``` ```
**3、安装后添加用户**
在安装pkgship软件后，会自动创建名为pkgshipuser的用户和名为pkgshipuser的用户组，无需手动创建，后续服务启动和运行时，都会以该用户角色操作。
## 配置参数 ## 配置参数
1.在配置文件中对相应参数进行配置，系统的默认配置文件存放在 /etc/pkgship/packge.ini，请根据实际情况进行配置更改。 1.在配置文件中对相应参数进行配置，系统的默认配置文件存放在 /etc/pkgship/packge.ini，请根据实际情况进行配置更改。
@ -120,12 +98,10 @@ vim /etc/pkgship/package.ini
; 初始化数据库时导入的yaml文件存放位置，该yaml中记录导入的sqlite文件位置。 ; 初始化数据库时导入的yaml文件存放位置，该yaml中记录导入的sqlite文件位置。
init_conf_path=/etc/pkgship/conf.yaml init_conf_path=/etc/pkgship/conf.yaml
; 若部署为客户端-服务端方式，服务端需保证query_ip_addr为本机ip或者0.0.0.0， ; 数据库端口。
; 并且客户端可通过query_ip_addr加query_port访问服务端，或者通过设置映射的remote_host访问服务端。
; 服务查询端口。
query_port=8090 query_port=8090
; 服务查询ip。 ; 数据库ip地址。
query_ip_addr=127.0.0.1 query_ip_addr=127.0.0.1
; 远程服务的地址，命令行可以直接调用远程服务来完成数据请求。 ; 远程服务的地址，命令行可以直接调用远程服务来完成数据请求。
@ -170,7 +146,7 @@ redis_port=6379
redis_max_connections=10 redis_max_connections=10
[DATABASE-数据库] [DATABASE-数据库]
;数据库访问地址，建议设置为本机地址 ;数据库访问地址，默认为本机地址
database_host=127.0.0.1 database_host=127.0.0.1
;数据库访问端口，默认为9200 ;数据库访问端口，默认为9200
@ -182,12 +158,12 @@ database_port=9200
conf.yaml 文件默认存放在 /etc/pkgship/ 路径下，pkgship会通过该配置读取要建立的数据库名称以及需要导入的sqlite文件，也支持配置sqlite文件所在的repo地址。conf.yaml 示例如下所示。 conf.yaml 文件默认存放在 /etc/pkgship/ 路径下，pkgship会通过该配置读取要建立的数据库名称以及需要导入的sqlite文件，也支持配置sqlite文件所在的repo地址。conf.yaml 示例如下所示。
```yaml ```yaml
dbname: oe20.03 #数据库名称 dbname: openEuler-20.03 #数据库名称
src_db_file: file:///etc/pkgship/repo/openEuler-20.09/src #源码包所在的本地路径 src_db_file: /etc/pkgship/repo/openEuler-20.09/src #源码包所在的本地路径
bin_db_file: file:///etc/pkgship/repo/openEuler-20.09/bin #二进制包所在的本地路径 bin_db_file: /etc/pkgship/repo/openEuler-20.09/bin #二进制包所在的本地路径
priority: 1 #数据库优先级 priority: 1 #数据库优先级
dbname: oe20.09 dbname: openEuler-20.09
src_db_file: https://repo.openeuler.org/openEuler-20.09/source #源码包所在的repo源 src_db_file: https://repo.openeuler.org/openEuler-20.09/source #源码包所在的repo源
bin_db_file: https://repo.openeuler.org/openEuler-20.09/everything/aarch64 #二进制包所在的repo源 bin_db_file: https://repo.openeuler.org/openEuler-20.09/everything/aarch64 #二进制包所在的repo源
priority: 2 priority: 2
@ -197,14 +173,7 @@ priority: 2
> 如需更改存放路径，请更改package.ini下的 init_conf_path 选项。 > 如需更改存放路径，请更改package.ini下的 init_conf_path 选项。
> >
> sqlite文件配置规格： > 不支持直接配置sqlite文件路径。
>
> - 为节省空间，不支持直接使用sqlite文件，请使用包含sqlite文件的压缩包，支持压缩格式为：.bz .gz .tar .zip
> - sqlite压缩包命名格式为xxx.primary.sqlite.xx，二进制包所在路径下还需使用filelists类型的sqlite压缩文件，命名格式为xxx.filelists.sqlite.xx,一般网络地址下的sqlite压缩包都符合该格式。
> - 本地路径请使用file://前缀。
> - 本地路径和网络地址路径都要满足sqlite压缩包位于：配置路径/repodata/xxx.primary.sqlite.xx
>
> dbname请使用小写字母或者数字，不支持大写字母。
## 服务启动和停止 ## 服务启动和停止
pkgship启动和停止方式有两种，systemctl方式和pkgshipd方式，其中systemctl方式启动可以有异常停止自启动的机制。两种方式的执行命令为： pkgship启动和停止方式有两种，systemctl方式和pkgshipd方式，其中systemctl方式启动可以有异常停止自启动的机制。两种方式的执行命令为：
@ -225,52 +194,50 @@ pkgshipd stop 停止服务
> 每次起停周期内仅支持一种方式，不允许两种操作同时使用。 > 每次起停周期内仅支持一种方式，不允许两种操作同时使用。
> >
> pkgshipd启动方式只允许在pkgshipuser用户下操作。 > pkgshipd启动方式只允许在pkgshipduser用户下操作。
>
> docker环境下如果不支持systemctl命令，请使用pkgshipd启停方式。
## 工具使用 ## 工具使用
1. 数据库初始化。 1. 数据库初始化。
> 使用场景：服务启动后，为了能查询对应的数据库（比如oe20.03，oe20.09）中的包信息及包依赖关系，需要将这些数据库通过createrepo生成的sqlite（分为源码库和二进制库）导入进服务内，生成对应的包信息json体然后插入Elasticsearch对应的数据库中。数据库名为根据config.yaml中配置的dbname生成的dbname-source/binary。 > 使用场景：服务启动后，为了能查询对应的数据库（比如openEuler-20.09， openEuler-21.03）中的包信息及包依赖关系，需要将这些数据库通过createrepo生成的sqlite（分为源码库和二进制库）导入进服务内，生成对应的包信息json体然后插入Elasticsearch对应的数据库中。数据库名为根据config.yaml中配置的dbname生成的dbname-source/binary，[-filepath]为可选参数。
```bash ```bash
pkgship init [-filepath path] pkgship init [-filepath path]
``` ```
> 参数说明： > 参数说明：
> -filepath：指定初始化配置文件config.yaml的路径，可以使用相对路径和绝对路径，不带参数则使用默认配置初始化，可选参数。 > -filepath：指定初始化配置文件config.yaml的路径，可以使用相对路径和绝对路径，不带参数则使用默认配置初始化。
2. 单包查询。 2. 单包查询。
用户可查询源码包或者二进制包(packagename)在指定数据库表（database）中的具体信息。 用户可查询源码包或者二进制包(packagename)在指定数据库表（database）中的具体信息。
> 使用场景：用户可查询源码包或者二进制包在指定数据库中的具体信息。 > 使用场景：用户可查询源码包或者二进制包在指定数据库中的具体信息，packagename,database为必选参数,-s为可选参数。
```bash ```bash
pkgship pkginfo $packageName $database [-s] pkgship pkginfo $packageName $database [-s]
``` ```
> 参数说明： > 参数说明：
> packagename：指定要查询的软件包名，必传参数。 > packagename：指定要查询的软件包名。
> database：指定具体的数据库名称，必传参数。 > database：指定具体的数据库名称。
> >
> -s: 指定`-s`将查询的是`src`源码包信息;若未指定 默认查询`bin`二进制包信息，可选参数。 > -s: 指定`-s`将查询的是`src`源码包信息;若未指定 默认查询`bin`二进制包信息
3. 所有包查询。 3. 所有包查询。
查询数据库下包含的所有包的信息。 查询数据库下包含的所有包的信息。
> 使用场景：用户可查询指定数据库下包含的所有软件包信息。 > 使用场景：用户可查询指定数据库下包含的所有软件包信息。其中tablename为必选参数，-s为可选参数。
```bash ```bash
pkgship list $database [-s] pkgship list $database [-s]
``` ```
> 参数说明： > 参数说明：
> database：指定具体的数据库名称，必传参数。 > database：指定具体的数据库名称。
> -s: 指定`-s`将查询的是`src`源码包信息;若未指定 默认查询`bin`二进制包信息，可选参数。 > -s: 指定`-s`将查询的是`src`源码包信息;若未指定 默认查询`bin`二进制包信息
4. 安装依赖查询。 4. 安装依赖查询。
@ -318,7 +285,7 @@ pkgshipd stop 停止服务
> 参数说明： > 参数说明：
> >
> pkgName：需要查询安装的依赖的软件包名字,支持传多个；必传参数。 > pkgName：需要查询安装的依赖的二进制包名字,支持传多个；必传参数。
> >
> -dbs： 指定需要查询的database优先级,不传按照系统默认优先级搜索；可选参数。 > -dbs： 指定需要查询的database优先级,不传按照系统默认优先级搜索；可选参数。
> >
@ -329,9 +296,9 @@ pkgshipd stop 停止服务
> -w：指定-s表示引入某个二进制包的时候，查询结果会显示出该二进制包对应的源码包以及该源码包生成的所有二进制包；如果不指定-w参数表示引入某个二进制包的时候，查询结果只显示对应的源码包；可选参数。 > -w：指定-s表示引入某个二进制包的时候，查询结果会显示出该二进制包对应的源码包以及该源码包生成的所有二进制包；如果不指定-w参数表示引入某个二进制包的时候，查询结果只显示对应的源码包；可选参数。
7. 被依赖查询。 7. 被依赖查询。
查询软件包(pkgName)在某数据库(dbName)中被哪些包所依赖。 查询源码包(sourceName)在某数据库(dbName)中被哪些包所依赖。
> 使用场景：针对软件包A，在升级或删除的情况下会影响哪些软件包，可通过该命令查询。该命令会显示源码包A(若为源码包)生成的所有二进制包（若输入为二进制包，那此处即为输入的二进制包）被哪些源码包（比如B）编译依赖，被哪些二进制包（比如C1）安装依赖；以及B生成的二进制包及C1被哪些源码包（比如D）编译依赖，被哪些二进制包（比如E1）安装依赖，以此类推，遍历这些二进制包的被依赖。 > 使用场景：针对软件源码包A，在升级或删除的情况下会影响哪些软件包，可通过该命令查询。该命令会显示源码包A生成的所有二进制包被哪些源码包（比如B）编译依赖，被哪些二进制包（比如C1）安装依赖；以及B生成的二进制包及C1被哪些源码包（比如D）编译依赖，被哪些二进制包（比如E1）安装依赖，以此类推，遍历这些二进制包的被依赖。
```bash ```bash
pkgship bedepend dbName [$pkgName1 $pkgName2 $pkgName3] [-w] [-b] [-install/build] pkgship bedepend dbName [$pkgName1 $pkgName2 $pkgName3] [-w] [-b] [-install/build]
@ -339,11 +306,9 @@ pkgshipd stop 停止服务
> 参数说明: > 参数说明:
> >
> dbName：需要查询依赖关系的仓库，不支持多个；必选参数。 > dbName：需要查询依赖关系的仓库，不支持多个；必选参数。
> >
> pkgName：待查询的软件包名称，支持多个；必选参数。 > -w ：当不指定-w 时，查询结果默认不包含对应二进制包的子包；当命令后指定配置参数[-w] 时，不仅会查询二进制包C1的被依赖关系，还会进一步去查询C1对应的源码包C生成的其他二进制包（比如：C2,C3）的被依赖关系；可选参数。
>
> -w ：当不指定-w 时，查询结果默认不包含对应源码包的子包；当命令后指定配置参数[-w] 时，不仅会查询二进制包C1的被依赖关系，还会进一步去查询C1对应的源码包C生成的其他二进制包（比如：C2,C3）的被依赖关系；可选参数。
> >
> -b：指定`-b`表示查询的包是二进制,默认查询源码包；可选参数。 > -b：指定`-b`表示查询的包是二进制,默认查询源码包；可选参数。
> >
@ -353,7 +318,7 @@ pkgshipd stop 停止服务
> 使用场景，查看Elasticsearch中初始化了哪些数据库，该功能会按照优先级顺序返回已经初始化的数据库列表。 > 使用场景，查看Elasticsearch中初始化了哪些数据库，该功能会按照优先级顺序返回已经初始化的数据库列表。
`pkgship dbs` `pkgship db`
9. 获取版本号。 9. 获取版本号。
@ -361,99 +326,3 @@ pkgshipd stop 停止服务
`pkgship -v` `pkgship -v`
10. 比较不同数据库的包信息的依赖关系。
查询输入数据库各自的所有源码包/二进制包信息，以及他们的一层编译依赖/安装依赖，然后以第一个数据库包信息为基准对比各个数据库依赖信息的差异并记录到csv文件中。
> 使用场景：比较各个系统依赖关系的差异，分析软件包依赖关系优化点。
```shell
pkgship compare -t build/install -dbs [database1 database2..] [-o out_path]
```
> 参数说明：
>
> -t: 查询的依赖类型，目前只支持编译依赖-build和安装依赖-install。
>
> -dbs: 查询的数据库列表，数据库之间用空格分隔。为了控制执行时间，目前最多支持4个数据库，对比时以第一个数据库的软件包作为对比基准包集合。
>
> -o: csv文件的保存路径。不填默认为/opt/pkgship/compare ，需要指定时要确保执行用户pkgshipuser有写入权限。最后csv文件保存在“${out_path}/时间戳” 路径下。csv文件为各个数据库的包依赖关系信息（以数据库名称命名）以及所有数据库的包依赖关系对比信息（compare.csv）。
## 日志查看和转储
**日志查看**
pkgship服务在运行时会产生两种日志，业务日志和操作日志。
1、业务日志:
路径：/var/log/pkgship/log_info.log（支持在conf.yaml中通过log_path字段自定义路径）。
功能：主要记录代码内部运行的日志，方便问题定位。
权限：路径权限755，日志文件权限644，普通用户可以查看。
2、操作日志：
路径：/var/log/pkgship-operation/uwsgi.log （支持在conf.yaml中通过daemonize字段自定义路径）。
功能：记录使用者操作信息，包括ip，访问时间，访问url，访问结果等，方便后续查阅以及记录攻击者信息。
权限：路径权限700，日志文件权限644，只有root和pkgshipuser可以查看。
**日志转储**
1、业务日志转储：
- 转储机制
使用python自带的logging内置函数的转储机制，按照日志大小来备份。
> 配置项，package.ini中配置每个日志的容量和备份数量
>
> ```ini
> ; Maximum capacity of each file, the unit is byte, default is 30M
> max_bytes=31457280
>
> ; Number of old logs to keep;default is 30
> backup_count=30
> ```
- 转储过程
当某次日志写入后，日志文件大小超过配置的日志容量时，会自动压缩转储，压缩后文件名为log_info.log.x.gz， x是数字，数字越小为越新的备份。
当备份日志数量到达配置的备份数量之后，最早的备份日志会被删除掉，然后备份一个最新的压缩日志文件。
2、操作日志转储：
- 转储机制
使用脚本进行转储，按照时间转储，每日转储一次，共保留30天，不支持自定义配置。
> 脚本位置：/etc/pkgship/uwsgi_logrotate.sh
- 转储过程
pkgship启动时转储脚本后台运行，从启动时，每隔1天进行转储压缩，共保留30份压缩文件，压缩文件名称为uwsgi.log-20201010x.zip， x为压缩时的小时数。
pkgship停止后转储脚本停止，不再进行转储，再次启动时，转储脚本重新执行。
## 扩展工具pkgship-panel
### 介绍
pkgship-panel旨在将软件包构建信息和维护信息集成到一起，方便版本维护人员可以快速识别构建异常软件包并快速邮件通知相关责任人去解决，保证构建工程稳定性，提高IOS构建成功率。
### 架构
![输入图片说明](https://gitee.com/openeuler/pkgship/raw/master/doc/design/pkgimg/panel逻辑视图.png)
### 工具使用
由于工具数据源不可配置，所以建议直接使用pkgship-panel官网地址：https://pkgmanage.openeuler.org/Infomanagement

BIN
pkgship-2.2.0.tar.gz Normal file
View File

Binary file not shown.

BIN
pkgship-3.0.0.tar.gz
View File

Binary file not shown.

157
pkgship.spec Normal file → Executable file
View File

@ -1,54 +1,40 @@
Name: pkgship Name: pkgship
Version: 3.0.0 Version: 2.2.0
Release: 2 Release: 6
Summary: Pkgship implements rpm package dependence ,maintainer, patch query and so on. Summary: Pkgship implements rpm package dependence ,maintainer, patch query and so on.
License: Mulan 2.0 License: Mulan 2.0
URL: https://gitee.com/openeuler/pkgship URL: https://gitee.com/openeuler/pkgship
Source0: https://gitee.com/src-openeuler/pkgship-%{version}.tar.gz Source0: https://gitee.com/openeuler/pkgship-%{version}.tar.gz
Patch0001: 0001-bugfix-pkginfo.patch
Patch0002: 0002-fix-install-and-start-script-info.patch
Patch0003: 0003-bugfix-binary-name-mapping-source.patch
Patch0004: 0004-fix-depend-query.patch
Patch0005: 0005-update-copyright.patch
BuildArch: noarch BuildArch: noarch
BuildRequires: shadow python3-mock python3-lxml BuildRequires: shadow python3-mock
BuildRequires: python3-flask-restful python3-flask python3 python3-pyyaml python3-redis BuildRequires: python3-flask-restful python3-flask python3 python3-pyyaml python3-redis
BuildRequires: python3-prettytable python3-requests python3-retrying python3-coverage BuildRequires: python3-prettytable python3-requests python3-retrying python3-coverage
BuildRequires: python3-marshmallow python3-uWSGI python3-gevent python3-Flask-Limiter BuildRequires: python3-marshmallow python3-uWSGI python3-gevent python3-Flask-Limiter
BuildRequires: python3-elasticsearch python3-concurrent-log-handler python3-pyrpm python3-aiohttp BuildRequires: python3-elasticsearch python3-concurrent-log-handler
Requires: shadow python3-mock
Requires: shadow python3-mock python3-lxml
Requires: python3-flask-restful python3-flask python3 python3-pyyaml python3-redis Requires: python3-flask-restful python3-flask python3 python3-pyyaml python3-redis
Requires: python3-prettytable python3-requests python3-retrying python3-coverage Requires: python3-prettytable python3-requests python3-retrying python3-coverage
Requires: python3-marshmallow python3-uWSGI python3-gevent python3-Flask-Limiter Requires: python3-marshmallow python3-uWSGI python3-gevent python3-Flask-Limiter
Requires: python3-elasticsearch python3-concurrent-log-handler python3-pyrpm python3-aiohttp Requires: python3-elasticsearch python3-concurrent-log-handler
%description %description
Pkgship implements rpm package dependence ,maintainer, patch query and so no. Pkgship implements rpm package dependence ,maintainer, patch query and so no.
%package -n pkgship-tools
Summary: Package Validation Tool
Requires: pkgship
%description -n pkgship-tools
Packages automatically analyze dependencies, create containers,
perform compilation and installation operations, and perform simple functional verification
%package -n pkgship-panel
Summary: openEuler Data panel
BuildRequires: pkgship python3-APScheduler
Requires: pkgship python3-APScheduler
%description -n pkgship-panel
A Kanban board that can view package compilation status,
package maintenance information, and sig group information
%prep %prep
%autosetup -n pkgship-%{version} -p1 %autosetup -n pkgship-%{version} -p1
%build %build
# pkgship build
pushd packageship
%py3_build %py3_build
## create version file current_path=`pwd`
cd $current_path'/packageship'
version_=%{version} version_=%{version}
release_=%{release} release_=%{release}
version_file=version.yaml version_file=version.yaml
@ -59,101 +45,76 @@ touch $version_file
echo "create version.yaml successfully." echo "create version.yaml successfully."
echo "Version: $version_" >> $version_file echo "Version: $version_" >> $version_file
echo "Release: $release_" >> $version_file echo "Release: $release_" >> $version_file
popd
# pkgship-panel build
pushd packageship_panel
%py3_build
popd
%install %install
# pkgship install
pushd packageship
%py3_install %py3_install
popd
# pkgship-panel install
pushd packageship_panel
%py3_install
popd
# install pkgship-tools
mkdir -p %{buildroot}/opt/pkgship/tools/
cp -r pkgship-tools/* %{buildroot}/opt/pkgship/tools/
%check %check
# pkgship check
current_path=`pwd` current_path=`pwd`
log_path=${current_path}/packageship/tmp/ log_path=${current_path}/packageship/tmp/
sed -i '/^LOG_PATH/d' ./packageship/packageship/libs/conf/global_config.py sed -i '/^LOG_PATH/d' ./packageship/libs/conf/global_config.py
echo "LOG_PATH=r\"${log_path}\"" >> ./packageship/packageship/libs/conf/global_config.py echo "LOG_PATH=r\"${log_path}\"" >> ./packageship/libs/conf/global_config.py
sed -i "/owner=('pkgshipuser', 'pkgshipuser')/d" ./packageship/packageship/libs/log.py
%{__python3} test/coverage_count.py %{__python3} test/coverage_count.py
%pre %pre
# add user and group
user=pkgshipuser user=pkgshipuser
group=pkgshipuser group=pkgshipuser
groupadd $group 2>/dev/null # create group if not exists
useradd -g $group $user 2>/dev/null egrep -w "^$group" /etc/group >& /dev/null
if [ $? -ne 0 ]
then
groupadd $group
fi
# create user if not exists
egrep -w "^$user" /etc/passwd >& /dev/null
if [ $? -ne 0 ]
then
useradd -g $group $user
fi
# create dir or file if not exists
function create_dir_file(){
if [ $3 = "d" ];then
if [ ! -d "$1" ];then
mkdir -p -m $2 $1
fi
elif [ $3 = "f" ];then
if [ -f $1 ];then
rm -rf $1
fi
touch $1
chmod $2 $1
fi
chown -R $user:$group $1
}
create_dir_file /opt/pkgship/ 750 d
create_dir_file /opt/pkgship/compare 755 d
create_dir_file /var/log/pkgship 755 d
create_dir_file /var/log/pkgship-operation 700 d
%post
%postun
mkdir -p /opt/pkgship/ 750
chown -R $user:$group /opt/pkgship/
mkdir -p /opt/pkgship/compare 755
chown -R $user:$group /opt/pkgship/compare
mkdir -p /opt/pkgship/tools 755
chown -R $user:$group /opt/pkgship/tools
mkdir -p /var/log/pkgship 755
chown -R $user:$group /var/log/pkgship
mkdir -p /var/log/pkgship-operation 700
chown -R $user:$group /var/log/pkgship-operation
%files %files
%doc README.md %doc README.md
%attr(0755,pkgshipuser,pkgshipuser) %{python3_sitelib}/packageship-*egg-info %attr(0755,pkgshipuser,pkgshipuser) %{python3_sitelib}/*
%attr(0755,pkgshipuser,pkgshipuser) %{python3_sitelib}/packageship/* %attr(0755,pkgshipuser,pkgshipuser) %config %{_sysconfdir}/pkgship/*
%attr(0755,pkgshipuser,pkgshipuser) %{_bindir}/pkgshipd %attr(0755,pkgshipuser,pkgshipuser) %{_bindir}/pkgshipd
%attr(0755,pkgshipuser,pkgshipuser) %{_bindir}/pkgship %attr(0755,pkgshipuser,pkgshipuser) %{_bindir}/pkgship
%attr(0750,root,root) /etc/pkgship/auto_install_pkgship_requires.sh %attr(0750,root,root) /etc/pkgship/auto_install_pkgship_requires.sh
%attr(0750,pkgshipuser,pkgshipuser) /etc/pkgship/uwsgi_logrotate.sh %attr(0750,pkgshipuser,pkgshipuser) /etc/pkgship/uwsgi_logrotate.sh
%attr(0750,pkgshipuser,pkgshipuser) /etc/pkgship/timer_sync
%attr(0640,pkgshipuser,pkgshipuser) /etc/pkgship/package.ini %attr(0640,pkgshipuser,pkgshipuser) /etc/pkgship/package.ini
%attr(0644,pkgshipuser,pkgshipuser) /etc/pkgship/conf.yaml %attr(0644,pkgshipuser,pkgshipuser) /etc/pkgship/conf.yaml
%attr(0644,pkgshipuser,pkgshipuser) /lib/systemd/system/pkgship.service %attr(0640,pkgshipuser,pkgshipuser) /lib/systemd/system/pkgship.service
# The file list of the package pkgship-tools
%files -n pkgship-tools
%attr(0755,root,root) /opt/pkgship/tools/*
# The file list of the package pkgship-panel
%files -n pkgship-panel
%attr(0755,pkgshipuser,pkgshipuser) %{python3_sitelib}/packageship_panel-*.egg-info
%attr(0755,pkgshipuser,pkgshipuser) %{python3_sitelib}/packageship_panel/*
%attr(0755,pkgshipuser,pkgshipuser) %{_bindir}/pkgship-paneld
%attr(0755,pkgshipuser,pkgshipuser) %{_bindir}/pkgship-panel
%attr(0644,pkgshipuser,pkgshipuser) /lib/systemd/system/pkgship-panel.service
%attr(0644,pkgshipuser,pkgshipuser) /etc/pkgship/timed_task.yaml
%attr(0755,pkgshipuser,pkgshipuser) /etc/pkgship/service-monitor.sh
%changelog %changelog
* Mon Jul 10 2023 gongzhengtang <gong_zhengtang@163.com> - 3.0.0-2
- Use case fails to adapt flask-limter upgrade
* Fri Feb 3 2023 liheavy <lihaiwei8@huawei.com> - 3.0.0-1
- add panel data synchronization
- Add dynamic version data update and search by multiple criteria
* Sun Jan 9 2022 Haiwei Li <lihaiwei8@huawei.com> - 2.2.0-9
- TerminalTable class Use the deprecated method of PrettyTable, resulting in compilation failure, delete the file.
* Thu Nov 18 2021 Haiwei Li <lihaiwei8@huawei.com> - 2.2.0-8
- Solve the problem of patch application failure due to the file encoding format being crlf.
- Solve the problem of json serialization caused by flask's new dependency on simple-json.
* Sun Sep 26 2021 Haiwei Li <lihaiwei8@huawei.com> - 2.2.0-7
- Due to a problem with docker, when pkgship is used in docker,
- it must be created with the --privileged parameter, otherwise it will fail. So update the instructions.
* Mon Sep 6 2021 Haiwei Li <lihaiwei8@huawei.com> - 2.2.0-6 * Mon Sep 6 2021 Haiwei Li <lihaiwei8@huawei.com> - 2.2.0-6
- Update the date in copyright. - Update the date in copyright.
@ -426,4 +387,4 @@ chown -R $user:$group /var/log/pkgship-operation
- add macro to build cli bin when rpm install - add macro to build cli bin when rpm install
* Sat Jun 6 2020 Feng Hu <solar.hu@foxmail.com> - 1.0-0 * Sat Jun 6 2020 Feng Hu <solar.hu@foxmail.com> - 1.0-0
- init package - init package