diff --git a/pyporter b/pyporter new file mode 100644 index 0000000..39119a5 --- /dev/null +++ b/pyporter @@ -0,0 +1,556 @@ +#!/usr/bin/python3 +""" +This is a packager bot for python modules from pypi.org +""" +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: Shinwell_Hu Myeuler +# Create: 2020-05-07 +# Description: provide a tool to package python module automatically +# ******************************************************************************/ + +import urllib +import urllib.request +from pprint import pprint +from os import path +import json +import sys +import re +import datetime +import argparse +import subprocess +import os +import platform +from pathlib import Path +# python3-wget is not default available on openEuler yet. +# import wget + + +json_file_template = '{pkg_name}.json' +name_tag_template = 'Name:\t\t{pkg_name}' +summary_tag_template = 'Summary:\t{pkg_sum}' +version_tag_template = 'Version:\t{pkg_ver}' +release_tag_template = 'Release:\t1' +license_tag_template = 'License:\t{pkg_lic}' +home_tag_template = 'URL:\t\t{pkg_home}' +source_tag_template = 'Source0:\t{pkg_source}' + +buildreq_tag_template = 'BuildRequires:\t{req}' + +# TODO List +# 1. Need a reliable way to get description of module .. Partially done +# 2. requires_dist has some dependency restirction, need to present +# 3. dependency outside python (i.e. pycurl depends on libcurl) doesn't exist in pipy + + + +class PyPorter: + __url_template = 'https://pypi.org/pypi/{pkg_name}/json' + __build_noarch = True + __json = None + __module_name = "" + __spec_name = "" + __pkg_name = "" + + def __init__(self, pkg): + """ + receive json from pypi.org + """ + url = self.__url_template.format(pkg_name=pkg) + resp = "" + with urllib.request.urlopen(url) as u: + self.__json = json.loads(u.read().decode('utf-8')) + if (self.__json is not None): + self.__module_name = self.__json["info"]["name"] + self.__spec_name = "python-" + self.__module_name + self.__pkg_name = "python3-" + self.__module_name + self.__build_noarch = self.__get_buildarch() + + + def get_spec_name(self): + return self.__spec_name + + def get_module_name(self): + return self.__module_name + + def get_pkg_name(self): + return self.__pkg_name + + def get_version(self): + return self.__json["info"]["version"] + + def get_summary(self): + return self.__json["info"]["summary"] + + def get_home(self): + return self.__json["info"]["project_urls"]["Homepage"] + + def get_license(self): + """ + By default, the license info can be achieved from json["info"]["license"] + in rare cases it doesn't work. + We fall back to json["info"]["classifiers"], it looks like License :: OSI Approved :: BSD Clause + """ + if self.__json["info"]["license"] != "": + return self.__json["info"]["license"] + for k in self.__json["info"]["classifiers"]: + if k.startswith("License"): + ks = k.split("::") + return ks[2].strip() + return "" + + def get_source_url(self): + """ + return URL for source file for the latest version + return "" in errors + """ + v = self.__json["info"]["version"] + rs = self.__json["releases"][v] + for r in rs: + if r["packagetype"] == "sdist": + return r["url"] + return "" + + def get_requires(self): + """ + return all requires no matter if extra is required. + """ + rs = self.__json["info"]["requires_dist"] + if rs is None: + return + for r in rs: + idx = r.find(";") + mod = transform_module_name(r[:idx]) + print("Requires:\t" + mod) + if self.__json["info"]["license"] != "": + return self.__json["info"]["license"] + for k in self.__json["info"]["classifiers"]: + if k.startswith("License"): + ks = k.split("::") + return ks[2].strip() + return "" + + def __get_buildarch(self): + """ + if this module has a prebuild package for amd64, then it is arch dependent. + print BuildArch tag if needed. + """ + v = self.__json["info"]["version"] + rs = self.__json["releases"][v] + for r in rs: + if r["packagetype"] == "bdist_wheel": + if r["url"].find("amd64") != -1: + return False + return True + + def is_build_noarch(self): + return self.__build_noarch + + def get_buildarch(self): + if (self.__build_noarch == True): + print("BuildArch:\tnoarch") + + def get_description(self): + """ + return description. + Usually it's json["info"]["description"] + If it's rst style, then only use the content for the first paragraph, and remove all tag line. + For empty description, use summary instead. + """ + desc = self.__json["info"]["description"].splitlines() + res = [] + paragraph = 0 + for d in desc: + if len(d.strip()) == 0: + continue + first_char = d.strip()[0] + ignore_line = False + if d.strip().startswith("===") or d.strip().startswith("---"): + paragraph = paragraph + 1 + ignore_line = True + elif d.strip().startswith(":") or d.strip().startswith(".."): + ignore_line = True + if ignore_line != True and paragraph == 1: + res.append(d) + if paragraph >= 2: + del res[-1] + return "\n".join(res) + if res != []: + return "\n".join(res) + elif paragraph == 0: + return self.__json["info"]["description"] + else: + return self.__json["info"]["summary"] + + def get_build_requires(self): + req_list=[] + rds = self.__json["info"]["requires_dist"] + if rds is not None: + for rp in rds: + br = refine_requires(rp) + if (br == ""): + continue + # + # Do not output BuildRequires: + # just collect all build requires and using pip to install + # than can help to build all rpm withoud trap into + # build dependency nightmare + # + #print(buildreq_tag_template.format(req=br)) + name=str.lstrip(br).split(" ") + req_list.append(name[0]) + return req_list + + def prepare_build_requires(self): + print(buildreq_tag_template.format(req='python3-devel')) + print(buildreq_tag_template.format(req='python3-setuptools')) + if (self.__build_noarch == False): + print(buildreq_tag_template.format(req='python3-cffi')) + print(buildreq_tag_template.format(req='gcc')) + print(buildreq_tag_template.format(req='gdb')) + + def prepare_pkg_build(self): + print("%py3_build") + def prepare_pkg_install(self): + print("%py3_install") + def prepare_pkg_files(self): + if self.__build_noarch: + print("%dir %{python3_sitelib}/*") + else: + print("%dir %{python3_sitearch}/*") + + def store_json(self, spath): + """ + save json file + """ + fname = json_file_template.format(pkg_name=self.__pkg_name) + json_file = os.path.join(spath, fname) + + # if file exist, do nothing + if path.exists(json_file) and path.isfile(json_file): + with open(json_file, 'r') as f: + resp = json.load(f) + else: + with open(json_file, 'w') as f: + json.dump(self.__json, f) + + +def transform_module_name(n): + """ + return module name with version restriction. + Any string with '.' or '/' is considered file, and will be ignored + Modules start with python- will be changed to python3- for consistency. + """ + # remove () + ns = re.split("[()]", n) + ver_constrain = [] + ns[0] = ns[0].strip() + if ns[0].startswith("python-"): + ns[0] = ns[0].replace("python-", "python3-") + else: + ns[0] = "python3-" + ns[0] + if ns[0].find("/") != -1 or ns[0].find(".") != -1: + return "" + """ + if len(ns) > 1: + vers = ns[1].split(",") + for ver in vers: + m = re.match("([<>=]+)( *)(\d.*)", ver.strip()) + ver_constrain.append(ns[0] + " " + m[1] + " " + m[3]) + return ", ".join(ver_constrain) + else: + """ + return ns[0] + + +def refine_requires(req): + """ + return only requires without ';' (thus no extra) + """ + ra = req.split(";", 1) + # + # Do not add requires which has ;, which is often has very complicated precondition + # TODO: need more parsing of the denpency after ; + return transform_module_name(ra[0]) + + +def download_source(porter, tgtpath): + """ + download source file from url, and save it to target path + """ + if (os.path.exists(tgtpath) == False): + print("download path %s does not exist\n", tgtpath) + return False + s_url = porter.get_source_url() + return subprocess.call(["wget", s_url, "-P", tgtpath]) + + +def prepare_rpm_build_env(root): + """ + prepare environment for rpmbuild + """ + if (os.path.exists(root) == False): + print("Root path %s does not exist\n" & buildroot) + return "" + + buildroot = os.path.join(root, "rpmbuild") + if (os.path.exists(buildroot) == False): + os.mkdir(buildroot) + + for sdir in ['SPECS', 'BUILD', 'SOURCES', 'SRPMS', 'RPMS', 'BUILDROOT']: + bpath = os.path.join(buildroot, sdir) + if (os.path.exists(bpath) == False): + os.mkdir(bpath) + + return buildroot + + +def try_pip_install_package(pkg): + """ + install packages listed in build requires + """ + # try pip installation + pip_name = pkg.split("-") + if len(pip_name) == 2: + ret = subprocess.call(["pip3", "install", "--user", pip_name[1]]) + else: + ret = subprocess.call(["pip3", "install", "--user", pip_name[0]]) + + if ret != 0: + print("%s can not be installed correctly, Fix it later, go ahead to do building..." % pip_name) + + # + # TODO: try to build anyway, fix it later + # + return True + +def package_installed(pkg): + print(pkg) + ret = subprocess.call(["rpm", "-qi", pkg]) + if ret == 0: + return True + + return False + + +def dependencies_ready(req_list): + """ + TODO: do not need to do dependency check here, do it in pyporter_run + """ + # if (try_pip_install_package(req) == False): + # return req + return "" + +def build_package(specfile): + """ + build rpm package with rpmbuild + """ + ret = subprocess.call(["rpmbuild", "-ba", specfile]) + return ret + + +def build_install_rpm(porter, rootpath): + ret = build_rpm(porter, rootpath) + if (ret != ""): + return ret + + arch = "noarch" + if (porter.is_build_noarch()): + arch = "noarch" + else: + arch = platform.machine() + + pkgname = os.path.join(rootpath, "rpmbuild", "RPMS", arch, porter.get_pkg_name() + "*") + ret = subprocess.call(["rpm", "-ivh", pkgname]) + if (ret != 0): + return "Install failed\n" + + return "" + +def build_rpm(porter, rootpath): + """ + full process to build rpm + """ + buildroot = prepare_rpm_build_env(rootpath) + if (buildroot == ""): + return False + + specfile = os.path.join(buildroot, "SPECS", porter.get_spec_name() + ".spec") + + req_list = build_spec(porter, specfile) + ret = dependencies_ready(req_list) + if ret != "": + print("%s can not be installed automatically, Please handle it" % ret) + return ret + + download_source(porter, os.path.join(buildroot, "SOURCES")) + + build_package(specfile) + + return "" + + +def build_spec(porter, output): + """ + print out the spec file + """ + if os.path.isdir(output): + output = os.path.join(output, porter.get_spec_name() + ".spec") + tmp = sys.stdout + if (output != ""): + sys.stdout = open(output, 'w+') + + print("%global _empty_manifest_terminate_build 0") + print(name_tag_template.format(pkg_name=porter.get_spec_name())) + print(version_tag_template.format(pkg_ver=porter.get_version())) + print(release_tag_template) + print(summary_tag_template.format(pkg_sum=porter.get_summary())) + print(license_tag_template.format(pkg_lic=porter.get_license())) + print(home_tag_template.format(pkg_home=porter.get_home())) + print(source_tag_template.format(pkg_source=porter.get_source_url())) + porter.get_buildarch() + print("") + porter.get_requires() + print("") + print("%description") + print(porter.get_description()) + print("") + + print("%package -n {name}".format(name=porter.get_pkg_name())) + print(summary_tag_template.format(pkg_sum=porter.get_summary())) + print("Provides:\t" + porter.get_spec_name()) + + porter.prepare_build_requires() + + build_req_list=porter.get_build_requires() + + print("%description -n " + porter.get_pkg_name()) + print(porter.get_description()) + print("") + print("%package help") + print("Summary:\tDevelopment documents and examples for {name}".format(name=porter.get_module_name())) + print("Provides:\t{name}-doc".format(name=porter.get_pkg_name())) + print("%description help") + print(porter.get_description()) + print("") + print("%prep") + print("%autosetup -n {name}-{ver}".format(name=porter.get_module_name(), ver=porter.get_version())) + print("") + print("%build") + porter.prepare_pkg_build() + print("") + print("%install") + porter.prepare_pkg_install() + print("install -d -m755 %{buildroot}/%{_pkgdocdir}") + print("if [ -d doc ]; then cp -arf doc %{buildroot}/%{_pkgdocdir}; fi") + print("if [ -d docs ]; then cp -arf docs %{buildroot}/%{_pkgdocdir}; fi") + print("if [ -d example ]; then cp -arf example %{buildroot}/%{_pkgdocdir}; fi") + print("if [ -d examples ]; then cp -arf examples %{buildroot}/%{_pkgdocdir}; fi") + print("pushd %{buildroot}") + print("if [ -d usr/lib ]; then") + print("\tfind usr/lib -type f -printf \"/%h/%f\\n\" >> filelist.lst") + print("fi") + print("if [ -d usr/lib64 ]; then") + print("\tfind usr/lib64 -type f -printf \"/%h/%f\\n\" >> filelist.lst") + print("fi") + print("if [ -d usr/bin ]; then") + print("\tfind usr/bin -type f -printf \"/%h/%f\\n\" >> filelist.lst") + print("fi") + print("if [ -d usr/sbin ]; then") + print("\tfind usr/sbin -type f -printf \"/%h/%f\\n\" >> filelist.lst") + print("fi") + print("touch doclist.lst") + print("if [ -d usr/share/man ]; then") + print("\tfind usr/share/man -type f -printf \"/%h/%f.gz\\n\" >> doclist.lst") + print("fi") + print("popd") + print("mv %{buildroot}/filelist.lst .") + print("mv %{buildroot}/doclist.lst .") + print("") + print("%files -n {name} -f filelist.lst".format(name=porter.get_pkg_name())) + + porter.prepare_pkg_files() + + print("") + print("%files help -f doclist.lst") + print("%{_pkgdocdir}") + print("") + print("%changelog") + date_str = datetime.date.today().strftime("%a %b %d %Y") + print("* {today} Python_Bot ".format(today=date_str)) + print("- Package Spec generated") + + sys.stdout = tmp + + return build_req_list + + +def do_args(root): + parser = argparse.ArgumentParser() + + parser.add_argument("-s", "--spec", help="Create spec file", action="store_true") + parser.add_argument("-R", "--requires", help="Get required python modules", action="store_true") + parser.add_argument("-b", "--build", help="Build rpm package", action="store_true") + parser.add_argument("-B", "--buildinstall", help="Build&Install rpm package", action="store_true") + parser.add_argument("-r", "--rootpath", help="Build rpm package in root path", type=str, default=dft_root_path) + parser.add_argument("-d", "--download", help="Download source file indicated path", action="store_true") + parser.add_argument("-p", "--path", help="indicated path to store files", type=str, default=os.getcwd()) + parser.add_argument("-j", "--json", help="Get Package JSON info", action="store_true") + parser.add_argument("-o", "--output", help="Output to file", type=str, default="") + parser.add_argument("-t", "--type", help="Build module type : python, perl...", type=str, default="python") + parser.add_argument("pkg", type=str, help="The Python Module Name") + + return parser + +def porter_creator(t_str, pkg): + if (t_str == "python"): + return PyPorter(pkg) + + return None + + + +if __name__ == "__main__": + + dft_root_path=os.path.join(str(Path.home())) + + parser = do_args(dft_root_path) + + args = parser.parse_args() + + porter = porter_creator(args.type, args.pkg) + if (porter is None): + print("Type %s is not supported now\n" % args.type) + sys.exit(1) + + + if (args.requires): + reqlist = porter.get_build_requires() + if reqlist is not None: + for req in reqlist: + print(req) + elif (args.spec): + build_spec(porter, args.output) + elif (args.build): + ret = build_rpm(porter, args.rootpath) + if ret != "": + print("build failed : BuildRequire : %s\n" % ret) + sys.exit(1) + elif (args.buildinstall): + ret = build_install_rpm(porter, args.rootpath) + if ret != "": + print("Build & install failed\n") + sys.exit(1) + elif (args.download): + download_source(porter, args.path) + elif (args.json): + porter.store_json(args.path) + diff --git a/python-uwsgi.spec b/python-uwsgi.spec new file mode 100644 index 0000000..0227856 --- /dev/null +++ b/python-uwsgi.spec @@ -0,0 +1,71 @@ +%global _empty_manifest_terminate_build 0 +Name: python-uWSGI +Version: 2.0.19.1 +Release: 1 +Summary: The uWSGI server +License: GPL2 +URL: https://uwsgi-docs.readthedocs.io/en/latest/ +Source0: https://files.pythonhosted.org/packages/c7/75/45234f7b441c59b1eefd31ba3d1041a7e3c89602af24488e2a22e11e7259/uWSGI-2.0.19.1.tar.gz +BuildArch: noarch + + +%description + + +%package -n python3-uWSGI +Summary: The uWSGI server +Provides: python-uWSGI +BuildRequires: python3-devel +BuildRequires: python3-setuptools +%description -n python3-uWSGI + + +%package help +Summary: Development documents and examples for uWSGI +Provides: python3-uWSGI-doc +%description help + + +%prep +%autosetup -c uwsgi-%{version} + +%build +%py3_build + +%install +%py3_install +install -d -m755 %{buildroot}/%{_pkgdocdir} +if [ -d doc ]; then cp -arf doc %{buildroot}/%{_pkgdocdir}; fi +if [ -d docs ]; then cp -arf docs %{buildroot}/%{_pkgdocdir}; fi +if [ -d example ]; then cp -arf example %{buildroot}/%{_pkgdocdir}; fi +if [ -d examples ]; then cp -arf examples %{buildroot}/%{_pkgdocdir}; fi +pushd %{buildroot} +if [ -d usr/lib ]; then + find usr/lib -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/lib64 ]; then + find usr/lib64 -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/bin ]; then + find usr/bin -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/sbin ]; then + find usr/sbin -type f -printf "/%h/%f\n" >> filelist.lst +fi +touch doclist.lst +if [ -d usr/share/man ]; then + find usr/share/man -type f -printf "/%h/%f.gz\n" >> doclist.lst +fi +popd +mv %{buildroot}/filelist.lst . +mv %{buildroot}/doclist.lst . + +%files -n python3-uWSGI -f filelist.lst +%dir %{python3_sitelib}/* + +%files help -f doclist.lst +%{_pkgdocdir} + +%changelog +* Wed Aug 05 2020 Python_Bot +- Package Spec generated diff --git a/python-uwsgi.yaml b/python-uwsgi.yaml new file mode 100644 index 0000000..0a5e5a6 --- /dev/null +++ b/python-uwsgi.yaml @@ -0,0 +1,4 @@ +version_control: pypi +src_repo: uWSGI +tag_prefix: "v" +seperator: "" diff --git a/uWSGI-2.0.19.1.tar.gz b/uWSGI-2.0.19.1.tar.gz new file mode 100644 index 0000000..39027e9 Binary files /dev/null and b/uWSGI-2.0.19.1.tar.gz differ