diff --git a/0001-urlgrabber-ext-down-another-python-3-compat.patch b/0001-urlgrabber-ext-down-another-python-3-compat.patch new file mode 100644 index 0000000..c3d3470 --- /dev/null +++ b/0001-urlgrabber-ext-down-another-python-3-compat.patch @@ -0,0 +1,48 @@ +From e7a01334f653661c657d4c4e6e1ad10aababfb5b Mon Sep 17 00:00:00 2001 +From: Pavel Raiskup +Date: Sun, 12 May 2019 08:54:01 +0200 +Subject: [PATCH] urlgrabber-ext-down: another python 3 compat + +Expect that _readlines() returns array of bytes objects in +Python 3 environments. + +Fixes rhbz #1707657 and #1688173 +--- + scripts/urlgrabber-ext-down | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +diff --git a/scripts/urlgrabber-ext-down b/scripts/urlgrabber-ext-down +index bbaebd5..13d6dc7 100755 +--- a/scripts/urlgrabber-ext-down ++++ b/scripts/urlgrabber-ext-down +@@ -19,12 +19,17 @@ + # Boston, MA 02111-1307 USA + + import time, os, errno, sys ++import six + from urlgrabber.grabber import \ + _readlines, URLGrabberOptions, _loads, \ + PyCurlFileObject, URLGrabError + + def write(fmt, *arg): +- try: os.write(1, fmt % arg) ++ buf = fmt % arg ++ if six.PY3: ++ buf = buf.encode() ++ try: ++ os.write(1, buf) + except OSError as e: + if e.args[0] != errno.EPIPE: raise + sys.exit(1) +@@ -46,6 +51,8 @@ def main(): + lines = _readlines(0) + if not lines: break + for line in lines: ++ if not isinstance(line, six.string_types): ++ line = line.decode('utf-8') + cnt += 1 + opts = URLGrabberOptions() + opts._id = cnt +-- +2.21.0 + diff --git a/0002-Revert-Simplify-mirror-conversion-to-utf8.patch b/0002-Revert-Simplify-mirror-conversion-to-utf8.patch new file mode 100644 index 0000000..464c14d --- /dev/null +++ b/0002-Revert-Simplify-mirror-conversion-to-utf8.patch @@ -0,0 +1,40 @@ +From 7607b9f408b71b6533ca4f8e8808090a5b930555 Mon Sep 17 00:00:00 2001 +From: Michal Domonkos +Date: Mon, 20 May 2019 15:06:38 +0200 +Subject: [PATCH 2/4] Revert "Simplify mirror conversion to utf8" + +This reverts commit be8ee10e35319e80200d4ff384434d46fe7783d9. + +A list of dicts (as opposed to strings) is valid input as well; see the +module-level doc string for details (section 2 under CUSTOMIZATION). In +fact, the nested estimate() function in MirrorGroup.__init__() accounts +for that, too. + +This fixes a traceback in YUM which does pass such a dict list. + +Closes #10. +--- + urlgrabber/mirror.py | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/urlgrabber/mirror.py b/urlgrabber/mirror.py +index 75f0bcb..d95863e 100644 +--- a/urlgrabber/mirror.py ++++ b/urlgrabber/mirror.py +@@ -297,7 +297,12 @@ class MirrorGroup: + self.default_action = kwargs.get('default_action') + + def _parse_mirrors(self, mirrors): +- return [{'mirror':_to_utf8(m)} for m in mirrors] ++ parsed_mirrors = [] ++ for m in mirrors: ++ if isinstance(m, string_types): ++ m = {'mirror': _to_utf8(m)} ++ parsed_mirrors.append(m) ++ return parsed_mirrors + + def _load_gr(self, gr): + # OVERRIDE IDEAS: +-- +2.21.0 + diff --git a/0003-urlgrabber-ext-down-convert-url-into-bytes.patch b/0003-urlgrabber-ext-down-convert-url-into-bytes.patch new file mode 100644 index 0000000..67eff79 --- /dev/null +++ b/0003-urlgrabber-ext-down-convert-url-into-bytes.patch @@ -0,0 +1,56 @@ +From c696255c5f50341688d718b3547efb1a9971927b Mon Sep 17 00:00:00 2001 +From: Michal Domonkos +Date: Tue, 21 May 2019 11:12:12 +0200 +Subject: [PATCH 3/4] urlgrabber-ext-down: convert url into bytes + +We need to convert the parsed url back into bytes before passing it to +the PyCurlFileObject constructor (since _set_opts() expects self.scheme, +constructed from the url, to be a bytes object). + +This caused the unit test "bypassing proxy cache on failure" to fail +(together with a bug in the test itself which is also being fixed here). + +Closes #14. +--- + scripts/urlgrabber-ext-down | 4 ++-- + test/test_mirror.py | 2 +- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/scripts/urlgrabber-ext-down b/scripts/urlgrabber-ext-down +index 13d6dc7..40469a7 100755 +--- a/scripts/urlgrabber-ext-down ++++ b/scripts/urlgrabber-ext-down +@@ -22,7 +22,7 @@ import time, os, errno, sys + import six + from urlgrabber.grabber import \ + _readlines, URLGrabberOptions, _loads, \ +- PyCurlFileObject, URLGrabError ++ PyCurlFileObject, URLGrabError, _to_utf8 + + def write(fmt, *arg): + buf = fmt % arg +@@ -65,7 +65,7 @@ def main(): + + dlsz = dltm = 0 + try: +- fo = PyCurlFileObject(opts.url, opts.filename, opts) ++ fo = PyCurlFileObject(_to_utf8(opts.url), opts.filename, opts) + fo._do_grab() + fo.fo.close() + size = fo._amount_read +diff --git a/test/test_mirror.py b/test/test_mirror.py +index a2daf19..66cafd0 100644 +--- a/test/test_mirror.py ++++ b/test/test_mirror.py +@@ -350,7 +350,7 @@ class HttpReplyCode(TestCase): + self.content = b'version1' + + def checkfunc_read(obj): +- if obj.data == 'version1': ++ if obj.data == b'version1': + raise URLGrabError(-1, 'Outdated version of foo') + + def checkfunc_grab(obj): +-- +2.21.0 + diff --git a/BZ-1051554-speed-on-404-mirror.patch b/BZ-1051554-speed-on-404-mirror.patch deleted file mode 100644 index b587c3d..0000000 --- a/BZ-1051554-speed-on-404-mirror.patch +++ /dev/null @@ -1,18 +0,0 @@ -commit fbc995805e9d860366c67819effc3bc7d5d8a8b2 -Author: Zdenek Pavlas -Date: Mon Jan 13 13:20:28 2014 +0100 - - Don't set speed=0 on a new mirror that 404'd. BZ 1051554 - -diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py -index ef18d6a..04f1179 100644 ---- a/urlgrabber/grabber.py -+++ b/urlgrabber/grabber.py -@@ -2418,6 +2418,7 @@ class _TH: - speed = (k1 * speed + k2 * dl_size / dl_time) / (k1 + k2) - fail = 0 - elif getattr(ug_err, 'code', None) == 404: -+ if not ts: return # 1st update, avoid speed=0 - fail = 0 # alive, at least - else: - fail += 1 # seems dead diff --git a/python-urlgrabber.spec b/python-urlgrabber.spec index bccdbc3..a47c3bb 100644 --- a/python-urlgrabber.spec +++ b/python-urlgrabber.spec @@ -1,14 +1,16 @@ Name: python-urlgrabber -Version: 3.10.1 -Release: 18 +Version: 4.0.0 +Release: 1 Summary: Cross-protocol urlgrabber License: LGPLv2+ Url: http://urlgrabber.baseurl.org/ Source0: http://urlgrabber.baseurl.org/download/urlgrabber-%{version}.tar.gz -Patch1: BZ-1051554-speed-on-404-mirror.patch + +Patch0001: 0001-urlgrabber-ext-down-another-python-3-compat.patch +Patch0002: 0002-Revert-Simplify-mirror-conversion-to-utf8.patch +Patch0003: 0003-urlgrabber-ext-down-convert-url-into-bytes.patch BuildArch: noarch -BuildRequires: python2-pycurl python2-devel %global _description\ It is a urlgrabber.We can use it to fetch data in three ways.Urlgrab copies\ @@ -18,34 +20,51 @@ and use this package. %description %_description -%global _package_python2 python2-urlgrabber - -%package -n %_package_python2 +%package -n python2-urlgrabber Summary: %summary -Provides: urlgrabber = %{version}-%{release} -Requires: python2-pycurl +BuildRequires: python2-devel python2dist(setuptools) python2dist(pycurl) python2dist(six) %{?python_provide:%python_provide python2-urlgrabber} -%description -n %_package_python2 %_description +%description -n python2-urlgrabber %_description + +%package -n python3-urlgrabber +Summary: %summary +Provides: urlgrabber = %{version}-%{release} +BuildRequires: python3-devel python3dist(setuptools) python3dist(pycurl) python3dist(six) +%{?python_provide:%python_provide python3-urlgrabber} + +%description -n python3-urlgrabber %_description %prep -%setup -q -n urlgrabber-%{version} -%patch1 -p1 +%autosetup -n urlgrabber-%{version} -p1 %build %py2_build +%py3_build %install %py2_install +%py3_install +sed -e "s|/usr/bin/python|%{__python3}|" -i $RPM_BUILD_ROOT/%{_libexecdir}/* rm -rf $RPM_BUILD_ROOT/%{_docdir}/urlgrabber-%{version} -%files -n %_package_python2 +%files -n python2-urlgrabber +%license LICENSE +%doc README ChangeLog TODO +%{python2_sitelib}/urlgrabber +%{python2_sitelib}/urlgrabber-%{version}-py?.?.egg-info + +%files -n python3-urlgrabber %license LICENSE %doc README ChangeLog TODO %{_bindir}/urlgrabber -%{python2_sitelib}/urlgrabber* -%attr(0755,root,root) %{_libexecdir}/urlgrabber-ext-down +%{_libexecdir}/urlgrabber-ext-down +%{python3_sitelib}/urlgrabber +%{python3_sitelib}/urlgrabber-%{version}-py?.?.egg-info %changelog +* Fri Feb 28 2020 lingsheng - 4.0.0-1 +- update version to 4.0.0 + * Fri Feb 14 2020 Jiangping Hu - 3.10.1-18 - Package init diff --git a/urlgrabber-3.10.1.tar.gz b/urlgrabber-3.10.1.tar.gz deleted file mode 100644 index efe223a..0000000 Binary files a/urlgrabber-3.10.1.tar.gz and /dev/null differ diff --git a/urlgrabber-4.0.0.tar.gz b/urlgrabber-4.0.0.tar.gz new file mode 100644 index 0000000..14d22bf Binary files /dev/null and b/urlgrabber-4.0.0.tar.gz differ