!69 回合上游社区补丁解决socketserver.ThreadingMixIn中存在的内存泄漏

From: @BruceGW
Reviewed-by: @hanxinke
Signed-off-by: @hanxinke

backport-Remove-thread-objects-which-finished-process-its-request.patch

@@ -0,0 +1,156 @@
+From de0359e236bba03b4aabaeed6af08e15a55dd4d9 Mon Sep 17 00:00:00 2001
+From: "Miss Islington (bot)"
+ <31488909+miss-islington@users.noreply.github.com>
+Date: Thu, 4 Mar 2021 08:55:24 -0800
+Subject: [PATCH] [3.8] bpo-37193: Remove thread objects which finished process
+ its request (GH-23127) (GH-24749)
+
+This reverts commit aca67da4fe68d5420401ac1782203d302875eb27.
+(cherry picked from commit b5711c940f70af89f2b4cf081a3fcd83924f3ae7)
+
+Co-authored-by: Jason R. Coombs <jaraco@jaraco.com>
+
+Automerge-Triggered-By: GH:jaraco
+---
+ Lib/socketserver.py                                | 51 +++++++++++++++++-----
+ Lib/test/test_socketserver.py                      | 23 ++++++++++
+ .../2020-06-12-21-23-20.bpo-37193.wJximU.rst       |  2 +
+ 3 files changed, 64 insertions(+), 12 deletions(-)
+ create mode 100644 Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst
+
+diff --git a/Lib/socketserver.py b/Lib/socketserver.py
+index 1ad028f..9e94c76 100644
+--- a/Lib/socketserver.py
++++ b/Lib/socketserver.py
+@@ -628,6 +628,39 @@ if hasattr(os, "fork"):
+             self.collect_children(blocking=self.block_on_close)
+ 
+ 
++class _Threads(list):
++    """
++    Joinable list of all non-daemon threads.
++    """
++    def append(self, thread):
++        self.reap()
++        if thread.daemon:
++            return
++        super().append(thread)
++
++    def pop_all(self):
++        self[:], result = [], self[:]
++        return result
++
++    def join(self):
++        for thread in self.pop_all():
++            thread.join()
++
++    def reap(self):
++        self[:] = (thread for thread in self if thread.is_alive())
++
++
++class _NoThreads:
++    """
++    Degenerate version of _Threads.
++    """
++    def append(self, thread):
++        pass
++
++    def join(self):
++        pass
++
++
+ class ThreadingMixIn:
+     """Mix-in class to handle each request in a new thread."""
+ 
+@@ -636,9 +669,9 @@ class ThreadingMixIn:
+     daemon_threads = False
+     # If true, server_close() waits until all non-daemonic threads terminate.
+     block_on_close = True
+-    # For non-daemonic threads, list of threading.Threading objects
++    # Threads object
+     # used by server_close() to wait for all threads completion.
+-    _threads = None
++    _threads = _NoThreads()
+ 
+     def process_request_thread(self, request, client_address):
+         """Same as in BaseServer but as a thread.
+@@ -655,23 +688,17 @@ class ThreadingMixIn:
+ 
+     def process_request(self, request, client_address):
+         """Start a new thread to process the request."""
++        if self.block_on_close:
++            vars(self).setdefault('_threads', _Threads())
+         t = threading.Thread(target = self.process_request_thread,
+                              args = (request, client_address))
+         t.daemon = self.daemon_threads
+-        if not t.daemon and self.block_on_close:
+-            if self._threads is None:
+-                self._threads = []
+-            self._threads.append(t)
++        self._threads.append(t)
+         t.start()
+ 
+     def server_close(self):
+         super().server_close()
+-        if self.block_on_close:
+-            threads = self._threads
+-            self._threads = None
+-            if threads:
+-                for thread in threads:
+-                    thread.join()
++        self._threads.join()
+ 
+ 
+ if hasattr(os, "fork"):
+diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py
+index 8aed4b6..1e5224d 100644
+--- a/Lib/test/test_socketserver.py
++++ b/Lib/test/test_socketserver.py
+@@ -276,6 +276,13 @@ class SocketServerTest(unittest.TestCase):
+             t.join()
+             s.server_close()
+ 
++    def test_close_immediately(self):
++        class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
++            pass
++
++        server = MyServer((HOST, 0), lambda: None)
++        server.server_close()
++
+     def test_tcpserver_bind_leak(self):
+         # Issue #22435: the server socket wouldn't be closed if bind()/listen()
+         # failed.
+@@ -490,6 +497,22 @@ class MiscTestCase(unittest.TestCase):
+         self.assertEqual(server.shutdown_called, 1)
+         server.server_close()
+ 
++    def test_threads_reaped(self):
++        """
++        In #37193, users reported a memory leak
++        due to the saving of every request thread. Ensure that
++        not all threads are kept forever.
++        """
++        class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
++            pass
++
++        server = MyServer((HOST, 0), socketserver.StreamRequestHandler)
++        for n in range(10):
++            with socket.create_connection(server.server_address):
++                server.handle_request()
++        self.assertLess(len(server._threads), 10)
++        server.server_close()
++
+ 
+ if __name__ == "__main__":
+     unittest.main()
+diff --git a/Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst b/Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst
+new file mode 100644
+index 0000000..fbf56d3
+--- /dev/null
++++ b/Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst
+@@ -0,0 +1,2 @@
++Fixed memory leak in ``socketserver.ThreadingMixIn`` introduced in Python
++3.7.
+-- 
+1.8.3.1
+

python3.spec

@@ -3,7 +3,7 @@ Summary: Interpreter of the Python3 programming language
 URL: https://www.python.org/
 
 Version: 3.8.5
-Release: 8
+Release: 9
 License: Python
 
 %global branchversion 3.8
@@ -98,6 +98,7 @@ Patch251: 00251-change-user-install-location.patch
 Patch252: CVE-2020-27619.patch
 Patch254: CVE-2021-3177.patch
 Patch255: backport-CVE-2021-23336.patch
+Patch256: backport-Remove-thread-objects-which-finished-process-its-request.patch
 
 Provides: python%{branchversion} = %{version}-%{release}
 Provides: python(abi) = %{branchversion}
@@ -193,6 +194,7 @@ rm Lib/ensurepip/_bundled/*.whl
 %patch252 -p1
 %patch254 -p1
 %patch255 -p1
+%patch256 -p1
 
 rm configure pyconfig.h.in
 
@@ -800,6 +802,12 @@ export BEP_GTDLIST="$BEP_GTDLIST_TMP"
 %{_mandir}/*/*
 
 %changelog
+* Tue Apr 27 2021 BruceGW<gyl93216@163.com> - 3.8.5-9
+- Type:bugfix
+- ID:NA
+- SUG:NA
+- DESC: fix memory leak in socketserver.ThreadingMixIn
+
 * Thu Mar 30 2021 shenyangyang<shenyangyang4@huawei.com> - 3.8.5-8
 - Type:bugfix
 - ID:NA