From 747a7c467d45354d8d1ea72bc9d2fce15e186479 Mon Sep 17 00:00:00 2001 From: Pandu E POLUAN Date: Tue, 9 Mar 2021 00:25:48 +0700 Subject: [PATCH 1/4] Implement Unthreaded Controller (#256) * Complete rewrite of aiosmtpd.controller * Implement Unthreaded Controllers * Implement tests of Unthreaded Controllers * Improve other tests * Improve coverage by replacing nocover's with conditional pragmas * Suppress exception ignored during __del__ * Blackification * Update badges * Tidy up table + link to Public PGP on GH * Bump version to 1.5.0a1 and update NEWS.rst --- DESCRIPTION.rst | 45 ++- README.rst | 24 +- aiosmtpd/__init__.py | 2 +- aiosmtpd/controller.py | 296 ++++++++++++---- aiosmtpd/docs/NEWS.rst | 12 + aiosmtpd/docs/controller.rst | 642 +++++++++++++++++++++++----------- aiosmtpd/docs/smtp.rst | 10 +- aiosmtpd/handlers.py | 11 +- aiosmtpd/proxy_protocol.py | 2 +- aiosmtpd/tests/conftest.py | 15 + aiosmtpd/tests/test_main.py | 27 +- aiosmtpd/tests/test_server.py | 248 ++++++++++--- pyproject.toml | 6 +- 13 files changed, 958 insertions(+), 382 deletions(-) diff --git a/DESCRIPTION.rst b/DESCRIPTION.rst index 9ec007b..caa9e7a 100644 --- a/DESCRIPTION.rst +++ b/DESCRIPTION.rst @@ -2,16 +2,22 @@ aiosmtpd - asyncio based SMTP server ###################################### -| |github license| |_| |PyPI Version| |PyPI Python| -| |GA badge| |codecov| |_| |LGTM.com| |readthedocs| |_| -| |GH Release| |_| |PullRequests| |_| |LastCommit| +| |github license| |_| |PyPI Version| |_| |PyPI Python| +| |GA badge| |_| |codecov| |_| |LGTM.com| |_| |readthedocs| +| |GH Release| |_| |GH PRs| |_| |GH LastCommit| | .. |_| unicode:: 0xA0 :trim: -.. |github license| image:: https://img.shields.io/github/license/aio-libs/aiosmtpd +.. |github license| image:: https://img.shields.io/github/license/aio-libs/aiosmtpd?logo=Open+Source+Initiative&logoColor=0F0 :target: https://github.com/aio-libs/aiosmtpd/blob/master/LICENSE :alt: Project License on GitHub +.. |PyPI Version| image:: https://img.shields.io/pypi/v/aiosmtpd?logo=pypi&logoColor=yellow + :target: https://pypi.org/project/aiosmtpd/ + :alt: PyPI Package +.. |PyPI Python| image:: https://img.shields.io/pypi/pyversions/aiosmtpd?logo=python&logoColor=yellow + :target: https://pypi.org/project/aiosmtpd/ + :alt: Supported Python Versions .. .. For |GA badge|, don't forget to check actual workflow name in unit-testing-and-coverage.yml .. |GA badge| image:: https://github.com/aio-libs/aiosmtpd/workflows/aiosmtpd%20CI/badge.svg :target: https://github.com/aio-libs/aiosmtpd/actions @@ -25,21 +31,15 @@ .. |readthedocs| image:: https://img.shields.io/readthedocs/aiosmtpd?logo=Read+the+Docs :target: https://aiosmtpd.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -.. |PyPI Version| image:: https://badge.fury.io/py/aiosmtpd.svg - :target: https://badge.fury.io/py/aiosmtpd - :alt: PyPI Package -.. |PyPI Python| image:: https://img.shields.io/pypi/pyversions/aiosmtpd.svg - :target: https://pypi.org/project/aiosmtpd/ - :alt: Supported Python Versions .. .. Do NOT include the Discourse badge! .. .. Below are badges just for PyPI .. |GH Release| image:: https://img.shields.io/github/v/release/aio-libs/aiosmtpd?logo=github :target: https://github.com/aio-libs/aiosmtpd/releases :alt: GitHub latest release -.. |PullRequests| image:: https://img.shields.io/github/issues-pr/aio-libs/aiosmtpd?logo=GitHub +.. |GH PRs| image:: https://img.shields.io/github/issues-pr/aio-libs/aiosmtpd?logo=GitHub :target: https://github.com/aio-libs/aiosmtpd/pulls :alt: GitHub pull requests -.. |LastCommit| image:: https://img.shields.io/github/last-commit/aio-libs/aiosmtpd?logo=GitHub +.. |GH LastCommit| image:: https://img.shields.io/github/last-commit/aio-libs/aiosmtpd?logo=GitHub :target: https://github.com/aio-libs/aiosmtpd/commits/master :alt: GitHub last commit @@ -61,10 +61,19 @@ Starting version 1.3.1, files provided through PyPI or `GitHub Releases`_ will be signed using one of the following GPG Keys: -+-------------------------+----------------+------------------------------+ -| GPG Key ID | Owner | Email | -+=========================+================+==============================+ -| ``5D60 CE28 9CD7 C258`` | Pandu E POLUAN | pepoluan at gmail period com | -+-------------------------+----------------+------------------------------+ - .. _`GitHub Releases`: https://github.com/aio-libs/aiosmtpd/releases + +.. .. In the second column of the table, prefix each line with "| " + .. In the third column, refrain from putting in a direct link to keep the table tidy. + Rather, use the |...|_ construct and do the replacement+linking directive below the table + ++-------------------------+--------------------------------+-----------+ +| GPG Key ID | Owner / Email | Key | ++=========================+================================+===========+ +| ``5D60 CE28 9CD7 C258`` | | Pandu POLUAN / | |pep_gh|_ | +| | | pepoluan at gmail period com | | ++-------------------------+--------------------------------+-----------+ + +.. .. The |_| contruct is U+00A0 (non-breaking space), defined at the start of the file +.. |pep_gh| replace:: On |_| GitHub +.. _`pep_gh`: https://github.com/pepoluan.gpg diff --git a/README.rst b/README.rst index 35dcb88..2c1bab7 100644 --- a/README.rst +++ b/README.rst @@ -2,14 +2,22 @@ aiosmtpd - An asyncio based SMTP server ========================================= -| |github license| |PyPI| |PyPI Python| -| |GA badge| |codecov| |LGTM.com| |readthedocs| +| |github license| |_| |PyPI Version| |_| |PyPI Python| +| |GA badge| |_| |codecov| |_| |LGTM.com| |_| |readthedocs| | | |Discourse| -.. |github license| image:: https://img.shields.io/github/license/aio-libs/aiosmtpd +.. |_| unicode:: 0xA0 + :trim: +.. |github license| image:: https://img.shields.io/github/license/aio-libs/aiosmtpd?logo=Open+Source+Initiative&logoColor=0F0 :target: https://github.com/aio-libs/aiosmtpd/blob/master/LICENSE :alt: Project License on GitHub +.. |PyPI Version| image:: https://img.shields.io/pypi/v/aiosmtpd?logo=pypi&logoColor=yellow + :target: https://pypi.org/project/aiosmtpd/ + :alt: PyPI Package +.. |PyPI Python| image:: https://img.shields.io/pypi/pyversions/aiosmtpd?logo=python&logoColor=yellow + :target: https://pypi.org/project/aiosmtpd/ + :alt: Supported Python Versions .. .. For |GA badge|, don't forget to check actual workflow name in unit-testing-and-coverage.yml .. |GA badge| image:: https://github.com/aio-libs/aiosmtpd/workflows/aiosmtpd%20CI/badge.svg :target: https://github.com/aio-libs/aiosmtpd/actions @@ -20,15 +28,9 @@ .. |LGTM.com| image:: https://img.shields.io/lgtm/grade/python/github/aio-libs/aiosmtpd.svg?logo=lgtm&logoWidth=18 :target: https://lgtm.com/projects/g/aio-libs/aiosmtpd/context:python :alt: Semmle/LGTM.com quality -.. |readthedocs| image:: https://readthedocs.org/projects/aiosmtpd/badge/?version=latest - :target: https://aiosmtpd.readthedocs.io/en/latest/?badge=latest +.. |readthedocs| image:: https://img.shields.io/readthedocs/aiosmtpd?logo=Read+the+Docs&logoColor=white + :target: https://aiosmtpd.readthedocs.io/en/latest/ :alt: Documentation Status -.. |PyPI| image:: https://badge.fury.io/py/aiosmtpd.svg - :target: https://badge.fury.io/py/aiosmtpd - :alt: PyPI Package -.. |PyPI Python| image:: https://img.shields.io/pypi/pyversions/aiosmtpd.svg - :target: https://pypi.org/project/aiosmtpd/ - :alt: Supported Python Versions .. .. If you edit the above badges, don't forget to edit setup.cfg .. .. The |Discourse| badge MUST NOT be included in setup.cfg .. |Discourse| image:: https://img.shields.io/discourse/status?server=https%3A%2F%2Faio-libs.discourse.group%2F&style=social diff --git a/aiosmtpd/__init__.py b/aiosmtpd/__init__.py index 7d459d8..9c7b938 100644 --- a/aiosmtpd/__init__.py +++ b/aiosmtpd/__init__.py @@ -1,4 +1,4 @@ # Copyright 2014-2021 The aiosmtpd Developers # SPDX-License-Identifier: Apache-2.0 -__version__ = "1.4.2" +__version__ = "1.5.0a1" diff --git a/aiosmtpd/controller.py b/aiosmtpd/controller.py index 2258c54..d3345b8 100644 --- a/aiosmtpd/controller.py +++ b/aiosmtpd/controller.py @@ -5,6 +5,7 @@ import asyncio import errno import os import ssl +import sys import threading import time from abc import ABCMeta, abstractmethod @@ -19,6 +20,11 @@ try: except ImportError: # pragma: on-not-win32 AF_UNIX = None from typing import Any, Coroutine, Dict, Optional, Union + +if sys.version_info >= (3, 8): + from typing import Literal # pragma: py-lt-38 +else: # pragma: py-ge-38 + from typing_extensions import Literal from warnings import warn from public import public @@ -38,13 +44,14 @@ class IP6_IS: YES = {errno.EADDRINUSE} -def _has_ipv6(): +def _has_ipv6() -> bool: # Helper function to assist in mocking return has_ipv6 @public -def get_localhost() -> str: +def get_localhost() -> Literal["::1", "127.0.0.1"]: + """Returns numeric address to localhost depending on IPv6 availability""" # Ref: # - https://github.com/urllib3/urllib3/pull/611#issuecomment-100954017 # - https://github.com/python/cpython/blob/ : @@ -91,24 +98,17 @@ class _FakeServer(asyncio.StreamReaderProtocol): @public -class BaseThreadedController(metaclass=ABCMeta): - """ - `Documentation can be found here - `_. - """ +class BaseController(metaclass=ABCMeta): + smtpd = None server: Optional[AsyncServer] = None server_coro: Optional[Coroutine] = None - smtpd = None - _factory_invoked: Optional[threading.Event] = None - _thread: Optional[threading.Thread] = None - _thread_exception: Optional[Exception] = None + _factory_invoked: threading.Event = None def __init__( self, - handler, - loop=None, + handler: Any, + loop: asyncio.AbstractEventLoop = None, *, - ready_timeout: float, ssl_context: Optional[ssl.SSLContext] = None, # SMTP parameters server_hostname: Optional[str] = None, @@ -119,9 +119,6 @@ class BaseThreadedController(metaclass=ABCMeta): self.loop = asyncio.new_event_loop() else: self.loop = loop - self.ready_timeout = float( - os.getenv("AIOSMTPD_CONTROLLER_TIMEOUT", ready_timeout) - ) self.ssl_context = ssl_context self.SMTP_kwargs: Dict[str, Any] = {} if "server_kwargs" in SMTP_parameters: @@ -139,9 +136,11 @@ class BaseThreadedController(metaclass=ABCMeta): # It actually conflicts with SMTP class's default, but the reasoning is # discussed in the docs. self.SMTP_kwargs.setdefault("enable_SMTPUTF8", True) + # + self._factory_invoked = threading.Event() def factory(self): - """Allow subclasses to customize the handler/server creation.""" + """Subclasses can override this to customize the handler/server creation.""" return SMTP(self.handler, **self.SMTP_kwargs) def _factory_invoker(self): @@ -159,13 +158,72 @@ class BaseThreadedController(metaclass=ABCMeta): @abstractmethod def _create_server(self) -> Coroutine: - raise NotImplementedError # pragma: nocover + """ + Overridden by subclasses to actually perform the async binding to the + listener endpoint. When overridden, MUST refer the _factory_invoker() method. + """ + raise NotImplementedError + + def _cleanup(self): + """Reset internal variables to prevent contamination""" + self._thread_exception = None + self._factory_invoked.clear() + self.server_coro = None + self.server = None + self.smtpd = None + + def cancel_tasks(self, stop_loop: bool = True): + """ + Convenience method to stop the loop and cancel all tasks. + Use loop.call_soon_threadsafe() to invoke this. + """ + if stop_loop: # pragma: nobranch + self.loop.stop() + try: + _all_tasks = asyncio.all_tasks # pytype: disable=module-attr + except AttributeError: # pragma: py-gt-36 + _all_tasks = asyncio.Task.all_tasks + for task in _all_tasks(self.loop): + # This needs to be invoked in a thread-safe way + task.cancel() + + +@public +class BaseThreadedController(BaseController, metaclass=ABCMeta): + _thread: Optional[threading.Thread] = None + _thread_exception: Optional[Exception] = None + + def __init__( + self, + handler: Any, + loop: asyncio.AbstractEventLoop = None, + *, + ready_timeout: float = DEFAULT_READY_TIMEOUT, + ssl_context: Optional[ssl.SSLContext] = None, + # SMTP parameters + server_hostname: Optional[str] = None, + **SMTP_parameters, + ): + super().__init__( + handler, + loop, + ssl_context=ssl_context, + server_hostname=server_hostname, + **SMTP_parameters, + ) + self.ready_timeout = float( + os.getenv("AIOSMTPD_CONTROLLER_TIMEOUT", ready_timeout) + ) @abstractmethod def _trigger_server(self): - raise NotImplementedError # pragma: nocover + """ + Overridden by subclasses to trigger asyncio to actually initialize the SMTP + class (it's lazy initialization, done only on initial connection). + """ + raise NotImplementedError - def _run(self, ready_event): + def _run(self, ready_event: threading.Event): asyncio.set_event_loop(self.loop) try: # Need to do two-step assignments here to ensure IDEs can properly @@ -187,14 +245,19 @@ class BaseThreadedController(metaclass=ABCMeta): return self.loop.call_soon(ready_event.set) self.loop.run_forever() + # We reach this point when loop is ended (by external code) + # Perform some stoppages to ensure endpoint no longer bound. self.server.close() self.loop.run_until_complete(self.server.wait_closed()) self.loop.close() self.server = None def start(self): + """ + Start a thread and run the asyncio event loop in that thread + """ assert self._thread is None, "SMTP daemon already running" - self._factory_invoked = threading.Event() + self._factory_invoked.clear() ready_event = threading.Event() self._thread = threading.Thread(target=self._run, args=(ready_event,)) @@ -240,43 +303,26 @@ class BaseThreadedController(metaclass=ABCMeta): if self.smtpd is None: raise RuntimeError("Unknown Error, failed to init SMTP server") - def _stop(self): - self.loop.stop() - try: - _all_tasks = asyncio.all_tasks # pytype: disable=module-attr - except AttributeError: # pragma: py-gt-36 - _all_tasks = asyncio.Task.all_tasks - for task in _all_tasks(self.loop): - task.cancel() - - def stop(self, no_assert=False): + def stop(self, no_assert: bool = False): + """ + Stop the loop, the tasks in the loop, and terminate the thread as well. + """ assert no_assert or self._thread is not None, "SMTP daemon not running" - self.loop.call_soon_threadsafe(self._stop) + self.loop.call_soon_threadsafe(self.cancel_tasks) if self._thread is not None: self._thread.join() self._thread = None - self._thread_exception = None - self._factory_invoked = None - self.server_coro = None - self.server = None - self.smtpd = None + self._cleanup() @public -class Controller(BaseThreadedController): - """ - `Documentation can be found here - `_. - """ +class BaseUnthreadedController(BaseController, metaclass=ABCMeta): def __init__( self, - handler, - hostname: Optional[str] = None, - port: int = 8025, - loop=None, + handler: Any, + loop: asyncio.AbstractEventLoop = None, *, - ready_timeout: float = DEFAULT_READY_TIMEOUT, - ssl_context: ssl.SSLContext = None, + ssl_context: Optional[ssl.SSLContext] = None, # SMTP parameters server_hostname: Optional[str] = None, **SMTP_parameters, @@ -284,15 +330,80 @@ class Controller(BaseThreadedController): super().__init__( handler, loop, - ready_timeout=ready_timeout, + ssl_context=ssl_context, server_hostname=server_hostname, - **SMTP_parameters + **SMTP_parameters, ) - self.hostname = get_localhost() if hostname is None else hostname + self.ended = threading.Event() + + def begin(self): + """ + Sets up the asyncio server task and inject it into the asyncio event loop. + Does NOT actually start the event loop itself. + """ + asyncio.set_event_loop(self.loop) + # Need to do two-step assignments here to ensure IDEs can properly + # detect the types of the vars. Cannot use `assert isinstance`, because + # Python 3.6 in asyncio debug mode has a bug wherein CoroWrapper is not + # an instance of Coroutine + self.server_coro = self._create_server() + srv: AsyncServer = self.loop.run_until_complete(self.server_coro) + self.server = srv + + async def finalize(self): + """ + Perform orderly closing of the server listener. + NOTE: This is an async method; await this from an async or use + loop.create_task() (if loop is still running), or + loop.run_until_complete() (if loop has stopped) + """ + self.ended.clear() + server = self.server + server.close() + await server.wait_closed() + self.server_coro.close() + self._cleanup() + self.ended.set() + + def end(self): + """ + Convenience method to asynchronously invoke finalize(). + Consider using loop.call_soon_threadsafe to invoke this method, especially + if your loop is running in a different thread. You can afterwards .wait() on + ended attribute (a threading.Event) to check for completion, if needed. + """ + self.ended.clear() + if self.loop.is_running(): + self.loop.create_task(self.finalize()) + else: + self.loop.run_until_complete(self.finalize()) + + +@public +class InetMixin(BaseController, metaclass=ABCMeta): + def __init__( + self, + handler: Any, + hostname: Optional[str] = None, + port: int = 8025, + loop: asyncio.AbstractEventLoop = None, + **kwargs, + ): + super().__init__( + handler, + loop, + **kwargs, + ) + self._localhost = get_localhost() + self.hostname = self._localhost if hostname is None else hostname self.port = port - self.ssl_context = ssl_context def _create_server(self) -> Coroutine: + """ + Creates a 'server task' that listens on an INET host:port. + Does NOT actually start the protocol object itself; + _factory_invoker() is only called upon fist connection attempt. + """ return self.loop.create_server( self._factory_invoker, host=self.hostname, @@ -308,42 +419,36 @@ class Controller(BaseThreadedController): """ # At this point, if self.hostname is Falsy, it most likely is "" (bind to all # addresses). In such case, it should be safe to connect to localhost) - hostname = self.hostname or get_localhost() + hostname = self.hostname or self._localhost with ExitStack() as stk: s = stk.enter_context(create_connection((hostname, self.port), 1.0)) if self.ssl_context: s = stk.enter_context(self.ssl_context.wrap_socket(s)) - _ = s.recv(1024) + s.recv(1024) -class UnixSocketController(BaseThreadedController): # pragma: on-win32 on-cygwin - """ - `Documentation can be found here - `_. - """ +@public +class UnixSocketMixin(BaseController, metaclass=ABCMeta): # pragma: no-unixsock def __init__( self, - handler, - unix_socket: Optional[Union[str, Path]], - loop=None, - *, - ready_timeout: float = DEFAULT_READY_TIMEOUT, - ssl_context: ssl.SSLContext = None, - # SMTP parameters - server_hostname: str = None, - **SMTP_parameters, + handler: Any, + unix_socket: Union[str, Path], + loop: asyncio.AbstractEventLoop = None, + **kwargs, ): super().__init__( handler, loop, - ready_timeout=ready_timeout, - ssl_context=ssl_context, - server_hostname=server_hostname, - **SMTP_parameters + **kwargs, ) self.unix_socket = str(unix_socket) def _create_server(self) -> Coroutine: + """ + Creates a 'server task' that listens on a Unix Socket file. + Does NOT actually start the protocol object itself; + _factory_invoker() is only called upon fist connection attempt. + """ return self.loop.create_unix_server( self._factory_invoker, path=self.unix_socket, @@ -351,9 +456,52 @@ class UnixSocketController(BaseThreadedController): # pragma: on-win32 on-cygwi ) def _trigger_server(self): + """ + Opens a socket connection to the newly launched server, wrapping in an SSL + Context if necessary, and read some data from it to ensure that factory() + gets invoked. + """ with ExitStack() as stk: s: makesock = stk.enter_context(makesock(AF_UNIX, SOCK_STREAM)) s.connect(self.unix_socket) if self.ssl_context: s = stk.enter_context(self.ssl_context.wrap_socket(s)) - _ = s.recv(1024) + s.recv(1024) + + +@public +class Controller(InetMixin, BaseThreadedController): + """Provides a multithreaded controller that listens on an INET endpoint""" + + def _trigger_server(self): + # Prevent confusion on which _trigger_server() to invoke. + # Or so LGTM.com claimed + InetMixin._trigger_server(self) + + +@public +class UnixSocketController( # pragma: no-unixsock + UnixSocketMixin, BaseThreadedController +): + """Provides a multithreaded controller that listens on a Unix Socket file""" + + def _trigger_server(self): # pragma: no-unixsock + # Prevent confusion on which _trigger_server() to invoke. + # Or so LGTM.com claimed + UnixSocketMixin._trigger_server(self) + + +@public +class UnthreadedController(InetMixin, BaseUnthreadedController): + """Provides an unthreaded controller that listens on an INET endpoint""" + + pass + + +@public +class UnixSocketUnthreadedController( # pragma: no-unixsock + UnixSocketMixin, BaseUnthreadedController +): + """Provides an unthreaded controller that listens on a Unix Socket file""" + + pass diff --git a/aiosmtpd/docs/NEWS.rst b/aiosmtpd/docs/NEWS.rst index fb32de4..ce627a7 100644 --- a/aiosmtpd/docs/NEWS.rst +++ b/aiosmtpd/docs/NEWS.rst @@ -3,6 +3,18 @@ ################### +1.5.0 (aiosmtpd-next-next) +========================== + +Added +----- +* Unthreaded Controllers (Closes #160) + +Fixed/Improved +-------------- +* All Controllers now have more rationale design, as they are now composited from a Base + a Mixin + + 1.4.2 (2021-03-08) ===================== diff --git a/aiosmtpd/docs/controller.rst b/aiosmtpd/docs/controller.rst index d3e08ed..e43720b 100644 --- a/aiosmtpd/docs/controller.rst +++ b/aiosmtpd/docs/controller.rst @@ -5,15 +5,15 @@ ==================== If you already have an `asyncio event loop`_, you can `create a server`_ using -the ``SMTP`` class as the *protocol factory*, and then run the loop forever. +the :class:`~aiosmtpd.smtp.SMTP` class as the *protocol factory*, and then run the loop forever. If you need to pass arguments to the ``SMTP`` constructor, use :func:`functools.partial` or write your own wrapper function. You might also want to add a signal handler so that the loop can be stopped, say when you hit control-C. -It's probably easier to use a *controller* which runs the SMTP server in a +It's probably easier to use a *threaded controller* which runs the SMTP server in a separate thread with a dedicated event loop. The controller provides useful -and reliable *start* and *stop* semantics so that the foreground thread +and reliable ``start`` and ``stop`` semantics so that the foreground thread doesn't block. Among other use cases, this makes it convenient to spin up an SMTP server for unit tests. @@ -30,7 +30,7 @@ Using the controller TCP-based Server ---------------- -The :class:`Controller` class creates a TCP-based server, +The :class:`~aiosmtpd.controller.Controller` class creates a TCP-based server, listening on an Internet endpoint (i.e., ``ip_address:port`` pair). Say you want to receive email for ``example.com`` and print incoming mail data @@ -100,11 +100,11 @@ Connect to the server and send a message, which then gets printed by End of message You'll notice that at the end of the ``DATA`` command, your handler's -``handle_DATA()`` method was called. The sender, recipients, and message +:meth:`handle_DATA` method was called. The sender, recipients, and message contents were taken from the envelope, and printed at the console. The handler methods also returns a successful status message. -The ``ExampleHandler`` class also implements a ``handle_RCPT()`` method. This +The ``ExampleHandler`` class also implements a :meth:`handle_RCPT` method. This gets called after the ``RCPT TO`` command is sanity checked. The method ensures that all recipients are local to the ``@example.com`` domain, returning an error status if not. It is the handler's responsibility to add @@ -148,10 +148,11 @@ use to do some common tasks, and it's easy to write your own handler. For a full overview of the methods that handler classes may implement, see the section on :ref:`handler hooks `. + Unix Socket-based Server ------------------------ -The :class:`UnixSocketController` class creates a server listening to +The :class:`~aiosmtpd.controller.UnixSocketController` class creates a server listening to a Unix Socket (i.e., a special file that can act as a 'pipe' for interprocess communication). @@ -168,8 +169,13 @@ with some differences: >>> controller = UnixSocketController(Sink(), unix_socket="smtp_socket~") >>> controller.start() +.. warning:: + + Do not exceed the Operating System limit for the length of the socket file path. + On Linux, the limit is 108 characters. On BSD OSes, it's 104 characters. + **Rather than connecting to IP:port, you connect to the Socket file.** -Python's :class:`smtplib.SMTP` sadly cannot connect to a Unix Socket, +Python's :class:`smtplib.SMTP` class sadly cannot connect to a Unix Socket, so we need to handle it on our own here: .. doctest:: unix_socket @@ -178,9 +184,8 @@ so we need to handle it on our own here: >>> import socket >>> sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) >>> sock.connect("smtp_socket~") - >>> resp = sock.recv(1024) - >>> resp[0:4] - b'220 ' + >>> sock.recv(1024) + b'220 ...' Try sending something, don't forget to end with ``"\r\n"``: @@ -189,9 +194,8 @@ Try sending something, don't forget to end with ``"\r\n"``: >>> sock.send(b"HELO example.org\r\n") 18 - >>> resp = sock.recv(1024) - >>> resp[0:4] - b'250 ' + >>> sock.recv(1024) + b'250 ...' And close everything when done: @@ -200,13 +204,116 @@ And close everything when done: >>> sock.send(b"QUIT\r\n") 6 - >>> resp = sock.recv(1024) - >>> resp[0:4] - b'221 ' + >>> sock.recv(1024) + b'221 Bye...' >>> sock.close() >>> controller.stop() +.. _unthreaded: + +Unthreaded Controllers +---------------------- + +In addition to the **threaded** controllers described above, +``aiosmtpd`` also provides the following **UNthreaded** controllers: + +* :class:`UnthreadedController` -- the unthreaded version of :class:`Controller` +* :class:`UnixSocketUnthreadedController` -- the unthreaded version of :class:`UnixSocketController` + +These classes are considered *advanced* classes, +because you'll have to manage the event loop yourself. + +For example, to start an unthreaded controller, +you'll have to do something similar to this: + +.. doctest:: unthreaded + + >>> import asyncio + >>> loop = asyncio.get_event_loop() + >>> from aiosmtpd.controller import UnthreadedController + >>> from aiosmtpd.handlers import Sink + >>> controller = UnthreadedController(Sink(), loop=loop) + >>> controller.begin() + +Note that unlike the threaded counterparts, +the method used to start the controller is named ``begin()``. +And unlike the method in the threaded version, +``begin()`` does NOT start the asyncio event loop; +you'll have to start it yourself. + +For the purposes of trying this, +let's create a thread and have it run the asyncio event loop; +we'll also schedule an autostop so it won't hang: + +.. doctest:: unthreaded + + >>> def runner(): + ... # Set the delay to something long enough so you have time + ... # to do some testing + ... loop.call_later(3.0, loop.stop) + ... loop.run_forever() + >>> import threading + >>> thread = threading.Thread(target=runner) + >>> thread.setDaemon(True) + >>> thread.start() + >>> import time + >>> time.sleep(0.1) # Allow the loop to begin + +At this point in time, the server would be listening: + +.. doctest:: unthreaded + + >>> from smtplib import SMTP as Client + >>> client = Client(controller.hostname, controller.port) + >>> client.helo("example.com") + (250, ...) + >>> client.quit() + (221, b'Bye') + +The complex thing will be to end it; +that is why we're marking these classes as "advanced". + +For our example here, +since we have created an "autostop loop", +all we have to do is wait for the runner thread to end: + +.. doctest:: unthreaded + + >>> thread.join() + >>> loop.is_running() + False + +We still need to do some cleanup to fully release the bound port. +Since the loop has ended, we can simply call the :meth:`end` method: + +.. doctest:: unthreaded + + >>> controller.end() + +If you want to end the controller *but* keep the loop running, +you'll have to do it like this:: + + loop.call_soon_threadsafe(controller.end) + # If you want to ensure that controller has stopped, you can wait() here: + controller.ended.wait(10.0) # Optional + +You must remember to cleanup the canceled tasks yourself. +We have provided a convenience method, +:meth:`~aiosmtpd.controller.BaseController.cancel_tasks`:: + + # Will also stop the loop! + loop.call_soon_threadsafe(controller.cancel_tasks) + +(If you invoke ``cancel_tasks`` with the parameter ``stop_loop=False``, +then loop will NOT be stopped. +That is a much too-advanced topic and we will not discuss it further in this documentation.) + +The Unix Socket variant, ``UnixSocketUnthreadedController``, works in the same way. +The difference is only in how to access the server, i.e., through a Unix Socket instead of TCP/IP. +We'll leave out the details for you to figure it out yourself. + + .. _enablesmtputf8: Enabling SMTPUTF8 @@ -253,265 +360,398 @@ Controller API .. py:module:: aiosmtpd.controller -.. class:: IP6_IS - .. py:attribute:: NO - :type: set +.. py:data:: DEFAULT_READY_TIMEOUT + :type: float + :value: 5.0 + + +.. py:function:: get_localhost() - Contains constants from :mod:`errno` that will be raised by `socket.bind()` - if IPv6 is not available on the system. + :return: The numeric address of the loopback interface; ``"::1"`` if IPv6 is supported, + ``"127.0.0.1"`` if IPv6 is not supported. + :rtype: Literal["::1", "127.0.0.1"] + + +.. class:: IP6_IS - .. important:: + .. py:attribute:: NO + :type: set[int] - If your system does not have IPv6 support but :func:`get_localhost` - raises an error instead of returning ``"127.0.0.1"``, - you can add the error number into this attribute. + Contains constants from :mod:`errno` that will be raised by :meth:`socket.socket.bind` + if IPv6 is NOT available on the system. .. py:attribute:: YES - :type: set + :type: set[int] - Contains constants from :mod:`errno` that will be raised by `socket.bind()` - if IPv6 is not available on the system. + Contains constants from :mod:`errno` that will be raised by :meth:`socket.socket.bind` + if IPv6 IS available on the system. -.. py:function:: get_localhost + .. note:: - :return: The numeric address of the loopback interface; ``"::1"`` if IPv6 is supported, - ``"127.0.0.1"`` if IPv6 is not supported. - :rtype: str + You can customize the contents of these attributes by adding/removing from them, + in case the behavior does not align with your expectations *and* + you cannot wait for a patch to be merged. -.. class:: BaseThreadedController(\ - handler, \ - loop=None, \ - *, \ - ready_timeout, \ - ssl_context=None, \ - server_hostname=None, server_kwargs=None, **SMTP_parameters) - :param handler: Handler object - :param loop: The asyncio event loop in which the server will run. - If not given, :func:`asyncio.new_event_loop` will be called to create the event loop. - :param ready_timeout: How long to wait until server starts. - The :envvar:`AIOSMTPD_CONTROLLER_TIMEOUT` takes precedence over this parameter. - See :attr:`ready_timeout` for more information. - :type ready_timeout: float - :param ssl_context: SSL Context to wrap the socket in. - Will be passed-through to :meth:`~asyncio.loop.create_server` method - :type ssl_context: ssl.SSLContext - :param server_hostname: Server's hostname, - will be passed-through as ``hostname`` parameter of :class:`~aiosmtpd.smtp.SMTP` - :type server_hostname: Optional[str] - :param server_kwargs: (DEPRECATED) A dict that - will be passed-through as keyword arguments of :class:`~aiosmtpd.smtp.SMTP`. - Explicitly listed keyword arguments going into ``**SMTP_parameters`` - will take precedence over this parameter - :type server_kwargs: Dict[str, Any] - :param SMTP_parameters: Optional keyword arguments that - will be passed-through as keyword arguments of :class:`~aiosmtpd.smtp.SMTP` +.. class:: BaseController(\ + handler, \ + loop=None, \ + *, \ + ssl_context=None, \ + server_hostname=None, \ + server_kwargs=None, \ + **SMTP_parameters, \ + ) - .. important:: + This **Abstract Base Class** defines parameters, attributes, and methods common between + all concrete controller classes. - Usually, setting the ``ssl_context`` parameter will switch the protocol to ``SMTPS`` mode, - implying unconditional encryption of the connection, - and preventing the use of the ``STARTTLS`` mechanism. + :param handler: Handler object + :param loop: The asyncio event loop in which the server will run. + If not given, :func:`asyncio.new_event_loop` will be called to create the event loop. + :type loop: asyncio.AbstractEventLoop + :param ssl_context: SSL Context to wrap the socket in. + Will be passed-through to :meth:`~asyncio.loop.create_server` method + :type ssl_context: ssl.SSLContext + :param server_hostname: Server's hostname, + will be passed-through as ``hostname`` parameter of :class:`~aiosmtpd.smtp.SMTP` + :type server_hostname: Optional[str] + :param server_kwargs: *(DEPRECATED)* A dict that will be passed-through as keyword + arguments of :class:`~aiosmtpd.smtp.SMTP`. + This is DEPRECATED; please use ``**SMTP_parameters`` instead. + :type server_kwargs: dict + :param SMTP_parameters: Optional keyword arguments that + will be passed-through as keyword arguments of :class:`~aiosmtpd.smtp.SMTP` - Actual behavior depends on the subclass's implementation. + | + | :part:`Attributes` - | - | :part:`Attributes` + .. attribute:: handler + :noindex: - .. attribute:: handler - :noindex: + The instance of the event *handler* passed to the constructor. - The instance of the event *handler* passed to the constructor. + .. attribute:: loop + :noindex: - .. attribute:: loop - :noindex: + The event loop being used. - The event loop being used. + .. attribute:: server - .. attribute:: ready_timeout - :type: float + This is the server instance returned by + :meth:`_create_server` after the server has started. - The timeout value used to wait for the server to start. + You can retrieve the :class:`~socket.socket` objects the server is listening on + from the ``server.sockets`` attribute. - This will either be the value of - the :envvar:`AIOSMTPD_CONTROLLER_TIMEOUT` environment variable (converted to float), - or the :attr:`ready_timeout` parameter. + .. py:attribute:: smtpd + :type: aiosmtpd.smtp.SMTP - Setting this to a high value will NOT slow down controller startup, - because it's a timeout limit rather than a sleep delay. - However, you may want to reduce the default value to something 'just enough' - so you don't have to wait too long for an exception, if problem arises. + The server instance (of class SMTP) created by :meth:`factory` after + the controller is started. - If this timeout is breached, a :class:`TimeoutError` exception will be raised. + | + | :part:`Methods` - .. attribute:: server + .. method:: factory() -> aiosmtpd.smtp.SMTP - This is the server instance returned by - :meth:`_create_server` after the server has started. + You can override this method to create custom instances of + the :class:`~aiosmtpd.smtp.SMTP` class being controlled. - .. py:attribute:: smtpd - :type: aiosmtpd.smtp.SMTP + By default, this creates an ``SMTP`` instance, + passing in your handler and setting flags from the :attr:`**SMTP_Parameters` parameter. - The server instance (of class SMTP) created by :meth:`factory` after - the controller is started. + Examples of why you would want to override this method include + creating an :ref:`LMTP ` server instance instead of the standard ``SMTP`` server. - | - | :part:`Methods` + .. py:method:: cancel_tasks(stop_loop=True) - .. py:method:: _create_server() -> Coroutine - :abstractmethod: + :param stop_loop: If ``True``, stops the loop before canceling tasks. + :type stop_loop: bool - This method will be called by :meth:`_run` during :meth:`start` procedure. + This is a convenience class that will stop the loop & + cancel all asyncio tasks for you. - It must return a ``Coroutine`` object which will be executed by the asyncio event loop. - .. py:method:: _trigger_server() -> None - :abstractmethod: +.. class:: Controller(\ + handler, \ + hostname=None, \ + port=8025, \ + loop=None, \ + *, \ + ready_timeout=DEFAULT_READY_TIMEOUT, \ + ssl_context=None, \ + server_hostname=None, \ + server_kwargs=None, \ + **SMTP_parameters) - The :meth:`asyncio.loop.create_server` method (or its parallel) - invokes :meth:`factory` "lazily", - so exceptions in :meth:`factory` can go undetected during :meth:`start`. + A concrete subclass of :class:`BaseController` that provides + a threaded, INET listener. - This method will create a connection to the started server and 'exchange' some traffic, - thus triggering :meth:`factory` invocation, - allowing the Controller to catch exceptions during initialization. + :param hostname: Will be given to the event loop's :meth:`~asyncio.loop.create_server` method + as the ``host`` parameter, with a slight processing (see below) + :type hostname: Optional[str] + :param port: Will be passed-through to :meth:`~asyncio.loop.create_server` method + :type port: int + :param ready_timeout: How long to wait until server starts. + The :envvar:`AIOSMTPD_CONTROLLER_TIMEOUT` takes precedence over this parameter. + See :attr:`ready_timeout` for more information. + :type ready_timeout: float - .. method:: start() -> None + Other parameters are defined in the :class:`BaseController` class. - :raises TimeoutError: if the server takes too long to get ready, - exceeding the ``ready_timeout`` parameter. - :raises RuntimeError: if an unrecognized & unhandled error happened, - resulting in non-creation of a server object - (:attr:`smtpd` remains ``None``) + The ``hostname`` parameter will be passed to the event loop's + :meth:`~asyncio.loop.create_server` method as the ``host`` parameter, + :boldital:`except` ``None`` (default) will be translated to ``::1``. - Start the server in the subthread. - The subthread is always a :class:`daemon thread ` - (i.e., we always set ``thread.daemon=True``). + * To bind `dual-stack`_ locally, use ``localhost``. + * To bind `dual-stack`_ on all interfaces, use ``""`` (empty string). - Exceptions can be raised - if the server does not start within :attr:`ready_timeout` seconds, - or if any other exception occurs in :meth:`factory` while creating the server. + .. important:: - .. important:: + The ``hostname`` parameter does NOT get passed through to the SMTP instance; + if you want to give the SMTP instance a custom hostname + (e.g., for use in HELO/EHLO greeting), + you must pass it through the :attr:`server_hostname` parameter. - If :meth:`start` raises an Exception, - cleanup is not performed automatically, - to support deep inspection post-exception (if you wish to do so.) - Cleanup must still be performed manually by calling :meth:`stop` + Explicitly defined SMTP keyword arguments will override keyword arguments of the + same names defined in the (deprecated) ``server_kwargs`` argument. - For example:: + .. doctest:: controller_kwargs - # Assume SomeController is a concrete subclass of BaseThreadedController - controller = SomeController(handler) - try: - controller.start() - except ...: - ... exception handling and/or inspection ... - finally: - controller.stop() + >>> from aiosmtpd.controller import Controller + >>> from aiosmtpd.handlers import Sink + >>> controller = Controller( + ... Sink(), timeout=200, server_kwargs=dict(timeout=400) + ... ) + >>> controller.SMTP_kwargs["timeout"] + 200 - .. method:: stop() -> None + Finally, setting the ``ssl_context`` parameter will switch the protocol to ``SMTPS`` mode, + implying unconditional encryption of the connection, + and preventing the use of the ``STARTTLS`` mechanism. - :raises AssertionError: if :meth:`stop` is called before :meth:`start` is called successfully + Actual behavior depends on the subclass's implementation. - Stop the server and the event loop, and cancel all tasks. + | + | :part:`Attributes` - .. method:: factory() -> aiosmtpd.smtp.SMTP + In addition to those provided by :class:`BaseController`, + this class provides the following: - You can override this method to create custom instances of the ``SMTP`` - class being controlled. + .. attribute:: hostname: str + port: int - By default, this creates an ``SMTP`` instance, - passing in your handler and setting flags from the :attr:`**SMTP_Parameters` parameter. + The values of the *hostname* and *port* arguments. - Examples of why you would want to override this method include - creating an :ref:`LMTP ` server instance instead of the standard ``SMTP`` server. + .. attribute:: ready_timeout + :type: float + The timeout value used to wait for the server to start. + This will either be the value of + the :envvar:`AIOSMTPD_CONTROLLER_TIMEOUT` environment variable (converted to float), + or the :attr:`ready_timeout` parameter. -.. class:: Controller(\ - handler, \ - hostname=None, port=8025, \ - loop=None, \ - *, \ - ready_timeout=3.0, \ - ssl_context=None, \ - server_hostname=None, server_kwargs=None, **SMTP_parameters) - - :param hostname: Will be given to the event loop's :meth:`~asyncio.loop.create_server` method - as the ``host`` parameter, with a slight processing (see below) - :type hostname: Optional[str] - :param port: Will be passed-through to :meth:`~asyncio.loop.create_server` method - :type port: int + Setting this to a high value will NOT slow down controller startup, + because it's a timeout limit rather than a sleep delay. + However, you may want to reduce the default value to something 'just enough' + so you don't have to wait too long for an exception, if problem arises. - .. note:: + If this timeout is breached, a :class:`TimeoutError` exception will be raised. + + | + | :part:`Methods` - The ``hostname`` parameter will be passed to the event loop's - :meth:`~asyncio.loop.create_server` method as the ``host`` parameter, - :boldital:`except` ``None`` (default) will be translated to ``::1``. + In addition to those provided by :class:`BaseController`, + this class provides the following: - * To bind `dual-stack`_ locally, use ``localhost``. + .. method:: start() -> None - * To bind `dual-stack`_ on all interfaces, use ``""`` (empty string). + :raises TimeoutError: if the server takes too long to get ready, + exceeding the ``ready_timeout`` parameter. + :raises RuntimeError: if an unrecognized & unhandled error happened, + resulting in non-creation of a server object + (:attr:`smtpd` remains ``None``) - .. important:: + Start the server in the subthread. + The subthread is always a :class:`daemon thread ` + (i.e., we always set ``thread.daemon=True``). - The ``hostname`` parameter does NOT get passed through to the SMTP instance; - if you want to give the SMTP instance a custom hostname - (e.g., for use in HELO/EHLO greeting), - you must pass it through the :attr:`server_hostname` parameter. + Exceptions can be raised + if the server does not start within :attr:`ready_timeout` seconds, + or if any other exception occurs in :meth:`~BaseController.factory` + while creating the server. - .. important:: + .. important:: - Explicitly defined SMTP keyword arguments will override keyword arguments of the - same names defined in the (deprecated) ``server_kwargs`` argument. + If :meth:`start` raises an Exception, + cleanup is not performed automatically, + to support deep inspection post-exception (if you wish to do so.) + Cleanup must still be performed manually by calling :meth:`stop` - >>> from aiosmtpd.handlers import Sink - >>> controller = Controller(Sink(), timeout=200, server_kwargs=dict(timeout=400)) - >>> controller.SMTP_kwargs["timeout"] - 200 + For example:: - One example is the ``enable_SMTPUTF8`` flag described in the - :ref:`Enabling SMTPUTF8 section ` above. + # Assume SomeController is a concrete subclass of BaseThreadedController + controller = SomeController(handler) + try: + controller.start() + except ...: + ... exception handling and/or inspection ... + finally: + controller.stop() - | - | :part:`Attributes` + .. method:: stop(no_assert=False) -> None - .. attribute:: hostname: str - port: int - :noindex: + :param no_assert: If ``True``, skip the assertion step so an ``AssertionError`` will + not be raised if thread had not been started successfully. + :type no_assert: bool - The values of the *hostname* and *port* arguments. + :raises AssertionError: if this method is called before + :meth:`start` is called successfully *AND* ``no_assert=False`` - Other parameters, attributes, and methods are identical to :class:`BaseThreadedController` - and thus are not repeated nor explained here. + Stop the server and the event loop, and cancel all tasks + via :meth:`~BaseController.cancel_tasks`. .. class:: UnixSocketController(\ - handler, \ - unix_socket, \ - loop=None, \ - *, \ - ready_timeout=3.0, \ - ssl_context=None, \ - server_hostname=None,\ - **SMTP_parameters) + handler, \ + unix_socket, \ + loop=None, \ + *, \ + ready_timeout=DEFAULT_READY_TIMEOUT, \ + ssl_context=None, \ + server_hostname=None, \ + **SMTP_parameters) + + A concrete subclass of :class:`BaseController` that provides + a threaded, Unix Socket listener. + + :param unix_socket: Socket file, + will be passed-through to :meth:`asyncio.loop.create_unix_server` + :type unix_socket: Union[str, pathlib.Path] + + For the other parameters, see the description under :class:`Controller` + + | + | :part:`Attributes` + + .. py:attribute:: unix_socket + :type: str + + The stringified version of the ``unix_socket`` parameter + + Other attributes (except ``hostname`` and ``port``) are identical to :class:`Controller` + and thus are not repeated nor explained here. + + | + | :part:`Methods` + + All methods are identical to :class:`Controller` + and thus are not repeated nor explained here. + + +.. class:: UnthreadedController(\ + handler, \ + hostname=None, \ + port=8025, \ + loop=None, \ + *, \ + ssl_context=None, \ + server_hostname=None, \ + server_kwargs=None, \ + **SMTP_parameters) + + .. versionadded:: 1.5.0 + + A concrete subclass of :class:`BaseController` that provides + an UNthreaded, INET listener. + + Parameters are identical to the :class:`Controller` class. + + | + | :part:`Attributes` + + Attributes are identical to the :class:`Controller` class with one addition: + + .. py:attribute:: ended + :type: threading.Event + + An ``Event`` that can be ``.wait()``-ed when ending the controller. + Please see the :ref:`Unthreaded Controllers ` section for more info. + + | + | :part:`Methods` + + In addition to those provided by :class:`BaseController`, + this class provides the following: + + .. py:method:: begin + + Initializes the server task and insert it into the asyncio event loop. + + .. note:: + + The SMTP class itself will only be initialized upon first connection + to the server task. + + .. py:method:: finalize + :async: + + Perform orderly closing of the server listener. + If you need to close the server from a non-async function, + you can use the :meth:`~UnthreadedController.end` method instead. + + Upon completion of this method, the :attr:`ended` attribute will be ``set()``. + + .. py:method:: end + + This is a convenience method that will asynchronously invoke the + :meth:`finalize` method. + This method non-async, and thus is callable from non-async functions. + + .. note:: + + If the asyncio event loop has been stopped, + then it is safe to invoke this method directly. + Otherwise, it is recommended to invoke this method + using the :meth:`~asyncio.loop.call_soon_threadsafe` method. + + +.. class:: UnixSocketUnthreadedController(\ + handler, \ + unix_socket, \ + loop=None, \ + *, \ + ssl_context=None, \ + server_hostname=None,\ + server_kwargs=None, \ + **SMTP_parameters) + + .. versionadded:: 1.5.0 + + A concrete subclass of :class:`BaseController` that provides + an UNthreaded, Unix Socket listener. + + Parameters are identical to the :class:`UnixSocketController` class. + + | + | :part:`Attributes` - :param unix_socket: Socket file, - will be passed-through to :meth:`asyncio.loop.create_unix_server` - :type unix_socket: Union[str, pathlib.Path] + Attributes are identical to the :class:`UnixSocketController` class, + with the following addition: - | - | :part:`Attributes` + .. py:attribute:: ended + :type: threading.Event - .. py:attribute:: unix_socket - :type: str + An ``Event`` that can be ``.wait()``-ed when ending the controller. + Please see the :ref:`Unthreaded Controllers ` section for more info. - The stringified version of the ``unix_socket`` parameter + | + | :part:`Methods` - Other parameters, attributes, and methods are identical to :class:`BaseThreadedController` - and thus are not repeated nor explained here. + Methods are identical to the :class:`UnthreadedController` class. .. _`asyncio event loop`: https://docs.python.org/3/library/asyncio-eventloop.html diff --git a/aiosmtpd/docs/smtp.rst b/aiosmtpd/docs/smtp.rst index f48b717..3305079 100644 --- a/aiosmtpd/docs/smtp.rst +++ b/aiosmtpd/docs/smtp.rst @@ -99,7 +99,8 @@ Server hooks The ``SMTP`` server class also implements some hooks which your subclass can override to provide additional responses. -``ehlo_hook()`` +.. py:function:: ehlo_hook() + This hook makes it possible for subclasses to return additional ``EHLO`` responses. This method, called *asynchronously* and taking no arguments, can do whatever it wants, including (most commonly) pushing new @@ -107,12 +108,17 @@ override to provide additional responses. before the standard ``250 HELP`` which ends the ``EHLO`` response from the server. -``rset_hook()`` + .. deprecated:: 1.2 + +.. py:function:: rset_hook() + This hook makes it possible to return additional ``RSET`` responses. This method, called *asynchronously* and taking no arguments, is called just before the standard ``250 OK`` which ends the ``RSET`` response from the server. + .. deprecated:: 1.2 + .. _smtp_api: diff --git a/aiosmtpd/handlers.py b/aiosmtpd/handlers.py index b13dd12..ada1e91 100644 --- a/aiosmtpd/handlers.py +++ b/aiosmtpd/handlers.py @@ -15,6 +15,7 @@ import mailbox import re import smtplib import sys +from abc import ABCMeta, abstractmethod from email import message_from_bytes, message_from_string from public import public @@ -148,7 +149,7 @@ class Sink: @public -class Message: +class Message(metaclass=ABCMeta): def __init__(self, message_class=None): self.message_class = message_class @@ -172,12 +173,13 @@ class Message: message['X-RcptTo'] = COMMASPACE.join(envelope.rcpt_tos) return message + @abstractmethod def handle_message(self, message): - raise NotImplementedError # pragma: nocover + raise NotImplementedError @public -class AsyncMessage(Message): +class AsyncMessage(Message, metaclass=ABCMeta): def __init__(self, message_class=None, *, loop=None): super().__init__(message_class) self.loop = loop or asyncio.get_event_loop() @@ -187,8 +189,9 @@ class AsyncMessage(Message): await self.handle_message(message) return '250 OK' + @abstractmethod async def handle_message(self, message): - raise NotImplementedError # pragma: nocover + raise NotImplementedError @public diff --git a/aiosmtpd/proxy_protocol.py b/aiosmtpd/proxy_protocol.py index a171211..621098c 100644 --- a/aiosmtpd/proxy_protocol.py +++ b/aiosmtpd/proxy_protocol.py @@ -99,7 +99,7 @@ class UnknownTypeTLV(KeyError): @public -class AsyncReader(Protocol): # pragma: nocover +class AsyncReader(Protocol): async def read(self, num_bytes: Optional[int] = None) -> bytes: ... return b"" diff --git a/aiosmtpd/tests/conftest.py b/aiosmtpd/tests/conftest.py index 08fc0e8..d0a6cd3 100644 --- a/aiosmtpd/tests/conftest.py +++ b/aiosmtpd/tests/conftest.py @@ -29,6 +29,7 @@ __all__ = [ "controller_data", "handler_data", "Global", + "AUTOSTOP_DELAY", "SERVER_CRT", "SERVER_KEY", ] @@ -64,6 +65,9 @@ class Global: cls.SrvAddr = HostPort(contr.hostname, contr.port) +# If less than 1.0, might cause intermittent error if test system +# is too busy/overloaded. +AUTOSTOP_DELAY = 1.0 SERVER_CRT = resource_filename("aiosmtpd.tests.certs", "server.crt") SERVER_KEY = resource_filename("aiosmtpd.tests.certs", "server.key") @@ -204,6 +208,17 @@ def temp_event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]: asyncio.set_event_loop(default_loop) +@pytest.fixture +def autostop_loop(temp_event_loop) -> Generator[asyncio.AbstractEventLoop, None, None]: + # Create a new event loop, and arrange for that loop to end almost + # immediately. This will allow the calls to main() in these tests to + # also exit almost immediately. Otherwise, the foreground test + # process will hang. + temp_event_loop.call_later(AUTOSTOP_DELAY, temp_event_loop.stop) + # + yield temp_event_loop + + @pytest.fixture def plain_controller(get_handler, get_controller) -> Generator[Controller, None, None]: """ diff --git a/aiosmtpd/tests/test_main.py b/aiosmtpd/tests/test_main.py index f9ac424..36992f3 100644 --- a/aiosmtpd/tests/test_main.py +++ b/aiosmtpd/tests/test_main.py @@ -16,8 +16,9 @@ import pytest from aiosmtpd import __version__ from aiosmtpd.handlers import Debugging from aiosmtpd.main import main, parseargs +from aiosmtpd.testing.helpers import catchup_delay from aiosmtpd.testing.statuscodes import SMTP_STATUS_CODES as S -from aiosmtpd.tests.conftest import SERVER_CRT, SERVER_KEY +from aiosmtpd.tests.conftest import AUTOSTOP_DELAY, SERVER_CRT, SERVER_KEY try: import pwd @@ -27,10 +28,6 @@ except ImportError: HAS_SETUID = hasattr(os, "setuid") MAIL_LOG = logging.getLogger("mail.log") -# If less than 1.0, might cause intermittent error if test system -# is too busy/overloaded. -AUTOSTOP_DELAY = 1.0 - # region ##### Custom Handlers ######################################################## @@ -53,17 +50,6 @@ class NullHandler: # region ##### Fixtures ############################################################### -@pytest.fixture -def autostop_loop(temp_event_loop) -> Generator[asyncio.AbstractEventLoop, None, None]: - # Create a new event loop, and arrange for that loop to end almost - # immediately. This will allow the calls to main() in these tests to - # also exit almost immediately. Otherwise, the foreground test - # process will hang. - temp_event_loop.call_later(AUTOSTOP_DELAY, temp_event_loop.stop) - # - yield temp_event_loop - - @pytest.fixture def nobody_uid() -> Generator[int, None, None]: if pwd is None: @@ -97,10 +83,10 @@ def watch_for_tls(ready_flag, retq: MP.Queue): req_tls = False ready_flag.set() start = time.monotonic() - delay = AUTOSTOP_DELAY * 1.5 + delay = AUTOSTOP_DELAY * 4 while (time.monotonic() - start) <= delay: try: - with SMTPClient("localhost", 8025) as client: + with SMTPClient("localhost", 8025, timeout=0.1) as client: resp = client.docmd("HELP", "HELO") if resp == S.S530_STARTTLS_FIRST: req_tls = True @@ -121,7 +107,7 @@ def watch_for_smtps(ready_flag, retq: MP.Queue): delay = AUTOSTOP_DELAY * 1.5 while (time.monotonic() - start) <= delay: try: - with SMTP_SSL("localhost", 8025) as client: + with SMTP_SSL("localhost", 8025, timeout=0.1) as client: client.ehlo("exemple.org") has_smtps = True break @@ -215,6 +201,7 @@ class TestMainByWatcher: with watcher_process(watch_for_tls) as retq: temp_event_loop.call_later(AUTOSTOP_DELAY, temp_event_loop.stop) main_n("--tlscert", str(SERVER_CRT), "--tlskey", str(SERVER_KEY)) + catchup_delay() has_starttls = retq.get() assert has_starttls is True require_tls = retq.get() @@ -230,6 +217,7 @@ class TestMainByWatcher: str(SERVER_KEY), "--no-requiretls", ) + catchup_delay() has_starttls = retq.get() assert has_starttls is True require_tls = retq.get() @@ -239,6 +227,7 @@ class TestMainByWatcher: with watcher_process(watch_for_smtps) as retq: temp_event_loop.call_later(AUTOSTOP_DELAY, temp_event_loop.stop) main_n("--smtpscert", str(SERVER_CRT), "--smtpskey", str(SERVER_KEY)) + catchup_delay() has_smtps = retq.get() assert has_smtps is True diff --git a/aiosmtpd/tests/test_server.py b/aiosmtpd/tests/test_server.py index 99c5630..41225dc 100644 --- a/aiosmtpd/tests/test_server.py +++ b/aiosmtpd/tests/test_server.py @@ -3,16 +3,18 @@ """Test other aspects of the server implementation.""" +import asyncio import errno import platform import socket -import ssl import time from contextlib import ExitStack from functools import partial from pathlib import Path +from smtplib import SMTP as SMTPClient, SMTPServerDisconnected from tempfile import mkdtemp -from typing import Generator +from threading import Thread +from typing import Generator, Optional import pytest from pytest_mock import MockFixture @@ -20,13 +22,17 @@ from pytest_mock import MockFixture from aiosmtpd.controller import ( Controller, UnixSocketController, + UnthreadedController, + UnixSocketMixin, + UnixSocketUnthreadedController, _FakeServer, get_localhost, ) from aiosmtpd.handlers import Sink from aiosmtpd.smtp import SMTP as Server +from aiosmtpd.testing.helpers import catchup_delay -from .conftest import Global +from .conftest import Global, AUTOSTOP_DELAY class SlowStartController(Controller): @@ -91,6 +97,45 @@ def safe_socket_dir() -> Generator[Path, None, None]: tmpdir.rmdir() +def assert_smtp_socket(controller: UnixSocketMixin): + assert Path(controller.unix_socket).exists() + sockfile = controller.unix_socket + ssl_context = controller.ssl_context + with ExitStack() as stk: + sock: socket.socket = stk.enter_context( + socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + ) + sock.settimeout(AUTOSTOP_DELAY) + sock.connect(str(sockfile)) + if ssl_context: + sock = stk.enter_context(ssl_context.wrap_socket(sock)) + catchup_delay() + try: + resp = sock.recv(1024) + except socket.timeout: + return False + if not resp: + return False + assert resp.startswith(b"220 ") + assert resp.endswith(b"\r\n") + sock.send(b"EHLO socket.test\r\n") + # We need to "build" resparr because, especially when socket is wrapped + # in SSL, the SMTP server takes it sweet time responding with the list + # of ESMTP features ... + resparr = bytearray() + while not resparr.endswith(b"250 HELP\r\n"): + catchup_delay() + resp = sock.recv(1024) + if not resp: + break + resparr += resp + assert resparr.endswith(b"250 HELP\r\n") + sock.send(b"QUIT\r\n") + catchup_delay() + resp = sock.recv(1024) + assert resp.startswith(b"221") + + class TestServer: """Tests for the aiosmtpd.smtp.SMTP class""" @@ -272,10 +317,7 @@ class TestController: # Apparently errno.E* constants adapts to the OS, so on Windows they will # automatically use the analogous WSAE* constants - @pytest.mark.parametrize( - "err", - [errno.EADDRNOTAVAIL, errno.EAFNOSUPPORT] - ) + @pytest.mark.parametrize("err", [errno.EADDRNOTAVAIL, errno.EAFNOSUPPORT]) def test_getlocalhost_6no(self, mocker, err): mock_makesock: mocker.Mock = mocker.patch( "aiosmtpd.controller.makesock", @@ -320,70 +362,176 @@ class TestController: @pytest.mark.skipif(in_cygwin(), reason="Cygwin AF_UNIX is problematic") @pytest.mark.skipif(in_win32(), reason="Win32 does not yet fully implement AF_UNIX") class TestUnixSocketController: - sockfile: Path = None - - def _assert_good_server(self, ssl_context: ssl.SSLContext = None): - # Note: all those time.sleep()s are necessary - # Remember that we're running in "Threaded" mode, and there's the GIL... - # The time.sleep()s lets go of the GIL allowing the asyncio loop to move - # forward - assert self.sockfile.exists() - with ExitStack() as stk: - sock: socket.socket = stk.enter_context( - socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - ) - sock.connect(str(self.sockfile)) - if ssl_context: - sock = stk.enter_context(ssl_context.wrap_socket(sock)) - time.sleep(0.1) - resp = sock.recv(1024) - assert resp.startswith(b"220 ") - assert resp.endswith(b"\r\n") - sock.send(b"EHLO socket.test\r\n") - # We need to "build" resparr because, especially when socket is wrapped - # in SSL, the SMTP server takes it sweet time responding with the list - # of ESMTP features ... - resparr = bytearray() - while not resparr.endswith(b"250 HELP\r\n"): - time.sleep(0.1) - resp = sock.recv(1024) - if not resp: - break - resparr += resp - assert resparr.endswith(b"250 HELP\r\n") - sock.send(b"QUIT\r\n") - time.sleep(0.1) - resp = sock.recv(1024) - assert resp.startswith(b"221") - def test_server_creation(self, safe_socket_dir): - self.sockfile = safe_socket_dir / "smtp" - cont = UnixSocketController(Sink(), unix_socket=self.sockfile) + sockfile = safe_socket_dir / "smtp" + cont = UnixSocketController(Sink(), unix_socket=sockfile) try: cont.start() - self._assert_good_server() + assert_smtp_socket(cont) finally: cont.stop() def test_server_creation_ssl(self, safe_socket_dir, ssl_context_server): - self.sockfile = safe_socket_dir / "smtp" + sockfile = safe_socket_dir / "smtp" cont = UnixSocketController( - Sink(), unix_socket=self.sockfile, ssl_context=ssl_context_server + Sink(), unix_socket=sockfile, ssl_context=ssl_context_server ) try: cont.start() # Allow additional time for SSL to kick in - time.sleep(0.1) - self._assert_good_server(ssl_context_server) + catchup_delay() + assert_smtp_socket(cont) finally: cont.stop() +class TestUnthreaded: + @pytest.fixture + def runner(self): + thread: Optional[Thread] = None + + def _runner(loop: asyncio.AbstractEventLoop): + loop.run_forever() + + def starter(loop: asyncio.AbstractEventLoop): + nonlocal thread + thread = Thread(target=_runner, args=(loop,)) + thread.setDaemon(True) + thread.start() + catchup_delay() + + def joiner(timeout: float = None): + nonlocal thread + assert isinstance(thread, Thread) + thread.join(timeout=timeout) + + def is_alive(): + nonlocal thread + assert isinstance(thread, Thread) + return thread.is_alive() + + starter.join = joiner + starter.is_alive = is_alive + return starter + + @pytest.mark.skipif(in_cygwin(), reason="Cygwin AF_UNIX is problematic") + @pytest.mark.skipif(in_win32(), reason="Win32 does not yet fully implement AF_UNIX") + def test_unixsocket(self, safe_socket_dir, autostop_loop, runner): + sockfile = safe_socket_dir / "smtp" + cont = UnixSocketUnthreadedController( + Sink(), unix_socket=sockfile, loop=autostop_loop + ) + cont.begin() + # Make sure event loop is not running (will be started in thread) + assert autostop_loop.is_running() is False + runner(autostop_loop) + # Make sure event loop is up and running (started within thread) + assert autostop_loop.is_running() is True + # Check we can connect + assert_smtp_socket(cont) + # Wait until thread ends, which it will be when the loop autostops + runner.join(timeout=AUTOSTOP_DELAY) + assert runner.is_alive() is False + catchup_delay() + assert autostop_loop.is_running() is False + # At this point, the loop _has_ stopped, but the task is still listening + assert assert_smtp_socket(cont) is False + # Stop the task + cont.end() + catchup_delay() + # Now the listener has gone away + # noinspection PyTypeChecker + with pytest.raises((socket.timeout, ConnectionError)): + assert_smtp_socket(cont) + + @pytest.mark.filterwarnings( + "ignore::pytest.PytestUnraisableExceptionWarning" + ) + def test_inet_loopstop(self, autostop_loop, runner): + """ + Verify behavior when the loop is stopped before controller is stopped + """ + autostop_loop.set_debug(True) + cont = UnthreadedController(Sink(), loop=autostop_loop) + cont.begin() + # Make sure event loop is not running (will be started in thread) + assert autostop_loop.is_running() is False + runner(autostop_loop) + # Make sure event loop is up and running (started within thread) + assert autostop_loop.is_running() is True + # Check we can connect + with SMTPClient(cont.hostname, cont.port, timeout=AUTOSTOP_DELAY) as client: + code, _ = client.helo("example.org") + assert code == 250 + # Wait until thread ends, which it will be when the loop autostops + runner.join(timeout=AUTOSTOP_DELAY) + assert runner.is_alive() is False + catchup_delay() + assert autostop_loop.is_running() is False + # At this point, the loop _has_ stopped, but the task is still listening, + # so rather than socket.timeout, we'll get a refusal instead, thus causing + # SMTPServerDisconnected + with pytest.raises(SMTPServerDisconnected): + SMTPClient(cont.hostname, cont.port, timeout=0.1) + cont.end() + catchup_delay() + cont.ended.wait() + # Now the listener has gone away, and thus we will end up with socket.timeout + # or ConnectionError (depending on OS) + # noinspection PyTypeChecker + with pytest.raises((socket.timeout, ConnectionError)): + SMTPClient(cont.hostname, cont.port, timeout=0.1) + + @pytest.mark.filterwarnings( + "ignore::pytest.PytestUnraisableExceptionWarning" + ) + def test_inet_contstop(self, temp_event_loop, runner): + """ + Verify behavior when the controller is stopped before loop is stopped + """ + cont = UnthreadedController(Sink(), loop=temp_event_loop) + cont.begin() + # Make sure event loop is not running (will be started in thread) + assert temp_event_loop.is_running() is False + runner(temp_event_loop) + # Make sure event loop is up and running + assert temp_event_loop.is_running() is True + try: + # Check that we can connect + with SMTPClient(cont.hostname, cont.port, timeout=AUTOSTOP_DELAY) as client: + code, _ = client.helo("example.org") + assert code == 250 + client.quit() + catchup_delay() + temp_event_loop.call_soon_threadsafe(cont.end) + for _ in range(10): # 10 is arbitrary + catchup_delay() # effectively yield to other threads/event loop + if cont.ended.wait(1.0): + break + assert temp_event_loop.is_running() is True + # Because we've called .end() there, the server listener should've gone + # away, so we should end up with a socket.timeout or ConnectionError or + # SMTPServerDisconnected (depending on lotsa factors) + expect_errs = (socket.timeout, ConnectionError, SMTPServerDisconnected) + # noinspection PyTypeChecker + with pytest.raises(expect_errs): + SMTPClient(cont.hostname, cont.port, timeout=0.1) + finally: + # Wrap up, or else we'll hang + temp_event_loop.call_soon_threadsafe(cont.cancel_tasks) + catchup_delay() + runner.join() + assert runner.is_alive() is False + assert temp_event_loop.is_running() is False + assert temp_event_loop.is_closed() is False + + class TestFactory: def test_normal_situation(self): cont = Controller(Sink()) try: cont.start() + catchup_delay() assert cont.smtpd is not None assert cont._thread_exception is None finally: diff --git a/pyproject.toml b/pyproject.toml index b61bfa6..e067d36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,6 @@ requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -# addopts = """--doctest-glob="*.rst" --strict-markers -rfEX""" addopts = """--strict-markers -rfEX""" markers = [ "client_data", @@ -37,6 +36,7 @@ source = [ [tool.coverage.coverage_conditional_plugin.rules] # Here we specify our pragma rules: py-ge-38 = "sys_version_info >= (3, 8)" +py-lt-38 = "sys_version_info < (3, 8)" py-gt-36 = "sys_version_info > (3, 6)" has-mypy = "is_installed('mypy')" has-pwd = "is_installed('pwd')" @@ -47,10 +47,14 @@ on-wsl = "'Microsoft' in platform_release" # As of 2021-02-07, only WSL has a kernel with "Microsoft" in the version. on-not-win32 = "sys_platform != 'win32'" on-cygwin = "sys_platform == 'cygwin'" +no-unixsock = "sys_platform in {'win32', 'cygwin'}" [tool.coverage.report] exclude_lines = [ "pragma: nocover", + "pragma: no cover", + "@abstract", + 'class \S+\(Protocol\):' ] fail_under = 100 show_missing = true -- 2.32.0