From 2fc3606671150b18633c66e7b3a411a544ce15d6 Mon Sep 17 00:00:00 2001 From: Roman Mogylatov Date: Wed, 5 Aug 2020 22:11:26 -0400 Subject: [PATCH] Asyncio daemon example (#276) * Add prototype * Update .pydocstylerc * Add tests * Remove one of the tests * Fix typo in the giphynav-aiohttp * Add README * Fix flake8 --- examples/.pydocstylerc | 2 +- examples/miniapps/giphynav-aiohttp/README.rst | 2 +- .../monitoring-daemon-asyncio/Dockerfile | 13 +++ .../monitoring-daemon-asyncio/README.rst | 67 ++++++++++++++++ .../monitoring-daemon-asyncio/config.yml | 17 ++++ .../docker-compose.yml | 9 +++ .../monitoringdaemon/__init__.py | 1 + .../monitoringdaemon/__main__.py | 18 +++++ .../monitoringdaemon/containers.py | 43 ++++++++++ .../monitoringdaemon/dispatcher.py | 72 +++++++++++++++++ .../monitoringdaemon/http.py | 11 +++ .../monitoringdaemon/monitors.py | 58 ++++++++++++++ .../monitoringdaemon/tests.py | 79 +++++++++++++++++++ .../requirements.txt | 6 ++ 14 files changed, 396 insertions(+), 2 deletions(-) create mode 100644 examples/miniapps/monitoring-daemon-asyncio/Dockerfile create mode 100644 examples/miniapps/monitoring-daemon-asyncio/README.rst create mode 100644 examples/miniapps/monitoring-daemon-asyncio/config.yml create mode 100644 examples/miniapps/monitoring-daemon-asyncio/docker-compose.yml create mode 100644 examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__init__.py create mode 100644 examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__main__.py create mode 100644 examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/containers.py create mode 100644 examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/dispatcher.py create mode 100644 examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/http.py create mode 100644 examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/monitors.py create mode 100644 examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/tests.py create mode 100644 examples/miniapps/monitoring-daemon-asyncio/requirements.txt diff --git a/examples/.pydocstylerc b/examples/.pydocstylerc index eda12aa0..c5b7bc0f 100644 --- a/examples/.pydocstylerc +++ b/examples/.pydocstylerc @@ -1,2 +1,2 @@ [pydocstyle] -ignore = D101,D103,D107,D203,D213 +ignore = D100,D101,D102,D103,D107,D203,D213 diff --git a/examples/miniapps/giphynav-aiohttp/README.rst b/examples/miniapps/giphynav-aiohttp/README.rst index c019d7da..9ac19d22 100644 --- a/examples/miniapps/giphynav-aiohttp/README.rst +++ b/examples/miniapps/giphynav-aiohttp/README.rst @@ -1,7 +1,7 @@ Aiohttp Dependency Injection Example ==================================== -Application ``giphynavigator`` is a `Aiohttp `_ + +Application ``giphynavigator`` is an `Aiohttp `_ + `Dependency Injector `_ application. Run diff --git a/examples/miniapps/monitoring-daemon-asyncio/Dockerfile b/examples/miniapps/monitoring-daemon-asyncio/Dockerfile new file mode 100644 index 00000000..c4bc7012 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-buster + +ENV PYTHONUNBUFFERED=1 + +WORKDIR /code +COPY . /code/ + +RUN apt-get install openssl \ + && pip install --upgrade pip \ + && pip install -r requirements.txt \ + && rm -rf ~/.cache + +CMD ["python", "-m", "monitoringdaemon"] diff --git a/examples/miniapps/monitoring-daemon-asyncio/README.rst b/examples/miniapps/monitoring-daemon-asyncio/README.rst new file mode 100644 index 00000000..cfda8b9a --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/README.rst @@ -0,0 +1,67 @@ +Asyncio Daemon Dependency Injection Example +=========================================== + +Application ``monitoringdaemon`` is an `asyncio `_ ++ `Dependency Injector `_ application. + +Run +--- + +Build the Docker image: + +.. code-block:: bash + + docker-compose build + +Run the docker-compose environment: + +.. code-block:: bash + + docker-compose up + +The output should be something like: + +.. code-block:: + + Starting monitoring-daemon-asyncio_monitor_1 ... done + Attaching to monitoring-daemon-asyncio_monitor_1 + monitor_1 | [2020-08-06 01:57:08,249] [INFO] [monitoringdaemon.dispatcher]: Dispatcher is starting up + monitor_1 | [2020-08-06 01:57:08,249] [INFO] [monitoringdaemon.dispatcher]: Monitoring task has been started monitoringdaemon.monitors.HttpMonitor(url="http://example.com") + monitor_1 | [2020-08-06 01:57:08,249] [INFO] [monitoringdaemon.dispatcher]: Monitoring task has been started monitoringdaemon.monitors.HttpMonitor(url="http://httpbin.org/get") + monitor_1 | [2020-08-06 01:57:08,318] [INFO] [monitoringdaemon.monitors.HttpMonitor(url="http://example.com")]: Response code: 200, content length: 648, request took: 0.067 seconds + monitor_1 | [2020-08-06 01:57:08,363] [INFO] [monitoringdaemon.monitors.HttpMonitor(url="http://httpbin.org/get")]: Response code: 200, content length: 309, request took: 0.112 seconds + +Test +---- + +This application comes with the unit tests. + +To run the tests do: + +.. code-block:: bash + + docker-compose run --rm monitor py.test monitoringdaemon/tests.py --cov=monitoringdaemon + +The output should be something like: + +.. code-block:: + + platform linux -- Python 3.8.3, pytest-6.0.1, py-1.9.0, pluggy-0.13.1 + rootdir: /code + plugins: asyncio-0.14.0, cov-2.10.0 + collected 2 items + + monitoringdaemon/tests.py .. [100%] + + ----------- coverage: platform linux, python 3.8.3-final-0 ----------- + Name Stmts Miss Cover + ---------------------------------------------------- + monitoringdaemon/__init__.py 0 0 100% + monitoringdaemon/__main__.py 9 9 0% + monitoringdaemon/containers.py 11 0 100% + monitoringdaemon/dispatcher.py 45 5 89% + monitoringdaemon/http.py 6 3 50% + monitoringdaemon/monitors.py 29 2 93% + monitoringdaemon/tests.py 37 0 100% + ---------------------------------------------------- + TOTAL 137 19 86% diff --git a/examples/miniapps/monitoring-daemon-asyncio/config.yml b/examples/miniapps/monitoring-daemon-asyncio/config.yml new file mode 100644 index 00000000..5bbe8795 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/config.yml @@ -0,0 +1,17 @@ +log: + level: "INFO" + format: "[%(asctime)s] [%(levelname)s] [%(name)s]: %(message)s" + +monitors: + + example: + method: "GET" + url: "http://example.com" + timeout: 5 + check_every: 5 + + httpbin: + method: "GET" + url: "http://httpbin.org/get" + timeout: 5 + check_every: 5 diff --git a/examples/miniapps/monitoring-daemon-asyncio/docker-compose.yml b/examples/miniapps/monitoring-daemon-asyncio/docker-compose.yml new file mode 100644 index 00000000..2857d812 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/docker-compose.yml @@ -0,0 +1,9 @@ +version: "3.7" + +services: + + monitor: + build: ./ + image: monitoring-daemon + volumes: + - "./:/code" diff --git a/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__init__.py b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__init__.py new file mode 100644 index 00000000..1c744ca5 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__init__.py @@ -0,0 +1 @@ +"""Top-level package.""" diff --git a/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__main__.py b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__main__.py new file mode 100644 index 00000000..69b66dd7 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__main__.py @@ -0,0 +1,18 @@ +"""Main module.""" + +from .containers import ApplicationContainer + + +def main() -> None: + """Run the application.""" + container = ApplicationContainer() + + container.config.from_yaml('config.yml') + container.configure_logging() + + dispatcher = container.dispatcher() + dispatcher.run() + + +if __name__ == '__main__': + main() diff --git a/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/containers.py b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/containers.py new file mode 100644 index 00000000..c66ec327 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/containers.py @@ -0,0 +1,43 @@ +"""Application containers module.""" + +import logging +import sys + +from dependency_injector import containers, providers + +from . import http, monitors, dispatcher + + +class ApplicationContainer(containers.DeclarativeContainer): + """Application container.""" + + config = providers.Configuration() + + configure_logging = providers.Callable( + logging.basicConfig, + stream=sys.stdout, + level=config.log.level, + format=config.log.format, + ) + + http_client = providers.Factory(http.HttpClient) + + example_monitor = providers.Factory( + monitors.HttpMonitor, + http_client=http_client, + options=config.monitors.example, + ) + + httpbin_monitor = providers.Factory( + monitors.HttpMonitor, + http_client=http_client, + options=config.monitors.httpbin, + ) + + dispatcher = providers.Factory( + dispatcher.Dispatcher, + monitors=providers.List( + example_monitor, + httpbin_monitor, + ), + ) diff --git a/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/dispatcher.py b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/dispatcher.py new file mode 100644 index 00000000..d12c22b9 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/dispatcher.py @@ -0,0 +1,72 @@ +"""Dispatcher module.""" + +import asyncio +import logging +import signal +import time +from typing import List + +from .monitors import Monitor + + +logger = logging.getLogger(__name__) + + +class Dispatcher: + + def __init__(self, monitors: List[Monitor]) -> None: + self._monitors = monitors + self._monitor_tasks: List[asyncio.Task] = [] + self._stopping = False + + def run(self) -> None: + asyncio.run(self.start()) + + async def start(self) -> None: + logger.info('Dispatcher is starting up') + + for monitor in self._monitors: + self._monitor_tasks.append( + asyncio.create_task(self._run_monitor(monitor)), + ) + logger.info( + 'Monitoring task has been started %s', + monitor.full_name, + ) + + asyncio.get_event_loop().add_signal_handler(signal.SIGTERM, self.stop) + asyncio.get_event_loop().add_signal_handler(signal.SIGINT, self.stop) + + await asyncio.gather(*self._monitor_tasks, return_exceptions=True) + + self.stop() + + def stop(self) -> None: + if self._stopping: + return + + self._stopping = True + + logger.info('Dispatcher is shutting down') + for task, monitor in zip(self._monitor_tasks, self._monitors): + task.cancel() + logger.info('Monitoring task has been stopped %s', monitor.full_name) + logger.info('Dispatcher shutting down finished successfully') + + @staticmethod + async def _run_monitor(monitor: Monitor) -> None: + def _until_next(last: float) -> float: + time_took = time.time() - last + return monitor.check_every - time_took + + while True: + time_start = time.time() + + try: + await monitor.check() + except asyncio.CancelledError: + break + except Exception: + monitor.logger.exception('Error running monitoring check') + + await asyncio.sleep(_until_next(last=time_start)) diff --git a/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/http.py b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/http.py new file mode 100644 index 00000000..9b07542a --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/http.py @@ -0,0 +1,11 @@ +"""Http client module.""" + +from aiohttp import ClientSession, ClientTimeout, ClientResponse + + +class HttpClient: + + async def request(self, method: str, url: str, timeout: int) -> ClientResponse: + async with ClientSession(timeout=ClientTimeout(timeout)) as session: + async with session.request(method, url) as response: + return response diff --git a/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/monitors.py b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/monitors.py new file mode 100644 index 00000000..3ffe71a7 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/monitors.py @@ -0,0 +1,58 @@ +"""Monitors module.""" + +import logging +import time +from typing import Dict, Any + +from .http import HttpClient + + +class Monitor: + + def __init__(self, check_every: int) -> None: + self.check_every = check_every + self.logger = logging.getLogger(self.full_name) + + @property + def full_name(self) -> str: + raise NotImplementedError() + + async def check(self) -> None: + raise NotImplementedError() + + +class HttpMonitor(Monitor): + + def __init__( + self, + http_client: HttpClient, + options: Dict[str, Any], + ) -> None: + self._client = http_client + self._method = options.pop('method') + self._url = options.pop('url') + self._timeout = options.pop('timeout') + super().__init__(check_every=options.pop('check_every')) + + @property + def full_name(self) -> str: + return '{0}.{1}(url="{2}")'.format(__name__, self.__class__.__name__, self._url) + + async def check(self) -> None: + time_start = time.time() + + response = await self._client.request( + method=self._method, + url=self._url, + timeout=self._timeout, + ) + + time_end = time.time() + time_took = time_end - time_start + + self.logger.info( + 'Response code: %s, content length: %s, request took: %s seconds', + response.status, + response.content_length, + round(time_took, 3) + ) diff --git a/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/tests.py b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/tests.py new file mode 100644 index 00000000..3fb712c6 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/tests.py @@ -0,0 +1,79 @@ +"""Tests module.""" + +import asyncio +import dataclasses +from unittest import mock + +import pytest + +from .containers import ApplicationContainer + + +@dataclasses.dataclass +class RequestStub: + status: int + content_length: int + + +@pytest.fixture +def container(): + container = ApplicationContainer() + container.config.from_dict({ + 'log': { + 'level': 'INFO', + 'formant': '[%(asctime)s] [%(levelname)s] [%(name)s]: %(message)s', + }, + 'monitors': { + 'example': { + 'method': 'GET', + 'url': 'http://fake-example.com', + 'timeout': 1, + 'check_every': 1, + }, + 'httpbin': { + 'method': 'GET', + 'url': 'http://fake-httpbin.org/get', + 'timeout': 1, + 'check_every': 1, + }, + }, + }) + return container + + +@pytest.mark.asyncio +async def test_example_monitor(container, caplog): + caplog.set_level('INFO') + + http_client_mock = mock.AsyncMock() + http_client_mock.request.return_value = RequestStub( + status=200, + content_length=635, + ) + + with container.http_client.override(http_client_mock): + example_monitor = container.example_monitor() + await example_monitor.check() + + assert 'http://fake-example.com' in caplog.text + assert 'Response code: 200' in caplog.text + assert 'content length: 635' in caplog.text + + +@pytest.mark.asyncio +async def test_dispatcher(container, caplog, event_loop): + caplog.set_level('INFO') + + example_monitor_mock = mock.AsyncMock() + httpbin_monitor_mock = mock.AsyncMock() + + with container.example_monitor.override(example_monitor_mock), \ + container.httpbin_monitor.override(httpbin_monitor_mock): + + dispatcher = container.dispatcher() + event_loop.create_task(dispatcher.start()) + await asyncio.sleep(0.1) + dispatcher.stop() + + assert example_monitor_mock.check.called + assert httpbin_monitor_mock.check.called diff --git a/examples/miniapps/monitoring-daemon-asyncio/requirements.txt b/examples/miniapps/monitoring-daemon-asyncio/requirements.txt new file mode 100644 index 00000000..ebaadda6 --- /dev/null +++ b/examples/miniapps/monitoring-daemon-asyncio/requirements.txt @@ -0,0 +1,6 @@ +dependency-injector +aiohttp +pyyaml +pytest +pytest-asyncio +pytest-cov