Asyncio daemon example (#276)

* Add prototype

* Update .pydocstylerc

* Add tests

* Remove one of the tests

* Fix typo in the giphynav-aiohttp

* Add README

* Fix flake8

examples/.pydocstylerc

@@ -1,2 +1,2 @@
 [pydocstyle]
-ignore = D101,D103,D107,D203,D213
+ignore = D100,D101,D102,D103,D107,D203,D213

examples/miniapps/giphynav-aiohttp/README.rst

@@ -1,7 +1,7 @@
 Aiohttp Dependency Injection Example
 ====================================
 
-Application ``giphynavigator`` is a `Aiohttp <https://docs.aiohttp.org/>`_ +
+Application ``giphynavigator`` is an `Aiohttp <https://docs.aiohttp.org/>`_ +
 `Dependency Injector <http://python-dependency-injector.ets-labs.org/>`_ application.
 
 Run

examples/miniapps/monitoring-daemon-asyncio/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.8-buster
+
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /code
+COPY . /code/
+
+RUN apt-get install openssl \
+ && pip install --upgrade pip \
+ && pip install -r requirements.txt \
+ && rm -rf ~/.cache
+
+CMD ["python", "-m", "monitoringdaemon"]

examples/miniapps/monitoring-daemon-asyncio/README.rst

@@ -0,0 +1,67 @@
+Asyncio Daemon Dependency Injection Example
+===========================================
+
+Application ``monitoringdaemon`` is an `asyncio <https://docs.python.org/3/library/asyncio.html>`_
++ `Dependency Injector <http://python-dependency-injector.ets-labs.org/>`_ application.
+
+Run
+---
+
+Build the Docker image:
+
+.. code-block:: bash
+
+   docker-compose build
+
+Run the docker-compose environment:
+
+.. code-block:: bash
+
+    docker-compose up
+
+The output should be something like:
+
+.. code-block::
+
+   Starting monitoring-daemon-asyncio_monitor_1 ... done
+   Attaching to monitoring-daemon-asyncio_monitor_1
+   monitor_1  | [2020-08-06 01:57:08,249] [INFO] [monitoringdaemon.dispatcher]: Dispatcher is starting up
+   monitor_1  | [2020-08-06 01:57:08,249] [INFO] [monitoringdaemon.dispatcher]: Monitoring task has been started monitoringdaemon.monitors.HttpMonitor(url="http://example.com")
+   monitor_1  | [2020-08-06 01:57:08,249] [INFO] [monitoringdaemon.dispatcher]: Monitoring task has been started monitoringdaemon.monitors.HttpMonitor(url="http://httpbin.org/get")
+   monitor_1  | [2020-08-06 01:57:08,318] [INFO] [monitoringdaemon.monitors.HttpMonitor(url="http://example.com")]: Response code: 200, content length: 648, request took: 0.067 seconds
+   monitor_1  | [2020-08-06 01:57:08,363] [INFO] [monitoringdaemon.monitors.HttpMonitor(url="http://httpbin.org/get")]: Response code: 200, content length: 309, request took: 0.112 seconds
+
+Test
+----
+
+This application comes with the unit tests.
+
+To run the tests do:
+
+.. code-block:: bash
+
+   docker-compose run --rm monitor py.test monitoringdaemon/tests.py --cov=monitoringdaemon
+
+The output should be something like:
+
+.. code-block::
+
+   platform linux -- Python 3.8.3, pytest-6.0.1, py-1.9.0, pluggy-0.13.1
+   rootdir: /code
+   plugins: asyncio-0.14.0, cov-2.10.0
+   collected 2 items
+
+   monitoringdaemon/tests.py ..                                     [100%]
+
+   ----------- coverage: platform linux, python 3.8.3-final-0 -----------
+   Name                             Stmts   Miss  Cover
+   ----------------------------------------------------
+   monitoringdaemon/__init__.py         0      0   100%
+   monitoringdaemon/__main__.py         9      9     0%
+   monitoringdaemon/containers.py      11      0   100%
+   monitoringdaemon/dispatcher.py      45      5    89%
+   monitoringdaemon/http.py             6      3    50%
+   monitoringdaemon/monitors.py        29      2    93%
+   monitoringdaemon/tests.py           37      0   100%
+   ----------------------------------------------------
+   TOTAL                              137     19    86%

examples/miniapps/monitoring-daemon-asyncio/config.yml

@@ -0,0 +1,17 @@
+log:
+  level: "INFO"
+  format: "[%(asctime)s] [%(levelname)s] [%(name)s]: %(message)s"
+
+monitors:
+
+  example:
+    method: "GET"
+    url: "http://example.com"
+    timeout: 5
+    check_every: 5
+
+  httpbin:
+    method: "GET"
+    url: "http://httpbin.org/get"
+    timeout: 5
+    check_every: 5

examples/miniapps/monitoring-daemon-asyncio/docker-compose.yml

@@ -0,0 +1,9 @@
+version: "3.7"
+
+services:
+
+  monitor:
+    build: ./
+    image: monitoring-daemon
+    volumes:
+      - "./:/code"

examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__init__.py

@@ -0,0 +1 @@
+"""Top-level package."""

examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/__main__.py

@@ -0,0 +1,18 @@
+"""Main module."""
+
+from .containers import ApplicationContainer
+
+
+def main() -> None:
+    """Run the application."""
+    container = ApplicationContainer()
+
+    container.config.from_yaml('config.yml')
+    container.configure_logging()
+
+    dispatcher = container.dispatcher()
+    dispatcher.run()
+
+
+if __name__ == '__main__':
+    main()

examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/containers.py

@@ -0,0 +1,43 @@
+"""Application containers module."""
+
+import logging
+import sys
+
+from dependency_injector import containers, providers
+
+from . import http, monitors, dispatcher
+
+
+class ApplicationContainer(containers.DeclarativeContainer):
+    """Application container."""
+
+    config = providers.Configuration()
+
+    configure_logging = providers.Callable(
+        logging.basicConfig,
+        stream=sys.stdout,
+        level=config.log.level,
+        format=config.log.format,
+    )
+
+    http_client = providers.Factory(http.HttpClient)
+
+    example_monitor = providers.Factory(
+        monitors.HttpMonitor,
+        http_client=http_client,
+        options=config.monitors.example,
+    )
+
+    httpbin_monitor = providers.Factory(
+        monitors.HttpMonitor,
+        http_client=http_client,
+        options=config.monitors.httpbin,
+    )
+
+    dispatcher = providers.Factory(
+        dispatcher.Dispatcher,
+        monitors=providers.List(
+            example_monitor,
+            httpbin_monitor,
+        ),
+    )

examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/dispatcher.py

@@ -0,0 +1,72 @@
+"""Dispatcher module."""
+
+import asyncio
+import logging
+import signal
+import time
+from typing import List
+
+from .monitors import Monitor
+
+
+logger = logging.getLogger(__name__)
+
+
+class Dispatcher:
+
+    def __init__(self, monitors: List[Monitor]) -> None:
+        self._monitors = monitors
+        self._monitor_tasks: List[asyncio.Task] = []
+        self._stopping = False
+
+    def run(self) -> None:
+        asyncio.run(self.start())
+
+    async def start(self) -> None:
+        logger.info('Dispatcher is starting up')
+
+        for monitor in self._monitors:
+            self._monitor_tasks.append(
+                asyncio.create_task(self._run_monitor(monitor)),
+            )
+            logger.info(
+                'Monitoring task has been started %s',
+                monitor.full_name,
+            )
+
+        asyncio.get_event_loop().add_signal_handler(signal.SIGTERM, self.stop)
+        asyncio.get_event_loop().add_signal_handler(signal.SIGINT, self.stop)
+
+        await asyncio.gather(*self._monitor_tasks, return_exceptions=True)
+
+        self.stop()
+
+    def stop(self) -> None:
+        if self._stopping:
+            return
+
+        self._stopping = True
+
+        logger.info('Dispatcher is shutting down')
+        for task, monitor in zip(self._monitor_tasks, self._monitors):
+            task.cancel()
+            logger.info('Monitoring task has been stopped %s', monitor.full_name)
+        logger.info('Dispatcher shutting down finished successfully')
+
+    @staticmethod
+    async def _run_monitor(monitor: Monitor) -> None:
+        def _until_next(last: float) -> float:
+            time_took = time.time() - last
+            return monitor.check_every - time_took
+
+        while True:
+            time_start = time.time()
+
+            try:
+                await monitor.check()
+            except asyncio.CancelledError:
+                break
+            except Exception:
+                monitor.logger.exception('Error running monitoring check')
+
+            await asyncio.sleep(_until_next(last=time_start))

examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/http.py

@@ -0,0 +1,11 @@
+"""Http client module."""
+
+from aiohttp import ClientSession, ClientTimeout, ClientResponse
+
+
+class HttpClient:
+
+    async def request(self, method: str, url: str, timeout: int) -> ClientResponse:
+        async with ClientSession(timeout=ClientTimeout(timeout)) as session:
+            async with session.request(method, url) as response:
+                return response

examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/monitors.py

@@ -0,0 +1,58 @@
+"""Monitors module."""
+
+import logging
+import time
+from typing import Dict, Any
+
+from .http import HttpClient
+
+
+class Monitor:
+
+    def __init__(self, check_every: int) -> None:
+        self.check_every = check_every
+        self.logger = logging.getLogger(self.full_name)
+
+    @property
+    def full_name(self) -> str:
+        raise NotImplementedError()
+
+    async def check(self) -> None:
+        raise NotImplementedError()
+
+
+class HttpMonitor(Monitor):
+
+    def __init__(
+            self,
+            http_client: HttpClient,
+            options: Dict[str, Any],
+    ) -> None:
+        self._client = http_client
+        self._method = options.pop('method')
+        self._url = options.pop('url')
+        self._timeout = options.pop('timeout')
+        super().__init__(check_every=options.pop('check_every'))
+
+    @property
+    def full_name(self) -> str:
+        return '{0}.{1}(url="{2}")'.format(__name__, self.__class__.__name__, self._url)
+
+    async def check(self) -> None:
+        time_start = time.time()
+
+        response = await self._client.request(
+            method=self._method,
+            url=self._url,
+            timeout=self._timeout,
+        )
+
+        time_end = time.time()
+        time_took = time_end - time_start
+
+        self.logger.info(
+            'Response code: %s, content length: %s, request took: %s seconds',
+            response.status,
+            response.content_length,
+            round(time_took, 3)
+        )

examples/miniapps/monitoring-daemon-asyncio/monitoringdaemon/tests.py

@@ -0,0 +1,79 @@
+"""Tests module."""
+
+import asyncio
+import dataclasses
+from unittest import mock
+
+import pytest
+
+from .containers import ApplicationContainer
+
+
+@dataclasses.dataclass
+class RequestStub:
+    status: int
+    content_length: int
+
+
+@pytest.fixture
+def container():
+    container = ApplicationContainer()
+    container.config.from_dict({
+        'log': {
+            'level': 'INFO',
+            'formant': '[%(asctime)s] [%(levelname)s] [%(name)s]: %(message)s',
+        },
+        'monitors': {
+            'example': {
+                'method': 'GET',
+                'url': 'http://fake-example.com',
+                'timeout': 1,
+                'check_every': 1,
+            },
+            'httpbin': {
+                'method': 'GET',
+                'url': 'http://fake-httpbin.org/get',
+                'timeout': 1,
+                'check_every': 1,
+            },
+        },
+    })
+    return container
+
+
+@pytest.mark.asyncio
+async def test_example_monitor(container, caplog):
+    caplog.set_level('INFO')
+
+    http_client_mock = mock.AsyncMock()
+    http_client_mock.request.return_value = RequestStub(
+        status=200,
+        content_length=635,
+    )
+
+    with container.http_client.override(http_client_mock):
+        example_monitor = container.example_monitor()
+        await example_monitor.check()
+
+    assert 'http://fake-example.com' in caplog.text
+    assert 'Response code: 200' in caplog.text
+    assert 'content length: 635' in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_dispatcher(container, caplog, event_loop):
+    caplog.set_level('INFO')
+
+    example_monitor_mock = mock.AsyncMock()
+    httpbin_monitor_mock = mock.AsyncMock()
+
+    with container.example_monitor.override(example_monitor_mock), \
+            container.httpbin_monitor.override(httpbin_monitor_mock):
+
+        dispatcher = container.dispatcher()
+        event_loop.create_task(dispatcher.start())
+        await asyncio.sleep(0.1)
+        dispatcher.stop()
+
+    assert example_monitor_mock.check.called
+    assert httpbin_monitor_mock.check.called

examples/miniapps/monitoring-daemon-asyncio/requirements.txt

@@ -0,0 +1,6 @@
+dependency-injector
+aiohttp
+pyyaml
+pytest
+pytest-asyncio
+pytest-cov