From d0e841b41d3019c88f09b237012790c96d5f3cc2 Mon Sep 17 00:00:00 2001 From: Sergey Klyuykov Date: Thu, 15 Aug 2019 09:58:34 +1000 Subject: [PATCH 01/13] Added support for executing from interpreter. For run from python interpreter as module use command: `python -m daphne [all daphne arguments]` --- daphne/__main__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 daphne/__main__.py diff --git a/daphne/__main__.py b/daphne/__main__.py new file mode 100644 index 0000000..2715716 --- /dev/null +++ b/daphne/__main__.py @@ -0,0 +1,3 @@ +from daphne.cli import CommandLineInterface + +CommandLineInterface.entrypoint() From b96720390fc2298617d9a86adac3fa7641ada9b4 Mon Sep 17 00:00:00 2001 From: Michael K Date: Tue, 20 Oct 2020 14:44:54 +0000 Subject: [PATCH 02/13] Switch from Travis CI to GitHub Actions (#336) * Add GitHub Actions * Remove Travis CI * Remove known first party from isort's config --- .github/workflows/tests.yml | 47 ++++++++++++++++++++++++++++++++++++ .travis.yml | 48 ------------------------------------- README.rst | 3 --- daphne/testing.py | 4 ++-- setup.cfg | 20 ++++++---------- tests/test_http_request.py | 3 +-- tests/test_http_response.py | 3 +-- tests/test_websocket.py | 3 +-- tox.ini | 23 ++++++++++++++++++ 9 files changed, 82 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/tests.yml delete mode 100644 .travis.yml create mode 100644 tox.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..0756665 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,47 @@ +name: Tests + +on: + push: + branches: + - master + pull_request: + +jobs: + tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - 3.6 + - 3.7 + - 3.8 + - 3.9 + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools tox + - name: Run tox targets for ${{ matrix.python-version }} + run: | + ENV_PREFIX=$(tr -C -d "0-9" <<< "${{ matrix.python-version }}") + TOXENV=$(tox --listenvs | grep "^py$ENV_PREFIX" | tr '\n' ',') python -m tox + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip tox + - name: Run lint + run: tox -e qa diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bf7be68..0000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -sudo: false - -language: python - -python: -- '3.8' -- '3.7' -- '3.6' -- '3.5' - -env: -- TWISTED="twisted" -- TWISTED="twisted==18.7.0" - -install: -- pip install $TWISTED -e .[tests] -- pip freeze - -script: -- pytest - -stages: - - lint - - test - - name: release - if: tag IS present - -jobs: - include: - - stage: lint - python: 3.6 - install: pip install -U -e .[tests] black pyflakes isort - script: - - pyflakes daphne tests - - black --check daphne tests - - isort --check-only --diff --recursive daphne tests - - - stage: release - python: 3.6 - script: skip - deploy: - provider: pypi - user: andrewgodwin_bot - on: - tags: true - distributions: sdist bdist_wheel - password: - secure: IA+dvSmMKN+fT47rgRb6zdmrExhK5QCVEDH8kheC6kAacw80ORBZKo6sMX9GQBJ3BlfhTqrzAhItHkDUxonb579rJDvmlJ7FPg7axZpsY9Fmls6q1rJC/La8iGWx20+ctberejKSH3wSwa0LH0imJXGDoKKzf1DLmk5pEEWjG2QqhKdEtyAcnzOPnDWcRCs+DKfQcMzETH7lMFN8oe3aBhHLLtcg4yA78cN5CeyyH92lmbaVp7k/b1FqXXFgf16bi5tlgLrb6DhmcnNjwLMSHRafNoPCXkWQOwh6gEHeHRR3OsHsBueyJHIikuHNrpmgpAqjYlVQ5WqmfgMlhCfRm9xL+G4G+KK9n8AJNGAszUfxVlPvMTw+nkOSd/bmxKrdCqqYnDIvDLucXJ86TstNzklfAwr3FL+wBlucRtOMLhQlHIaPTXYcNpOuh6B4ELjC+WjDGh8EdRKvcsZz7+5AS5ZaDDccuviMzQFsXVcE2d4HiosbARVrkxJ7j3MWp0OGgWVxXgRO2EQIksbgGSIjI8PqFjBqht2WT6MhVZPCc9XHUlP2CiAR5+QY8JgTIztbEDuhpgr0cRAtiHwJEAxDR9tJR/j/v4X/Pau2ZdR0C0yW77lVgD75spLL0khAnU7q+qgiF0hyQ7gRRVy0tElT0HBenVbzjzHowdJX8lSPjRg= diff --git a/README.rst b/README.rst index 72fb333..b0e1f2c 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,6 @@ daphne ====== -.. image:: https://api.travis-ci.org/django/daphne.svg - :target: https://travis-ci.org/django/daphne - .. image:: https://img.shields.io/pypi/v/daphne.svg :target: https://pypi.python.org/pypi/daphne diff --git a/daphne/testing.py b/daphne/testing.py index 7cc1182..c302045 100644 --- a/daphne/testing.py +++ b/daphne/testing.py @@ -127,8 +127,8 @@ class DaphneProcess(multiprocessing.Process): from twisted.internet import reactor - from .server import Server from .endpoints import build_endpoint_description_strings + from .server import Server try: # Create the server class @@ -266,8 +266,8 @@ class TestApplication: def _reinstall_reactor(): - import sys import asyncio + import sys from twisted.internet import asyncioreactor diff --git a/setup.cfg b/setup.cfg index e50af68..ddb3d5d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,16 +1,10 @@ -[bdist_wheel] -universal=1 - -[tool:pytest] -addopts = tests/ - -[isort] -include_trailing_comma = True -multi_line_output = 3 -known_first_party = channels,daphne,asgiref,channels_redis -line_length = 88 - [flake8] exclude = venv/*,tox/*,docs/*,testproject/*,js_client/*,.eggs/* -ignore = E123,E128,E266,E402,W503,E731,W601 +extend-ignore = E123, E128, E266, E402, W503, E731, W601 max-line-length = 120 + +[isort] +profile = black + +[tool:pytest] +testpaths = tests diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 0bd2d82..aae7f99 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,10 +3,9 @@ import collections from urllib import parse -from hypothesis import assume, given, settings - import http_strategies from http_base import DaphneTestCase +from hypothesis import assume, given, settings class TestHTTPRequest(DaphneTestCase): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 08c34bb..9dd728d 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,9 +1,8 @@ # coding: utf8 -from hypothesis import given, settings - import http_strategies from http_base import DaphneTestCase +from hypothesis import given, settings class TestHTTPResponse(DaphneTestCase): diff --git a/tests/test_websocket.py b/tests/test_websocket.py index 9ec2c0d..c27e7a9 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -4,10 +4,9 @@ import collections import time from urllib import parse -from hypothesis import given, settings - import http_strategies from http_base import DaphneTestCase, DaphneTestingInstance +from hypothesis import given, settings class TestWebsocket(DaphneTestCase): diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..a4010d7 --- /dev/null +++ b/tox.ini @@ -0,0 +1,23 @@ +[tox] +envlist = + py{36,37,38,39}-twisted{187,latest} + qa + +[testenv] +usedevelop = true +extras = tests +commands = + pytest -v {posargs} +deps = + twisted187: twisted==18.7.0 + twistedlatest: twisted>=20.3.0 + +[testenv:qa] +deps = + black + flake8 + isort +commands = + flake8 daphne tests + black --check daphne tests + isort --check-only --diff daphne tests From e1b77e930bdb94961fbee526d725a9951a93c000 Mon Sep 17 00:00:00 2001 From: Samori Gorse Date: Wed, 21 Oct 2020 16:38:03 +0200 Subject: [PATCH 03/13] Added request body chunking (#335) The entire body was previously read in memory which would lead the server to be killed by the scheduler. This change allows 8Kb chunks to be read until the entire body is consummed. Co-authored-by: Samori Gorse --- README.rst | 2 +- daphne/http_protocol.py | 16 +++++++++++++--- daphne/server.py | 2 ++ daphne/testing.py | 5 ++++- setup.py | 3 ++- tests/http_base.py | 25 ++++++++++++++++++++++--- tests/test_http_request.py | 24 ++++++++++++++++++++++++ 7 files changed, 68 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index b0e1f2c..cc77aea 100644 --- a/README.rst +++ b/README.rst @@ -112,7 +112,7 @@ should start with a slash, but not end with one; for example:: Python Support -------------- -Daphne requires Python 3.5 or later. +Daphne requires Python 3.6 or later. Contributing diff --git a/daphne/http_protocol.py b/daphne/http_protocol.py index ccbfdb9..7df7bae 100755 --- a/daphne/http_protocol.py +++ b/daphne/http_protocol.py @@ -185,9 +185,19 @@ class WebRequest(http.Request): # Not much we can do, the request is prematurely abandoned. return # Run application against request - self.application_queue.put_nowait( - {"type": "http.request", "body": self.content.read()} - ) + buffer_size = self.server.request_buffer_size + while True: + chunk = self.content.read(buffer_size) + more_body = not (len(chunk) < buffer_size) + payload = { + "type": "http.request", + "body": chunk, + "more_body": more_body, + } + self.application_queue.put_nowait(payload) + if not more_body: + break + except Exception: logger.error(traceback.format_exc()) self.basic_error( diff --git a/daphne/server.py b/daphne/server.py index 5ede808..9ca01d4 100755 --- a/daphne/server.py +++ b/daphne/server.py @@ -42,6 +42,7 @@ class Server(object): signal_handlers=True, action_logger=None, http_timeout=None, + request_buffer_size=8192, websocket_timeout=86400, websocket_connect_timeout=20, ping_interval=20, @@ -67,6 +68,7 @@ class Server(object): self.http_timeout = http_timeout self.ping_interval = ping_interval self.ping_timeout = ping_timeout + self.request_buffer_size = request_buffer_size self.proxy_forwarded_address_header = proxy_forwarded_address_header self.proxy_forwarded_port_header = proxy_forwarded_port_header self.proxy_forwarded_proto_header = proxy_forwarded_proto_header diff --git a/daphne/testing.py b/daphne/testing.py index c302045..3fd27ee 100644 --- a/daphne/testing.py +++ b/daphne/testing.py @@ -18,11 +18,12 @@ class DaphneTestingInstance: startup_timeout = 2 - def __init__(self, xff=False, http_timeout=None): + def __init__(self, xff=False, http_timeout=None, request_buffer_size=None): self.xff = xff self.http_timeout = http_timeout self.host = "127.0.0.1" self.lock = multiprocessing.Lock() + self.request_buffer_size = request_buffer_size def __enter__(self): # Clear result storage @@ -30,6 +31,8 @@ class DaphneTestingInstance: TestApplication.delete_result() # Option Daphne features kwargs = {} + if self.request_buffer_size: + kwargs["request_buffer_size"] = self.request_buffer_size # Optionally enable X-Forwarded-For support. if self.xff: kwargs["proxy_forwarded_address_header"] = "X-Forwarded-For" diff --git a/setup.py b/setup.py index 357e294..c6df474 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ setup( packages=find_packages() + ["twisted.plugins"], include_package_data=True, install_requires=["twisted[tls]>=18.7", "autobahn>=0.18", "asgiref~=3.2"], + python_requires='>=3.6', setup_requires=["pytest-runner"], extras_require={ "tests": ["hypothesis==4.23", "pytest~=3.10", "pytest-asyncio~=0.8"] @@ -38,10 +39,10 @@ setup( "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Topic :: Internet :: WWW/HTTP", ], ) diff --git a/tests/http_base.py b/tests/http_base.py index f3a8340..7ea796c 100644 --- a/tests/http_base.py +++ b/tests/http_base.py @@ -20,13 +20,24 @@ class DaphneTestCase(unittest.TestCase): ### Plain HTTP helpers def run_daphne_http( - self, method, path, params, body, responses, headers=None, timeout=1, xff=False + self, + method, + path, + params, + body, + responses, + headers=None, + timeout=1, + xff=False, + request_buffer_size=None, ): """ Runs Daphne with the given request callback (given the base URL) and response messages. """ - with DaphneTestingInstance(xff=xff) as test_app: + with DaphneTestingInstance( + xff=xff, request_buffer_size=request_buffer_size + ) as test_app: # Add the response messages test_app.add_send_messages(responses) # Send it the request. We have to do this the long way to allow @@ -79,7 +90,14 @@ class DaphneTestCase(unittest.TestCase): ) def run_daphne_request( - self, method, path, params=None, body=None, headers=None, xff=False + self, + method, + path, + params=None, + body=None, + headers=None, + xff=False, + request_buffer_size=None, ): """ Convenience method for just testing request handling. @@ -92,6 +110,7 @@ class DaphneTestCase(unittest.TestCase): body=body, headers=headers, xff=xff, + request_buffer_size=request_buffer_size, responses=[ {"type": "http.response.start", "status": 200}, {"type": "http.response.body", "body": b"OK"}, diff --git a/tests/test_http_request.py b/tests/test_http_request.py index aae7f99..fbe3f6c 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -6,6 +6,7 @@ from urllib import parse import http_strategies from http_base import DaphneTestCase from hypothesis import assume, given, settings +from hypothesis.strategies import integers class TestHTTPRequest(DaphneTestCase): @@ -119,6 +120,29 @@ class TestHTTPRequest(DaphneTestCase): self.assert_valid_http_scope(scope, "GET", request_path, params=request_params) self.assert_valid_http_request_message(messages[0], body=b"") + @given( + request_path=http_strategies.http_path(), + chunk_size=integers(min_value=1), + ) + @settings(max_examples=5, deadline=5000) + def test_request_body_chunking(self, request_path, chunk_size): + """ + Tests request body chunking logic. + """ + body = b"The quick brown fox jumps over the lazy dog" + _, messages = self.run_daphne_request( + "POST", + request_path, + body=body, + request_buffer_size=chunk_size, + ) + + # Avoid running those asserts when there's a single "http.disconnect" + if len(messages) > 1: + assert messages[0]["body"].decode() == body.decode()[:chunk_size] + assert not messages[-2]["more_body"] + assert messages[-1] == {"type": "http.disconnect"} + @given( request_path=http_strategies.http_path(), request_body=http_strategies.http_body(), From 15ba5c6776bcefc0f3e4a27b06636883ae5b7cf4 Mon Sep 17 00:00:00 2001 From: Avinash Raj Date: Wed, 28 Oct 2020 00:20:50 +0530 Subject: [PATCH 04/13] Updated to use ASGI v3 applications internally. (#275) Used guarantee_single_callable(). Removed unneeded --asgi-protocol CLI option. Updated tests. Co-authored-by: Carlton Gibson --- daphne/cli.py | 32 +++----------------------------- daphne/server.py | 12 +++++++----- daphne/testing.py | 9 ++++----- tests/test_http_request.py | 7 +++---- tests/test_websocket.py | 10 +++++++++- 5 files changed, 26 insertions(+), 44 deletions(-) diff --git a/daphne/cli.py b/daphne/cli.py index 2e65b12..8c42c43 100755 --- a/daphne/cli.py +++ b/daphne/cli.py @@ -1,10 +1,9 @@ import argparse -import functools import logging import sys from argparse import ArgumentError, Namespace -from asgiref.compatibility import is_double_callable +from asgiref.compatibility import guarantee_single_callable from .access import AccessLogGenerator from .endpoints import build_endpoint_description_strings @@ -17,19 +16,6 @@ DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 8000 -class ASGI3Middleware: - def __init__(self, app): - self.app = app - - def __call__(self, scope): - scope.setdefault("asgi", {}) - scope["asgi"]["version"] = "3.0" - return functools.partial(self.asgi, scope=scope) - - async def asgi(self, receive, send, scope): - await self.app(scope, receive, send) - - class CommandLineInterface(object): """ Acts as the main CLI entry point for running the server. @@ -129,13 +115,6 @@ class CommandLineInterface(object): help="The WebSocket protocols you wish to support", default=None, ) - self.parser.add_argument( - "--asgi-protocol", - dest="asgi_protocol", - help="The version of the ASGI protocol to use", - default="auto", - choices=["asgi2", "asgi3", "auto"], - ) self.parser.add_argument( "--root-path", dest="root_path", @@ -247,16 +226,11 @@ class CommandLineInterface(object): access_log_stream = open(args.access_log, "a", 1) elif args.verbosity >= 1: access_log_stream = sys.stdout + # Import application sys.path.insert(0, ".") application = import_by_path(args.application) - - asgi_protocol = args.asgi_protocol - if asgi_protocol == "auto": - asgi_protocol = "asgi2" if is_double_callable(application) else "asgi3" - - if asgi_protocol == "asgi3": - application = ASGI3Middleware(application) + application = guarantee_single_callable(application) # Set up port/host bindings if not any( diff --git a/daphne/server.py b/daphne/server.py index 9ca01d4..f367e06 100755 --- a/daphne/server.py +++ b/daphne/server.py @@ -199,15 +199,17 @@ class Server(object): assert "application_instance" not in self.connections[protocol] # Make an instance of the application input_queue = asyncio.Queue() - application_instance = self.application(scope=scope) + scope.setdefault("asgi", {"version": "3.0"}) + application_instance = self.application( + scope=scope, + receive=input_queue.get, + send=lambda message: self.handle_reply(protocol, message), + ) # Run it, and stash the future for later checking if protocol not in self.connections: return None self.connections[protocol]["application_instance"] = asyncio.ensure_future( - application_instance( - receive=input_queue.get, - send=lambda message: self.handle_reply(protocol, message), - ), + application_instance, loop=asyncio.get_event_loop(), ) return input_queue diff --git a/daphne/testing.py b/daphne/testing.py index 3fd27ee..1632516 100644 --- a/daphne/testing.py +++ b/daphne/testing.py @@ -5,7 +5,6 @@ import pickle import tempfile import traceback from concurrent.futures import CancelledError -from functools import partial class DaphneTestingInstance: @@ -43,7 +42,7 @@ class DaphneTestingInstance: # Start up process self.process = DaphneProcess( host=self.host, - application=partial(TestApplication, lock=self.lock), + application=TestApplication(lock=self.lock), kwargs=kwargs, setup=self.process_setup, teardown=self.process_teardown, @@ -173,12 +172,12 @@ class TestApplication: setup_storage = os.path.join(tempfile.gettempdir(), "setup.testio") result_storage = os.path.join(tempfile.gettempdir(), "result.testio") - def __init__(self, scope, lock): - self.scope = scope + def __init__(self, lock): self.lock = lock self.messages = [] - async def __call__(self, send, receive): + async def __call__(self, scope, receive, send): + self.scope = scope # Receive input and send output logging.debug("test app coroutine alive") try: diff --git a/tests/test_http_request.py b/tests/test_http_request.py index fbe3f6c..7048326 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -24,6 +24,7 @@ class TestHTTPRequest(DaphneTestCase): # Check overall keys self.assert_key_sets( required_keys={ + "asgi", "type", "http_version", "method", @@ -35,6 +36,7 @@ class TestHTTPRequest(DaphneTestCase): optional_keys={"scheme", "root_path", "client", "server"}, actual_keys=scope.keys(), ) + self.assertEqual(scope["asgi"]["version"], "3.0") # Check that it is the right type self.assertEqual(scope["type"], "http") # Method (uppercased unicode string) @@ -120,10 +122,7 @@ class TestHTTPRequest(DaphneTestCase): self.assert_valid_http_scope(scope, "GET", request_path, params=request_params) self.assert_valid_http_request_message(messages[0], body=b"") - @given( - request_path=http_strategies.http_path(), - chunk_size=integers(min_value=1), - ) + @given(request_path=http_strategies.http_path(), chunk_size=integers(min_value=1)) @settings(max_examples=5, deadline=5000) def test_request_body_chunking(self, request_path, chunk_size): """ diff --git a/tests/test_websocket.py b/tests/test_websocket.py index c27e7a9..862e71c 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -23,10 +23,18 @@ class TestWebsocket(DaphneTestCase): """ # Check overall keys self.assert_key_sets( - required_keys={"type", "path", "raw_path", "query_string", "headers"}, + required_keys={ + "asgi", + "type", + "path", + "raw_path", + "query_string", + "headers", + }, optional_keys={"scheme", "root_path", "client", "server", "subprotocols"}, actual_keys=scope.keys(), ) + self.assertEqual(scope["asgi"]["version"], "3.0") # Check that it is the right type self.assertEqual(scope["type"], "websocket") # Path From 525b6d2dbb986cd9acdfd8c1c8c2d4bbc03ab2b8 Mon Sep 17 00:00:00 2001 From: Ryan Fredericks <41584817+mrfredericks@users.noreply.github.com> Date: Wed, 28 Oct 2020 15:45:41 -0400 Subject: [PATCH 05/13] Update README for shell compatibility. (#327) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index cc77aea..7e99aef 100644 --- a/README.rst +++ b/README.rst @@ -58,7 +58,7 @@ Daphne supports terminating HTTP/2 connections natively. You'll need to do a couple of things to get it working, though. First, you need to make sure you install the Twisted ``http2`` and ``tls`` extras:: - pip install -U Twisted[tls,http2] + pip install -U 'Twisted[tls,http2]' Next, because all current browsers only support HTTP/2 when using TLS, you will need to start Daphne with TLS turned on, which can be done using the Twisted endpoint syntax:: @@ -124,7 +124,7 @@ Please refer to the To run tests, make sure you have installed the ``tests`` extra with the package:: cd daphne/ - pip install -e .[tests] + pip install -e '.[tests]' pytest From a69723ca3faf9c1d8d42b9cbe3bf06e8f1b9d986 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 28 Oct 2020 20:26:26 +0100 Subject: [PATCH 06/13] Version 3.0 release. * Bump version number. * Changelog. * README. * Update asgiref dependency specifier to match Django 3.1. --- CHANGELOG.txt | 24 +++++++++++++++++++----- README.rst | 4 ---- daphne/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9462217..9a48f5f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,12 +1,26 @@ +3.0.0 (2020-10-28) +------------------ + +* Updates internals to use ASGI v3 throughout. ``asgiref.compatibility`` is + used for older applications. + +* Consequently, the `--asgi-protocol` command-line option is removed. + +* HTTP request bodies are now read, and passed to the application, in chunks. + +* Added support for Python 3.9. + +* Dropped support for Python 3.5. + 2.5.0 (2020-04-15) ------------------ -* Fixes compatability for twisted when running Python 3.8+ on Windows, by - setting ``asyncio.WindowsSelectorEventLoopPolicy`` as the event loop policy - in this case. +* Fixes compatability for twisted when running Python 3.8+ on Windows, by + setting ``asyncio.WindowsSelectorEventLoopPolicy`` as the event loop policy + in this case. -* The internal ``daphne.testing.TestApplication`` now requires an addition - ``lock`` argument to ``__init__()``. This is expected to be an instance of +* The internal ``daphne.testing.TestApplication`` now requires an addition + ``lock`` argument to ``__init__()``. This is expected to be an instance of ``multiprocessing.Lock``. 2.4.1 (2019-12-18) diff --git a/README.rst b/README.rst index 7e99aef..16e806c 100644 --- a/README.rst +++ b/README.rst @@ -12,10 +12,6 @@ developed to power Django Channels. It supports automatic negotiation of protocols; there's no need for URL prefixing to determine WebSocket endpoints versus HTTP endpoints. -*Note:* Daphne 2 is not compatible with Channels 1.x applications, only with -Channels 2.x and other ASGI applications. Install a 1.x version of Daphne -for Channels 1.x support. - Running ------- diff --git a/daphne/__init__.py b/daphne/__init__.py index 2c5059a..0292642 100755 --- a/daphne/__init__.py +++ b/daphne/__init__.py @@ -1,6 +1,6 @@ import sys -__version__ = "2.5.0" +__version__ = "3.0.0" # Windows on Python 3.8+ uses ProactorEventLoop, which is not compatible with diff --git a/setup.py b/setup.py index c6df474..57341ca 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( package_dir={"twisted": "daphne/twisted"}, packages=find_packages() + ["twisted.plugins"], include_package_data=True, - install_requires=["twisted[tls]>=18.7", "autobahn>=0.18", "asgiref~=3.2"], + install_requires=["twisted[tls]>=18.7", "autobahn>=0.18", "asgiref>=3.2.10,<4"], python_requires='>=3.6', setup_requires=["pytest-runner"], extras_require={ From aae0870971cff742a09a0491dae330a1f6cc8ea2 Mon Sep 17 00:00:00 2001 From: Patrick Gingras <775.pg.12@gmail.com> Date: Wed, 11 Nov 2020 10:12:33 -0500 Subject: [PATCH 07/13] Handle asyncio.CancelledError in Server.application_checker (#341) As of [bpo-32528](https://bugs.python.org/issue32528), asyncio.CancelledError is not a subclass of concurrent.futures.CancelledError. This means that if an asyncio future raises an exception, it won't be caught. Therefore, the exception will bubble past the try-except within the loop in application_checker, resulting in done applications not being cleaned up, and the application_checker task not being queued again. --- daphne/server.py | 2 +- daphne/testing.py | 30 ++++++++++++++++------- tests/test_websocket.py | 53 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/daphne/server.py b/daphne/server.py index f367e06..77bd2d1 100755 --- a/daphne/server.py +++ b/daphne/server.py @@ -277,7 +277,7 @@ class Server(object): if application_instance and application_instance.done(): try: exception = application_instance.exception() - except CancelledError: + except (CancelledError, asyncio.CancelledError): # Future cancellation. We can ignore this. pass else: diff --git a/daphne/testing.py b/daphne/testing.py index 1632516..e2c7200 100644 --- a/daphne/testing.py +++ b/daphne/testing.py @@ -7,7 +7,7 @@ import traceback from concurrent.futures import CancelledError -class DaphneTestingInstance: +class BaseDaphneTestingInstance: """ Launches an instance of Daphne in a subprocess, with a host and port attribute allowing you to call it. @@ -17,17 +17,16 @@ class DaphneTestingInstance: startup_timeout = 2 - def __init__(self, xff=False, http_timeout=None, request_buffer_size=None): + def __init__( + self, xff=False, http_timeout=None, request_buffer_size=None, *, application + ): self.xff = xff self.http_timeout = http_timeout self.host = "127.0.0.1" - self.lock = multiprocessing.Lock() self.request_buffer_size = request_buffer_size + self.application = application def __enter__(self): - # Clear result storage - TestApplication.delete_setup() - TestApplication.delete_result() # Option Daphne features kwargs = {} if self.request_buffer_size: @@ -42,7 +41,7 @@ class DaphneTestingInstance: # Start up process self.process = DaphneProcess( host=self.host, - application=TestApplication(lock=self.lock), + application=self.application, kwargs=kwargs, setup=self.process_setup, teardown=self.process_teardown, @@ -76,6 +75,21 @@ class DaphneTestingInstance: """ pass + def get_received(self): + pass + + +class DaphneTestingInstance(BaseDaphneTestingInstance): + def __init__(self, *args, **kwargs): + self.lock = multiprocessing.Lock() + super().__init__(*args, **kwargs, application=TestApplication(lock=self.lock)) + + def __enter__(self): + # Clear result storage + TestApplication.delete_setup() + TestApplication.delete_result() + return super().__enter__() + def get_received(self): """ Returns the scope and messages the test application has received @@ -149,7 +163,7 @@ class DaphneProcess(multiprocessing.Process): self.server.run() finally: self.teardown() - except Exception as e: + except BaseException as e: # Put the error on our queue so the parent gets it self.errors.put((e, traceback.format_exc())) diff --git a/tests/test_websocket.py b/tests/test_websocket.py index 862e71c..26572e7 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -8,6 +8,8 @@ import http_strategies from http_base import DaphneTestCase, DaphneTestingInstance from hypothesis import given, settings +from daphne.testing import BaseDaphneTestingInstance + class TestWebsocket(DaphneTestCase): """ @@ -261,3 +263,54 @@ class TestWebsocket(DaphneTestCase): self.websocket_send_frame(sock, "still alive?") # Receive a frame and make sure it's correct assert self.websocket_receive_frame(sock) == "cake" + + def test_application_checker_handles_asyncio_cancellederror(self): + with CancellingTestingInstance() as app: + # Connect to the websocket app, it will immediately raise + # asyncio.CancelledError + sock, _ = self.websocket_handshake(app) + # Disconnect from the socket + sock.close() + # Wait for application_checker to clean up the applications for + # disconnected clients, and for the server to be stopped. + time.sleep(3) + # Make sure we received either no error, or a ConnectionsNotEmpty + while not app.process.errors.empty(): + err, _tb = app.process.errors.get() + if not isinstance(err, ConnectionsNotEmpty): + raise err + self.fail( + "Server connections were not cleaned up after an asyncio.CancelledError was raised" + ) + + +async def cancelling_application(scope, receive, send): + import asyncio + + from twisted.internet import reactor + + # Stop the server after a short delay so that the teardown is run. + reactor.callLater(2, lambda: reactor.stop()) + await send({"type": "websocket.accept"}) + raise asyncio.CancelledError() + + +class ConnectionsNotEmpty(Exception): + pass + + +class CancellingTestingInstance(BaseDaphneTestingInstance): + def __init__(self): + super().__init__(application=cancelling_application) + + def process_teardown(self): + import multiprocessing + + # Get a hold of the enclosing DaphneProcess (we're currently running in + # the same process as the application). + proc = multiprocessing.current_process() + # By now the (only) socket should have disconnected, and the + # application_checker should have run. If there are any connections + # still, it means that the application_checker did not clean them up. + if proc.server.connections: + raise ConnectionsNotEmpty() From aac4708a61574206405efdd40d278c5357ec75dc Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Thu, 12 Nov 2020 20:28:42 +0100 Subject: [PATCH 08/13] Bumped version and change notes for 3.0.1 release. --- CHANGELOG.txt | 12 ++++++++++++ daphne/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9a48f5f..4e0b0a9 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,10 @@ +3.0.1 (2020-11-12) +------------------ + +* Fixed a bug where ``asyncio.CancelledError`` was not correctly handled on + Python 3.8+, resulting in incorrect protocol application cleanup. + + 3.0.0 (2020-10-28) ------------------ @@ -12,6 +19,7 @@ * Dropped support for Python 3.5. + 2.5.0 (2020-04-15) ------------------ @@ -23,6 +31,7 @@ ``lock`` argument to ``__init__()``. This is expected to be an instance of ``multiprocessing.Lock``. + 2.4.1 (2019-12-18) ------------------ @@ -30,6 +39,7 @@ 3.0's ``async_unsafe()`` decorator in threaded contexts, such as using the auto-reloader. + 2.4.0 (2019-11-20) ------------------ @@ -47,11 +57,13 @@ * Adds missing LICENSE to distribution. + 2.3.0 (2019-04-09) ------------------ * Added support for ASGI v3. + 2.2.5 (2019-01-31) ------------------ diff --git a/daphne/__init__.py b/daphne/__init__.py index 0292642..9a235f3 100755 --- a/daphne/__init__.py +++ b/daphne/__init__.py @@ -1,6 +1,6 @@ import sys -__version__ = "3.0.0" +__version__ = "3.0.1" # Windows on Python 3.8+ uses ProactorEventLoop, which is not compatible with From ca611621299bcfab04a22cd6ae7c569b72d5877c Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 7 Apr 2021 19:11:21 +0100 Subject: [PATCH 09/13] Lint with pre-commit (#365) * Lint with pre-commit * Move existing tox qa hooks into pre-commit. * Set up GitHub Action based on https://github.com/pre-commit/action/ (we could also use https://pre-commit.ci ). * Add `pyupgrade` to drop old Python syntax. * Add `flake8-bugbear` plugin to prevent flake8 errors. * Drop custom GHA --- .github/workflows/pre-commit.yml | 24 ++++++++++++++++++++++++ .github/workflows/tests.yml | 14 -------------- .pre-commit-config.yaml | 21 +++++++++++++++++++++ daphne/access.py | 2 +- daphne/cli.py | 4 ++-- daphne/server.py | 2 +- daphne/twisted/plugins/fd_endpoint.py | 2 +- daphne/ws_protocol.py | 4 ++-- setup.py | 2 +- tests/http_base.py | 4 ++-- tests/test_cli.py | 2 -- tests/test_http_request.py | 2 -- tests/test_http_response.py | 2 -- tests/test_utils.py | 2 -- tests/test_websocket.py | 2 -- tox.ini | 11 ----------- 16 files changed, 55 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..d939432 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,24 @@ +name: pre-commit + +on: + push: + branches: + - main + pull_request: + +jobs: + pre-commit: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - uses: pre-commit/action@v2.0.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0756665..d388dea 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,17 +31,3 @@ jobs: run: | ENV_PREFIX=$(tr -C -d "0-9" <<< "${{ matrix.python-version }}") TOXENV=$(tox --listenvs | grep "^py$ENV_PREFIX" | tr '\n' ',') python -m tox - - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip tox - - name: Run lint - run: tox -e qa diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..5de3ae5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: + - repo: https://github.com/asottile/pyupgrade + rev: v2.11.0 + hooks: + - id: pyupgrade + args: [--py36-plus] + - repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + language_version: python3 + - repo: https://github.com/pycqa/isort + rev: 5.8.0 + hooks: + - id: isort + - repo: https://github.com/PyCQA/flake8 + rev: 3.9.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear diff --git a/daphne/access.py b/daphne/access.py index 2b3b1cd..e18138a 100644 --- a/daphne/access.py +++ b/daphne/access.py @@ -1,7 +1,7 @@ import datetime -class AccessLogGenerator(object): +class AccessLogGenerator: """ Object that implements the Daphne "action logger" internal interface in order to provide an access log in something resembling NCSA format. diff --git a/daphne/cli.py b/daphne/cli.py index 8c42c43..923b9d3 100755 --- a/daphne/cli.py +++ b/daphne/cli.py @@ -16,7 +16,7 @@ DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 8000 -class CommandLineInterface(object): +class CommandLineInterface: """ Acts as the main CLI entry point for running the server. """ @@ -258,7 +258,7 @@ class CommandLineInterface(object): ) endpoints = sorted(args.socket_strings + endpoints) # Start the server - logger.info("Starting server at %s" % (", ".join(endpoints),)) + logger.info("Starting server at {}".format(", ".join(endpoints))) self.server = self.server_class( application=application, endpoints=endpoints, diff --git a/daphne/server.py b/daphne/server.py index 77bd2d1..0ec72cc 100755 --- a/daphne/server.py +++ b/daphne/server.py @@ -34,7 +34,7 @@ from .ws_protocol import WebSocketFactory logger = logging.getLogger(__name__) -class Server(object): +class Server: def __init__( self, application, diff --git a/daphne/twisted/plugins/fd_endpoint.py b/daphne/twisted/plugins/fd_endpoint.py index ff5a4c7..313a315 100644 --- a/daphne/twisted/plugins/fd_endpoint.py +++ b/daphne/twisted/plugins/fd_endpoint.py @@ -7,7 +7,7 @@ from zope.interface import implementer @implementer(IPlugin, IStreamServerEndpointStringParser) -class _FDParser(object): +class _FDParser: prefix = "fd" def _parseServer(self, reactor, fileno, domain=socket.AF_INET): diff --git a/daphne/ws_protocol.py b/daphne/ws_protocol.py index 1962450..975b1a9 100755 --- a/daphne/ws_protocol.py +++ b/daphne/ws_protocol.py @@ -297,7 +297,7 @@ class WebSocketProtocol(WebSocketServerProtocol): return id(self) == id(other) def __repr__(self): - return "" % (self.client_addr, self.path) + return f"" class WebSocketFactory(WebSocketServerFactory): @@ -318,7 +318,7 @@ class WebSocketFactory(WebSocketServerFactory): Builds protocol instances. We use this to inject the factory object into the protocol. """ try: - protocol = super(WebSocketFactory, self).buildProtocol(addr) + protocol = super().buildProtocol(addr) protocol.factory = self return protocol except Exception: diff --git a/setup.py b/setup.py index 57341ca..af3b3b9 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ setup( packages=find_packages() + ["twisted.plugins"], include_package_data=True, install_requires=["twisted[tls]>=18.7", "autobahn>=0.18", "asgiref>=3.2.10,<4"], - python_requires='>=3.6', + python_requires=">=3.6", setup_requires=["pytest-runner"], extras_require={ "tests": ["hypothesis==4.23", "pytest~=3.10", "pytest-asyncio~=0.8"] diff --git a/tests/http_base.py b/tests/http_base.py index 7ea796c..e5a80c2 100644 --- a/tests/http_base.py +++ b/tests/http_base.py @@ -182,7 +182,7 @@ class DaphneTestCase(unittest.TestCase): if response.status != 101: raise RuntimeError("WebSocket upgrade did not result in status code 101") # Prepare headers for subprotocol searching - response_headers = dict((n.lower(), v) for n, v in response.getheaders()) + response_headers = {n.lower(): v for n, v in response.getheaders()} response.read() assert not response.closed # Return the raw socket and any subprotocol @@ -252,7 +252,7 @@ class DaphneTestCase(unittest.TestCase): """ try: socket.inet_aton(address) - except socket.error: + except OSError: self.fail("'%s' is not a valid IP address." % address) def assert_key_sets(self, required_keys, optional_keys, actual_keys): diff --git a/tests/test_cli.py b/tests/test_cli.py index 2bbcc42..17335ed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,3 @@ -# coding: utf8 - import logging from argparse import ArgumentError from unittest import TestCase diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7048326..52f6dd1 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,5 +1,3 @@ -# coding: utf8 - import collections from urllib import parse diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 9dd728d..0f42df2 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,5 +1,3 @@ -# coding: utf8 - import http_strategies from http_base import DaphneTestCase from hypothesis import given, settings diff --git a/tests/test_utils.py b/tests/test_utils.py index 6b04939..b8ef1e1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,3 @@ -# coding: utf8 - from unittest import TestCase from twisted.web.http_headers import Headers diff --git a/tests/test_websocket.py b/tests/test_websocket.py index 26572e7..9b67aa1 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -1,5 +1,3 @@ -# coding: utf8 - import collections import time from urllib import parse diff --git a/tox.ini b/tox.ini index a4010d7..c507a8b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] envlist = py{36,37,38,39}-twisted{187,latest} - qa [testenv] usedevelop = true @@ -11,13 +10,3 @@ commands = deps = twisted187: twisted==18.7.0 twistedlatest: twisted>=20.3.0 - -[testenv:qa] -deps = - black - flake8 - isort -commands = - flake8 daphne tests - black --check daphne tests - isort --check-only --diff daphne tests From 2b6f15361646093e76e990675bf24304aad205bd Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 7 Apr 2021 19:14:02 +0100 Subject: [PATCH 10/13] Used partial() to wrap Server.handle_reply() (#364) Fixes #332. --- CHANGELOG.txt | 4 ++++ daphne/server.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 4e0b0a9..0584eb9 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,7 @@ +* Fixed a bug where ``send`` passed to applications wasn't a true async + function but a lambda wrapper, preventing it from being used with + ``asgiref.sync.async_to_sync()``. + 3.0.1 (2020-11-12) ------------------ diff --git a/daphne/server.py b/daphne/server.py index 0ec72cc..0d463d0 100755 --- a/daphne/server.py +++ b/daphne/server.py @@ -22,6 +22,7 @@ else: import logging import time from concurrent.futures import CancelledError +from functools import partial from twisted.internet import defer, reactor from twisted.internet.endpoints import serverFromString @@ -203,7 +204,7 @@ class Server: application_instance = self.application( scope=scope, receive=input_queue.get, - send=lambda message: self.handle_reply(protocol, message), + send=partial(self.handle_reply, protocol), ) # Run it, and stash the future for later checking if protocol not in self.connections: From d5c41bf6418a26b1a672e718dfa1e18c707268eb Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 7 Apr 2021 20:18:05 +0200 Subject: [PATCH 11/13] Updated various README URLs. --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 16e806c..7525b27 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,8 @@ daphne :target: https://pypi.python.org/pypi/daphne Daphne is a HTTP, HTTP2 and WebSocket protocol server for -`ASGI `_ and -`ASGI-HTTP `_, +`ASGI `_ and +`ASGI-HTTP `_, developed to power Django Channels. It supports automatic negotiation of protocols; there's no need for URL @@ -115,7 +115,7 @@ Contributing ------------ Please refer to the -`main Channels contributing docs `_. +`main Channels contributing docs `_. To run tests, make sure you have installed the ``tests`` extra with the package:: @@ -134,4 +134,4 @@ https://docs.djangoproject.com/en/dev/internals/security/. To report bugs or request new features, please open a new GitHub issue. This repository is part of the Channels project. For the shepherd and maintenance team, please see the -`main Channels readme `_. +`main Channels readme `_. From e480917c1a68baf9beb5d8b61981750c4e49dcb0 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 7 Apr 2021 20:21:36 +0200 Subject: [PATCH 12/13] Bumped version and change notes for 3.0.2 release. --- CHANGELOG.txt | 4 ++++ daphne/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0584eb9..8463751 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,7 +1,11 @@ +3.0.2 (2021-04-07) +------------------ + * Fixed a bug where ``send`` passed to applications wasn't a true async function but a lambda wrapper, preventing it from being used with ``asgiref.sync.async_to_sync()``. + 3.0.1 (2020-11-12) ------------------ diff --git a/daphne/__init__.py b/daphne/__init__.py index 9a235f3..530cbd0 100755 --- a/daphne/__init__.py +++ b/daphne/__init__.py @@ -1,6 +1,6 @@ import sys -__version__ = "3.0.1" +__version__ = "3.0.2" # Windows on Python 3.8+ uses ProactorEventLoop, which is not compatible with From 36ce9fd1edd9fcc1826ecc3968503aa6fd43726a Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Fri, 16 Apr 2021 17:21:51 +0100 Subject: [PATCH 13/13] Use tox-py in CI (#369) --- .github/workflows/tests.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d388dea..ea93fa7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,14 +20,16 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | - python -m pip install --upgrade pip wheel setuptools tox + python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade tox tox-py + - name: Run tox targets for ${{ matrix.python-version }} - run: | - ENV_PREFIX=$(tr -C -d "0-9" <<< "${{ matrix.python-version }}") - TOXENV=$(tox --listenvs | grep "^py$ENV_PREFIX" | tr '\n' ',') python -m tox + run: tox --py current