From 7032f8e0f82afdc3bd606b08f1443c1245e2774d Mon Sep 17 00:00:00 2001 From: Joonhyung Shin Date: Thu, 7 Nov 2019 03:51:00 +0900 Subject: [PATCH 01/10] Resolve asyncio + multiprocessing problem when testing. (#247) --- .gitignore | 1 + .travis.yml | 37 +++++++++++++++++++++++++++++++++++++ daphne/testing.py | 39 ++++++++++++++++++++++++++++++++++----- 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 830060b..b3cff06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea/ *.egg-info *.pyc __pycache__ diff --git a/.travis.yml b/.travis.yml index d87df46..02274ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,43 @@ jobs: dist: xenial sudo: required + - os: osx + language: generic + python: '3.5' + env: TWISTED="twisted==18.7.0" + before_install: + - eval "$(pyenv init -)" + - pyenv install 3.5.5 + - pyenv global 3.5.5 + sudo: required + - os: osx + language: generic + python: '3.5' + env: TWISTED="twisted" + before_install: + - eval "$(pyenv init -)" + - pyenv install 3.5.5 + - pyenv global 3.5.5 + sudo: required + - os: osx + language: generic + python: '3.6' + env: TWISTED="twisted==18.7.0" + before_install: + - eval "$(pyenv init -)" + - pyenv install 3.6.5 + - pyenv global 3.6.5 + sudo: required + - os: osx + language: generic + python: '3.6' + env: TWISTED="twisted" + before_install: + - eval "$(pyenv init -)" + - pyenv install 3.6.5 + - pyenv global 3.6.5 + sudo: required + - stage: lint install: pip install -U -e .[tests] black pyflakes isort script: diff --git a/daphne/testing.py b/daphne/testing.py index f5f3724..0da03d0 100644 --- a/daphne/testing.py +++ b/daphne/testing.py @@ -6,11 +6,6 @@ import tempfile import traceback from concurrent.futures import CancelledError -from twisted.internet import reactor - -from .endpoints import build_endpoint_description_strings -from .server import Server - class DaphneTestingInstance: """ @@ -121,6 +116,17 @@ class DaphneProcess(multiprocessing.Process): self.errors = multiprocessing.Queue() def run(self): + # OK, now we are in a forked child process, and want to use the reactor. + # However, FreeBSD systems like MacOS do not fork the underlying Kqueue, + # which asyncio (hence asyncioreactor) is built on. + # Therefore, we should uninstall the broken reactor and install a new one. + _reinstall_reactor() + + from twisted.internet import reactor + + from .server import Server + from .endpoints import build_endpoint_description_strings + try: # Create the server class endpoints = build_endpoint_description_strings(host=self.host, port=0) @@ -143,6 +149,8 @@ class DaphneProcess(multiprocessing.Process): self.errors.put((e, traceback.format_exc())) def resolve_port(self): + from twisted.internet import reactor + if self.server.listening_addresses: self.port.value = self.server.listening_addresses[0][1] self.ready.set() @@ -249,3 +257,24 @@ class TestApplication: os.unlink(cls.result_storage) except OSError: pass + + +def _reinstall_reactor(): + import sys + import asyncio + + from twisted.internet import asyncioreactor + + # Uninstall the reactor. + if "twisted.internet.reactor" in sys.modules: + del sys.modules["twisted.internet.reactor"] + + # The daphne.server module may have already installed the reactor. + # If so, using this module will use uninstalled one, thus we should + # reimport this module too. + if "daphne.server" in sys.modules: + del sys.modules["daphne.server"] + + event_loop = asyncio.new_event_loop() + asyncioreactor.install(event_loop) + asyncio.set_event_loop(event_loop) From 78be865eb40f1dc1a761c5f5d82a33a33efff33d Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Thu, 14 Nov 2019 07:13:16 +0100 Subject: [PATCH 02/10] Fixed #276 -- Ensured 500 response when app sends malformed headers. (#281) --- daphne/server.py | 7 ++++++- tests/http_base.py | 8 ++++++-- tests/test_http_response.py | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/daphne/server.py b/daphne/server.py index de04b8a..f980989 100755 --- a/daphne/server.py +++ b/daphne/server.py @@ -219,7 +219,12 @@ class Server(object): "disconnected", None ): return - self.check_headers_type(message) + try: + self.check_headers_type(message) + except ValueError: + # Ensure to send SOME reply. + protocol.basic_error(500, b"Server Error", "Server Error") + raise # Let the protocol handle it protocol.handle_reply(message) diff --git a/tests/http_base.py b/tests/http_base.py index 71d9618..f3a8340 100644 --- a/tests/http_base.py +++ b/tests/http_base.py @@ -56,12 +56,16 @@ class DaphneTestCase(unittest.TestCase): # Return scope, messages, response return test_app.get_received() + (response,) - def run_daphne_raw(self, data, timeout=1): + def run_daphne_raw(self, data, *, responses=None, timeout=1): """ - Runs daphne and sends it the given raw bytestring over a socket. Returns what it sends back. + Runs Daphne and sends it the given raw bytestring over a socket. + Accepts list of response messages the application will reply with. + Returns what Daphne sends back. """ assert isinstance(data, bytes) with DaphneTestingInstance() as test_app: + if responses is not None: + test_app.add_send_messages(responses) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index afb8e39..08c34bb 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -169,3 +169,21 @@ class TestHTTPResponse(DaphneTestCase): str(context.exception), "Header value 'True' expected to be `bytes`, but got ``", ) + + def test_headers_type_raw(self): + """ + Daphne returns a 500 error response if the application sends invalid + headers. + """ + response = self.run_daphne_raw( + b"GET / HTTP/1.0\r\n\r\n", + responses=[ + { + "type": "http.response.start", + "status": 200, + "headers": [["foo", b"bar"]], + }, + {"type": "http.response.body", "body": b""}, + ], + ) + self.assertTrue(response.startswith(b"HTTP/1.0 500 Internal Server Error")) From a4efcd5c1de99ef42170cb7f056567e9d7df2e69 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 20 Nov 2019 20:08:06 +0100 Subject: [PATCH 03/10] Reduced macOS Travis builds to single env. Slow, and not any benefit in multiple runs. --- .travis.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 02274ba..0b93f89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,33 +34,6 @@ jobs: dist: xenial sudo: required - - os: osx - language: generic - python: '3.5' - env: TWISTED="twisted==18.7.0" - before_install: - - eval "$(pyenv init -)" - - pyenv install 3.5.5 - - pyenv global 3.5.5 - sudo: required - - os: osx - language: generic - python: '3.5' - env: TWISTED="twisted" - before_install: - - eval "$(pyenv init -)" - - pyenv install 3.5.5 - - pyenv global 3.5.5 - sudo: required - - os: osx - language: generic - python: '3.6' - env: TWISTED="twisted==18.7.0" - before_install: - - eval "$(pyenv init -)" - - pyenv install 3.6.5 - - pyenv global 3.6.5 - sudo: required - os: osx language: generic python: '3.6' From 4b7c027b988fef3eed0033f59810a9ac93da06f3 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 20 Nov 2019 20:13:51 +0100 Subject: [PATCH 04/10] Add testing against Python 3.8. --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0b93f89..537ecc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,10 @@ jobs: env: TWISTED="twisted" dist: xenial sudo: required + - python: '3.8' + env: TWISTED="twisted" + dist: xenial + sudo: required - os: osx language: generic From beb836acce5a3fd1f348cb551b898ee7f0fa36ba Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 20 Nov 2019 20:19:33 +0100 Subject: [PATCH 05/10] Remove macOS Travis build. Travis' infrastructure is just too slow. --- .travis.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 537ecc1..2034a3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,16 +38,6 @@ jobs: dist: xenial sudo: required - - os: osx - language: generic - python: '3.6' - env: TWISTED="twisted" - before_install: - - eval "$(pyenv init -)" - - pyenv install 3.6.5 - - pyenv global 3.6.5 - sudo: required - - stage: lint install: pip install -U -e .[tests] black pyflakes isort script: From eb582d1d43c74fc9d2cf9414c0cc8006b26a65aa Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 20 Nov 2019 20:41:07 +0100 Subject: [PATCH 06/10] Releasing 2.4.0 --- CHANGELOG.txt | 17 +++++++++++++++++ daphne/__init__.py | 2 +- setup.py | 3 ++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index af0dc4b..17c8485 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,20 @@ +2.4.0 (2019-11-20) +------------------ + +* Adds CI testing against and support for Python 3.8. + +* Adds support for ``raw_path`` in ASGI scope. + +* Ensures an error response is sent to the client if the application sends + malformed headers. + +* Resolves an asyncio + multiprocessing problem when testing that would cause + the test suite to fail/hang on macOS. + +* Requires installing Twisted's TLS extras, via ``install_requires``. + +* Adds missing LICENSE to distribution. + 2.3.0 (2019-04-09) ------------------ diff --git a/daphne/__init__.py b/daphne/__init__.py index 55e4709..3d67cd6 100755 --- a/daphne/__init__.py +++ b/daphne/__init__.py @@ -1 +1 @@ -__version__ = "2.3.0" +__version__ = "2.4.0" diff --git a/setup.py b/setup.py index b6cfdc8..357e294 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( package_dir={"twisted": "daphne/twisted"}, packages=find_packages() + ["twisted.plugins"], include_package_data=True, - install_requires=["twisted[tls]>=18.7", "autobahn>=0.18", "asgiref~=3.0"], + install_requires=["twisted[tls]>=18.7", "autobahn>=0.18", "asgiref~=3.2"], setup_requires=["pytest-runner"], extras_require={ "tests": ["hypothesis==4.23", "pytest~=3.10", "pytest-asyncio~=0.8"] @@ -41,6 +41,7 @@ setup( "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Topic :: Internet :: WWW/HTTP", ], ) From 27f760a8147efd0c2f4cb842cbe9d50cfdf40d46 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 18 Dec 2019 20:05:50 +0100 Subject: [PATCH 07/10] Avoid Twisted using the default event loop When switching threads, e.g. when run via Django auto-reloader, the default run loop causes issues detecting async contexts. Fixes https://github.com/django/channels/issues/1374 --- daphne/server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/daphne/server.py b/daphne/server.py index f980989..f441285 100755 --- a/daphne/server.py +++ b/daphne/server.py @@ -1,8 +1,10 @@ # This has to be done first as Twisted is import-order-sensitive with reactors +import asyncio # isort:skip import sys # isort:skip import warnings # isort:skip from twisted.internet import asyncioreactor # isort:skip +twisted_loop = asyncio.new_event_loop() current_reactor = sys.modules.get("twisted.internet.reactor", None) if current_reactor is not None: if not isinstance(current_reactor, asyncioreactor.AsyncioSelectorReactor): @@ -13,11 +15,10 @@ if current_reactor is not None: UserWarning, ) del sys.modules["twisted.internet.reactor"] - asyncioreactor.install() + asyncioreactor.install(twisted_loop) else: - asyncioreactor.install() + asyncioreactor.install(twisted_loop) -import asyncio import logging import time import traceback From 18f2d67f34e33a5f8c87a7bca093dfffea44d7a7 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 18 Dec 2019 20:42:24 +0100 Subject: [PATCH 08/10] Releasing 2.4.1 --- CHANGELOG.txt | 7 +++++++ daphne/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 17c8485..d096da4 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,10 @@ +2.4.1 (2019-12-18) +------------------ + +* Avoids Twisted using the default event loop, for compatibility with Django + 3.0's ``async_unsafe()`` decorator in threaded contexts, such as using the + auto-reloader. + 2.4.0 (2019-11-20) ------------------ diff --git a/daphne/__init__.py b/daphne/__init__.py index 3d67cd6..54499df 100755 --- a/daphne/__init__.py +++ b/daphne/__init__.py @@ -1 +1 @@ -__version__ = "2.4.0" +__version__ = "2.4.1" From 61c8633c5d834e24c8d5473df522efa849a60e8d Mon Sep 17 00:00:00 2001 From: LittlePony Date: Wed, 5 Feb 2020 22:40:44 +0300 Subject: [PATCH 09/10] Add logger traceback on application error. (#308) --- daphne/server.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/daphne/server.py b/daphne/server.py index f441285..5ede808 100755 --- a/daphne/server.py +++ b/daphne/server.py @@ -21,7 +21,6 @@ else: import logging import time -import traceback from concurrent.futures import CancelledError from twisted.internet import defer, reactor @@ -283,13 +282,10 @@ class Server(object): # Protocol is asking the server to exit (likely during test) self.stop() else: - exception_output = "{}\n{}{}".format( - exception, - "".join(traceback.format_tb(exception.__traceback__)), - " {}".format(exception), - ) logger.error( - "Exception inside application: %s", exception_output + "Exception inside application: %s", + exception, + exc_info=exception, ) if not disconnected: protocol.handle_exception(exception) From 59b57a9f4b7a419f1ab3ecb792ca802671045075 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 5 Feb 2020 20:05:51 +0000 Subject: [PATCH 10/10] Simplify travis config (#295) --- .travis.yml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2034a3b..bf7be68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ sudo: false language: python python: +- '3.8' +- '3.7' - '3.6' - '3.5' @@ -25,20 +27,8 @@ stages: jobs: include: - - python: '3.7' - env: TWISTED="twisted==18.7.0" - dist: xenial - sudo: required - - python: '3.7' - env: TWISTED="twisted" - dist: xenial - sudo: required - - python: '3.8' - env: TWISTED="twisted" - dist: xenial - sudo: required - - stage: lint + python: 3.6 install: pip install -U -e .[tests] black pyflakes isort script: - pyflakes daphne tests @@ -46,6 +36,7 @@ jobs: - isort --check-only --diff --recursive daphne tests - stage: release + python: 3.6 script: skip deploy: provider: pypi