From 821816f656ed8be43dc8a5ebf90fa16920355f1a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 3 Jun 2015 18:17:46 +0100 Subject: [PATCH 001/746] Initial commit --- .gitignore | 1 + README | 0 channel/__init__.py | 7 +++ channel/adapters.py | 18 ++++++ channel/channels/__init__.py | 0 channel/channels/base.py | 53 ++++++++++++++++ channel/channels/memory.py | 48 +++++++++++++++ channel/consumer_registry.py | 40 +++++++++++++ channel/management/__init__.py | 0 channel/management/commands/__init__.py | 0 .../management/commands/runinterfaceserver.py | 60 +++++++++++++++++++ channel/request.py | 34 +++++++++++ channel/response.py | 29 +++++++++ channel/utils.py | 14 +++++ channel/worker.py | 19 ++++++ setup.py | 12 ++++ 16 files changed, 335 insertions(+) create mode 100644 .gitignore create mode 100644 README create mode 100755 channel/__init__.py create mode 100755 channel/adapters.py create mode 100644 channel/channels/__init__.py create mode 100644 channel/channels/base.py create mode 100644 channel/channels/memory.py create mode 100755 channel/consumer_registry.py create mode 100644 channel/management/__init__.py create mode 100644 channel/management/commands/__init__.py create mode 100755 channel/management/commands/runinterfaceserver.py create mode 100755 channel/request.py create mode 100755 channel/response.py create mode 100755 channel/utils.py create mode 100755 channel/worker.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..11041c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.egg-info diff --git a/README b/README new file mode 100644 index 0000000..e69de29 diff --git a/channel/__init__.py b/channel/__init__.py new file mode 100755 index 0000000..d5e0e00 --- /dev/null +++ b/channel/__init__.py @@ -0,0 +1,7 @@ +from .consumer_registry import ConsumerRegistry + +# Make a site-wide registry +coreg = ConsumerRegistry() + +# Load an implementation of Channel +from .channels.memory import Channel diff --git a/channel/adapters.py b/channel/adapters.py new file mode 100755 index 0000000..7f3474e --- /dev/null +++ b/channel/adapters.py @@ -0,0 +1,18 @@ +from django.core.handlers.base import BaseHandler +from channel import Channel +from .response import encode_response +from .request import decode_request + + +class DjangoUrlAdapter(object): + """ + Adapts the channel-style HTTP requests to the URL-router/handler style + """ + + def __init__(self): + self.handler = BaseHandler() + self.handler.load_middleware() + + def __call__(self, request, response_channel): + response = self.handler.get_response(decode_request(request)) + Channel(response_channel).send(**encode_response(response)) diff --git a/channel/channels/__init__.py b/channel/channels/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channel/channels/base.py b/channel/channels/base.py new file mode 100644 index 0000000..7841d99 --- /dev/null +++ b/channel/channels/base.py @@ -0,0 +1,53 @@ +class Channel(object): + """ + Base class for all channel layer implementations. + """ + + class ClosedError(Exception): + """ + Raised when you try to send to a closed channel. + """ + pass + + def __init__(self, name): + """ + Create an instance for the channel named "name" + """ + self.name = name + + def send(self, **kwargs): + """ + Send a message over the channel, taken from the kwargs. + """ + raise NotImplementedError() + + def close(self): + """ + Closes the channel, allowing no more messages to be sent over it. + """ + raise NotImplementedError() + + @property + def closed(self): + """ + Says if the channel is closed. + """ + raise NotImplementedError() + + @classmethod + def receive_many(self, channel_names): + """ + Block and return the first message available on one of the + channels passed, as a (channel_name, message) tuple. + """ + raise NotImplementedError() + + @classmethod + def new_name(self, prefix): + """ + Returns a new channel name that's unique and not closed + with the given prefix. Does not need to be called before sending + on a channel name - just provides a way to avoid clashing for + response channels. + """ + raise NotImplementedError() diff --git a/channel/channels/memory.py b/channel/channels/memory.py new file mode 100644 index 0000000..6a66ceb --- /dev/null +++ b/channel/channels/memory.py @@ -0,0 +1,48 @@ +import time +import string +import random +from collections import deque +from .base import Channel as BaseChannel + +queues = {} +closed = set() + +class Channel(BaseChannel): + """ + In-memory channel implementation. Intended only for use with threading, + in low-throughput development environments. + """ + + def send(self, **kwargs): + # Don't allow if closed + if self.name in closed: + raise Channel.ClosedError("%s is closed" % self.name) + # Add to the deque, making it if needs be + queues.setdefault(self.name, deque()).append(kwargs) + + @property + def closed(self): + # Check closed set + return self.name in closed + + def close(self): + # Add to closed set + closed.add(self.name) + + @classmethod + def receive_many(self, channel_names): + while True: + # Try to pop a message from each channel + for channel_name in channel_names: + try: + # This doesn't clean up empty channels - OK for testing. + # For later versions, have cleanup w/lock. + return channel_name, queues[channel_name].popleft() + except (IndexError, KeyError): + pass + # If all empty, sleep for a little bit + time.sleep(0.01) + + @classmethod + def new_name(self, prefix): + return "%s.%s" % (prefix, "".join(random.choice(string.ascii_letters) for i in range(16))) diff --git a/channel/consumer_registry.py b/channel/consumer_registry.py new file mode 100755 index 0000000..0e6baaa --- /dev/null +++ b/channel/consumer_registry.py @@ -0,0 +1,40 @@ +import functools + +class ConsumerRegistry(object): + """ + Manages the available consumers in the project and which channels they + listen to. + + Generally a single project-wide instance of this is used. + """ + + def __init__(self): + self.consumers = {} + + def add_consumer(self, consumer, channels): + for channel in channels: + if channel in self.consumers: + raise ValueError("Cannot register consumer %s - channel %s already consumed by %s" % ( + consumer, + channel, + self.consumers[channel], + )) + self.consumers[channel] = consumer + + def consumer(self, channels): + """ + Decorator that registers a function as a consumer. + """ + def inner(func): + self.add_consumer(func, channels) + return func + return inner + + def all_channel_names(self): + return self.consumers.keys() + + def consumer_for_channel(self, channel): + try: + return self.consumers[channel] + except KeyError: + return None diff --git a/channel/management/__init__.py b/channel/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channel/management/commands/__init__.py b/channel/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channel/management/commands/runinterfaceserver.py b/channel/management/commands/runinterfaceserver.py new file mode 100755 index 0000000..796606d --- /dev/null +++ b/channel/management/commands/runinterfaceserver.py @@ -0,0 +1,60 @@ +import django +import threading +from django.core.management.commands.runserver import Command as RunserverCommand +from django.core.handlers.wsgi import WSGIHandler +from channel import Channel, coreg +from channel.request import encode_request +from channel.response import decode_response +from channel.worker import Worker +from channel.utils import auto_import_consumers + + +class Command(RunserverCommand): + + def get_handler(self, *args, **options): + """ + Returns the default WSGI handler for the runner. + """ + django.setup() + return WSGIInterfaceHandler() + + def run(self, *args, **options): + # Force disable reloader for now + options['use_reloader'] = False + # Check a handler is registered for http reqs + auto_import_consumers() + if not coreg.consumer_for_channel("django.wsgi.request"): + raise RuntimeError("No consumer registered for WSGI requests") + # Launch a worker thread + worker = WorkerThread() + worker.daemon = True + worker.start() + # Run the rest + return super(Command, self).run(*args, **options) + + +class WSGIInterfaceHandler(WSGIHandler): + """ + New WSGI handler that pushes requests to channels. + """ + + def get_response(self, request): + response_channel = Channel.new_name("django.wsgi.response") + Channel("django.wsgi.request").send( + request = encode_request(request), + response_channel = response_channel, + ) + channel, message = Channel.receive_many([response_channel]) + return decode_response(message) + + +class WorkerThread(threading.Thread): + """ + Class that runs a worker + """ + + def run(self): + Worker( + consumer_registry = coreg, + channel_class = Channel, + ).run() diff --git a/channel/request.py b/channel/request.py new file mode 100755 index 0000000..c319ac5 --- /dev/null +++ b/channel/request.py @@ -0,0 +1,34 @@ +from django.http import HttpRequest +from django.utils.datastructures import MultiValueDict + + +def encode_request(request): + """ + Encodes a request to JSON-compatible datastructures + """ + # TODO: More stuff + value = { + "GET": request.GET.items(), + "POST": request.POST.items(), + "COOKIES": request.COOKIES, + "META": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, + "path": request.path, + "path_info": request.path_info, + "method": request.method, + } + return value + + +def decode_request(value): + """ + Decodes a request JSONish value to a HttpRequest object. + """ + request = HttpRequest() + request.GET = MultiValueDict(value['GET']) + request.POST = MultiValueDict(value['POST']) + request.COOKIES = value['COOKIES'] + request.META = value['META'] + request.path = value['path'] + request.method = value['method'] + request.path_info = value['path_info'] + return request diff --git a/channel/response.py b/channel/response.py new file mode 100755 index 0000000..542482a --- /dev/null +++ b/channel/response.py @@ -0,0 +1,29 @@ +from django.http import HttpResponse + + +def encode_response(response): + """ + Encodes a response to JSON-compatible datastructures + """ + # TODO: Entirely useful things like cookies + value = { + "content_type": getattr(response, "content_type", None), + "content": response.content, + "status_code": response.status_code, + "headers": response._headers.values(), + } + response.close() + return value + + +def decode_response(value): + """ + Decodes a response JSONish value to a HttpResponse object. + """ + response = HttpResponse( + content = value['content'], + content_type = value['content_type'], + status = value['status_code'], + ) + response._headers = {k.lower: (k, v) for k, v in value['headers']} + return response diff --git a/channel/utils.py b/channel/utils.py new file mode 100755 index 0000000..2615e72 --- /dev/null +++ b/channel/utils.py @@ -0,0 +1,14 @@ +from django.apps import apps + + +def auto_import_consumers(): + """ + Auto-import consumers modules in apps + """ + for app_config in apps.get_app_configs(): + consumer_module_name = "%s.consumers" % (app_config.name,) + try: + __import__(consumer_module_name) + except ImportError as e: + if "no module named" not in str(e).lower(): + raise diff --git a/channel/worker.py b/channel/worker.py new file mode 100755 index 0000000..73a2c1f --- /dev/null +++ b/channel/worker.py @@ -0,0 +1,19 @@ +class Worker(object): + """ + A "worker" process that continually looks for available messages to run + and runs their consumers. + """ + + def __init__(self, consumer_registry, channel_class): + self.consumer_registry = consumer_registry + self.channel_class = channel_class + + def run(self): + """ + Tries to continually dispatch messages to consumers. + """ + channels = self.consumer_registry.all_channel_names() + while True: + channel, message = self.channel_class.receive_many(channels) + consumer = self.consumer_registry.consumer_for_channel(channel) + consumer(**message) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e8274ab --- /dev/null +++ b/setup.py @@ -0,0 +1,12 @@ +from setuptools import find_packages, setup + +setup( + name='django-channel', + version="0.1", + url='http://github.com/andrewgodwin/django-channel', + author='Andrew Godwin', + author_email='andrew@aeracode.org', + license='BSD', + packages=find_packages(), + include_package_data=True, +) From 6cd01e2bc19dcde88bae74cde7c200c6c992a908 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 8 Jun 2015 10:51:51 -0700 Subject: [PATCH 002/746] Rename to plural "channels", start fleshing out req/resp cycle --- channel/__init__.py | 7 --- channel/adapters.py | 18 ------- channel/management/commands/__init__.py | 0 channels/__init__.py | 11 +++++ channels/adapters.py | 47 +++++++++++++++++++ channels/backends/__init__.py | 2 + .../channels => channels/backends}/base.py | 11 ++++- .../channels => channels/backends}/memory.py | 4 +- {channel => channels}/consumer_registry.py | 0 channels/docs/integration-changes.rst | 23 +++++++++ channels/docs/message-standards.rst | 23 +++++++++ channels/hacks.py | 27 +++++++++++ .../management}/__init__.py | 0 .../management/commands}/__init__.py | 0 .../management/commands/runserver.py | 24 +++++----- {channel => channels}/request.py | 2 + {channel => channels}/response.py | 9 ++++ {channel => channels}/utils.py | 2 +- {channel => channels}/worker.py | 0 19 files changed, 168 insertions(+), 42 deletions(-) delete mode 100755 channel/__init__.py delete mode 100755 channel/adapters.py delete mode 100644 channel/management/commands/__init__.py create mode 100644 channels/__init__.py create mode 100644 channels/adapters.py create mode 100644 channels/backends/__init__.py rename {channel/channels => channels/backends}/base.py (78%) rename {channel/channels => channels/backends}/memory.py (92%) rename {channel => channels}/consumer_registry.py (100%) mode change 100755 => 100644 create mode 100644 channels/docs/integration-changes.rst create mode 100644 channels/docs/message-standards.rst create mode 100644 channels/hacks.py rename {channel/channels => channels/management}/__init__.py (100%) rename {channel/management => channels/management/commands}/__init__.py (100%) rename channel/management/commands/runinterfaceserver.py => channels/management/commands/runserver.py (64%) mode change 100755 => 100644 rename {channel => channels}/request.py (87%) mode change 100755 => 100644 rename {channel => channels}/response.py (71%) mode change 100755 => 100644 rename {channel => channels}/utils.py (81%) mode change 100755 => 100644 rename {channel => channels}/worker.py (100%) mode change 100755 => 100644 diff --git a/channel/__init__.py b/channel/__init__.py deleted file mode 100755 index d5e0e00..0000000 --- a/channel/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .consumer_registry import ConsumerRegistry - -# Make a site-wide registry -coreg = ConsumerRegistry() - -# Load an implementation of Channel -from .channels.memory import Channel diff --git a/channel/adapters.py b/channel/adapters.py deleted file mode 100755 index 7f3474e..0000000 --- a/channel/adapters.py +++ /dev/null @@ -1,18 +0,0 @@ -from django.core.handlers.base import BaseHandler -from channel import Channel -from .response import encode_response -from .request import decode_request - - -class DjangoUrlAdapter(object): - """ - Adapts the channel-style HTTP requests to the URL-router/handler style - """ - - def __init__(self): - self.handler = BaseHandler() - self.handler.load_middleware() - - def __call__(self, request, response_channel): - response = self.handler.get_response(decode_request(request)) - Channel(response_channel).send(**encode_response(response)) diff --git a/channel/management/commands/__init__.py b/channel/management/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/channels/__init__.py b/channels/__init__.py new file mode 100644 index 0000000..1aa77ce --- /dev/null +++ b/channels/__init__.py @@ -0,0 +1,11 @@ +from .consumer_registry import ConsumerRegistry + +# Make a site-wide registry +coreg = ConsumerRegistry() + +# Load an implementation of Channel +from .backends import InMemoryChannel as Channel + +# Ensure monkeypatching +from .hacks import monkeypatch_django +monkeypatch_django() diff --git a/channels/adapters.py b/channels/adapters.py new file mode 100644 index 0000000..03c02c9 --- /dev/null +++ b/channels/adapters.py @@ -0,0 +1,47 @@ +from django.core.handlers.base import BaseHandler +from django.http import HttpRequest, HttpResponse +from channels import Channel, coreg + + +class UrlConsumer(object): + """ + Dispatches channel HTTP requests into django's URL system. + """ + + def __init__(self): + self.handler = BaseHandler() + self.handler.load_middleware() + + def __call__(self, **kwargs): + request = HttpRequest.channel_decode(kwargs) + try: + response = self.handler.get_response(request) + except HttpResponse.ResponseLater: + return + Channel(request.response_channel).send(**response.channel_encode()) + + +def view_producer(channel_name): + """ + Returns a new view function that actually writes the request to a channel + and abandons the response (with an exception the Worker will catch) + """ + def producing_view(request): + Channel(channel_name).send(**request.channel_encode()) + raise HttpResponse.ResponseLater() + return producing_view + + +def view_consumer(channel_name): + """ + Decorates a normal Django view to be a channel consumer. + Does not run any middleware. + """ + def inner(func): + def consumer(**kwargs): + request = HttpRequest.channel_decode(kwargs) + response = func(request) + Channel(request.response_channel).send(**response.channel_encode()) + coreg.add_consumer(consumer, [channel_name]) + return func + return inner diff --git a/channels/backends/__init__.py b/channels/backends/__init__.py new file mode 100644 index 0000000..0627c0a --- /dev/null +++ b/channels/backends/__init__.py @@ -0,0 +1,2 @@ +from .base import BaseChannel +from .memory import InMemoryChannel diff --git a/channel/channels/base.py b/channels/backends/base.py similarity index 78% rename from channel/channels/base.py rename to channels/backends/base.py index 7841d99..5341e0c 100644 --- a/channel/channels/base.py +++ b/channels/backends/base.py @@ -1,4 +1,4 @@ -class Channel(object): +class BaseChannel(object): """ Base class for all channel layer implementations. """ @@ -51,3 +51,12 @@ class Channel(object): response channels. """ raise NotImplementedError() + + def as_view(self): + """ + Returns a view version of this channel - one that takes + the request passed in and dispatches it to our channel, + serialized. + """ + from channels.adapters import view_producer + return view_producer(self.name) diff --git a/channel/channels/memory.py b/channels/backends/memory.py similarity index 92% rename from channel/channels/memory.py rename to channels/backends/memory.py index 6a66ceb..91bd1a2 100644 --- a/channel/channels/memory.py +++ b/channels/backends/memory.py @@ -2,12 +2,12 @@ import time import string import random from collections import deque -from .base import Channel as BaseChannel +from .base import BaseChannel queues = {} closed = set() -class Channel(BaseChannel): +class InMemoryChannel(BaseChannel): """ In-memory channel implementation. Intended only for use with threading, in low-throughput development environments. diff --git a/channel/consumer_registry.py b/channels/consumer_registry.py old mode 100755 new mode 100644 similarity index 100% rename from channel/consumer_registry.py rename to channels/consumer_registry.py diff --git a/channels/docs/integration-changes.rst b/channels/docs/integration-changes.rst new file mode 100644 index 0000000..2f6d204 --- /dev/null +++ b/channels/docs/integration-changes.rst @@ -0,0 +1,23 @@ +Message Standards +================= + +Some standardised message formats are used for common message types - they +are detailed below. + +HTTP Request +------------ + +Represents a full-fledged, single HTTP request coming in from a client. +Contains the following keys: + +* request: An encoded Django HTTP request +* response_channel: The channel name to write responses to + + +HTTP Response +------------- + +Sends a whole response to a client. +Contains the following keys: + +* response: An encoded Django HTTP response diff --git a/channels/docs/message-standards.rst b/channels/docs/message-standards.rst new file mode 100644 index 0000000..2f6d204 --- /dev/null +++ b/channels/docs/message-standards.rst @@ -0,0 +1,23 @@ +Message Standards +================= + +Some standardised message formats are used for common message types - they +are detailed below. + +HTTP Request +------------ + +Represents a full-fledged, single HTTP request coming in from a client. +Contains the following keys: + +* request: An encoded Django HTTP request +* response_channel: The channel name to write responses to + + +HTTP Response +------------- + +Sends a whole response to a client. +Contains the following keys: + +* response: An encoded Django HTTP response diff --git a/channels/hacks.py b/channels/hacks.py new file mode 100644 index 0000000..5d5e329 --- /dev/null +++ b/channels/hacks.py @@ -0,0 +1,27 @@ +from django.http.request import HttpRequest +from django.http.response import HttpResponseBase +from django.core.handlers.base import BaseHandler +from .request import encode_request, decode_request +from .response import encode_response, decode_response, ResponseLater + + +def monkeypatch_django(): + """ + Monkeypatches support for us into parts of Django. + """ + # Request encode/decode + HttpRequest.channel_encode = encode_request + HttpRequest.channel_decode = staticmethod(decode_request) + # Response encode/decode + HttpResponseBase.channel_encode = encode_response + HttpResponseBase.channel_decode = staticmethod(decode_response) + HttpResponseBase.ResponseLater = ResponseLater + # Allow ResponseLater to propagate above handler + BaseHandler.old_handle_uncaught_exception = BaseHandler.handle_uncaught_exception + BaseHandler.handle_uncaught_exception = new_handle_uncaught_exception + + +def new_handle_uncaught_exception(self, request, resolver, exc_info): + if exc_info[0] is ResponseLater: + raise + return BaseHandler.old_handle_uncaught_exception(self, request, resolver, exc_info) diff --git a/channel/channels/__init__.py b/channels/management/__init__.py similarity index 100% rename from channel/channels/__init__.py rename to channels/management/__init__.py diff --git a/channel/management/__init__.py b/channels/management/commands/__init__.py similarity index 100% rename from channel/management/__init__.py rename to channels/management/commands/__init__.py diff --git a/channel/management/commands/runinterfaceserver.py b/channels/management/commands/runserver.py old mode 100755 new mode 100644 similarity index 64% rename from channel/management/commands/runinterfaceserver.py rename to channels/management/commands/runserver.py index 796606d..910bac9 --- a/channel/management/commands/runinterfaceserver.py +++ b/channels/management/commands/runserver.py @@ -2,11 +2,11 @@ import django import threading from django.core.management.commands.runserver import Command as RunserverCommand from django.core.handlers.wsgi import WSGIHandler -from channel import Channel, coreg -from channel.request import encode_request -from channel.response import decode_response -from channel.worker import Worker -from channel.utils import auto_import_consumers +from django.http import HttpResponse +from channels import Channel, coreg +from channels.worker import Worker +from channels.utils import auto_import_consumers +from channels.adapters import UrlConsumer class Command(RunserverCommand): @@ -24,7 +24,8 @@ class Command(RunserverCommand): # Check a handler is registered for http reqs auto_import_consumers() if not coreg.consumer_for_channel("django.wsgi.request"): - raise RuntimeError("No consumer registered for WSGI requests") + # Register the default one + coreg.add_consumer(UrlConsumer(), ["django.wsgi.request"]) # Launch a worker thread worker = WorkerThread() worker.daemon = True @@ -39,13 +40,10 @@ class WSGIInterfaceHandler(WSGIHandler): """ def get_response(self, request): - response_channel = Channel.new_name("django.wsgi.response") - Channel("django.wsgi.request").send( - request = encode_request(request), - response_channel = response_channel, - ) - channel, message = Channel.receive_many([response_channel]) - return decode_response(message) + request.response_channel = Channel.new_name("django.wsgi.response") + Channel("django.wsgi.request").send(**request.channel_encode()) + channel, message = Channel.receive_many([request.response_channel]) + return HttpResponse.channel_decode(message) class WorkerThread(threading.Thread): diff --git a/channel/request.py b/channels/request.py old mode 100755 new mode 100644 similarity index 87% rename from channel/request.py rename to channels/request.py index c319ac5..7062d62 --- a/channel/request.py +++ b/channels/request.py @@ -15,6 +15,7 @@ def encode_request(request): "path": request.path, "path_info": request.path_info, "method": request.method, + "response_channel": request.response_channel, } return value @@ -31,4 +32,5 @@ def decode_request(value): request.path = value['path'] request.method = value['method'] request.path_info = value['path_info'] + request.response_channel = value['response_channel'] return request diff --git a/channel/response.py b/channels/response.py old mode 100755 new mode 100644 similarity index 71% rename from channel/response.py rename to channels/response.py index 542482a..c71d333 --- a/channel/response.py +++ b/channels/response.py @@ -27,3 +27,12 @@ def decode_response(value): ) response._headers = {k.lower: (k, v) for k, v in value['headers']} return response + + +class ResponseLater(Exception): + """ + Class that represents a response which will be sent doown the response + channel later. Used to move a django view-based segment onto the next + task, as otherwise we'd need to write some kind of fake response. + """ + pass diff --git a/channel/utils.py b/channels/utils.py old mode 100755 new mode 100644 similarity index 81% rename from channel/utils.py rename to channels/utils.py index 2615e72..35f56d6 --- a/channel/utils.py +++ b/channels/utils.py @@ -10,5 +10,5 @@ def auto_import_consumers(): try: __import__(consumer_module_name) except ImportError as e: - if "no module named" not in str(e).lower(): + if "no module named consumers" not in str(e).lower(): raise diff --git a/channel/worker.py b/channels/worker.py old mode 100755 new mode 100644 similarity index 100% rename from channel/worker.py rename to channels/worker.py From 2cc1d00e1806f396ff1ef52d0af3dda1fe9fddf0 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 8 Jun 2015 12:22:23 -0700 Subject: [PATCH 003/746] Separate channel backend from user-facing class --- channels/__init__.py | 9 +++- channels/adapters.py | 2 + channels/backends/__init__.py | 2 - channels/backends/base.py | 60 +++++------------------ channels/backends/memory.py | 31 +++--------- channels/channel.py | 48 ++++++++++++++++++ channels/consumer_registry.py | 10 ++-- channels/docs/integration-changes.rst | 33 +++++++------ channels/management/commands/runserver.py | 6 +-- channels/utils.py | 27 +++++++--- channels/worker.py | 8 +-- 11 files changed, 128 insertions(+), 108 deletions(-) create mode 100644 channels/channel.py diff --git a/channels/__init__.py b/channels/__init__.py index 1aa77ce..d679bd6 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,10 +1,15 @@ +from .channel import Channel from .consumer_registry import ConsumerRegistry # Make a site-wide registry coreg = ConsumerRegistry() -# Load an implementation of Channel -from .backends import InMemoryChannel as Channel +# Load a backend +from .backends.memory import InMemoryChannelBackend +DEFAULT_CHANNEL_LAYER = "default" +channel_layers = { + DEFAULT_CHANNEL_LAYER: InMemoryChannelBackend(), +} # Ensure monkeypatching from .hacks import monkeypatch_django diff --git a/channels/adapters.py b/channels/adapters.py index 03c02c9..e078310 100644 --- a/channels/adapters.py +++ b/channels/adapters.py @@ -1,3 +1,4 @@ +import functools from django.core.handlers.base import BaseHandler from django.http import HttpRequest, HttpResponse from channels import Channel, coreg @@ -38,6 +39,7 @@ def view_consumer(channel_name): Does not run any middleware. """ def inner(func): + @functools.wraps(func) def consumer(**kwargs): request = HttpRequest.channel_decode(kwargs) response = func(request) diff --git a/channels/backends/__init__.py b/channels/backends/__init__.py index 0627c0a..e69de29 100644 --- a/channels/backends/__init__.py +++ b/channels/backends/__init__.py @@ -1,2 +0,0 @@ -from .base import BaseChannel -from .memory import InMemoryChannel diff --git a/channels/backends/base.py b/channels/backends/base.py index 5341e0c..5ff69db 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -1,62 +1,24 @@ -class BaseChannel(object): +class ChannelClosed(Exception): + """ + Raised when you try to send to a closed channel. + """ + pass + + +class BaseChannelBackend(object): """ Base class for all channel layer implementations. """ - class ClosedError(Exception): - """ - Raised when you try to send to a closed channel. - """ - pass - - def __init__(self, name): - """ - Create an instance for the channel named "name" - """ - self.name = name - - def send(self, **kwargs): + def send(self, channel, message): """ Send a message over the channel, taken from the kwargs. """ raise NotImplementedError() - def close(self): - """ - Closes the channel, allowing no more messages to be sent over it. - """ - raise NotImplementedError() - - @property - def closed(self): - """ - Says if the channel is closed. - """ - raise NotImplementedError() - - @classmethod - def receive_many(self, channel_names): + def receive_many(self, channels): """ Block and return the first message available on one of the - channels passed, as a (channel_name, message) tuple. + channels passed, as a (channel, message) tuple. """ raise NotImplementedError() - - @classmethod - def new_name(self, prefix): - """ - Returns a new channel name that's unique and not closed - with the given prefix. Does not need to be called before sending - on a channel name - just provides a way to avoid clashing for - response channels. - """ - raise NotImplementedError() - - def as_view(self): - """ - Returns a view version of this channel - one that takes - the request passed in and dispatches it to our channel, - serialized. - """ - from channels.adapters import view_producer - return view_producer(self.name) diff --git a/channels/backends/memory.py b/channels/backends/memory.py index 91bd1a2..8b4bd14 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -2,47 +2,30 @@ import time import string import random from collections import deque -from .base import BaseChannel +from .base import BaseChannelBackend queues = {} -closed = set() -class InMemoryChannel(BaseChannel): +class InMemoryChannelBackend(BaseChannelBackend): """ In-memory channel implementation. Intended only for use with threading, in low-throughput development environments. """ - def send(self, **kwargs): - # Don't allow if closed - if self.name in closed: - raise Channel.ClosedError("%s is closed" % self.name) + def send(self, channel, message): # Add to the deque, making it if needs be - queues.setdefault(self.name, deque()).append(kwargs) + queues.setdefault(channel, deque()).append(message) - @property - def closed(self): - # Check closed set - return self.name in closed - - def close(self): - # Add to closed set - closed.add(self.name) - - @classmethod - def receive_many(self, channel_names): + def receive_many(self, channels): while True: # Try to pop a message from each channel - for channel_name in channel_names: + for channel in channels: try: # This doesn't clean up empty channels - OK for testing. # For later versions, have cleanup w/lock. - return channel_name, queues[channel_name].popleft() + return channel, queues[channel].popleft() except (IndexError, KeyError): pass # If all empty, sleep for a little bit time.sleep(0.01) - @classmethod - def new_name(self, prefix): - return "%s.%s" % (prefix, "".join(random.choice(string.ascii_letters) for i in range(16))) diff --git a/channels/channel.py b/channels/channel.py new file mode 100644 index 0000000..ea4a133 --- /dev/null +++ b/channels/channel.py @@ -0,0 +1,48 @@ +import random +import string + + +class Channel(object): + """ + Public interaction class for the channel layer. + + This is separate to the backends so we can: + a) Hide receive_many from end-users, as it is only for interface servers + b) Keep a stable-ish backend interface for third parties + + You can pass an alternate Channel Layer alias in, but it will use the + "default" one by default. + """ + + def __init__(self, name, alias=None): + """ + Create an instance for the channel named "name" + """ + from channels import channel_layers, DEFAULT_CHANNEL_LAYER + self.name = name + self.channel_layer = channel_layers[alias or DEFAULT_CHANNEL_LAYER] + + def send(self, **kwargs): + """ + Send a message over the channel, taken from the kwargs. + """ + self.channel_layer.send(self.name, kwargs) + + @classmethod + def new_name(self, prefix): + """ + Returns a new channel name that's unique and not closed + with the given prefix. Does not need to be called before sending + on a channel name - just provides a way to avoid clashing for + response channels. + """ + return "%s.%s" % (prefix, "".join(random.choice(string.ascii_letters) for i in range(32))) + + def as_view(self): + """ + Returns a view version of this channel - one that takes + the request passed in and dispatches it to our channel, + serialized. + """ + from channels.adapters import view_producer + return view_producer(self.name) diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index 0e6baaa..e308282 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -1,4 +1,6 @@ import functools +from django.utils import six +from .utils import name_that_thing class ConsumerRegistry(object): """ @@ -14,10 +16,10 @@ class ConsumerRegistry(object): def add_consumer(self, consumer, channels): for channel in channels: if channel in self.consumers: - raise ValueError("Cannot register consumer %s - channel %s already consumed by %s" % ( - consumer, + raise ValueError("Cannot register consumer %s - channel %r already consumed by %s" % ( + name_that_thing(consumer), channel, - self.consumers[channel], + name_that_thing(self.consumers[channel]), )) self.consumers[channel] = consumer @@ -25,6 +27,8 @@ class ConsumerRegistry(object): """ Decorator that registers a function as a consumer. """ + if isinstance(channels, six.string_types): + channels = [channels] def inner(func): self.add_consumer(func, channels) return func diff --git a/channels/docs/integration-changes.rst b/channels/docs/integration-changes.rst index 2f6d204..c59fa38 100644 --- a/channels/docs/integration-changes.rst +++ b/channels/docs/integration-changes.rst @@ -1,23 +1,24 @@ -Message Standards +Integration Notes ================= -Some standardised message formats are used for common message types - they -are detailed below. +Django Channels is intended to be merged into Django itself; these are the +planned changes the codebase will need to undertake in that transition. -HTTP Request ------------- +* The ``channels`` package will become ``django.channels``. The expected way + of interacting with the system will be via the ``Channel`` object, -Represents a full-fledged, single HTTP request coming in from a client. -Contains the following keys: +* Obviously, the monkeypatches in ``channels.hacks`` will be replaced by + placing methods onto the objects themselves. The ``request`` and ``response`` + modules will thus no longer exist separately. -* request: An encoded Django HTTP request -* response_channel: The channel name to write responses to +Things to ponder +---------------- +* The mismatch between signals (broadcast) and channels (single-worker) means + we should probably leave patching signals into channels for the end developer. + This would also ensure the speedup improvements for empty signals keep working. -HTTP Response -------------- - -Sends a whole response to a client. -Contains the following keys: - -* response: An encoded Django HTTP response +* It's likely that the decorator-based approach of consumer registration will + mean extending Django's auto-module-loading beyond ``models`` and + ``admin`` app modules to include ``views`` and ``consumers``. There may be + a better unified approach to this. diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 910bac9..8ed1563 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -3,7 +3,7 @@ import threading from django.core.management.commands.runserver import Command as RunserverCommand from django.core.handlers.wsgi import WSGIHandler from django.http import HttpResponse -from channels import Channel, coreg +from channels import Channel, coreg, channel_layers, DEFAULT_CHANNEL_LAYER from channels.worker import Worker from channels.utils import auto_import_consumers from channels.adapters import UrlConsumer @@ -42,7 +42,7 @@ class WSGIInterfaceHandler(WSGIHandler): def get_response(self, request): request.response_channel = Channel.new_name("django.wsgi.response") Channel("django.wsgi.request").send(**request.channel_encode()) - channel, message = Channel.receive_many([request.response_channel]) + channel, message = channel_layers[DEFAULT_CHANNEL_LAYER].receive_many([request.response_channel]) return HttpResponse.channel_decode(message) @@ -54,5 +54,5 @@ class WorkerThread(threading.Thread): def run(self): Worker( consumer_registry = coreg, - channel_class = Channel, + channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER], ).run() diff --git a/channels/utils.py b/channels/utils.py index 35f56d6..b110e7a 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -1,3 +1,4 @@ +import types from django.apps import apps @@ -6,9 +7,23 @@ def auto_import_consumers(): Auto-import consumers modules in apps """ for app_config in apps.get_app_configs(): - consumer_module_name = "%s.consumers" % (app_config.name,) - try: - __import__(consumer_module_name) - except ImportError as e: - if "no module named consumers" not in str(e).lower(): - raise + for submodule in ["consumers", "views"]: + module_name = "%s.%s" % (app_config.name, submodule) + try: + __import__(module_name) + except ImportError as e: + if "no module named %s" % submodule not in str(e).lower(): + raise + + +def name_that_thing(thing): + """ + Returns either the function/class path or just the object's repr + """ + if hasattr(thing, "__name__"): + if hasattr(thing, "__class__") and not isinstance(thing, types.FunctionType): + if thing.__class__ is not type: + return name_that_thing(thing.__class__) + if hasattr(thing, "__module__"): + return "%s.%s" % (thing.__module__, thing.__name__) + return repr(thing) diff --git a/channels/worker.py b/channels/worker.py index 73a2c1f..4f4d129 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -4,16 +4,18 @@ class Worker(object): and runs their consumers. """ - def __init__(self, consumer_registry, channel_class): + def __init__(self, consumer_registry, channel_layer): + from channels import channel_layers, DEFAULT_CHANNEL_LAYER self.consumer_registry = consumer_registry - self.channel_class = channel_class + self.channel_layer = channel_layer def run(self): """ Tries to continually dispatch messages to consumers. """ + channels = self.consumer_registry.all_channel_names() while True: - channel, message = self.channel_class.receive_many(channels) + channel, message = self.channel_layer.receive_many(channels) consumer = self.consumer_registry.consumer_for_channel(channel) consumer(**message) From c2c1ffc5bd59443463456b91d85598ab25bca8a1 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 8 Jun 2015 12:40:47 -0700 Subject: [PATCH 004/746] Make everything hang off of channel_layers --- channels/__init__.py | 4 ---- channels/adapters.py | 12 ++++++++---- channels/backends/base.py | 10 +++++++++- channels/channel.py | 19 +++++++++++++++++++ channels/consumer_registry.py | 20 ++++++++------------ channels/management/commands/runserver.py | 18 ++++++++++-------- channels/worker.py | 8 +++----- 7 files changed, 57 insertions(+), 34 deletions(-) diff --git a/channels/__init__.py b/channels/__init__.py index d679bd6..8d02ab0 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,8 +1,4 @@ from .channel import Channel -from .consumer_registry import ConsumerRegistry - -# Make a site-wide registry -coreg = ConsumerRegistry() # Load a backend from .backends.memory import InMemoryChannelBackend diff --git a/channels/adapters.py b/channels/adapters.py index e078310..bad2276 100644 --- a/channels/adapters.py +++ b/channels/adapters.py @@ -1,7 +1,9 @@ import functools + from django.core.handlers.base import BaseHandler from django.http import HttpRequest, HttpResponse -from channels import Channel, coreg + +from channels import Channel, channel_layers, DEFAULT_CHANNEL_LAYER class UrlConsumer(object): @@ -33,10 +35,10 @@ def view_producer(channel_name): return producing_view -def view_consumer(channel_name): +def view_consumer(channel_name, alias=None): """ Decorates a normal Django view to be a channel consumer. - Does not run any middleware. + Does not run any middleware """ def inner(func): @functools.wraps(func) @@ -44,6 +46,8 @@ def view_consumer(channel_name): request = HttpRequest.channel_decode(kwargs) response = func(request) Channel(request.response_channel).send(**response.channel_encode()) - coreg.add_consumer(consumer, [channel_name]) + # Get the channel layer and register + channel_layer = channel_layers[alias or DEFAULT_CHANNEL_LAYER] + channel_layer.registry.add_consumer(consumer, [channel_name]) return func return inner diff --git a/channels/backends/base.py b/channels/backends/base.py index 5ff69db..a35fcdc 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -1,3 +1,6 @@ +from channels.consumer_registry import ConsumerRegistry + + class ChannelClosed(Exception): """ Raised when you try to send to a closed channel. @@ -7,9 +10,14 @@ class ChannelClosed(Exception): class BaseChannelBackend(object): """ - Base class for all channel layer implementations. + Base class for all channel layer implementations. Manages both sending + and receving messages from the backend, and each comes with its own + registry of consumers. """ + def __init__(self): + self.registry = ConsumerRegistry() + def send(self, channel, message): """ Send a message over the channel, taken from the kwargs. diff --git a/channels/channel.py b/channels/channel.py index ea4a133..a911548 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -1,6 +1,8 @@ import random import string +from django.utils import six + class Channel(object): """ @@ -46,3 +48,20 @@ class Channel(object): """ from channels.adapters import view_producer return view_producer(self.name) + + @classmethod + def consumer(self, channels, alias=None): + """ + Decorator that registers a function as a consumer. + """ + from channels import channel_layers, DEFAULT_CHANNEL_LAYER + # Upconvert if you just pass in a string + if isinstance(channels, six.string_types): + channels = [channels] + # Get the channel + channel_layer = channel_layers[alias or DEFAULT_CHANNEL_LAYER] + # Return a function that'll register whatever it wraps + def inner(func): + channel_layer.registry.add_consumer(func, channels) + return func + return inner diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index e308282..1bfe4e0 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -1,19 +1,26 @@ import functools + from django.utils import six + from .utils import name_that_thing + class ConsumerRegistry(object): """ Manages the available consumers in the project and which channels they listen to. - Generally a single project-wide instance of this is used. + Generally this is attached to a backend instance as ".registry" """ def __init__(self): self.consumers = {} def add_consumer(self, consumer, channels): + # Upconvert if you just pass in a string + if isinstance(channels, six.string_types): + channels = [channels] + # Register on each channel, checking it's unique for channel in channels: if channel in self.consumers: raise ValueError("Cannot register consumer %s - channel %r already consumed by %s" % ( @@ -23,17 +30,6 @@ class ConsumerRegistry(object): )) self.consumers[channel] = consumer - def consumer(self, channels): - """ - Decorator that registers a function as a consumer. - """ - if isinstance(channels, six.string_types): - channels = [channels] - def inner(func): - self.add_consumer(func, channels) - return func - return inner - def all_channel_names(self): return self.consumers.keys() diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 8ed1563..f17aa95 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -3,7 +3,7 @@ import threading from django.core.management.commands.runserver import Command as RunserverCommand from django.core.handlers.wsgi import WSGIHandler from django.http import HttpResponse -from channels import Channel, coreg, channel_layers, DEFAULT_CHANNEL_LAYER +from channels import Channel, channel_layers, DEFAULT_CHANNEL_LAYER from channels.worker import Worker from channels.utils import auto_import_consumers from channels.adapters import UrlConsumer @@ -22,12 +22,13 @@ class Command(RunserverCommand): # Force disable reloader for now options['use_reloader'] = False # Check a handler is registered for http reqs + channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] auto_import_consumers() - if not coreg.consumer_for_channel("django.wsgi.request"): + if not channel_layer.registry.consumer_for_channel("django.wsgi.request"): # Register the default one - coreg.add_consumer(UrlConsumer(), ["django.wsgi.request"]) + channel_layer.registry.add_consumer(UrlConsumer(), ["django.wsgi.request"]) # Launch a worker thread - worker = WorkerThread() + worker = WorkerThread(channel_layer) worker.daemon = True worker.start() # Run the rest @@ -51,8 +52,9 @@ class WorkerThread(threading.Thread): Class that runs a worker """ + def __init__(self, channel_layer): + super(WorkerThread, self).__init__() + self.channel_layer = channel_layer + def run(self): - Worker( - consumer_registry = coreg, - channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER], - ).run() + Worker(channel_layer=self.channel_layer).run() diff --git a/channels/worker.py b/channels/worker.py index 4f4d129..9ce0606 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -4,9 +4,7 @@ class Worker(object): and runs their consumers. """ - def __init__(self, consumer_registry, channel_layer): - from channels import channel_layers, DEFAULT_CHANNEL_LAYER - self.consumer_registry = consumer_registry + def __init__(self, channel_layer): self.channel_layer = channel_layer def run(self): @@ -14,8 +12,8 @@ class Worker(object): Tries to continually dispatch messages to consumers. """ - channels = self.consumer_registry.all_channel_names() + channels = self.channel_layer.registry.all_channel_names() while True: channel, message = self.channel_layer.receive_many(channels) - consumer = self.consumer_registry.consumer_for_channel(channel) + consumer = self.channel_layer.registry.consumer_for_channel(channel) consumer(**message) From 5a7af1e3afc4656585e5f278e5be2dfbfc0f79d2 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 8 Jun 2015 13:35:27 -0700 Subject: [PATCH 005/746] Add channel to every consumer signature, update msg fmt docs --- channels/adapters.py | 4 +- channels/backends/base.py | 3 +- channels/docs/message-standards.rst | 64 +++++++++++++++++++++++++++-- channels/worker.py | 2 +- 4 files changed, 66 insertions(+), 7 deletions(-) diff --git a/channels/adapters.py b/channels/adapters.py index bad2276..85ac5b6 100644 --- a/channels/adapters.py +++ b/channels/adapters.py @@ -15,7 +15,7 @@ class UrlConsumer(object): self.handler = BaseHandler() self.handler.load_middleware() - def __call__(self, **kwargs): + def __call__(self, channel, **kwargs): request = HttpRequest.channel_decode(kwargs) try: response = self.handler.get_response(request) @@ -42,7 +42,7 @@ def view_consumer(channel_name, alias=None): """ def inner(func): @functools.wraps(func) - def consumer(**kwargs): + def consumer(channel, **kwargs): request = HttpRequest.channel_decode(kwargs) response = func(request) Channel(request.response_channel).send(**response.channel_encode()) diff --git a/channels/backends/base.py b/channels/backends/base.py index a35fcdc..62866f0 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -15,8 +15,9 @@ class BaseChannelBackend(object): registry of consumers. """ - def __init__(self): + def __init__(self, expiry=60): self.registry = ConsumerRegistry() + self.expiry = expiry def send(self, channel, message): """ diff --git a/channels/docs/message-standards.rst b/channels/docs/message-standards.rst index 2f6d204..ef225cb 100644 --- a/channels/docs/message-standards.rst +++ b/channels/docs/message-standards.rst @@ -4,20 +4,78 @@ Message Standards Some standardised message formats are used for common message types - they are detailed below. +Note: All consumers also receive the channel name as the keyword argument +"channel", so there is no need for separate type information to let +multi-channel consumers distinguish. + HTTP Request ------------ Represents a full-fledged, single HTTP request coming in from a client. + Contains the following keys: -* request: An encoded Django HTTP request -* response_channel: The channel name to write responses to +* GET: List of (key, value) tuples of GET variables +* POST: List of (key, value) tuples of POST variables +* COOKIES: Same as ``request.COOKIES`` +* META: Same as ``request.META`` +* path: Same as ``request.path`` +* path_info: Same as ``request.path_info`` +* method: Upper-cased HTTP method +* response_channel: Channel name to write response to HTTP Response ------------- Sends a whole response to a client. + Contains the following keys: -* response: An encoded Django HTTP response +* content: String of content to send +* content_type: Mimetype of content +* status_code: Numerical HTTP status code +* headers: Dictionary of headers (key is header name, value is value) + + +HTTP Disconnect +--------------- + +Send when a client disconnects early, before the response has been sent. +Only sent by long-polling-capable HTTP interface servers. + +Contains the same keys as HTTP Request. + + +WebSocket Connection +-------------------- + +Sent when a new WebSocket is connected. + +Contains the following keys: + +* GET: List of (key, value) tuples of GET variables +* COOKIES: Same as ``request.COOKIES`` +* META: Same as ``request.META`` +* path: Same as ``request.path`` +* path_info: Same as ``request.path_info`` +* send_channel: Channel name to send responses on + + +WebSocket Receive +----------------- + +Sent when a datagram is received on the WebSocket. + +Contains the same keys as WebSocket Connection, plus: + +* content: String content of the datagram + + +WebSocket Close +--------------- + +Sent when the WebSocket is closed by either the client or the server. + +Contains the same keys as WebSocket Connection, including send_channel, +though nothing should be sent on it. diff --git a/channels/worker.py b/channels/worker.py index 9ce0606..47fa21a 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -16,4 +16,4 @@ class Worker(object): while True: channel, message = self.channel_layer.receive_many(channels) consumer = self.channel_layer.registry.consumer_for_channel(channel) - consumer(**message) + consumer(channel=channel, **message) From c9eb683ed8b4c2df43582997868ea1393a0e0cb4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Jun 2015 09:40:57 -0700 Subject: [PATCH 006/746] Switch to settings-based backend list, start ORM backend --- channels/__init__.py | 22 +++++--- channels/adapters.py | 6 +- channels/backends/__init__.py | 27 +++++++++ channels/backends/memory.py | 5 +- channels/backends/orm.py | 68 +++++++++++++++++++++++ channels/channel.py | 12 ++-- channels/docs/message-standards.rst | 3 + channels/management/commands/runserver.py | 6 +- 8 files changed, 127 insertions(+), 22 deletions(-) create mode 100644 channels/backends/orm.py diff --git a/channels/__init__.py b/channels/__init__.py index 8d02ab0..6203a7b 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,12 +1,18 @@ -from .channel import Channel - -# Load a backend -from .backends.memory import InMemoryChannelBackend -DEFAULT_CHANNEL_LAYER = "default" -channel_layers = { - DEFAULT_CHANNEL_LAYER: InMemoryChannelBackend(), -} +# Load backends +DEFAULT_CHANNEL_BACKEND = "default" +from .backends import BackendManager +from django.conf import settings +channel_backends = BackendManager( + getattr(settings, "CHANNEL_BACKENDS", { + DEFAULT_CHANNEL_BACKEND: { + "BACKEND": "channels.backends.memory.InMemoryChannelBackend", + } + }) +) # Ensure monkeypatching from .hacks import monkeypatch_django monkeypatch_django() + +# Promote channel to top-level (down here to avoid circular import errs) +from .channel import Channel diff --git a/channels/adapters.py b/channels/adapters.py index 85ac5b6..6e31647 100644 --- a/channels/adapters.py +++ b/channels/adapters.py @@ -3,7 +3,7 @@ import functools from django.core.handlers.base import BaseHandler from django.http import HttpRequest, HttpResponse -from channels import Channel, channel_layers, DEFAULT_CHANNEL_LAYER +from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND class UrlConsumer(object): @@ -35,7 +35,7 @@ def view_producer(channel_name): return producing_view -def view_consumer(channel_name, alias=None): +def view_consumer(channel_name, alias=DEFAULT_CHANNEL_BACKEND): """ Decorates a normal Django view to be a channel consumer. Does not run any middleware @@ -47,7 +47,7 @@ def view_consumer(channel_name, alias=None): response = func(request) Channel(request.response_channel).send(**response.channel_encode()) # Get the channel layer and register - channel_layer = channel_layers[alias or DEFAULT_CHANNEL_LAYER] + channel_layer = channel_backends[DEFAULT_CHANNEL_BACKEND] channel_layer.registry.add_consumer(consumer, [channel_name]) return func return inner diff --git a/channels/backends/__init__.py b/channels/backends/__init__.py index e69de29..83c74f4 100644 --- a/channels/backends/__init__.py +++ b/channels/backends/__init__.py @@ -0,0 +1,27 @@ +from django.utils.module_loading import import_string + + +class InvalidChannelBackendError(ValueError): + pass + + +class BackendManager(object): + """ + Takes a settings dictionary of backends and initialises them. + """ + + def __init__(self, backend_configs): + self.backends = {} + for name, config in backend_configs.items(): + # Load the backend class + try: + backend_class = import_string(config['BACKEND']) + except KeyError: + raise InvalidChannelBackendError("No BACKEND specified for %s" % name) + except ImportError: + raise InvalidChannelBackendError("Cannot import BACKEND %s specified for %s" % (config['BACKEND'], name)) + # Initialise and pass config + self.backends[name] = backend_class(**{k.lower(): v for k, v in config.items() if k != "BACKEND"}) + + def __getitem__(self, key): + return self.backends[key] diff --git a/channels/backends/memory.py b/channels/backends/memory.py index 8b4bd14..770e602 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -1,6 +1,5 @@ import time -import string -import random +import json from collections import deque from .base import BaseChannelBackend @@ -13,6 +12,8 @@ class InMemoryChannelBackend(BaseChannelBackend): """ def send(self, channel, message): + # Try JSON encoding it to make sure it would, but store the native version + json.dumps(message) # Add to the deque, making it if needs be queues.setdefault(channel, deque()).append(message) diff --git a/channels/backends/orm.py b/channels/backends/orm.py new file mode 100644 index 0000000..225a9f7 --- /dev/null +++ b/channels/backends/orm.py @@ -0,0 +1,68 @@ +import time +import datetime + +from django.apps.registry import Apps +from django.db import models, connections, DEFAULT_DB_ALIAS + +from .base import BaseChannelBackend + +queues = {} + +class ORMChannelBackend(BaseChannelBackend): + """ + ORM-backed channel environment. For development use only; it will span + multiple processes fine, but it's going to be pretty bad at throughput. + """ + + def __init__(self, expiry, db_alias=DEFAULT_DB_ALIAS): + super(ORMChannelBackend, self).__init__(expiry) + self.connection = connections[db_alias] + self.model = self.make_model() + self.ensure_schema() + + def make_model(self): + """ + Initialises a new model to store messages; not done as part of a + models.py as we don't want to make it for most installs. + """ + class Message(models.Model): + # We assume an autoincrementing PK for message order + channel = models.CharField(max_length=200, db_index=True) + content = models.TextField() + expiry = models.DateTimeField(db_index=True) + class Meta: + apps = Apps() + app_label = "channels" + db_table = "django_channels" + return Message + + def ensure_schema(self): + """ + Ensures the table exists and has the correct schema. + """ + # If the table's there, that's fine - we've never changed its schema + # in the codebase. + if self.model._meta.db_table in self.connection.introspection.table_names(self.connection.cursor()): + return + # Make the table + with self.connection.schema_editor() as editor: + editor.create_model(self.model) + + def send(self, channel, message): + self.model.objects.create( + channel = channel, + message = json.dumps(message), + expiry = datetime.datetime.utcnow() + datetime.timedelta(seconds=self.expiry) + ) + + def receive_many(self, channels): + while True: + # Delete all expired messages (add 10 second grace period for clock sync) + self.model.objects.filter(expiry__lt=datetime.datetime.utcnow() - datetime.timedelta(seconds=10)).delete() + # Get a message from one of our channels + message = self.model.objects.filter(channel__in=channels).order_by("id").first() + if message: + return message.channel, json.loads(message.content) + # If all empty, sleep for a little bit + time.sleep(0.2) + diff --git a/channels/channel.py b/channels/channel.py index a911548..b4e887d 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -3,6 +3,8 @@ import string from django.utils import six +from channels import channel_backends, DEFAULT_CHANNEL_BACKEND + class Channel(object): """ @@ -16,13 +18,12 @@ class Channel(object): "default" one by default. """ - def __init__(self, name, alias=None): + def __init__(self, name, alias=DEFAULT_CHANNEL_BACKEND): """ Create an instance for the channel named "name" """ - from channels import channel_layers, DEFAULT_CHANNEL_LAYER self.name = name - self.channel_layer = channel_layers[alias or DEFAULT_CHANNEL_LAYER] + self.channel_layer = channel_backends[alias] def send(self, **kwargs): """ @@ -50,16 +51,15 @@ class Channel(object): return view_producer(self.name) @classmethod - def consumer(self, channels, alias=None): + def consumer(self, channels, alias=DEFAULT_CHANNEL_BACKEND): """ Decorator that registers a function as a consumer. """ - from channels import channel_layers, DEFAULT_CHANNEL_LAYER # Upconvert if you just pass in a string if isinstance(channels, six.string_types): channels = [channels] # Get the channel - channel_layer = channel_layers[alias or DEFAULT_CHANNEL_LAYER] + channel_layer = channel_backends[alias] # Return a function that'll register whatever it wraps def inner(func): channel_layer.registry.add_consumer(func, channels) diff --git a/channels/docs/message-standards.rst b/channels/docs/message-standards.rst index ef225cb..d22d2df 100644 --- a/channels/docs/message-standards.rst +++ b/channels/docs/message-standards.rst @@ -8,6 +8,9 @@ Note: All consumers also receive the channel name as the keyword argument "channel", so there is no need for separate type information to let multi-channel consumers distinguish. +The length limit on channel names will be 200 characters. + + HTTP Request ------------ diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index f17aa95..7858359 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -3,7 +3,7 @@ import threading from django.core.management.commands.runserver import Command as RunserverCommand from django.core.handlers.wsgi import WSGIHandler from django.http import HttpResponse -from channels import Channel, channel_layers, DEFAULT_CHANNEL_LAYER +from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND from channels.worker import Worker from channels.utils import auto_import_consumers from channels.adapters import UrlConsumer @@ -22,7 +22,7 @@ class Command(RunserverCommand): # Force disable reloader for now options['use_reloader'] = False # Check a handler is registered for http reqs - channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + channel_layer = channel_backends[DEFAULT_CHANNEL_BACKEND] auto_import_consumers() if not channel_layer.registry.consumer_for_channel("django.wsgi.request"): # Register the default one @@ -43,7 +43,7 @@ class WSGIInterfaceHandler(WSGIHandler): def get_response(self, request): request.response_channel = Channel.new_name("django.wsgi.response") Channel("django.wsgi.request").send(**request.channel_encode()) - channel, message = channel_layers[DEFAULT_CHANNEL_LAYER].receive_many([request.response_channel]) + channel, message = channel_backends[DEFAULT_CHANNEL_BACKEND].receive_many([request.response_channel]) return HttpResponse.channel_decode(message) From 95e706f71f0ef6c9911a59fbe32b54698bf4fc98 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Jun 2015 09:42:34 -0700 Subject: [PATCH 007/746] Fix line endings --- channels/__init__.py | 36 +++--- channels/adapters.py | 106 ++++++++--------- channels/backends/base.py | 66 +++++------ channels/backends/memory.py | 64 +++++----- channels/backends/orm.py | 136 +++++++++++----------- channels/consumer_registry.py | 80 ++++++------- channels/hacks.py | 54 ++++----- channels/management/commands/runserver.py | 120 +++++++++---------- channels/request.py | 72 ++++++------ channels/response.py | 76 ++++++------ channels/utils.py | 58 ++++----- channels/worker.py | 38 +++--- 12 files changed, 453 insertions(+), 453 deletions(-) diff --git a/channels/__init__.py b/channels/__init__.py index 6203a7b..1010c41 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,18 +1,18 @@ -# Load backends -DEFAULT_CHANNEL_BACKEND = "default" -from .backends import BackendManager -from django.conf import settings -channel_backends = BackendManager( - getattr(settings, "CHANNEL_BACKENDS", { - DEFAULT_CHANNEL_BACKEND: { - "BACKEND": "channels.backends.memory.InMemoryChannelBackend", - } - }) -) - -# Ensure monkeypatching -from .hacks import monkeypatch_django -monkeypatch_django() - -# Promote channel to top-level (down here to avoid circular import errs) -from .channel import Channel +# Load backends +DEFAULT_CHANNEL_BACKEND = "default" +from .backends import BackendManager +from django.conf import settings +channel_backends = BackendManager( + getattr(settings, "CHANNEL_BACKENDS", { + DEFAULT_CHANNEL_BACKEND: { + "BACKEND": "channels.backends.memory.InMemoryChannelBackend", + } + }) +) + +# Ensure monkeypatching +from .hacks import monkeypatch_django +monkeypatch_django() + +# Promote channel to top-level (down here to avoid circular import errs) +from .channel import Channel diff --git a/channels/adapters.py b/channels/adapters.py index 6e31647..0879241 100644 --- a/channels/adapters.py +++ b/channels/adapters.py @@ -1,53 +1,53 @@ -import functools - -from django.core.handlers.base import BaseHandler -from django.http import HttpRequest, HttpResponse - -from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND - - -class UrlConsumer(object): - """ - Dispatches channel HTTP requests into django's URL system. - """ - - def __init__(self): - self.handler = BaseHandler() - self.handler.load_middleware() - - def __call__(self, channel, **kwargs): - request = HttpRequest.channel_decode(kwargs) - try: - response = self.handler.get_response(request) - except HttpResponse.ResponseLater: - return - Channel(request.response_channel).send(**response.channel_encode()) - - -def view_producer(channel_name): - """ - Returns a new view function that actually writes the request to a channel - and abandons the response (with an exception the Worker will catch) - """ - def producing_view(request): - Channel(channel_name).send(**request.channel_encode()) - raise HttpResponse.ResponseLater() - return producing_view - - -def view_consumer(channel_name, alias=DEFAULT_CHANNEL_BACKEND): - """ - Decorates a normal Django view to be a channel consumer. - Does not run any middleware - """ - def inner(func): - @functools.wraps(func) - def consumer(channel, **kwargs): - request = HttpRequest.channel_decode(kwargs) - response = func(request) - Channel(request.response_channel).send(**response.channel_encode()) - # Get the channel layer and register - channel_layer = channel_backends[DEFAULT_CHANNEL_BACKEND] - channel_layer.registry.add_consumer(consumer, [channel_name]) - return func - return inner +import functools + +from django.core.handlers.base import BaseHandler +from django.http import HttpRequest, HttpResponse + +from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND + + +class UrlConsumer(object): + """ + Dispatches channel HTTP requests into django's URL system. + """ + + def __init__(self): + self.handler = BaseHandler() + self.handler.load_middleware() + + def __call__(self, channel, **kwargs): + request = HttpRequest.channel_decode(kwargs) + try: + response = self.handler.get_response(request) + except HttpResponse.ResponseLater: + return + Channel(request.response_channel).send(**response.channel_encode()) + + +def view_producer(channel_name): + """ + Returns a new view function that actually writes the request to a channel + and abandons the response (with an exception the Worker will catch) + """ + def producing_view(request): + Channel(channel_name).send(**request.channel_encode()) + raise HttpResponse.ResponseLater() + return producing_view + + +def view_consumer(channel_name, alias=DEFAULT_CHANNEL_BACKEND): + """ + Decorates a normal Django view to be a channel consumer. + Does not run any middleware + """ + def inner(func): + @functools.wraps(func) + def consumer(channel, **kwargs): + request = HttpRequest.channel_decode(kwargs) + response = func(request) + Channel(request.response_channel).send(**response.channel_encode()) + # Get the channel layer and register + channel_layer = channel_backends[DEFAULT_CHANNEL_BACKEND] + channel_layer.registry.add_consumer(consumer, [channel_name]) + return func + return inner diff --git a/channels/backends/base.py b/channels/backends/base.py index 62866f0..a3481cb 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -1,33 +1,33 @@ -from channels.consumer_registry import ConsumerRegistry - - -class ChannelClosed(Exception): - """ - Raised when you try to send to a closed channel. - """ - pass - - -class BaseChannelBackend(object): - """ - Base class for all channel layer implementations. Manages both sending - and receving messages from the backend, and each comes with its own - registry of consumers. - """ - - def __init__(self, expiry=60): - self.registry = ConsumerRegistry() - self.expiry = expiry - - def send(self, channel, message): - """ - Send a message over the channel, taken from the kwargs. - """ - raise NotImplementedError() - - def receive_many(self, channels): - """ - Block and return the first message available on one of the - channels passed, as a (channel, message) tuple. - """ - raise NotImplementedError() +from channels.consumer_registry import ConsumerRegistry + + +class ChannelClosed(Exception): + """ + Raised when you try to send to a closed channel. + """ + pass + + +class BaseChannelBackend(object): + """ + Base class for all channel layer implementations. Manages both sending + and receving messages from the backend, and each comes with its own + registry of consumers. + """ + + def __init__(self, expiry=60): + self.registry = ConsumerRegistry() + self.expiry = expiry + + def send(self, channel, message): + """ + Send a message over the channel, taken from the kwargs. + """ + raise NotImplementedError() + + def receive_many(self, channels): + """ + Block and return the first message available on one of the + channels passed, as a (channel, message) tuple. + """ + raise NotImplementedError() diff --git a/channels/backends/memory.py b/channels/backends/memory.py index 770e602..123ec40 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -1,32 +1,32 @@ -import time -import json -from collections import deque -from .base import BaseChannelBackend - -queues = {} - -class InMemoryChannelBackend(BaseChannelBackend): - """ - In-memory channel implementation. Intended only for use with threading, - in low-throughput development environments. - """ - - def send(self, channel, message): - # Try JSON encoding it to make sure it would, but store the native version - json.dumps(message) - # Add to the deque, making it if needs be - queues.setdefault(channel, deque()).append(message) - - def receive_many(self, channels): - while True: - # Try to pop a message from each channel - for channel in channels: - try: - # This doesn't clean up empty channels - OK for testing. - # For later versions, have cleanup w/lock. - return channel, queues[channel].popleft() - except (IndexError, KeyError): - pass - # If all empty, sleep for a little bit - time.sleep(0.01) - +import time +import json +from collections import deque +from .base import BaseChannelBackend + +queues = {} + +class InMemoryChannelBackend(BaseChannelBackend): + """ + In-memory channel implementation. Intended only for use with threading, + in low-throughput development environments. + """ + + def send(self, channel, message): + # Try JSON encoding it to make sure it would, but store the native version + json.dumps(message) + # Add to the deque, making it if needs be + queues.setdefault(channel, deque()).append(message) + + def receive_many(self, channels): + while True: + # Try to pop a message from each channel + for channel in channels: + try: + # This doesn't clean up empty channels - OK for testing. + # For later versions, have cleanup w/lock. + return channel, queues[channel].popleft() + except (IndexError, KeyError): + pass + # If all empty, sleep for a little bit + time.sleep(0.01) + diff --git a/channels/backends/orm.py b/channels/backends/orm.py index 225a9f7..fdf6645 100644 --- a/channels/backends/orm.py +++ b/channels/backends/orm.py @@ -1,68 +1,68 @@ -import time -import datetime - -from django.apps.registry import Apps -from django.db import models, connections, DEFAULT_DB_ALIAS - -from .base import BaseChannelBackend - -queues = {} - -class ORMChannelBackend(BaseChannelBackend): - """ - ORM-backed channel environment. For development use only; it will span - multiple processes fine, but it's going to be pretty bad at throughput. - """ - - def __init__(self, expiry, db_alias=DEFAULT_DB_ALIAS): - super(ORMChannelBackend, self).__init__(expiry) - self.connection = connections[db_alias] - self.model = self.make_model() - self.ensure_schema() - - def make_model(self): - """ - Initialises a new model to store messages; not done as part of a - models.py as we don't want to make it for most installs. - """ - class Message(models.Model): - # We assume an autoincrementing PK for message order - channel = models.CharField(max_length=200, db_index=True) - content = models.TextField() - expiry = models.DateTimeField(db_index=True) - class Meta: - apps = Apps() - app_label = "channels" - db_table = "django_channels" - return Message - - def ensure_schema(self): - """ - Ensures the table exists and has the correct schema. - """ - # If the table's there, that's fine - we've never changed its schema - # in the codebase. - if self.model._meta.db_table in self.connection.introspection.table_names(self.connection.cursor()): - return - # Make the table - with self.connection.schema_editor() as editor: - editor.create_model(self.model) - - def send(self, channel, message): - self.model.objects.create( - channel = channel, - message = json.dumps(message), - expiry = datetime.datetime.utcnow() + datetime.timedelta(seconds=self.expiry) - ) - - def receive_many(self, channels): - while True: - # Delete all expired messages (add 10 second grace period for clock sync) - self.model.objects.filter(expiry__lt=datetime.datetime.utcnow() - datetime.timedelta(seconds=10)).delete() - # Get a message from one of our channels - message = self.model.objects.filter(channel__in=channels).order_by("id").first() - if message: - return message.channel, json.loads(message.content) - # If all empty, sleep for a little bit - time.sleep(0.2) - +import time +import datetime + +from django.apps.registry import Apps +from django.db import models, connections, DEFAULT_DB_ALIAS + +from .base import BaseChannelBackend + +queues = {} + +class ORMChannelBackend(BaseChannelBackend): + """ + ORM-backed channel environment. For development use only; it will span + multiple processes fine, but it's going to be pretty bad at throughput. + """ + + def __init__(self, expiry, db_alias=DEFAULT_DB_ALIAS): + super(ORMChannelBackend, self).__init__(expiry) + self.connection = connections[db_alias] + self.model = self.make_model() + self.ensure_schema() + + def make_model(self): + """ + Initialises a new model to store messages; not done as part of a + models.py as we don't want to make it for most installs. + """ + class Message(models.Model): + # We assume an autoincrementing PK for message order + channel = models.CharField(max_length=200, db_index=True) + content = models.TextField() + expiry = models.DateTimeField(db_index=True) + class Meta: + apps = Apps() + app_label = "channels" + db_table = "django_channels" + return Message + + def ensure_schema(self): + """ + Ensures the table exists and has the correct schema. + """ + # If the table's there, that's fine - we've never changed its schema + # in the codebase. + if self.model._meta.db_table in self.connection.introspection.table_names(self.connection.cursor()): + return + # Make the table + with self.connection.schema_editor() as editor: + editor.create_model(self.model) + + def send(self, channel, message): + self.model.objects.create( + channel = channel, + message = json.dumps(message), + expiry = datetime.datetime.utcnow() + datetime.timedelta(seconds=self.expiry) + ) + + def receive_many(self, channels): + while True: + # Delete all expired messages (add 10 second grace period for clock sync) + self.model.objects.filter(expiry__lt=datetime.datetime.utcnow() - datetime.timedelta(seconds=10)).delete() + # Get a message from one of our channels + message = self.model.objects.filter(channel__in=channels).order_by("id").first() + if message: + return message.channel, json.loads(message.content) + # If all empty, sleep for a little bit + time.sleep(0.2) + diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index 1bfe4e0..5e80729 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -1,40 +1,40 @@ -import functools - -from django.utils import six - -from .utils import name_that_thing - - -class ConsumerRegistry(object): - """ - Manages the available consumers in the project and which channels they - listen to. - - Generally this is attached to a backend instance as ".registry" - """ - - def __init__(self): - self.consumers = {} - - def add_consumer(self, consumer, channels): - # Upconvert if you just pass in a string - if isinstance(channels, six.string_types): - channels = [channels] - # Register on each channel, checking it's unique - for channel in channels: - if channel in self.consumers: - raise ValueError("Cannot register consumer %s - channel %r already consumed by %s" % ( - name_that_thing(consumer), - channel, - name_that_thing(self.consumers[channel]), - )) - self.consumers[channel] = consumer - - def all_channel_names(self): - return self.consumers.keys() - - def consumer_for_channel(self, channel): - try: - return self.consumers[channel] - except KeyError: - return None +import functools + +from django.utils import six + +from .utils import name_that_thing + + +class ConsumerRegistry(object): + """ + Manages the available consumers in the project and which channels they + listen to. + + Generally this is attached to a backend instance as ".registry" + """ + + def __init__(self): + self.consumers = {} + + def add_consumer(self, consumer, channels): + # Upconvert if you just pass in a string + if isinstance(channels, six.string_types): + channels = [channels] + # Register on each channel, checking it's unique + for channel in channels: + if channel in self.consumers: + raise ValueError("Cannot register consumer %s - channel %r already consumed by %s" % ( + name_that_thing(consumer), + channel, + name_that_thing(self.consumers[channel]), + )) + self.consumers[channel] = consumer + + def all_channel_names(self): + return self.consumers.keys() + + def consumer_for_channel(self, channel): + try: + return self.consumers[channel] + except KeyError: + return None diff --git a/channels/hacks.py b/channels/hacks.py index 5d5e329..e3003cf 100644 --- a/channels/hacks.py +++ b/channels/hacks.py @@ -1,27 +1,27 @@ -from django.http.request import HttpRequest -from django.http.response import HttpResponseBase -from django.core.handlers.base import BaseHandler -from .request import encode_request, decode_request -from .response import encode_response, decode_response, ResponseLater - - -def monkeypatch_django(): - """ - Monkeypatches support for us into parts of Django. - """ - # Request encode/decode - HttpRequest.channel_encode = encode_request - HttpRequest.channel_decode = staticmethod(decode_request) - # Response encode/decode - HttpResponseBase.channel_encode = encode_response - HttpResponseBase.channel_decode = staticmethod(decode_response) - HttpResponseBase.ResponseLater = ResponseLater - # Allow ResponseLater to propagate above handler - BaseHandler.old_handle_uncaught_exception = BaseHandler.handle_uncaught_exception - BaseHandler.handle_uncaught_exception = new_handle_uncaught_exception - - -def new_handle_uncaught_exception(self, request, resolver, exc_info): - if exc_info[0] is ResponseLater: - raise - return BaseHandler.old_handle_uncaught_exception(self, request, resolver, exc_info) +from django.http.request import HttpRequest +from django.http.response import HttpResponseBase +from django.core.handlers.base import BaseHandler +from .request import encode_request, decode_request +from .response import encode_response, decode_response, ResponseLater + + +def monkeypatch_django(): + """ + Monkeypatches support for us into parts of Django. + """ + # Request encode/decode + HttpRequest.channel_encode = encode_request + HttpRequest.channel_decode = staticmethod(decode_request) + # Response encode/decode + HttpResponseBase.channel_encode = encode_response + HttpResponseBase.channel_decode = staticmethod(decode_response) + HttpResponseBase.ResponseLater = ResponseLater + # Allow ResponseLater to propagate above handler + BaseHandler.old_handle_uncaught_exception = BaseHandler.handle_uncaught_exception + BaseHandler.handle_uncaught_exception = new_handle_uncaught_exception + + +def new_handle_uncaught_exception(self, request, resolver, exc_info): + if exc_info[0] is ResponseLater: + raise + return BaseHandler.old_handle_uncaught_exception(self, request, resolver, exc_info) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 7858359..ade88a2 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -1,60 +1,60 @@ -import django -import threading -from django.core.management.commands.runserver import Command as RunserverCommand -from django.core.handlers.wsgi import WSGIHandler -from django.http import HttpResponse -from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND -from channels.worker import Worker -from channels.utils import auto_import_consumers -from channels.adapters import UrlConsumer - - -class Command(RunserverCommand): - - def get_handler(self, *args, **options): - """ - Returns the default WSGI handler for the runner. - """ - django.setup() - return WSGIInterfaceHandler() - - def run(self, *args, **options): - # Force disable reloader for now - options['use_reloader'] = False - # Check a handler is registered for http reqs - channel_layer = channel_backends[DEFAULT_CHANNEL_BACKEND] - auto_import_consumers() - if not channel_layer.registry.consumer_for_channel("django.wsgi.request"): - # Register the default one - channel_layer.registry.add_consumer(UrlConsumer(), ["django.wsgi.request"]) - # Launch a worker thread - worker = WorkerThread(channel_layer) - worker.daemon = True - worker.start() - # Run the rest - return super(Command, self).run(*args, **options) - - -class WSGIInterfaceHandler(WSGIHandler): - """ - New WSGI handler that pushes requests to channels. - """ - - def get_response(self, request): - request.response_channel = Channel.new_name("django.wsgi.response") - Channel("django.wsgi.request").send(**request.channel_encode()) - channel, message = channel_backends[DEFAULT_CHANNEL_BACKEND].receive_many([request.response_channel]) - return HttpResponse.channel_decode(message) - - -class WorkerThread(threading.Thread): - """ - Class that runs a worker - """ - - def __init__(self, channel_layer): - super(WorkerThread, self).__init__() - self.channel_layer = channel_layer - - def run(self): - Worker(channel_layer=self.channel_layer).run() +import django +import threading +from django.core.management.commands.runserver import Command as RunserverCommand +from django.core.handlers.wsgi import WSGIHandler +from django.http import HttpResponse +from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND +from channels.worker import Worker +from channels.utils import auto_import_consumers +from channels.adapters import UrlConsumer + + +class Command(RunserverCommand): + + def get_handler(self, *args, **options): + """ + Returns the default WSGI handler for the runner. + """ + django.setup() + return WSGIInterfaceHandler() + + def run(self, *args, **options): + # Force disable reloader for now + options['use_reloader'] = False + # Check a handler is registered for http reqs + channel_layer = channel_backends[DEFAULT_CHANNEL_BACKEND] + auto_import_consumers() + if not channel_layer.registry.consumer_for_channel("django.wsgi.request"): + # Register the default one + channel_layer.registry.add_consumer(UrlConsumer(), ["django.wsgi.request"]) + # Launch a worker thread + worker = WorkerThread(channel_layer) + worker.daemon = True + worker.start() + # Run the rest + return super(Command, self).run(*args, **options) + + +class WSGIInterfaceHandler(WSGIHandler): + """ + New WSGI handler that pushes requests to channels. + """ + + def get_response(self, request): + request.response_channel = Channel.new_name("django.wsgi.response") + Channel("django.wsgi.request").send(**request.channel_encode()) + channel, message = channel_backends[DEFAULT_CHANNEL_BACKEND].receive_many([request.response_channel]) + return HttpResponse.channel_decode(message) + + +class WorkerThread(threading.Thread): + """ + Class that runs a worker + """ + + def __init__(self, channel_layer): + super(WorkerThread, self).__init__() + self.channel_layer = channel_layer + + def run(self): + Worker(channel_layer=self.channel_layer).run() diff --git a/channels/request.py b/channels/request.py index 7062d62..d7cd4b2 100644 --- a/channels/request.py +++ b/channels/request.py @@ -1,36 +1,36 @@ -from django.http import HttpRequest -from django.utils.datastructures import MultiValueDict - - -def encode_request(request): - """ - Encodes a request to JSON-compatible datastructures - """ - # TODO: More stuff - value = { - "GET": request.GET.items(), - "POST": request.POST.items(), - "COOKIES": request.COOKIES, - "META": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, - "path": request.path, - "path_info": request.path_info, - "method": request.method, - "response_channel": request.response_channel, - } - return value - - -def decode_request(value): - """ - Decodes a request JSONish value to a HttpRequest object. - """ - request = HttpRequest() - request.GET = MultiValueDict(value['GET']) - request.POST = MultiValueDict(value['POST']) - request.COOKIES = value['COOKIES'] - request.META = value['META'] - request.path = value['path'] - request.method = value['method'] - request.path_info = value['path_info'] - request.response_channel = value['response_channel'] - return request +from django.http import HttpRequest +from django.utils.datastructures import MultiValueDict + + +def encode_request(request): + """ + Encodes a request to JSON-compatible datastructures + """ + # TODO: More stuff + value = { + "GET": request.GET.items(), + "POST": request.POST.items(), + "COOKIES": request.COOKIES, + "META": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, + "path": request.path, + "path_info": request.path_info, + "method": request.method, + "response_channel": request.response_channel, + } + return value + + +def decode_request(value): + """ + Decodes a request JSONish value to a HttpRequest object. + """ + request = HttpRequest() + request.GET = MultiValueDict(value['GET']) + request.POST = MultiValueDict(value['POST']) + request.COOKIES = value['COOKIES'] + request.META = value['META'] + request.path = value['path'] + request.method = value['method'] + request.path_info = value['path_info'] + request.response_channel = value['response_channel'] + return request diff --git a/channels/response.py b/channels/response.py index c71d333..e790250 100644 --- a/channels/response.py +++ b/channels/response.py @@ -1,38 +1,38 @@ -from django.http import HttpResponse - - -def encode_response(response): - """ - Encodes a response to JSON-compatible datastructures - """ - # TODO: Entirely useful things like cookies - value = { - "content_type": getattr(response, "content_type", None), - "content": response.content, - "status_code": response.status_code, - "headers": response._headers.values(), - } - response.close() - return value - - -def decode_response(value): - """ - Decodes a response JSONish value to a HttpResponse object. - """ - response = HttpResponse( - content = value['content'], - content_type = value['content_type'], - status = value['status_code'], - ) - response._headers = {k.lower: (k, v) for k, v in value['headers']} - return response - - -class ResponseLater(Exception): - """ - Class that represents a response which will be sent doown the response - channel later. Used to move a django view-based segment onto the next - task, as otherwise we'd need to write some kind of fake response. - """ - pass +from django.http import HttpResponse + + +def encode_response(response): + """ + Encodes a response to JSON-compatible datastructures + """ + # TODO: Entirely useful things like cookies + value = { + "content_type": getattr(response, "content_type", None), + "content": response.content, + "status_code": response.status_code, + "headers": response._headers.values(), + } + response.close() + return value + + +def decode_response(value): + """ + Decodes a response JSONish value to a HttpResponse object. + """ + response = HttpResponse( + content = value['content'], + content_type = value['content_type'], + status = value['status_code'], + ) + response._headers = {k.lower: (k, v) for k, v in value['headers']} + return response + + +class ResponseLater(Exception): + """ + Class that represents a response which will be sent doown the response + channel later. Used to move a django view-based segment onto the next + task, as otherwise we'd need to write some kind of fake response. + """ + pass diff --git a/channels/utils.py b/channels/utils.py index b110e7a..a44fe9c 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -1,29 +1,29 @@ -import types -from django.apps import apps - - -def auto_import_consumers(): - """ - Auto-import consumers modules in apps - """ - for app_config in apps.get_app_configs(): - for submodule in ["consumers", "views"]: - module_name = "%s.%s" % (app_config.name, submodule) - try: - __import__(module_name) - except ImportError as e: - if "no module named %s" % submodule not in str(e).lower(): - raise - - -def name_that_thing(thing): - """ - Returns either the function/class path or just the object's repr - """ - if hasattr(thing, "__name__"): - if hasattr(thing, "__class__") and not isinstance(thing, types.FunctionType): - if thing.__class__ is not type: - return name_that_thing(thing.__class__) - if hasattr(thing, "__module__"): - return "%s.%s" % (thing.__module__, thing.__name__) - return repr(thing) +import types +from django.apps import apps + + +def auto_import_consumers(): + """ + Auto-import consumers modules in apps + """ + for app_config in apps.get_app_configs(): + for submodule in ["consumers", "views"]: + module_name = "%s.%s" % (app_config.name, submodule) + try: + __import__(module_name) + except ImportError as e: + if "no module named %s" % submodule not in str(e).lower(): + raise + + +def name_that_thing(thing): + """ + Returns either the function/class path or just the object's repr + """ + if hasattr(thing, "__name__"): + if hasattr(thing, "__class__") and not isinstance(thing, types.FunctionType): + if thing.__class__ is not type: + return name_that_thing(thing.__class__) + if hasattr(thing, "__module__"): + return "%s.%s" % (thing.__module__, thing.__name__) + return repr(thing) diff --git a/channels/worker.py b/channels/worker.py index 47fa21a..bb0c669 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -1,19 +1,19 @@ -class Worker(object): - """ - A "worker" process that continually looks for available messages to run - and runs their consumers. - """ - - def __init__(self, channel_layer): - self.channel_layer = channel_layer - - def run(self): - """ - Tries to continually dispatch messages to consumers. - """ - - channels = self.channel_layer.registry.all_channel_names() - while True: - channel, message = self.channel_layer.receive_many(channels) - consumer = self.channel_layer.registry.consumer_for_channel(channel) - consumer(channel=channel, **message) +class Worker(object): + """ + A "worker" process that continually looks for available messages to run + and runs their consumers. + """ + + def __init__(self, channel_layer): + self.channel_layer = channel_layer + + def run(self): + """ + Tries to continually dispatch messages to consumers. + """ + + channels = self.channel_layer.registry.all_channel_names() + while True: + channel, message = self.channel_layer.receive_many(channels) + consumer = self.channel_layer.registry.consumer_for_channel(channel) + consumer(channel=channel, **message) From 80627d8e372d90d466455094eec876fe90e63ebf Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Jun 2015 11:17:32 -0700 Subject: [PATCH 008/746] Working database backend and "runworker" command --- channels/__init__.py | 2 +- channels/adapters.py | 4 +- channels/backends/__init__.py | 26 ++++++----- channels/backends/base.py | 7 +++ channels/backends/{orm.py => database.py} | 53 +++++++++++++---------- channels/backends/memory.py | 4 ++ channels/channel.py | 8 ++-- channels/management/commands/runserver.py | 15 ++++--- channels/management/commands/runworker.py | 41 ++++++++++++++++++ channels/worker.py | 13 +++--- 10 files changed, 121 insertions(+), 52 deletions(-) rename channels/backends/{orm.py => database.py} (54%) create mode 100644 channels/management/commands/runworker.py diff --git a/channels/__init__.py b/channels/__init__.py index 1010c41..c0b7065 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -# Load backends +# Load backends, using settings if available (else falling back to a default) DEFAULT_CHANNEL_BACKEND = "default" from .backends import BackendManager from django.conf import settings diff --git a/channels/adapters.py b/channels/adapters.py index 0879241..cf365cd 100644 --- a/channels/adapters.py +++ b/channels/adapters.py @@ -47,7 +47,7 @@ def view_consumer(channel_name, alias=DEFAULT_CHANNEL_BACKEND): response = func(request) Channel(request.response_channel).send(**response.channel_encode()) # Get the channel layer and register - channel_layer = channel_backends[DEFAULT_CHANNEL_BACKEND] - channel_layer.registry.add_consumer(consumer, [channel_name]) + channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] + channel_backend.registry.add_consumer(consumer, [channel_name]) return func return inner diff --git a/channels/backends/__init__.py b/channels/backends/__init__.py index 83c74f4..5582b73 100644 --- a/channels/backends/__init__.py +++ b/channels/backends/__init__.py @@ -11,17 +11,23 @@ class BackendManager(object): """ def __init__(self, backend_configs): + self.configs = backend_configs self.backends = {} - for name, config in backend_configs.items(): - # Load the backend class - try: - backend_class = import_string(config['BACKEND']) - except KeyError: - raise InvalidChannelBackendError("No BACKEND specified for %s" % name) - except ImportError: - raise InvalidChannelBackendError("Cannot import BACKEND %s specified for %s" % (config['BACKEND'], name)) - # Initialise and pass config - self.backends[name] = backend_class(**{k.lower(): v for k, v in config.items() if k != "BACKEND"}) + + def make_backend(self, name): + # Load the backend class + try: + backend_class = import_string(self.configs[name]['BACKEND']) + except KeyError: + raise InvalidChannelBackendError("No BACKEND specified for %s" % name) + except ImportError as e: + raise InvalidChannelBackendError("Cannot import BACKEND %r specified for %s" % (self.configs[name]['BACKEND'], name)) + # Initialise and pass config + instance = backend_class(**{k.lower(): v for k, v in self.configs[name].items() if k != "BACKEND"}) + instance.alias = name + return instance def __getitem__(self, key): + if key not in self.backends: + self.backends[key] = self.make_backend(key) return self.backends[key] diff --git a/channels/backends/base.py b/channels/backends/base.py index a3481cb..c2d8f7c 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -15,6 +15,10 @@ class BaseChannelBackend(object): registry of consumers. """ + # Flags if this backend can only be used inside one process. + # Causes errors if you try to run workers/interfaces separately with it. + local_only = False + def __init__(self, expiry=60): self.registry = ConsumerRegistry() self.expiry = expiry @@ -31,3 +35,6 @@ class BaseChannelBackend(object): channels passed, as a (channel, message) tuple. """ raise NotImplementedError() + + def __str__(self): + return self.__class__.__name__ diff --git a/channels/backends/orm.py b/channels/backends/database.py similarity index 54% rename from channels/backends/orm.py rename to channels/backends/database.py index fdf6645..eb48a19 100644 --- a/channels/backends/orm.py +++ b/channels/backends/database.py @@ -1,30 +1,40 @@ import time +import json import datetime from django.apps.registry import Apps from django.db import models, connections, DEFAULT_DB_ALIAS +from django.utils.functional import cached_property +from django.utils.timezone import now from .base import BaseChannelBackend queues = {} -class ORMChannelBackend(BaseChannelBackend): +class DatabaseChannelBackend(BaseChannelBackend): """ ORM-backed channel environment. For development use only; it will span multiple processes fine, but it's going to be pretty bad at throughput. """ - def __init__(self, expiry, db_alias=DEFAULT_DB_ALIAS): - super(ORMChannelBackend, self).__init__(expiry) - self.connection = connections[db_alias] - self.model = self.make_model() - self.ensure_schema() + def __init__(self, expiry=60, db_alias=DEFAULT_DB_ALIAS): + super(DatabaseChannelBackend, self).__init__(expiry) + self.db_alias = db_alias - def make_model(self): + @property + def connection(self): + """ + Returns the correct connection for the current thread. + """ + return connections[self.db_alias] + + @property + def model(self): """ Initialises a new model to store messages; not done as part of a models.py as we don't want to make it for most installs. """ + # Make the model class class Message(models.Model): # We assume an autoincrementing PK for message order channel = models.CharField(max_length=200, db_index=True) @@ -34,35 +44,32 @@ class ORMChannelBackend(BaseChannelBackend): apps = Apps() app_label = "channels" db_table = "django_channels" + # Ensure its table exists + if Message._meta.db_table not in self.connection.introspection.table_names(self.connection.cursor()): + with self.connection.schema_editor() as editor: + editor.create_model(Message) return Message - def ensure_schema(self): - """ - Ensures the table exists and has the correct schema. - """ - # If the table's there, that's fine - we've never changed its schema - # in the codebase. - if self.model._meta.db_table in self.connection.introspection.table_names(self.connection.cursor()): - return - # Make the table - with self.connection.schema_editor() as editor: - editor.create_model(self.model) - def send(self, channel, message): self.model.objects.create( channel = channel, - message = json.dumps(message), - expiry = datetime.datetime.utcnow() + datetime.timedelta(seconds=self.expiry) + content = json.dumps(message), + expiry = now() + datetime.timedelta(seconds=self.expiry) ) def receive_many(self, channels): + if not channels: + raise ValueError("Cannot receive on empty channel list!") while True: # Delete all expired messages (add 10 second grace period for clock sync) - self.model.objects.filter(expiry__lt=datetime.datetime.utcnow() - datetime.timedelta(seconds=10)).delete() + self.model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() # Get a message from one of our channels message = self.model.objects.filter(channel__in=channels).order_by("id").first() if message: + self.model.objects.filter(pk=message.pk).delete() return message.channel, json.loads(message.content) # If all empty, sleep for a little bit - time.sleep(0.2) + time.sleep(0.1) + def __str__(self): + return "%s(alias=%s)" % (self.__class__.__name__, self.connection.alias) diff --git a/channels/backends/memory.py b/channels/backends/memory.py index 123ec40..3b7baca 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -11,6 +11,8 @@ class InMemoryChannelBackend(BaseChannelBackend): in low-throughput development environments. """ + local_only = True + def send(self, channel, message): # Try JSON encoding it to make sure it would, but store the native version json.dumps(message) @@ -18,6 +20,8 @@ class InMemoryChannelBackend(BaseChannelBackend): queues.setdefault(channel, deque()).append(message) def receive_many(self, channels): + if not channels: + raise ValueError("Cannot receive on empty channel list!") while True: # Try to pop a message from each channel for channel in channels: diff --git a/channels/channel.py b/channels/channel.py index b4e887d..c1b8330 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -23,13 +23,13 @@ class Channel(object): Create an instance for the channel named "name" """ self.name = name - self.channel_layer = channel_backends[alias] + self.channel_backend = channel_backends[alias] def send(self, **kwargs): """ Send a message over the channel, taken from the kwargs. """ - self.channel_layer.send(self.name, kwargs) + self.channel_backend.send(self.name, kwargs) @classmethod def new_name(self, prefix): @@ -59,9 +59,9 @@ class Channel(object): if isinstance(channels, six.string_types): channels = [channels] # Get the channel - channel_layer = channel_backends[alias] + channel_backend = channel_backends[alias] # Return a function that'll register whatever it wraps def inner(func): - channel_layer.registry.add_consumer(func, channels) + channel_backend.registry.add_consumer(func, channels) return func return inner diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index ade88a2..0179b65 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -1,6 +1,7 @@ import django import threading from django.core.management.commands.runserver import Command as RunserverCommand +from django.core.management import CommandError from django.core.handlers.wsgi import WSGIHandler from django.http import HttpResponse from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND @@ -22,13 +23,13 @@ class Command(RunserverCommand): # Force disable reloader for now options['use_reloader'] = False # Check a handler is registered for http reqs - channel_layer = channel_backends[DEFAULT_CHANNEL_BACKEND] + channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] auto_import_consumers() - if not channel_layer.registry.consumer_for_channel("django.wsgi.request"): + if not channel_backend.registry.consumer_for_channel("django.wsgi.request"): # Register the default one - channel_layer.registry.add_consumer(UrlConsumer(), ["django.wsgi.request"]) + channel_backend.registry.add_consumer(UrlConsumer(), ["django.wsgi.request"]) # Launch a worker thread - worker = WorkerThread(channel_layer) + worker = WorkerThread(channel_backend) worker.daemon = True worker.start() # Run the rest @@ -52,9 +53,9 @@ class WorkerThread(threading.Thread): Class that runs a worker """ - def __init__(self, channel_layer): + def __init__(self, channel_backend): super(WorkerThread, self).__init__() - self.channel_layer = channel_layer + self.channel_backend = channel_backend def run(self): - Worker(channel_layer=self.channel_layer).run() + Worker(channel_backend=self.channel_backend).run() diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py new file mode 100644 index 0000000..8d8b2ae --- /dev/null +++ b/channels/management/commands/runworker.py @@ -0,0 +1,41 @@ +import time +from wsgiref.simple_server import BaseHTTPRequestHandler +from django.core.management import BaseCommand, CommandError +from channels import channel_backends, DEFAULT_CHANNEL_BACKEND +from channels.worker import Worker +from channels.utils import auto_import_consumers + + +class Command(BaseCommand): + + def handle(self, *args, **options): + # Get the backend to use + channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] + auto_import_consumers() + if channel_backend.local_only: + raise CommandError( + "You have a process-local channel backend configured, and so cannot run separate workers.\n" + "Configure a network-based backend in CHANNEL_BACKENDS to use this command." + ) + # Launch a worker + self.stdout.write("Running worker against backend %s" % channel_backend) + # Optionally provide an output callback + callback = None + if options.get("verbosity", 1) > 1: + callback = self.consumer_called + # Run the worker + try: + Worker(channel_backend=channel_backend, callback=callback).run() + except KeyboardInterrupt: + pass + + def consumer_called(self, channel, message): + self.stdout.write("[%s] %s" % (self.log_date_time_string(), channel)) + + def log_date_time_string(self): + """Return the current time formatted for logging.""" + now = time.time() + year, month, day, hh, mm, ss, x, y, z = time.localtime(now) + s = "%02d/%3s/%04d %02d:%02d:%02d" % ( + day, BaseHTTPRequestHandler.monthname[month], year, hh, mm, ss) + return s diff --git a/channels/worker.py b/channels/worker.py index bb0c669..44522cd 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -4,16 +4,19 @@ class Worker(object): and runs their consumers. """ - def __init__(self, channel_layer): - self.channel_layer = channel_layer + def __init__(self, channel_backend, callback=None): + self.channel_backend = channel_backend + self.callback = callback def run(self): """ Tries to continually dispatch messages to consumers. """ - channels = self.channel_layer.registry.all_channel_names() + channels = self.channel_backend.registry.all_channel_names() while True: - channel, message = self.channel_layer.receive_many(channels) - consumer = self.channel_layer.registry.consumer_for_channel(channel) + channel, message = self.channel_backend.receive_many(channels) + consumer = self.channel_backend.registry.consumer_for_channel(channel) + if self.callback: + self.callback(channel, message) consumer(channel=channel, **message) From 433625da1ee6984ea1fe907d585098166e309c3e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Jun 2015 20:39:36 -0700 Subject: [PATCH 009/746] Start working on a WebSocket interface server --- channels/backends/base.py | 29 +++++- channels/backends/database.py | 19 ++-- channels/backends/memory.py | 20 ++-- channels/channel.py | 7 +- channels/interfaces/__init__.py | 0 channels/interfaces/websocket_twisted.py | 106 ++++++++++++++++++++ channels/interfaces/wsgi.py | 21 ++++ channels/management/commands/runserver.py | 28 ++---- channels/management/commands/runwsserver.py | 25 +++++ channels/worker.py | 7 +- 10 files changed, 214 insertions(+), 48 deletions(-) create mode 100644 channels/interfaces/__init__.py create mode 100644 channels/interfaces/websocket_twisted.py create mode 100644 channels/interfaces/wsgi.py create mode 100644 channels/management/commands/runwsserver.py diff --git a/channels/backends/base.py b/channels/backends/base.py index c2d8f7c..1cc27e2 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -1,3 +1,4 @@ +import time from channels.consumer_registry import ConsumerRegistry @@ -31,10 +32,34 @@ class BaseChannelBackend(object): def receive_many(self, channels): """ - Block and return the first message available on one of the - channels passed, as a (channel, message) tuple. + Return the first message available on one of the + channels passed, as a (channel, message) tuple, or return (None, None) + if no channels are available. + + Should not block, but is allowed to be moderately slow/have a short + timeout - it needs to return so we can refresh the list of channels, + not because the rest of the process is waiting on it. + + Better performance can be achieved for interface servers by directly + integrating the server and the backend code; this is merely for a + generic support-everything pattern. """ raise NotImplementedError() + def receive_many_blocking(self, channels): + """ + Blocking version of receive_many, if the calling context knows it + doesn't ever want to change the channels list until something happens. + + This base class provides a default implementation; can be overridden + to be more efficient by subclasses. + """ + while True: + channel, message = self.receive_many(channels) + if channel is None: + time.sleep(0.05) + continue + return channel, message + def __str__(self): return self.__class__.__name__ diff --git a/channels/backends/database.py b/channels/backends/database.py index eb48a19..1553216 100644 --- a/channels/backends/database.py +++ b/channels/backends/database.py @@ -60,16 +60,15 @@ class DatabaseChannelBackend(BaseChannelBackend): def receive_many(self, channels): if not channels: raise ValueError("Cannot receive on empty channel list!") - while True: - # Delete all expired messages (add 10 second grace period for clock sync) - self.model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() - # Get a message from one of our channels - message = self.model.objects.filter(channel__in=channels).order_by("id").first() - if message: - self.model.objects.filter(pk=message.pk).delete() - return message.channel, json.loads(message.content) - # If all empty, sleep for a little bit - time.sleep(0.1) + # Delete all expired messages (add 10 second grace period for clock sync) + self.model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() + # Get a message from one of our channels + message = self.model.objects.filter(channel__in=channels).order_by("id").first() + if message: + self.model.objects.filter(pk=message.pk).delete() + return message.channel, json.loads(message.content) + else: + return None, None def __str__(self): return "%s(alias=%s)" % (self.__class__.__name__, self.connection.alias) diff --git a/channels/backends/memory.py b/channels/backends/memory.py index 3b7baca..1d53d45 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -22,15 +22,13 @@ class InMemoryChannelBackend(BaseChannelBackend): def receive_many(self, channels): if not channels: raise ValueError("Cannot receive on empty channel list!") - while True: - # Try to pop a message from each channel - for channel in channels: - try: - # This doesn't clean up empty channels - OK for testing. - # For later versions, have cleanup w/lock. - return channel, queues[channel].popleft() - except (IndexError, KeyError): - pass - # If all empty, sleep for a little bit - time.sleep(0.01) + # Try to pop a message from each channel + for channel in channels: + try: + # This doesn't clean up empty channels - OK for testing. + # For later versions, have cleanup w/lock. + return channel, queues[channel].popleft() + except (IndexError, KeyError): + pass + return None, None diff --git a/channels/channel.py b/channels/channel.py index c1b8330..9572525 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -18,12 +18,15 @@ class Channel(object): "default" one by default. """ - def __init__(self, name, alias=DEFAULT_CHANNEL_BACKEND): + def __init__(self, name, alias=DEFAULT_CHANNEL_BACKEND, channel_backend=None): """ Create an instance for the channel named "name" """ self.name = name - self.channel_backend = channel_backends[alias] + if channel_backend: + self.channel_backend = channel_backend + else: + self.channel_backend = channel_backends[alias] def send(self, **kwargs): """ diff --git a/channels/interfaces/__init__.py b/channels/interfaces/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py new file mode 100644 index 0000000..6650c82 --- /dev/null +++ b/channels/interfaces/websocket_twisted.py @@ -0,0 +1,106 @@ +import django +import time +from collections import deque +from twisted.internet import reactor +from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND +from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory + + +class InterfaceProtocol(WebSocketServerProtocol): + """ + Protocol which supports WebSockets and forwards incoming messages to + the django.websocket channels. + """ + + def onConnect(self, request): + self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] + self.request = request + + def onOpen(self): + # Make sending channel + self.send_channel = Channel.new_name("django.websocket.send") + self.factory.protocols[self.send_channel] = self + # Send news that this channel is open + Channel("django.websocket.connect").send( + send_channel = self.send_channel, + ) + + def onMessage(self, payload, isBinary): + if isBinary: + Channel("django.websocket.receive").send( + send_channel = self.send_channel, + content = payload, + binary = True, + ) + else: + Channel("django.websocket.receive").send( + send_channel = self.send_channel, + content = payload.decode("utf8"), + binary = False, + ) + + def onChannelSend(self, content, binary=False, **kwargs): + self.sendMessage(content, binary) + + def onClose(self, wasClean, code, reason): + del self.factory.protocols[self.send_channel] + Channel("django.websocket.disconnect").send( + send_channel = self.send_channel, + ) + + +class InterfaceFactory(WebSocketServerFactory): + """ + Factory which keeps track of its open protocols' receive channels + and can dispatch to them. + """ + + # TODO: Clean up dead protocols if needed? + + def __init__(self, *args, **kwargs): + super(InterfaceFactory, self).__init__(*args, **kwargs) + self.protocols = {} + + def send_channels(self): + return self.protocols.keys() + + def dispatch_send(self, channel, message): + self.protocols[channel].onChannelSend(**message) + + +class WebsocketTwistedInterface(object): + """ + Easy API to run a WebSocket interface server using Twisted. + Integrates the channel backend by running it in a separate thread, as we don't + know if the backend is Twisted-compliant. + """ + + def __init__(self, channel_backend, port=9000): + self.channel_backend = channel_backend + self.port = port + + def run(self): + self.factory = InterfaceFactory("ws://localhost:%i" % self.port, debug=False) + self.factory.protocol = InterfaceProtocol + reactor.listenTCP(self.port, self.factory) + reactor.callInThread(self.backend_reader) + reactor.run() + + def backend_reader(self): + """ + Run in a separate thread; reads messages from the backend. + """ + while True: + channels = self.factory.send_channels() + # Don't do anything if there's no channels to listen on + if channels: + channel, message = self.channel_backend.receive_many(channels) + else: + time.sleep(0.1) + continue + # Wait around if there's nothing received + if channel is None: + time.sleep(0.05) + continue + # Deal with the message + self.factory.dispatch_send(channel, message) diff --git a/channels/interfaces/wsgi.py b/channels/interfaces/wsgi.py new file mode 100644 index 0000000..60b19c2 --- /dev/null +++ b/channels/interfaces/wsgi.py @@ -0,0 +1,21 @@ +import django +from django.core.handlers.wsgi import WSGIHandler +from django.http import HttpResponse +from channels import Channel + + +class WSGIInterface(WSGIHandler): + """ + WSGI application that pushes requests to channels. + """ + + def __init__(self, channel_backend, *args, **kwargs): + self.channel_backend = channel_backend + django.setup() + super(WSGIInterface, self).__init__(*args, **kwargs) + + def get_response(self, request): + request.response_channel = Channel.new_name("django.wsgi.response") + Channel("django.wsgi.request", channel_backend=self.channel_backend).send(**request.channel_encode()) + channel, message = self.channel_backend.receive_many_blocking([request.response_channel]) + return HttpResponse.channel_decode(message) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 0179b65..7a795bc 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -2,12 +2,11 @@ import django import threading from django.core.management.commands.runserver import Command as RunserverCommand from django.core.management import CommandError -from django.core.handlers.wsgi import WSGIHandler -from django.http import HttpResponse -from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND +from channels import channel_backends, DEFAULT_CHANNEL_BACKEND from channels.worker import Worker from channels.utils import auto_import_consumers from channels.adapters import UrlConsumer +from channels.interfaces.wsgi import WSGIInterface class Command(RunserverCommand): @@ -16,38 +15,25 @@ class Command(RunserverCommand): """ Returns the default WSGI handler for the runner. """ - django.setup() - return WSGIInterfaceHandler() + return WSGIInterface(self.channel_backend) def run(self, *args, **options): # Force disable reloader for now options['use_reloader'] = False # Check a handler is registered for http reqs - channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] + self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] auto_import_consumers() - if not channel_backend.registry.consumer_for_channel("django.wsgi.request"): + if not self.channel_backend.registry.consumer_for_channel("django.wsgi.request"): # Register the default one - channel_backend.registry.add_consumer(UrlConsumer(), ["django.wsgi.request"]) + self.channel_backend.registry.add_consumer(UrlConsumer(), ["django.wsgi.request"]) # Launch a worker thread - worker = WorkerThread(channel_backend) + worker = WorkerThread(self.channel_backend) worker.daemon = True worker.start() # Run the rest return super(Command, self).run(*args, **options) -class WSGIInterfaceHandler(WSGIHandler): - """ - New WSGI handler that pushes requests to channels. - """ - - def get_response(self, request): - request.response_channel = Channel.new_name("django.wsgi.response") - Channel("django.wsgi.request").send(**request.channel_encode()) - channel, message = channel_backends[DEFAULT_CHANNEL_BACKEND].receive_many([request.response_channel]) - return HttpResponse.channel_decode(message) - - class WorkerThread(threading.Thread): """ Class that runs a worker diff --git a/channels/management/commands/runwsserver.py b/channels/management/commands/runwsserver.py new file mode 100644 index 0000000..d5d072c --- /dev/null +++ b/channels/management/commands/runwsserver.py @@ -0,0 +1,25 @@ +import time +from django.core.management import BaseCommand, CommandError +from channels import channel_backends, DEFAULT_CHANNEL_BACKEND +from channels.interfaces.websocket_twisted import WebsocketTwistedInterface +from channels.utils import auto_import_consumers + + +class Command(BaseCommand): + + def handle(self, *args, **options): + # Get the backend to use + channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] + auto_import_consumers() + if channel_backend.local_only: + raise CommandError( + "You have a process-local channel backend configured, and so cannot run separate interface servers.\n" + "Configure a network-based backend in CHANNEL_BACKENDS to use this command." + ) + # Launch a worker + self.stdout.write("Running Twisted/Autobahn WebSocket interface against backend %s" % channel_backend) + # Run the interface + try: + WebsocketTwistedInterface(channel_backend=channel_backend).run() + except KeyboardInterrupt: + pass diff --git a/channels/worker.py b/channels/worker.py index 44522cd..f39327a 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -1,3 +1,6 @@ +import time + + class Worker(object): """ A "worker" process that continually looks for available messages to run @@ -12,10 +15,10 @@ class Worker(object): """ Tries to continually dispatch messages to consumers. """ - channels = self.channel_backend.registry.all_channel_names() while True: - channel, message = self.channel_backend.receive_many(channels) + channel, message = self.channel_backend.receive_many_blocking(channels) + # Handle the message consumer = self.channel_backend.registry.consumer_for_channel(channel) if self.callback: self.callback(channel, message) From 75ee13ff9c25d82166b74d5d37bb21124e9a0fb4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Jun 2015 23:02:53 -0700 Subject: [PATCH 010/746] Add redis backend --- channels/backends/database.py | 1 - channels/backends/redis_py.py | 56 +++++++++++++++++++++ channels/interfaces/websocket_twisted.py | 14 ++++-- channels/management/commands/runwsserver.py | 15 +++--- 4 files changed, 75 insertions(+), 11 deletions(-) create mode 100644 channels/backends/redis_py.py diff --git a/channels/backends/database.py b/channels/backends/database.py index 1553216..bf8e842 100644 --- a/channels/backends/database.py +++ b/channels/backends/database.py @@ -9,7 +9,6 @@ from django.utils.timezone import now from .base import BaseChannelBackend -queues = {} class DatabaseChannelBackend(BaseChannelBackend): """ diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py new file mode 100644 index 0000000..13ea81d --- /dev/null +++ b/channels/backends/redis_py.py @@ -0,0 +1,56 @@ +import time +import json +import datetime +import redis +import uuid + +from .base import BaseChannelBackend + + +class RedisChannelBackend(BaseChannelBackend): + """ + ORM-backed channel environment. For development use only; it will span + multiple processes fine, but it's going to be pretty bad at throughput. + """ + + def __init__(self, expiry=60, host="localhost", port=6379, prefix="django-channels:"): + super(RedisChannelBackend, self).__init__(expiry) + self.host = host + self.port = port + self.prefix = prefix + + @property + def connection(self): + """ + Returns the correct connection for the current thread. + """ + return redis.Redis(host=self.host, port=self.port) + + def send(self, channel, message): + key = uuid.uuid4() + self.connection.set( + key, + json.dumps(message), + ex = self.expiry + 10, + ) + self.connection.rpush( + self.prefix + channel, + key, + ) + + def receive_many(self, channels): + if not channels: + raise ValueError("Cannot receive on empty channel list!") + # Get a message from one of our channels + while True: + result = self.connection.blpop([self.prefix + channel for channel in channels], timeout=1) + if result: + content = self.connection.get(result[1]) + if content is None: + continue + return result[0][len(self.prefix):], json.loads(content) + else: + return None, None + + def __str__(self): + return "%s(host=%s, port=%s)" % (self.__class__.__name__, self.host, self.port) diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index 6650c82..a24f233 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -40,7 +40,10 @@ class InterfaceProtocol(WebSocketServerProtocol): ) def onChannelSend(self, content, binary=False, **kwargs): - self.sendMessage(content, binary) + if binary: + self.sendMessage(content, binary) + else: + self.sendMessage(content.encode("utf8"), binary) def onClose(self, wasClean, code, reason): del self.factory.protocols[self.send_channel] @@ -71,8 +74,8 @@ class InterfaceFactory(WebSocketServerFactory): class WebsocketTwistedInterface(object): """ Easy API to run a WebSocket interface server using Twisted. - Integrates the channel backend by running it in a separate thread, as we don't - know if the backend is Twisted-compliant. + Integrates the channel backend by running it in a separate thread, using + the always-compatible polling style. """ def __init__(self, channel_backend, port=9000): @@ -80,7 +83,7 @@ class WebsocketTwistedInterface(object): self.port = port def run(self): - self.factory = InterfaceFactory("ws://localhost:%i" % self.port, debug=False) + self.factory = InterfaceFactory("ws://0.0.0.0:%i" % self.port, debug=False) self.factory.protocol = InterfaceProtocol reactor.listenTCP(self.port, self.factory) reactor.callInThread(self.backend_reader) @@ -92,6 +95,9 @@ class WebsocketTwistedInterface(object): """ while True: channels = self.factory.send_channels() + # Quit if reactor is stopping + if not reactor.running: + return # Don't do anything if there's no channels to listen on if channels: channel, message = self.channel_backend.receive_many(channels) diff --git a/channels/management/commands/runwsserver.py b/channels/management/commands/runwsserver.py index d5d072c..9434be0 100644 --- a/channels/management/commands/runwsserver.py +++ b/channels/management/commands/runwsserver.py @@ -7,6 +7,10 @@ from channels.utils import auto_import_consumers class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('port', nargs='?', + help='Optional port number') + def handle(self, *args, **options): # Get the backend to use channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] @@ -16,10 +20,9 @@ class Command(BaseCommand): "You have a process-local channel backend configured, and so cannot run separate interface servers.\n" "Configure a network-based backend in CHANNEL_BACKENDS to use this command." ) - # Launch a worker - self.stdout.write("Running Twisted/Autobahn WebSocket interface against backend %s" % channel_backend) # Run the interface - try: - WebsocketTwistedInterface(channel_backend=channel_backend).run() - except KeyboardInterrupt: - pass + port = options.get("port", None) or 9000 + self.stdout.write("Running Twisted/Autobahn WebSocket interface server") + self.stdout.write(" Channel backend: %s" % channel_backend) + self.stdout.write(" Listening on: ws://0.0.0.0:%i" % port) + WebsocketTwistedInterface(channel_backend=channel_backend, port=port).run() From d9681df69a767e3ee08c338a51e5826b12c2350d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 15 Jun 2015 00:13:32 -0700 Subject: [PATCH 011/746] Few more WS tweaks --- README | 0 README.rst | 85 ++++++++++++++++++++++++ channels/__init__.py | 2 + channels/backends/redis_py.py | 8 +++ channels/docs/message-standards.rst | 25 ++++++- channels/interfaces/websocket_twisted.py | 24 ++++++- setup.py | 5 +- 7 files changed, 142 insertions(+), 7 deletions(-) delete mode 100644 README create mode 100644 README.rst diff --git a/README b/README deleted file mode 100644 index e69de29..0000000 diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..30421e4 --- /dev/null +++ b/README.rst @@ -0,0 +1,85 @@ +django-channels +=============== + +This is a work-in-progress code branch of Django implemented as a third-party +app, which aims to bring some asynchrony to Django and expand the options +for code beyond the request-response model. + +The proposal itself is detailed in `a very long Gist `_ +and there is discussion about it on `django-developers `_. + +If you wish to use this in your own project, there's basic integration +instructions below - but be warned! This is not stable and may change massively +at any time! + +Integration +----------- + +Make sure you're running Django 1.8. This doesn't work with 1.7 (yet?) + +If you want to use WebSockets (and that's kind of the point) you'll need +``autobahn`` and ``twisted`` packages too. Python 3/asyncio support coming soon. + +``pip install django-channels`` and then add ``channels`` to the **TOP** +of your ``INSTALLED_APPS`` list (if it is not at the top you won't get the +new runserver command). + +You now have a ``runserver`` that actually runs a WSGI interface and a +worker in two different threads, ``runworker`` to run separate workers, +and ``runwsserver`` to run a Twisted-based WebSocket server. + +You should place consumers in either your ``views.py`` or a ``consumers.py``. +Here's an example of WebSocket consumers for basic chat:: + + import redis + from channels import Channel + + redis_conn = redis.Redis("localhost", 6379) + + @Channel.consumer("django.websockets.connect") + def ws_connect(path, send_channel, **kwargs): + redis_conn.sadd("chatroom", send_channel) + + @Channel.consumer("django.websocket.receive") + def ws_receive(channel, send_channel, content, binary, **kwargs): + # Ignore binary messages + if binary: + return + # Re-dispatch message + for channel in redis_conn.smembers("chatroom"): + Channel(channel).send(content=content, binary=False) + + @Channel.consumer("django.websocket.disconnect") + def ws_disconnect(channel, send_channel, **kwargs): + redis_conn.srem("chatroom", send_channel) + # NOTE: this does not clean up server crash disconnects, + # you'd want expiring keys here in real life. + +Alternately, you can just push some code outside of a normal view into a worker +thread:: + + + from django.shortcuts import render + from channels import Channel + + def my_view(request): + # Dispatch a task to run outside the req/response cycle + Channel("a_task_channel").send(value=3) + # Return a response + return render(request, "test.html") + + @Channel.consumer("a_task_channel") + def some_task(channel, value): + print "My value was %s from channel %s" % (value, channel) + +Limitations +----------- + +The ``runserver`` this command provides currently does not support static +media serving, streamed responses or autoreloading. + +In addition, this library is a preview and basically might do anything to your +code, or change drastically at any time. + +If you have opinions, please provide feedback via the appropriate +`django-developers thread `_. diff --git a/channels/__init__.py b/channels/__init__.py index c0b7065..883c43a 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,3 +1,5 @@ +__version__ = "0.1" + # Load backends, using settings if available (else falling back to a default) DEFAULT_CHANNEL_BACKEND = "default" from .backends import BackendManager diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 13ea81d..7c65fe9 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -27,16 +27,24 @@ class RedisChannelBackend(BaseChannelBackend): return redis.Redis(host=self.host, port=self.port) def send(self, channel, message): + # Write out message into expiring key (avoids big items in list) key = uuid.uuid4() self.connection.set( key, json.dumps(message), ex = self.expiry + 10, ) + # Add key to list self.connection.rpush( self.prefix + channel, key, ) + # Set list to expire when message does (any later messages will bump this) + self.connection.expire( + self.prefix + channel, + self.expiry + 10, + ) + # TODO: Prune expired messages from same list (in case nobody consumes) def receive_many(self, channels): if not channels: diff --git a/channels/docs/message-standards.rst b/channels/docs/message-standards.rst index d22d2df..c1c6528 100644 --- a/channels/docs/message-standards.rst +++ b/channels/docs/message-standards.rst @@ -73,12 +73,33 @@ Sent when a datagram is received on the WebSocket. Contains the same keys as WebSocket Connection, plus: * content: String content of the datagram +* binary: If the content is to be interpreted as text or binary -WebSocket Close ---------------- +WebSocket Client Close +---------------------- Sent when the WebSocket is closed by either the client or the server. Contains the same keys as WebSocket Connection, including send_channel, though nothing should be sent on it. + + +WebSocket Send +-------------- + +Sent by a Django consumer to send a message back over the WebSocket to +the client. + +Contains the keys: + +* content: String content of the datagram +* binary: If the content is to be interpreted as text or binary + + +WebSocket Server Close +---------------------- + +Sent by a Django consumer to close the client's WebSocket. + +Contains no keys. diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index a24f233..4cb7083 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -14,7 +14,9 @@ class InterfaceProtocol(WebSocketServerProtocol): def onConnect(self, request): self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - self.request = request + self.request_info = { + "path": request.path, + } def onOpen(self): # Make sending channel @@ -23,6 +25,7 @@ class InterfaceProtocol(WebSocketServerProtocol): # Send news that this channel is open Channel("django.websocket.connect").send( send_channel = self.send_channel, + **self.request_info ) def onMessage(self, payload, isBinary): @@ -31,24 +34,36 @@ class InterfaceProtocol(WebSocketServerProtocol): send_channel = self.send_channel, content = payload, binary = True, + **self.request_info ) else: Channel("django.websocket.receive").send( send_channel = self.send_channel, content = payload.decode("utf8"), binary = False, + **self.request_info ) - def onChannelSend(self, content, binary=False, **kwargs): + def serverSend(self, content, binary=False, **kwargs): + """ + Server-side channel message to send a message. + """ if binary: self.sendMessage(content, binary) else: self.sendMessage(content.encode("utf8"), binary) + def serverClose(self): + """ + Server-side channel message to close the socket + """ + self.sendClose() + def onClose(self, wasClean, code, reason): del self.factory.protocols[self.send_channel] Channel("django.websocket.disconnect").send( send_channel = self.send_channel, + **self.request_info ) @@ -68,7 +83,10 @@ class InterfaceFactory(WebSocketServerFactory): return self.protocols.keys() def dispatch_send(self, channel, message): - self.protocols[channel].onChannelSend(**message) + if message.get("close", False): + self.protocols[channel].serverClose() + else: + self.protocols[channel].serverSend(**message) class WebsocketTwistedInterface(object): diff --git a/setup.py b/setup.py index e8274ab..ae21988 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,12 @@ from setuptools import find_packages, setup setup( - name='django-channel', + name='django-channels', version="0.1", - url='http://github.com/andrewgodwin/django-channel', + url='http://github.com/andrewgodwin/django-channels', author='Andrew Godwin', author_email='andrew@aeracode.org', + description="Brings event-driven capabilities to Django with a channel system. Django 1.8 and up only.", license='BSD', packages=find_packages(), include_package_data=True, From 2096d6a9e63ee40e56ed5dc7cf8e753806fd106d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 15 Jun 2015 00:22:28 -0700 Subject: [PATCH 012/746] Release bits and bobs --- .gitignore | 1 + README.rst | 4 ++-- setup.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 11041c7..08407a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.egg-info +dist/ diff --git a/README.rst b/README.rst index 30421e4..32bc90b 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -django-channels +Django Channels =============== This is a work-in-progress code branch of Django implemented as a third-party @@ -20,7 +20,7 @@ Make sure you're running Django 1.8. This doesn't work with 1.7 (yet?) If you want to use WebSockets (and that's kind of the point) you'll need ``autobahn`` and ``twisted`` packages too. Python 3/asyncio support coming soon. -``pip install django-channels`` and then add ``channels`` to the **TOP** +``pip install channels`` and then add ``channels`` to the **TOP** of your ``INSTALLED_APPS`` list (if it is not at the top you won't get the new runserver command). diff --git a/setup.py b/setup.py index ae21988..04eb552 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages, setup setup( - name='django-channels', + name='channels', version="0.1", url='http://github.com/andrewgodwin/django-channels', author='Andrew Godwin', From e9b8632b6c44df078e55e5478d6ba9a92bfb11e4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 15 Jun 2015 00:44:52 -0700 Subject: [PATCH 013/746] Make redis backend more portable to older versions --- channels/backends/redis_py.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 7c65fe9..602e749 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -32,7 +32,10 @@ class RedisChannelBackend(BaseChannelBackend): self.connection.set( key, json.dumps(message), - ex = self.expiry + 10, + ) + self.connection.expire( + key, + self.expiry + 10, ) # Add key to list self.connection.rpush( From 884dad9277f6c127c695fd9e032c9b397c4b77e5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 15 Jun 2015 00:45:15 -0700 Subject: [PATCH 014/746] Version bump --- channels/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/__init__.py b/channels/__init__.py index 883c43a..2b9a471 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1" +__version__ = "0.1.1" # Load backends, using settings if available (else falling back to a default) DEFAULT_CHANNEL_BACKEND = "default" diff --git a/setup.py b/setup.py index 04eb552..01fe972 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup setup( name='channels', - version="0.1", + version="0.1.1", url='http://github.com/andrewgodwin/django-channels', author='Andrew Godwin', author_email='andrew@aeracode.org', From 423e3125ed980b6a7e23c8cd53dedcab4abad549 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 15 Jun 2015 00:54:01 -0700 Subject: [PATCH 015/746] Fix bad readme example --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 32bc90b..005785e 100644 --- a/README.rst +++ b/README.rst @@ -36,7 +36,7 @@ Here's an example of WebSocket consumers for basic chat:: redis_conn = redis.Redis("localhost", 6379) - @Channel.consumer("django.websockets.connect") + @Channel.consumer("django.websocket.connect") def ws_connect(path, send_channel, **kwargs): redis_conn.sadd("chatroom", send_channel) From 7a99995eb8300627001d5063b2e8b5c4a65f8930 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 19 Jun 2015 11:25:57 +0800 Subject: [PATCH 016/746] Fix crash when non-handshaked websocket closes --- channels/interfaces/websocket_twisted.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index 4cb7083..dc9aff9 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -60,11 +60,12 @@ class InterfaceProtocol(WebSocketServerProtocol): self.sendClose() def onClose(self, wasClean, code, reason): - del self.factory.protocols[self.send_channel] - Channel("django.websocket.disconnect").send( - send_channel = self.send_channel, - **self.request_info - ) + if hasattr(self, "send_channel"): + del self.factory.protocols[self.send_channel] + Channel("django.websocket.disconnect").send( + send_channel = self.send_channel, + **self.request_info + ) class InterfaceFactory(WebSocketServerFactory): From 217afe034815f39e6b4ac16acd3740a1de853ae1 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Jun 2015 10:27:22 +0800 Subject: [PATCH 017/746] Start sphinx-ified docs --- .gitignore | 1 + channels/backends/base.py | 30 ++ docs/Makefile | 177 ++++++++++++ docs/backend-requirements.rst | 46 ++++ docs/concepts.rst | 172 ++++++++++++ docs/conf.py | 258 ++++++++++++++++++ docs/getting-started.rst | 6 + docs/index.rst | 29 ++ docs/installation.rst | 21 ++ .../docs => docs}/integration-changes.rst | 0 docs/make.bat | 242 ++++++++++++++++ {channels/docs => docs}/message-standards.rst | 0 docs/scaling.rst | 30 ++ 13 files changed, 1012 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/backend-requirements.rst create mode 100644 docs/concepts.rst create mode 100644 docs/conf.py create mode 100644 docs/getting-started.rst create mode 100644 docs/index.rst create mode 100644 docs/installation.rst rename {channels/docs => docs}/integration-changes.rst (100%) create mode 100644 docs/make.bat rename {channels/docs => docs}/message-standards.rst (100%) create mode 100644 docs/scaling.rst diff --git a/.gitignore b/.gitignore index 08407a7..a49fd66 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.egg-info dist/ +docs/_build diff --git a/channels/backends/base.py b/channels/backends/base.py index 1cc27e2..e7dd139 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -61,5 +61,35 @@ class BaseChannelBackend(object): continue return channel, message + def group_add(self, group, channel, expiry=None): + """ + Adds the channel to the named group for at least 'expiry' + seconds (expiry defaults to message expiry if not provided). + """ + raise NotImplementedError() + + def group_discard(self, group, channel): + """ + Removes the channel from the named group if it is in the group; + does nothing otherwise (does not error) + """ + raise NotImplementedError() + + def group_channels(self, group): + """ + Returns an iterable of all channels in the group. + """ + raise NotImplementedError() + + def send_group(self, group, message): + """ + Sends a message to the entire group. + + This base class provides a default implementation; can be overridden + to be more efficient by subclasses. + """ + for channel in self.group_channels(): + self.send(channel, message) + def __str__(self): return self.__class__.__name__ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..21894a7 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Channels.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Channels.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Channels" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Channels" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/backend-requirements.rst b/docs/backend-requirements.rst new file mode 100644 index 0000000..f2dc028 --- /dev/null +++ b/docs/backend-requirements.rst @@ -0,0 +1,46 @@ +Channel Backend Requirements +============================ + +While the channel backends are pluggable, there's a minimum standard they +must meet in terms of the way they perform. + +In particular, a channel backend MUST: + +* Provide a ``send()`` method which sends a message onto a named channel + +* Provide a ``receive_many()`` method which returns an available message on the + provided channels, or returns no message either instantly or after a short + delay (it must not block indefinitely) + +* Provide a ``group_add()`` method which adds a channel to a named group + for at least the provided expiry period. + +* Provide a ``group_discard()`` method with removes a channel from a named + group if it was added, and nothing otherwise. + +* Provide a ``group_members()`` method which returns an iterable of all + channels currently in the group. + +* Preserve the ordering of messages inside a channel + +* Never deliver a message more than once + +* Never block on sending of a message (dropping the message/erroring is preferable to blocking) + +* Be able to store messages of at least 5MB in size + +* Expire messages only after the expiry period provided is up (a backend may + keep them longer if it wishes, but should expire them at some reasonable + point to ensure users do not come to rely on permanent messages) + +In addition, it SHOULD: + +* Provide a ``receive_many_blocking()`` method which is like ``receive_many()`` + but blocks until a message is available. + +* Provide a ``send_group()`` method which sends a message to every channel + in a group. + +* Try and preserve a rough global ordering, so that one busy channel does not + drown out an old message in another channel if a worker is listening on both. + diff --git a/docs/concepts.rst b/docs/concepts.rst new file mode 100644 index 0000000..3171b01 --- /dev/null +++ b/docs/concepts.rst @@ -0,0 +1,172 @@ +Concepts +======== + +Django's traditional view of the world revolves around requests and responses; +a request comes in, Django is fired up to serve it, generates a response to +send, and then Django goes away and waits for the next request. + +That was fine when the internet was all driven by simple browser interactions, +but the modern Web includes things like WebSockets and HTTP2 server push, +which allow websites to communicate outside of this traditional cycle. + +And, beyond that, there are plenty of non-critical tasks that applications +could easily offload until after a response as been sent - like saving things +into a cache, or thumbnailing newly-uploaded images. + +Channels changes the way Django runs to be "event oriented" - rather than +just responding to requests, instead Django responses to a wide array of events +sent on *channels*. There's still no persistent state - each event handler, +or *consumer* as we call them, is called independently in a way much like a +view is called. + +Let's look at what *channels* are first. + +What is a channel? +------------------ + +The core of Channels is, unsurprisingly, a datastructure called a *channel*. +What is a channel? It is an *ordered*, *first-in first-out queue* with +*at-most-once delivery* to *only one listener at a time*. + +You can think of it as analagous to a task queue - messages are put onto +the channel by *producers*, and then given to just one of the *consumers* +listening to that channnel. + +By *at-most-once* we say that either one consumer gets the message or nobody +does (if the channel implementation crashes, let's say). The +alternative is *at-least-once*, where normally one consumer gets the message +but when things crash it's sent to more than one, which is not the trade-off +we want. + +There are a couple of other limitations - messages must be JSON-serialisable, +and not be more than 1MB in size - but these are to make the whole thing +practical, and not too important to think about up front. + +The channels have capacity, so a load of producers can write lots of messages +into a channel with no consumers and then a consumer can come along later and +will start getting served those queued messages. + +If you've used channels in Go, these are reasonably similar to those. The key +difference is that these channels are network-transparent; the implementations +of channels we provide are all accessible across a network to consumers +and producers running in different processes or on different machines. + +Inside a network, we identify channels uniquely by a name string - you can +send to any named channel from any machine connected to the same channel +backend. If two different machines both write to the ``django.wsgi.request`` +channel, they're writing into the same channel. + +How do we use channels? +----------------------- + +That's what a channel is, but how is Django using them? Well, inside Django +you can connect a function to consume a channel, like so:: + + @Channel.consumer("channel-name") + def my_consumer(something, **kwargs): + pass + +This means that for every message on the channel, Django will call that +consumer function with the message as keyword arguments (messages are always +a dict, and are mapped to keyword arguments for send/receive). + +Django can do this as rather than run in a request-response mode, Channels +changes Django so that it runs in a worker mode - it listens on all channels +that have consumers declared, and when a message arrives on one, runs the +relevant consumer. + +In fact, this is illustrative of the new way Django runs to enable Channels to +work. Rather than running in just a single process tied to a WSGI server, +Django runs in three separate layers: + +* Interface servers, which communicate between Django and the outside world. + This includes a WSGI adapter as well as a separate WebSocket server - we'll + cover this later. + +* The channel backend, which is a combination of pluggable Python code and + a datastore (a database, or Redis) and responsible for transporting messages. + +* The workers, that listen on all relevant channels and run consumer code + when a message is ready. + +This may seem quite simplistic, but that's part of the design; rather than +try and have a full asynchronous architecture, we're just introducing a +slightly more complex abstraction than that presented by Django views. + +A view takes a request and returns a response; a consumer takes a channel +message and can write out zero to many other channel messages. + +Now, let's make a channel for requests (called ``django.wsgi.request``), +and a channel per client for responses (e.g. ``django.wsgi.respsonse.o4F2h2Fd``), +with the response channel a property (``send_channel``) of the request message. +Suddenly, a view is merely another example of a consumer:: + + @Channel.consumer("django.wsgi.request") + def my_consumer(send_channel, **request_data): + # Decode the request from JSON-compat to a full object + django_request = Request.decode(request_data) + # Run view + django_response = view(django_request) + # Encode the response into JSON-compat format + Channel(send_channel).send(django_response.encode()) + +In fact, this is how Channels works. The interface servers transform connections +from the outside world (HTTP, WebSockets, etc.) into messages on channels, +and then you write workers to handle these messages. + +This may seem like it's still not very well designed to handle push-style +code - where you use HTTP2's server-sent events or a WebSocket to notify +clients of changes in real time (messages in a chat, perhaps, or live updates +in an admin as another user edits something). + +However, the key here is that you can run code (and so send on channels) in +response to any event - and that includes ones you create. You can trigger +on model saves, on other incoming messages, or from code paths inside views +and forms. + +.. _channel-types: + +Channel Types +------------- + +Now, if you think about it, there are actually two major uses for channels in +this model. The first, and more obvious one, is the dispatching of work to +consumers - a message gets added to a channel, and then any one of the workers +can pick it up and run the consumer. + +The second kind of channel, however, is used for responses. Notably, these only +have one thing listening on them - the interface server. Each response channel +is individually named and has to be routed back to the interface server where +its client is terminated. + +This is not a massive difference - they both still behave according to the core +definition of a *channel* - but presents some problems when we're looking to +scale things up. We can happily randomly load-balance normal channels across +clusters of channel servers and workers - after all, any worker can process +the message - but response channels would have to have their messages sent +to the channel server they're listening on. + +For this reason, Channels treats these as two different *channel types*, and +denotes a response channel by having the first character of the channel name +be the character ``!`` - e.g. ``!django.wsgi.response.f5G3fE21f``. Normal +channels have no special prefix, but along with the rest of the response +channel name, they must contain only the characters ``a-z A-Z 0-9 - _``, +and be less than 200 characters long. + +It's optional for a backend implementation to understand this - after all, +it's only important at scale, where you want to shard the two types differently +- but it's present nonetheless. For more on scaling, and how to handle channel +types if you're writing a backend or interface server, read :doc:`scaling`. + +Groups +------ + +Because channels only deliver to a single listener, they can't do broadcast; +if you want to send a message to an arbitrary group of clients, you need to +keep track of which response channels of those you wish to send to. + +Say I had a live blog where I wanted to push out updates whenever a new post is +saved, I would register a handler for the ``post_save`` signal and keep a +set of channels to send updates to:: + + (todo) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..9be3d8a --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,258 @@ +# -*- coding: utf-8 -*- +# +# Channels documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 19 11:37:58 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Channels' +copyright = u'2015, Andrew Godwin' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.2' +# The full version, including alpha/beta/rc tags. +release = '0.2' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Channelsdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'Channels.tex', u'Channels Documentation', + u'Andrew Godwin', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'channels', u'Channels Documentation', + [u'Andrew Godwin'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'Channels', u'Channels Documentation', + u'Andrew Godwin', 'Channels', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/getting-started.rst b/docs/getting-started.rst new file mode 100644 index 0000000..059f831 --- /dev/null +++ b/docs/getting-started.rst @@ -0,0 +1,6 @@ +Getting Started +=============== + +(If you haven't yet, make sure you :doc:`install Channels ` +and read up on :doc:`the concepts behind Channels `) + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..65b34e5 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,29 @@ +Django Channels +=============== + +Channels is a project to make Django able to handle more than just plain +HTTP requests, including WebSockets and HTTP2, as well as the ability to +run code after a response has been sent for things like thumbnailing or +background calculation. + +It's an easy-to-understand extension of the Django view model, and easy +to integrate and deploy. + +First, read our :doc:`concepts` documentation to get an idea of the +data model underlying Channels and how they're used inside Django. + +Then, read :doc:`getting-started` to see how to get up and running with +WebSockets with only 30 lines of code. + +Contents: + +.. toctree:: + :maxdepth: 2 + + concepts + installation + getting-started + integration-changes + message-standards + scaling + backend-requirements diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..06a7c5c --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,21 @@ +Installation +============ + +Channels is available on PyPI - to install it, just run:: + + pip install -U channels + +Once that's done, you should add ``channels`` to your +``INSTALLED_APPS`` setting:: + + INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + ... + 'channels', + ) + +That's it! Once enabled, ``channels`` will integrate itself into Django and +take control of the ``runserver`` command. See :doc:`getting-started` for more. diff --git a/channels/docs/integration-changes.rst b/docs/integration-changes.rst similarity index 100% rename from channels/docs/integration-changes.rst rename to docs/integration-changes.rst diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..33f68e4 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,242 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Channels.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Channels.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/channels/docs/message-standards.rst b/docs/message-standards.rst similarity index 100% rename from channels/docs/message-standards.rst rename to docs/message-standards.rst diff --git a/docs/scaling.rst b/docs/scaling.rst new file mode 100644 index 0000000..8d9e87d --- /dev/null +++ b/docs/scaling.rst @@ -0,0 +1,30 @@ +Scaling +======= + +Of course, one of the downsides of introducing a channel layer to Django it +that it's something else that must scale. Scaling traditional Django as a +WSGI application is easy - you just add more servers and a loadbalancer. Your +database is likely to be the thing that stopped scaling before, and there's +a relatively large amount of knowledge about tackling that problem. + +By comparison, there's not as much knowledge about scaling something like this +(though as it is very similar to a task queue, we have some work to build from). +In particular, the fact that messages are at-most-once - we do not guarantee +delivery, in the same way a webserver doesn't guarantee a response - means +we can loosen a lot of restrictions that slow down more traditional task queues. + +In addition, because channels can only have single consumers and they're handled +by a fleet of workers all running the same code, we could easily split out +incoming work by sharding into separate clusters of channel backends +and worker servers - any cluster can handle any request, so we can just +loadbalance over them. + +Of course, that doesn't work for interface servers, where only a single +particular server is listening to each response channel - if we broke things +into clusters, it might end up that a response is sent on a different cluster +to the one that the interface server is listening on. + +That's why Channels labels any *response channel* with a leading ``!``, letting +you know that only one server is listening for it, and thus letting you scale +and shard the two different types of channels accordingly (for more on +the difference, see :ref:`channel-types`). From a488a3adfe4f5869405cc7df28e2ddf2699a99f8 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 23 Jun 2015 17:50:56 +0200 Subject: [PATCH 018/746] Fixed docstring typo in ResponseLater doown => down --- channels/response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/response.py b/channels/response.py index e790250..22d410a 100644 --- a/channels/response.py +++ b/channels/response.py @@ -31,7 +31,7 @@ def decode_response(value): class ResponseLater(Exception): """ - Class that represents a response which will be sent doown the response + Class that represents a response which will be sent down the response channel later. Used to move a django view-based segment onto the next task, as otherwise we'd need to write some kind of fake response. """ From 60f0680ec21102d73c4a2044ed3a7d11fa9c6455 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 25 Jun 2015 21:33:22 -0700 Subject: [PATCH 019/746] Update docs a bit more --- docs/concepts.rst | 91 +++++++++++++++++++++++++++++++++++++--- docs/getting-started.rst | 47 ++++++++++++++++++++- 2 files changed, 130 insertions(+), 8 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 3171b01..e4d0edd 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -26,7 +26,7 @@ What is a channel? The core of Channels is, unsurprisingly, a datastructure called a *channel*. What is a channel? It is an *ordered*, *first-in first-out queue* with -*at-most-once delivery* to *only one listener at a time*. +*message expiry* and *at-most-once delivery* to *only one listener at a time*. You can think of it as analagous to a task queue - messages are put onto the channel by *producers*, and then given to just one of the *consumers* @@ -147,9 +147,9 @@ the message - but response channels would have to have their messages sent to the channel server they're listening on. For this reason, Channels treats these as two different *channel types*, and -denotes a response channel by having the first character of the channel name -be the character ``!`` - e.g. ``!django.wsgi.response.f5G3fE21f``. Normal -channels have no special prefix, but along with the rest of the response +denotes a *response channel* by having the first character of the channel name +be the character ``!`` - e.g. ``!django.wsgi.response.f5G3fE21f``. *Normal +channels* have no special prefix, but along with the rest of the response channel name, they must contain only the characters ``a-z A-Z 0-9 - _``, and be less than 200 characters long. @@ -167,6 +167,85 @@ keep track of which response channels of those you wish to send to. Say I had a live blog where I wanted to push out updates whenever a new post is saved, I would register a handler for the ``post_save`` signal and keep a -set of channels to send updates to:: +set of channels (here, using Redis) to send updates to:: - (todo) + + redis_conn = redis.Redis("localhost", 6379) + + @receiver(post_save, sender=BlogUpdate) + def send_update(sender, instance, **kwargs): + # Loop through all response channels and send the update + for send_channel in redis_conn.smembers("readers"): + Channel(send_channel).send( + id=instance.id, + content=instance.content, + ) + + @Channel.consumer("django.websocket.connect") + def ws_connect(path, send_channel, **kwargs): + # Add to reader set + redis_conn.sadd("readers", send_channel) + +While this will work, there's a small problem - we never remove people from +the ``readers`` set when they disconnect. We could add a consumer that +listens to ``django.websocket.disconnect`` to do that, but we'd also need to +have some kind of expiry in case an interface server is forced to quit or +loses power before it can send disconnect signals - your code will never +see any disconnect notification but the response channel is completely +invalid and messages you send there will never get consumed and just expire. + +Because the basic design of channels is stateless, the channel server has no +concept of "closing" a channel if an interface server goes away - after all, +channels are meant to hold messages until a consumer comes along (and some +types of interface server, e.g. an SMS gateway, could theoretically serve +any client from any interface server). + +That means that we need to follow a keepalive model, where the interface server +(or, if you want even better accuracy, the client browser/connection) sends +a periodic message saying it's still connected (though only for persistent +connection types like WebSockets; normal HTTP doesn't need this as it won't +stay connected for more than its own timeout). + +Now, we could go back into our example above and add an expiring set and keep +track of expiry times and so forth, but this is such a common pattern that +we don't need to; Channels has it built in, as a feature called Groups:: + + @receiver(post_save, sender=BlogUpdate) + def send_update(sender, instance, **kwargs): + Group("liveblog").send( + id=instance.id, + content=instance.content, + ) + + @Channel.consumer("django.websocket.connect") + @Channel.consumer("django.websocket.keepalive") + def ws_connect(path, send_channel, **kwargs): + # Add to reader group + Group("liveblog").add(send_channel) + +Not only do groups have their own ``send()`` method (which backends can provide +an efficient implementation of), they also automatically manage expiry of +the group members. You'll have to re-call ``Group.add()`` every so often to +keep existing members from expiring, but that's easy, and can be done in the +same handler for both ``connect`` and ``keepalive``, as you can see above. + +Groups are generally only useful for response channels (ones starting with +the character ``!``), as these are unique-per-client. + +Next Steps +---------- + +That's the high-level overview of channels and groups, and how you should +starting thinking about them - remember, Django provides some channels +but you're free to make and consume your own, and all channels are +network-transparent. + +One thing channels are not, however, is guaranteed delivery. If you want tasks +you're sure will complete, use a system designed for this with retries and +persistence like Celery, or you'll need to make a management command that +checks for completion and re-submits a message to the channel if nothing +is completed (rolling your own retry logic, essentially). + +We'll cover more about what kind of tasks fit well into Channels in the rest +of the documentation, but for now, let's progress to :doc:`getting-started` +and writing some code. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 059f831..67054cc 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -1,6 +1,49 @@ Getting Started =============== -(If you haven't yet, make sure you :doc:`install Channels ` -and read up on :doc:`the concepts behind Channels `) +(If you haven't yet, make sure you :doc:`install Channels `) +Now, let's get to writing some consumers. If you've not read it already, +you should read :doc:`concepts`, as it covers the basic description of what +channels and groups are, and lays out some of the important implementation +patterns and caveats. + +First Consumers +--------------- + +Now, by default, Django will run things through Channels but it will also +tie in the URL router and view subsystem to the default ``django.wsgi.request`` +channel if you don't provide another consumer that listens to it - remember, +only one consumer can listen to any given channel. + +As a very basic example, let's write a consumer that overrides the built-in +handling and handles every HTTP request directly. Make a new project, a new +app, and put this in a ``consumers.py`` file in the app:: + + from channels import Channel + from django.http import HttpResponse + + @Channel.consumer("django.wsgi.request") + def http_consumer(response_channel, path, **kwargs): + response = HttpResponse("Hello world! You asked for %s" % path) + Channel(response_channel).send(response.channel_encode()) + +The most important thing to note here is that, because things we send in +messages must be JSON-serialisable, the request and response messages +are in a key-value format. There are ``channel_decode()`` and +``channel_encode()`` methods on both Django's request and response classes, +but here we just take two of the request variables directly as keyword +arguments for simplicity. + +If you start up ``python manage.py runserver`` and go to +``http://localhost:8000``, you'll see that, rather than a default Django page, +you get the Hello World response, so things are working. If you don't see +a response, check you :doc:`installed Channels correctly `. + +Now, that's not very exciting - raw HTTP responses are something Django can +do any time. Let's try some WebSockets! + +Delete that consumer from above - we'll need the normal Django view layer to +serve templates later - and make this WebSocket consumer instead:: + + # todo From 768722dc64540a2e825891993139bbfe0b864859 Mon Sep 17 00:00:00 2001 From: Hiroki Kiyohara Date: Thu, 9 Jul 2015 16:10:39 +0900 Subject: [PATCH 020/746] Fixed wrong code on doc `.send()` won't get positional arguments. --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 67054cc..cf97437 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -26,7 +26,7 @@ app, and put this in a ``consumers.py`` file in the app:: @Channel.consumer("django.wsgi.request") def http_consumer(response_channel, path, **kwargs): response = HttpResponse("Hello world! You asked for %s" % path) - Channel(response_channel).send(response.channel_encode()) + Channel(response_channel).send(**response.channel_encode()) The most important thing to note here is that, because things we send in messages must be JSON-serialisable, the request and response messages From aa921b16598c74d2da7061abdc906aace4ab3719 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 12 Jul 2015 20:19:10 -0500 Subject: [PATCH 021/746] Channel groups, tests and docs --- channels/backends/base.py | 2 +- channels/backends/database.py | 68 ++++++++++++++-- channels/backends/memory.py | 44 +++++++++- channels/backends/redis_py.py | 38 ++++++++- channels/channel.py | 28 ++++++- channels/tests/__init__.py | 0 channels/tests/test_backends.py | 91 +++++++++++++++++++++ docs/getting-started.rst | 138 +++++++++++++++++++++++++++++++- 8 files changed, 395 insertions(+), 14 deletions(-) create mode 100644 channels/tests/__init__.py create mode 100644 channels/tests/test_backends.py diff --git a/channels/backends/base.py b/channels/backends/base.py index e7dd139..7a99568 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -88,7 +88,7 @@ class BaseChannelBackend(object): This base class provides a default implementation; can be overridden to be more efficient by subclasses. """ - for channel in self.group_channels(): + for channel in self.group_channels(group): self.send(channel, message) def __str__(self): diff --git a/channels/backends/database.py b/channels/backends/database.py index bf8e842..ed59a9e 100644 --- a/channels/backends/database.py +++ b/channels/backends/database.py @@ -27,8 +27,8 @@ class DatabaseChannelBackend(BaseChannelBackend): """ return connections[self.db_alias] - @property - def model(self): + @cached_property + def channel_model(self): """ Initialises a new model to store messages; not done as part of a models.py as we don't want to make it for most installs. @@ -49,8 +49,30 @@ class DatabaseChannelBackend(BaseChannelBackend): editor.create_model(Message) return Message + @cached_property + def group_model(self): + """ + Initialises a new model to store groups; not done as part of a + models.py as we don't want to make it for most installs. + """ + # Make the model class + class Group(models.Model): + group = models.CharField(max_length=200) + channel = models.CharField(max_length=200) + expiry = models.DateTimeField(db_index=True) + class Meta: + apps = Apps() + app_label = "channels" + db_table = "django_channel_groups" + unique_together = [["group", "channel"]] + # Ensure its table exists + if Group._meta.db_table not in self.connection.introspection.table_names(self.connection.cursor()): + with self.connection.schema_editor() as editor: + editor.create_model(Group) + return Group + def send(self, channel, message): - self.model.objects.create( + self.channel_model.objects.create( channel = channel, content = json.dumps(message), expiry = now() + datetime.timedelta(seconds=self.expiry) @@ -59,15 +81,47 @@ class DatabaseChannelBackend(BaseChannelBackend): def receive_many(self, channels): if not channels: raise ValueError("Cannot receive on empty channel list!") - # Delete all expired messages (add 10 second grace period for clock sync) - self.model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() + self._clean_expired() # Get a message from one of our channels - message = self.model.objects.filter(channel__in=channels).order_by("id").first() + message = self.channel_model.objects.filter(channel__in=channels).order_by("id").first() if message: - self.model.objects.filter(pk=message.pk).delete() + self.channel_model.objects.filter(pk=message.pk).delete() return message.channel, json.loads(message.content) else: return None, None + def _clean_expired(self): + """ + Cleans out expired groups and messages. + """ + # Include a 10-second grace period because that solves some clock sync + self.channel_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() + self.group_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() + + def group_add(self, group, channel, expiry=None): + """ + Adds the channel to the named group for at least 'expiry' + seconds (expiry defaults to message expiry if not provided). + """ + self.group_model.objects.update_or_create( + group = group, + channel = channel, + defaults = {"expiry": now() + datetime.timedelta(seconds=expiry or self.expiry)}, + ) + + def group_discard(self, group, channel): + """ + Removes the channel from the named group if it is in the group; + does nothing otherwise (does not error) + """ + self.group_model.objects.filter(group=group, channel=channel).delete() + + def group_channels(self, group): + """ + Returns an iterable of all channels in the group. + """ + self._clean_expired() + return list(self.group_model.objects.filter(group=group).values_list("channel", flat=True)) + def __str__(self): return "%s(alias=%s)" % (self.__class__.__name__, self.connection.alias) diff --git a/channels/backends/memory.py b/channels/backends/memory.py index 1d53d45..c398653 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -4,6 +4,7 @@ from collections import deque from .base import BaseChannelBackend queues = {} +groups = {} class InMemoryChannelBackend(BaseChannelBackend): """ @@ -17,18 +18,57 @@ class InMemoryChannelBackend(BaseChannelBackend): # Try JSON encoding it to make sure it would, but store the native version json.dumps(message) # Add to the deque, making it if needs be - queues.setdefault(channel, deque()).append(message) + queues.setdefault(channel, deque()).append((message, time.time() + self.expiry)) def receive_many(self, channels): if not channels: raise ValueError("Cannot receive on empty channel list!") # Try to pop a message from each channel + self._clean_expired() for channel in channels: try: # This doesn't clean up empty channels - OK for testing. # For later versions, have cleanup w/lock. - return channel, queues[channel].popleft() + return channel, queues[channel].popleft()[0] except (IndexError, KeyError): pass return None, None + def _clean_expired(self): + # Handle expired messages + for channel, messages in queues.items(): + while len(messages) and messages[0][1] < time.time(): + messages.popleft() + # Handle expired groups + for group, channels in list(groups.items()): + for channel, expiry in list(channels.items()): + if expiry < (time.time() - 10): + try: + del groups[group][channel] + except KeyError: + # Another thread might have got there first + pass + + def group_add(self, group, channel, expiry=None): + """ + Adds the channel to the named group for at least 'expiry' + seconds (expiry defaults to message expiry if not provided). + """ + groups.setdefault(group, {})[channel] = time.time() + (expiry or self.expiry) + + def group_discard(self, group, channel): + """ + Removes the channel from the named group if it is in the group; + does nothing otherwise (does not error) + """ + try: + del groups[group][channel] + except KeyError: + pass + + def group_channels(self, group): + """ + Returns an iterable of all channels in the group. + """ + self._clean_expired() + return groups.get(group, {}).keys() diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 602e749..0557fc1 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -28,7 +28,7 @@ class RedisChannelBackend(BaseChannelBackend): def send(self, channel, message): # Write out message into expiring key (avoids big items in list) - key = uuid.uuid4() + key = self.prefix + uuid.uuid4().get_hex() self.connection.set( key, json.dumps(message), @@ -63,5 +63,41 @@ class RedisChannelBackend(BaseChannelBackend): else: return None, None + def group_add(self, group, channel, expiry=None): + """ + Adds the channel to the named group for at least 'expiry' + seconds (expiry defaults to message expiry if not provided). + """ + key = "%s:group:%s" % (self.prefix, group) + self.connection.zadd( + key, + **{channel: time.time() + (expiry or self.expiry)} + ) + + def group_discard(self, group, channel): + """ + Removes the channel from the named group if it is in the group; + does nothing otherwise (does not error) + """ + key = "%s:group:%s" % (self.prefix, group) + self.connection.zrem( + key, + channel, + ) + + def group_channels(self, group): + """ + Returns an iterable of all channels in the group. + """ + key = "%s:group:%s" % (self.prefix, group) + # Discard old channels + self.connection.zremrangebyscore(key, 0, int(time.time()) - 10) + # Return current lot + return self.connection.zrange( + key, + 0, + -1, + ) + def __str__(self): return "%s(host=%s, port=%s)" % (self.__class__.__name__, self.host, self.port) diff --git a/channels/channel.py b/channels/channel.py index 9572525..ac24c4b 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -54,7 +54,7 @@ class Channel(object): return view_producer(self.name) @classmethod - def consumer(self, channels, alias=DEFAULT_CHANNEL_BACKEND): + def consumer(self, *channels, alias=DEFAULT_CHANNEL_BACKEND): """ Decorator that registers a function as a consumer. """ @@ -68,3 +68,29 @@ class Channel(object): channel_backend.registry.add_consumer(func, channels) return func return inner + + +class Group(object): + """ + A group of channels that can be messaged at once, and that expire out + of the group after an expiry time (keep re-adding to keep them in). + """ + + def __init__(self, alias=DEFAULT_CHANNEL_BACKEND, channel_backend=None): + self.name = name + if channel_backend: + self.channel_backend = channel_backend + else: + self.channel_backend = channel_backends[alias] + + def add(self, channel): + self.channel_backend.add(self.name, channel) + + def discard(self, channel): + self.channel_backend.discard(self.name, channel) + + def channels(self): + self.channel_backend.channels(self.name) + + def send(self, **kwargs): + self.channel_backend.send_group(self, self.name, kwargs) diff --git a/channels/tests/__init__.py b/channels/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channels/tests/test_backends.py b/channels/tests/test_backends.py new file mode 100644 index 0000000..1846bdc --- /dev/null +++ b/channels/tests/test_backends.py @@ -0,0 +1,91 @@ +from django.test import TestCase +from ..channel import Channel +from ..backends.database import DatabaseChannelBackend +from ..backends.redis_py import RedisChannelBackend +from ..backends.memory import InMemoryChannelBackend + + +class MemoryBackendTests(TestCase): + + backend_class = InMemoryChannelBackend + + def setUp(self): + self.backend = self.backend_class() + + def test_send_recv(self): + """ + Tests that channels can send and receive messages. + """ + self.backend.send("test", {"value": "blue"}) + self.backend.send("test", {"value": "green"}) + self.backend.send("test2", {"value": "red"}) + # Get just one first + channel, message = self.backend.receive_many(["test"]) + self.assertEqual(channel, "test") + self.assertEqual(message, {"value": "blue"}) + # And the second + channel, message = self.backend.receive_many(["test"]) + self.assertEqual(channel, "test") + self.assertEqual(message, {"value": "green"}) + # And the other channel with multi select + channel, message = self.backend.receive_many(["test", "test2"]) + self.assertEqual(channel, "test2") + self.assertEqual(message, {"value": "red"}) + + def test_message_expiry(self): + self.backend = self.backend_class(expiry=-100) + self.backend.send("test", {"value": "blue"}) + channel, message = self.backend.receive_many(["test"]) + self.assertIs(channel, None) + self.assertIs(message, None) + + def test_groups(self): + """ + Tests that group addition and removal and listing works + """ + self.backend.group_add("tgroup", "test") + self.backend.group_add("tgroup", "test2") + self.backend.group_add("tgroup2", "test3") + self.assertEqual( + set(self.backend.group_channels("tgroup")), + {"test", "test2"}, + ) + self.backend.group_discard("tgroup", "test2") + self.backend.group_discard("tgroup", "test2") + self.assertEqual( + self.backend.group_channels("tgroup"), + ["test"], + ) + + def test_group_send(self): + """ + Tests sending to groups. + """ + self.backend.group_add("tgroup", "test") + self.backend.group_add("tgroup", "test2") + self.backend.send_group("tgroup", {"value": "orange"}) + channel, message = self.backend.receive_many(["test"]) + self.assertEqual(channel, "test") + self.assertEqual(message, {"value": "orange"}) + channel, message = self.backend.receive_many(["test2"]) + self.assertEqual(channel, "test2") + self.assertEqual(message, {"value": "orange"}) + + def test_group_expiry(self): + self.backend = self.backend_class(expiry=-100) + self.backend.group_add("tgroup", "test") + self.backend.group_add("tgroup", "test2") + self.assertEqual( + self.backend.group_channels("tgroup"), + [], + ) + + +class RedisBackendTests(MemoryBackendTests): + + backend_class = RedisChannelBackend + + +class DatabaseBackendTests(MemoryBackendTests): + + backend_class = DatabaseChannelBackend diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 67054cc..9313844 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -41,9 +41,143 @@ you get the Hello World response, so things are working. If you don't see a response, check you :doc:`installed Channels correctly `. Now, that's not very exciting - raw HTTP responses are something Django can -do any time. Let's try some WebSockets! +do any time. Let's try some WebSockets, and make a basic chat server! Delete that consumer from above - we'll need the normal Django view layer to serve templates later - and make this WebSocket consumer instead:: - # todo + @Channel.consumer("django.websocket.connect") + def ws_connect(channel, send_channel, **kwargs): + Group("chat").add(send_channel) + +Now, let's look at what this is doing. It's tied to the +``django.websocket.connect`` channel, which means that it'll get a message +whenever a new WebSocket connection is opened by a client. + +When it gets that message, it takes the ``send_channel`` key from it, which +is the unique response channel for that client, and adds it to the ``chat`` +group, which means we can send messages to all connected chat clients. + +Of course, if you've read through :doc:`concepts`, you'll know that channels +added to groups expire out after a while unless you keep renewing their +membership. This is because Channels is stateless; the worker processes +don't keep track of the open/close states of the potentially thousands of +connections you have open at any one time. + +The solution to this is that the WebSocket interface servers will send +periodic "keepalive" messages on the ``django.websocket.keepalive`` channel, +so we can hook that up to re-add the channel (it's safe to add the channel to +a group it's already in - similarly, it's safe to discard a channel from a +group it's not in):: + + @Channel.consumer("django.websocket.keepalive") + def ws_keepalive(channel, send_channel, **kwargs): + Group("chat").add(send_channel) + +Of course, this is exactly the same code as the ``connect`` handler, so let's +just combine them:: + + @Channel.consumer("django.websocket.connect", "django.websocket.keepalive") + def ws_add(channel, send_channel, **kwargs): + Group("chat").add(send_channel) + +And, even though channels will expire out, let's add an explicit ``disconnect`` +handler to clean up as people disconnect (most channels will cleanly disconnect +and get this called):: + + @Channel.consumer("django.websocket.disconnect") + def ws_disconnect(channel, send_channel, **kwargs): + Group("chat").discard(send_channel) + +Now, that's taken care of adding and removing WebSocket send channels for the +``chat`` group; all we need to do now is take care of message sending. For now, +we're not going to store a history of messages or anything and just replay +any message sent in to all connected clients. Here's all the code:: + + @Channel.consumer("django.websocket.connect", "django.websocket.keepalive") + def ws_add(channel, send_channel, **kwargs): + Group("chat").add(send_channel) + + @Channel.consumer("django.websocket.receive") + def ws_message(channel, send_channel, content, **kwargs): + Group("chat").send(content=content) + + @Channel.consumer("django.websocket.disconnect") + def ws_disconnect(channel, send_channel, **kwargs): + Group("chat").discard(send_channel) + +With all that code in your ``consumers.py`` file, you now have a working +set of a logic for a chat server. All you need to do now is get it deployed, +and as we'll see, that's not too hard. + +Running with Channels +--------------------- + +Because Channels takes Django into a multi-process model, you can no longer +just run one process if you want to serve more than one protocol type. + +There are multiple kinds of "interface server", and each one will service a +different type of request - one might do WSGI requests, one might handle +WebSockets, or you might have one that handles both. + +These are separate from the "worker servers" where Django will run actual logic, +though, and so you'll need to configure a channel backend to allow the +channels to run over the network. By default, when you're using Django out of +the box, the channel backend is set to an in-memory one that only works in +process; this is enough to serve normal WSGI style requests (``runserver`` is +just running a WSGI interface and a worker in two threads), but now we want +WebSocket support we'll need a separate process to keep things clean. + +For simplicity, we'll configure the database backend - this uses two tables +in the database to do message handling, and isn't particularly fast but +requires no extra dependencies. Put this in your ``settings.py`` file:: + + CHANNEL_BACKENDS = { + "default": { + "BACKEND": "channels.backends.database.DatabaseChannelBackend", + }, + } + +As you can see, the format is quite similar to the ``DATABASES`` setting in +Django, but for this case much simpler, as it just uses the default database +(you can set which alias it uses with the ``DB_ALIAS`` key). + +In production, we'd recommend you use something like the Redis channel backend; +you can :doc:`read about the backends ` and see how to set them up +and their performance considerations if you wish. + +The second thing, once we have a networked channel backend set up, is to make +sure we're running the WebSocket interface server. Even in development, we need +to do this; ``runserver`` will take care of normal Web requests and running +a worker for us, but WebSockets require an in-process async solution. + +The easiest way to do this is to use the ``runwsserver`` management command +that ships with Django; just make sure you've installed the latest release +of ``autobahn`` first:: + + pip install -U autobahn + python manage.py runwsserver + +Run that alongside ``runserver`` and you'll have two interface servers, a +worker thread, and the channel backend all connected and running. You can +even launch separate worker processes with ``runworker`` if you like (you'll +need at least one of those if you're not also running ``runserver``). + +Now, just open a browser and put the following into the JavaScript console +to test your new code:: + + socket = new WebSocket("ws://127.0.0.1:9000"); + socket.onmessage = function(e) { + alert(e.data); + } + socket.send("hello world"); + +You should see an alert come back immediately saying "hello world" - your +message has round-tripped through the server and come back to trigger the alert. +You can open another tab and do the same there if you like, and both tabs will +receive the message and show an alert. + +Feel free to put some calls to ``print`` in your handler functions too, if you +like, so you can understand when they're called. If you run three or four +copies of ``runworker``, too, you will probably be able to see the tasks running +on different workers. From 8492dcde4874786e8c0d4ca29fcf6873b3a4468a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 12 Jul 2015 21:25:53 -0500 Subject: [PATCH 022/746] More docs, some API permutation --- channels/backends/redis_py.py | 2 + channels/channel.py | 18 -- channels/decorators.py | 24 ++ channels/interfaces/websocket_twisted.py | 1 + ...{backend-requirements.rst => backends.rst} | 31 ++- docs/getting-started.rst | 234 +++++++++++++++++- docs/index.rst | 2 +- 7 files changed, 278 insertions(+), 34 deletions(-) create mode 100644 channels/decorators.py rename docs/{backend-requirements.rst => backends.rst} (68%) diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 0557fc1..11d3834 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -99,5 +99,7 @@ class RedisChannelBackend(BaseChannelBackend): -1, ) + # TODO: send_group efficient implementation using Lua + def __str__(self): return "%s(host=%s, port=%s)" % (self.__class__.__name__, self.host, self.port) diff --git a/channels/channel.py b/channels/channel.py index ac24c4b..6724eba 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -1,8 +1,6 @@ import random import string -from django.utils import six - from channels import channel_backends, DEFAULT_CHANNEL_BACKEND @@ -53,22 +51,6 @@ class Channel(object): from channels.adapters import view_producer return view_producer(self.name) - @classmethod - def consumer(self, *channels, alias=DEFAULT_CHANNEL_BACKEND): - """ - Decorator that registers a function as a consumer. - """ - # Upconvert if you just pass in a string - if isinstance(channels, six.string_types): - channels = [channels] - # Get the channel - channel_backend = channel_backends[alias] - # Return a function that'll register whatever it wraps - def inner(func): - channel_backend.registry.add_consumer(func, channels) - return func - return inner - class Group(object): """ diff --git a/channels/decorators.py b/channels/decorators.py new file mode 100644 index 0000000..63ba59d --- /dev/null +++ b/channels/decorators.py @@ -0,0 +1,24 @@ +import functools + +from django.utils import six + +from channels import channel_backends, DEFAULT_CHANNEL_BACKEND + + +def consumer(self, *channels, alias=DEFAULT_CHANNEL_BACKEND): + """ + Decorator that registers a function as a consumer. + """ + # Upconvert if you just pass in a string + if isinstance(channels, six.string_types): + channels = [channels] + # Get the channel + channel_backend = channel_backends[alias] + # Return a function that'll register whatever it wraps + def inner(func): + channel_backend.registry.add_consumer(func, channels) + return func + return inner + + +# TODO: Sessions, auth diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index dc9aff9..5e3dbe1 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -114,6 +114,7 @@ class WebsocketTwistedInterface(object): """ while True: channels = self.factory.send_channels() + # TODO: Send keepalives # Quit if reactor is stopping if not reactor.running: return diff --git a/docs/backend-requirements.rst b/docs/backends.rst similarity index 68% rename from docs/backend-requirements.rst rename to docs/backends.rst index f2dc028..be9faac 100644 --- a/docs/backend-requirements.rst +++ b/docs/backends.rst @@ -1,5 +1,24 @@ -Channel Backend Requirements -============================ +Backends +======== + +Multiple choices of backend are available, to fill different tradeoffs of +complexity, throughput and scalability. You can also write your own backend if +you wish; the API is very simple and documented below. + +In-memory +--------- + +Database +-------- + +Redis +----- + +Writing Custom Backends +----------------------- + +Backend Requirements +^^^^^^^^^^^^^^^^^^^^ While the channel backends are pluggable, there's a minimum standard they must meet in terms of the way they perform. @@ -23,12 +42,14 @@ In particular, a channel backend MUST: * Preserve the ordering of messages inside a channel -* Never deliver a message more than once +* Never deliver a message more than once (design for at-most-once delivery) * Never block on sending of a message (dropping the message/erroring is preferable to blocking) * Be able to store messages of at least 5MB in size +* Allow for channel and group names of up to 200 printable ASCII characters + * Expire messages only after the expiry period provided is up (a backend may keep them longer if it wishes, but should expire them at some reasonable point to ensure users do not come to rely on permanent messages) @@ -41,6 +62,10 @@ In addition, it SHOULD: * Provide a ``send_group()`` method which sends a message to every channel in a group. +* Make ``send_group()`` perform better than ``O(n)``, where ``n`` is the + number of members in the group; preferably send the messages to all + members in a single call to your backing datastore or protocol. + * Try and preserve a rough global ordering, so that one busy channel does not drown out an old message in another channel if a worker is listening on both. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 9313844..892c291 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -21,9 +21,10 @@ handling and handles every HTTP request directly. Make a new project, a new app, and put this in a ``consumers.py`` file in the app:: from channels import Channel + from channels.decorators import consumer from django.http import HttpResponse - @Channel.consumer("django.wsgi.request") + @consumer("django.wsgi.request") def http_consumer(response_channel, path, **kwargs): response = HttpResponse("Hello world! You asked for %s" % path) Channel(response_channel).send(response.channel_encode()) @@ -46,7 +47,7 @@ do any time. Let's try some WebSockets, and make a basic chat server! Delete that consumer from above - we'll need the normal Django view layer to serve templates later - and make this WebSocket consumer instead:: - @Channel.consumer("django.websocket.connect") + @consumer("django.websocket.connect") def ws_connect(channel, send_channel, **kwargs): Group("chat").add(send_channel) @@ -70,14 +71,14 @@ so we can hook that up to re-add the channel (it's safe to add the channel to a group it's already in - similarly, it's safe to discard a channel from a group it's not in):: - @Channel.consumer("django.websocket.keepalive") + @consumer("django.websocket.keepalive") def ws_keepalive(channel, send_channel, **kwargs): Group("chat").add(send_channel) Of course, this is exactly the same code as the ``connect`` handler, so let's just combine them:: - @Channel.consumer("django.websocket.connect", "django.websocket.keepalive") + @consumer("django.websocket.connect", "django.websocket.keepalive") def ws_add(channel, send_channel, **kwargs): Group("chat").add(send_channel) @@ -85,7 +86,7 @@ And, even though channels will expire out, let's add an explicit ``disconnect`` handler to clean up as people disconnect (most channels will cleanly disconnect and get this called):: - @Channel.consumer("django.websocket.disconnect") + @consumer("django.websocket.disconnect") def ws_disconnect(channel, send_channel, **kwargs): Group("chat").discard(send_channel) @@ -94,15 +95,18 @@ Now, that's taken care of adding and removing WebSocket send channels for the we're not going to store a history of messages or anything and just replay any message sent in to all connected clients. Here's all the code:: - @Channel.consumer("django.websocket.connect", "django.websocket.keepalive") + from channels import Channel + from channels.decorators import consumer + + @consumer("django.websocket.connect", "django.websocket.keepalive") def ws_add(channel, send_channel, **kwargs): Group("chat").add(send_channel) - @Channel.consumer("django.websocket.receive") + @consumer("django.websocket.receive") def ws_message(channel, send_channel, content, **kwargs): Group("chat").send(content=content) - @Channel.consumer("django.websocket.disconnect") + @consumer("django.websocket.disconnect") def ws_disconnect(channel, send_channel, **kwargs): Group("chat").discard(send_channel) @@ -125,10 +129,10 @@ though, and so you'll need to configure a channel backend to allow the channels to run over the network. By default, when you're using Django out of the box, the channel backend is set to an in-memory one that only works in process; this is enough to serve normal WSGI style requests (``runserver`` is -just running a WSGI interface and a worker in two threads), but now we want +just running a WSGI interface and a worker in two separate threads), but now we want WebSocket support we'll need a separate process to keep things clean. -For simplicity, we'll configure the database backend - this uses two tables +For simplicity, we'll use the database channel backend - this uses two tables in the database to do message handling, and isn't particularly fast but requires no extra dependencies. Put this in your ``settings.py`` file:: @@ -149,7 +153,8 @@ and their performance considerations if you wish. The second thing, once we have a networked channel backend set up, is to make sure we're running the WebSocket interface server. Even in development, we need to do this; ``runserver`` will take care of normal Web requests and running -a worker for us, but WebSockets require an in-process async solution. +a worker for us, but WebSockets isn't compatible with WSGI and needs to run +separately. The easiest way to do this is to use the ``runwsserver`` management command that ships with Django; just make sure you've installed the latest release @@ -179,5 +184,210 @@ receive the message and show an alert. Feel free to put some calls to ``print`` in your handler functions too, if you like, so you can understand when they're called. If you run three or four -copies of ``runworker``, too, you will probably be able to see the tasks running +copies of ``runworker`` you'll probably be able to see the tasks running on different workers. + +Authentication +-------------- + +Now, of course, a WebSocket solution is somewhat limited in scope without the +ability to live with the rest of your website - in particular, we want to make +sure we know what user we're talking to, in case we have things like private +chat channels (we don't want a solution where clients just ask for the right +channels, as anyone could change the code and just put in private channel names) + +It can also save you having to manually make clients ask for what they want to +see; if I see you open a WebSocket to my "updates" endpoint, and I know which +user ID, I can just auto-add that channel to all the relevant groups (mentions +of that user, for example). + +Handily, as WebSockets start off using the HTTP protocol, they have a lot of +familiar features, including a path, GET parameters, and cookies. Notably, +the cookies allow us to perform authentication using the same methods the +normal Django middleware does. Middleware only runs on requests to views, +however, and not on raw consumer calls; it's tailored to work with single +HTTP requests, and so we need a different solution to authenticate WebSockets. + +In addition, we don't want the interface servers storing data or trying to run +authentication; they're meant to be simple, lean, fast processes without much +state, and so we'll need to do our authentication inside our consumer functions. + +Fortunately, because Channels has standardised WebSocket event +:doc:`message-standards`, it ships with decorators that help you with +authentication, as well as using Django's session framework (which authentication +relies on). + +All we need to do is add the ``websocket_auth`` decorator to our views, +and we'll get extra ``session`` and ``user`` keyword arguments we can use; +let's make one where users can only chat to people with the same first letter +of their username:: + + from channels import Channel + from channels.decorators import consumer, websocket_auth + + @consumer("django.websocket.connect", "django.websocket.keepalive") + @websocket_auth + def ws_add(channel, send_channel, user, **kwargs): + Group("chat-%s" % user.username[0]).add(send_channel) + + @consumer("django.websocket.receive") + @websocket_auth + def ws_message(channel, send_channel, content, user, **kwargs): + Group("chat-%s" % user.username[0]).send(content=content) + + @consumer("django.websocket.disconnect") + @websocket_auth + def ws_disconnect(channel, send_channel, user, **kwargs): + Group("chat-%s" % user.username[0]).discard(send_channel) + +(Note that we always end consumers with ``**kwargs``; this is to save us +from writing out all variables we might get sent and to allow forwards-compatibility +with any additions to the message formats in the future) + +Persisting Data +--------------- + +Doing chatrooms by username first letter is a nice simple example, but it's +skirting around the real design pattern - persistent state for connections. +A user may open our chat site and select the chatroom to join themselves, so we +should let them send this request in the initial WebSocket connection, +check they're allowed to access it, and then remember which room a socket is +connected to when they send a message in so we know which group to send it to. + +The ``send_channel`` is our unique pointer to the open WebSocket - as you've +seen, we do all our operations on it - but it's not something we can annotate +with data; it's just a simple string, and even if we hack around and set +attributes on it that's not going to carry over to other workers. + +Instead, the solution is to persist information keyed by the send channel in +some other data store - sound familiar? This is what Django's session framework +does for HTTP requests, only there it uses cookies as the lookup key rather +than the ``send_channel``. + +Now, as you saw above, you can use the ``websocket_auth`` decorator to get +both a ``user`` and a ``session`` variable in your message arguments - and, +indeed, there is a ``websocket_session`` decorator that will just give you +the ``session`` attribute. + +However, that session is based on cookies, and so follows the user round the +site - it's great for information that should persist across all WebSocket and +HTTP connections, but not great for information that is specific to a single +WebSocket (such as "which chatroom should this socket be connected to"). For +this reason, Channels also provides a ``websocker_channel_session`` decorator, +which adds a ``channel_session`` attribute to the message; this works just like +the normal ``session`` attribute, and persists to the same storage, but varies +per-channel rather than per-cookie. + +Let's use it now to build a chat server that expects you to pass a chatroom +name in the path of your WebSocket request (we'll ignore auth for now):: + + from channels import Channel + from channels.decorators import consumer, websocket_channel_session + + @consumer("django.websocket.connect") + @websocket_channel_session + def ws_connect(channel, send_channel, path, channel_session, **kwargs): + # Work out room name from path (ignore slashes) + room = path.strip("/") + # Save room in session and add us to the group + channel_session['room'] = room + Group("chat-%s" % room).add(send_channel) + + @consumer("django.websocket.keepalive") + @websocket_channel_session + def ws_add(channel, send_channel, channel_session, **kwargs): + Group("chat-%s" % channel_session['room']).add(send_channel) + + @consumer("django.websocket.receive") + @websocket_channel_session + def ws_message(channel, send_channel, content, channel_session, **kwargs): + Group("chat-%s" % channel_session['room']).send(content=content) + + @consumer("django.websocket.disconnect") + @websocket_channel_session + def ws_disconnect(channel, send_channel, channel_session, **kwargs): + Group("chat-%s" % channel_session['room']).discard(send_channel) + +If you play around with it from the console (or start building a simple +JavaScript chat client that appends received messages to a div), you'll see +that you can now request which chat room you want in the initial request. We +could easily add in the auth decorator here too and do an initial check in +``connect`` that the user had permission to join that chatroom. + +Models +------ + +So far, we've just been taking incoming messages and rebroadcasting them to +other clients connected to the same group, but this isn't that great; really, +we want to persist messages to a datastore, and we'd probably like to be +able to inject messages into chatrooms from things other than WebSocket client +connections (perhaps a built-in bot, or server status messages). + +Thankfully, we can just use Django's ORM to handle persistence of messages and +easily integrate the send into the save flow of the model, rather than the +message receive - that way, any new message saved will be broadcast to all +the appropriate clients, no matter where it's saved from. + +We'll even take some performance considerations into account - We'll make our +own custom channel for new chat messages and move the model save and the chat +broadcast into that, meaning the sending process/consumer can move on +immediately and not spend time waiting for the database save and the +(slow on some backends) ``Group.send()`` call. + +Let's see what that looks like, assuming we +have a ChatMessage model with ``message`` and ``room`` fields:: + + from channels import Channel + from channels.decorators import consumer, websocket_channel_session + from .models import ChatMessage + + @consumer("chat-messages") + def msg_consumer(room, message): + # Save to model + ChatMessage.objects.create(room=room, message=message) + # Broadcast to listening sockets + Group("chat-%s" % room).send(message) + + @consumer("django.websocket.connect") + @websocket_channel_session + def ws_connect(channel, send_channel, path, channel_session, **kwargs): + # Work out room name from path (ignore slashes) + room = path.strip("/") + # Save room in session and add us to the group + channel_session['room'] = room + Group("chat-%s" % room).add(send_channel) + + @consumer("django.websocket.keepalive") + @websocket_channel_session + def ws_add(channel, send_channel, channel_session, **kwargs): + Group("chat-%s" % channel_session['room']).add(send_channel) + + @consumer("django.websocket.receive") + @websocket_channel_session + def ws_message(channel, send_channel, content, channel_session, **kwargs): + # Stick the message onto the processing queue + Channel("chat-messages").send(room=channel_session['room'], message=content) + + @consumer("django.websocket.disconnect") + @websocket_channel_session + def ws_disconnect(channel, send_channel, channel_session, **kwargs): + Group("chat-%s" % channel_session['room']).discard(send_channel) + +Note that we could add messages onto the ``chat-messages`` channel from anywhere; +inside a View, inside another model's ``post_save`` signal, inside a management +command run via ``cron``. If we wanted to write a bot, too, we could put its +listening logic inside the ``chat-messages`` consumer, as every message would +pass through it. + +Next Steps +---------- + +That covers the basics of using Channels; you've seen not only how to use basic +channels, but also seen how they integrate with WebSockets, how to use groups +to manage logical sets of channels, and how Django's session and authentication +systems easily integrate with WebSockets. + +We recommend you read through the rest of the reference documentation to see +all of what Channels has to offer; in particular, you may want to look at +our :doc:`deployment` and :doc:`scaling` resources to get an idea of how to +design and run apps in production environments. diff --git a/docs/index.rst b/docs/index.rst index 65b34e5..3301360 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,4 +26,4 @@ Contents: integration-changes message-standards scaling - backend-requirements + backends From 3a4847887e7b49cefb61f153bb7ffdd2ac888141 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 12 Jul 2015 23:37:43 -0500 Subject: [PATCH 023/746] Fix invalid py2 syntax --- channels/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/decorators.py b/channels/decorators.py index 63ba59d..5a94850 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -5,7 +5,7 @@ from django.utils import six from channels import channel_backends, DEFAULT_CHANNEL_BACKEND -def consumer(self, *channels, alias=DEFAULT_CHANNEL_BACKEND): +def consumer(self, alias=DEFAULT_CHANNEL_BACKEND, *channels): """ Decorator that registers a function as a consumer. """ From 964c457df1f59c06880353db7f76a1cd6b974f1c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 12 Jul 2015 23:52:02 -0500 Subject: [PATCH 024/746] Make groups actually work --- channels/__init__.py | 2 +- channels/channel.py | 10 +++++----- channels/decorators.py | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/channels/__init__.py b/channels/__init__.py index 2b9a471..e5d43b7 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -17,4 +17,4 @@ from .hacks import monkeypatch_django monkeypatch_django() # Promote channel to top-level (down here to avoid circular import errs) -from .channel import Channel +from .channel import Channel, Group diff --git a/channels/channel.py b/channels/channel.py index 6724eba..faf5f3a 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -58,7 +58,7 @@ class Group(object): of the group after an expiry time (keep re-adding to keep them in). """ - def __init__(self, alias=DEFAULT_CHANNEL_BACKEND, channel_backend=None): + def __init__(self, name, alias=DEFAULT_CHANNEL_BACKEND, channel_backend=None): self.name = name if channel_backend: self.channel_backend = channel_backend @@ -66,13 +66,13 @@ class Group(object): self.channel_backend = channel_backends[alias] def add(self, channel): - self.channel_backend.add(self.name, channel) + self.channel_backend.group_add(self.name, channel) def discard(self, channel): - self.channel_backend.discard(self.name, channel) + self.channel_backend.group_discard(self.name, channel) def channels(self): - self.channel_backend.channels(self.name) + self.channel_backend.group_channels(self.name) def send(self, **kwargs): - self.channel_backend.send_group(self, self.name, kwargs) + self.channel_backend.send_group(self.name, kwargs) diff --git a/channels/decorators.py b/channels/decorators.py index 5a94850..584f66a 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -5,10 +5,12 @@ from django.utils import six from channels import channel_backends, DEFAULT_CHANNEL_BACKEND -def consumer(self, alias=DEFAULT_CHANNEL_BACKEND, *channels): +def consumer(*channels, **kwargs): """ Decorator that registers a function as a consumer. """ + # We can't put a kwarg after *args in py2 + alias = kwargs.get("alias", DEFAULT_CHANNEL_BACKEND) # Upconvert if you just pass in a string if isinstance(channels, six.string_types): channels = [channels] From 9d86aaad34658fb5d9114f063867d1640e91e296 Mon Sep 17 00:00:00 2001 From: HawkOwl Date: Mon, 13 Jul 2015 12:53:39 +0800 Subject: [PATCH 025/746] hawkie fixes --- channels/request.py | 6 +++--- channels/response.py | 5 ++++- channels/utils.py | 10 ++++++++-- docs/getting-started.rst | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/channels/request.py b/channels/request.py index d7cd4b2..7cbdcc5 100644 --- a/channels/request.py +++ b/channels/request.py @@ -8,10 +8,10 @@ def encode_request(request): """ # TODO: More stuff value = { - "GET": request.GET.items(), - "POST": request.POST.items(), + "GET": list(request.GET.items()), + "POST": list(request.POST.items()), "COOKIES": request.COOKIES, - "META": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, + "META": dict({k: v for k, v in request.META.items() if not k.startswith("wsgi")}), "path": request.path, "path_info": request.path_info, "method": request.method, diff --git a/channels/response.py b/channels/response.py index e790250..0719c47 100644 --- a/channels/response.py +++ b/channels/response.py @@ -1,4 +1,5 @@ from django.http import HttpResponse +from six import PY3 def encode_response(response): @@ -10,8 +11,10 @@ def encode_response(response): "content_type": getattr(response, "content_type", None), "content": response.content, "status_code": response.status_code, - "headers": response._headers.values(), + "headers": list(response._headers.values()), } + if PY3: + value["content"] = value["content"].decode('utf8') response.close() return value diff --git a/channels/utils.py b/channels/utils.py index a44fe9c..4a7fb65 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -1,6 +1,7 @@ import types from django.apps import apps +from six import PY3 def auto_import_consumers(): """ @@ -12,8 +13,13 @@ def auto_import_consumers(): try: __import__(module_name) except ImportError as e: - if "no module named %s" % submodule not in str(e).lower(): - raise + err = str(e).lower() + if PY3: + if "no module named '%s'" % (module_name,) not in err: + raise + else: + if "no module named %s" % (submodule,) not in err: + raise def name_that_thing(thing): diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 892c291..00c8db2 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -27,7 +27,7 @@ app, and put this in a ``consumers.py`` file in the app:: @consumer("django.wsgi.request") def http_consumer(response_channel, path, **kwargs): response = HttpResponse("Hello world! You asked for %s" % path) - Channel(response_channel).send(response.channel_encode()) + Channel(response_channel).send(**response.channel_encode()) The most important thing to note here is that, because things we send in messages must be JSON-serialisable, the request and response messages From d048527ce877ffa802c4e2227b08a6919cb81174 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 00:00:47 -0500 Subject: [PATCH 026/746] Update README --- README.rst | 81 +++--------------------------------------------------- 1 file changed, 4 insertions(+), 77 deletions(-) diff --git a/README.rst b/README.rst index 005785e..58568ab 100644 --- a/README.rst +++ b/README.rst @@ -5,81 +5,8 @@ This is a work-in-progress code branch of Django implemented as a third-party app, which aims to bring some asynchrony to Django and expand the options for code beyond the request-response model. -The proposal itself is detailed in `a very long Gist `_ -and there is discussion about it on `django-developers `_. +This is still **pre-alpha** software, and you use it at your own risk; the +API is not yet stable. -If you wish to use this in your own project, there's basic integration -instructions below - but be warned! This is not stable and may change massively -at any time! - -Integration ------------ - -Make sure you're running Django 1.8. This doesn't work with 1.7 (yet?) - -If you want to use WebSockets (and that's kind of the point) you'll need -``autobahn`` and ``twisted`` packages too. Python 3/asyncio support coming soon. - -``pip install channels`` and then add ``channels`` to the **TOP** -of your ``INSTALLED_APPS`` list (if it is not at the top you won't get the -new runserver command). - -You now have a ``runserver`` that actually runs a WSGI interface and a -worker in two different threads, ``runworker`` to run separate workers, -and ``runwsserver`` to run a Twisted-based WebSocket server. - -You should place consumers in either your ``views.py`` or a ``consumers.py``. -Here's an example of WebSocket consumers for basic chat:: - - import redis - from channels import Channel - - redis_conn = redis.Redis("localhost", 6379) - - @Channel.consumer("django.websocket.connect") - def ws_connect(path, send_channel, **kwargs): - redis_conn.sadd("chatroom", send_channel) - - @Channel.consumer("django.websocket.receive") - def ws_receive(channel, send_channel, content, binary, **kwargs): - # Ignore binary messages - if binary: - return - # Re-dispatch message - for channel in redis_conn.smembers("chatroom"): - Channel(channel).send(content=content, binary=False) - - @Channel.consumer("django.websocket.disconnect") - def ws_disconnect(channel, send_channel, **kwargs): - redis_conn.srem("chatroom", send_channel) - # NOTE: this does not clean up server crash disconnects, - # you'd want expiring keys here in real life. - -Alternately, you can just push some code outside of a normal view into a worker -thread:: - - - from django.shortcuts import render - from channels import Channel - - def my_view(request): - # Dispatch a task to run outside the req/response cycle - Channel("a_task_channel").send(value=3) - # Return a response - return render(request, "test.html") - - @Channel.consumer("a_task_channel") - def some_task(channel, value): - print "My value was %s from channel %s" % (value, channel) - -Limitations ------------ - -The ``runserver`` this command provides currently does not support static -media serving, streamed responses or autoreloading. - -In addition, this library is a preview and basically might do anything to your -code, or change drastically at any time. - -If you have opinions, please provide feedback via the appropriate -`django-developers thread `_. +Documentation, installation and getting started instructions are at +http://channels.readthedocs.org From da7cb9eb0a4f10336481656e9449209fb3537c79 Mon Sep 17 00:00:00 2001 From: HawkOwl Date: Mon, 13 Jul 2015 13:13:05 +0800 Subject: [PATCH 027/746] minor doc fix --- docs/getting-started.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 00c8db2..63b433d 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -175,7 +175,9 @@ to test your new code:: socket.onmessage = function(e) { alert(e.data); } - socket.send("hello world"); + socket.onopen = function() { + socket.send("hello world"); + } You should see an alert come back immediately saying "hello world" - your message has round-tripped through the server and come back to trigger the alert. From 8497b081a72e4d3683c542db9d4d7ce64c7a1bf0 Mon Sep 17 00:00:00 2001 From: HawkOwl Date: Mon, 13 Jul 2015 13:15:05 +0800 Subject: [PATCH 028/746] not required --- channels/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/request.py b/channels/request.py index 7cbdcc5..e7fd49f 100644 --- a/channels/request.py +++ b/channels/request.py @@ -11,7 +11,7 @@ def encode_request(request): "GET": list(request.GET.items()), "POST": list(request.POST.items()), "COOKIES": request.COOKIES, - "META": dict({k: v for k, v in request.META.items() if not k.startswith("wsgi")}), + "META": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, "path": request.path, "path_info": request.path_info, "method": request.method, From 7cfb3139dd21b84b1e23876881403177b9254888 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 00:36:41 -0500 Subject: [PATCH 029/746] Fix doc imports and a function signature --- docs/getting-started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 63b433d..6811b78 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -95,7 +95,7 @@ Now, that's taken care of adding and removing WebSocket send channels for the we're not going to store a history of messages or anything and just replay any message sent in to all connected clients. Here's all the code:: - from channels import Channel + from channels import Channel, Group from channels.decorators import consumer @consumer("django.websocket.connect", "django.websocket.keepalive") @@ -344,7 +344,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: from .models import ChatMessage @consumer("chat-messages") - def msg_consumer(room, message): + def msg_consumer(channel, room, message): # Save to model ChatMessage.objects.create(room=room, message=message) # Broadcast to listening sockets From 6e91ea0040d4d83b2a57ee2631d646371609948d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 15:51:10 -0500 Subject: [PATCH 030/746] Make the location of channels in INSTALLED_APPS not matter --- channels/__init__.py | 4 +--- channels/apps.py | 11 +++++++++++ channels/hacks.py | 5 +++++ channels/management/commands/runserver.py | 4 ++++ 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 channels/apps.py diff --git a/channels/__init__.py b/channels/__init__.py index e5d43b7..2ee5420 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -12,9 +12,7 @@ channel_backends = BackendManager( }) ) -# Ensure monkeypatching -from .hacks import monkeypatch_django -monkeypatch_django() +default_app_config = 'channels.apps.ChannelsConfig' # Promote channel to top-level (down here to avoid circular import errs) from .channel import Channel, Group diff --git a/channels/apps.py b/channels/apps.py new file mode 100644 index 0000000..2d31d8d --- /dev/null +++ b/channels/apps.py @@ -0,0 +1,11 @@ +from django.apps import AppConfig + +class ChannelsConfig(AppConfig): + + name = "channels" + verbose_name = "Channels" + + def ready(self): + # Do django monkeypatches + from .hacks import monkeypatch_django + monkeypatch_django() diff --git a/channels/hacks.py b/channels/hacks.py index e3003cf..e782eb6 100644 --- a/channels/hacks.py +++ b/channels/hacks.py @@ -19,6 +19,11 @@ def monkeypatch_django(): # Allow ResponseLater to propagate above handler BaseHandler.old_handle_uncaught_exception = BaseHandler.handle_uncaught_exception BaseHandler.handle_uncaught_exception = new_handle_uncaught_exception + # Ensure that the staticfiles version of runserver bows down to us + # This one is particularly horrible + from django.contrib.staticfiles.management.commands.runserver import Command as StaticRunserverCommand + from .management.commands.runserver import Command as RunserverCommand + StaticRunserverCommand.__bases__ = (RunserverCommand, ) def new_handle_uncaught_exception(self, request, resolver, exc_info): diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 7a795bc..50dca7e 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -30,6 +30,10 @@ class Command(RunserverCommand): worker = WorkerThread(self.channel_backend) worker.daemon = True worker.start() + # Note that this is the right one on the console + self.stdout.write("Worker thread running, channels enabled") + if self.channel_backend.local_only: + self.stdout.write("Local channel backend detected, no remote channels support") # Run the rest return super(Command, self).run(*args, **options) From 804a4c561efcd21b52f9392ba649f678c2875d7a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 17:58:52 -0700 Subject: [PATCH 031/746] Implement send_channel_session --- channels/decorators.py | 34 ++++++++++++++++++++++++++++++++++ docs/getting-started.rst | 22 +++++++++++----------- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/channels/decorators.py b/channels/decorators.py index 584f66a..6012938 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -1,5 +1,8 @@ import functools +import hashlib +from importlib import import_module +from django.conf import settings from django.utils import six from channels import channel_backends, DEFAULT_CHANNEL_BACKEND @@ -24,3 +27,34 @@ def consumer(*channels, **kwargs): # TODO: Sessions, auth + +def send_channel_session(func): + """ + Provides a session-like object called "channel_session" to consumers + as a message attribute that will auto-persist across consumers with + the same incoming "send_channel" value. + """ + @functools.wraps(func) + def inner(*args, **kwargs): + # Make sure there's a send_channel in kwargs + if "send_channel" not in kwargs: + raise ValueError("No send_channel sent to consumer; this decorator can only be used on messages containing it.") + # Turn the send_channel into a valid session key length thing. + # We take the last 24 bytes verbatim, as these are the random section, + # and then hash the remaining ones onto the start, and add a prefix + # TODO: See if there's a better way of doing this + session_key = "skt" + hashlib.md5(kwargs['send_channel'][:-24]).hexdigest()[:8] + kwargs['send_channel'][-24:] + # Make a session storage + session_engine = import_module(settings.SESSION_ENGINE) + session = session_engine.SessionStore(session_key=session_key) + # If the session does not already exist, save to force our session key to be valid + if not session.exists(session.session_key): + session.save() + kwargs['channel_session'] = session + # Run the consumer + result = func(*args, **kwargs) + # Persist session if needed (won't be saved if error happens) + if session.modified: + session.save() + return result + return inner diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 6811b78..0f94062 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -275,7 +275,7 @@ However, that session is based on cookies, and so follows the user round the site - it's great for information that should persist across all WebSocket and HTTP connections, but not great for information that is specific to a single WebSocket (such as "which chatroom should this socket be connected to"). For -this reason, Channels also provides a ``websocker_channel_session`` decorator, +this reason, Channels also provides a ``send_channel_session`` decorator, which adds a ``channel_session`` attribute to the message; this works just like the normal ``session`` attribute, and persists to the same storage, but varies per-channel rather than per-cookie. @@ -284,10 +284,10 @@ Let's use it now to build a chat server that expects you to pass a chatroom name in the path of your WebSocket request (we'll ignore auth for now):: from channels import Channel - from channels.decorators import consumer, websocket_channel_session + from channels.decorators import consumer, send_channel_session @consumer("django.websocket.connect") - @websocket_channel_session + @send_channel_session def ws_connect(channel, send_channel, path, channel_session, **kwargs): # Work out room name from path (ignore slashes) room = path.strip("/") @@ -296,17 +296,17 @@ name in the path of your WebSocket request (we'll ignore auth for now):: Group("chat-%s" % room).add(send_channel) @consumer("django.websocket.keepalive") - @websocket_channel_session + @send_channel_session def ws_add(channel, send_channel, channel_session, **kwargs): Group("chat-%s" % channel_session['room']).add(send_channel) @consumer("django.websocket.receive") - @websocket_channel_session + @send_channel_session def ws_message(channel, send_channel, content, channel_session, **kwargs): Group("chat-%s" % channel_session['room']).send(content=content) @consumer("django.websocket.disconnect") - @websocket_channel_session + @send_channel_session def ws_disconnect(channel, send_channel, channel_session, **kwargs): Group("chat-%s" % channel_session['room']).discard(send_channel) @@ -340,7 +340,7 @@ Let's see what that looks like, assuming we have a ChatMessage model with ``message`` and ``room`` fields:: from channels import Channel - from channels.decorators import consumer, websocket_channel_session + from channels.decorators import consumer, send_channel_session from .models import ChatMessage @consumer("chat-messages") @@ -351,7 +351,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: Group("chat-%s" % room).send(message) @consumer("django.websocket.connect") - @websocket_channel_session + @send_channel_session def ws_connect(channel, send_channel, path, channel_session, **kwargs): # Work out room name from path (ignore slashes) room = path.strip("/") @@ -360,18 +360,18 @@ have a ChatMessage model with ``message`` and ``room`` fields:: Group("chat-%s" % room).add(send_channel) @consumer("django.websocket.keepalive") - @websocket_channel_session + @send_channel_session def ws_add(channel, send_channel, channel_session, **kwargs): Group("chat-%s" % channel_session['room']).add(send_channel) @consumer("django.websocket.receive") - @websocket_channel_session + @send_channel_session def ws_message(channel, send_channel, content, channel_session, **kwargs): # Stick the message onto the processing queue Channel("chat-messages").send(room=channel_session['room'], message=content) @consumer("django.websocket.disconnect") - @websocket_channel_session + @send_channel_session def ws_disconnect(channel, send_channel, channel_session, **kwargs): Group("chat-%s" % channel_session['room']).discard(send_channel) From b5210e38c43bd81be317493760ca9c68c3ef1cd7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 19:15:44 -0700 Subject: [PATCH 032/746] Keepalive messages for WebSockets, and ! response chan prefix --- channels/interfaces/websocket_twisted.py | 29 ++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index 5e3dbe1..824a0cf 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -1,9 +1,11 @@ import django import time + +from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory from collections import deque from twisted.internet import reactor + from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND -from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory class InterfaceProtocol(WebSocketServerProtocol): @@ -20,7 +22,8 @@ class InterfaceProtocol(WebSocketServerProtocol): def onOpen(self): # Make sending channel - self.send_channel = Channel.new_name("django.websocket.send") + self.send_channel = Channel.new_name("!django.websocket.send") + self.last_keepalive = time.time() self.factory.protocols[self.send_channel] = self # Send news that this channel is open Channel("django.websocket.connect").send( @@ -67,6 +70,16 @@ class InterfaceProtocol(WebSocketServerProtocol): **self.request_info ) + def sendKeepalive(self): + """ + Sends a keepalive packet on the keepalive channel. + """ + Channel("django.websocket.keepalive").send( + send_channel = self.send_channel, + **self.request_info + ) + self.last_keepalive = time.time() + class InterfaceFactory(WebSocketServerFactory): """ @@ -106,6 +119,7 @@ class WebsocketTwistedInterface(object): self.factory.protocol = InterfaceProtocol reactor.listenTCP(self.port, self.factory) reactor.callInThread(self.backend_reader) + reactor.callLater(1, self.keepalive_sender) reactor.run() def backend_reader(self): @@ -130,3 +144,14 @@ class WebsocketTwistedInterface(object): continue # Deal with the message self.factory.dispatch_send(channel, message) + + def keepalive_sender(self): + """ + Sends keepalive messages for open WebSockets every + (channel_backend expiry / 2) seconds. + """ + expiry_window = int(self.channel_backend.expiry / 2) + for protocol in self.factory.protocols.values(): + if time.time() - protocol.last_keepalive > expiry_window: + protocol.sendKeepalive() + reactor.callLater(1, self.keepalive_sender) From 8a991056ba0f214c31475555b36391a206bb3454 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 19:56:49 -0700 Subject: [PATCH 033/746] Add first draft of deployment docs --- docs/deploying.rst | 142 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 docs/deploying.rst diff --git a/docs/deploying.rst b/docs/deploying.rst new file mode 100644 index 0000000..5a47e36 --- /dev/null +++ b/docs/deploying.rst @@ -0,0 +1,142 @@ +Deploying +========= + +Deploying applications using Channels requires a few more steps than a normal +Django WSGI application, but it's not very many. + +Firstly, remember that even if you have Channels installed, it's an entirely +optional part of Django. If you leave a project with the default settings +(no ``CHANNEL_BACKENDS``), it'll just run and work like a normal WSGI app. + +When you want to enable channels in production, you need to do three things: + +* Set up a channel backend +* Run worker servers +* Run interface servers + + +Setting up a channel backend +---------------------------- + +The first step is to set up a channel backend. If you followed the +:doc:`getting-started` guide, you will have ended up using the database +backend, which is great for getting started quickly in development but totally +unsuitable for production use; it will hammer your database with lots of +queries as it polls for new messages. + +Instead, take a look at the list of :doc:`backends`, and choose one that +fits your requirements (additionally, you could use a third-party pluggable +backend or write your own - that page also explains the interface and rules +a backend has to follow). + +Typically a channel backend will connect to one or more central servers that +serve as the communication layer - for example, the Redis backend connects +to a Redis server. All this goes into the ``CHANNEL_BACKENDS`` setting; +here's an example for a remote Redis server:: + + CHANNEL_BACKENDS = { + "default": { + "BACKEND": "channels.backends.redis.RedisChannelBackend", + "HOST": "redis-channel", + }, + } + +Make sure the same setting file is used across all your workers, interfaces +and WSGI apps; without it, they won't be able to talk to each other and things +will just fail to work. + + +Run worker servers +------------------ + +Because the work of running consumers is decoupled from the work of talking +to HTTP, WebSocket and other client connections, you need to run a cluster +of "worker servers" to do all the processing. + +Each server is single-threaded, so it's recommended you run around one per +core on each machine; it's safe to run as many concurrent workers on the same +machine as you like, as they don't open any ports (all they do is talk to +the channel backend). + +To run a worker server, just run:: + + python manage.py runworker + +Make sure you run this inside an init system or a program like supervisord that +can take care of restarting the process when it exits; the worker server has +no retry-on-exit logic, though it will absorb tracebacks from inside consumers +and forward them to stderr. + +Make sure you keep an eye on how busy your workers are; if they get overloaded, +requests will take longer and longer to return as the messages queue up +(until the expiry limit is reached, at which point HTTP connections will +start dropping). + +TODO: We should probably ship some kind of latency measuring tooling. + + +Run interface servers +--------------------- + +The final piece of the puzzle is the "interface servers", the processes that +do the work of taking incoming requests and loading them into the channels +system. + +You can just keep running your Django code as a WSGI app if you like, behind +something like uwsgi or gunicorn, and just use the WSGI interface as the app +you load into the server - just set it to use +``channels.interfaces.wsgi:WSGIHandler``. + +If you want to support WebSockets, however, you'll need to run another +interface server, as the WSGI protocol has no support for WebSockets. +Channels ships with an Autobahn-based WebSocket interface server +that should suit your needs; however, you could also use a third-party +interface server or write one yourself, as long as it follows the +:doc:`message-standards`. + +Notably, it's possible to combine more than one protocol into the same +interface server, and the one Channels ships with does just this; it can +optionally serve HTTP requests as well as WebSockets, though by default +it will just serve WebSockets and assume you're routing requests to the right +kind of server using your load balancer or reverse proxy. + +To run a normal WebSocket server, just run:: + + python manage.py runwsserver + +Like ``runworker``, you should place this inside an init system or something +like supervisord to ensure it is re-run if it exits unexpectedly. + +If you want to enable serving of normal HTTP requests as well, just run:: + + python manage.py runwsserver --accept-all + +This interface server is built on in-process asynchronous solutions +(Twisted for Python 2, and asyncio for Python 3) and so should be able to +handle a lot of simultaneous connections. That said, you should still plan to +run a cluster of them and load-balance between them; the per-connection memory +overhead is moderately high. + +Finally, note that it's entirely possible for interface servers to be written +in a language other than Python, though this would mean they could not take +advantage of the channel backend abstraction code and so they'd likely be +custom-written for a single channel backend. + + +Deploying new versions of code +------------------------------ + +One of the benefits of decoupling the client connection handling from work +processing is that it means you can run new code without dropping client +connections; this is especially useful for WebSockets. + +Just restart your workers when you have new code (by default, if you send +them SIGTERM they'll cleanly exit and finish running any in-process +consumers), and any queued messages or new connections will go to the new +workers. As long as the new code is session-compatible, you can even do staged +rollouts to make sure workers on new code aren't experiencing high error rates. + +There's no need to restart the WSGI or WebSocket interface servers unless +you've upgraded your version of Channels or changed any settings; +none of your code is used by them, and all middleware and code that can +customise requests is run on the consumers. From 8186aa22f72d67a4ee1115876b3d87c5b1a2c8c9 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 23:39:58 -0700 Subject: [PATCH 034/746] Fix deployment docs links --- docs/getting-started.rst | 2 +- docs/index.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 0f94062..29c509c 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -391,5 +391,5 @@ systems easily integrate with WebSockets. We recommend you read through the rest of the reference documentation to see all of what Channels has to offer; in particular, you may want to look at -our :doc:`deployment` and :doc:`scaling` resources to get an idea of how to +our :doc:`deploying` and :doc:`scaling` resources to get an idea of how to design and run apps in production environments. diff --git a/docs/index.rst b/docs/index.rst index 3301360..78e811c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ Contents: concepts installation getting-started + deploying integration-changes message-standards scaling From edd499938435c143d22050d286fbf26a00992832 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 23:41:19 -0700 Subject: [PATCH 035/746] Add some kind of license --- LICENSE | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5f4f225 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) Django Software Foundation and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of Django nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 3c001716b77106dd486f0452a75d7fd6a5fdd618 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 23:42:45 -0700 Subject: [PATCH 036/746] Remove last vestige of Channel.consumer --- docs/concepts.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index e4d0edd..8dfb7b6 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -62,7 +62,9 @@ How do we use channels? That's what a channel is, but how is Django using them? Well, inside Django you can connect a function to consume a channel, like so:: - @Channel.consumer("channel-name") + from channels.decorators import consumer + + @consumer("channel-name") def my_consumer(something, **kwargs): pass @@ -101,7 +103,7 @@ and a channel per client for responses (e.g. ``django.wsgi.respsonse.o4F2h2Fd``) with the response channel a property (``send_channel``) of the request message. Suddenly, a view is merely another example of a consumer:: - @Channel.consumer("django.wsgi.request") + @consumer("django.wsgi.request") def my_consumer(send_channel, **request_data): # Decode the request from JSON-compat to a full object django_request = Request.decode(request_data) @@ -181,7 +183,7 @@ set of channels (here, using Redis) to send updates to:: content=instance.content, ) - @Channel.consumer("django.websocket.connect") + @consumer("django.websocket.connect") def ws_connect(path, send_channel, **kwargs): # Add to reader set redis_conn.sadd("readers", send_channel) @@ -217,8 +219,8 @@ we don't need to; Channels has it built in, as a feature called Groups:: content=instance.content, ) - @Channel.consumer("django.websocket.connect") - @Channel.consumer("django.websocket.keepalive") + @consumer("django.websocket.connect") + @consumer("django.websocket.keepalive") def ws_connect(path, send_channel, **kwargs): # Add to reader group Group("liveblog").add(send_channel) From 5264e7a416bb563d5f226e7b99c5b82c906b3f4d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 13 Jul 2015 23:43:51 -0700 Subject: [PATCH 037/746] Typo --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 8dfb7b6..4c16092 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -99,7 +99,7 @@ A view takes a request and returns a response; a consumer takes a channel message and can write out zero to many other channel messages. Now, let's make a channel for requests (called ``django.wsgi.request``), -and a channel per client for responses (e.g. ``django.wsgi.respsonse.o4F2h2Fd``), +and a channel per client for responses (e.g. ``django.wsgi.response.o4F2h2Fd``), with the response channel a property (``send_channel``) of the request message. Suddenly, a view is merely another example of a consumer:: From feb3017c0e4ec7cc641b125f232fd5c8ac81fcfd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 4 Aug 2015 23:54:10 +0100 Subject: [PATCH 038/746] Fix request GET/POST format --- channels/request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/request.py b/channels/request.py index e7fd49f..bc5bf88 100644 --- a/channels/request.py +++ b/channels/request.py @@ -8,8 +8,8 @@ def encode_request(request): """ # TODO: More stuff value = { - "GET": list(request.GET.items()), - "POST": list(request.POST.items()), + "GET": list(request.GET.lists()), + "POST": list(request.POST.lists()), "COOKIES": request.COOKIES, "META": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, "path": request.path, From 15b54b48879c5548aeca8dd0fd7ae054bec20ecb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Sep 2015 00:09:01 -0700 Subject: [PATCH 039/746] Update bad example code --- docs/getting-started.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 29c509c..c216c0b 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -219,26 +219,26 @@ Fortunately, because Channels has standardised WebSocket event authentication, as well as using Django's session framework (which authentication relies on). -All we need to do is add the ``websocket_auth`` decorator to our views, +All we need to do is add the ``django_http_auth`` decorator to our views, and we'll get extra ``session`` and ``user`` keyword arguments we can use; let's make one where users can only chat to people with the same first letter of their username:: - from channels import Channel - from channels.decorators import consumer, websocket_auth + from channels import Channel, Group + from channels.decorators import consumer, django_http_auth @consumer("django.websocket.connect", "django.websocket.keepalive") - @websocket_auth + @django_http_auth def ws_add(channel, send_channel, user, **kwargs): Group("chat-%s" % user.username[0]).add(send_channel) @consumer("django.websocket.receive") - @websocket_auth + @django_http_auth def ws_message(channel, send_channel, content, user, **kwargs): Group("chat-%s" % user.username[0]).send(content=content) @consumer("django.websocket.disconnect") - @websocket_auth + @django_http_auth def ws_disconnect(channel, send_channel, user, **kwargs): Group("chat-%s" % user.username[0]).discard(send_channel) @@ -266,7 +266,7 @@ some other data store - sound familiar? This is what Django's session framework does for HTTP requests, only there it uses cookies as the lookup key rather than the ``send_channel``. -Now, as you saw above, you can use the ``websocket_auth`` decorator to get +Now, as you saw above, you can use the ``django_http_auth`` decorator to get both a ``user`` and a ``session`` variable in your message arguments - and, indeed, there is a ``websocket_session`` decorator that will just give you the ``session`` attribute. From 39a82e2a5c935230b762f80801f31e0d2ab4dec8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Sep 2015 00:09:14 -0700 Subject: [PATCH 040/746] Make worker print exceptions rather than explode --- channels/worker.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/channels/worker.py b/channels/worker.py index f39327a..1e7f2ea 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -1,4 +1,4 @@ -import time +import traceback class Worker(object): @@ -22,4 +22,7 @@ class Worker(object): consumer = self.channel_backend.registry.consumer_for_channel(channel) if self.callback: self.callback(channel, message) - consumer(channel=channel, **message) + try: + consumer(channel=channel, **message) + except: + traceback.print_exc() From 062035f992dc9b0a41f7c23b9f9ea1a5fa577613 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Sep 2015 00:09:38 -0700 Subject: [PATCH 041/746] Change to sending GET/POST as dicts of lists --- channels/interfaces/websocket_twisted.py | 1 + channels/request.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index 824a0cf..72e85a7 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -18,6 +18,7 @@ class InterfaceProtocol(WebSocketServerProtocol): self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] self.request_info = { "path": request.path, + "GET": request.params, } def onOpen(self): diff --git a/channels/request.py b/channels/request.py index bc5bf88..8dfcec6 100644 --- a/channels/request.py +++ b/channels/request.py @@ -8,8 +8,8 @@ def encode_request(request): """ # TODO: More stuff value = { - "GET": list(request.GET.lists()), - "POST": list(request.POST.lists()), + "GET": dict(request.GET.lists()), + "POST": dict(request.POST.lists()), "COOKIES": request.COOKIES, "META": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, "path": request.path, From a0830f88f7a18b9547c492de1c336737a9159821 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Sep 2015 00:09:53 -0700 Subject: [PATCH 042/746] Initial django session and auth decorators --- channels/decorators.py | 70 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/channels/decorators.py b/channels/decorators.py index 6012938..f19db2c 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -4,6 +4,7 @@ from importlib import import_module from django.conf import settings from django.utils import six +from django.contrib import auth from channels import channel_backends, DEFAULT_CHANNEL_BACKEND @@ -28,6 +29,75 @@ def consumer(*channels, **kwargs): # TODO: Sessions, auth +def http_session(func): + """ + Wraps a HTTP or WebSocket consumer (or any consumer of messages + that provides a "COOKIES" or "GET" attribute) to provide a "session" + attribute that behaves like request.session; that is, it's hung off of + a per-user session key that is saved in a cookie or passed as the + "session_key" GET parameter. + + It won't automatically create and set a session cookie for users who + don't have one - that's what SessionMiddleware is for, this is a simpler + read-only version for more low-level code. + + If a user does not have a session we can inflate, the "session" attribute will + be None, rather than an empty session you can write to. + """ + @functools.wraps(func) + def inner(*args, **kwargs): + if "COOKIES" not in kwargs and "GET" not in kwargs: + print kwargs + raise ValueError("No COOKIES or GET sent to consumer; this decorator can only be used on messages containing at least one.") + # Make sure there's a session key + session_key = None + if "GET" in kwargs: + session_key = kwargs['GET'].get("session_key") + if "COOKIES" in kwargs and session_key is None: + session_key = kwargs['COOKIES'].get(settings.SESSION_COOKIE_NAME) + # Make a session storage + if session_key: + session_engine = import_module(settings.SESSION_ENGINE) + session = session_engine.SessionStore(session_key=session_key) + else: + session = None + kwargs['session'] = session + # Run the consumer + result = func(*args, **kwargs) + # Persist session if needed (won't be saved if error happens) + if session is not None and session.modified: + session.save() + return result + return inner + + +def http_django_auth(func): + """ + Wraps a HTTP or WebSocket consumer (or any consumer of messages + that provides a "COOKIES" attribute) to provide both a "session" + attribute and a "user" attibute, like AuthMiddleware does. + + This runs http_session() to get a session to hook auth off of. + If the user does not have a session cookie set, both "session" + and "user" will be None. + """ + @http_session + @functools.wraps(func) + def inner(*args, **kwargs): + # If we didn't get a session, then we don't get a user + if kwargs['session'] is None: + kwargs['user'] = None + # Otherwise, be a bit naughty and make a fake Request with just + # a "session" attribute (later on, perhaps refactor contrib.auth to + # pass around session rather than request) + else: + fake_request = type("FakeRequest", (object, ), {"session": kwargs['session']}) + kwargs['user'] = auth.get_user(fake_request) + # Run the consumer + return func(*args, **kwargs) + return inner + + def send_channel_session(func): """ Provides a session-like object called "channel_session" to consumers From 29eb75326fb8094e44a95e1e5c29d35b25e7930d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 3 Sep 2015 00:06:58 -0700 Subject: [PATCH 043/746] Make session decorator work right with ?session_key= --- channels/decorators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/channels/decorators.py b/channels/decorators.py index f19db2c..367681a 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -47,12 +47,14 @@ def http_session(func): @functools.wraps(func) def inner(*args, **kwargs): if "COOKIES" not in kwargs and "GET" not in kwargs: - print kwargs raise ValueError("No COOKIES or GET sent to consumer; this decorator can only be used on messages containing at least one.") # Make sure there's a session key session_key = None if "GET" in kwargs: - session_key = kwargs['GET'].get("session_key") + try: + session_key = kwargs['GET'].get("session_key", [])[0] + except IndexError: + pass if "COOKIES" in kwargs and session_key is None: session_key = kwargs['COOKIES'].get(settings.SESSION_COOKIE_NAME) # Make a session storage From 832809ca25bc7b42ae7d92ee1a2fcde8f4ea9eb9 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 3 Sep 2015 00:07:30 -0700 Subject: [PATCH 044/746] Stop using @consumer, move to explicit routing --- channels/backends/base.py | 4 +- channels/backends/database.py | 4 +- channels/backends/redis_py.py | 4 +- channels/consumer_registry.py | 28 +++++-- channels/decorators.py | 20 ----- docs/getting-started.rst | 145 +++++++++++++++++++++++----------- 6 files changed, 127 insertions(+), 78 deletions(-) diff --git a/channels/backends/base.py b/channels/backends/base.py index 7a99568..84e874e 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -20,8 +20,8 @@ class BaseChannelBackend(object): # Causes errors if you try to run workers/interfaces separately with it. local_only = False - def __init__(self, expiry=60): - self.registry = ConsumerRegistry() + def __init__(self, routing, expiry=60): + self.registry = ConsumerRegistry(routing) self.expiry = expiry def send(self, channel, message): diff --git a/channels/backends/database.py b/channels/backends/database.py index ed59a9e..504f3ad 100644 --- a/channels/backends/database.py +++ b/channels/backends/database.py @@ -16,8 +16,8 @@ class DatabaseChannelBackend(BaseChannelBackend): multiple processes fine, but it's going to be pretty bad at throughput. """ - def __init__(self, expiry=60, db_alias=DEFAULT_DB_ALIAS): - super(DatabaseChannelBackend, self).__init__(expiry) + def __init__(self, routing, expiry=60, db_alias=DEFAULT_DB_ALIAS): + super(DatabaseChannelBackend, self).__init__(routing=routing, expiry=expiry) self.db_alias = db_alias @property diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 11d3834..5c8671c 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -13,8 +13,8 @@ class RedisChannelBackend(BaseChannelBackend): multiple processes fine, but it's going to be pretty bad at throughput. """ - def __init__(self, expiry=60, host="localhost", port=6379, prefix="django-channels:"): - super(RedisChannelBackend, self).__init__(expiry) + def __init__(self, routing, expiry=60, host="localhost", port=6379, prefix="django-channels:"): + super(RedisChannelBackend, self).__init__(routing=routing, expiry=expiry) self.host = host self.port = port self.prefix = prefix diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index 5e80729..cefe4c6 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -1,7 +1,6 @@ -import functools - +import importlib from django.utils import six - +from django.core.exceptions import ImproperlyConfigured from .utils import name_that_thing @@ -13,13 +12,32 @@ class ConsumerRegistry(object): Generally this is attached to a backend instance as ".registry" """ - def __init__(self): + def __init__(self, routing=None): self.consumers = {} + # Initialise with any routing that was passed in + if routing: + # If the routing was a string, import it + if isinstance(routing, six.string_types): + module_name, variable_name = routing.rsplit(".", 1) + try: + routing = getattr(importlib.import_module(module_name), variable_name) + except (ImportError, AttributeError): + raise ImproperlyConfigured("Cannot import channel routing %r" % routing) + # Load consumers into us + for channel, handler in routing.items(): + self.add_consumer(handler, [channel]) def add_consumer(self, consumer, channels): - # Upconvert if you just pass in a string + # Upconvert if you just pass in a string for channels if isinstance(channels, six.string_types): channels = [channels] + # Import any consumer referenced as string + if isinstance(consumer, six.string_types): + module_name, variable_name = consumer.rsplit(".", 1) + try: + consumer = getattr(importlib.import_module(module_name), variable_name) + except (ImportError, AttributeError): + raise ImproperlyConfigured("Cannot import consumer %r" % consumer) # Register on each channel, checking it's unique for channel in channels: if channel in self.consumers: diff --git a/channels/decorators.py b/channels/decorators.py index 367681a..bd70058 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -9,26 +9,6 @@ from django.contrib import auth from channels import channel_backends, DEFAULT_CHANNEL_BACKEND -def consumer(*channels, **kwargs): - """ - Decorator that registers a function as a consumer. - """ - # We can't put a kwarg after *args in py2 - alias = kwargs.get("alias", DEFAULT_CHANNEL_BACKEND) - # Upconvert if you just pass in a string - if isinstance(channels, six.string_types): - channels = [channels] - # Get the channel - channel_backend = channel_backends[alias] - # Return a function that'll register whatever it wraps - def inner(func): - channel_backend.registry.add_consumer(func, channels) - return func - return inner - - -# TODO: Sessions, auth - def http_session(func): """ Wraps a HTTP or WebSocket consumer (or any consumer of messages diff --git a/docs/getting-started.rst b/docs/getting-started.rst index c216c0b..9a79b80 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -17,14 +17,15 @@ channel if you don't provide another consumer that listens to it - remember, only one consumer can listen to any given channel. As a very basic example, let's write a consumer that overrides the built-in -handling and handles every HTTP request directly. Make a new project, a new -app, and put this in a ``consumers.py`` file in the app:: +handling and handles every HTTP request directly. This isn't something you'd +usually do in a project, but it's a good illustration of how channels +now underlie every part of Django. + +Make a new project, a new app, and put this in a ``consumers.py`` file in the app:: from channels import Channel - from channels.decorators import consumer from django.http import HttpResponse - @consumer("django.wsgi.request") def http_consumer(response_channel, path, **kwargs): response = HttpResponse("Hello world! You asked for %s" % path) Channel(response_channel).send(**response.channel_encode()) @@ -36,6 +37,30 @@ are in a key-value format. There are ``channel_decode()`` and but here we just take two of the request variables directly as keyword arguments for simplicity. +Now, go into your ``settings.py`` file, and set up a channel backend; by default, +Django will just use a local backend and route HTTP requests to the normal +URL resolver (we'll come back to backends in a minute). + +For now, we want to override the *channel routing* so that, rather than going +to the URL resolver and our normal view stack, all HTTP requests go to our +custom consumer we wrote above. Here's what that looks like:: + + CHANNEL_BACKENDS = { + "default": { + "BACKEND": "channels.backends.database.DatabaseChannelBackend", + "ROUTING": { + "django.wsgi.request": "myproject.myapp.consumers.http_consumer", + }, + }, + } + +As you can see, this is a little like Django's ``DATABASES`` setting; there are +named channel backends, with a default one called ``default``. Each backend +needs a class specified which powers it - we'll come to the options there later - +and a routing scheme, which can either be defined directly as a dict or as +a string pointing to a dict in another file (if you'd rather keep it outside +settings). + If you start up ``python manage.py runserver`` and go to ``http://localhost:8000``, you'll see that, rather than a default Django page, you get the Hello World response, so things are working. If you don't see @@ -44,13 +69,23 @@ a response, check you :doc:`installed Channels correctly `. Now, that's not very exciting - raw HTTP responses are something Django can do any time. Let's try some WebSockets, and make a basic chat server! -Delete that consumer from above - we'll need the normal Django view layer to -serve templates later - and make this WebSocket consumer instead:: +Delete that consumer and its routing - we'll want the normal Django view layer to +serve HTTP requests from now on - and make this WebSocket consumer instead:: - @consumer("django.websocket.connect") - def ws_connect(channel, send_channel, **kwargs): + def ws_add(channel, send_channel, **kwargs): Group("chat").add(send_channel) +Hook it up to the ``django.websocket.connect`` channel like this:: + + CHANNEL_BACKENDS = { + "default": { + "BACKEND": "channels.backends.database.DatabaseChannelBackend", + "ROUTING": { + "django.websocket.connect": "myproject.myapp.consumers.ws_add", + }, + }, + } + Now, let's look at what this is doing. It's tied to the ``django.websocket.connect`` channel, which means that it'll get a message whenever a new WebSocket connection is opened by a client. @@ -71,22 +106,25 @@ so we can hook that up to re-add the channel (it's safe to add the channel to a group it's already in - similarly, it's safe to discard a channel from a group it's not in):: - @consumer("django.websocket.keepalive") + # Connected to django.websocket.keepalive def ws_keepalive(channel, send_channel, **kwargs): Group("chat").add(send_channel) Of course, this is exactly the same code as the ``connect`` handler, so let's -just combine them:: +just route both channels to the same consumer:: - @consumer("django.websocket.connect", "django.websocket.keepalive") - def ws_add(channel, send_channel, **kwargs): - Group("chat").add(send_channel) + ... + "ROUTING": { + "django.websocket.connect": "myproject.myapp.consumers.ws_add", + "django.websocket.keepalive": "myproject.myapp.consumers.ws_add", + }, + ... And, even though channels will expire out, let's add an explicit ``disconnect`` handler to clean up as people disconnect (most channels will cleanly disconnect and get this called):: - @consumer("django.websocket.disconnect") + # Connected to django.websocket.disconnect def ws_disconnect(channel, send_channel, **kwargs): Group("chat").discard(send_channel) @@ -96,20 +134,33 @@ we're not going to store a history of messages or anything and just replay any message sent in to all connected clients. Here's all the code:: from channels import Channel, Group - from channels.decorators import consumer - @consumer("django.websocket.connect", "django.websocket.keepalive") + # Connected to django.websocket.connect and django.websocket.keepalive def ws_add(channel, send_channel, **kwargs): Group("chat").add(send_channel) - @consumer("django.websocket.receive") + # Connected to django.websocket.receive def ws_message(channel, send_channel, content, **kwargs): Group("chat").send(content=content) - @consumer("django.websocket.disconnect") + # Connected to django.websocket.disconnect def ws_disconnect(channel, send_channel, **kwargs): Group("chat").discard(send_channel) +And what our routing should look like in ``settings.py``:: + + CHANNEL_BACKENDS = { + "default": { + "BACKEND": "channels.backends.database.DatabaseChannelBackend", + "ROUTING": { + "django.websocket.connect": "myproject.myapp.consumers.ws_add", + "django.websocket.keepalive": "myproject.myapp.consumers.ws_add", + "django.websocket.receive": "myproject.myapp.consumers.ws_message", + "django.websocket.disconnect": "myproject.myapp.consumers.ws_disconnect", + }, + }, + } + With all that code in your ``consumers.py`` file, you now have a working set of a logic for a chat server. All you need to do now is get it deployed, and as we'll see, that's not too hard. @@ -132,23 +183,11 @@ process; this is enough to serve normal WSGI style requests (``runserver`` is just running a WSGI interface and a worker in two separate threads), but now we want WebSocket support we'll need a separate process to keep things clean. -For simplicity, we'll use the database channel backend - this uses two tables +If you notice, in the example above we switched our default backend to the +database channel backend. This uses two tables in the database to do message handling, and isn't particularly fast but -requires no extra dependencies. Put this in your ``settings.py`` file:: - - CHANNEL_BACKENDS = { - "default": { - "BACKEND": "channels.backends.database.DatabaseChannelBackend", - }, - } - -As you can see, the format is quite similar to the ``DATABASES`` setting in -Django, but for this case much simpler, as it just uses the default database -(you can set which alias it uses with the ``DB_ALIAS`` key). - -In production, we'd recommend you use something like the Redis channel backend; -you can :doc:`read about the backends ` and see how to set them up -and their performance considerations if you wish. +requires no extra dependencies. When you deploy to production, you'll want to +use a backend like the Redis backend that has much better throughput. The second thing, once we have a networked channel backend set up, is to make sure we're running the WebSocket interface server. Even in development, we need @@ -182,7 +221,9 @@ to test your new code:: You should see an alert come back immediately saying "hello world" - your message has round-tripped through the server and come back to trigger the alert. You can open another tab and do the same there if you like, and both tabs will -receive the message and show an alert. +receive the message and show an alert, as any incoming message is sent to the +``chat`` group by the ``ws_message`` consumer, and both your tabs will have +been put into the ``chat`` group when they connected. Feel free to put some calls to ``print`` in your handler functions too, if you like, so you can understand when they're called. If you run three or four @@ -204,11 +245,10 @@ user ID, I can just auto-add that channel to all the relevant groups (mentions of that user, for example). Handily, as WebSockets start off using the HTTP protocol, they have a lot of -familiar features, including a path, GET parameters, and cookies. Notably, -the cookies allow us to perform authentication using the same methods the -normal Django middleware does. Middleware only runs on requests to views, -however, and not on raw consumer calls; it's tailored to work with single -HTTP requests, and so we need a different solution to authenticate WebSockets. +familiar features, including a path, GET parameters, and cookies. We'd like to +use these to hook into the familiar Django session and authentication systems; +after all, WebSockets are no good unless we can identify who they belong to +and do things securely. In addition, we don't want the interface servers storing data or trying to run authentication; they're meant to be simple, lean, fast processes without much @@ -217,7 +257,11 @@ state, and so we'll need to do our authentication inside our consumer functions. Fortunately, because Channels has standardised WebSocket event :doc:`message-standards`, it ships with decorators that help you with authentication, as well as using Django's session framework (which authentication -relies on). +relies on). Channels can use Django sessions either from cookies (if you're running your websocket +server on the same port as your main site, which requires a reverse proxy that +understands WebSockets), or from a ``session_key`` GET parameter, which +is much more portable, and works in development where you need to run a separate +WebSocket server (by default, on port 9000). All we need to do is add the ``django_http_auth`` decorator to our views, and we'll get extra ``session`` and ``user`` keyword arguments we can use; @@ -225,26 +269,33 @@ let's make one where users can only chat to people with the same first letter of their username:: from channels import Channel, Group - from channels.decorators import consumer, django_http_auth + from channels.decorators import django_http_auth - @consumer("django.websocket.connect", "django.websocket.keepalive") @django_http_auth def ws_add(channel, send_channel, user, **kwargs): Group("chat-%s" % user.username[0]).add(send_channel) - @consumer("django.websocket.receive") @django_http_auth def ws_message(channel, send_channel, content, user, **kwargs): Group("chat-%s" % user.username[0]).send(content=content) - @consumer("django.websocket.disconnect") @django_http_auth def ws_disconnect(channel, send_channel, user, **kwargs): Group("chat-%s" % user.username[0]).discard(send_channel) -(Note that we always end consumers with ``**kwargs``; this is to save us +Now, when we connect to the WebSocket we'll have to remember to provide the +Django session ID as part of the URL, like this:: + + socket = new WebSocket("ws://127.0.0.1:9000/?session_key=abcdefg"); + +You can get the current session key in a template with ``{{ request.session.session_key }}``. +Note that Channels can't work with signed cookie sessions - since only HTTP +responses can set cookies, it needs a backend it can write to separately to +store state. + +(Also note that we always end consumers with ``**kwargs``; this is to save us from writing out all variables we might get sent and to allow forwards-compatibility -with any additions to the message formats in the future) +with any additions to the message formats in the future.) Persisting Data --------------- From 16a0e73c6ce262bb2022b1ff59eef94a0380d517 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 3 Sep 2015 00:07:57 -0700 Subject: [PATCH 045/746] Make runserver work with autoreload --- channels/management/commands/runserver.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 50dca7e..ddc1731 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -18,24 +18,26 @@ class Command(RunserverCommand): return WSGIInterface(self.channel_backend) def run(self, *args, **options): - # Force disable reloader for now - options['use_reloader'] = False + # Run the rest + return super(Command, self).run(*args, **options) + + def inner_run(self, *args, **options): # Check a handler is registered for http reqs self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] auto_import_consumers() if not self.channel_backend.registry.consumer_for_channel("django.wsgi.request"): # Register the default one self.channel_backend.registry.add_consumer(UrlConsumer(), ["django.wsgi.request"]) - # Launch a worker thread - worker = WorkerThread(self.channel_backend) - worker.daemon = True - worker.start() # Note that this is the right one on the console self.stdout.write("Worker thread running, channels enabled") if self.channel_backend.local_only: self.stdout.write("Local channel backend detected, no remote channels support") - # Run the rest - return super(Command, self).run(*args, **options) + # Launch a worker thread + worker = WorkerThread(self.channel_backend) + worker.daemon = True + worker.start() + # Run rest of inner run + super(Command, self).inner_run(*args, **options) class WorkerThread(threading.Thread): From 4c65ad4c0998911b56d0a9b87efe547282ae0548 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 3 Sep 2015 00:08:05 -0700 Subject: [PATCH 046/746] Remove done comment --- channels/interfaces/websocket_twisted.py | 1 - 1 file changed, 1 deletion(-) diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index 72e85a7..e378b61 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -129,7 +129,6 @@ class WebsocketTwistedInterface(object): """ while True: channels = self.factory.send_channels() - # TODO: Send keepalives # Quit if reactor is stopping if not reactor.running: return From 9b92eec43a0b06ef33b1c9aa399369082c9b8a5a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 7 Sep 2015 15:04:10 -0500 Subject: [PATCH 047/746] Fix request/response compatability --- channels/request.py | 15 +++++++++++++-- channels/response.py | 5 ++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/channels/request.py b/channels/request.py index 8dfcec6..7e5a71e 100644 --- a/channels/request.py +++ b/channels/request.py @@ -1,5 +1,7 @@ from django.http import HttpRequest from django.utils.datastructures import MultiValueDict +from django.http.request import QueryDict +from django.conf import settings def encode_request(request): @@ -25,8 +27,8 @@ def decode_request(value): Decodes a request JSONish value to a HttpRequest object. """ request = HttpRequest() - request.GET = MultiValueDict(value['GET']) - request.POST = MultiValueDict(value['POST']) + request.GET = CustomQueryDict(value['GET']) + request.POST = CustomQueryDict(value['POST']) request.COOKIES = value['COOKIES'] request.META = value['META'] request.path = value['path'] @@ -34,3 +36,12 @@ def decode_request(value): request.path_info = value['path_info'] request.response_channel = value['response_channel'] return request + + +class CustomQueryDict(QueryDict): + """ + Custom override of QueryDict that sets things directly. + """ + + def __init__(self, values): + MultiValueDict.__init__(self, values) diff --git a/channels/response.py b/channels/response.py index c8abdd7..2b8d761 100644 --- a/channels/response.py +++ b/channels/response.py @@ -1,4 +1,5 @@ from django.http import HttpResponse +from django.http.cookie import SimpleCookie from six import PY3 @@ -6,12 +7,12 @@ def encode_response(response): """ Encodes a response to JSON-compatible datastructures """ - # TODO: Entirely useful things like cookies value = { "content_type": getattr(response, "content_type", None), "content": response.content, "status_code": response.status_code, "headers": list(response._headers.values()), + "cookies": {k: v.output(header="") for k, v in response.cookies.items()} } if PY3: value["content"] = value["content"].decode('utf8') @@ -28,6 +29,8 @@ def decode_response(value): content_type = value['content_type'], status = value['status_code'], ) + for cookie in value['cookies'].values(): + response.cookies.load(cookie) response._headers = {k.lower: (k, v) for k, v in value['headers']} return response From 48d6f63fb29bb274432e7bdcd6c66cda2d620ff9 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 8 Sep 2015 01:04:36 -0500 Subject: [PATCH 048/746] Change to consumers taking a single "message" argument --- channels/adapters.py | 29 ++--- channels/channel.py | 21 +++- channels/decorators.py | 51 +++++---- channels/interfaces/websocket_twisted.py | 42 +++---- channels/interfaces/wsgi.py | 6 +- channels/message.py | 18 +++ channels/request.py | 4 +- channels/worker.py | 11 +- docs/concepts.rst | 18 +-- docs/getting-started.rst | 138 ++++++++++++----------- docs/message-standards.rst | 4 +- 11 files changed, 185 insertions(+), 157 deletions(-) create mode 100644 channels/message.py diff --git a/channels/adapters.py b/channels/adapters.py index cf365cd..add1339 100644 --- a/channels/adapters.py +++ b/channels/adapters.py @@ -3,7 +3,7 @@ import functools from django.core.handlers.base import BaseHandler from django.http import HttpRequest, HttpResponse -from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND +from channels import Channel class UrlConsumer(object): @@ -15,13 +15,13 @@ class UrlConsumer(object): self.handler = BaseHandler() self.handler.load_middleware() - def __call__(self, channel, **kwargs): - request = HttpRequest.channel_decode(kwargs) + def __call__(self, message): + request = HttpRequest.channel_decode(message.content) try: response = self.handler.get_response(request) except HttpResponse.ResponseLater: return - Channel(request.response_channel).send(**response.channel_encode()) + message.reply_channel.send(response.channel_encode()) def view_producer(channel_name): @@ -30,24 +30,19 @@ def view_producer(channel_name): and abandons the response (with an exception the Worker will catch) """ def producing_view(request): - Channel(channel_name).send(**request.channel_encode()) + Channel(channel_name).send(request.channel_encode()) raise HttpResponse.ResponseLater() return producing_view -def view_consumer(channel_name, alias=DEFAULT_CHANNEL_BACKEND): +def view_consumer(func): """ Decorates a normal Django view to be a channel consumer. Does not run any middleware """ - def inner(func): - @functools.wraps(func) - def consumer(channel, **kwargs): - request = HttpRequest.channel_decode(kwargs) - response = func(request) - Channel(request.response_channel).send(**response.channel_encode()) - # Get the channel layer and register - channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - channel_backend.registry.add_consumer(consumer, [channel_name]) - return func - return inner + @functools.wraps(func) + def consumer(message): + request = HttpRequest.channel_decode(message.content) + response = func(request) + message.reply_channel.send(response.channel_encode()) + return func diff --git a/channels/channel.py b/channels/channel.py index faf5f3a..0e047b5 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -26,11 +26,13 @@ class Channel(object): else: self.channel_backend = channel_backends[alias] - def send(self, **kwargs): + def send(self, content): """ - Send a message over the channel, taken from the kwargs. + Send a message over the channel - messages are always dicts. """ - self.channel_backend.send(self.name, kwargs) + if not isinstance(content, dict): + raise ValueError("You can only send dicts as content on channels.") + self.channel_backend.send(self.name, content) @classmethod def new_name(self, prefix): @@ -51,6 +53,9 @@ class Channel(object): from channels.adapters import view_producer return view_producer(self.name) + def __str__(self): + return self.name + class Group(object): """ @@ -66,13 +71,19 @@ class Group(object): self.channel_backend = channel_backends[alias] def add(self, channel): + if isinstance(channel, Channel): + channel = channel.name self.channel_backend.group_add(self.name, channel) def discard(self, channel): + if isinstance(channel, Channel): + channel = channel.name self.channel_backend.group_discard(self.name, channel) def channels(self): self.channel_backend.group_channels(self.name) - def send(self, **kwargs): - self.channel_backend.send_group(self.name, kwargs) + def send(self, content): + if not isinstance(content, dict): + raise ValueError("You can only send dicts as content on channels.") + self.channel_backend.send_group(self.name, content) diff --git a/channels/decorators.py b/channels/decorators.py index bd70058..bc5f116 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -25,27 +25,27 @@ def http_session(func): be None, rather than an empty session you can write to. """ @functools.wraps(func) - def inner(*args, **kwargs): - if "COOKIES" not in kwargs and "GET" not in kwargs: + def inner(message, *args, **kwargs): + if "COOKIES" not in message.content and "GET" not in message.content: raise ValueError("No COOKIES or GET sent to consumer; this decorator can only be used on messages containing at least one.") # Make sure there's a session key session_key = None - if "GET" in kwargs: + if "GET" in message.content: try: - session_key = kwargs['GET'].get("session_key", [])[0] + session_key = message.content['GET'].get("session_key", [])[0] except IndexError: pass - if "COOKIES" in kwargs and session_key is None: - session_key = kwargs['COOKIES'].get(settings.SESSION_COOKIE_NAME) + if "COOKIES" in message.content and session_key is None: + session_key = message.content['COOKIES'].get(settings.SESSION_COOKIE_NAME) # Make a session storage if session_key: session_engine = import_module(settings.SESSION_ENGINE) session = session_engine.SessionStore(session_key=session_key) else: session = None - kwargs['session'] = session + message.session = session # Run the consumer - result = func(*args, **kwargs) + result = func(message, *args, **kwargs) # Persist session if needed (won't be saved if error happens) if session is not None and session.modified: session.save() @@ -65,46 +65,49 @@ def http_django_auth(func): """ @http_session @functools.wraps(func) - def inner(*args, **kwargs): + def inner(message, *args, **kwargs): # If we didn't get a session, then we don't get a user - if kwargs['session'] is None: - kwargs['user'] = None + if not hasattr(message, "session"): + raise ValueError("Did not see a session to get auth from") + if message.session is None: + message.user = None # Otherwise, be a bit naughty and make a fake Request with just # a "session" attribute (later on, perhaps refactor contrib.auth to # pass around session rather than request) else: - fake_request = type("FakeRequest", (object, ), {"session": kwargs['session']}) - kwargs['user'] = auth.get_user(fake_request) + fake_request = type("FakeRequest", (object, ), {"session": message.session}) + message.user = auth.get_user(fake_request) # Run the consumer - return func(*args, **kwargs) + return func(message, *args, **kwargs) return inner -def send_channel_session(func): +def channel_session(func): """ Provides a session-like object called "channel_session" to consumers as a message attribute that will auto-persist across consumers with - the same incoming "send_channel" value. + the same incoming "reply_channel" value. """ @functools.wraps(func) - def inner(*args, **kwargs): - # Make sure there's a send_channel in kwargs - if "send_channel" not in kwargs: - raise ValueError("No send_channel sent to consumer; this decorator can only be used on messages containing it.") - # Turn the send_channel into a valid session key length thing. + def inner(message, *args, **kwargs): + # Make sure there's a reply_channel in kwargs + if not message.reply_channel: + raise ValueError("No reply_channel sent to consumer; this decorator can only be used on messages containing it.") + # Turn the reply_channel into a valid session key length thing. # We take the last 24 bytes verbatim, as these are the random section, # and then hash the remaining ones onto the start, and add a prefix # TODO: See if there's a better way of doing this - session_key = "skt" + hashlib.md5(kwargs['send_channel'][:-24]).hexdigest()[:8] + kwargs['send_channel'][-24:] + reply_name = message.reply_channel.name + session_key = "skt" + hashlib.md5(reply_name[:-24]).hexdigest()[:8] + reply_name[-24:] # Make a session storage session_engine = import_module(settings.SESSION_ENGINE) session = session_engine.SessionStore(session_key=session_key) # If the session does not already exist, save to force our session key to be valid if not session.exists(session.session_key): session.save() - kwargs['channel_session'] = session + message.channel_session = session # Run the consumer - result = func(*args, **kwargs) + result = func(message, *args, **kwargs) # Persist session if needed (won't be saved if error happens) if session.modified: session.save() diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index e378b61..8aa0b31 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -23,30 +23,26 @@ class InterfaceProtocol(WebSocketServerProtocol): def onOpen(self): # Make sending channel - self.send_channel = Channel.new_name("!django.websocket.send") + self.reply_channel = Channel.new_name("!django.websocket.send") + self.request_info["reply_channel"] = self.reply_channel self.last_keepalive = time.time() - self.factory.protocols[self.send_channel] = self + self.factory.protocols[self.reply_channel] = self # Send news that this channel is open - Channel("django.websocket.connect").send( - send_channel = self.send_channel, - **self.request_info - ) + Channel("django.websocket.connect").send(self.request_info) def onMessage(self, payload, isBinary): if isBinary: - Channel("django.websocket.receive").send( - send_channel = self.send_channel, + Channel("django.websocket.receive").send(dict( + self.request_info, content = payload, binary = True, - **self.request_info - ) + )) else: - Channel("django.websocket.receive").send( - send_channel = self.send_channel, + Channel("django.websocket.receive").send(dict( + self.request_info, content = payload.decode("utf8"), binary = False, - **self.request_info - ) + )) def serverSend(self, content, binary=False, **kwargs): """ @@ -64,21 +60,15 @@ class InterfaceProtocol(WebSocketServerProtocol): self.sendClose() def onClose(self, wasClean, code, reason): - if hasattr(self, "send_channel"): - del self.factory.protocols[self.send_channel] - Channel("django.websocket.disconnect").send( - send_channel = self.send_channel, - **self.request_info - ) + if hasattr(self, "reply_channel"): + del self.factory.protocols[self.reply_channel] + Channel("django.websocket.disconnect").send(self.request_info) def sendKeepalive(self): """ Sends a keepalive packet on the keepalive channel. """ - Channel("django.websocket.keepalive").send( - send_channel = self.send_channel, - **self.request_info - ) + Channel("django.websocket.keepalive").send(self.request_info) self.last_keepalive = time.time() @@ -94,7 +84,7 @@ class InterfaceFactory(WebSocketServerFactory): super(InterfaceFactory, self).__init__(*args, **kwargs) self.protocols = {} - def send_channels(self): + def reply_channels(self): return self.protocols.keys() def dispatch_send(self, channel, message): @@ -128,7 +118,7 @@ class WebsocketTwistedInterface(object): Run in a separate thread; reads messages from the backend. """ while True: - channels = self.factory.send_channels() + channels = self.factory.reply_channels() # Quit if reactor is stopping if not reactor.running: return diff --git a/channels/interfaces/wsgi.py b/channels/interfaces/wsgi.py index 60b19c2..8df49c0 100644 --- a/channels/interfaces/wsgi.py +++ b/channels/interfaces/wsgi.py @@ -15,7 +15,7 @@ class WSGIInterface(WSGIHandler): super(WSGIInterface, self).__init__(*args, **kwargs) def get_response(self, request): - request.response_channel = Channel.new_name("django.wsgi.response") - Channel("django.wsgi.request", channel_backend=self.channel_backend).send(**request.channel_encode()) - channel, message = self.channel_backend.receive_many_blocking([request.response_channel]) + request.reply_channel = Channel.new_name("django.wsgi.response") + Channel("django.wsgi.request", channel_backend=self.channel_backend).send(request.channel_encode()) + channel, message = self.channel_backend.receive_many_blocking([request.reply_channel]) return HttpResponse.channel_decode(message) diff --git a/channels/message.py b/channels/message.py new file mode 100644 index 0000000..bc7eda5 --- /dev/null +++ b/channels/message.py @@ -0,0 +1,18 @@ +from .channel import Channel + + +class Message(object): + """ + Represents a message sent over a Channel. + + The message content is a dict called .content, while + reply_channel is an optional extra attribute representing a channel + to use to reply to this message's end user, if that makes sense. + """ + + def __init__(self, content, channel, channel_backend, reply_channel=None): + self.content = content + self.channel = channel + self.channel_backend = channel_backend + if reply_channel: + self.reply_channel = Channel(reply_channel, channel_backend=self.channel_backend) diff --git a/channels/request.py b/channels/request.py index 7e5a71e..dafcf49 100644 --- a/channels/request.py +++ b/channels/request.py @@ -17,7 +17,7 @@ def encode_request(request): "path": request.path, "path_info": request.path_info, "method": request.method, - "response_channel": request.response_channel, + "reply_channel": request.reply_channel, } return value @@ -34,7 +34,7 @@ def decode_request(value): request.path = value['path'] request.method = value['method'] request.path_info = value['path_info'] - request.response_channel = value['response_channel'] + request.reply_channel = value['reply_channel'] return request diff --git a/channels/worker.py b/channels/worker.py index 1e7f2ea..db0a896 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -1,4 +1,5 @@ import traceback +from .message import Message class Worker(object): @@ -17,12 +18,18 @@ class Worker(object): """ channels = self.channel_backend.registry.all_channel_names() while True: - channel, message = self.channel_backend.receive_many_blocking(channels) + channel, content = self.channel_backend.receive_many_blocking(channels) + message = Message( + content=content, + channel=channel, + channel_backend=self.channel_backend, + reply_channel=content.get("reply_channel", None), + ) # Handle the message consumer = self.channel_backend.registry.consumer_for_channel(channel) if self.callback: self.callback(channel, message) try: - consumer(channel=channel, **message) + consumer(message) except: traceback.print_exc() diff --git a/docs/concepts.rst b/docs/concepts.rst index 4c16092..d331b37 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -100,17 +100,17 @@ message and can write out zero to many other channel messages. Now, let's make a channel for requests (called ``django.wsgi.request``), and a channel per client for responses (e.g. ``django.wsgi.response.o4F2h2Fd``), -with the response channel a property (``send_channel``) of the request message. +with the response channel a property (``reply_channel``) of the request message. Suddenly, a view is merely another example of a consumer:: @consumer("django.wsgi.request") - def my_consumer(send_channel, **request_data): + def my_consumer(reply_channel, **request_data): # Decode the request from JSON-compat to a full object django_request = Request.decode(request_data) # Run view django_response = view(django_request) # Encode the response into JSON-compat format - Channel(send_channel).send(django_response.encode()) + Channel(reply_channel).send(django_response.encode()) In fact, this is how Channels works. The interface servers transform connections from the outside world (HTTP, WebSockets, etc.) into messages on channels, @@ -177,16 +177,16 @@ set of channels (here, using Redis) to send updates to:: @receiver(post_save, sender=BlogUpdate) def send_update(sender, instance, **kwargs): # Loop through all response channels and send the update - for send_channel in redis_conn.smembers("readers"): - Channel(send_channel).send( + for reply_channel in redis_conn.smembers("readers"): + Channel(reply_channel).send( id=instance.id, content=instance.content, ) @consumer("django.websocket.connect") - def ws_connect(path, send_channel, **kwargs): + def ws_connect(path, reply_channel, **kwargs): # Add to reader set - redis_conn.sadd("readers", send_channel) + redis_conn.sadd("readers", reply_channel) While this will work, there's a small problem - we never remove people from the ``readers`` set when they disconnect. We could add a consumer that @@ -221,9 +221,9 @@ we don't need to; Channels has it built in, as a feature called Groups:: @consumer("django.websocket.connect") @consumer("django.websocket.keepalive") - def ws_connect(path, send_channel, **kwargs): + def ws_connect(path, reply_channel, **kwargs): # Add to reader group - Group("liveblog").add(send_channel) + Group("liveblog").add(reply_channel) Not only do groups have their own ``send()`` method (which backends can provide an efficient implementation of), they also automatically manage expiry of diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 9a79b80..8d906e8 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -26,16 +26,16 @@ Make a new project, a new app, and put this in a ``consumers.py`` file in the ap from channels import Channel from django.http import HttpResponse - def http_consumer(response_channel, path, **kwargs): - response = HttpResponse("Hello world! You asked for %s" % path) - Channel(response_channel).send(**response.channel_encode()) + def http_consumer(message): + response = HttpResponse("Hello world! You asked for %s" % message.content['path']) + message.reply_channel.send(response.channel_encode()) The most important thing to note here is that, because things we send in messages must be JSON-serialisable, the request and response messages are in a key-value format. There are ``channel_decode()`` and ``channel_encode()`` methods on both Django's request and response classes, -but here we just take two of the request variables directly as keyword -arguments for simplicity. +but here we just use the message's ``content`` attribute directly for simplicity +(message content is always a dict). Now, go into your ``settings.py`` file, and set up a channel backend; by default, Django will just use a local backend and route HTTP requests to the normal @@ -72,8 +72,8 @@ do any time. Let's try some WebSockets, and make a basic chat server! Delete that consumer and its routing - we'll want the normal Django view layer to serve HTTP requests from now on - and make this WebSocket consumer instead:: - def ws_add(channel, send_channel, **kwargs): - Group("chat").add(send_channel) + def ws_add(message): + Group("chat").add(message.reply_channel) Hook it up to the ``django.websocket.connect`` channel like this:: @@ -90,7 +90,7 @@ Now, let's look at what this is doing. It's tied to the ``django.websocket.connect`` channel, which means that it'll get a message whenever a new WebSocket connection is opened by a client. -When it gets that message, it takes the ``send_channel`` key from it, which +When it gets that message, it takes the ``reply_channel`` attribute from it, which is the unique response channel for that client, and adds it to the ``chat`` group, which means we can send messages to all connected chat clients. @@ -107,8 +107,8 @@ a group it's already in - similarly, it's safe to discard a channel from a group it's not in):: # Connected to django.websocket.keepalive - def ws_keepalive(channel, send_channel, **kwargs): - Group("chat").add(send_channel) + def ws_keepalive(message): + Group("chat").add(message.reply_channel) Of course, this is exactly the same code as the ``connect`` handler, so let's just route both channels to the same consumer:: @@ -125,8 +125,8 @@ handler to clean up as people disconnect (most channels will cleanly disconnect and get this called):: # Connected to django.websocket.disconnect - def ws_disconnect(channel, send_channel, **kwargs): - Group("chat").discard(send_channel) + def ws_disconnect(message): + Group("chat").discard(message.reply_channel) Now, that's taken care of adding and removing WebSocket send channels for the ``chat`` group; all we need to do now is take care of message sending. For now, @@ -136,16 +136,16 @@ any message sent in to all connected clients. Here's all the code:: from channels import Channel, Group # Connected to django.websocket.connect and django.websocket.keepalive - def ws_add(channel, send_channel, **kwargs): - Group("chat").add(send_channel) + def ws_add(message): + Group("chat").add(message.reply_channel) # Connected to django.websocket.receive - def ws_message(channel, send_channel, content, **kwargs): - Group("chat").send(content=content) + def ws_message(message): + Group("chat").send(message.content) # Connected to django.websocket.disconnect - def ws_disconnect(channel, send_channel, **kwargs): - Group("chat").discard(send_channel) + def ws_disconnect(message): + Group("chat").discard(message.reply_channel) And what our routing should look like in ``settings.py``:: @@ -264,7 +264,7 @@ is much more portable, and works in development where you need to run a separate WebSocket server (by default, on port 9000). All we need to do is add the ``django_http_auth`` decorator to our views, -and we'll get extra ``session`` and ``user`` keyword arguments we can use; +and we'll get extra ``session`` and ``user`` keyword attributes on ``message`` we can use; let's make one where users can only chat to people with the same first letter of their username:: @@ -272,16 +272,16 @@ of their username:: from channels.decorators import django_http_auth @django_http_auth - def ws_add(channel, send_channel, user, **kwargs): - Group("chat-%s" % user.username[0]).add(send_channel) + def ws_add(message): + Group("chat-%s" % message.user.username[0]).add(message.reply_channel) @django_http_auth - def ws_message(channel, send_channel, content, user, **kwargs): - Group("chat-%s" % user.username[0]).send(content=content) + def ws_message(message): + Group("chat-%s" % message.user.username[0]).send(message.content) @django_http_auth - def ws_disconnect(channel, send_channel, user, **kwargs): - Group("chat-%s" % user.username[0]).discard(send_channel) + def ws_disconnect(message): + Group("chat-%s" % message.user.username[0]).discard(message.reply_channel) Now, when we connect to the WebSocket we'll have to remember to provide the Django session ID as part of the URL, like this:: @@ -293,10 +293,6 @@ Note that Channels can't work with signed cookie sessions - since only HTTP responses can set cookies, it needs a backend it can write to separately to store state. -(Also note that we always end consumers with ``**kwargs``; this is to save us -from writing out all variables we might get sent and to allow forwards-compatibility -with any additions to the message formats in the future.) - Persisting Data --------------- @@ -307,7 +303,7 @@ should let them send this request in the initial WebSocket connection, check they're allowed to access it, and then remember which room a socket is connected to when they send a message in so we know which group to send it to. -The ``send_channel`` is our unique pointer to the open WebSocket - as you've +The ``reply_channel`` is our unique pointer to the open WebSocket - as you've seen, we do all our operations on it - but it's not something we can annotate with data; it's just a simple string, and even if we hack around and set attributes on it that's not going to carry over to other workers. @@ -315,18 +311,18 @@ attributes on it that's not going to carry over to other workers. Instead, the solution is to persist information keyed by the send channel in some other data store - sound familiar? This is what Django's session framework does for HTTP requests, only there it uses cookies as the lookup key rather -than the ``send_channel``. +than the ``reply_channel``. Now, as you saw above, you can use the ``django_http_auth`` decorator to get -both a ``user`` and a ``session`` variable in your message arguments - and, -indeed, there is a ``websocket_session`` decorator that will just give you +both a ``user`` and a ``session`` attribute on your message - and, +indeed, there is a ``http_session`` decorator that will just give you the ``session`` attribute. However, that session is based on cookies, and so follows the user round the site - it's great for information that should persist across all WebSocket and HTTP connections, but not great for information that is specific to a single WebSocket (such as "which chatroom should this socket be connected to"). For -this reason, Channels also provides a ``send_channel_session`` decorator, +this reason, Channels also provides a ``channel_session`` decorator, which adds a ``channel_session`` attribute to the message; this works just like the normal ``session`` attribute, and persists to the same storage, but varies per-channel rather than per-cookie. @@ -335,31 +331,31 @@ Let's use it now to build a chat server that expects you to pass a chatroom name in the path of your WebSocket request (we'll ignore auth for now):: from channels import Channel - from channels.decorators import consumer, send_channel_session + from channels.decorators import consumer, channel_session @consumer("django.websocket.connect") - @send_channel_session - def ws_connect(channel, send_channel, path, channel_session, **kwargs): + @channel_session + def ws_connect(message): # Work out room name from path (ignore slashes) - room = path.strip("/") + room = message.content['path'].strip("/") # Save room in session and add us to the group - channel_session['room'] = room - Group("chat-%s" % room).add(send_channel) + message.channel_session['room'] = room + Group("chat-%s" % room).add(message.reply_channel) @consumer("django.websocket.keepalive") - @send_channel_session - def ws_add(channel, send_channel, channel_session, **kwargs): - Group("chat-%s" % channel_session['room']).add(send_channel) + @channel_session + def ws_add(message): + Group("chat-%s" % message.channel_session['room']).add(message.reply_channel) @consumer("django.websocket.receive") - @send_channel_session - def ws_message(channel, send_channel, content, channel_session, **kwargs): - Group("chat-%s" % channel_session['room']).send(content=content) + @channel_session + def ws_message(message): + Group("chat-%s" % message.channel_session['room']).send(content) @consumer("django.websocket.disconnect") - @send_channel_session - def ws_disconnect(channel, send_channel, channel_session, **kwargs): - Group("chat-%s" % channel_session['room']).discard(send_channel) + @channel_session + def ws_disconnect(message): + Group("chat-%s" % message.channel_session['room']).discard(message.reply_channel) If you play around with it from the console (or start building a simple JavaScript chat client that appends received messages to a div), you'll see @@ -391,40 +387,48 @@ Let's see what that looks like, assuming we have a ChatMessage model with ``message`` and ``room`` fields:: from channels import Channel - from channels.decorators import consumer, send_channel_session + from channels.decorators import consumer, channel_session from .models import ChatMessage @consumer("chat-messages") - def msg_consumer(channel, room, message): + def msg_consumer(message): # Save to model - ChatMessage.objects.create(room=room, message=message) + ChatMessage.objects.create( + room=message.content['room'], + message=message.content['message'], + ) # Broadcast to listening sockets - Group("chat-%s" % room).send(message) + Group("chat-%s" % room).send({ + "content": message.content['message'], + }) @consumer("django.websocket.connect") - @send_channel_session - def ws_connect(channel, send_channel, path, channel_session, **kwargs): + @channel_session + def ws_connect(message): # Work out room name from path (ignore slashes) - room = path.strip("/") + room = message.content['path'].strip("/") # Save room in session and add us to the group - channel_session['room'] = room - Group("chat-%s" % room).add(send_channel) + message.channel_session['room'] = room + Group("chat-%s" % room).add(message.reply_channel) @consumer("django.websocket.keepalive") - @send_channel_session - def ws_add(channel, send_channel, channel_session, **kwargs): - Group("chat-%s" % channel_session['room']).add(send_channel) + @channel_session + def ws_add(message): + Group("chat-%s" % message.channel_session['room']).add(message.reply_channel) @consumer("django.websocket.receive") - @send_channel_session - def ws_message(channel, send_channel, content, channel_session, **kwargs): + @channel_session + def ws_message(message): # Stick the message onto the processing queue - Channel("chat-messages").send(room=channel_session['room'], message=content) + Channel("chat-messages").send({ + "room": channel_session['room'], + "message": content, + }) @consumer("django.websocket.disconnect") - @send_channel_session - def ws_disconnect(channel, send_channel, channel_session, **kwargs): - Group("chat-%s" % channel_session['room']).discard(send_channel) + @channel_session + def ws_disconnect(message): + Group("chat-%s" % message.channel_session['room']).discard(message.reply_channel) Note that we could add messages onto the ``chat-messages`` channel from anywhere; inside a View, inside another model's ``post_save`` signal, inside a management diff --git a/docs/message-standards.rst b/docs/message-standards.rst index c1c6528..c018c39 100644 --- a/docs/message-standards.rst +++ b/docs/message-standards.rst @@ -62,7 +62,7 @@ Contains the following keys: * META: Same as ``request.META`` * path: Same as ``request.path`` * path_info: Same as ``request.path_info`` -* send_channel: Channel name to send responses on +* reply_channel: Channel name to send responses on WebSocket Receive @@ -81,7 +81,7 @@ WebSocket Client Close Sent when the WebSocket is closed by either the client or the server. -Contains the same keys as WebSocket Connection, including send_channel, +Contains the same keys as WebSocket Connection, including reply_channel, though nothing should be sent on it. From c6527bebf1fa55a95dcf4ff50d4bc9b4acd1b0ca Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 8 Sep 2015 10:33:24 -0500 Subject: [PATCH 049/746] Version 0.7 --- README.rst | 7 ++++--- setup.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 58568ab..0207815 100644 --- a/README.rst +++ b/README.rst @@ -3,10 +3,11 @@ Django Channels This is a work-in-progress code branch of Django implemented as a third-party app, which aims to bring some asynchrony to Django and expand the options -for code beyond the request-response model. +for code beyond the request-response model, in particular enabling WebSocket, +HTTP2 push, and background task support. -This is still **pre-alpha** software, and you use it at your own risk; the -API is not yet stable. +This is still **beta** software: the API is mostly settled, but might change +a bit as things develop. Documentation, installation and getting started instructions are at http://channels.readthedocs.org diff --git a/setup.py b/setup.py index 01fe972..a5e0547 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,11 @@ from setuptools import find_packages, setup setup( name='channels', - version="0.1.1", + version="0.7", url='http://github.com/andrewgodwin/django-channels', author='Andrew Godwin', author_email='andrew@aeracode.org', - description="Brings event-driven capabilities to Django with a channel system. Django 1.8 and up only.", + description="Brings event-driven capabilities to Django with a channel system. Django 1.7 and up only.", license='BSD', packages=find_packages(), include_package_data=True, From d4c7f2db2056499a75ccfeda97b7921bf3f89484 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 8 Sep 2015 16:03:33 -0500 Subject: [PATCH 050/746] Update concepts doc to not use decorators --- docs/concepts.rst | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index d331b37..1f52294 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -60,21 +60,25 @@ How do we use channels? ----------------------- That's what a channel is, but how is Django using them? Well, inside Django -you can connect a function to consume a channel, like so:: +you can write a function to consume a channel, like so:: - from channels.decorators import consumer - - @consumer("channel-name") - def my_consumer(something, **kwargs): + def my_consumer(message): pass +And then assign a channel to it like this in the channel backend settings:: + + "ROUTING": { + "some-channel": "myapp.consumers.my_consumer", + } + This means that for every message on the channel, Django will call that -consumer function with the message as keyword arguments (messages are always -a dict, and are mapped to keyword arguments for send/receive). +consumer function with a message object (message objects have a "content" +attribute which is always a dict of data, and a "channel" attribute which +is the channel it came from, as well as some others). Django can do this as rather than run in a request-response mode, Channels changes Django so that it runs in a worker mode - it listens on all channels -that have consumers declared, and when a message arrives on one, runs the +that have consumers assigned, and when a message arrives on one, runs the relevant consumer. In fact, this is illustrative of the new way Django runs to enable Channels to @@ -98,15 +102,15 @@ slightly more complex abstraction than that presented by Django views. A view takes a request and returns a response; a consumer takes a channel message and can write out zero to many other channel messages. -Now, let's make a channel for requests (called ``django.wsgi.request``), +Now, let's make a channel for requests (called ``django.wsgi.request``), and a channel per client for responses (e.g. ``django.wsgi.response.o4F2h2Fd``), with the response channel a property (``reply_channel``) of the request message. Suddenly, a view is merely another example of a consumer:: - @consumer("django.wsgi.request") - def my_consumer(reply_channel, **request_data): + # Listens on django.wsgi.request. + def my_consumer(message): # Decode the request from JSON-compat to a full object - django_request = Request.decode(request_data) + django_request = Request.decode(message.content) # Run view django_response = view(django_request) # Encode the response into JSON-compat format @@ -171,7 +175,6 @@ Say I had a live blog where I wanted to push out updates whenever a new post is saved, I would register a handler for the ``post_save`` signal and keep a set of channels (here, using Redis) to send updates to:: - redis_conn = redis.Redis("localhost", 6379) @receiver(post_save, sender=BlogUpdate) @@ -183,10 +186,10 @@ set of channels (here, using Redis) to send updates to:: content=instance.content, ) - @consumer("django.websocket.connect") - def ws_connect(path, reply_channel, **kwargs): + # Connected to django.websocket.connect + def ws_connect(message): # Add to reader set - redis_conn.sadd("readers", reply_channel) + redis_conn.sadd("readers", message.reply_channel.name) While this will work, there's a small problem - we never remove people from the ``readers`` set when they disconnect. We could add a consumer that @@ -219,14 +222,13 @@ we don't need to; Channels has it built in, as a feature called Groups:: content=instance.content, ) - @consumer("django.websocket.connect") - @consumer("django.websocket.keepalive") - def ws_connect(path, reply_channel, **kwargs): + # Connected to django.websocket.connect and django.websocket.keepalive + def ws_connect(message): # Add to reader group - Group("liveblog").add(reply_channel) + Group("liveblog").add(message.reply_channel) Not only do groups have their own ``send()`` method (which backends can provide -an efficient implementation of), they also automatically manage expiry of +an efficient implementation of), they also automatically manage expiry of the group members. You'll have to re-call ``Group.add()`` every so often to keep existing members from expiring, but that's easy, and can be done in the same handler for both ``connect`` and ``keepalive``, as you can see above. From 056082325fd53b26abb527cf442fa67e1525da45 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 8 Sep 2015 18:20:18 -0500 Subject: [PATCH 051/746] Improve doc linking a little. --- README.rst | 2 ++ docs/index.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 0207815..9a174fc 100644 --- a/README.rst +++ b/README.rst @@ -11,3 +11,5 @@ a bit as things develop. Documentation, installation and getting started instructions are at http://channels.readthedocs.org + +You can also install channels from PyPI as the ``channels`` package. diff --git a/docs/index.rst b/docs/index.rst index 78e811c..39bb19d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,8 @@ data model underlying Channels and how they're used inside Django. Then, read :doc:`getting-started` to see how to get up and running with WebSockets with only 30 lines of code. +You can find the Channels repository `on GitHub `_. + Contents: .. toctree:: From bbe0d14fc4d8694e2e58e1169046568be9b606c4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 8 Sep 2015 18:32:25 -0500 Subject: [PATCH 052/746] Remove more old @consumer decoators from docs --- docs/getting-started.rst | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 8d906e8..21e9d63 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -331,9 +331,9 @@ Let's use it now to build a chat server that expects you to pass a chatroom name in the path of your WebSocket request (we'll ignore auth for now):: from channels import Channel - from channels.decorators import consumer, channel_session + from channels.decorators import channel_session - @consumer("django.websocket.connect") + # Connected to django.websocket.connect @channel_session def ws_connect(message): # Work out room name from path (ignore slashes) @@ -342,17 +342,17 @@ name in the path of your WebSocket request (we'll ignore auth for now):: message.channel_session['room'] = room Group("chat-%s" % room).add(message.reply_channel) - @consumer("django.websocket.keepalive") + # Connected to django.websocket.keepalive @channel_session def ws_add(message): Group("chat-%s" % message.channel_session['room']).add(message.reply_channel) - @consumer("django.websocket.receive") + # Connected to django.websocket.receive @channel_session def ws_message(message): Group("chat-%s" % message.channel_session['room']).send(content) - @consumer("django.websocket.disconnect") + # Connected to django.websocket.disconnect @channel_session def ws_disconnect(message): Group("chat-%s" % message.channel_session['room']).discard(message.reply_channel) @@ -387,10 +387,9 @@ Let's see what that looks like, assuming we have a ChatMessage model with ``message`` and ``room`` fields:: from channels import Channel - from channels.decorators import consumer, channel_session + from channels.decorators import channel_session from .models import ChatMessage - @consumer("chat-messages") def msg_consumer(message): # Save to model ChatMessage.objects.create( @@ -402,7 +401,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: "content": message.content['message'], }) - @consumer("django.websocket.connect") + # Connected to django.websocket.connect @channel_session def ws_connect(message): # Work out room name from path (ignore slashes) @@ -411,12 +410,12 @@ have a ChatMessage model with ``message`` and ``room`` fields:: message.channel_session['room'] = room Group("chat-%s" % room).add(message.reply_channel) - @consumer("django.websocket.keepalive") + # Connected to django.websocket.keepalive @channel_session def ws_add(message): Group("chat-%s" % message.channel_session['room']).add(message.reply_channel) - @consumer("django.websocket.receive") + # Connected to django.websocket.receive @channel_session def ws_message(message): # Stick the message onto the processing queue @@ -425,7 +424,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: "message": content, }) - @consumer("django.websocket.disconnect") + # Connected to django.websocket.disconnect @channel_session def ws_disconnect(message): Group("chat-%s" % message.channel_session['room']).discard(message.reply_channel) From 39bead9de90f09803eeaffe97aaa15512e6b98ee Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 8 Sep 2015 18:45:59 -0500 Subject: [PATCH 053/746] Remove useless import from doc example --- docs/getting-started.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 21e9d63..72bff85 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -23,7 +23,6 @@ now underlie every part of Django. Make a new project, a new app, and put this in a ``consumers.py`` file in the app:: - from channels import Channel from django.http import HttpResponse def http_consumer(message): From 041ea3fa5c3d9d127a06460c71adad83671d8031 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 9 Sep 2015 13:57:52 -0500 Subject: [PATCH 054/746] Added some FAQs --- docs/faqs.rst | 126 +++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 3 +- 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 docs/faqs.rst diff --git a/docs/faqs.rst b/docs/faqs.rst new file mode 100644 index 0000000..21e8caf --- /dev/null +++ b/docs/faqs.rst @@ -0,0 +1,126 @@ +Frequently Asked Questions +========================== + +Why are you doing this rather than just using Tornado/gevent/asyncio/etc.? +-------------------------------------------------------------------------- + +They're kind of solving different problems. Tornado, gevent and other +in-process async solutions are a way of making a single Python process act +asynchronously - doing other things while a HTTP request is going on, or +juggling hundreds of incoming connections without blocking on a single one. + +Channels is different - all the code you write for consumers runs synchronously. +You can do all the blocking filesystem calls and CPU-bound tasks you like +and all you'll do is block the one worker you're running on; the other +worker processes will just keep on going and handling other messages. + +This is partially because Django is all written in a synchronous manner, and +rewriting it to all be asynchronous would be a near-impossible task, but also +because we believe that normal developers should not have to write +asynchronous-friendly code. It's really easy to shoot yourself in the foot; +do a tight loop without yielding in the middle, or access a file that happens +to be on a slow NFS share, and you've just blocked the entire process. + +Channels still uses asynchronous code, but it confines it to the interface +layer - the processes that serve HTTP, WebSocket and other requests. These do +indeed use asynchronous frameworks (currently, asyncio and Twisted) to handle +managing all the concurrent connections, but they're also fixed pieces of code; +as an end developer, you'll likely never have to touch them. + +All of your work can be with standard Python libraries and patterns and the +only thing you need to look out for is worker contention - if you flood your +workers with infinite loops, of course they'll all stop working, but that's +better than a single thread of execution stopping the entire site. + + +Why aren't you using node/go/etc. to proxy to Django? +----------------------------------------------------- + +There are a couple of solutions where you can use a more "async-friendly" +language (or Python framework) to bridge things like WebSockets to Django - +terminate them in (say) a Node process, and then bridge it to Django using +either a reverse proxy model, or Redis signalling, or some other mechanism. + +The thing is, Channels actually makes it easier to do this if you wish. The +key part of Channels is introducing a standardised way to run event-triggered +pieces of code, and a standardised way to route messages via named channels +that hits the right balance between flexibility and simplicity. + +While our interface servers are written in Python, there's nothing stopping +you from writing an interface server in another language, providing it follows +the same serialisation standards for HTTP/WebSocket/etc. messages. In fact, +we may ship an alternative server implementation ourselves at some point. + + +Why isn't there guaranteed delivery/a retry mechanism? +------------------------------------------------------ + +Channels' design is such that anything is allowed to fail - a consumer can +error and not send replies, the channel layer can restart and drop a few messages, +a dogpile can happen and a few incoming clients get rejected. + +This is because designing a system that was fully guaranteed, end-to-end, would +result in something with incredibly low throughput, and almost no problem needs +that level of guarantee. If you want some level of guarantee, you can build on +top of what Channels provides and add it in (for example, use a database to +mark things that need to be cleaned up and resend messages if they aren't after +a while, or make idempotent consumers and over-send messages rather than +under-send). + +That said, it's a good way to design a system to presume any part of it can +fail, and design for detection and recovery of that state, rather than hanging +your entire livelihood on a system working perfectly as designed. Channels +takes this idea and uses it to provide a high-throughput solution that is +mostly reliable, rather than a low-throughput one that is *nearly* completely +reliable. + + +Can I run HTTP requests/service calls/etc. in parallel from Django without blocking? +------------------------------------------------------------------------------------ + +Not directly - Channels only allows a consumer function to listen to channels +at the start, which is what kicks it off; you can't send tasks off on channels +to other consumers and then *wait on the result*. You can send them off and keep +going, but you cannot ever block waiting on a channel in a consumer, as otherwise +you'd hit deadlocks, livelocks, and similar issues. + +This is partially a design feature - this falls into the class of "difficult +async concepts that it's easy to shoot yourself in the foot with" - but also +to keep the underlying channels implementation simple. By not allowing this sort +of blocking, we can have specifications for channel layers that allows horizontal +scaling and sharding. + +What you can do is: + +* Dispatch a whole load of tasks to run later in the background and then finish + your current task - for example, dispatching an avatar thumbnailing task in + the avatar upload view, then returning a "we got it!" HTTP response. + +* Pass details along to the other task about how to continue, in particular + a channel name linked to another consumer that will finish the job, or + IDs or other details of the data (remember, message contents are just a dict + you can put stuff into). For example, you might have a generic image fetching + task for a variety of models that should fetch an image, store it, and pass + the resultant ID and the ID of the object you're attaching it to onto a different + channel depending on the model - you'd pass the next channel name and the + ID of the target object in the message, and then the consumer could send + a new message onto that channel name when it's done. + +* Have interface servers that perform requests or slow tasks (remember, interface + servers are the specialist code which *is* written to be highly asynchronous) + and then send their results onto a channel when finished. Again, you can't wait + around inside a consumer and block on the results, but you can provide another + consumer on a new channel that will do the second half. + + +How do I associate data with incoming connections? +-------------------------------------------------- + +Channels provides full integration with Django's session and auth system for its +WebSockets support, as well as per-websocket sessions for persisting data, so +you can easily persist data on a per-connection or per-user basis. + +You can also provide your own solution if you wish, keyed off of ``message.reply_channel``, +which is the unique channel representing the connection, but remember that +whatever you store in must be **network-transparent** - storing things in a +global variable won't work outside of development. diff --git a/docs/index.rst b/docs/index.rst index 39bb19d..a1c8d5f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,7 +21,7 @@ Contents: .. toctree:: :maxdepth: 2 - + concepts installation getting-started @@ -30,3 +30,4 @@ Contents: message-standards scaling backends + faqs From fc52e3c5a23aaacb078a5f357e125285fa22c597 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 9 Sep 2015 20:58:32 -0500 Subject: [PATCH 055/746] Remove auto-importing of modules --- channels/management/commands/runserver.py | 6 ++---- channels/management/commands/runworker.py | 2 -- channels/management/commands/runwsserver.py | 2 -- channels/utils.py | 21 --------------------- 4 files changed, 2 insertions(+), 29 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index ddc1731..34154b9 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -4,7 +4,6 @@ from django.core.management.commands.runserver import Command as RunserverComman from django.core.management import CommandError from channels import channel_backends, DEFAULT_CHANNEL_BACKEND from channels.worker import Worker -from channels.utils import auto_import_consumers from channels.adapters import UrlConsumer from channels.interfaces.wsgi import WSGIInterface @@ -24,10 +23,9 @@ class Command(RunserverCommand): def inner_run(self, *args, **options): # Check a handler is registered for http reqs self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - auto_import_consumers() - if not self.channel_backend.registry.consumer_for_channel("django.wsgi.request"): + if not self.channel_backend.registry.consumer_for_channel("http.request"): # Register the default one - self.channel_backend.registry.add_consumer(UrlConsumer(), ["django.wsgi.request"]) + self.channel_backend.registry.add_consumer(UrlConsumer(), ["http.request"]) # Note that this is the right one on the console self.stdout.write("Worker thread running, channels enabled") if self.channel_backend.local_only: diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 8d8b2ae..dd0bb9c 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -3,7 +3,6 @@ from wsgiref.simple_server import BaseHTTPRequestHandler from django.core.management import BaseCommand, CommandError from channels import channel_backends, DEFAULT_CHANNEL_BACKEND from channels.worker import Worker -from channels.utils import auto_import_consumers class Command(BaseCommand): @@ -11,7 +10,6 @@ class Command(BaseCommand): def handle(self, *args, **options): # Get the backend to use channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - auto_import_consumers() if channel_backend.local_only: raise CommandError( "You have a process-local channel backend configured, and so cannot run separate workers.\n" diff --git a/channels/management/commands/runwsserver.py b/channels/management/commands/runwsserver.py index 9434be0..7b4d296 100644 --- a/channels/management/commands/runwsserver.py +++ b/channels/management/commands/runwsserver.py @@ -2,7 +2,6 @@ import time from django.core.management import BaseCommand, CommandError from channels import channel_backends, DEFAULT_CHANNEL_BACKEND from channels.interfaces.websocket_twisted import WebsocketTwistedInterface -from channels.utils import auto_import_consumers class Command(BaseCommand): @@ -14,7 +13,6 @@ class Command(BaseCommand): def handle(self, *args, **options): # Get the backend to use channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - auto_import_consumers() if channel_backend.local_only: raise CommandError( "You have a process-local channel backend configured, and so cannot run separate interface servers.\n" diff --git a/channels/utils.py b/channels/utils.py index 4a7fb65..2cddb58 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -1,25 +1,4 @@ import types -from django.apps import apps - -from six import PY3 - -def auto_import_consumers(): - """ - Auto-import consumers modules in apps - """ - for app_config in apps.get_app_configs(): - for submodule in ["consumers", "views"]: - module_name = "%s.%s" % (app_config.name, submodule) - try: - __import__(module_name) - except ImportError as e: - err = str(e).lower() - if PY3: - if "no module named '%s'" % (module_name,) not in err: - raise - else: - if "no module named %s" % (submodule,) not in err: - raise def name_that_thing(thing): From 70caf7d1711d8b8d371e128bd4a621a86d283eca Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 9 Sep 2015 21:21:43 -0500 Subject: [PATCH 056/746] Rename channels and change message format docs --- channels/interfaces/websocket_twisted.py | 14 +-- channels/interfaces/wsgi.py | 4 +- channels/response.py | 7 +- docs/concepts.rst | 16 ++-- docs/getting-started.rst | 50 +++++----- docs/message-standards.rst | 114 ++++++++++++++--------- 6 files changed, 115 insertions(+), 90 deletions(-) diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index 8aa0b31..478d03b 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -11,7 +11,7 @@ from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND class InterfaceProtocol(WebSocketServerProtocol): """ Protocol which supports WebSockets and forwards incoming messages to - the django.websocket channels. + the websocket channels. """ def onConnect(self, request): @@ -23,22 +23,22 @@ class InterfaceProtocol(WebSocketServerProtocol): def onOpen(self): # Make sending channel - self.reply_channel = Channel.new_name("!django.websocket.send") + self.reply_channel = Channel.new_name("!websocket.send") self.request_info["reply_channel"] = self.reply_channel self.last_keepalive = time.time() self.factory.protocols[self.reply_channel] = self # Send news that this channel is open - Channel("django.websocket.connect").send(self.request_info) + Channel("websocket.connect").send(self.request_info) def onMessage(self, payload, isBinary): if isBinary: - Channel("django.websocket.receive").send(dict( + Channel("websocket.receive").send(dict( self.request_info, content = payload, binary = True, )) else: - Channel("django.websocket.receive").send(dict( + Channel("websocket.receive").send(dict( self.request_info, content = payload.decode("utf8"), binary = False, @@ -62,13 +62,13 @@ class InterfaceProtocol(WebSocketServerProtocol): def onClose(self, wasClean, code, reason): if hasattr(self, "reply_channel"): del self.factory.protocols[self.reply_channel] - Channel("django.websocket.disconnect").send(self.request_info) + Channel("websocket.disconnect").send(self.request_info) def sendKeepalive(self): """ Sends a keepalive packet on the keepalive channel. """ - Channel("django.websocket.keepalive").send(self.request_info) + Channel("websocket.keepalive").send(self.request_info) self.last_keepalive = time.time() diff --git a/channels/interfaces/wsgi.py b/channels/interfaces/wsgi.py index 8df49c0..b856b93 100644 --- a/channels/interfaces/wsgi.py +++ b/channels/interfaces/wsgi.py @@ -15,7 +15,7 @@ class WSGIInterface(WSGIHandler): super(WSGIInterface, self).__init__(*args, **kwargs) def get_response(self, request): - request.reply_channel = Channel.new_name("django.wsgi.response") - Channel("django.wsgi.request", channel_backend=self.channel_backend).send(request.channel_encode()) + request.reply_channel = Channel.new_name("http.response") + Channel("http.request", channel_backend=self.channel_backend).send(request.channel_encode()) channel, message = self.channel_backend.receive_many_blocking([request.reply_channel]) return HttpResponse.channel_decode(message) diff --git a/channels/response.py b/channels/response.py index 2b8d761..fa4557f 100644 --- a/channels/response.py +++ b/channels/response.py @@ -1,5 +1,4 @@ from django.http import HttpResponse -from django.http.cookie import SimpleCookie from six import PY3 @@ -12,7 +11,7 @@ def encode_response(response): "content": response.content, "status_code": response.status_code, "headers": list(response._headers.values()), - "cookies": {k: v.output(header="") for k, v in response.cookies.items()} + "cookies": [v.output(header="") for _, v in response.cookies.items()] } if PY3: value["content"] = value["content"].decode('utf8') @@ -29,9 +28,9 @@ def decode_response(value): content_type = value['content_type'], status = value['status_code'], ) - for cookie in value['cookies'].values(): + for cookie in value['cookies']: response.cookies.load(cookie) - response._headers = {k.lower: (k, v) for k, v in value['headers']} + response._headers = {k.lower(): (k, v) for k, v in value['headers']} return response diff --git a/docs/concepts.rst b/docs/concepts.rst index 1f52294..e91cc64 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -53,7 +53,7 @@ and producers running in different processes or on different machines. Inside a network, we identify channels uniquely by a name string - you can send to any named channel from any machine connected to the same channel -backend. If two different machines both write to the ``django.wsgi.request`` +backend. If two different machines both write to the ``http.request`` channel, they're writing into the same channel. How do we use channels? @@ -102,12 +102,12 @@ slightly more complex abstraction than that presented by Django views. A view takes a request and returns a response; a consumer takes a channel message and can write out zero to many other channel messages. -Now, let's make a channel for requests (called ``django.wsgi.request``), -and a channel per client for responses (e.g. ``django.wsgi.response.o4F2h2Fd``), +Now, let's make a channel for requests (called ``http.request``), +and a channel per client for responses (e.g. ``http.response.o4F2h2Fd``), with the response channel a property (``reply_channel``) of the request message. Suddenly, a view is merely another example of a consumer:: - # Listens on django.wsgi.request. + # Listens on http.request def my_consumer(message): # Decode the request from JSON-compat to a full object django_request = Request.decode(message.content) @@ -154,7 +154,7 @@ to the channel server they're listening on. For this reason, Channels treats these as two different *channel types*, and denotes a *response channel* by having the first character of the channel name -be the character ``!`` - e.g. ``!django.wsgi.response.f5G3fE21f``. *Normal +be the character ``!`` - e.g. ``!http.response.f5G3fE21f``. *Normal channels* have no special prefix, but along with the rest of the response channel name, they must contain only the characters ``a-z A-Z 0-9 - _``, and be less than 200 characters long. @@ -186,14 +186,14 @@ set of channels (here, using Redis) to send updates to:: content=instance.content, ) - # Connected to django.websocket.connect + # Connected to websocket.connect def ws_connect(message): # Add to reader set redis_conn.sadd("readers", message.reply_channel.name) While this will work, there's a small problem - we never remove people from the ``readers`` set when they disconnect. We could add a consumer that -listens to ``django.websocket.disconnect`` to do that, but we'd also need to +listens to ``websocket.disconnect`` to do that, but we'd also need to have some kind of expiry in case an interface server is forced to quit or loses power before it can send disconnect signals - your code will never see any disconnect notification but the response channel is completely @@ -222,7 +222,7 @@ we don't need to; Channels has it built in, as a feature called Groups:: content=instance.content, ) - # Connected to django.websocket.connect and django.websocket.keepalive + # Connected to websocket.connect and websocket.keepalive def ws_connect(message): # Add to reader group Group("liveblog").add(message.reply_channel) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 72bff85..77ce6b0 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -12,7 +12,7 @@ First Consumers --------------- Now, by default, Django will run things through Channels but it will also -tie in the URL router and view subsystem to the default ``django.wsgi.request`` +tie in the URL router and view subsystem to the default ``http.request`` channel if you don't provide another consumer that listens to it - remember, only one consumer can listen to any given channel. @@ -48,7 +48,7 @@ custom consumer we wrote above. Here's what that looks like:: "default": { "BACKEND": "channels.backends.database.DatabaseChannelBackend", "ROUTING": { - "django.wsgi.request": "myproject.myapp.consumers.http_consumer", + "http.request": "myproject.myapp.consumers.http_consumer", }, }, } @@ -74,19 +74,19 @@ serve HTTP requests from now on - and make this WebSocket consumer instead:: def ws_add(message): Group("chat").add(message.reply_channel) -Hook it up to the ``django.websocket.connect`` channel like this:: +Hook it up to the ``websocket.connect`` channel like this:: CHANNEL_BACKENDS = { "default": { "BACKEND": "channels.backends.database.DatabaseChannelBackend", "ROUTING": { - "django.websocket.connect": "myproject.myapp.consumers.ws_add", + "websocket.connect": "myproject.myapp.consumers.ws_add", }, }, } Now, let's look at what this is doing. It's tied to the -``django.websocket.connect`` channel, which means that it'll get a message +``websocket.connect`` channel, which means that it'll get a message whenever a new WebSocket connection is opened by a client. When it gets that message, it takes the ``reply_channel`` attribute from it, which @@ -100,12 +100,12 @@ don't keep track of the open/close states of the potentially thousands of connections you have open at any one time. The solution to this is that the WebSocket interface servers will send -periodic "keepalive" messages on the ``django.websocket.keepalive`` channel, +periodic "keepalive" messages on the ``websocket.keepalive`` channel, so we can hook that up to re-add the channel (it's safe to add the channel to a group it's already in - similarly, it's safe to discard a channel from a group it's not in):: - # Connected to django.websocket.keepalive + # Connected to websocket.keepalive def ws_keepalive(message): Group("chat").add(message.reply_channel) @@ -114,8 +114,8 @@ just route both channels to the same consumer:: ... "ROUTING": { - "django.websocket.connect": "myproject.myapp.consumers.ws_add", - "django.websocket.keepalive": "myproject.myapp.consumers.ws_add", + "websocket.connect": "myproject.myapp.consumers.ws_add", + "websocket.keepalive": "myproject.myapp.consumers.ws_add", }, ... @@ -123,7 +123,7 @@ And, even though channels will expire out, let's add an explicit ``disconnect`` handler to clean up as people disconnect (most channels will cleanly disconnect and get this called):: - # Connected to django.websocket.disconnect + # Connected to websocket.disconnect def ws_disconnect(message): Group("chat").discard(message.reply_channel) @@ -134,15 +134,15 @@ any message sent in to all connected clients. Here's all the code:: from channels import Channel, Group - # Connected to django.websocket.connect and django.websocket.keepalive + # Connected to websocket.connect and websocket.keepalive def ws_add(message): Group("chat").add(message.reply_channel) - # Connected to django.websocket.receive + # Connected to websocket.receive def ws_message(message): Group("chat").send(message.content) - # Connected to django.websocket.disconnect + # Connected to websocket.disconnect def ws_disconnect(message): Group("chat").discard(message.reply_channel) @@ -152,10 +152,10 @@ And what our routing should look like in ``settings.py``:: "default": { "BACKEND": "channels.backends.database.DatabaseChannelBackend", "ROUTING": { - "django.websocket.connect": "myproject.myapp.consumers.ws_add", - "django.websocket.keepalive": "myproject.myapp.consumers.ws_add", - "django.websocket.receive": "myproject.myapp.consumers.ws_message", - "django.websocket.disconnect": "myproject.myapp.consumers.ws_disconnect", + "websocket.connect": "myproject.myapp.consumers.ws_add", + "websocket.keepalive": "myproject.myapp.consumers.ws_add", + "websocket.receive": "myproject.myapp.consumers.ws_message", + "websocket.disconnect": "myproject.myapp.consumers.ws_disconnect", }, }, } @@ -332,7 +332,7 @@ name in the path of your WebSocket request (we'll ignore auth for now):: from channels import Channel from channels.decorators import channel_session - # Connected to django.websocket.connect + # Connected to websocket.connect @channel_session def ws_connect(message): # Work out room name from path (ignore slashes) @@ -341,17 +341,17 @@ name in the path of your WebSocket request (we'll ignore auth for now):: message.channel_session['room'] = room Group("chat-%s" % room).add(message.reply_channel) - # Connected to django.websocket.keepalive + # Connected to websocket.keepalive @channel_session def ws_add(message): Group("chat-%s" % message.channel_session['room']).add(message.reply_channel) - # Connected to django.websocket.receive + # Connected to websocket.receive @channel_session def ws_message(message): Group("chat-%s" % message.channel_session['room']).send(content) - # Connected to django.websocket.disconnect + # Connected to websocket.disconnect @channel_session def ws_disconnect(message): Group("chat-%s" % message.channel_session['room']).discard(message.reply_channel) @@ -400,7 +400,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: "content": message.content['message'], }) - # Connected to django.websocket.connect + # Connected to websocket.connect @channel_session def ws_connect(message): # Work out room name from path (ignore slashes) @@ -409,12 +409,12 @@ have a ChatMessage model with ``message`` and ``room`` fields:: message.channel_session['room'] = room Group("chat-%s" % room).add(message.reply_channel) - # Connected to django.websocket.keepalive + # Connected to websocket.keepalive @channel_session def ws_add(message): Group("chat-%s" % message.channel_session['room']).add(message.reply_channel) - # Connected to django.websocket.receive + # Connected to websocket.receive @channel_session def ws_message(message): # Stick the message onto the processing queue @@ -423,7 +423,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: "message": content, }) - # Connected to django.websocket.disconnect + # Connected to websocket.disconnect @channel_session def ws_disconnect(message): Group("chat-%s" % message.channel_session['room']).discard(message.reply_channel) diff --git a/docs/message-standards.rst b/docs/message-standards.rst index c018c39..405b23c 100644 --- a/docs/message-standards.rst +++ b/docs/message-standards.rst @@ -2,13 +2,27 @@ Message Standards ================= Some standardised message formats are used for common message types - they -are detailed below. +are detailed below. Message formats are meant to be generic and offload as +much protocol-specific processing to the interface server as is reasonable; +thus, they should generally represent things at as high a level as makes sense. -Note: All consumers also receive the channel name as the keyword argument -"channel", so there is no need for separate type information to let -multi-channel consumers distinguish. +In addition to the standards outlined below, each message may contain a +``reply_channel``, which details where to send responses. Protocols with +separate connection and data receiving messages (like WebSockets) will only +contain the connection and detailed client information in the first message; +use the ``@channel_session`` decorator to persist this data to consumers of +the received data (the decorator will take care of handling persistence and +ordering guarantees on messages). -The length limit on channel names will be 200 characters. +All messages must be able to be encoded as JSON; channel backends don't +necessarily have to use JSON, but we consider it the lowest common denominator +for serialisation format compatability. + +The size limit on messages is 1MB (while channel backends may support larger +sizes, all message formats should stay under this limit, which might include +multi-part messages where large content must be transferred). + +The length limit on channel names is 200 printable ASCII characters. HTTP Request @@ -16,29 +30,45 @@ HTTP Request Represents a full-fledged, single HTTP request coming in from a client. +Standard channel name is ``http.request``. + Contains the following keys: -* GET: List of (key, value) tuples of GET variables -* POST: List of (key, value) tuples of POST variables -* COOKIES: Same as ``request.COOKIES`` -* META: Same as ``request.META`` -* path: Same as ``request.path`` -* path_info: Same as ``request.path_info`` -* method: Upper-cased HTTP method -* response_channel: Channel name to write response to +* GET: List of (key, value) tuples of GET variables (keys and values are strings) +* POST: List of (key, value) tuples of POST variables (keys and values are strings) +* COOKIES: Dict of cookies as {cookie_name: cookie_value} (names and values are strings) +* META: Dict of HTTP headers and info as defined in the Django Request docs (names and values are strings) +* path: String, full path to the requested page, without query string or domain +* path_info: String, like ``path`` but without any script prefix. Often just ``path``. +* method: String, upper-cased HTTP method + +Should come with an associated ``reply_channel`` which accepts HTTP Responses. HTTP Response ------------- -Sends a whole response to a client. +Sends either a part of a response or a whole response to a HTTP client - to do +streaming responses, several response messages are sent with ``more_content: True`` +and the final one has the key omitted. Normal, single-shot responses do not +need the key at all. -Contains the following keys: +Due to the 1MB size limit on messages, some larger responses will have to be +sent multi-part to stay within the limit. + +Only sent on reply channels. + +Keys that must only be in the first message of a set: + +* content_type: String, mimetype of content +* status_code: Integer, numerical HTTP status code +* cookies: List of cookies to set (as encoded cookie strings suitable for headers) +* headers: Dictionary of headers (key is header name, value is value, both strings) + +All messages in a set can the following keys: * content: String of content to send -* content_type: Mimetype of content -* status_code: Numerical HTTP status code -* headers: Dictionary of headers (key is header name, value is value) +* more_content: Boolean, signals the interface server should wait for another response chunk to stream. HTTP Disconnect @@ -47,7 +77,9 @@ HTTP Disconnect Send when a client disconnects early, before the response has been sent. Only sent by long-polling-capable HTTP interface servers. -Contains the same keys as HTTP Request. +Standard channel name is ``http.disconnect``. + +Contains no keys. WebSocket Connection @@ -55,14 +87,9 @@ WebSocket Connection Sent when a new WebSocket is connected. -Contains the following keys: +Standard channel name is ``websocket.connect``. -* GET: List of (key, value) tuples of GET variables -* COOKIES: Same as ``request.COOKIES`` -* META: Same as ``request.META`` -* path: Same as ``request.path`` -* path_info: Same as ``request.path_info`` -* reply_channel: Channel name to send responses on +Contains the same keys as HTTP Request, without the ``POST`` or ``method`` keys. WebSocket Receive @@ -70,10 +97,12 @@ WebSocket Receive Sent when a datagram is received on the WebSocket. -Contains the same keys as WebSocket Connection, plus: +Standard channel name is ``websocket.receive``. -* content: String content of the datagram -* binary: If the content is to be interpreted as text or binary +Contains the following keys: + +* content: String content of the datagram. +* binary: Boolean, saying if the content is binary. If not present or false, content is a UTF8 string. WebSocket Client Close @@ -81,25 +110,22 @@ WebSocket Client Close Sent when the WebSocket is closed by either the client or the server. -Contains the same keys as WebSocket Connection, including reply_channel, -though nothing should be sent on it. +Standard channel name is ``websocket.disconnect``. + +Contains no keys. -WebSocket Send --------------- +WebSocket Send/Close +-------------------- Sent by a Django consumer to send a message back over the WebSocket to -the client. +the client or close the client connection. The content is optional if close +is set, and close will happen after any content is sent, if some is present. + +Only sent on reply channels. Contains the keys: -* content: String content of the datagram -* binary: If the content is to be interpreted as text or binary - - -WebSocket Server Close ----------------------- - -Sent by a Django consumer to close the client's WebSocket. - -Contains no keys. +* content: String content of the datagram. +* binary: If the content is to be interpreted as text or binary. +* close: Boolean. If set to True, will close the client connection. From 2dd7f589eb970916a308095e4e9de1cc9dd72188 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 9 Sep 2015 21:25:36 -0500 Subject: [PATCH 057/746] Lowercase request GET/POST etc. --- channels/decorators.py | 12 ++++++------ channels/request.py | 16 ++++++++-------- docs/message-standards.rst | 8 ++++---- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/channels/decorators.py b/channels/decorators.py index bc5f116..03a4ecc 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -26,17 +26,17 @@ def http_session(func): """ @functools.wraps(func) def inner(message, *args, **kwargs): - if "COOKIES" not in message.content and "GET" not in message.content: - raise ValueError("No COOKIES or GET sent to consumer; this decorator can only be used on messages containing at least one.") + if "cookies" not in message.content and "get" not in message.content: + raise ValueError("No cookies or get sent to consumer; this decorator can only be used on messages containing at least one.") # Make sure there's a session key session_key = None - if "GET" in message.content: + if "get" in message.content: try: - session_key = message.content['GET'].get("session_key", [])[0] + session_key = message.content['get'].get("session_key", [])[0] except IndexError: pass - if "COOKIES" in message.content and session_key is None: - session_key = message.content['COOKIES'].get(settings.SESSION_COOKIE_NAME) + if "cookies" in message.content and session_key is None: + session_key = message.content['cookies'].get(settings.SESSION_COOKIE_NAME) # Make a session storage if session_key: session_engine = import_module(settings.SESSION_ENGINE) diff --git a/channels/request.py b/channels/request.py index dafcf49..1c975ef 100644 --- a/channels/request.py +++ b/channels/request.py @@ -10,10 +10,10 @@ def encode_request(request): """ # TODO: More stuff value = { - "GET": dict(request.GET.lists()), - "POST": dict(request.POST.lists()), - "COOKIES": request.COOKIES, - "META": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, + "get": dict(request.GET.lists()), + "post": dict(request.POST.lists()), + "cookies": request.COOKIES, + "meta": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, "path": request.path, "path_info": request.path_info, "method": request.method, @@ -27,10 +27,10 @@ def decode_request(value): Decodes a request JSONish value to a HttpRequest object. """ request = HttpRequest() - request.GET = CustomQueryDict(value['GET']) - request.POST = CustomQueryDict(value['POST']) - request.COOKIES = value['COOKIES'] - request.META = value['META'] + request.GET = CustomQueryDict(value['get']) + request.POST = CustomQueryDict(value['post']) + request.COOKIES = value['cookies'] + request.META = value['meta'] request.path = value['path'] request.method = value['method'] request.path_info = value['path_info'] diff --git a/docs/message-standards.rst b/docs/message-standards.rst index 405b23c..722e218 100644 --- a/docs/message-standards.rst +++ b/docs/message-standards.rst @@ -34,10 +34,10 @@ Standard channel name is ``http.request``. Contains the following keys: -* GET: List of (key, value) tuples of GET variables (keys and values are strings) -* POST: List of (key, value) tuples of POST variables (keys and values are strings) -* COOKIES: Dict of cookies as {cookie_name: cookie_value} (names and values are strings) -* META: Dict of HTTP headers and info as defined in the Django Request docs (names and values are strings) +* get: List of (key, value) tuples of GET variables (keys and values are strings) +* post: List of (key, value) tuples of POST variables (keys and values are strings) +* cookies: Dict of cookies as {cookie_name: cookie_value} (names and values are strings) +* meta: Dict of HTTP headers and info as defined in the Django Request docs (names and values are strings) * path: String, full path to the requested page, without query string or domain * path_info: String, like ``path`` but without any script prefix. Often just ``path``. * method: String, upper-cased HTTP method From 2839a4265e2d786826dbb76512dfcb6d8fd0aa55 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 9 Sep 2015 22:07:22 -0500 Subject: [PATCH 058/746] Fix tests. --- channels/tests/test_backends.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/channels/tests/test_backends.py b/channels/tests/test_backends.py index 1846bdc..a626d2b 100644 --- a/channels/tests/test_backends.py +++ b/channels/tests/test_backends.py @@ -10,7 +10,7 @@ class MemoryBackendTests(TestCase): backend_class = InMemoryChannelBackend def setUp(self): - self.backend = self.backend_class() + self.backend = self.backend_class(routing={}) def test_send_recv(self): """ @@ -33,7 +33,7 @@ class MemoryBackendTests(TestCase): self.assertEqual(message, {"value": "red"}) def test_message_expiry(self): - self.backend = self.backend_class(expiry=-100) + self.backend = self.backend_class(routing={}, expiry=-100) self.backend.send("test", {"value": "blue"}) channel, message = self.backend.receive_many(["test"]) self.assertIs(channel, None) @@ -72,7 +72,7 @@ class MemoryBackendTests(TestCase): self.assertEqual(message, {"value": "orange"}) def test_group_expiry(self): - self.backend = self.backend_class(expiry=-100) + self.backend = self.backend_class(routing={}, expiry=-100) self.backend.group_add("tgroup", "test") self.backend.group_add("tgroup", "test2") self.assertEqual( From e042da5bc14afc41671ee89bfcf568a7c3f41e24 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 9 Sep 2015 22:07:30 -0500 Subject: [PATCH 059/746] Add a bit of in-memory docs --- docs/backends.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/backends.rst b/docs/backends.rst index be9faac..e79dda9 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -8,6 +8,13 @@ you wish; the API is very simple and documented below. In-memory --------- +The in-memory backend is the simplest, and not really a backend as such; +it exists purely to enable Django to run in a "normal" mode where no Channels +functionality is available, just normal HTTP request processing. You should +never need to set it explicitly. + +This backend provides no network transparency or non-blocking guarantees. + Database -------- From eed6e5e607d36851ab3d5fbbf09994495abb8f00 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 9 Sep 2015 22:07:52 -0500 Subject: [PATCH 060/746] Don't waffle about ordering guarantees --- docs/message-standards.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/message-standards.rst b/docs/message-standards.rst index 722e218..8924048 100644 --- a/docs/message-standards.rst +++ b/docs/message-standards.rst @@ -11,8 +11,7 @@ In addition to the standards outlined below, each message may contain a separate connection and data receiving messages (like WebSockets) will only contain the connection and detailed client information in the first message; use the ``@channel_session`` decorator to persist this data to consumers of -the received data (the decorator will take care of handling persistence and -ordering guarantees on messages). +the received data (or something else based on ``reply_channel``). All messages must be able to be encoded as JSON; channel backends don't necessarily have to use JSON, but we consider it the lowest common denominator From 638bf260f823e333dd281f921aa8fb5f6d73e4ec Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Sep 2015 11:52:49 -0500 Subject: [PATCH 061/746] Fixed #6: Linearise decorator and better user session stuff --- channels/auth.py | 66 +++++++++++ channels/backends/base.py | 13 ++ channels/backends/database.py | 45 ++++++- channels/backends/memory.py | 20 ++++ channels/backends/redis_py.py | 15 +++ channels/decorators.py | 144 ++++++++++++----------- channels/interfaces/websocket_twisted.py | 2 +- channels/message.py | 7 ++ channels/worker.py | 4 + 9 files changed, 243 insertions(+), 73 deletions(-) create mode 100644 channels/auth.py diff --git a/channels/auth.py b/channels/auth.py new file mode 100644 index 0000000..261010c --- /dev/null +++ b/channels/auth.py @@ -0,0 +1,66 @@ +import functools + +from django.contrib import auth +from .decorators import channel_session, http_session + + +def transfer_user(from_session, to_session): + """ + Transfers user from HTTP session to channel session. + """ + to_session[auth.BACKEND_SESSION_KEY] = from_session[auth.BACKEND_SESSION_KEY] + to_session[auth.SESSION_KEY] = from_session[auth.SESSION_KEY] + to_session[auth.HASH_SESSION_KEY] = from_session[auth.HASH_SESSION_KEY] + + +def channel_session_user(func): + """ + Presents a message.user attribute obtained from a user ID in the channel + session, rather than in the http_session. Turns on channel session implicitly. + """ + @channel_session + @functools.wraps(func) + def inner(message, *args, **kwargs): + # If we didn't get a session, then we don't get a user + if not hasattr(message, "channel_session"): + raise ValueError("Did not see a channel session to get auth from") + if message.channel_session is None: + message.user = None + # Otherwise, be a bit naughty and make a fake Request with just + # a "session" attribute (later on, perhaps refactor contrib.auth to + # pass around session rather than request) + else: + fake_request = type("FakeRequest", (object, ), {"session": message.channel_session}) + message.user = auth.get_user(fake_request) + # Run the consumer + return func(message, *args, **kwargs) + return inner + + +def http_session_user(func): + """ + Wraps a HTTP or WebSocket consumer (or any consumer of messages + that provides a "COOKIES" attribute) to provide both a "session" + attribute and a "user" attibute, like AuthMiddleware does. + + This runs http_session() to get a session to hook auth off of. + If the user does not have a session cookie set, both "session" + and "user" will be None. + """ + @http_session + @functools.wraps(func) + def inner(message, *args, **kwargs): + # If we didn't get a session, then we don't get a user + if not hasattr(message, "http_session"): + raise ValueError("Did not see a http session to get auth from") + if message.http_session is None: + message.user = None + # Otherwise, be a bit naughty and make a fake Request with just + # a "session" attribute (later on, perhaps refactor contrib.auth to + # pass around session rather than request) + else: + fake_request = type("FakeRequest", (object, ), {"session": message.http_session}) + message.user = auth.get_user(fake_request) + # Run the consumer + return func(message, *args, **kwargs) + return inner diff --git a/channels/backends/base.py b/channels/backends/base.py index 84e874e..9baaf6c 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -93,3 +93,16 @@ class BaseChannelBackend(object): def __str__(self): return self.__class__.__name__ + + def lock_channel(self, channel): + """ + Attempts to get a lock on the named channel. Returns True if lock + obtained, False if lock not obtained. + """ + raise NotImplementedError() + + def unlock_channel(self, channel): + """ + Unlocks the named channel. Always succeeds. + """ + raise NotImplementedError() diff --git a/channels/backends/database.py b/channels/backends/database.py index 504f3ad..fdc866a 100644 --- a/channels/backends/database.py +++ b/channels/backends/database.py @@ -3,7 +3,7 @@ import json import datetime from django.apps.registry import Apps -from django.db import models, connections, DEFAULT_DB_ALIAS +from django.db import models, connections, DEFAULT_DB_ALIAS, IntegrityError from django.utils.functional import cached_property from django.utils.timezone import now @@ -71,6 +71,26 @@ class DatabaseChannelBackend(BaseChannelBackend): editor.create_model(Group) return Group + @cached_property + def lock_model(self): + """ + Initialises a new model to store groups; not done as part of a + models.py as we don't want to make it for most installs. + """ + # Make the model class + class Lock(models.Model): + channel = models.CharField(max_length=200, unique=True) + expiry = models.DateTimeField(db_index=True) + class Meta: + apps = Apps() + app_label = "channels" + db_table = "django_channel_locks" + # Ensure its table exists + if Lock._meta.db_table not in self.connection.introspection.table_names(self.connection.cursor()): + with self.connection.schema_editor() as editor: + editor.create_model(Lock) + return Lock + def send(self, channel, message): self.channel_model.objects.create( channel = channel, @@ -97,6 +117,7 @@ class DatabaseChannelBackend(BaseChannelBackend): # Include a 10-second grace period because that solves some clock sync self.channel_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() self.group_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() + self.lock_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() def group_add(self, group, channel, expiry=None): """ @@ -123,5 +144,27 @@ class DatabaseChannelBackend(BaseChannelBackend): self._clean_expired() return list(self.group_model.objects.filter(group=group).values_list("channel", flat=True)) + def lock_channel(self, channel, expiry=None): + """ + Attempts to get a lock on the named channel. Returns True if lock + obtained, False if lock not obtained. + """ + # We rely on the UNIQUE constraint for only-one-thread-wins on locks + try: + self.lock_model.objects.create( + channel = channel, + expiry = now() + datetime.timedelta(seconds=expiry or self.expiry), + ) + except IntegrityError: + return False + else: + return True + + def unlock_channel(self, channel): + """ + Unlocks the named channel. Always succeeds. + """ + self.lock_model.objects.filter(channel=channel).delete() + def __str__(self): return "%s(alias=%s)" % (self.__class__.__name__, self.connection.alias) diff --git a/channels/backends/memory.py b/channels/backends/memory.py index c398653..1d78363 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -5,6 +5,7 @@ from .base import BaseChannelBackend queues = {} groups = {} +locks = set() class InMemoryChannelBackend(BaseChannelBackend): """ @@ -72,3 +73,22 @@ class InMemoryChannelBackend(BaseChannelBackend): """ self._clean_expired() return groups.get(group, {}).keys() + + def lock_channel(self, channel): + """ + Attempts to get a lock on the named channel. Returns True if lock + obtained, False if lock not obtained. + """ + # Probably not perfect for race conditions, but close enough considering + # it shouldn't be used. + if channel not in locks: + locks.add(channel) + return True + else: + return False + + def unlock_channel(self, channel): + """ + Unlocks the named channel. Always succeeds. + """ + locks.discard(channel) diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 5c8671c..6af662f 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -101,5 +101,20 @@ class RedisChannelBackend(BaseChannelBackend): # TODO: send_group efficient implementation using Lua + def lock_channel(self, channel, expiry=None): + """ + Attempts to get a lock on the named channel. Returns True if lock + obtained, False if lock not obtained. + """ + key = "%s:lock:%s" % (self.prefix, channel) + return bool(self.connection.setnx(key, "1")) + + def unlock_channel(self, channel): + """ + Unlocks the named channel. Always succeeds. + """ + key = "%s:lock:%s" % (self.prefix, channel) + self.connection.delete(key) + def __str__(self): return "%s(host=%s, port=%s)" % (self.__class__.__name__, self.host, self.port) diff --git a/channels/decorators.py b/channels/decorators.py index 03a4ecc..280b493 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -3,16 +3,77 @@ import hashlib from importlib import import_module from django.conf import settings -from django.utils import six -from django.contrib import auth -from channels import channel_backends, DEFAULT_CHANNEL_BACKEND + +def linearize(func): + """ + Makes sure the contained consumer does not run at the same time other + consumers are running on messages with the same reply_channel. + + Required if you don't want weird things like a second consumer starting + up before the first has exited and saved its session. Doesn't guarantee + ordering, just linearity. + """ + @functools.wraps(func) + def inner(message, *args, **kwargs): + # Make sure there's a reply channel + if not message.reply_channel: + raise ValueError("No reply_channel sent to consumer; @no_overlap can only be used on messages containing it.") + # Get the lock, or re-queue + locked = message.channel_backend.lock_channel(message.reply_channel) + if not locked: + raise message.Requeue() + # OK, keep going + try: + return func(message, *args, **kwargs) + finally: + message.channel_backend.unlock_channel(message.reply_channel) + return inner + + +def channel_session(func): + """ + Provides a session-like object called "channel_session" to consumers + as a message attribute that will auto-persist across consumers with + the same incoming "reply_channel" value. + + Use this to persist data across the lifetime of a connection. + """ + @functools.wraps(func) + def inner(message, *args, **kwargs): + # Make sure there's a reply_channel + if not message.reply_channel: + raise ValueError("No reply_channel sent to consumer; @channel_session can only be used on messages containing it.") + # Make sure there's NOT a channel_session already + if hasattr(message, "channel_session"): + raise ValueError("channel_session decorator wrapped inside another channel_session decorator") + # Turn the reply_channel into a valid session key length thing. + # We take the last 24 bytes verbatim, as these are the random section, + # and then hash the remaining ones onto the start, and add a prefix + reply_name = message.reply_channel.name + session_key = "skt" + hashlib.md5(reply_name[:-24]).hexdigest()[:8] + reply_name[-24:] + # Make a session storage + session_engine = import_module(settings.SESSION_ENGINE) + session = session_engine.SessionStore(session_key=session_key) + # If the session does not already exist, save to force our + # session key to be valid. + if not session.exists(session.session_key): + session.save(must_create=True) + message.channel_session = session + # Run the consumer + try: + return func(message, *args, **kwargs) + finally: + # Persist session if needed + if session.modified: + session.save() + return inner def http_session(func): """ - Wraps a HTTP or WebSocket consumer (or any consumer of messages - that provides a "COOKIES" or "GET" attribute) to provide a "session" + Wraps a HTTP or WebSocket connect consumer (or any consumer of messages + that provides a "cooikies" or "get" attribute) to provide a "http_session" attribute that behaves like request.session; that is, it's hung off of a per-user session key that is saved in a cookie or passed as the "session_key" GET parameter. @@ -21,13 +82,16 @@ def http_session(func): don't have one - that's what SessionMiddleware is for, this is a simpler read-only version for more low-level code. - If a user does not have a session we can inflate, the "session" attribute will - be None, rather than an empty session you can write to. + If a message does not have a session we can inflate, the "session" attribute + will be None, rather than an empty session you can write to. """ @functools.wraps(func) def inner(message, *args, **kwargs): if "cookies" not in message.content and "get" not in message.content: - raise ValueError("No cookies or get sent to consumer; this decorator can only be used on messages containing at least one.") + raise ValueError("No cookies or get sent to consumer - cannot initialise http_session") + # Make sure there's NOT a http_session already + if hasattr(message, "http_session"): + raise ValueError("http_session decorator wrapped inside another http_session decorator") # Make sure there's a session key session_key = None if "get" in message.content: @@ -43,7 +107,7 @@ def http_session(func): session = session_engine.SessionStore(session_key=session_key) else: session = None - message.session = session + message.http_session = session # Run the consumer result = func(message, *args, **kwargs) # Persist session if needed (won't be saved if error happens) @@ -51,65 +115,3 @@ def http_session(func): session.save() return result return inner - - -def http_django_auth(func): - """ - Wraps a HTTP or WebSocket consumer (or any consumer of messages - that provides a "COOKIES" attribute) to provide both a "session" - attribute and a "user" attibute, like AuthMiddleware does. - - This runs http_session() to get a session to hook auth off of. - If the user does not have a session cookie set, both "session" - and "user" will be None. - """ - @http_session - @functools.wraps(func) - def inner(message, *args, **kwargs): - # If we didn't get a session, then we don't get a user - if not hasattr(message, "session"): - raise ValueError("Did not see a session to get auth from") - if message.session is None: - message.user = None - # Otherwise, be a bit naughty and make a fake Request with just - # a "session" attribute (later on, perhaps refactor contrib.auth to - # pass around session rather than request) - else: - fake_request = type("FakeRequest", (object, ), {"session": message.session}) - message.user = auth.get_user(fake_request) - # Run the consumer - return func(message, *args, **kwargs) - return inner - - -def channel_session(func): - """ - Provides a session-like object called "channel_session" to consumers - as a message attribute that will auto-persist across consumers with - the same incoming "reply_channel" value. - """ - @functools.wraps(func) - def inner(message, *args, **kwargs): - # Make sure there's a reply_channel in kwargs - if not message.reply_channel: - raise ValueError("No reply_channel sent to consumer; this decorator can only be used on messages containing it.") - # Turn the reply_channel into a valid session key length thing. - # We take the last 24 bytes verbatim, as these are the random section, - # and then hash the remaining ones onto the start, and add a prefix - # TODO: See if there's a better way of doing this - reply_name = message.reply_channel.name - session_key = "skt" + hashlib.md5(reply_name[:-24]).hexdigest()[:8] + reply_name[-24:] - # Make a session storage - session_engine = import_module(settings.SESSION_ENGINE) - session = session_engine.SessionStore(session_key=session_key) - # If the session does not already exist, save to force our session key to be valid - if not session.exists(session.session_key): - session.save() - message.channel_session = session - # Run the consumer - result = func(message, *args, **kwargs) - # Persist session if needed (won't be saved if error happens) - if session.modified: - session.save() - return result - return inner diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index 478d03b..ff5e947 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -18,7 +18,7 @@ class InterfaceProtocol(WebSocketServerProtocol): self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] self.request_info = { "path": request.path, - "GET": request.params, + "get": request.params, } def onOpen(self): diff --git a/channels/message.py b/channels/message.py index bc7eda5..c9a80bd 100644 --- a/channels/message.py +++ b/channels/message.py @@ -10,6 +10,13 @@ class Message(object): to use to reply to this message's end user, if that makes sense. """ + class Requeue(Exception): + """ + Raise this while processing a message to requeue it back onto the + channel. Useful if you're manually ensuring partial ordering, etc. + """ + pass + def __init__(self, content, channel, channel_backend, reply_channel=None): self.content = content self.channel = channel diff --git a/channels/worker.py b/channels/worker.py index db0a896..50de9bd 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -1,5 +1,6 @@ import traceback from .message import Message +from .utils import name_that_thing class Worker(object): @@ -31,5 +32,8 @@ class Worker(object): self.callback(channel, message) try: consumer(message) + except Message.Requeue: + self.channel_backend.send(channel, content) except: + print "Error processing message with consumer %s:" % name_that_thing(consumer) traceback.print_exc() From 814aa64e776e69bb54c37347f859d1bb7d4579b1 Mon Sep 17 00:00:00 2001 From: Faris Chebib Date: Thu, 10 Sep 2015 11:04:16 -0600 Subject: [PATCH 062/746] updated docs, updated gitignore --- .gitignore | 3 +++ docs/installation.rst | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.gitignore b/.gitignore index a49fd66..b8252fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.egg-info dist/ docs/_build +__pycache__/ +*.swp + diff --git a/docs/installation.rst b/docs/installation.rst index 06a7c5c..2becc63 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -19,3 +19,16 @@ Once that's done, you should add ``channels`` to your That's it! Once enabled, ``channels`` will integrate itself into Django and take control of the ``runserver`` command. See :doc:`getting-started` for more. + + +Installing the lastest development version +------------------------------------------ + +To install the latest version of Channels, clone the repo, change to the repo, +change to the repo directory, and pip install it into your current virtual +environment:: + + $ git clone git@github.com:andrewgodwin/channels.git + $ cd channels + $ + (environment) $ pip install -e . # the dot specifies the current repo From 15d29a0230d31aa981a11c63bba6ca9414421009 Mon Sep 17 00:00:00 2001 From: Faris Chebib Date: Thu, 10 Sep 2015 11:30:35 -0600 Subject: [PATCH 063/746] added external link to golang example --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index e91cc64..1f2de1e 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -46,7 +46,7 @@ The channels have capacity, so a load of producers can write lots of messages into a channel with no consumers and then a consumer can come along later and will start getting served those queued messages. -If you've used channels in Go, these are reasonably similar to those. The key +If you've used `channels in Go `_, these are reasonably similar to those. The key difference is that these channels are network-transparent; the implementations of channels we provide are all accessible across a network to consumers and producers running in different processes or on different machines. From 19130ebb05a2eea7fe4b9af942826fe58463af5c Mon Sep 17 00:00:00 2001 From: Faris Chebib Date: Thu, 10 Sep 2015 11:54:51 -0600 Subject: [PATCH 064/746] updated concepts and added six to reqs --- docs/concepts.rst | 2 +- setup.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 1f2de1e..cb5febb 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -161,7 +161,7 @@ and be less than 200 characters long. It's optional for a backend implementation to understand this - after all, it's only important at scale, where you want to shard the two types differently -- but it's present nonetheless. For more on scaling, and how to handle channel +— but it's present nonetheless. For more on scaling, and how to handle channel types if you're writing a backend or interface server, read :doc:`scaling`. Groups diff --git a/setup.py b/setup.py index a5e0547..845e3c1 100644 --- a/setup.py +++ b/setup.py @@ -10,4 +10,7 @@ setup( license='BSD', packages=find_packages(), include_package_data=True, + install_requires=[ + 'six', + ] ) From bd6f61de98597463ab7b198cf3e07b4cb575a90b Mon Sep 17 00:00:00 2001 From: Faris Chebib Date: Thu, 10 Sep 2015 12:00:47 -0600 Subject: [PATCH 065/746] updated getting started to include Group ref --- docs/getting-started.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 77ce6b0..6ed982e 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -123,6 +123,8 @@ And, even though channels will expire out, let's add an explicit ``disconnect`` handler to clean up as people disconnect (most channels will cleanly disconnect and get this called):: + from channels import Group + # Connected to websocket.disconnect def ws_disconnect(message): Group("chat").discard(message.reply_channel) From 275bfdf1e50954eef6d3349b1ecf6880c278d5b8 Mon Sep 17 00:00:00 2001 From: Faris Chebib Date: Thu, 10 Sep 2015 12:04:01 -0600 Subject: [PATCH 066/746] Group ref on keepalive --- docs/getting-started.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 6ed982e..5d210d9 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -105,6 +105,8 @@ so we can hook that up to re-add the channel (it's safe to add the channel to a group it's already in - similarly, it's safe to discard a channel from a group it's not in):: + from channels import Group + # Connected to websocket.keepalive def ws_keepalive(message): Group("chat").add(message.reply_channel) From d4de42d3b26212f25209c302e0679df860292316 Mon Sep 17 00:00:00 2001 From: Faris Chebib Date: Thu, 10 Sep 2015 12:04:52 -0600 Subject: [PATCH 067/746] Group ref on keepalive --- docs/getting-started.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 5d210d9..3d4c112 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -71,6 +71,8 @@ do any time. Let's try some WebSockets, and make a basic chat server! Delete that consumer and its routing - we'll want the normal Django view layer to serve HTTP requests from now on - and make this WebSocket consumer instead:: + from channels import Group + def ws_add(message): Group("chat").add(message.reply_channel) From 27f54ad23b53aad4331e4fc1efb4aa3614936528 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Sep 2015 13:08:09 -0500 Subject: [PATCH 068/746] Rework getting started --- docs/getting-started.rst | 205 ++++++++++++++++++++++----------------- 1 file changed, 115 insertions(+), 90 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 77ce6b0..bedf07a 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -229,105 +229,31 @@ like, so you can understand when they're called. If you run three or four copies of ``runworker`` you'll probably be able to see the tasks running on different workers. -Authentication --------------- - -Now, of course, a WebSocket solution is somewhat limited in scope without the -ability to live with the rest of your website - in particular, we want to make -sure we know what user we're talking to, in case we have things like private -chat channels (we don't want a solution where clients just ask for the right -channels, as anyone could change the code and just put in private channel names) - -It can also save you having to manually make clients ask for what they want to -see; if I see you open a WebSocket to my "updates" endpoint, and I know which -user ID, I can just auto-add that channel to all the relevant groups (mentions -of that user, for example). - -Handily, as WebSockets start off using the HTTP protocol, they have a lot of -familiar features, including a path, GET parameters, and cookies. We'd like to -use these to hook into the familiar Django session and authentication systems; -after all, WebSockets are no good unless we can identify who they belong to -and do things securely. - -In addition, we don't want the interface servers storing data or trying to run -authentication; they're meant to be simple, lean, fast processes without much -state, and so we'll need to do our authentication inside our consumer functions. - -Fortunately, because Channels has standardised WebSocket event -:doc:`message-standards`, it ships with decorators that help you with -authentication, as well as using Django's session framework (which authentication -relies on). Channels can use Django sessions either from cookies (if you're running your websocket -server on the same port as your main site, which requires a reverse proxy that -understands WebSockets), or from a ``session_key`` GET parameter, which -is much more portable, and works in development where you need to run a separate -WebSocket server (by default, on port 9000). - -All we need to do is add the ``django_http_auth`` decorator to our views, -and we'll get extra ``session`` and ``user`` keyword attributes on ``message`` we can use; -let's make one where users can only chat to people with the same first letter -of their username:: - - from channels import Channel, Group - from channels.decorators import django_http_auth - - @django_http_auth - def ws_add(message): - Group("chat-%s" % message.user.username[0]).add(message.reply_channel) - - @django_http_auth - def ws_message(message): - Group("chat-%s" % message.user.username[0]).send(message.content) - - @django_http_auth - def ws_disconnect(message): - Group("chat-%s" % message.user.username[0]).discard(message.reply_channel) - -Now, when we connect to the WebSocket we'll have to remember to provide the -Django session ID as part of the URL, like this:: - - socket = new WebSocket("ws://127.0.0.1:9000/?session_key=abcdefg"); - -You can get the current session key in a template with ``{{ request.session.session_key }}``. -Note that Channels can't work with signed cookie sessions - since only HTTP -responses can set cookies, it needs a backend it can write to separately to -store state. - Persisting Data --------------- -Doing chatrooms by username first letter is a nice simple example, but it's +Echoing messages is a nice simple example, but it's skirting around the real design pattern - persistent state for connections. -A user may open our chat site and select the chatroom to join themselves, so we -should let them send this request in the initial WebSocket connection, -check they're allowed to access it, and then remember which room a socket is -connected to when they send a message in so we know which group to send it to. +Let's consider a basic chat site where a user requests a chat room upon initial +connection, as part of the query string (e.g. ``http://host/websocket?room=abc``). -The ``reply_channel`` is our unique pointer to the open WebSocket - as you've -seen, we do all our operations on it - but it's not something we can annotate -with data; it's just a simple string, and even if we hack around and set -attributes on it that's not going to carry over to other workers. +The ``reply_channel`` attribute you've seen before is our unique pointer to the +open WebSocket - because it varies between different clients, it's how we can +keep track of "who" a message is from. Remember, Channels is network-trasparent +and can run on multiple workers, so you can't just store things locally in +global variables or similar. -Instead, the solution is to persist information keyed by the send channel in +Instead, the solution is to persist information keyed by the ``reply_channel`` in some other data store - sound familiar? This is what Django's session framework does for HTTP requests, only there it uses cookies as the lookup key rather than the ``reply_channel``. -Now, as you saw above, you can use the ``django_http_auth`` decorator to get -both a ``user`` and a ``session`` attribute on your message - and, -indeed, there is a ``http_session`` decorator that will just give you -the ``session`` attribute. - -However, that session is based on cookies, and so follows the user round the -site - it's great for information that should persist across all WebSocket and -HTTP connections, but not great for information that is specific to a single -WebSocket (such as "which chatroom should this socket be connected to"). For -this reason, Channels also provides a ``channel_session`` decorator, -which adds a ``channel_session`` attribute to the message; this works just like -the normal ``session`` attribute, and persists to the same storage, but varies -per-channel rather than per-cookie. +Channels provides a ``channel_session`` decorator for this purpose - it +provides you with an attribute called ``message.channel_session`` that acts +just like a normal Django session. Let's use it now to build a chat server that expects you to pass a chatroom -name in the path of your WebSocket request (we'll ignore auth for now):: +name in the path of your WebSocket request (we'll ignore auth for now - that's next):: from channels import Channel from channels.decorators import channel_session @@ -358,9 +284,102 @@ name in the path of your WebSocket request (we'll ignore auth for now):: If you play around with it from the console (or start building a simple JavaScript chat client that appends received messages to a div), you'll see -that you can now request which chat room you want in the initial request. We -could easily add in the auth decorator here too and do an initial check in -``connect`` that the user had permission to join that chatroom. +that you can now request which chat room you want in the initial request. + +Authentication +-------------- + +Now, of course, a WebSocket solution is somewhat limited in scope without the +ability to live with the rest of your website - in particular, we want to make +sure we know what user we're talking to, in case we have things like private +chat channels (we don't want a solution where clients just ask for the right +channels, as anyone could change the code and just put in private channel names) + +It can also save you having to manually make clients ask for what they want to +see; if I see you open a WebSocket to my "updates" endpoint, and I know which +user you are, I can just auto-add that channel to all the relevant groups (mentions +of that user, for example). + +Handily, as WebSockets start off using the HTTP protocol, they have a lot of +familiar features, including a path, GET parameters, and cookies. We'd like to +use these to hook into the familiar Django session and authentication systems; +after all, WebSockets are no good unless we can identify who they belong to +and do things securely. + +In addition, we don't want the interface servers storing data or trying to run +authentication; they're meant to be simple, lean, fast processes without much +state, and so we'll need to do our authentication inside our consumer functions. + +Fortunately, because Channels has standardised WebSocket event +:doc:`message-standards`, it ships with decorators that help you with +both authentication and getting the underlying Django session (which is what +Django authentication relies on). + +Channels can use Django sessions either from cookies (if you're running your websocket +server on the same port as your main site, which requires a reverse proxy that +understands WebSockets), or from a ``session_key`` GET parameter, which +is much more portable, and works in development where you need to run a separate +WebSocket server (by default, on port 9000). + +You get access to a user's normal Django session using the ``http_session`` +decorator - that gives you a ``message.http_session`` attribute that behaves +just like ``request.session``. You can go one further and use ``http_session_user`` +which will provide a ``message.user`` attribute as well as the session attribute. + +Now, one thing to note is that you only get the detailed HTTP information +during the ``connect`` message of a WebSocket connection (you can read more +about what you get when in :doc:`message-standards`) - this means we're not +wasting bandwidth sending the same information over the wire needlessly. + +This also means we'll have to grab the user in the connection handler and then +store it in the session; thankfully, Channels ships with both a ``channel_session_user`` +decorator that works like the ``http_session_user`` decorator you saw above but +loads the user from the *channel* session rather than the *HTTP* session, +and a function called ``transfer_user`` which replicates a user from one session +to another. + +Bringing that all together, let's make a chat server one where users can only +chat to people with the same first letter of their username:: + + from channels import Channel, Group + from channels.decorators import channel_session + from channels.auth import http_session_user, channel_session_user, transfer_user + + # Connected to websocket.connect + @channel_session + @http_session_user + def ws_add(message): + # Copy user from HTTP to channel session + transfer_user(message.http_session, message.channel_session) + # Add them to the right group + Group("chat-%s" % message.user.username[0]).add(message.reply_channel) + + # Connected to websocket.keepalive + @channel_session_user + def ws_keepalive(message): + # Keep them in the right group + Group("chat-%s" % message.user.username[0]).add(message.reply_channel) + + # Connected to websocket.receive + @channel_session_user + def ws_message(message): + Group("chat-%s" % message.user.username[0]).send(message.content) + + # Connected to websocket.disconnect + @channel_session_user + def ws_disconnect(message): + Group("chat-%s" % message.user.username[0]).discard(message.reply_channel) + +Now, when we connect to the WebSocket we'll have to remember to provide the +Django session ID as part of the URL, like this:: + + socket = new WebSocket("ws://127.0.0.1:9000/?session_key=abcdefg"); + +You can get the current session key in a template with ``{{ request.session.session_key }}``. +Note that Channels can't work with signed cookie sessions - since only HTTP +responses can set cookies, it needs a backend it can write to separately to +store state. + Models ------ @@ -434,6 +453,12 @@ command run via ``cron``. If we wanted to write a bot, too, we could put its listening logic inside the ``chat-messages`` consumer, as every message would pass through it. +Linearization +------------- + +TODO + + Next Steps ---------- From d563f7748c08c0a05f4d176fcd34617f9af80176 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Sep 2015 13:20:58 -0500 Subject: [PATCH 069/746] Linearize docs --- docs/getting-started.rst | 62 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index bedf07a..2287f24 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -456,7 +456,67 @@ pass through it. Linearization ------------- -TODO +There's one final concept we want to introduce you to before you go on to build +sites with Channels - linearizing consumers. + +Because Channels is a distributed system that can have many workers, by default +it's entirely feasible for a WebSocket interface server to send out a ``connect`` +and a ``receive`` message close enough together that a second worker will pick +up and start processing the ``receive`` message before the first worker has +finished processing the ``connect`` worker. + +This is particularly annoying if you're storing things in the session in the +``connect`` consumer and trying to get them in the ``receive`` consumer - because +the ``connect`` consumer hasn't exited, its session hasn't saved. You'd get the +same effect if someone tried to request a view before the login view had finished +processing, but there you're not expecting that page to run after the login, +whereas you'd naturally expect ``receive`` to run after ``connect``. + +But, of course, Channels has a solution - the ``linearize`` decorator. Any +handler decorated with this will use locking to ensure it does not run at the +same time as any other view with ``linearize`` **on messages with the same reply channel**. +That means your site will happily mutitask with lots of different people's messages, +but if two happen to try to run at the same time for the same client, they'll +be deconflicted. + +There's a small cost to using ``linearize``, which is why it's an optional +decorator, but generally you'll want to use it for most session-based WebSocket +and other "continuous protocol" things. Here's an example, improving our +first-letter-of-username chat from earlier:: + + from channels import Channel, Group + from channels.decorators import channel_session, linearize + from channels.auth import http_session_user, channel_session_user, transfer_user + + # Connected to websocket.connect + @linearize + @channel_session + @http_session_user + def ws_add(message): + # Copy user from HTTP to channel session + transfer_user(message.http_session, message.channel_session) + # Add them to the right group + Group("chat-%s" % message.user.username[0]).add(message.reply_channel) + + # Connected to websocket.keepalive + # We don't linearize as we know this will happen a decent time after add + @channel_session_user + def ws_keepalive(message): + # Keep them in the right group + Group("chat-%s" % message.user.username[0]).add(message.reply_channel) + + # Connected to websocket.receive + @linearize + @channel_session_user + def ws_message(message): + Group("chat-%s" % message.user.username[0]).send(message.content) + + # Connected to websocket.disconnect + # We don't linearize as even if this gets an empty session, the group + # will auto-discard after the expiry anyway. + @channel_session_user + def ws_disconnect(message): + Group("chat-%s" % message.user.username[0]).discard(message.reply_channel) Next Steps From 414a4ab460893d85f82371f3af62bbeb44578e28 Mon Sep 17 00:00:00 2001 From: Faris Chebib Date: Thu, 10 Sep 2015 12:26:08 -0600 Subject: [PATCH 070/746] updated print statements --- channels/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/worker.py b/channels/worker.py index 50de9bd..b54dc44 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -35,5 +35,5 @@ class Worker(object): except Message.Requeue: self.channel_backend.send(channel, content) except: - print "Error processing message with consumer %s:" % name_that_thing(consumer) + print("Error processing message with consumer {}:".format(name_that_thing(consumer))) traceback.print_exc() From f69ad33747515eb224204eee4e142bf41f371514 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Sep 2015 14:41:38 -0500 Subject: [PATCH 071/746] Fixed #9: Add asyncio websocket server --- channels/interfaces/websocket_asyncio.py | 155 ++++++++++++++++++++ channels/management/commands/runwsserver.py | 19 ++- 2 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 channels/interfaces/websocket_asyncio.py diff --git a/channels/interfaces/websocket_asyncio.py b/channels/interfaces/websocket_asyncio.py new file mode 100644 index 0000000..5dc3fe3 --- /dev/null +++ b/channels/interfaces/websocket_asyncio.py @@ -0,0 +1,155 @@ +import asyncio +import django +import time + +from autobahn.asyncio.websocket import WebSocketServerProtocol, WebSocketServerFactory +from collections import deque + +from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND + + +class InterfaceProtocol(WebSocketServerProtocol): + """ + Protocol which supports WebSockets and forwards incoming messages to + the websocket channels. + """ + + def onConnect(self, request): + self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] + self.request_info = { + "path": request.path, + "get": request.params, + } + + def onOpen(self): + # Make sending channel + self.reply_channel = Channel.new_name("!websocket.send") + self.request_info["reply_channel"] = self.reply_channel + self.last_keepalive = time.time() + self.factory.protocols[self.reply_channel] = self + # Send news that this channel is open + Channel("websocket.connect").send(self.request_info) + + def onMessage(self, payload, isBinary): + if isBinary: + Channel("websocket.receive").send(dict( + self.request_info, + content = payload, + binary = True, + )) + else: + Channel("websocket.receive").send(dict( + self.request_info, + content = payload.decode("utf8"), + binary = False, + )) + + def serverSend(self, content, binary=False, **kwargs): + """ + Server-side channel message to send a message. + """ + if binary: + self.sendMessage(content, binary) + else: + self.sendMessage(content.encode("utf8"), binary) + + def serverClose(self): + """ + Server-side channel message to close the socket + """ + self.sendClose() + + def onClose(self, wasClean, code, reason): + if hasattr(self, "reply_channel"): + del self.factory.protocols[self.reply_channel] + Channel("websocket.disconnect").send(self.request_info) + + def sendKeepalive(self): + """ + Sends a keepalive packet on the keepalive channel. + """ + Channel("websocket.keepalive").send(self.request_info) + self.last_keepalive = time.time() + + +class InterfaceFactory(WebSocketServerFactory): + """ + Factory which keeps track of its open protocols' receive channels + and can dispatch to them. + """ + + # TODO: Clean up dead protocols if needed? + + def __init__(self, *args, **kwargs): + super(InterfaceFactory, self).__init__(*args, **kwargs) + self.protocols = {} + + def reply_channels(self): + return self.protocols.keys() + + def dispatch_send(self, channel, message): + if message.get("close", False): + self.protocols[channel].serverClose() + else: + self.protocols[channel].serverSend(**message) + + +class WebsocketAsyncioInterface(object): + """ + Easy API to run a WebSocket interface server using Twisted. + Integrates the channel backend by running it in a separate thread, using + the always-compatible polling style. + """ + + def __init__(self, channel_backend, port=9000): + self.channel_backend = channel_backend + self.port = port + + def run(self): + self.factory = InterfaceFactory("ws://0.0.0.0:%i" % self.port, debug=False) + self.factory.protocol = InterfaceProtocol + self.loop = asyncio.get_event_loop() + coro = self.loop.create_server(self.factory, '0.0.0.0', 9000) + server = self.loop.run_until_complete(coro) + self.loop.run_in_executor(None, self.backend_reader) + self.loop.call_later(1, self.keepalive_sender) + try: + self.loop.run_forever() + except KeyboardInterrupt: + pass + finally: + server.close() + self.loop.close() + + def backend_reader(self): + """ + Run in a separate thread; reads messages from the backend. + """ + while True: + channels = self.factory.reply_channels() + # Quit if reactor is stopping + if not self.loop.is_running(): + return + # Don't do anything if there's no channels to listen on + if channels: + channel, message = self.channel_backend.receive_many(channels) + else: + time.sleep(0.1) + continue + # Wait around if there's nothing received + if channel is None: + time.sleep(0.05) + continue + # Deal with the message + self.factory.dispatch_send(channel, message) + + def keepalive_sender(self): + """ + Sends keepalive messages for open WebSockets every + (channel_backend expiry / 2) seconds. + """ + expiry_window = int(self.channel_backend.expiry / 2) + for protocol in self.factory.protocols.values(): + if time.time() - protocol.last_keepalive > expiry_window: + protocol.sendKeepalive() + self.loop.call_later(1, self.keepalive_sender) diff --git a/channels/management/commands/runwsserver.py b/channels/management/commands/runwsserver.py index 7b4d296..b1525d9 100644 --- a/channels/management/commands/runwsserver.py +++ b/channels/management/commands/runwsserver.py @@ -1,7 +1,6 @@ import time from django.core.management import BaseCommand, CommandError from channels import channel_backends, DEFAULT_CHANNEL_BACKEND -from channels.interfaces.websocket_twisted import WebsocketTwistedInterface class Command(BaseCommand): @@ -20,7 +19,17 @@ class Command(BaseCommand): ) # Run the interface port = options.get("port", None) or 9000 - self.stdout.write("Running Twisted/Autobahn WebSocket interface server") - self.stdout.write(" Channel backend: %s" % channel_backend) - self.stdout.write(" Listening on: ws://0.0.0.0:%i" % port) - WebsocketTwistedInterface(channel_backend=channel_backend, port=port).run() + try: + import asyncio + except ImportError: + from channels.interfaces.websocket_twisted import WebsocketTwistedInterface + self.stdout.write("Running Twisted/Autobahn WebSocket interface server") + self.stdout.write(" Channel backend: %s" % channel_backend) + self.stdout.write(" Listening on: ws://0.0.0.0:%i" % port) + WebsocketTwistedInterface(channel_backend=channel_backend, port=port).run() + else: + from channels.interfaces.websocket_asyncio import WebsocketAsyncioInterface + self.stdout.write("Running asyncio/Autobahn WebSocket interface server") + self.stdout.write(" Channel backend: %s" % channel_backend) + self.stdout.write(" Listening on: ws://0.0.0.0:%i" % port) + WebsocketAsyncioInterface(channel_backend=channel_backend, port=port).run() From 73d50a46952a8b13614e4cea6dc14ca63b6b4765 Mon Sep 17 00:00:00 2001 From: Faris Chebib Date: Thu, 10 Sep 2015 14:46:24 -0600 Subject: [PATCH 072/746] updated typo in docs example --- docs/getting-started.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 5b1d88e..f2d4a36 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -138,7 +138,7 @@ Now, that's taken care of adding and removing WebSocket send channels for the we're not going to store a history of messages or anything and just replay any message sent in to all connected clients. Here's all the code:: - from channels import Channel, Group + from channels import Group # Connected to websocket.connect and websocket.keepalive def ws_add(message): @@ -261,7 +261,7 @@ just like a normal Django session. Let's use it now to build a chat server that expects you to pass a chatroom name in the path of your WebSocket request (we'll ignore auth for now - that's next):: - from channels import Channel + from channels import Group from channels.decorators import channel_session # Connected to websocket.connect @@ -281,7 +281,7 @@ name in the path of your WebSocket request (we'll ignore auth for now - that's n # Connected to websocket.receive @channel_session def ws_message(message): - Group("chat-%s" % message.channel_session['room']).send(content) + Group("chat-%s" % message.channel_session['room']).send(message.content) # Connected to websocket.disconnect @channel_session From bd1553556d38f61d88d8cbbbde0bc50f4afc45b8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Sep 2015 16:00:31 -0500 Subject: [PATCH 073/746] Refactor websocket servers a bit to share logic --- channels/interfaces/websocket_asyncio.py | 96 ++------------------ channels/interfaces/websocket_autobahn.py | 101 ++++++++++++++++++++++ channels/interfaces/websocket_twisted.py | 94 +------------------- 3 files changed, 109 insertions(+), 182 deletions(-) create mode 100644 channels/interfaces/websocket_autobahn.py diff --git a/channels/interfaces/websocket_asyncio.py b/channels/interfaces/websocket_asyncio.py index 5dc3fe3..55c9ea3 100644 --- a/channels/interfaces/websocket_asyncio.py +++ b/channels/interfaces/websocket_asyncio.py @@ -1,97 +1,9 @@ import asyncio -import django import time from autobahn.asyncio.websocket import WebSocketServerProtocol, WebSocketServerFactory -from collections import deque -from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND - - -class InterfaceProtocol(WebSocketServerProtocol): - """ - Protocol which supports WebSockets and forwards incoming messages to - the websocket channels. - """ - - def onConnect(self, request): - self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - self.request_info = { - "path": request.path, - "get": request.params, - } - - def onOpen(self): - # Make sending channel - self.reply_channel = Channel.new_name("!websocket.send") - self.request_info["reply_channel"] = self.reply_channel - self.last_keepalive = time.time() - self.factory.protocols[self.reply_channel] = self - # Send news that this channel is open - Channel("websocket.connect").send(self.request_info) - - def onMessage(self, payload, isBinary): - if isBinary: - Channel("websocket.receive").send(dict( - self.request_info, - content = payload, - binary = True, - )) - else: - Channel("websocket.receive").send(dict( - self.request_info, - content = payload.decode("utf8"), - binary = False, - )) - - def serverSend(self, content, binary=False, **kwargs): - """ - Server-side channel message to send a message. - """ - if binary: - self.sendMessage(content, binary) - else: - self.sendMessage(content.encode("utf8"), binary) - - def serverClose(self): - """ - Server-side channel message to close the socket - """ - self.sendClose() - - def onClose(self, wasClean, code, reason): - if hasattr(self, "reply_channel"): - del self.factory.protocols[self.reply_channel] - Channel("websocket.disconnect").send(self.request_info) - - def sendKeepalive(self): - """ - Sends a keepalive packet on the keepalive channel. - """ - Channel("websocket.keepalive").send(self.request_info) - self.last_keepalive = time.time() - - -class InterfaceFactory(WebSocketServerFactory): - """ - Factory which keeps track of its open protocols' receive channels - and can dispatch to them. - """ - - # TODO: Clean up dead protocols if needed? - - def __init__(self, *args, **kwargs): - super(InterfaceFactory, self).__init__(*args, **kwargs) - self.protocols = {} - - def reply_channels(self): - return self.protocols.keys() - - def dispatch_send(self, channel, message): - if message.get("close", False): - self.protocols[channel].serverClose() - else: - self.protocols[channel].serverSend(**message) +from .websocket_autobahn import get_protocol, get_factory class WebsocketAsyncioInterface(object): @@ -106,8 +18,8 @@ class WebsocketAsyncioInterface(object): self.port = port def run(self): - self.factory = InterfaceFactory("ws://0.0.0.0:%i" % self.port, debug=False) - self.factory.protocol = InterfaceProtocol + self.factory = get_factory(WebSocketServerFactory)("ws://0.0.0.0:%i" % self.port, debug=False) + self.factory.protocol = get_protocol(WebSocketServerProtocol) self.loop = asyncio.get_event_loop() coro = self.loop.create_server(self.factory, '0.0.0.0', 9000) server = self.loop.run_until_complete(coro) @@ -125,6 +37,8 @@ class WebsocketAsyncioInterface(object): """ Run in a separate thread; reads messages from the backend. """ + # Wait for main loop to start + time.sleep(0.5) while True: channels = self.factory.reply_channels() # Quit if reactor is stopping diff --git a/channels/interfaces/websocket_autobahn.py b/channels/interfaces/websocket_autobahn.py new file mode 100644 index 0000000..dae31e0 --- /dev/null +++ b/channels/interfaces/websocket_autobahn.py @@ -0,0 +1,101 @@ +import time + +from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND + + +def get_protocol(base): + + class InterfaceProtocol(base): + """ + Protocol which supports WebSockets and forwards incoming messages to + the websocket channels. + """ + + def onConnect(self, request): + self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] + self.request_info = { + "path": request.path, + "get": request.params, + } + + def onOpen(self): + # Make sending channel + self.reply_channel = Channel.new_name("!websocket.send") + self.request_info["reply_channel"] = self.reply_channel + self.last_keepalive = time.time() + self.factory.protocols[self.reply_channel] = self + # Send news that this channel is open + Channel("websocket.connect").send(self.request_info) + + def onMessage(self, payload, isBinary): + if isBinary: + Channel("websocket.receive").send({ + "reply_channel": self.reply_channel, + "content": payload, + "binary": True, + }) + else: + Channel("websocket.receive").send({ + "reply_channel": self.reply_channel, + "content": payload.decode("utf8"), + "binary": False, + }) + + def serverSend(self, content, binary=False, **kwargs): + """ + Server-side channel message to send a message. + """ + if binary: + self.sendMessage(content, binary) + else: + self.sendMessage(content.encode("utf8"), binary) + + def serverClose(self): + """ + Server-side channel message to close the socket + """ + self.sendClose() + + def onClose(self, wasClean, code, reason): + if hasattr(self, "reply_channel"): + del self.factory.protocols[self.reply_channel] + Channel("websocket.disconnect").send({ + "reply_channel": self.reply_channel, + }) + + def sendKeepalive(self): + """ + Sends a keepalive packet on the keepalive channel. + """ + Channel("websocket.keepalive").send({ + "reply_channel": self.reply_channel, + }) + self.last_keepalive = time.time() + + return InterfaceProtocol + + +def get_factory(base): + + class InterfaceFactory(base): + """ + Factory which keeps track of its open protocols' receive channels + and can dispatch to them. + """ + + # TODO: Clean up dead protocols if needed? + + def __init__(self, *args, **kwargs): + super(InterfaceFactory, self).__init__(*args, **kwargs) + self.protocols = {} + + def reply_channels(self): + return self.protocols.keys() + + def dispatch_send(self, channel, message): + if message.get("close", False): + self.protocols[channel].serverClose() + else: + self.protocols[channel].serverSend(**message) + + return InterfaceFactory diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index ff5e947..e6055f1 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -1,97 +1,9 @@ -import django import time from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory -from collections import deque from twisted.internet import reactor -from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND - - -class InterfaceProtocol(WebSocketServerProtocol): - """ - Protocol which supports WebSockets and forwards incoming messages to - the websocket channels. - """ - - def onConnect(self, request): - self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - self.request_info = { - "path": request.path, - "get": request.params, - } - - def onOpen(self): - # Make sending channel - self.reply_channel = Channel.new_name("!websocket.send") - self.request_info["reply_channel"] = self.reply_channel - self.last_keepalive = time.time() - self.factory.protocols[self.reply_channel] = self - # Send news that this channel is open - Channel("websocket.connect").send(self.request_info) - - def onMessage(self, payload, isBinary): - if isBinary: - Channel("websocket.receive").send(dict( - self.request_info, - content = payload, - binary = True, - )) - else: - Channel("websocket.receive").send(dict( - self.request_info, - content = payload.decode("utf8"), - binary = False, - )) - - def serverSend(self, content, binary=False, **kwargs): - """ - Server-side channel message to send a message. - """ - if binary: - self.sendMessage(content, binary) - else: - self.sendMessage(content.encode("utf8"), binary) - - def serverClose(self): - """ - Server-side channel message to close the socket - """ - self.sendClose() - - def onClose(self, wasClean, code, reason): - if hasattr(self, "reply_channel"): - del self.factory.protocols[self.reply_channel] - Channel("websocket.disconnect").send(self.request_info) - - def sendKeepalive(self): - """ - Sends a keepalive packet on the keepalive channel. - """ - Channel("websocket.keepalive").send(self.request_info) - self.last_keepalive = time.time() - - -class InterfaceFactory(WebSocketServerFactory): - """ - Factory which keeps track of its open protocols' receive channels - and can dispatch to them. - """ - - # TODO: Clean up dead protocols if needed? - - def __init__(self, *args, **kwargs): - super(InterfaceFactory, self).__init__(*args, **kwargs) - self.protocols = {} - - def reply_channels(self): - return self.protocols.keys() - - def dispatch_send(self, channel, message): - if message.get("close", False): - self.protocols[channel].serverClose() - else: - self.protocols[channel].serverSend(**message) +from .websocket_autobahn import get_protocol, get_factory class WebsocketTwistedInterface(object): @@ -106,8 +18,8 @@ class WebsocketTwistedInterface(object): self.port = port def run(self): - self.factory = InterfaceFactory("ws://0.0.0.0:%i" % self.port, debug=False) - self.factory.protocol = InterfaceProtocol + self.factory = get_factory(WebSocketServerFactory)("ws://0.0.0.0:%i" % self.port, debug=False) + self.factory.protocol = get_protocol(WebSocketServerProtocol) reactor.listenTCP(self.port, self.factory) reactor.callInThread(self.backend_reader) reactor.callLater(1, self.keepalive_sender) From 655213eff9f7d9643e264df6d3fff411434a2d73 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Sep 2015 16:00:39 -0500 Subject: [PATCH 074/746] 0.8 --- channels/__init__.py | 2 +- docs/index.rst | 1 + docs/releases/0.8.rst | 22 ++++++++++++++++++++++ docs/releases/index.rst | 7 +++++++ setup.py | 2 +- 5 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 docs/releases/0.8.rst create mode 100644 docs/releases/index.rst diff --git a/channels/__init__.py b/channels/__init__.py index 2ee5420..8fd164c 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.1" +__version__ = "0.8" # Load backends, using settings if available (else falling back to a default) DEFAULT_CHANNEL_BACKEND = "default" diff --git a/docs/index.rst b/docs/index.rst index a1c8d5f..d246cee 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,3 +31,4 @@ Contents: scaling backends faqs + releases/index diff --git a/docs/releases/0.8.rst b/docs/releases/0.8.rst new file mode 100644 index 0000000..ddeac26 --- /dev/null +++ b/docs/releases/0.8.rst @@ -0,0 +1,22 @@ +0.8 (2015-09-10) +---------------- + +This release reworks a few of the core concepts to make the channel layer +more efficient and user friendly: + +* Channel names now do not start with ``django``, and are instead just ``http.request``, etc. + +* HTTP headers/GET/etc are only sent with ``websocket.connect`` rather than all websocket requests, + to save a lot of bandwidth in the channel layer. + +* The session/user decorators were renamed, and a ``@channel_session_user`` and ``transfer_user`` set of functions + added to allow moving the user details from the HTTP session to the channel session in the ``connect`` consumer. + +* A ``@linearize`` decorator was added to help ensure a ``connect``/``receive`` pair are not executed + simultanously on two different workers. + +* Channel backends gained locking mechanisms to support the ``linearize`` feature. + +* ``runwsserver`` will use asyncio rather than Twisted if it's available. + +* Message formats have been made a bit more consistent. diff --git a/docs/releases/index.rst b/docs/releases/index.rst new file mode 100644 index 0000000..231771f --- /dev/null +++ b/docs/releases/index.rst @@ -0,0 +1,7 @@ +Release Notes +------------- + +.. toctree:: + :maxdepth: 1 + + 0.8 diff --git a/setup.py b/setup.py index 845e3c1..d8c5394 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup setup( name='channels', - version="0.7", + version="0.8", url='http://github.com/andrewgodwin/django-channels', author='Andrew Godwin', author_email='andrew@aeracode.org', From 4a8bae272b5a8285104daef96b7f21b11129c1f0 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Sep 2015 16:34:28 -0500 Subject: [PATCH 075/746] Update docs to recommend doing routing not in settings --- docs/concepts.rst | 4 ++-- docs/getting-started.rst | 48 +++++++++++++++++----------------------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index cb5febb..5bbf21b 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -65,9 +65,9 @@ you can write a function to consume a channel, like so:: def my_consumer(message): pass -And then assign a channel to it like this in the channel backend settings:: +And then assign a channel to it like this in the channel routing:: - "ROUTING": { + channel_routing = { "some-channel": "myapp.consumers.my_consumer", } diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 5b1d88e..6158fdf 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -44,21 +44,25 @@ For now, we want to override the *channel routing* so that, rather than going to the URL resolver and our normal view stack, all HTTP requests go to our custom consumer we wrote above. Here's what that looks like:: + # In settings.py CHANNEL_BACKENDS = { "default": { "BACKEND": "channels.backends.database.DatabaseChannelBackend", - "ROUTING": { - "http.request": "myproject.myapp.consumers.http_consumer", - }, + "ROUTING": "myproject.routing.channel_routing", }, } + # In routing.py + channel_routing = { + "http.request": "myproject.myapp.consumers.http_consumer", + } + As you can see, this is a little like Django's ``DATABASES`` setting; there are named channel backends, with a default one called ``default``. Each backend needs a class specified which powers it - we'll come to the options there later - -and a routing scheme, which can either be defined directly as a dict or as -a string pointing to a dict in another file (if you'd rather keep it outside -settings). +and a routing scheme, which points to a dict containing the routing settings. +It's recommended you call this ``routing.py`` and put it alongside ``urls.py`` +in your project. If you start up ``python manage.py runserver`` and go to ``http://localhost:8000``, you'll see that, rather than a default Django page, @@ -78,13 +82,8 @@ serve HTTP requests from now on - and make this WebSocket consumer instead:: Hook it up to the ``websocket.connect`` channel like this:: - CHANNEL_BACKENDS = { - "default": { - "BACKEND": "channels.backends.database.DatabaseChannelBackend", - "ROUTING": { - "websocket.connect": "myproject.myapp.consumers.ws_add", - }, - }, + channel_routing = { + "websocket.connect": "myproject.myapp.consumers.ws_add", } Now, let's look at what this is doing. It's tied to the @@ -116,12 +115,10 @@ group it's not in):: Of course, this is exactly the same code as the ``connect`` handler, so let's just route both channels to the same consumer:: - ... - "ROUTING": { + channel_routing = { "websocket.connect": "myproject.myapp.consumers.ws_add", "websocket.keepalive": "myproject.myapp.consumers.ws_add", - }, - ... + } And, even though channels will expire out, let's add an explicit ``disconnect`` handler to clean up as people disconnect (most channels will cleanly disconnect @@ -152,18 +149,13 @@ any message sent in to all connected clients. Here's all the code:: def ws_disconnect(message): Group("chat").discard(message.reply_channel) -And what our routing should look like in ``settings.py``:: +And what our routing should look like in ``routing.py``:: - CHANNEL_BACKENDS = { - "default": { - "BACKEND": "channels.backends.database.DatabaseChannelBackend", - "ROUTING": { - "websocket.connect": "myproject.myapp.consumers.ws_add", - "websocket.keepalive": "myproject.myapp.consumers.ws_add", - "websocket.receive": "myproject.myapp.consumers.ws_message", - "websocket.disconnect": "myproject.myapp.consumers.ws_disconnect", - }, - }, + channel_routing = { + "websocket.connect": "myproject.myapp.consumers.ws_add", + "websocket.keepalive": "myproject.myapp.consumers.ws_add", + "websocket.receive": "myproject.myapp.consumers.ws_message", + "websocket.disconnect": "myproject.myapp.consumers.ws_disconnect", } With all that code in your ``consumers.py`` file, you now have a working From ff9cdb71132794d343aad475f7399e8d7fc2c23d Mon Sep 17 00:00:00 2001 From: Faris Chebib Date: Thu, 10 Sep 2015 16:57:57 -0600 Subject: [PATCH 076/746] updated example and made decorators py3-ready --- channels/decorators.py | 4 ++-- docs/getting-started.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/channels/decorators.py b/channels/decorators.py index 280b493..7c28143 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -50,8 +50,8 @@ def channel_session(func): # Turn the reply_channel into a valid session key length thing. # We take the last 24 bytes verbatim, as these are the random section, # and then hash the remaining ones onto the start, and add a prefix - reply_name = message.reply_channel.name - session_key = "skt" + hashlib.md5(reply_name[:-24]).hexdigest()[:8] + reply_name[-24:] + reply_name = str(message.reply_channel.name).encode() + session_key = b"skt" + str(hashlib.md5(reply_name[:-24]).hexdigest()[:8]).encode() + reply_name[-24:] # Make a session storage session_engine = import_module(settings.SESSION_ENGINE) session = session_engine.SessionStore(session_key=session_key) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 68b8b4a..7370f40 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -267,7 +267,7 @@ name in the path of your WebSocket request (we'll ignore auth for now - that's n # Connected to websocket.keepalive @channel_session - def ws_add(message): + def ws_keepalive(message): Group("chat-%s" % message.channel_session['room']).add(message.reply_channel) # Connected to websocket.receive From 29c5ee4c3f8e0e076a0b83619b90a9630989763f Mon Sep 17 00:00:00 2001 From: Arne Schauf Date: Thu, 17 Sep 2015 11:03:36 +0200 Subject: [PATCH 077/746] Add cast to int when reading port argument --- channels/management/commands/runwsserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/management/commands/runwsserver.py b/channels/management/commands/runwsserver.py index b1525d9..a1f9350 100644 --- a/channels/management/commands/runwsserver.py +++ b/channels/management/commands/runwsserver.py @@ -18,7 +18,7 @@ class Command(BaseCommand): "Configure a network-based backend in CHANNEL_BACKENDS to use this command." ) # Run the interface - port = options.get("port", None) or 9000 + port = int(options.get("port", None) or 9000) try: import asyncio except ImportError: From c1990a92ed68e157bfef3ef8e2f986ca5e039745 Mon Sep 17 00:00:00 2001 From: Arne Schauf Date: Thu, 17 Sep 2015 15:54:44 +0200 Subject: [PATCH 078/746] decode incoming data before loading --- channels/backends/redis_py.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 6af662f..315c026 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -27,6 +27,10 @@ class RedisChannelBackend(BaseChannelBackend): return redis.Redis(host=self.host, port=self.port) def send(self, channel, message): + # if channel is no str (=> bytes) convert it + if not isinstance(channel, str): + channel = channel.decode('utf-8') + # Write out message into expiring key (avoids big items in list) key = self.prefix + uuid.uuid4().get_hex() self.connection.set( @@ -59,7 +63,7 @@ class RedisChannelBackend(BaseChannelBackend): content = self.connection.get(result[1]) if content is None: continue - return result[0][len(self.prefix):], json.loads(content) + return result[0][len(self.prefix):].decode("utf-8"), json.loads(content.decode("utf-8")) else: return None, None From 7cd5a02ee93e96bd503e0a0992ff66695eb9956f Mon Sep 17 00:00:00 2001 From: Arne Schauf Date: Thu, 17 Sep 2015 18:06:26 +0200 Subject: [PATCH 079/746] update some doc about the redis backend --- docs/backends.rst | 14 ++++++++++++++ docs/deploying.rst | 7 ++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/backends.rst b/docs/backends.rst index e79dda9..b80911a 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -21,6 +21,20 @@ Database Redis ----- +To use the Redis backend you have to install the redis package:: + + pip install -U redis + +Also you need to set the following in the ``CHANNEL_BACKENDS`` setting:: + + CHANNEL_BACKENDS = { + "default": { + "BACKEND": "channels.backends.redis_py.RedisChannelBackend", + "HOST": "redis-hostname", + }, + } + + Writing Custom Backends ----------------------- diff --git a/docs/deploying.rst b/docs/deploying.rst index 5a47e36..49433b7 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -36,11 +36,16 @@ here's an example for a remote Redis server:: CHANNEL_BACKENDS = { "default": { - "BACKEND": "channels.backends.redis.RedisChannelBackend", + "BACKEND": "channels.backends.redis_py.RedisChannelBackend", "HOST": "redis-channel", }, } +To use the Redis backend you have to install the redis package:: + + pip install -U redis + + Make sure the same setting file is used across all your workers, interfaces and WSGI apps; without it, they won't be able to talk to each other and things will just fail to work. From 07bdaf020b569ed6bf4ceb063516ec62920df28b Mon Sep 17 00:00:00 2001 From: Arne Schauf Date: Fri, 18 Sep 2015 10:34:25 +0200 Subject: [PATCH 080/746] fix hardcoded port in asyncio --- channels/interfaces/websocket_asyncio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/interfaces/websocket_asyncio.py b/channels/interfaces/websocket_asyncio.py index 55c9ea3..7c7f13c 100644 --- a/channels/interfaces/websocket_asyncio.py +++ b/channels/interfaces/websocket_asyncio.py @@ -21,7 +21,7 @@ class WebsocketAsyncioInterface(object): self.factory = get_factory(WebSocketServerFactory)("ws://0.0.0.0:%i" % self.port, debug=False) self.factory.protocol = get_protocol(WebSocketServerProtocol) self.loop = asyncio.get_event_loop() - coro = self.loop.create_server(self.factory, '0.0.0.0', 9000) + coro = self.loop.create_server(self.factory, '0.0.0.0', self.port) server = self.loop.run_until_complete(coro) self.loop.run_in_executor(None, self.backend_reader) self.loop.call_later(1, self.keepalive_sender) From 503c5fd54ba8aa83f367bab6fa3ff2eef11eefe3 Mon Sep 17 00:00:00 2001 From: Arne Schauf Date: Fri, 18 Sep 2015 10:29:28 +0200 Subject: [PATCH 081/746] fix uuid generation --- channels/backends/redis_py.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 315c026..35d8f28 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -32,7 +32,7 @@ class RedisChannelBackend(BaseChannelBackend): channel = channel.decode('utf-8') # Write out message into expiring key (avoids big items in list) - key = self.prefix + uuid.uuid4().get_hex() + key = self.prefix + str(uuid.uuid4()) self.connection.set( key, json.dumps(message), From 56c375d46010e6f5a1a9aa9a2ab251f547bf19c9 Mon Sep 17 00:00:00 2001 From: John Leith Date: Wed, 14 Oct 2015 08:20:23 -0600 Subject: [PATCH 082/746] keeping the parent class signature --- channels/request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/channels/request.py b/channels/request.py index 1c975ef..0d34c37 100644 --- a/channels/request.py +++ b/channels/request.py @@ -43,5 +43,6 @@ class CustomQueryDict(QueryDict): Custom override of QueryDict that sets things directly. """ - def __init__(self, values): + def __init__(self, values, mutable=False, encoding=None): + """ mutable and encoding are ignored :( """ MultiValueDict.__init__(self, values) From aaf2321db1ea8913cf231b9f3ebbac7f2cc5a404 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Sep 2015 17:41:48 -0500 Subject: [PATCH 083/746] Make sure content on a send/close is sent before close --- channels/interfaces/websocket_autobahn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/interfaces/websocket_autobahn.py b/channels/interfaces/websocket_autobahn.py index dae31e0..7517b8c 100644 --- a/channels/interfaces/websocket_autobahn.py +++ b/channels/interfaces/websocket_autobahn.py @@ -93,9 +93,9 @@ def get_factory(base): return self.protocols.keys() def dispatch_send(self, channel, message): + if message.get("content", None): + self.protocols[channel].serverSend(**message) if message.get("close", False): self.protocols[channel].serverClose() - else: - self.protocols[channel].serverSend(**message) return InterfaceFactory From b9f07475b7bf99d6fa0069081c28c70ae62cd70c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 6 Nov 2015 13:33:51 +0100 Subject: [PATCH 084/746] Shuffle fetch order for channels to prevent starvation --- channels/backends/redis_py.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 35d8f28..02d683e 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -56,6 +56,8 @@ class RedisChannelBackend(BaseChannelBackend): def receive_many(self, channels): if not channels: raise ValueError("Cannot receive on empty channel list!") + # Shuffle channels to avoid the first ones starving others of workers + random.shuffle(channels) # Get a message from one of our channels while True: result = self.connection.blpop([self.prefix + channel for channel in channels], timeout=1) From c937c4da6df1025c4159233cddbc3e51d309eb6e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 6 Nov 2015 13:34:50 +0100 Subject: [PATCH 085/746] Don't keep doing keepalives if main loop dies --- channels/interfaces/websocket_asyncio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/channels/interfaces/websocket_asyncio.py b/channels/interfaces/websocket_asyncio.py index 7c7f13c..b0d852a 100644 --- a/channels/interfaces/websocket_asyncio.py +++ b/channels/interfaces/websocket_asyncio.py @@ -66,4 +66,5 @@ class WebsocketAsyncioInterface(object): for protocol in self.factory.protocols.values(): if time.time() - protocol.last_keepalive > expiry_window: protocol.sendKeepalive() - self.loop.call_later(1, self.keepalive_sender) + if self.loop.is_running(): + self.loop.call_later(1, self.keepalive_sender) From 5106c7822cbda3b0b9d14f12d2b6af733e18deb0 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 6 Nov 2015 13:35:33 +0100 Subject: [PATCH 086/746] Remove path_info from request message format --- channels/request.py | 4 ++-- docs/message-standards.rst | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/channels/request.py b/channels/request.py index 0d34c37..2af4098 100644 --- a/channels/request.py +++ b/channels/request.py @@ -15,7 +15,6 @@ def encode_request(request): "cookies": request.COOKIES, "meta": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, "path": request.path, - "path_info": request.path_info, "method": request.method, "reply_channel": request.reply_channel, } @@ -33,8 +32,9 @@ def decode_request(value): request.META = value['meta'] request.path = value['path'] request.method = value['method'] - request.path_info = value['path_info'] request.reply_channel = value['reply_channel'] + # We don't support non-/ script roots + request.path_info = value['path'] return request diff --git a/docs/message-standards.rst b/docs/message-standards.rst index 8924048..ae5d75b 100644 --- a/docs/message-standards.rst +++ b/docs/message-standards.rst @@ -38,7 +38,6 @@ Contains the following keys: * cookies: Dict of cookies as {cookie_name: cookie_value} (names and values are strings) * meta: Dict of HTTP headers and info as defined in the Django Request docs (names and values are strings) * path: String, full path to the requested page, without query string or domain -* path_info: String, like ``path`` but without any script prefix. Often just ``path``. * method: String, upper-cased HTTP method Should come with an associated ``reply_channel`` which accepts HTTP Responses. From a41516fa6b188a6440bc538d7865491a34b1ae00 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 6 Nov 2015 07:57:26 -0800 Subject: [PATCH 087/746] Make Redis backend shardable --- channels/backends/redis_py.py | 104 +++++++++++++++++++++++++++------- docs/backends.rst | 62 ++++++++++++++------ docs/deploying.rst | 4 +- docs/faqs.rst | 0 docs/scaling.rst | 7 +++ 5 files changed, 136 insertions(+), 41 deletions(-) mode change 100644 => 100755 docs/faqs.rst mode change 100644 => 100755 docs/scaling.rst diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 02d683e..8c263a7 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -1,9 +1,14 @@ import time import json import datetime +import math import redis +import random +import binascii import uuid +from django.utils import six + from .base import BaseChannelBackend @@ -13,41 +18,81 @@ class RedisChannelBackend(BaseChannelBackend): multiple processes fine, but it's going to be pretty bad at throughput. """ - def __init__(self, routing, expiry=60, host="localhost", port=6379, prefix="django-channels:"): + def __init__(self, routing, expiry=60, hosts=None, prefix="django-channels:"): super(RedisChannelBackend, self).__init__(routing=routing, expiry=expiry) - self.host = host - self.port = port + # Make sure they provided some hosts, or provide a default + if not hosts: + hosts = [("localhost", 6379)] + for host, port in hosts: + assert isinstance(host, six.string_types) + assert int(port) + self.hosts = hosts self.prefix = prefix + # Precalculate some values for ring selection + self.ring_size = len(self.hosts) + self.ring_divisor = int(math.ceil(4096 / float(self.ring_size))) - @property - def connection(self): + def consistent_hash(self, value): + """ + Maps the value to a node value between 0 and 4095 + using MD5, then down to one of the ring nodes. + """ + bigval = binascii.crc32(value) & 0xffffffff + return (bigval // 0x100000) // self.ring_divisor + + def random_index(self): + return random.randint(0, len(self.hosts) - 1) + + def connection(self, index): """ Returns the correct connection for the current thread. + + Pass key to use a server based on consistent hashing of the key value; + pass None to use a random server instead. """ - return redis.Redis(host=self.host, port=self.port) + # If index is explicitly None, pick a random server + if index is None: + index = self.random_index() + # Catch bad indexes + if not (0 <= index < self.ring_size): + raise ValueError("There are only %s hosts - you asked for %s!" % (self.ring_size, index)) + host, port = self.hosts[index] + return redis.Redis(host=host, port=port) + + @property + def connections(self): + for i in range(len(self.hosts)): + return self.connection(i) def send(self, channel, message): # if channel is no str (=> bytes) convert it if not isinstance(channel, str): channel = channel.decode('utf-8') - + # Pick a connection to the right server - consistent for response + # channels, random for normal channels + if channel.startswith("!"): + index = self.consistent_hash(key) + connection = self.connection(index) + else: + connection = self.connection(None) # Write out message into expiring key (avoids big items in list) - key = self.prefix + str(uuid.uuid4()) - self.connection.set( + # TODO: Use extended set, drop support for older redis? + key = self.prefix + uuid.uuid4().get_hex() + connection.set( key, json.dumps(message), ) - self.connection.expire( + connection.expire( key, self.expiry + 10, ) # Add key to list - self.connection.rpush( + connection.rpush( self.prefix + channel, key, ) # Set list to expire when message does (any later messages will bump this) - self.connection.expire( + connection.expire( self.prefix + channel, self.expiry + 10, ) @@ -56,13 +101,27 @@ class RedisChannelBackend(BaseChannelBackend): def receive_many(self, channels): if not channels: raise ValueError("Cannot receive on empty channel list!") - # Shuffle channels to avoid the first ones starving others of workers - random.shuffle(channels) + # Work out what servers to listen on for the given channels + indexes = {} + random_index = self.random_index() + for channel in channels: + if channel.startswith("!"): + indexes.setdefault(self.consistent_hash(channel), []).append(channel) + else: + indexes.setdefault(random_index, []).append(channel) # Get a message from one of our channels while True: - result = self.connection.blpop([self.prefix + channel for channel in channels], timeout=1) + # Select a random connection to use + # TODO: Would we be better trying to do this truly async? + index = random.choice(indexes.keys()) + connection = self.connection(index) + channels = indexes[index] + # Shuffle channels to avoid the first ones starving others of workers + random.shuffle(channels) + # Pop off any waiting message + result = connection.blpop([self.prefix + channel for channel in channels], timeout=1) if result: - content = self.connection.get(result[1]) + content = connection.get(result[1]) if content is None: continue return result[0][len(self.prefix):].decode("utf-8"), json.loads(content.decode("utf-8")) @@ -75,7 +134,7 @@ class RedisChannelBackend(BaseChannelBackend): seconds (expiry defaults to message expiry if not provided). """ key = "%s:group:%s" % (self.prefix, group) - self.connection.zadd( + self.connection(self.consistent_hash(group)).zadd( key, **{channel: time.time() + (expiry or self.expiry)} ) @@ -86,7 +145,7 @@ class RedisChannelBackend(BaseChannelBackend): does nothing otherwise (does not error) """ key = "%s:group:%s" % (self.prefix, group) - self.connection.zrem( + self.connection(self.consistent_hash(group)).zrem( key, channel, ) @@ -96,10 +155,11 @@ class RedisChannelBackend(BaseChannelBackend): Returns an iterable of all channels in the group. """ key = "%s:group:%s" % (self.prefix, group) + connection = self.connection(self.consistent_hash(group)) # Discard old channels - self.connection.zremrangebyscore(key, 0, int(time.time()) - 10) + connection.zremrangebyscore(key, 0, int(time.time()) - 10) # Return current lot - return self.connection.zrange( + return connection.zrange( key, 0, -1, @@ -113,14 +173,14 @@ class RedisChannelBackend(BaseChannelBackend): obtained, False if lock not obtained. """ key = "%s:lock:%s" % (self.prefix, channel) - return bool(self.connection.setnx(key, "1")) + return bool(self.connection(self.consistent_hash(channel)).setnx(key, "1")) def unlock_channel(self, channel): """ Unlocks the named channel. Always succeeds. """ key = "%s:lock:%s" % (self.prefix, channel) - self.connection.delete(key) + self.connection(self.consistent_hash(channel)).delete(key) def __str__(self): return "%s(host=%s, port=%s)" % (self.__class__.__name__, self.host, self.port) diff --git a/docs/backends.rst b/docs/backends.rst index b80911a..bb92930 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -5,6 +5,50 @@ Multiple choices of backend are available, to fill different tradeoffs of complexity, throughput and scalability. You can also write your own backend if you wish; the API is very simple and documented below. +Redis +----- + +The Redis backend is the recommended backend to run Channels with, as it +supports both high throughput on a single Redis server as well as the ability +to run against a set of Redis servers in a sharded mode. + +To use the Redis backend you have to install the redis package:: + + pip install -U redis + +By default, it will attempt to connect to a Redis server on ``localhost:6379``, +but you can override this with the ``HOSTS`` setting:: + + CHANNEL_BACKENDS = { + "default": { + "BACKEND": "channels.backends.redis.RedisChannelBackend", + "HOSTS": [("redis-channel-1", 6379), ("redis-channel-2", 6379)], + }, + } + +Sharding +~~~~~~~~ + +The sharding model is based on consistent hashing - in particular, +:ref:`response channels ` are hashed and used to pick a single +Redis server that both the interface server and the worker will use. + +For normal channels, since any worker can service any channel request, messages +are simply distributed randomly among all possible servers, and workers will +pick a single server to listen to. Note that if you run more Redis servers than +workers, it's very likely that some servers will not have workers listening to +them; we recommend you always have at least ten workers for each Redis server +to ensure good distribution. Workers will, however, change server periodically +(every five seconds or so) so queued messages should eventually get a response. + +Note that if you change the set of sharding servers you will need to restart +all interface servers and workers with the new set before anything works, +and any in-flight messages will be lost (even with persistence, some will); +the consistent hashing model relies on all running clients having the same +settings. Any misconfigured interface server or worker will drop some or all +messages. + + In-memory --------- @@ -18,23 +62,7 @@ This backend provides no network transparency or non-blocking guarantees. Database -------- -Redis ------ - -To use the Redis backend you have to install the redis package:: - - pip install -U redis - -Also you need to set the following in the ``CHANNEL_BACKENDS`` setting:: - - CHANNEL_BACKENDS = { - "default": { - "BACKEND": "channels.backends.redis_py.RedisChannelBackend", - "HOST": "redis-hostname", - }, - } - - +======= Writing Custom Backends ----------------------- diff --git a/docs/deploying.rst b/docs/deploying.rst index 49433b7..f9fcd1a 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -36,8 +36,8 @@ here's an example for a remote Redis server:: CHANNEL_BACKENDS = { "default": { - "BACKEND": "channels.backends.redis_py.RedisChannelBackend", - "HOST": "redis-channel", + "BACKEND": "channels.backends.redis.RedisChannelBackend", + "HOSTS": [("redis-channel", 6379)], }, } diff --git a/docs/faqs.rst b/docs/faqs.rst old mode 100644 new mode 100755 diff --git a/docs/scaling.rst b/docs/scaling.rst old mode 100644 new mode 100755 index 8d9e87d..3bedf23 --- a/docs/scaling.rst +++ b/docs/scaling.rst @@ -28,3 +28,10 @@ That's why Channels labels any *response channel* with a leading ``!``, letting you know that only one server is listening for it, and thus letting you scale and shard the two different types of channels accordingly (for more on the difference, see :ref:`channel-types`). + +This is the underlying theory behind Channels' sharding model - normal channels +are sent to random Redis servers, while response channels are sent to a +predictable server that both the interface server and worker can derive. + +Currently, sharding is implemented as part of the Redis backend only; +see the :doc:`backend documentation ` for more information. From f3c3a239b9b8d937a00b01ac451a4b113f341bb2 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 6 Nov 2015 08:12:51 -0800 Subject: [PATCH 088/746] Add echo channel and fix __str__ on redis backend --- channels/backends/redis_py.py | 2 +- channels/consumer_registry.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 8c263a7..a4f22d0 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -183,4 +183,4 @@ class RedisChannelBackend(BaseChannelBackend): self.connection(self.consistent_hash(channel)).delete(key) def __str__(self): - return "%s(host=%s, port=%s)" % (self.__class__.__name__, self.host, self.port) + return "%s(hosts=%s)" % (self.__class__.__name__, self.hosts) diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index cefe4c6..baf1945 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -14,6 +14,8 @@ class ConsumerRegistry(object): def __init__(self, routing=None): self.consumers = {} + # Add basic internal consumers + self.add_consumer(self.echo_consumer, ["__channels__.echo"]) # Initialise with any routing that was passed in if routing: # If the routing was a string, import it @@ -56,3 +58,11 @@ class ConsumerRegistry(object): return self.consumers[channel] except KeyError: return None + + def echo_consumer(self, message): + """ + Implements the echo message standard. + """ + message.reply_channel.send({ + "content": message.content.get("content", None), + }) From cac848fc8924d79584ff21a3f10ae4b388870899 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 7 Nov 2015 03:13:05 -0800 Subject: [PATCH 089/746] Python 3 / unicode fixes --- channels/backends/redis_py.py | 13 +++++++++---- channels/consumer_registry.py | 4 ++-- channels/tests/test_backends.py | 14 ++++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index a4f22d0..df81a35 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -37,6 +37,8 @@ class RedisChannelBackend(BaseChannelBackend): Maps the value to a node value between 0 and 4095 using MD5, then down to one of the ring nodes. """ + if isinstance(value, six.text_type): + value = value.encode("utf8") bigval = binascii.crc32(value) & 0xffffffff return (bigval // 0x100000) // self.ring_divisor @@ -77,7 +79,7 @@ class RedisChannelBackend(BaseChannelBackend): connection = self.connection(None) # Write out message into expiring key (avoids big items in list) # TODO: Use extended set, drop support for older redis? - key = self.prefix + uuid.uuid4().get_hex() + key = self.prefix + uuid.uuid4().hex connection.set( key, json.dumps(message), @@ -113,7 +115,7 @@ class RedisChannelBackend(BaseChannelBackend): while True: # Select a random connection to use # TODO: Would we be better trying to do this truly async? - index = random.choice(indexes.keys()) + index = random.choice(list(indexes.keys())) connection = self.connection(index) channels = indexes[index] # Shuffle channels to avoid the first ones starving others of workers @@ -134,6 +136,7 @@ class RedisChannelBackend(BaseChannelBackend): seconds (expiry defaults to message expiry if not provided). """ key = "%s:group:%s" % (self.prefix, group) + key = key.encode("utf8") self.connection(self.consistent_hash(group)).zadd( key, **{channel: time.time() + (expiry or self.expiry)} @@ -145,6 +148,7 @@ class RedisChannelBackend(BaseChannelBackend): does nothing otherwise (does not error) """ key = "%s:group:%s" % (self.prefix, group) + key = key.encode("utf8") self.connection(self.consistent_hash(group)).zrem( key, channel, @@ -155,15 +159,16 @@ class RedisChannelBackend(BaseChannelBackend): Returns an iterable of all channels in the group. """ key = "%s:group:%s" % (self.prefix, group) + key = key.encode("utf8") connection = self.connection(self.consistent_hash(group)) # Discard old channels connection.zremrangebyscore(key, 0, int(time.time()) - 10) # Return current lot - return connection.zrange( + return [x.decode("utf8") for x in connection.zrange( key, 0, -1, - ) + )] # TODO: send_group efficient implementation using Lua diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index baf1945..359fcd7 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -23,8 +23,8 @@ class ConsumerRegistry(object): module_name, variable_name = routing.rsplit(".", 1) try: routing = getattr(importlib.import_module(module_name), variable_name) - except (ImportError, AttributeError): - raise ImproperlyConfigured("Cannot import channel routing %r" % routing) + except (ImportError, AttributeError) as e: + raise ImproperlyConfigured("Cannot import channel routing %r: %s" % (routing, e)) # Load consumers into us for channel, handler in routing.items(): self.add_consumer(handler, [channel]) diff --git a/channels/tests/test_backends.py b/channels/tests/test_backends.py index a626d2b..3d63e0e 100644 --- a/channels/tests/test_backends.py +++ b/channels/tests/test_backends.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from django.test import TestCase from ..channel import Channel from ..backends.database import DatabaseChannelBackend @@ -44,16 +46,16 @@ class MemoryBackendTests(TestCase): Tests that group addition and removal and listing works """ self.backend.group_add("tgroup", "test") - self.backend.group_add("tgroup", "test2") + self.backend.group_add("tgroup", "test2€") self.backend.group_add("tgroup2", "test3") self.assertEqual( set(self.backend.group_channels("tgroup")), - {"test", "test2"}, + {"test", "test2€"}, ) - self.backend.group_discard("tgroup", "test2") - self.backend.group_discard("tgroup", "test2") + self.backend.group_discard("tgroup", "test2€") + self.backend.group_discard("tgroup", "test2€") self.assertEqual( - self.backend.group_channels("tgroup"), + list(self.backend.group_channels("tgroup")), ["test"], ) @@ -76,7 +78,7 @@ class MemoryBackendTests(TestCase): self.backend.group_add("tgroup", "test") self.backend.group_add("tgroup", "test2") self.assertEqual( - self.backend.group_channels("tgroup"), + list(self.backend.group_channels("tgroup")), [], ) From e1bf2f5982a870cc01c6a5853f57a3cd5b4075aa Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 7 Nov 2015 03:13:15 -0800 Subject: [PATCH 090/746] Small project to test with --- testproject/chtest/__init__.py | 0 testproject/chtest/consumers.py | 4 ++++ testproject/chtest/models.py | 3 +++ testproject/manage.py | 10 ++++++++ testproject/testproject/__init__.py | 0 testproject/testproject/settings.py | 36 +++++++++++++++++++++++++++++ testproject/testproject/urls.py | 7 ++++++ testproject/testproject/wsgi.py | 16 +++++++++++++ 8 files changed, 76 insertions(+) create mode 100644 testproject/chtest/__init__.py create mode 100644 testproject/chtest/consumers.py create mode 100644 testproject/chtest/models.py create mode 100644 testproject/manage.py create mode 100644 testproject/testproject/__init__.py create mode 100644 testproject/testproject/settings.py create mode 100644 testproject/testproject/urls.py create mode 100644 testproject/testproject/wsgi.py diff --git a/testproject/chtest/__init__.py b/testproject/chtest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/testproject/chtest/consumers.py b/testproject/chtest/consumers.py new file mode 100644 index 0000000..ba7385c --- /dev/null +++ b/testproject/chtest/consumers.py @@ -0,0 +1,4 @@ + +def ws_message(message): + "Echoes messages back to the client" + message.reply_channel.send(message.content) diff --git a/testproject/chtest/models.py b/testproject/chtest/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/testproject/chtest/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/testproject/manage.py b/testproject/manage.py new file mode 100644 index 0000000..97ed576 --- /dev/null +++ b/testproject/manage.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/testproject/testproject/__init__.py b/testproject/testproject/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/testproject/testproject/settings.py b/testproject/testproject/settings.py new file mode 100644 index 0000000..5bae6cb --- /dev/null +++ b/testproject/testproject/settings.py @@ -0,0 +1,36 @@ +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +SECRET_KEY = '-3yt98bfvxe)7+^h#(@8k#1(1m_fpd9x3q2wolfbf^!r5ma62u' + +DEBUG = True + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'channels', +) + +ROOT_URLCONF = 'testproject.urls' + +WSGI_APPLICATION = 'testproject.wsgi.application' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + +CHANNEL_BACKENDS = { + "default": { + "BACKEND": "channels.backends.database.DatabaseChannelBackend", + "ROUTING": "testproject.urls.channel_routing", + }, +} + +if os.environ.get("USEREDIS", None): + CHANNEL_BACKENDS['default']['BACKEND'] = "channels.backends.redis_py.RedisChannelBackend" diff --git a/testproject/testproject/urls.py b/testproject/testproject/urls.py new file mode 100644 index 0000000..0337b72 --- /dev/null +++ b/testproject/testproject/urls.py @@ -0,0 +1,7 @@ +from django.conf.urls import include, url +from chtest import consumers +urlpatterns = [] + +channel_routing = { + "websocket.message": consumers.ws_message, +} diff --git a/testproject/testproject/wsgi.py b/testproject/testproject/wsgi.py new file mode 100644 index 0000000..c24d001 --- /dev/null +++ b/testproject/testproject/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for testproject project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/dev/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings") + +application = get_wsgi_application() From b928846391499436559a979c4820889132001603 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 7 Nov 2015 03:16:39 -0800 Subject: [PATCH 091/746] Add basic benchmark script --- testproject/benchmark.py | 98 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 testproject/benchmark.py diff --git a/testproject/benchmark.py b/testproject/benchmark.py new file mode 100644 index 0000000..dcdcbb4 --- /dev/null +++ b/testproject/benchmark.py @@ -0,0 +1,98 @@ +import random + +from autobahn.twisted.websocket import WebSocketClientProtocol, \ + WebSocketClientFactory + + +NUM_CONNECTIONS = 100 +PER_SECOND = 10 +stats = {} + + +class MyClientProtocol(WebSocketClientProtocol): + + num_messages = 5 + message_gap = 1 + + def onConnect(self, response): + self.sent = 0 + self.received = 0 + self.corrupted = 0 + self.out_of_order = 0 + self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) + stats[self.fingerprint] = {} + + def onOpen(self): + def hello(): + self.sendMessage("%s:%s" % (self.sent, self.fingerprint)) + self.sent += 1 + if self.sent < self.num_messages: + self.factory.reactor.callLater(1, hello) + else: + self.sendClose() + hello() + + def onMessage(self, payload, isBinary): + num, fingerprint = payload.split(":") + if fingerprint != self.fingerprint: + self.corrupted += 1 + if num != self.received: + self.out_of_order += 1 + self.received += 1 + + def onClose(self, wasClean, code, reason): + stats[self.fingerprint] = { + "sent": self.sent, + "received": self.received, + "corrupted": self.corrupted, + "out_of_order": self.out_of_order, + } + + +def spawn_connections(): + if len(stats) >= NUM_CONNECTIONS: + return + for i in range(PER_SECOND): + reactor.connectTCP("127.0.0.1", 9000, factory) + reactor.callLater(1, spawn_connections) + + +def print_progress(): + open_protocols = len([x for x in stats.values() if not x]) + print "%s open, %s total" % ( + open_protocols, + len(stats), + ) + reactor.callLater(1, print_progress) + if open_protocols == 0 and len(stats) >= NUM_CONNECTIONS: + reactor.stop() + print_stats() + + +def print_stats(): + num_incomplete = len([x for x in stats.values() if x['sent'] != x['received']]) + num_corruption = len([x for x in stats.values() if x['corrupted']]) + num_out_of_order = len([x for x in stats.values() if x['out_of_order']]) + print "-------" + print "Sockets opened: %s" % len(stats) + print "Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100) + print "Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100) + print "Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100) + + +if __name__ == '__main__': + + import sys + + from twisted.python import log + from twisted.internet import reactor + +# log.startLogging(sys.stdout) + + factory = WebSocketClientFactory(u"ws://127.0.0.1:9000", debug=False) + factory.protocol = MyClientProtocol + + reactor.callLater(1, spawn_connections) + reactor.callLater(1, print_progress) + + reactor.run() From 50cb6d13d8d52fc9f320bbee557906501b79a53c Mon Sep 17 00:00:00 2001 From: Maik Hoepfel Date: Sat, 7 Nov 2015 12:12:34 +0100 Subject: [PATCH 092/746] Wording changes to docs I eagerly read through the (excellent, thanks!) documentation for channels, and had to re-read one or two sentences. So being a good citizen, I'm suggesting a few fixes here and there. --- docs/concepts.rst | 53 ++++++++++++++++++---------------------- docs/getting-started.rst | 44 +++++++++++++++++++++------------ 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 5bbf21b..9d21582 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -5,13 +5,13 @@ Django's traditional view of the world revolves around requests and responses; a request comes in, Django is fired up to serve it, generates a response to send, and then Django goes away and waits for the next request. -That was fine when the internet was all driven by simple browser interactions, +That was fine when the internet was driven by simple browser interactions, but the modern Web includes things like WebSockets and HTTP2 server push, which allow websites to communicate outside of this traditional cycle. And, beyond that, there are plenty of non-critical tasks that applications could easily offload until after a response as been sent - like saving things -into a cache, or thumbnailing newly-uploaded images. +into a cache, sending emails or thumbnailing newly-uploaded images. Channels changes the way Django runs to be "event oriented" - rather than just responding to requests, instead Django responses to a wide array of events @@ -38,7 +38,7 @@ alternative is *at-least-once*, where normally one consumer gets the message but when things crash it's sent to more than one, which is not the trade-off we want. -There are a couple of other limitations - messages must be JSON-serialisable, +There are a couple of other limitations - messages must be JSON serialisable, and not be more than 1MB in size - but these are to make the whole thing practical, and not too important to think about up front. @@ -59,13 +59,13 @@ channel, they're writing into the same channel. How do we use channels? ----------------------- -That's what a channel is, but how is Django using them? Well, inside Django -you can write a function to consume a channel, like so:: +So how is Django using those channels? Inside Django +you can write a function to consume a channel:: def my_consumer(message): pass -And then assign a channel to it like this in the channel routing:: +And then assign a channel to it in the channel routing:: channel_routing = { "some-channel": "myapp.consumers.my_consumer", @@ -76,14 +76,11 @@ consumer function with a message object (message objects have a "content" attribute which is always a dict of data, and a "channel" attribute which is the channel it came from, as well as some others). -Django can do this as rather than run in a request-response mode, Channels -changes Django so that it runs in a worker mode - it listens on all channels -that have consumers assigned, and when a message arrives on one, runs the -relevant consumer. - -In fact, this is illustrative of the new way Django runs to enable Channels to -work. Rather than running in just a single process tied to a WSGI server, -Django runs in three separate layers: +Instead of having Django run in the traditional request-response mode, +Channels changes Django so that it runs in a worker mode - it listens on +all channels that have consumers assigned, and when a message arrives on +one, it runs the relevant consumer. So rather than running in just a +single process tied to a WSGI server, Django runs in three separate layers: * Interface servers, which communicate between Django and the outside world. This includes a WSGI adapter as well as a separate WebSocket server - we'll @@ -104,8 +101,8 @@ message and can write out zero to many other channel messages. Now, let's make a channel for requests (called ``http.request``), and a channel per client for responses (e.g. ``http.response.o4F2h2Fd``), -with the response channel a property (``reply_channel``) of the request message. -Suddenly, a view is merely another example of a consumer:: +where the response channel is a property (``reply_channel``) of the request +message. Suddenly, a view is merely another example of a consumer:: # Listens on http.request def my_consumer(message): @@ -120,22 +117,20 @@ In fact, this is how Channels works. The interface servers transform connections from the outside world (HTTP, WebSockets, etc.) into messages on channels, and then you write workers to handle these messages. -This may seem like it's still not very well designed to handle push-style -code - where you use HTTP2's server-sent events or a WebSocket to notify -clients of changes in real time (messages in a chat, perhaps, or live updates -in an admin as another user edits something). - However, the key here is that you can run code (and so send on channels) in response to any event - and that includes ones you create. You can trigger on model saves, on other incoming messages, or from code paths inside views -and forms. +and forms. That approach comes in handy for push-style +code - where you use HTTP2's server-sent events or a WebSocket to notify +clients of changes in real time (messages in a chat, perhaps, or live updates +in an admin as another user edits something). .. _channel-types: Channel Types ------------- -Now, if you think about it, there are actually two major uses for channels in +There are actually two major uses for channels in this model. The first, and more obvious one, is the dispatching of work to consumers - a message gets added to a channel, and then any one of the workers can pick it up and run the consumer. @@ -240,15 +235,15 @@ Next Steps ---------- That's the high-level overview of channels and groups, and how you should -starting thinking about them - remember, Django provides some channels +start thinking about them. Remember, Django provides some channels but you're free to make and consume your own, and all channels are network-transparent. -One thing channels are not, however, is guaranteed delivery. If you want tasks -you're sure will complete, use a system designed for this with retries and -persistence like Celery, or you'll need to make a management command that -checks for completion and re-submits a message to the channel if nothing -is completed (rolling your own retry logic, essentially). +One thing channels do not, however, is guaranteeing delivery. If you need +certainty that tasks will complete, use a system designed for this with +retries and persistence (e.g. Celery), or alternatively make a management +command that checks for completion and re-submits a message to the channel +if nothing is completed (rolling your own retry logic, essentially). We'll cover more about what kind of tasks fit well into Channels in the rest of the documentation, but for now, let's progress to :doc:`getting-started` diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 7370f40..61bd9a2 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -30,7 +30,7 @@ Make a new project, a new app, and put this in a ``consumers.py`` file in the ap message.reply_channel.send(response.channel_encode()) The most important thing to note here is that, because things we send in -messages must be JSON-serialisable, the request and response messages +messages must be JSON serialisable, the request and response messages are in a key-value format. There are ``channel_decode()`` and ``channel_encode()`` methods on both Django's request and response classes, but here we just use the message's ``content`` attribute directly for simplicity @@ -69,12 +69,14 @@ If you start up ``python manage.py runserver`` and go to you get the Hello World response, so things are working. If you don't see a response, check you :doc:`installed Channels correctly `. -Now, that's not very exciting - raw HTTP responses are something Django can -do any time. Let's try some WebSockets, and make a basic chat server! +Now, that's not very exciting - raw HTTP responses are something Django has +been able to do for a long time. Let's try some WebSockets, and make a basic +chat server! Delete that consumer and its routing - we'll want the normal Django view layer to serve HTTP requests from now on - and make this WebSocket consumer instead:: + # In consumers.py from channels import Group def ws_add(message): @@ -82,6 +84,7 @@ serve HTTP requests from now on - and make this WebSocket consumer instead:: Hook it up to the ``websocket.connect`` channel like this:: + # In routing.py channel_routing = { "websocket.connect": "myproject.myapp.consumers.ws_add", } @@ -102,19 +105,21 @@ connections you have open at any one time. The solution to this is that the WebSocket interface servers will send periodic "keepalive" messages on the ``websocket.keepalive`` channel, -so we can hook that up to re-add the channel (it's safe to add the channel to -a group it's already in - similarly, it's safe to discard a channel from a -group it's not in):: +so we can hook that up to re-add the channel:: + # In consumers.py from channels import Group # Connected to websocket.keepalive def ws_keepalive(message): Group("chat").add(message.reply_channel) +It's safe to add the channel to a group it's already in - similarly, it's +safe to discard a channel from a group it's not in. Of course, this is exactly the same code as the ``connect`` handler, so let's just route both channels to the same consumer:: + # In routing.py channel_routing = { "websocket.connect": "myproject.myapp.consumers.ws_add", "websocket.keepalive": "myproject.myapp.consumers.ws_add", @@ -124,6 +129,7 @@ And, even though channels will expire out, let's add an explicit ``disconnect`` handler to clean up as people disconnect (most channels will cleanly disconnect and get this called):: + # In consumers.py from channels import Group # Connected to websocket.disconnect @@ -135,6 +141,7 @@ Now, that's taken care of adding and removing WebSocket send channels for the we're not going to store a history of messages or anything and just replay any message sent in to all connected clients. Here's all the code:: + # In consumers.py from channels import Group # Connected to websocket.connect and websocket.keepalive @@ -158,9 +165,9 @@ And what our routing should look like in ``routing.py``:: "websocket.disconnect": "myproject.myapp.consumers.ws_disconnect", } -With all that code in your ``consumers.py`` file, you now have a working -set of a logic for a chat server. All you need to do now is get it deployed, -and as we'll see, that's not too hard. +With all that code, you now have a working set of a logic for a chat server. +All you need to do now is get it deployed, and as we'll see, that's not too +hard. Running with Channels --------------------- @@ -168,7 +175,7 @@ Running with Channels Because Channels takes Django into a multi-process model, you can no longer just run one process if you want to serve more than one protocol type. -There are multiple kinds of "interface server", and each one will service a +There are multiple kinds of "interface servers", and each one will service a different type of request - one might do WSGI requests, one might handle WebSockets, or you might have one that handles both. @@ -233,7 +240,7 @@ Persisting Data Echoing messages is a nice simple example, but it's skirting around the real design pattern - persistent state for connections. Let's consider a basic chat site where a user requests a chat room upon initial -connection, as part of the query string (e.g. ``http://host/websocket?room=abc``). +connection, as part of the query string (e.g. ``https://host/websocket?room=abc``). The ``reply_channel`` attribute you've seen before is our unique pointer to the open WebSocket - because it varies between different clients, it's how we can @@ -253,6 +260,7 @@ just like a normal Django session. Let's use it now to build a chat server that expects you to pass a chatroom name in the path of your WebSocket request (we'll ignore auth for now - that's next):: + # In consumers.py from channels import Group from channels.decorators import channel_session @@ -282,7 +290,7 @@ name in the path of your WebSocket request (we'll ignore auth for now - that's n If you play around with it from the console (or start building a simple JavaScript chat client that appends received messages to a div), you'll see -that you can now request which chat room you want in the initial request. +that you can set a chat room with the initial request. Authentication -------------- @@ -336,9 +344,10 @@ loads the user from the *channel* session rather than the *HTTP* session, and a function called ``transfer_user`` which replicates a user from one session to another. -Bringing that all together, let's make a chat server one where users can only +Bringing that all together, let's make a chat server where users can only chat to people with the same first letter of their username:: + # In consumers.py from channels import Channel, Group from channels.decorators import channel_session from channels.auth import http_session_user, channel_session_user, transfer_user @@ -375,7 +384,7 @@ Django session ID as part of the URL, like this:: You can get the current session key in a template with ``{{ request.session.session_key }}``. Note that Channels can't work with signed cookie sessions - since only HTTP -responses can set cookies, it needs a backend it can write to separately to +responses can set cookies, it needs a backend it can write to to separately to store state. @@ -393,7 +402,7 @@ easily integrate the send into the save flow of the model, rather than the message receive - that way, any new message saved will be broadcast to all the appropriate clients, no matter where it's saved from. -We'll even take some performance considerations into account - We'll make our +We'll even take some performance considerations into account: We'll make our own custom channel for new chat messages and move the model save and the chat broadcast into that, meaning the sending process/consumer can move on immediately and not spend time waiting for the database save and the @@ -402,10 +411,12 @@ immediately and not spend time waiting for the database save and the Let's see what that looks like, assuming we have a ChatMessage model with ``message`` and ``room`` fields:: + # In consumers.py from channels import Channel from channels.decorators import channel_session from .models import ChatMessage + # Connected to chat-messages def msg_consumer(message): # Save to model ChatMessage.objects.create( @@ -451,7 +462,7 @@ command run via ``cron``. If we wanted to write a bot, too, we could put its listening logic inside the ``chat-messages`` consumer, as every message would pass through it. -Linearization +Linearisation ------------- There's one final concept we want to introduce you to before you go on to build @@ -482,6 +493,7 @@ decorator, but generally you'll want to use it for most session-based WebSocket and other "continuous protocol" things. Here's an example, improving our first-letter-of-username chat from earlier:: + # In consumers.py from channels import Channel, Group from channels.decorators import channel_session, linearize from channels.auth import http_session_user, channel_session_user, transfer_user From dae0b257d646abd0de65fa297eafe7b0328de296 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 7 Nov 2015 04:45:10 -0800 Subject: [PATCH 093/746] Fix tests with unicode and isolation --- channels/backends/database.py | 3 +++ channels/backends/memory.py | 6 ++++++ channels/backends/redis_py.py | 9 ++++----- channels/tests/test_backends.py | 1 + 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/channels/backends/database.py b/channels/backends/database.py index fdc866a..d1514fa 100644 --- a/channels/backends/database.py +++ b/channels/backends/database.py @@ -168,3 +168,6 @@ class DatabaseChannelBackend(BaseChannelBackend): def __str__(self): return "%s(alias=%s)" % (self.__class__.__name__, self.connection.alias) + + def flush(self): + pass diff --git a/channels/backends/memory.py b/channels/backends/memory.py index 1d78363..9fff16c 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -92,3 +92,9 @@ class InMemoryChannelBackend(BaseChannelBackend): Unlocks the named channel. Always succeeds. """ locks.discard(channel) + + def flush(self): + global queues, groups, locks + queues = {} + groups = {} + locks = set() diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index df81a35..f48f799 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -61,11 +61,6 @@ class RedisChannelBackend(BaseChannelBackend): host, port = self.hosts[index] return redis.Redis(host=host, port=port) - @property - def connections(self): - for i in range(len(self.hosts)): - return self.connection(i) - def send(self, channel, message): # if channel is no str (=> bytes) convert it if not isinstance(channel, str): @@ -189,3 +184,7 @@ class RedisChannelBackend(BaseChannelBackend): def __str__(self): return "%s(hosts=%s)" % (self.__class__.__name__, self.hosts) + + def flush(self): + for i in range(self.ring_size): + self.connection(i).flushdb() diff --git a/channels/tests/test_backends.py b/channels/tests/test_backends.py index 3d63e0e..c2cbba6 100644 --- a/channels/tests/test_backends.py +++ b/channels/tests/test_backends.py @@ -13,6 +13,7 @@ class MemoryBackendTests(TestCase): def setUp(self): self.backend = self.backend_class(routing={}) + self.backend.flush() def test_send_recv(self): """ From 21b54e7db8f08a61af934a56d6832a3065d37676 Mon Sep 17 00:00:00 2001 From: Ben Cole Date: Sat, 7 Nov 2015 11:49:47 +0100 Subject: [PATCH 094/746] Use Django's six and require Django instead of six --- channels/response.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/response.py b/channels/response.py index fa4557f..21eb8e8 100644 --- a/channels/response.py +++ b/channels/response.py @@ -1,5 +1,5 @@ from django.http import HttpResponse -from six import PY3 +from django.utils.six import PY3 def encode_response(response): diff --git a/setup.py b/setup.py index d8c5394..7cde525 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,6 @@ setup( packages=find_packages(), include_package_data=True, install_requires=[ - 'six', + 'Django' ] ) From b6714c8a65c19a08947c9cd1143b1f881f28912d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 7 Nov 2015 05:02:03 -0800 Subject: [PATCH 095/746] Fixed #20: Bad routing setup for default backend --- channels/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/__init__.py b/channels/__init__.py index 8fd164c..923d05c 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -8,6 +8,7 @@ channel_backends = BackendManager( getattr(settings, "CHANNEL_BACKENDS", { DEFAULT_CHANNEL_BACKEND: { "BACKEND": "channels.backends.memory.InMemoryChannelBackend", + "ROUTING": {}, } }) ) From 9222676bf721ab07711760da104d793a89ea84a7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 7 Nov 2015 05:31:46 -0800 Subject: [PATCH 096/746] Change status_code to status in http.response --- channels/response.py | 4 ++-- docs/message-standards.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/channels/response.py b/channels/response.py index 21eb8e8..4b253d7 100644 --- a/channels/response.py +++ b/channels/response.py @@ -9,7 +9,7 @@ def encode_response(response): value = { "content_type": getattr(response, "content_type", None), "content": response.content, - "status_code": response.status_code, + "status": response.status_code, "headers": list(response._headers.values()), "cookies": [v.output(header="") for _, v in response.cookies.items()] } @@ -26,7 +26,7 @@ def decode_response(value): response = HttpResponse( content = value['content'], content_type = value['content_type'], - status = value['status_code'], + status = value['status'], ) for cookie in value['cookies']: response.cookies.load(cookie) diff --git a/docs/message-standards.rst b/docs/message-standards.rst index ae5d75b..665ac2c 100644 --- a/docs/message-standards.rst +++ b/docs/message-standards.rst @@ -59,7 +59,7 @@ Only sent on reply channels. Keys that must only be in the first message of a set: * content_type: String, mimetype of content -* status_code: Integer, numerical HTTP status code +* status: Integer, numerical HTTP status code * cookies: List of cookies to set (as encoded cookie strings suitable for headers) * headers: Dictionary of headers (key is header name, value is value, both strings) From 57f21d71e2b7d064ea44b5bb9b24c72f9191091c Mon Sep 17 00:00:00 2001 From: Ben Cole Date: Sat, 7 Nov 2015 17:37:09 +0100 Subject: [PATCH 097/746] Added logging --- channels/log.py | 16 ++++++++++++++++ channels/management/commands/runserver.py | 16 +++++++++++----- channels/management/commands/runworker.py | 20 +++++++------------- channels/management/commands/runwsserver.py | 17 ++++++++++------- channels/worker.py | 8 +++++--- 5 files changed, 49 insertions(+), 28 deletions(-) create mode 100644 channels/log.py diff --git a/channels/log.py b/channels/log.py new file mode 100644 index 0000000..c3a8103 --- /dev/null +++ b/channels/log.py @@ -0,0 +1,16 @@ +import logging + + +def setup_logger(name, verbosity=1): + formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(module)s - %(message)s') + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + logger.addHandler(handler) + if verbosity > 1: + logger.setLevel(logging.DEBUG) + logger.propagate = False + return logger diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 34154b9..25be5d8 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -1,15 +1,21 @@ -import django import threading + from django.core.management.commands.runserver import Command as RunserverCommand -from django.core.management import CommandError + from channels import channel_backends, DEFAULT_CHANNEL_BACKEND -from channels.worker import Worker from channels.adapters import UrlConsumer from channels.interfaces.wsgi import WSGIInterface +from channels.log import setup_logger +from channels.worker import Worker class Command(RunserverCommand): + def handle(self, *args, **options): + self.verbosity = options.get("verbosity", 1) + self.logger = setup_logger('django.channels', self.verbosity) + super(Command, self).handle(*args, **options) + def get_handler(self, *args, **options): """ Returns the default WSGI handler for the runner. @@ -27,9 +33,9 @@ class Command(RunserverCommand): # Register the default one self.channel_backend.registry.add_consumer(UrlConsumer(), ["http.request"]) # Note that this is the right one on the console - self.stdout.write("Worker thread running, channels enabled") + self.logger.info("Worker thread running, channels enabled") if self.channel_backend.local_only: - self.stdout.write("Local channel backend detected, no remote channels support") + self.logger.info("Local channel backend detected, no remote channels support") # Launch a worker thread worker = WorkerThread(self.channel_backend) worker.daemon = True diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index dd0bb9c..721e42f 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -1,7 +1,7 @@ -import time -from wsgiref.simple_server import BaseHTTPRequestHandler from django.core.management import BaseCommand, CommandError + from channels import channel_backends, DEFAULT_CHANNEL_BACKEND +from channels.log import setup_logger from channels.worker import Worker @@ -9,6 +9,8 @@ class Command(BaseCommand): def handle(self, *args, **options): # Get the backend to use + self.verbosity = options.get("verbosity", 1) + self.logger = setup_logger('django.channels', self.verbosity) channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] if channel_backend.local_only: raise CommandError( @@ -16,10 +18,10 @@ class Command(BaseCommand): "Configure a network-based backend in CHANNEL_BACKENDS to use this command." ) # Launch a worker - self.stdout.write("Running worker against backend %s" % channel_backend) + self.logger.info("Running worker against backend %s", channel_backend) # Optionally provide an output callback callback = None - if options.get("verbosity", 1) > 1: + if self.verbosity > 1: callback = self.consumer_called # Run the worker try: @@ -28,12 +30,4 @@ class Command(BaseCommand): pass def consumer_called(self, channel, message): - self.stdout.write("[%s] %s" % (self.log_date_time_string(), channel)) - - def log_date_time_string(self): - """Return the current time formatted for logging.""" - now = time.time() - year, month, day, hh, mm, ss, x, y, z = time.localtime(now) - s = "%02d/%3s/%04d %02d:%02d:%02d" % ( - day, BaseHTTPRequestHandler.monthname[month], year, hh, mm, ss) - return s + self.logger.debug("%s", channel) diff --git a/channels/management/commands/runwsserver.py b/channels/management/commands/runwsserver.py index a1f9350..72e3cb7 100644 --- a/channels/management/commands/runwsserver.py +++ b/channels/management/commands/runwsserver.py @@ -1,6 +1,7 @@ -import time from django.core.management import BaseCommand, CommandError + from channels import channel_backends, DEFAULT_CHANNEL_BACKEND +from channels.log import setup_logger class Command(BaseCommand): @@ -10,6 +11,8 @@ class Command(BaseCommand): help='Optional port number') def handle(self, *args, **options): + self.verbosity = options.get("verbosity", 1) + self.logger = setup_logger('django.channels', self.verbosity) # Get the backend to use channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] if channel_backend.local_only: @@ -23,13 +26,13 @@ class Command(BaseCommand): import asyncio except ImportError: from channels.interfaces.websocket_twisted import WebsocketTwistedInterface - self.stdout.write("Running Twisted/Autobahn WebSocket interface server") - self.stdout.write(" Channel backend: %s" % channel_backend) - self.stdout.write(" Listening on: ws://0.0.0.0:%i" % port) + self.logger.info("Running Twisted/Autobahn WebSocket interface server") + self.logger.info(" Channel backend: %s", channel_backend) + self.logger.info(" Listening on: ws://0.0.0.0:%i" % port) WebsocketTwistedInterface(channel_backend=channel_backend, port=port).run() else: from channels.interfaces.websocket_asyncio import WebsocketAsyncioInterface - self.stdout.write("Running asyncio/Autobahn WebSocket interface server") - self.stdout.write(" Channel backend: %s" % channel_backend) - self.stdout.write(" Listening on: ws://0.0.0.0:%i" % port) + self.logger.info("Running asyncio/Autobahn WebSocket interface server") + self.logger.info(" Channel backend: %s", channel_backend) + self.logger.info(" Listening on: ws://0.0.0.0:%i", port) WebsocketAsyncioInterface(channel_backend=channel_backend, port=port).run() diff --git a/channels/worker.py b/channels/worker.py index b54dc44..e4012c6 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -1,7 +1,10 @@ -import traceback +import logging + from .message import Message from .utils import name_that_thing +logger = logging.getLogger('django.channels') + class Worker(object): """ @@ -35,5 +38,4 @@ class Worker(object): except Message.Requeue: self.channel_backend.send(channel, content) except: - print("Error processing message with consumer {}:".format(name_that_thing(consumer))) - traceback.print_exc() + logger.exception("Error processing message with consumer %s:", name_that_thing(consumer)) From 351a18ba07fba57ca3b2c1c5b098665cbc179ee1 Mon Sep 17 00:00:00 2001 From: ekmartin Date: Sat, 7 Nov 2015 03:54:56 +0100 Subject: [PATCH 098/746] Add tox and run tests for python 2.7/3.5 and Django 1.6-1.8 --- .gitignore | 3 ++- channels/tests/settings.py | 7 +++++++ runtests.py | 15 +++++++++++++++ tox.ini | 16 ++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 channels/tests/settings.py create mode 100644 runtests.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index b8252fc..1c527d5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,6 @@ dist/ docs/_build __pycache__/ +.tox/ *.swp - +*.pyc diff --git a/channels/tests/settings.py b/channels/tests/settings.py new file mode 100644 index 0000000..e1c7f80 --- /dev/null +++ b/channels/tests/settings.py @@ -0,0 +1,7 @@ +SECRET_KEY = 'cat' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + } +} diff --git a/runtests.py b/runtests.py new file mode 100644 index 0000000..1d60d79 --- /dev/null +++ b/runtests.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import os +import sys + +import django +from django.conf import settings +from django.test.utils import get_runner + +if __name__ == "__main__": + os.environ['DJANGO_SETTINGS_MODULE'] = "channels.tests.settings" + django.setup() + TestRunner = get_runner(settings) + test_runner = TestRunner() + failures = test_runner.run_tests(["channels.tests"]) + sys.exit(bool(failures)) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..18142eb --- /dev/null +++ b/tox.ini @@ -0,0 +1,16 @@ +[tox] +skipsdist = True +envlist = + {py27,py35}-django-{16,17,18} + +[testenv] +setenv = + PYTHONPATH = {toxinidir}:{toxinidir} +deps = + six + redis==2.10.5 + django-16: Django>=1.6,<1.7 + django-17: Django>=1.7,<1.8 + django-18: Django>=1.8,<1.9 +commands = + python {toxinidir}/runtests.py From 4469b55d96742b4c9039ecfce309c638d6e0d8af Mon Sep 17 00:00:00 2001 From: ekmartin Date: Sat, 7 Nov 2015 04:10:49 +0100 Subject: [PATCH 099/746] Fix pep8 errors using the Django core's flake8 preset --- channels/__init__.py | 2 +- channels/apps.py | 3 ++- channels/backends/__init__.py | 10 ++++++---- channels/backends/database.py | 20 +++++++++++--------- channels/backends/memory.py | 1 + channels/backends/redis_py.py | 11 ++++++----- channels/decorators.py | 9 ++++++--- channels/management/commands/runserver.py | 1 - channels/management/commands/runworker.py | 3 ++- channels/management/commands/runwsserver.py | 2 +- channels/request.py | 1 - channels/response.py | 6 +++--- channels/tests/settings.py | 2 ++ channels/tests/test_backends.py | 1 - tox.ini | 14 ++++++++++++-- 15 files changed, 53 insertions(+), 33 deletions(-) diff --git a/channels/__init__.py b/channels/__init__.py index 923d05c..185fbc0 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -16,4 +16,4 @@ channel_backends = BackendManager( default_app_config = 'channels.apps.ChannelsConfig' # Promote channel to top-level (down here to avoid circular import errs) -from .channel import Channel, Group +from .channel import Channel, Group # NOQA diff --git a/channels/apps.py b/channels/apps.py index 2d31d8d..389fc99 100644 --- a/channels/apps.py +++ b/channels/apps.py @@ -1,10 +1,11 @@ from django.apps import AppConfig + class ChannelsConfig(AppConfig): name = "channels" verbose_name = "Channels" - + def ready(self): # Do django monkeypatches from .hacks import monkeypatch_django diff --git a/channels/backends/__init__.py b/channels/backends/__init__.py index 5582b73..71efe76 100644 --- a/channels/backends/__init__.py +++ b/channels/backends/__init__.py @@ -13,17 +13,19 @@ class BackendManager(object): def __init__(self, backend_configs): self.configs = backend_configs self.backends = {} - + def make_backend(self, name): # Load the backend class try: backend_class = import_string(self.configs[name]['BACKEND']) except KeyError: raise InvalidChannelBackendError("No BACKEND specified for %s" % name) - except ImportError as e: - raise InvalidChannelBackendError("Cannot import BACKEND %r specified for %s" % (self.configs[name]['BACKEND'], name)) + except ImportError: + raise InvalidChannelBackendError("Cannot import BACKEND %r specified for %s" % + (self.configs[name]['BACKEND'], name)) # Initialise and pass config - instance = backend_class(**{k.lower(): v for k, v in self.configs[name].items() if k != "BACKEND"}) + instance = backend_class( + **{k.lower(): v for k, v in self.configs[name].items() if k != "BACKEND"}) instance.alias = name return instance diff --git a/channels/backends/database.py b/channels/backends/database.py index d1514fa..d6966e9 100644 --- a/channels/backends/database.py +++ b/channels/backends/database.py @@ -1,4 +1,3 @@ -import time import json import datetime @@ -39,6 +38,7 @@ class DatabaseChannelBackend(BaseChannelBackend): channel = models.CharField(max_length=200, db_index=True) content = models.TextField() expiry = models.DateTimeField(db_index=True) + class Meta: apps = Apps() app_label = "channels" @@ -60,6 +60,7 @@ class DatabaseChannelBackend(BaseChannelBackend): group = models.CharField(max_length=200) channel = models.CharField(max_length=200) expiry = models.DateTimeField(db_index=True) + class Meta: apps = Apps() app_label = "channels" @@ -81,6 +82,7 @@ class DatabaseChannelBackend(BaseChannelBackend): class Lock(models.Model): channel = models.CharField(max_length=200, unique=True) expiry = models.DateTimeField(db_index=True) + class Meta: apps = Apps() app_label = "channels" @@ -93,9 +95,9 @@ class DatabaseChannelBackend(BaseChannelBackend): def send(self, channel, message): self.channel_model.objects.create( - channel = channel, - content = json.dumps(message), - expiry = now() + datetime.timedelta(seconds=self.expiry) + channel=channel, + content=json.dumps(message), + expiry=now() + datetime.timedelta(seconds=self.expiry) ) def receive_many(self, channels): @@ -125,9 +127,9 @@ class DatabaseChannelBackend(BaseChannelBackend): seconds (expiry defaults to message expiry if not provided). """ self.group_model.objects.update_or_create( - group = group, - channel = channel, - defaults = {"expiry": now() + datetime.timedelta(seconds=expiry or self.expiry)}, + group=group, + channel=channel, + defaults={"expiry": now() + datetime.timedelta(seconds=expiry or self.expiry)}, ) def group_discard(self, group, channel): @@ -152,8 +154,8 @@ class DatabaseChannelBackend(BaseChannelBackend): # We rely on the UNIQUE constraint for only-one-thread-wins on locks try: self.lock_model.objects.create( - channel = channel, - expiry = now() + datetime.timedelta(seconds=expiry or self.expiry), + channel=channel, + expiry=now() + datetime.timedelta(seconds=expiry or self.expiry), ) except IntegrityError: return False diff --git a/channels/backends/memory.py b/channels/backends/memory.py index 9fff16c..6f4ca4b 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -7,6 +7,7 @@ queues = {} groups = {} locks = set() + class InMemoryChannelBackend(BaseChannelBackend): """ In-memory channel implementation. Intended only for use with threading, diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index f48f799..b0071f6 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -1,6 +1,5 @@ import time import json -import datetime import math import redis import random @@ -64,7 +63,11 @@ class RedisChannelBackend(BaseChannelBackend): def send(self, channel, message): # if channel is no str (=> bytes) convert it if not isinstance(channel, str): - channel = channel.decode('utf-8') + channel = channel.decode("utf-8") + # Write out message into expiring key (avoids big items in list) + # TODO: Use extended set, drop support for older redis? + key = self.prefix + uuid.uuid4().hex + # Pick a connection to the right server - consistent for response # channels, random for normal channels if channel.startswith("!"): @@ -72,9 +75,7 @@ class RedisChannelBackend(BaseChannelBackend): connection = self.connection(index) else: connection = self.connection(None) - # Write out message into expiring key (avoids big items in list) - # TODO: Use extended set, drop support for older redis? - key = self.prefix + uuid.uuid4().hex + connection.set( key, json.dumps(message), diff --git a/channels/decorators.py b/channels/decorators.py index 7c28143..6b0ca7e 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -18,7 +18,8 @@ def linearize(func): def inner(message, *args, **kwargs): # Make sure there's a reply channel if not message.reply_channel: - raise ValueError("No reply_channel sent to consumer; @no_overlap can only be used on messages containing it.") + raise ValueError("No reply_channel sent to consumer; " + "@no_overlap can only be used on messages containing it.") # Get the lock, or re-queue locked = message.channel_backend.lock_channel(message.reply_channel) if not locked: @@ -43,10 +44,12 @@ def channel_session(func): def inner(message, *args, **kwargs): # Make sure there's a reply_channel if not message.reply_channel: - raise ValueError("No reply_channel sent to consumer; @channel_session can only be used on messages containing it.") + raise ValueError("No reply_channel sent to consumer; " + "@channel_session can only be used on messages containing it.") # Make sure there's NOT a channel_session already if hasattr(message, "channel_session"): - raise ValueError("channel_session decorator wrapped inside another channel_session decorator") + raise ValueError("channel_session decorator wrapped " + "inside another channel_session decorator") # Turn the reply_channel into a valid session key length thing. # We take the last 24 bytes verbatim, as these are the random section, # and then hash the remaining ones onto the start, and add a prefix diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 25be5d8..d9e91ea 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -1,7 +1,6 @@ import threading from django.core.management.commands.runserver import Command as RunserverCommand - from channels import channel_backends, DEFAULT_CHANNEL_BACKEND from channels.adapters import UrlConsumer from channels.interfaces.wsgi import WSGIInterface diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 721e42f..1c7df73 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -14,7 +14,8 @@ class Command(BaseCommand): channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] if channel_backend.local_only: raise CommandError( - "You have a process-local channel backend configured, and so cannot run separate workers.\n" + "You have a process-local channel backend configured, " + "and so cannot run separate workers.\n" "Configure a network-based backend in CHANNEL_BACKENDS to use this command." ) # Launch a worker diff --git a/channels/management/commands/runwsserver.py b/channels/management/commands/runwsserver.py index 72e3cb7..fae1e2c 100644 --- a/channels/management/commands/runwsserver.py +++ b/channels/management/commands/runwsserver.py @@ -23,7 +23,7 @@ class Command(BaseCommand): # Run the interface port = int(options.get("port", None) or 9000) try: - import asyncio + import asyncio # NOQA except ImportError: from channels.interfaces.websocket_twisted import WebsocketTwistedInterface self.logger.info("Running Twisted/Autobahn WebSocket interface server") diff --git a/channels/request.py b/channels/request.py index 2af4098..e29d711 100644 --- a/channels/request.py +++ b/channels/request.py @@ -1,7 +1,6 @@ from django.http import HttpRequest from django.utils.datastructures import MultiValueDict from django.http.request import QueryDict -from django.conf import settings def encode_request(request): diff --git a/channels/response.py b/channels/response.py index 4b253d7..f484da7 100644 --- a/channels/response.py +++ b/channels/response.py @@ -24,9 +24,9 @@ def decode_response(value): Decodes a response JSONish value to a HttpResponse object. """ response = HttpResponse( - content = value['content'], - content_type = value['content_type'], - status = value['status'], + content=value['content'], + content_type=value['content_type'], + status=value['status'], ) for cookie in value['cookies']: response.cookies.load(cookie) diff --git a/channels/tests/settings.py b/channels/tests/settings.py index e1c7f80..c472c7e 100644 --- a/channels/tests/settings.py +++ b/channels/tests/settings.py @@ -5,3 +5,5 @@ DATABASES = { 'ENGINE': 'django.db.backends.sqlite3', } } + +MIDDLEWARE_CLASSES = [] diff --git a/channels/tests/test_backends.py b/channels/tests/test_backends.py index c2cbba6..90089fd 100644 --- a/channels/tests/test_backends.py +++ b/channels/tests/test_backends.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals from django.test import TestCase -from ..channel import Channel from ..backends.database import DatabaseChannelBackend from ..backends.redis_py import RedisChannelBackend from ..backends.memory import InMemoryChannelBackend diff --git a/tox.ini b/tox.ini index 18142eb..ab262df 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,14 @@ [tox] skipsdist = True envlist = - {py27,py35}-django-{16,17,18} + {py27}-django-{17,18,19} + {py35}-django-{18,19} + {py27,py35}-flake8 + +[flake8] +exclude = venv/*,tox/*,docs/* +ignore = E123,E128,E402,W503,E731,W601 +max-line-length = 119 [testenv] setenv = @@ -9,8 +16,11 @@ setenv = deps = six redis==2.10.5 + flake8: flake8 django-16: Django>=1.6,<1.7 django-17: Django>=1.7,<1.8 django-18: Django>=1.8,<1.9 + django-19: Django==1.9b1 commands = - python {toxinidir}/runtests.py + flake8: flake8 + django: python {toxinidir}/runtests.py From 14bc3062143acaee72d968b4bdd42bb4cc436ae1 Mon Sep 17 00:00:00 2001 From: ekmartin Date: Sat, 7 Nov 2015 20:33:13 +0100 Subject: [PATCH 100/746] Make sure channels() actually returns the channel list --- channels/channel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/channel.py b/channels/channel.py index 0e047b5..5a0160e 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -81,7 +81,7 @@ class Group(object): self.channel_backend.group_discard(self.name, channel) def channels(self): - self.channel_backend.group_channels(self.name) + return self.channel_backend.group_channels(self.name) def send(self, content): if not isinstance(content, dict): From 1804d66b8597974754d8562d1793a18b6cb8eef5 Mon Sep 17 00:00:00 2001 From: ekmartin Date: Sat, 7 Nov 2015 20:51:50 +0100 Subject: [PATCH 101/746] Ignore testproject for flake8 --- channels/tests/test_backends.py | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/channels/tests/test_backends.py b/channels/tests/test_backends.py index 90089fd..f4bc461 100644 --- a/channels/tests/test_backends.py +++ b/channels/tests/test_backends.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import unicode_literals from django.test import TestCase diff --git a/tox.ini b/tox.ini index ab262df..cff47ce 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ envlist = {py27,py35}-flake8 [flake8] -exclude = venv/*,tox/*,docs/* +exclude = venv/*,tox/*,docs/*,testproject/* ignore = E123,E128,E402,W503,E731,W601 max-line-length = 119 From cf2de79d64745f9328225fa7e3666b2d5f4a4dd4 Mon Sep 17 00:00:00 2001 From: ekmartin Date: Sat, 7 Nov 2015 21:03:31 +0100 Subject: [PATCH 102/746] Add isort and fix errors --- channels/__init__.py | 8 +++++--- channels/auth.py | 1 + channels/backends/__init__.py | 9 +++++---- channels/backends/base.py | 1 + channels/backends/database.py | 4 ++-- channels/backends/memory.py | 3 ++- channels/backends/redis_py.py | 6 +++--- channels/channel.py | 2 +- channels/consumer_registry.py | 4 +++- channels/decorators.py | 16 ++++++++++------ channels/hacks.py | 7 ++++--- channels/interfaces/websocket_asyncio.py | 8 +++++--- channels/interfaces/websocket_autobahn.py | 2 +- channels/interfaces/websocket_twisted.py | 6 ++++-- channels/interfaces/wsgi.py | 1 + channels/management/commands/runserver.py | 6 ++++-- channels/management/commands/runworker.py | 6 +++--- channels/management/commands/runwsserver.py | 2 +- channels/request.py | 2 +- channels/tests/test_backends.py | 3 ++- setup.cfg | 12 ++++++++++++ tox.ini | 8 +++----- 22 files changed, 74 insertions(+), 43 deletions(-) create mode 100644 setup.cfg diff --git a/channels/__init__.py b/channels/__init__.py index 185fbc0..dde140b 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -2,8 +2,10 @@ __version__ = "0.8" # Load backends, using settings if available (else falling back to a default) DEFAULT_CHANNEL_BACKEND = "default" -from .backends import BackendManager -from django.conf import settings + +from .backends import BackendManager # isort:skip +from django.conf import settings # isort:skip + channel_backends = BackendManager( getattr(settings, "CHANNEL_BACKENDS", { DEFAULT_CHANNEL_BACKEND: { @@ -16,4 +18,4 @@ channel_backends = BackendManager( default_app_config = 'channels.apps.ChannelsConfig' # Promote channel to top-level (down here to avoid circular import errs) -from .channel import Channel, Group # NOQA +from .channel import Channel, Group # NOQA isort:skip diff --git a/channels/auth.py b/channels/auth.py index 261010c..d0c786a 100644 --- a/channels/auth.py +++ b/channels/auth.py @@ -1,6 +1,7 @@ import functools from django.contrib import auth + from .decorators import channel_session, http_session diff --git a/channels/backends/__init__.py b/channels/backends/__init__.py index 71efe76..e52d24f 100644 --- a/channels/backends/__init__.py +++ b/channels/backends/__init__.py @@ -21,11 +21,12 @@ class BackendManager(object): except KeyError: raise InvalidChannelBackendError("No BACKEND specified for %s" % name) except ImportError: - raise InvalidChannelBackendError("Cannot import BACKEND %r specified for %s" % - (self.configs[name]['BACKEND'], name)) + raise InvalidChannelBackendError( + "Cannot import BACKEND %r specified for %s" % (self.configs[name]['BACKEND'], name) + ) + # Initialise and pass config - instance = backend_class( - **{k.lower(): v for k, v in self.configs[name].items() if k != "BACKEND"}) + instance = backend_class(**{k.lower(): v for k, v in self.configs[name].items() if k != "BACKEND"}) instance.alias = name return instance diff --git a/channels/backends/base.py b/channels/backends/base.py index 9baaf6c..fbf9c5f 100644 --- a/channels/backends/base.py +++ b/channels/backends/base.py @@ -1,4 +1,5 @@ import time + from channels.consumer_registry import ConsumerRegistry diff --git a/channels/backends/database.py b/channels/backends/database.py index d6966e9..d0b7f69 100644 --- a/channels/backends/database.py +++ b/channels/backends/database.py @@ -1,8 +1,8 @@ -import json import datetime +import json from django.apps.registry import Apps -from django.db import models, connections, DEFAULT_DB_ALIAS, IntegrityError +from django.db import DEFAULT_DB_ALIAS, IntegrityError, connections, models from django.utils.functional import cached_property from django.utils.timezone import now diff --git a/channels/backends/memory.py b/channels/backends/memory.py index 6f4ca4b..0cb28ba 100644 --- a/channels/backends/memory.py +++ b/channels/backends/memory.py @@ -1,6 +1,7 @@ -import time import json +import time from collections import deque + from .base import BaseChannelBackend queues = {} diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index b0071f6..0cfbea9 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -1,11 +1,11 @@ -import time +import binascii import json import math -import redis import random -import binascii +import time import uuid +import redis from django.utils import six from .base import BaseChannelBackend diff --git a/channels/channel.py b/channels/channel.py index 5a0160e..33112d8 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -1,7 +1,7 @@ import random import string -from channels import channel_backends, DEFAULT_CHANNEL_BACKEND +from channels import DEFAULT_CHANNEL_BACKEND, channel_backends class Channel(object): diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index 359fcd7..5102e7a 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -1,6 +1,8 @@ import importlib -from django.utils import six + from django.core.exceptions import ImproperlyConfigured +from django.utils import six + from .utils import name_that_thing diff --git a/channels/decorators.py b/channels/decorators.py index 6b0ca7e..6f78c49 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -18,8 +18,10 @@ def linearize(func): def inner(message, *args, **kwargs): # Make sure there's a reply channel if not message.reply_channel: - raise ValueError("No reply_channel sent to consumer; " - "@no_overlap can only be used on messages containing it.") + raise ValueError( + "No reply_channel sent to consumer; @no_overlap can only be used on messages containing it." + ) + # Get the lock, or re-queue locked = message.channel_backend.lock_channel(message.reply_channel) if not locked: @@ -44,12 +46,14 @@ def channel_session(func): def inner(message, *args, **kwargs): # Make sure there's a reply_channel if not message.reply_channel: - raise ValueError("No reply_channel sent to consumer; " - "@channel_session can only be used on messages containing it.") + raise ValueError( + "No reply_channel sent to consumer; @no_overlap can only be used on messages containing it." + ) + # Make sure there's NOT a channel_session already if hasattr(message, "channel_session"): - raise ValueError("channel_session decorator wrapped " - "inside another channel_session decorator") + raise ValueError("channel_session decorator wrapped inside another channel_session decorator") + # Turn the reply_channel into a valid session key length thing. # We take the last 24 bytes verbatim, as these are the random section, # and then hash the remaining ones onto the start, and add a prefix diff --git a/channels/hacks.py b/channels/hacks.py index e782eb6..b8050cb 100644 --- a/channels/hacks.py +++ b/channels/hacks.py @@ -1,8 +1,9 @@ +from django.core.handlers.base import BaseHandler from django.http.request import HttpRequest from django.http.response import HttpResponseBase -from django.core.handlers.base import BaseHandler -from .request import encode_request, decode_request -from .response import encode_response, decode_response, ResponseLater + +from .request import decode_request, encode_request +from .response import ResponseLater, decode_response, encode_response def monkeypatch_django(): diff --git a/channels/interfaces/websocket_asyncio.py b/channels/interfaces/websocket_asyncio.py index b0d852a..899bfcf 100644 --- a/channels/interfaces/websocket_asyncio.py +++ b/channels/interfaces/websocket_asyncio.py @@ -1,9 +1,11 @@ -import asyncio import time -from autobahn.asyncio.websocket import WebSocketServerProtocol, WebSocketServerFactory +import asyncio +from autobahn.asyncio.websocket import ( + WebSocketServerFactory, WebSocketServerProtocol, +) -from .websocket_autobahn import get_protocol, get_factory +from .websocket_autobahn import get_factory, get_protocol class WebsocketAsyncioInterface(object): diff --git a/channels/interfaces/websocket_autobahn.py b/channels/interfaces/websocket_autobahn.py index 7517b8c..1445ad1 100644 --- a/channels/interfaces/websocket_autobahn.py +++ b/channels/interfaces/websocket_autobahn.py @@ -1,6 +1,6 @@ import time -from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND +from channels import DEFAULT_CHANNEL_BACKEND, Channel, channel_backends def get_protocol(base): diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index e6055f1..e8122a7 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -1,9 +1,11 @@ import time -from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory +from autobahn.twisted.websocket import ( + WebSocketServerFactory, WebSocketServerProtocol, +) from twisted.internet import reactor -from .websocket_autobahn import get_protocol, get_factory +from .websocket_autobahn import get_factory, get_protocol class WebsocketTwistedInterface(object): diff --git a/channels/interfaces/wsgi.py b/channels/interfaces/wsgi.py index b856b93..004e6ea 100644 --- a/channels/interfaces/wsgi.py +++ b/channels/interfaces/wsgi.py @@ -1,6 +1,7 @@ import django from django.core.handlers.wsgi import WSGIHandler from django.http import HttpResponse + from channels import Channel diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index d9e91ea..6fa54d6 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -1,7 +1,9 @@ import threading -from django.core.management.commands.runserver import Command as RunserverCommand -from channels import channel_backends, DEFAULT_CHANNEL_BACKEND +from django.core.management.commands.runserver import \ + Command as RunserverCommand + +from channels import DEFAULT_CHANNEL_BACKEND, channel_backends from channels.adapters import UrlConsumer from channels.interfaces.wsgi import WSGIInterface from channels.log import setup_logger diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 1c7df73..2c157bc 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -1,6 +1,7 @@ + from django.core.management import BaseCommand, CommandError -from channels import channel_backends, DEFAULT_CHANNEL_BACKEND +from channels import DEFAULT_CHANNEL_BACKEND, channel_backends from channels.log import setup_logger from channels.worker import Worker @@ -14,8 +15,7 @@ class Command(BaseCommand): channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] if channel_backend.local_only: raise CommandError( - "You have a process-local channel backend configured, " - "and so cannot run separate workers.\n" + "You have a process-local channel backend configured, and so cannot run separate workers.\n" "Configure a network-based backend in CHANNEL_BACKENDS to use this command." ) # Launch a worker diff --git a/channels/management/commands/runwsserver.py b/channels/management/commands/runwsserver.py index fae1e2c..54f2f0c 100644 --- a/channels/management/commands/runwsserver.py +++ b/channels/management/commands/runwsserver.py @@ -1,6 +1,6 @@ from django.core.management import BaseCommand, CommandError -from channels import channel_backends, DEFAULT_CHANNEL_BACKEND +from channels import DEFAULT_CHANNEL_BACKEND, channel_backends from channels.log import setup_logger diff --git a/channels/request.py b/channels/request.py index e29d711..4060335 100644 --- a/channels/request.py +++ b/channels/request.py @@ -1,6 +1,6 @@ from django.http import HttpRequest -from django.utils.datastructures import MultiValueDict from django.http.request import QueryDict +from django.utils.datastructures import MultiValueDict def encode_request(request): diff --git a/channels/tests/test_backends.py b/channels/tests/test_backends.py index f4bc461..b25750f 100644 --- a/channels/tests/test_backends.py +++ b/channels/tests/test_backends.py @@ -2,9 +2,10 @@ from __future__ import unicode_literals from django.test import TestCase + from ..backends.database import DatabaseChannelBackend -from ..backends.redis_py import RedisChannelBackend from ..backends.memory import InMemoryChannelBackend +from ..backends.redis_py import RedisChannelBackend class MemoryBackendTests(TestCase): diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..445b946 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,12 @@ +[flake8] +exclude = venv/*,tox/*,docs/*,testproject/* +ignore = E123,E128,E402,W503,E731,W601 +max-line-length = 119 + +[isort] +combine_as_imports = true +default_section = THIRDPARTY +include_trailing_comma = true +known_first_party = channels +multi_line_output = 5 +not_skip = __init__.py diff --git a/tox.ini b/tox.ini index cff47ce..f42ff19 100644 --- a/tox.ini +++ b/tox.ini @@ -4,11 +4,7 @@ envlist = {py27}-django-{17,18,19} {py35}-django-{18,19} {py27,py35}-flake8 - -[flake8] -exclude = venv/*,tox/*,docs/*,testproject/* -ignore = E123,E128,E402,W503,E731,W601 -max-line-length = 119 + isort [testenv] setenv = @@ -17,10 +13,12 @@ deps = six redis==2.10.5 flake8: flake8 + isort: isort django-16: Django>=1.6,<1.7 django-17: Django>=1.7,<1.8 django-18: Django>=1.8,<1.9 django-19: Django==1.9b1 commands = flake8: flake8 + isort: isort -c -rc channels django: python {toxinidir}/runtests.py From 28e132468f990dffa184d9135f637515118ce79a Mon Sep 17 00:00:00 2001 From: ekmartin Date: Sat, 7 Nov 2015 23:48:15 +0100 Subject: [PATCH 103/746] Store the session key as a string --- channels/decorators.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/channels/decorators.py b/channels/decorators.py index 7c28143..9a0d084 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -50,8 +50,9 @@ def channel_session(func): # Turn the reply_channel into a valid session key length thing. # We take the last 24 bytes verbatim, as these are the random section, # and then hash the remaining ones onto the start, and add a prefix - reply_name = str(message.reply_channel.name).encode() - session_key = b"skt" + str(hashlib.md5(reply_name[:-24]).hexdigest()[:8]).encode() + reply_name[-24:] + reply_name = message.reply_channel.name + hashed = hashlib.md5(reply_name[:-24].encode()).hexdigest()[:8] + session_key = "skt" + hashed + reply_name[-24:] # Make a session storage session_engine = import_module(settings.SESSION_ENGINE) session = session_engine.SessionStore(session_key=session_key) From 4f87d53adc334ccda9390edb61795a2270c0adb4 Mon Sep 17 00:00:00 2001 From: ekmartin Date: Wed, 18 Nov 2015 17:48:03 +0100 Subject: [PATCH 104/746] Include cookies in request_info for websockets --- channels/interfaces/websocket_autobahn.py | 3 ++ channels/tests/test_interfaces.py | 53 +++++++++++++++++++++++ tox.ini | 2 + 3 files changed, 58 insertions(+) create mode 100644 channels/tests/test_interfaces.py diff --git a/channels/interfaces/websocket_autobahn.py b/channels/interfaces/websocket_autobahn.py index 1445ad1..eff1bc6 100644 --- a/channels/interfaces/websocket_autobahn.py +++ b/channels/interfaces/websocket_autobahn.py @@ -1,5 +1,7 @@ import time +from django.http import parse_cookie + from channels import DEFAULT_CHANNEL_BACKEND, Channel, channel_backends @@ -16,6 +18,7 @@ def get_protocol(base): self.request_info = { "path": request.path, "get": request.params, + "cookies": parse_cookie(request.headers.get('cookie', '')) } def onOpen(self): diff --git a/channels/tests/test_interfaces.py b/channels/tests/test_interfaces.py new file mode 100644 index 0000000..fd830d7 --- /dev/null +++ b/channels/tests/test_interfaces.py @@ -0,0 +1,53 @@ +from django.test import TestCase + +from channels.interfaces.websocket_autobahn import get_protocol + +try: + from unittest import mock +except ImportError: + import mock + + +def generate_connection_request(path, params, headers): + request = mock.Mock() + request.path = path + request.params = params + request.headers = headers + return request + + +class WebsocketAutobahnInterfaceProtocolTestCase(TestCase): + def test_on_connect_cookie(self): + protocol = get_protocol(object)() + session = "123cat" + cookie = "somethingelse=test; sessionid={0}".format(session) + headers = { + "cookie": cookie + } + + test_request = generate_connection_request("path", {}, headers) + protocol.onConnect(test_request) + self.assertEqual(session, protocol.request_info["cookies"]["sessionid"]) + + def test_on_connect_no_cookie(self): + protocol = get_protocol(object)() + test_request = generate_connection_request("path", {}, {}) + protocol.onConnect(test_request) + self.assertEqual({}, protocol.request_info["cookies"]) + + def test_on_connect_params(self): + protocol = get_protocol(object)() + params = { + "session_key": ["123cat"] + } + + test_request = generate_connection_request("path", params, {}) + protocol.onConnect(test_request) + self.assertEqual(params, protocol.request_info["get"]) + + def test_on_connect_path(self): + protocol = get_protocol(object)() + path = "path" + test_request = generate_connection_request(path, {}, {}) + protocol.onConnect(test_request) + self.assertEqual(path, protocol.request_info["path"]) diff --git a/tox.ini b/tox.ini index f42ff19..4cde18b 100644 --- a/tox.ini +++ b/tox.ini @@ -10,8 +10,10 @@ envlist = setenv = PYTHONPATH = {toxinidir}:{toxinidir} deps = + autobahn six redis==2.10.5 + py27: mock flake8: flake8 isort: isort django-16: Django>=1.6,<1.7 From 2ff6388ef2334c770f7f5b614d92bdb9f386bd01 Mon Sep 17 00:00:00 2001 From: Maik Hoepfel Date: Thu, 19 Nov 2015 14:08:14 +0100 Subject: [PATCH 105/746] Changes after PR review --- docs/concepts.rst | 8 ++++---- docs/getting-started.rst | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 9d21582..fbe4275 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -11,7 +11,7 @@ which allow websites to communicate outside of this traditional cycle. And, beyond that, there are plenty of non-critical tasks that applications could easily offload until after a response as been sent - like saving things -into a cache, sending emails or thumbnailing newly-uploaded images. +into a cache or thumbnailing newly-uploaded images. Channels changes the way Django runs to be "event oriented" - rather than just responding to requests, instead Django responses to a wide array of events @@ -38,7 +38,7 @@ alternative is *at-least-once*, where normally one consumer gets the message but when things crash it's sent to more than one, which is not the trade-off we want. -There are a couple of other limitations - messages must be JSON serialisable, +There are a couple of other limitations - messages must be JSON serializable, and not be more than 1MB in size - but these are to make the whole thing practical, and not too important to think about up front. @@ -121,7 +121,7 @@ However, the key here is that you can run code (and so send on channels) in response to any event - and that includes ones you create. You can trigger on model saves, on other incoming messages, or from code paths inside views and forms. That approach comes in handy for push-style -code - where you use HTTP2's server-sent events or a WebSocket to notify +code - where you use HTML5's server-sent events or a WebSocket to notify clients of changes in real time (messages in a chat, perhaps, or live updates in an admin as another user edits something). @@ -239,7 +239,7 @@ start thinking about them. Remember, Django provides some channels but you're free to make and consume your own, and all channels are network-transparent. -One thing channels do not, however, is guaranteeing delivery. If you need +One thing channels do not, however, is guarantee delivery. If you need certainty that tasks will complete, use a system designed for this with retries and persistence (e.g. Celery), or alternatively make a management command that checks for completion and re-submits a message to the channel diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 61bd9a2..e11b0db 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -30,7 +30,7 @@ Make a new project, a new app, and put this in a ``consumers.py`` file in the ap message.reply_channel.send(response.channel_encode()) The most important thing to note here is that, because things we send in -messages must be JSON serialisable, the request and response messages +messages must be JSON serializable, the request and response messages are in a key-value format. There are ``channel_decode()`` and ``channel_encode()`` methods on both Django's request and response classes, but here we just use the message's ``content`` attribute directly for simplicity @@ -240,7 +240,7 @@ Persisting Data Echoing messages is a nice simple example, but it's skirting around the real design pattern - persistent state for connections. Let's consider a basic chat site where a user requests a chat room upon initial -connection, as part of the query string (e.g. ``https://host/websocket?room=abc``). +connection, as part of the query string (e.g. ``wss://host/websocket?room=abc``). The ``reply_channel`` attribute you've seen before is our unique pointer to the open WebSocket - because it varies between different clients, it's how we can @@ -462,7 +462,7 @@ command run via ``cron``. If we wanted to write a bot, too, we could put its listening logic inside the ``chat-messages`` consumer, as every message would pass through it. -Linearisation +Linearization ------------- There's one final concept we want to introduce you to before you go on to build From bf9a4232112a3c74d1d8b77ddb71a43f6a225786 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 10 Nov 2015 17:28:06 -0800 Subject: [PATCH 106/746] Twisted HTTP interface now serves basic requests --- channels/management/commands/runworker.py | 8 ++++++-- channels/request.py | 18 ++++++++++++++++-- docs/getting-started.rst | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 2c157bc..32692e9 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -1,8 +1,8 @@ from django.core.management import BaseCommand, CommandError - -from channels import DEFAULT_CHANNEL_BACKEND, channel_backends +from channels import channel_backends, DEFAULT_CHANNEL_BACKEND from channels.log import setup_logger +from channels.adapters import UrlConsumer from channels.worker import Worker @@ -18,6 +18,10 @@ class Command(BaseCommand): "You have a process-local channel backend configured, and so cannot run separate workers.\n" "Configure a network-based backend in CHANNEL_BACKENDS to use this command." ) + # Check a handler is registered for http reqs + if not channel_backend.registry.consumer_for_channel("http.request"): + # Register the default one + channel_backend.registry.add_consumer(UrlConsumer(), ["http.request"]) # Launch a worker self.logger.info("Running worker against backend %s", channel_backend) # Optionally provide an output callback diff --git a/channels/request.py b/channels/request.py index 4060335..0c5de55 100644 --- a/channels/request.py +++ b/channels/request.py @@ -12,7 +12,11 @@ def encode_request(request): "get": dict(request.GET.lists()), "post": dict(request.POST.lists()), "cookies": request.COOKIES, - "meta": {k: v for k, v in request.META.items() if not k.startswith("wsgi")}, + "headers": { + k[5:].lower(): v + for k, v in request.META.items() + if k.lower().startswith("http_") + }, "path": request.path, "method": request.method, "reply_channel": request.reply_channel, @@ -28,10 +32,20 @@ def decode_request(value): request.GET = CustomQueryDict(value['get']) request.POST = CustomQueryDict(value['post']) request.COOKIES = value['cookies'] - request.META = value['meta'] request.path = value['path'] request.method = value['method'] request.reply_channel = value['reply_channel'] + # Channels requests are more high-level than the dumping ground that is + # META; re-combine back into it + request.META = { + "REQUEST_METHOD": value["method"], + "SERVER_NAME": value["server"][0], + "SERVER_PORT": value["server"][1], + "REMOTE_ADDR": value["client"][0], + "REMOTE_HOST": value["client"][0], # Not the DNS name, hopefully fine. + } + for header, header_value in value.get("headers", {}).items(): + request.META["HTTP_%s" % header.upper()] = header_value # We don't support non-/ script roots request.path_info = value['path'] return request diff --git a/docs/getting-started.rst b/docs/getting-started.rst index e11b0db..283cdcf 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -203,7 +203,7 @@ The easiest way to do this is to use the ``runwsserver`` management command that ships with Django; just make sure you've installed the latest release of ``autobahn`` first:: - pip install -U autobahn + pip install -U autobahn[twisted] python manage.py runwsserver Run that alongside ``runserver`` and you'll have two interface servers, a From b6f38910dea3ed6c0fe8382d22b60a8ad2da0767 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Dec 2015 12:54:52 -0800 Subject: [PATCH 107/746] Working mixed-mode HTTP/WebSocket server! --- channels/interfaces/http_twisted.py | 249 +++++++++++++++++++ channels/interfaces/websocket_asyncio.py | 2 +- channels/interfaces/websocket_autobahn.py | 12 +- channels/interfaces/websocket_twisted.py | 2 +- channels/management/commands/runallserver.py | 26 ++ channels/request.py | 8 + docs/message-standards.rst | 4 +- 7 files changed, 293 insertions(+), 10 deletions(-) create mode 100644 channels/interfaces/http_twisted.py create mode 100644 channels/management/commands/runallserver.py diff --git a/channels/interfaces/http_twisted.py b/channels/interfaces/http_twisted.py new file mode 100644 index 0000000..fcee728 --- /dev/null +++ b/channels/interfaces/http_twisted.py @@ -0,0 +1,249 @@ +import time + +from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory +from twisted.python.compat import _PY3 +from twisted.web.http import HTTPFactory, HTTPChannel, Request, _respondToBadRequestAndDisconnect, parse_qs, _parseHeader +from twisted.protocols.policies import ProtocolWrapper +from twisted.internet import reactor + +from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND +from .websocket_autobahn import get_protocol, get_factory + + +WebsocketProtocol = get_protocol(WebSocketServerProtocol) + + +class WebRequest(Request): + """ + Request that either hands off information to channels, or offloads + to a WebSocket class. + + Does some extra processing over the normal Twisted Web request to separate + GET and POST out. + """ + + def __init__(self, *args, **kwargs): + Request.__init__(self, *args, **kwargs) + self.reply_channel = Channel.new_name("!http.response") + self.channel.factory.reply_protocols[self.reply_channel] = self + + def process(self): + # Get upgrade header + upgrade_header = None + if self.requestHeaders.hasHeader("Upgrade"): + upgrade_header = self.requestHeaders.getRawHeaders("Upgrade")[0] + # Is it WebSocket? IS IT?! + if upgrade_header == "websocket": + # Make WebSocket protocol to hand off to + protocol = self.channel.factory.ws_factory.buildProtocol(self.transport.getPeer()) + if not protocol: + # If protocol creation fails, we signal "internal server error" + self.setResponseCode(500) + self.finish() + # Port across transport + transport, self.transport = self.transport, None + if isinstance(transport, ProtocolWrapper): + # i.e. TLS is a wrapping protocol + transport.wrappedProtocol = protocol + else: + transport.protocol = protocol + protocol.makeConnection(transport) + # Re-inject request + if _PY3: + data = self.method + b' ' + self.uri + b' HTTP/1.1\x0d\x0a' + for h in self.requestHeaders.getAllRawHeaders(): + data += h[0] + b': ' + b",".join(h[1]) + b'\x0d\x0a' + data += b"\x0d\x0a" + data += self.content.read() + else: + data = "%s %s HTTP/1.1\x0d\x0a" % (self.method, self.uri) + for h in self.requestHeaders.getAllRawHeaders(): + data += "%s: %s\x0d\x0a" % (h[0], ",".join(h[1])) + data += "\x0d\x0a" + protocol.dataReceived(data) + # Remove our HTTP reply channel association + self.channel.factory.reply_protocols[self.reply_channel] = None + self.reply_channel = None + # Boring old HTTP. + else: + # Send request message + Channel("http.request").send({ + "reply_channel": self.reply_channel, + "method": self.method, + "get": self.get, + "post": self.post, + "cookies": self.received_cookies, + "headers": {k: v[0] for k, v in self.requestHeaders.getAllRawHeaders()}, + "client": [self.client.host, self.client.port], + "server": [self.host.host, self.host.port], + "path": self.path, + }) + + def connectionLost(self, reason): + """ + Cleans up reply channel on close. + """ + if self.reply_channel: + del self.channel.factory.reply_protocols[self.reply_channel] + Request.connectionLost(self, reason) + + def serverResponse(self, message): + """ + Writes a received HTTP response back out to the transport. + """ + # Write code + self.setResponseCode(message['status']) + # Write headers + for header, value in message.get("headers", {}): + self.setHeader(header.encode("utf8"), value.encode("utf8")) + # Write cookies + for cookie in message.get("cookies"): + self.cookies.append(cookie.encode("utf8")) + # Write out body + if "content" in message: + Request.write(self, message['content'].encode("utf8")) + self.finish() + + def requestReceived(self, command, path, version): + """ + Called by channel when all data has been received. + Overridden because Twisted merges GET and POST into one thing by default. + """ + self.content.seek(0,0) + self.get = {} + self.post = {} + + self.method, self.uri = command, path + self.clientproto = version + x = self.uri.split(b'?', 1) + + print self.method + + # URI and GET args assignment + if len(x) == 1: + self.path = self.uri + else: + self.path, argstring = x + self.get = parse_qs(argstring, 1) + + # cache the client and server information, we'll need this later to be + # serialized and sent with the request so CGIs will work remotely + self.client = self.channel.transport.getPeer() + self.host = self.channel.transport.getHost() + + # Argument processing + ctype = self.requestHeaders.getRawHeaders(b'content-type') + if ctype is not None: + ctype = ctype[0] + + # Process POST data if present + if self.method == b"POST" and ctype: + mfd = b'multipart/form-data' + key, pdict = _parseHeader(ctype) + if key == b'application/x-www-form-urlencoded': + self.post.update(parse_qs(self.content.read(), 1)) + elif key == mfd: + try: + cgiArgs = cgi.parse_multipart(self.content, pdict) + + if _PY3: + # parse_multipart on Python 3 decodes the header bytes + # as iso-8859-1 and returns a str key -- we want bytes + # so encode it back + self.post.update({x.encode('iso-8859-1'): y + for x, y in cgiArgs.items()}) + else: + self.post.update(cgiArgs) + except: + # It was a bad request. + _respondToBadRequestAndDisconnect(self.channel.transport) + return + self.content.seek(0, 0) + + # Continue with rest of request handling + self.process() + + +class WebProtocol(HTTPChannel): + + requestFactory = WebRequest + + +class WebFactory(HTTPFactory): + + protocol = WebProtocol + + def __init__(self): + HTTPFactory.__init__(self) + # We track all sub-protocols for response channel mapping + self.reply_protocols = {} + # Make a factory for WebSocket protocols + self.ws_factory = WebSocketServerFactory("ws://127.0.0.1:8000") + self.ws_factory.protocol = WebsocketProtocol + self.ws_factory.reply_protocols = self.reply_protocols + + def reply_channels(self): + return self.reply_protocols.keys() + + def dispatch_reply(self, channel, message): + if channel.startswith("!http") and isinstance(self.reply_protocols[channel], WebRequest): + self.reply_protocols[channel].serverResponse(message) + elif channel.startswith("!websocket") and isinstance(self.reply_protocols[channel], WebsocketProtocol): + if message.get("content", None): + self.reply_protocols[channel].serverSend(**message) + if message.get("close", False): + self.reply_protocols[channel].serverClose() + else: + raise ValueError("Cannot dispatch message on channel %r" % channel) + + +class HttpTwistedInterface(object): + """ + Easy API to run a HTTP1 & WebSocket interface server using Twisted. + Integrates the channel backend by running it in a separate thread, using + the always-compatible polling style. + """ + + def __init__(self, channel_backend, port=8000): + self.channel_backend = channel_backend + self.port = port + + def run(self): + self.factory = WebFactory() + reactor.listenTCP(self.port, self.factory) + reactor.callInThread(self.backend_reader) + #reactor.callLater(1, self.keepalive_sender) + reactor.run() + + def backend_reader(self): + """ + Run in a separate thread; reads messages from the backend. + """ + while True: + channels = self.factory.reply_channels() + # Quit if reactor is stopping + if not reactor.running: + return + # Don't do anything if there's no channels to listen on + if channels: + channel, message = self.channel_backend.receive_many(channels) + else: + time.sleep(0.1) + continue + # Wait around if there's nothing received + if channel is None: + time.sleep(0.05) + continue + # Deal with the message + self.factory.dispatch_reply(channel, message) + + def keepalive_sender(self): + """ + Sends keepalive messages for open WebSockets every + (channel_backend expiry / 2) seconds. + """ + expiry_window = int(self.channel_backend.expiry / 2) + for protocol in self.factory.reply_protocols.values(): + if time.time() - protocol.last_keepalive > expiry_window: + protocol.sendKeepalive() + reactor.callLater(1, self.keepalive_sender) diff --git a/channels/interfaces/websocket_asyncio.py b/channels/interfaces/websocket_asyncio.py index 899bfcf..c41871e 100644 --- a/channels/interfaces/websocket_asyncio.py +++ b/channels/interfaces/websocket_asyncio.py @@ -65,7 +65,7 @@ class WebsocketAsyncioInterface(object): (channel_backend expiry / 2) seconds. """ expiry_window = int(self.channel_backend.expiry / 2) - for protocol in self.factory.protocols.values(): + for protocol in self.factory.reply_protocols.values(): if time.time() - protocol.last_keepalive > expiry_window: protocol.sendKeepalive() if self.loop.is_running(): diff --git a/channels/interfaces/websocket_autobahn.py b/channels/interfaces/websocket_autobahn.py index eff1bc6..cd3dd3e 100644 --- a/channels/interfaces/websocket_autobahn.py +++ b/channels/interfaces/websocket_autobahn.py @@ -26,7 +26,7 @@ def get_protocol(base): self.reply_channel = Channel.new_name("!websocket.send") self.request_info["reply_channel"] = self.reply_channel self.last_keepalive = time.time() - self.factory.protocols[self.reply_channel] = self + self.factory.reply_protocols[self.reply_channel] = self # Send news that this channel is open Channel("websocket.connect").send(self.request_info) @@ -61,7 +61,7 @@ def get_protocol(base): def onClose(self, wasClean, code, reason): if hasattr(self, "reply_channel"): - del self.factory.protocols[self.reply_channel] + del self.factory.reply_protocols[self.reply_channel] Channel("websocket.disconnect").send({ "reply_channel": self.reply_channel, }) @@ -90,15 +90,15 @@ def get_factory(base): def __init__(self, *args, **kwargs): super(InterfaceFactory, self).__init__(*args, **kwargs) - self.protocols = {} + self.reply_protocols = {} def reply_channels(self): - return self.protocols.keys() + return self.reply_protocols.keys() def dispatch_send(self, channel, message): if message.get("content", None): - self.protocols[channel].serverSend(**message) + self.reply_protocols[channel].serverSend(**message) if message.get("close", False): - self.protocols[channel].serverClose() + self.reply_protocols[channel].serverClose() return InterfaceFactory diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index e8122a7..f8a4c4b 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -55,7 +55,7 @@ class WebsocketTwistedInterface(object): (channel_backend expiry / 2) seconds. """ expiry_window = int(self.channel_backend.expiry / 2) - for protocol in self.factory.protocols.values(): + for protocol in self.factory.reply_protocols.values(): if time.time() - protocol.last_keepalive > expiry_window: protocol.sendKeepalive() reactor.callLater(1, self.keepalive_sender) diff --git a/channels/management/commands/runallserver.py b/channels/management/commands/runallserver.py new file mode 100644 index 0000000..1762be5 --- /dev/null +++ b/channels/management/commands/runallserver.py @@ -0,0 +1,26 @@ +import time +from django.core.management import BaseCommand, CommandError +from channels import channel_backends, DEFAULT_CHANNEL_BACKEND + + +class Command(BaseCommand): + + def add_arguments(self, parser): + parser.add_argument('port', nargs='?', + help='Optional port number') + + def handle(self, *args, **options): + # Get the backend to use + channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] + if channel_backend.local_only: + raise CommandError( + "You have a process-local channel backend configured, and so cannot run separate interface servers.\n" + "Configure a network-based backend in CHANNEL_BACKENDS to use this command." + ) + # Run the interface + port = int(options.get("port", None) or 8000) + from channels.interfaces.http_twisted import HttpTwistedInterface + self.stdout.write("Running twisted/Autobahn HTTP & WebSocket interface server") + self.stdout.write(" Channel backend: %s" % channel_backend) + self.stdout.write(" Listening on: 0.0.0.0:%i" % port) + HttpTwistedInterface(channel_backend=channel_backend, port=port).run() diff --git a/channels/request.py b/channels/request.py index 0c5de55..01be5b5 100644 --- a/channels/request.py +++ b/channels/request.py @@ -20,6 +20,14 @@ def encode_request(request): "path": request.path, "method": request.method, "reply_channel": request.reply_channel, + "server": [ + request.META.get("SERVER_NAME", None), + request.META.get("SERVER_PORT", None), + ], + "client": [ + request.META.get("REMOTE_ADDR", None), + request.META.get("REMOTE_PORT", None), + ], } return value diff --git a/docs/message-standards.rst b/docs/message-standards.rst index 665ac2c..44d4852 100644 --- a/docs/message-standards.rst +++ b/docs/message-standards.rst @@ -33,8 +33,8 @@ Standard channel name is ``http.request``. Contains the following keys: -* get: List of (key, value) tuples of GET variables (keys and values are strings) -* post: List of (key, value) tuples of POST variables (keys and values are strings) +* get: Dict of {key: [value, ...]} of GET variables (keys and values are strings) +* post: Dict of {key: [value, ...]} of POST variables (keys and values are strings) * cookies: Dict of cookies as {cookie_name: cookie_value} (names and values are strings) * meta: Dict of HTTP headers and info as defined in the Django Request docs (names and values are strings) * path: String, full path to the requested page, without query string or domain From 0cf2a132342b1d16d722287b3c16ce827f386ccf Mon Sep 17 00:00:00 2001 From: Yehonatan Daniv Date: Sun, 6 Dec 2015 17:47:23 +0200 Subject: [PATCH 108/746] Fixed typo --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index fbe4275..1b3bc46 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -10,7 +10,7 @@ but the modern Web includes things like WebSockets and HTTP2 server push, which allow websites to communicate outside of this traditional cycle. And, beyond that, there are plenty of non-critical tasks that applications -could easily offload until after a response as been sent - like saving things +could easily offload until after a response has been sent - like saving things into a cache or thumbnailing newly-uploaded images. Channels changes the way Django runs to be "event oriented" - rather than From eafb3c728dadd43514c155d294751ed2c915a3b4 Mon Sep 17 00:00:00 2001 From: Martin Pajuste Date: Sat, 12 Dec 2015 00:18:57 +0200 Subject: [PATCH 109/746] Fix some typos --- docs/concepts.rst | 2 +- docs/getting-started.rst | 6 +++--- docs/installation.rst | 4 ++-- docs/message-standards.rst | 2 +- docs/releases/0.8.rst | 2 +- docs/scaling.rst | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 1b3bc46..2b90fcf 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -28,7 +28,7 @@ The core of Channels is, unsurprisingly, a datastructure called a *channel*. What is a channel? It is an *ordered*, *first-in first-out queue* with *message expiry* and *at-most-once delivery* to *only one listener at a time*. -You can think of it as analagous to a task queue - messages are put onto +You can think of it as analogous to a task queue - messages are put onto the channel by *producers*, and then given to just one of the *consumers* listening to that channnel. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 283cdcf..f7a0b7e 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -244,7 +244,7 @@ connection, as part of the query string (e.g. ``wss://host/websocket?room=abc``) The ``reply_channel`` attribute you've seen before is our unique pointer to the open WebSocket - because it varies between different clients, it's how we can -keep track of "who" a message is from. Remember, Channels is network-trasparent +keep track of "who" a message is from. Remember, Channels is network-transparent and can run on multiple workers, so you can't just store things locally in global variables or similar. @@ -384,7 +384,7 @@ Django session ID as part of the URL, like this:: You can get the current session key in a template with ``{{ request.session.session_key }}``. Note that Channels can't work with signed cookie sessions - since only HTTP -responses can set cookies, it needs a backend it can write to to separately to +responses can set cookies, it needs a backend it can write to to separately store state. @@ -484,7 +484,7 @@ whereas you'd naturally expect ``receive`` to run after ``connect``. But, of course, Channels has a solution - the ``linearize`` decorator. Any handler decorated with this will use locking to ensure it does not run at the same time as any other view with ``linearize`` **on messages with the same reply channel**. -That means your site will happily mutitask with lots of different people's messages, +That means your site will happily multitask with lots of different people's messages, but if two happen to try to run at the same time for the same client, they'll be deconflicted. diff --git a/docs/installation.rst b/docs/installation.rst index 2becc63..288f12b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -21,8 +21,8 @@ That's it! Once enabled, ``channels`` will integrate itself into Django and take control of the ``runserver`` command. See :doc:`getting-started` for more. -Installing the lastest development version ------------------------------------------- +Installing the latest development version +----------------------------------------- To install the latest version of Channels, clone the repo, change to the repo, change to the repo directory, and pip install it into your current virtual diff --git a/docs/message-standards.rst b/docs/message-standards.rst index 44d4852..927d3b0 100644 --- a/docs/message-standards.rst +++ b/docs/message-standards.rst @@ -15,7 +15,7 @@ the received data (or something else based on ``reply_channel``). All messages must be able to be encoded as JSON; channel backends don't necessarily have to use JSON, but we consider it the lowest common denominator -for serialisation format compatability. +for serialisation format compatibility. The size limit on messages is 1MB (while channel backends may support larger sizes, all message formats should stay under this limit, which might include diff --git a/docs/releases/0.8.rst b/docs/releases/0.8.rst index ddeac26..9c4d15c 100644 --- a/docs/releases/0.8.rst +++ b/docs/releases/0.8.rst @@ -13,7 +13,7 @@ more efficient and user friendly: added to allow moving the user details from the HTTP session to the channel session in the ``connect`` consumer. * A ``@linearize`` decorator was added to help ensure a ``connect``/``receive`` pair are not executed - simultanously on two different workers. + simultaneously on two different workers. * Channel backends gained locking mechanisms to support the ``linearize`` feature. diff --git a/docs/scaling.rst b/docs/scaling.rst index 3bedf23..71cfd98 100755 --- a/docs/scaling.rst +++ b/docs/scaling.rst @@ -1,7 +1,7 @@ Scaling ======= -Of course, one of the downsides of introducing a channel layer to Django it +Of course, one of the downsides of introducing a channel layer to Django is that it's something else that must scale. Scaling traditional Django as a WSGI application is easy - you just add more servers and a loadbalancer. Your database is likely to be the thing that stopped scaling before, and there's From c9135ddcbe282ff19d9347a64deb9eba5ac368d2 Mon Sep 17 00:00:00 2001 From: knbk Date: Sun, 13 Dec 2015 02:31:52 +0100 Subject: [PATCH 110/746] Fixed typo --- channels/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/decorators.py b/channels/decorators.py index b123d86..7be50ee 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -81,7 +81,7 @@ def channel_session(func): def http_session(func): """ Wraps a HTTP or WebSocket connect consumer (or any consumer of messages - that provides a "cooikies" or "get" attribute) to provide a "http_session" + that provides a "cookies" or "get" attribute) to provide a "http_session" attribute that behaves like request.session; that is, it's hung off of a per-user session key that is saved in a cookie or passed as the "session_key" GET parameter. From ccad3d8e2f829f5a3be9a969fc69ea5125e02a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pau=20Ru=C5=80lan?= Date: Mon, 14 Dec 2015 10:29:10 +0100 Subject: [PATCH 111/746] add question about messagepack in faq from request #37 --- docs/faqs.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/faqs.rst b/docs/faqs.rst index 21e8caf..cbc0635 100755 --- a/docs/faqs.rst +++ b/docs/faqs.rst @@ -124,3 +124,19 @@ You can also provide your own solution if you wish, keyed off of ``message.reply which is the unique channel representing the connection, but remember that whatever you store in must be **network-transparent** - storing things in a global variable won't work outside of development. + + +Would you support messagepack or any other format? +-------------------------------------------------- + +Although we've evaluated msgpack it does not offer enough over JSON to be +reasonable - the encoding/decoding is often slower, the language support is +much poorer, and in general we would rather just have one version of a standard, +especially since there's plans to write parts of the Channels system in other +languages. + +That said, at some point it's up to the individual channel backend to support +whatever it likes, as long as it spits out dicts at either end. So this is +something that could be implemented by someone else as a pluggable backend to +see. We might always come back and revisit this if message size/bandwidth turns +out to be a limiting factor, though. From ff45689c46fb8f48f707afe766698f653f16c288 Mon Sep 17 00:00:00 2001 From: Reinout van Rees Date: Mon, 14 Dec 2015 14:26:37 +0100 Subject: [PATCH 112/746] Changed three "these/those" into explicit names The original version was unclear to me. It sounded like Go channels were the ones that are network-transparent even though we mean to say that Django's are. So I simply removed these/those and used "Go channels" and "Django channels" explicitly. (But *do* check whether I got it right! :-) ) --- docs/concepts.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 2b90fcf..53a4904 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -46,8 +46,9 @@ The channels have capacity, so a load of producers can write lots of messages into a channel with no consumers and then a consumer can come along later and will start getting served those queued messages. -If you've used `channels in Go `_, these are reasonably similar to those. The key -difference is that these channels are network-transparent; the implementations +If you've used `channels in Go `_: Go channels +are reasonably similar to Django ones. The key difference is that +Django channels channels are network-transparent; the implementations of channels we provide are all accessible across a network to consumers and producers running in different processes or on different machines. From 42d437eaf842ad4e331d698ae6a61ece7d5007e3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 15 Dec 2015 19:54:58 +0000 Subject: [PATCH 113/746] Fixed #44: Don't make autobahn check host/path --- channels/interfaces/websocket_asyncio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/interfaces/websocket_asyncio.py b/channels/interfaces/websocket_asyncio.py index c41871e..043c7d2 100644 --- a/channels/interfaces/websocket_asyncio.py +++ b/channels/interfaces/websocket_asyncio.py @@ -20,7 +20,7 @@ class WebsocketAsyncioInterface(object): self.port = port def run(self): - self.factory = get_factory(WebSocketServerFactory)("ws://0.0.0.0:%i" % self.port, debug=False) + self.factory = get_factory(WebSocketServerFactory)(debug=False) self.factory.protocol = get_protocol(WebSocketServerProtocol) self.loop = asyncio.get_event_loop() coro = self.loop.create_server(self.factory, '0.0.0.0', self.port) From a6073157f4845c37f7711fb0c203862c0ab8d17c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 15 Dec 2015 19:56:49 +0000 Subject: [PATCH 114/746] Fixed #47: Wrong import path for redis backend in docs --- docs/backends.rst | 2 +- docs/deploying.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/backends.rst b/docs/backends.rst index bb92930..e132654 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -21,7 +21,7 @@ but you can override this with the ``HOSTS`` setting:: CHANNEL_BACKENDS = { "default": { - "BACKEND": "channels.backends.redis.RedisChannelBackend", + "BACKEND": "channels.backends.redis_py.RedisChannelBackend", "HOSTS": [("redis-channel-1", 6379), ("redis-channel-2", 6379)], }, } diff --git a/docs/deploying.rst b/docs/deploying.rst index f9fcd1a..1d133c4 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -36,7 +36,7 @@ here's an example for a remote Redis server:: CHANNEL_BACKENDS = { "default": { - "BACKEND": "channels.backends.redis.RedisChannelBackend", + "BACKEND": "channels.backends.redis_py.RedisChannelBackend", "HOSTS": [("redis-channel", 6379)], }, } From c5fdf16568faeeeb81a0bf98e8696ab25e635d7f Mon Sep 17 00:00:00 2001 From: Bastian Hoyer Date: Wed, 16 Dec 2015 20:10:34 +0100 Subject: [PATCH 115/746] Fixes #44 also for twisted backend --- channels/interfaces/websocket_twisted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py index f8a4c4b..1a924ac 100644 --- a/channels/interfaces/websocket_twisted.py +++ b/channels/interfaces/websocket_twisted.py @@ -20,7 +20,7 @@ class WebsocketTwistedInterface(object): self.port = port def run(self): - self.factory = get_factory(WebSocketServerFactory)("ws://0.0.0.0:%i" % self.port, debug=False) + self.factory = get_factory(WebSocketServerFactory)(debug=False) self.factory.protocol = get_protocol(WebSocketServerProtocol) reactor.listenTCP(self.port, self.factory) reactor.callInThread(self.backend_reader) From 70bc429d0ba4461c6b50ee6b21f6214663b27f85 Mon Sep 17 00:00:00 2001 From: Bastian Hoyer Date: Wed, 16 Dec 2015 20:42:56 +0100 Subject: [PATCH 116/746] Fixed #39: Allow redis:// URL redis configuration --- channels/backends/redis_py.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py index 0cfbea9..3ef5c07 100644 --- a/channels/backends/redis_py.py +++ b/channels/backends/redis_py.py @@ -22,10 +22,12 @@ class RedisChannelBackend(BaseChannelBackend): # Make sure they provided some hosts, or provide a default if not hosts: hosts = [("localhost", 6379)] - for host, port in hosts: - assert isinstance(host, six.string_types) - assert int(port) - self.hosts = hosts + self.hosts = [] + for entry in hosts: + if isinstance(entry, six.string_types): + self.hosts.append(entry) + else: + self.hosts.append("redis://%s:%d/0" % (entry[0],entry[1])) self.prefix = prefix # Precalculate some values for ring selection self.ring_size = len(self.hosts) @@ -57,8 +59,7 @@ class RedisChannelBackend(BaseChannelBackend): # Catch bad indexes if not (0 <= index < self.ring_size): raise ValueError("There are only %s hosts - you asked for %s!" % (self.ring_size, index)) - host, port = self.hosts[index] - return redis.Redis(host=host, port=port) + return redis.Redis.from_url(self.hosts[index]) def send(self, channel, message): # if channel is no str (=> bytes) convert it From 5461c2db036b8a0c75b26a4062b2364f730867b6 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 17 Dec 2015 00:40:11 +0000 Subject: [PATCH 117/746] Add first draft of integration plan --- docs/index.rst | 1 + docs/integration-plan.rst | 84 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 docs/integration-plan.rst diff --git a/docs/index.rst b/docs/index.rst index d246cee..1de45f8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,5 +30,6 @@ Contents: message-standards scaling backends + integration-plan faqs releases/index diff --git a/docs/integration-plan.rst b/docs/integration-plan.rst new file mode 100644 index 0000000..f0f4a0c --- /dev/null +++ b/docs/integration-plan.rst @@ -0,0 +1,84 @@ +Integration Plan +================ + +*DRAFT VERSION - NOT FINAL* + +Channels is to become a built-in feature for Django 1.10, but in order to aid adoption and to encourage community usage and support, it will also be backported to Django 1.8 and 1.9 via a third-party app. + +Obviously, we want to do this with as little code duplication as is sensible, so this document outlines the plan of how Channels will be integrated into Django and how it will be importable and usable from the supported versions. + +Separate Components +------------------- + +The major step in code reuse will be developing the main interface server as a separate project still released under the Django umbrella (current codename is ``daphne``). + +This piece of software will likely have its own release cycle but still be developed and maintained by Django as part of the core project, and will speak HTTP, WebSockets, and likely HTTP2 natively. It will be designed to be used either directly exposed to the Internet or behind a reverse proxy that serves static files, much like gunicorn or uwsgi would be deployed. + +This would then be supplemented by two implementations of the "worker" end of the Channels stack - one native in Django 1.10, and one as a third-party app for earlier Django versions. They would act very similarly, and there will be some code duplication between them, but the 1.10 version should be cleaner and faster as there's no need to re-route around the existing Django internals (though it can be done, as the current third-party app approach shows). + +All three components would need to share the channel backend implementations - there is still an unanswered question here about if those should be separate packages, or somehow bundled into the implementations themselves. + +Preserving Simplicty +-------------------- + +A main goal of the channels project is to keep the ability to just download and use Django as simple as it is now, which means that for every external dependency we introduce, there needs to be a way to work round it to just run Django in "classic" mode if the user wants to. + +The first key part of this is the WSGI interface server - it just plugs in as a WSGI application and routes things to the channel layer. Combined with an in-memory channel backend option, this means that the existing out-of-the-box setup will continue to work, with no need to have Redis around, or Autobahn/Twisted/asyncio installed. + +It's possible that Django 1.10 would come bundled with the interface server (``daphne``) in the tarball download, but bring it in as a dependency for package-manager-based installations (``pip`` as well as OS packages). This could also be how we handle sharing channel backend code between them. + +Standardization +--------------- + +Given the intention to develop this as two codebases (though Django will still mean the combination of both), it will likely be sensible if not necessary to standardise the way the two talk to each other. + +There are two levels to this standardisation - firstly, the message formats for how to encode different events, requests and responses (which you can see the initial version of over in :doc:`message-standards`), and secondly, the actual transports themselves, the channel backends. + +Message format standardisation shouldn't be too hard; the top-level constraint on messages will be that they must be JSON-serializable dicts with no size limit; from there, developing sensible mappings for HTTP and WebSocket is not too hard, especially drawing on the WSGI spec for the former (indeed, Channels will have to be able to reconstitude WSGI-style properties and variables to let existing view code continue to work) + +Transport standardisation is different, though; the current approach is to have a standard backend interface that either core backends or third-party ones can implement, much like Django's database support; this would seem to fill the immediate need of both having core, tested and scalable transports as well as the option for more complex projects with special requirements to write their own. + +That said, the current proposed system is just one transport standard away from being able to interoperate with other languages; specifically, one can imagine an interface server written in a compiled language like Go or Rust that is more efficient that its Python equivalent (or even a Python implementation that uses a concurrency model that doesn't fit the channel backend's poll-style interface). + +It may be that the Redis backend itself is written up and standardised as well to provide a clear spec that third parties can code against from scratch; however, this will need to be after a period of iterative testing to ensure whatever is proposed can scale to handle both large messages and large volumes of messages, and can shard horizontally. + +There is no intention to propose a WSGI replacement at this point, but there's potential to do that at the fundamental Python level of "your code will get given messages, and can send them, and here's the message formats". However, such an effort should not be taken lightly and only attempted if and when channels proves itself to work as effectively as WSGI as the abstraction for handling web requests, and its servers are as stable and mature as those for WSGI; Django will continue to support WSGI for the foreseeable future. + +Imports +------- + +Having Channels as a third-party package on some versions of Django and as a core package on others presents an issue for anyone trying to write portable code; the import path will differ. + +Specifically, for the third party app you might import from ``channels``, whereas for the native one you might import from ``django.channels``. + +There are two sensible solutions to this: + +1. Make both versions available under the import path ``django.channels`` +2. Make people do try/except imports for portable code + +Neither are perfect; the first likely means some nasty monkeypatching or more, especially on Python 2.7, while the second involves verbose code. More research is needed here. + +WSGI attribute access +--------------------- + +While Channels transports across a lot of the data available on a WSGI request - like paths, remote client IPs, POST data and so forth - it cannot reproduce everything. Specifically, the following changes will happen to Django request objects: + +- The SCRIPT_NAME attribute will always be ``""``, and so the ``PATH_INFO`` will always be the full path. +- ``request.META`` will not contain any environment variables from the system. +- The ``wsgi`` keys in ``request.META`` will change as follows: + - ``wsgi.version`` will be set to ``(1, 0)`` + - ``wsgi.url_scheme`` will be populated correctly + - ``wsgi.input`` will work, but point to a StringIO or File object with the entire request body buffered already + - ``wsgi.errors`` will point to ``sys.stderr`` + - ``wsgi.multithread`` will always be ``True`` + - ``wsgi.multiprocess`` will always be ``True`` + - ``wsgi.run_once`` will always be ``False`` + +This is a best-attempt effort to mirror these variables' meanings to allow code to keep working on multiple Django versions; we will encourage people using ``url_scheme`` to switch to ``request.is_secure``, however, and people using ``input`` to use the ``read()`` or ``body`` attributes on ``request``. + +All of the ``wsgi`` variable emulation will be subject to the usual Django deprecation cycle and after this will not be available unless Django is running in a WSGI environment. + +Running Workers +--------------- + +It is not intended for there to be a separate command to run a Django worker; instead, ``manage.py runworker`` will be the recommended method, along with a wrapping process manager that handles logging and auto-restart (such as systemd or supervisord). From 84e78ea503ca9ef7244a5af16a73c2e7d8de2d7c Mon Sep 17 00:00:00 2001 From: Arne Schauf Date: Thu, 17 Dec 2015 14:42:31 +0100 Subject: [PATCH 118/746] fix 2 typos in integration plan docs --- docs/integration-plan.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/integration-plan.rst b/docs/integration-plan.rst index f0f4a0c..011bdfd 100644 --- a/docs/integration-plan.rst +++ b/docs/integration-plan.rst @@ -18,8 +18,8 @@ This would then be supplemented by two implementations of the "worker" end of th All three components would need to share the channel backend implementations - there is still an unanswered question here about if those should be separate packages, or somehow bundled into the implementations themselves. -Preserving Simplicty --------------------- +Preserving Simplicity +--------------------- A main goal of the channels project is to keep the ability to just download and use Django as simple as it is now, which means that for every external dependency we introduce, there needs to be a way to work round it to just run Django in "classic" mode if the user wants to. @@ -38,7 +38,7 @@ Message format standardisation shouldn't be too hard; the top-level constraint o Transport standardisation is different, though; the current approach is to have a standard backend interface that either core backends or third-party ones can implement, much like Django's database support; this would seem to fill the immediate need of both having core, tested and scalable transports as well as the option for more complex projects with special requirements to write their own. -That said, the current proposed system is just one transport standard away from being able to interoperate with other languages; specifically, one can imagine an interface server written in a compiled language like Go or Rust that is more efficient that its Python equivalent (or even a Python implementation that uses a concurrency model that doesn't fit the channel backend's poll-style interface). +That said, the current proposed system is just one transport standard away from being able to interoperate with other languages; specifically, one can imagine an interface server written in a compiled language like Go or Rust that is more efficient than its Python equivalent (or even a Python implementation that uses a concurrency model that doesn't fit the channel backend's poll-style interface). It may be that the Redis backend itself is written up and standardised as well to provide a clear spec that third parties can code against from scratch; however, this will need to be after a period of iterative testing to ensure whatever is proposed can scale to handle both large messages and large volumes of messages, and can shard horizontally. From 5a95248b3fb23683b476ddd0a1f2454f9e1af4ab Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 18 Dec 2015 20:55:58 +0000 Subject: [PATCH 119/746] Initial script name support --- channels/request.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/channels/request.py b/channels/request.py index 01be5b5..528a8b8 100644 --- a/channels/request.py +++ b/channels/request.py @@ -18,6 +18,7 @@ def encode_request(request): if k.lower().startswith("http_") }, "path": request.path, + "root_path": request.META.get("SCRIPT_NAME", ""), "method": request.method, "reply_channel": request.reply_channel, "server": [ @@ -51,11 +52,14 @@ def decode_request(value): "SERVER_PORT": value["server"][1], "REMOTE_ADDR": value["client"][0], "REMOTE_HOST": value["client"][0], # Not the DNS name, hopefully fine. + "SCRIPT_NAME": value["root_path"], } for header, header_value in value.get("headers", {}).items(): request.META["HTTP_%s" % header.upper()] = header_value - # We don't support non-/ script roots - request.path_info = value['path'] + # Derive path_info from script root + request.path_info = request.path + if request.META.get("SCRIPT_NAME", ""): + request.path_info = request.path_info[len(request.META["SCRIPT_NAME"]):] return request From 76ca034e63ed4d3812f8ab550d2d1c18adbf4a4b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 18 Dec 2015 21:00:24 +0000 Subject: [PATCH 120/746] Update message standards doc --- docs/message-standards.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/message-standards.rst b/docs/message-standards.rst index 927d3b0..c1e6108 100644 --- a/docs/message-standards.rst +++ b/docs/message-standards.rst @@ -36,9 +36,13 @@ Contains the following keys: * get: Dict of {key: [value, ...]} of GET variables (keys and values are strings) * post: Dict of {key: [value, ...]} of POST variables (keys and values are strings) * cookies: Dict of cookies as {cookie_name: cookie_value} (names and values are strings) +* headers: Dict of {header name: value}. Multiple headers of the same name are concatenated into one value separated by commas. * meta: Dict of HTTP headers and info as defined in the Django Request docs (names and values are strings) * path: String, full path to the requested page, without query string or domain +* root_path: Path designated as the "root" of the application (SCRIPT_NAME) * method: String, upper-cased HTTP method +* server: [host, port] showing the address the client connected to +* client: [host, port] of the remote client Should come with an associated ``reply_channel`` which accepts HTTP Responses. From ecb2e4c22b80100637197e77f192a53e62f10681 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 18 Dec 2015 21:34:47 +0000 Subject: [PATCH 121/746] Remove old note --- docs/integration-changes.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/integration-changes.rst b/docs/integration-changes.rst index c59fa38..7fa5c6e 100644 --- a/docs/integration-changes.rst +++ b/docs/integration-changes.rst @@ -4,8 +4,7 @@ Integration Notes Django Channels is intended to be merged into Django itself; these are the planned changes the codebase will need to undertake in that transition. -* The ``channels`` package will become ``django.channels``. The expected way - of interacting with the system will be via the ``Channel`` object, +* The ``channels`` package will become ``django.channels``, and main objects will keep their import path. * Obviously, the monkeypatches in ``channels.hacks`` will be replaced by placing methods onto the objects themselves. The ``request`` and ``response`` @@ -17,8 +16,3 @@ Things to ponder * The mismatch between signals (broadcast) and channels (single-worker) means we should probably leave patching signals into channels for the end developer. This would also ensure the speedup improvements for empty signals keep working. - -* It's likely that the decorator-based approach of consumer registration will - mean extending Django's auto-module-loading beyond ``models`` and - ``admin`` app modules to include ``views`` and ``consumers``. There may be - a better unified approach to this. From 03c9c90f4ce0231a546ac47682a287c2d1c19612 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 18 Dec 2015 22:44:33 +0000 Subject: [PATCH 122/746] Clarify WSGI in integration plan --- docs/integration-plan.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/integration-plan.rst b/docs/integration-plan.rst index 011bdfd..4fd1a7a 100644 --- a/docs/integration-plan.rst +++ b/docs/integration-plan.rst @@ -23,9 +23,9 @@ Preserving Simplicity A main goal of the channels project is to keep the ability to just download and use Django as simple as it is now, which means that for every external dependency we introduce, there needs to be a way to work round it to just run Django in "classic" mode if the user wants to. -The first key part of this is the WSGI interface server - it just plugs in as a WSGI application and routes things to the channel layer. Combined with an in-memory channel backend option, this means that the existing out-of-the-box setup will continue to work, with no need to have Redis around, or Autobahn/Twisted/asyncio installed. +To this end, Django will still be deployable with a WSGI server as it is now, with all channels-dependent features disabled, and when it ships, will fall back to a standard WSGI `runserver` if the dependencies to run a more complex interface server aren't installed (but if they are, `runserver` will become websocket-aware). -It's possible that Django 1.10 would come bundled with the interface server (``daphne``) in the tarball download, but bring it in as a dependency for package-manager-based installations (``pip`` as well as OS packages). This could also be how we handle sharing channel backend code between them. +There will also be options to plug Channels into a WSGI server as a WSGI application that then forwards into a channel backend, if users wish to keep using familiar webservers but still run a worker cluster and gain things like background tasks (or WebSockets using a second server process) Standardization --------------- @@ -61,9 +61,8 @@ Neither are perfect; the first likely means some nasty monkeypatching or more, e WSGI attribute access --------------------- -While Channels transports across a lot of the data available on a WSGI request - like paths, remote client IPs, POST data and so forth - it cannot reproduce everything. Specifically, the following changes will happen to Django request objects: +While Channels transports across a lot of the data available on a WSGI request - like paths, remote client IPs, POST data and so forth - it cannot reproduce everything. Specifically, the following changes will happen to Django request objects when they come via Channels: -- The SCRIPT_NAME attribute will always be ``""``, and so the ``PATH_INFO`` will always be the full path. - ``request.META`` will not contain any environment variables from the system. - The ``wsgi`` keys in ``request.META`` will change as follows: - ``wsgi.version`` will be set to ``(1, 0)`` From 44b10836e06e309cea531fd8628982811c18ad5c Mon Sep 17 00:00:00 2001 From: NiiEquity Date: Mon, 21 Dec 2015 15:03:55 +0000 Subject: [PATCH 123/746] Update concepts.rst Correct the repetition of "channels" --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 53a4904..02c047d 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -48,7 +48,7 @@ will start getting served those queued messages. If you've used `channels in Go `_: Go channels are reasonably similar to Django ones. The key difference is that -Django channels channels are network-transparent; the implementations +Django channels are network-transparent; the implementations of channels we provide are all accessible across a network to consumers and producers running in different processes or on different machines. From 3bcfea0421d2805d0d5b23c1b9e95b12f471d2fd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 22 Dec 2015 16:57:20 +0000 Subject: [PATCH 124/746] Start on some kind of general spec. --- docs/asgi.rst | 194 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 docs/asgi.rst diff --git a/docs/asgi.rst b/docs/asgi.rst new file mode 100644 index 0000000..05a1c41 --- /dev/null +++ b/docs/asgi.rst @@ -0,0 +1,194 @@ +PEP: XXX +Title: ASGI (Asynchronous Server Gateway Interface) +Version: $Revision$ +Last-Modified: $Date$ +Author: Andrew Godwin +Status: Draft +Type: Informational +Content-Type: text/x-rst +Created: ? +Post-History: ? + + +Abstract +======== + +This document proposes a standard interface between network protocol +servers and Python applications, intended to allow handling of multiple +protocol styles (including HTTP, HTTP2, and WebSocket). + +It is intended to replace and expand on WSGI, though the design +deliberately includes provisions to allow WSGI-to-ASGI and ASGI-to-WGSI +adapters to be easily written for the HTTP protocol. + + +Rationale +========= + +The WSGI specification has worked well since it was introduced, and +allowed for great flexibility in Python framework and webserver choice. +However, its design is irrevocably tied to the HTTP-style +request/response cycle, and more and more protocols are becoming a +standard part of web programming that do not follow this pattern +(most notably, WebSocket). + +ASGI attempts to preserve a simple application interface, but provide +an abstraction that allows for data to be sent and received at any time, +and from different application threads or processes. + +It also lays out new, serialisation-compatible formats for things like +HTTP requests and responses, to allow these to be transported over a +network or local socket, and thus allow separation of protocol handling +and application logic. + +Part of this design is ensuring there is an easy path to use both +existing WSGI servers and applications, as a large majority of Python +web usage relies on WSGI and providing an easy path forwards is critical +to adoption. + + +Overview +======== + +ASGI consists of three different components - *protocol servers*, +a *channel layer*, and *application code*. Channel layers are the core +part of the implementation, and provide an interface to both protocol +servers and applications. + +A channel layer provides a protocol server or an application server +with a ``send`` callable, which takes a channel name and message +``dict``, and a ``receive_many`` callable, which takes a list of +channel names and returns the next message available on any named channel. + +Thus, rather than under WSGI, where you point the protocol server to the +application, under ASGI you point both the protocol server and the application +to a channel layer instance. It is intended that applications and protocol +servers always run in separate processes or threads, and always communicate +via the channel layer. + +Despite the name of the proposal, ASGI does not specify or design to any +specific in-process async solution, such as ``asyncio``, ``twisted``, or +``gevent``. Instead, the ``receive_many`` function is nonblocking - it either +returns ``None`` or a ``(channel, message)`` tuple immediately. Integrating +this into both synchronous and asynchronous code should be easy, and it's +still possible for channel layers to use an async solution (such as +``asyncio``) if one is provided, and just provide the nonblocking call +via an in-memory queue. + +The distinction between protocol servers and applications in this document +is mostly to distinguish their roles and to make illustrating concepts easier. +There is no code-level distinction between the two, and it's entirely possible +to have a process that does both, or middleware-like code that transforms +messages between two different channel layers or channel names. It is +expected, however, that most deployments will fall into this pattern. + + +Channels and Messages +--------------------- + +All communication in an ASGI stack is using messages sent over channels. +All messages must be a ``dict`` at the top level of the object, and be +serialisable by the built-in ``json`` serialiser module (though the +actual serialisation a channel layer uses is up to the implementation; +``json`` is just considered the lowest common denominator). + +Channels are identified by a bytestring name consisting only of ASCII +letters, numbers, numerical digits, periods (``.``), dashes (``-``) +and underscores (``_``), plus an optional prefix character (see below). + +Channels are a first-in, first out queue with at-most-once delivery +semantics. They can have multiple writers and multiple readers; only a single +reader should get each written message. Implementations should never +deliver a message more than once or to more than one reader, and should +drop messages if this is necessary to achieve this restriction. + +In order to aid with scaling and network architecture, a distinction +is made between channels that have multiple readers (such as the +``http.request`` channel that web applications would listen on from every +application worker process) and *single-reader channels* +(such as a ``http.response.ABCDEF`` channel tied to a client socket). + +*Single-reader channel* names are prefixed with an exclamation mark +(``!``) character in order to indicate to the channel layer that it may +have to route these channels' data differently to ensure it reaches the +single process that needs it; these channels are nearly always tied to +incoming connections from the outside world. Some channel layers may not +need this, and can simply treat the prefix as part of the name. + +Messages should expire after a set time sitting unread in a channel; +the recommendation is one minute, though the best value depends on the +channel layer and the way it is deployed. + + +Handling Protocols +------------------ + +ASGI messages represent two main things - internal application events +(for example, a channel might be used to queue thumbnails of previously +uploaded videos), and protocol events to/from connected clients. + +As such, this specification outlines encodings to and from ASGI messages +for three common protocols (HTTP, WebSocket and raw UDP); this allows any ASGI +web server to talk to any ASGI web application, and the same for any other +protocol with a common specification. It is recommended that if other +protocols become commonplace they should gain standardised formats in a +supplementary PEP of their own. + +The message formats are a key part of the specification; without them, +the protocol server and web application might be able to talk to each other, +but may not understand some of what they're saying. It's equivalent to the +standard keys in the ``environ`` dict for WSGI. + +The key abstraction is that most protocols will share a few channels for +incoming data (for example, ``http.request``, ``websocket.connect`` and +``websocket.receive``), but will have individual channels for sending to +each client (such as ``!http.response.kj2daj23``). This allows incoming +data to be dispatched into a cluster of application servers that can all +handle it, while responses are routed to the individual protocol server +that has the other end of the client's socket. + +Some protocols, however, do not have the concept of a unique socket +connection; for example, an SMS gateway protocol server might just have +``sms.receive`` and ``sms.send``, and the protocol server cluster would +take messages from ``sms.send`` and route them into the normal phone +network based on attributes in the message (in this case, a telephone +number). + + +Groups +------ + +While the basic channel model is sufficient to handle basic application +needs, many more advanced uses of asynchronous messaging require +notifying many users at once when an event occurs - imagine a live blog, +for example, where every viewer should get a long poll response or +WebSocket packet when a new entry is posted. + +While the concept of a *group* of channels could be + + +Linearization +------------- + +The design of ASGI is meant to enable a shared-nothing architecture, +where messages + + +Specification Details +===================== + +A *channel layer* should provide an object with XXX attributes: + +* ``send(channel, message)``, a callable that takes two positional + arguments; the channel to send on, as a byte string, and the message + to send, as a serialisable ``dict``. + +* ``receive_many(channels)``, a callable that takes a list of channel + names as byte strings, and returns immediately with either ``None`` + or ``(channel, message)`` if a message is available. + + +Copyright +========= + +This document has been placed in the public domain. From 31ee80757e1405fb01463fc412dff76cac05288c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 22 Dec 2015 16:59:19 +0000 Subject: [PATCH 125/746] Big ASGI disclaimer. --- docs/asgi.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/asgi.rst b/docs/asgi.rst index 05a1c41..ab10280 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -9,6 +9,8 @@ Content-Type: text/x-rst Created: ? Post-History: ? +**NOTE: This is still heavily in-progress, and should not even be +considered draft yet. Even the name might change.** Abstract ======== From daa8d1aca1b801cb65205837215e218c8a64d41a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 23 Dec 2015 15:32:16 +0000 Subject: [PATCH 126/746] More work on the ASGI spec; pruning out some things. --- docs/asgi.rst | 637 ++++++++++++++++++++++++++++++++++++++++++++++--- docs/index.rst | 1 + 2 files changed, 601 insertions(+), 37 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index ab10280..870f0c4 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -1,23 +1,31 @@ -PEP: XXX -Title: ASGI (Asynchronous Server Gateway Interface) -Version: $Revision$ -Last-Modified: $Date$ -Author: Andrew Godwin -Status: Draft -Type: Informational -Content-Type: text/x-rst -Created: ? -Post-History: ? +=============== +Draft ASGI Spec +=============== **NOTE: This is still heavily in-progress, and should not even be -considered draft yet. Even the name might change.** +considered draft yet. Even the name might change; this is being written +as development progresses.** + +:: + + PEP: XXX + Title: ASGI (Asynchronous Server Gateway Interface) + Version: $Revision$ + Last-Modified: $Date$ + Author: Andrew Godwin + Status: Draft + Type: Informational + Content-Type: text/x-rst + Created: ? + Post-History: ? Abstract ======== This document proposes a standard interface between network protocol -servers and Python applications, intended to allow handling of multiple -protocol styles (including HTTP, HTTP2, and WebSocket). +servers (particularly webservers) and Python applications, intended +to allow handling of multiple common protocol styles (including HTTP, HTTP2, +and WebSocket). It is intended to replace and expand on WSGI, though the design deliberately includes provisions to allow WSGI-to-ASGI and ASGI-to-WGSI @@ -38,10 +46,10 @@ ASGI attempts to preserve a simple application interface, but provide an abstraction that allows for data to be sent and received at any time, and from different application threads or processes. -It also lays out new, serialisation-compatible formats for things like -HTTP requests and responses, to allow these to be transported over a -network or local socket, and thus allow separation of protocol handling -and application logic. +It also lays out new, serialization-compatible formats for things like +HTTP requests and responses and WebSocket data frames, to allow these to +be transported over a network or local socket, and allow separation +of protocol handling and application logic into different processes. Part of this design is ensuring there is an easy path to use both existing WSGI servers and applications, as a large majority of Python @@ -71,11 +79,10 @@ via the channel layer. Despite the name of the proposal, ASGI does not specify or design to any specific in-process async solution, such as ``asyncio``, ``twisted``, or ``gevent``. Instead, the ``receive_many`` function is nonblocking - it either -returns ``None`` or a ``(channel, message)`` tuple immediately. Integrating -this into both synchronous and asynchronous code should be easy, and it's -still possible for channel layers to use an async solution (such as -``asyncio``) if one is provided, and just provide the nonblocking call -via an in-memory queue. +returns ``None`` or a ``(channel, message)`` tuple immediately. This approach +should work with either synchronous or asynchronous code; part of the design +of ASGI is to allow developers to still write synchronous code where possible, +as this makes for easier maintenance and less bugs, at the cost of performance. The distinction between protocol servers and applications in this document is mostly to distinguish their roles and to make illustrating concepts easier. @@ -88,13 +95,13 @@ expected, however, that most deployments will fall into this pattern. Channels and Messages --------------------- -All communication in an ASGI stack is using messages sent over channels. +All communication in an ASGI stack uses *messages* sent over *channels*. All messages must be a ``dict`` at the top level of the object, and be -serialisable by the built-in ``json`` serialiser module (though the -actual serialisation a channel layer uses is up to the implementation; -``json`` is just considered the lowest common denominator). +serializable by the built-in ``json`` serializer module (though the +actual serialization a channel layer uses is up to the implementation; +we use ``json`` as the lowest common denominator). -Channels are identified by a bytestring name consisting only of ASCII +Channels are identified by a byte string name consisting only of ASCII letters, numbers, numerical digits, periods (``.``), dashes (``-``) and underscores (``_``), plus an optional prefix character (see below). @@ -121,6 +128,14 @@ Messages should expire after a set time sitting unread in a channel; the recommendation is one minute, though the best value depends on the channel layer and the way it is deployed. +Message size is finite, though the maximum varies based on the channel layer +and the encoding it's using. Channel layers may reject messages at ``send()`` +time with a ``MessageTooLarge`` exception; the calling code should take +appropriate action (e.g. HTTP responses can be chunked, while HTTP +requests should be closed with a ``413 Request Entity Too Large`` response). +It is intended that some channel layers will only support messages of around a +megabyte, while others will be able to take a gigabyte or more, and that it +may be configurable. Handling Protocols ------------------ @@ -133,15 +148,15 @@ As such, this specification outlines encodings to and from ASGI messages for three common protocols (HTTP, WebSocket and raw UDP); this allows any ASGI web server to talk to any ASGI web application, and the same for any other protocol with a common specification. It is recommended that if other -protocols become commonplace they should gain standardised formats in a -supplementary PEP of their own. +protocols become commonplace they should gain standardized formats in a +supplementary specification of their own. The message formats are a key part of the specification; without them, the protocol server and web application might be able to talk to each other, -but may not understand some of what they're saying. It's equivalent to the +but may not understand some of what the other is saying. It's equivalent to the standard keys in the ``environ`` dict for WSGI. -The key abstraction is that most protocols will share a few channels for +The design pattern is that most protocols will share a few channels for incoming data (for example, ``http.request``, ``websocket.connect`` and ``websocket.receive``), but will have individual channels for sending to each client (such as ``!http.response.kj2daj23``). This allows incoming @@ -157,6 +172,27 @@ network based on attributes in the message (in this case, a telephone number). +Extensions +---------- + +ASGI has the concept of *extensions*, of which one is specified in this +document. Extensions are functionality that is +not required for basic application code and nearly all protocol server +code, and so has been made optional in order to encourage lighter-weight +channel layers to be written. + +The only extension in this document is the ``groups`` extension, defined +below. + +There is potential to add further extensions; these may be defined by +a separate specification, or a new version of this specification. + +If application code requires an extension, it should check for it as soon +as possible, and hard error if it is not provided. Frameworks should +encourage optional use of extensions, while attempting to move any +extension-not-found errors to process startup rather than message handling. + + Groups ------ @@ -166,29 +202,556 @@ notifying many users at once when an event occurs - imagine a live blog, for example, where every viewer should get a long poll response or WebSocket packet when a new entry is posted. -While the concept of a *group* of channels could be +This concept could be kept external to the ASGI spec, and would be, if it +were not for the significant performance gains a channel layer implementation +could make on the send-group operation by having it included - the +alternative being a ``send_many`` callable that might have to take +tens of thousands of destination channel names in a single call. However, +the group feature is still optional; its presence is indicated by the +``supports_groups`` attribute on the channel layer object. + +Thus, there is a simple Group concept in ASGI, which acts as the +broadcast/multicast mechanism across channels. Channels are added to a group, +and then messages sent to that group are sent to all members of the group. +Channels expire from being in a group after a certain amount of time, +and must be refreshed periodically to remain in it, and can also be +explicitly removed. + +The expiry is because this specification assumes that at some point +message delivery will fail, and so disconnection events by themselves +are not sufficient to tie to an explicit group removal - over time, the +number of group members will slowly increase as old response channels +leak as disconnections get dropped. + +Instead, all protocol servers that have an ongoing connection +(for example, long-poll HTTP or WebSockets) will instead send periodic +"keepalive" messages, which can be used to refresh the response channel's +group membership - each call to ``group_add`` should reset the expiry timer. + +Keepalive message intervals should be one-third as long as the group expiry +timeout, to allow for slow or missed delivery of keepalives; protocol servers +and anything else sending keepalives can retrieve the group expiry time from +the channel layer in order to do this correctly. + +*Implementation of the group functionality is optional*. If it is not provided +and an application or protocol server requires it, they should hard error +and exit with an appropriate error message. It is expected that protocol +servers will not need to use groups. Linearization ------------- The design of ASGI is meant to enable a shared-nothing architecture, -where messages +where messages can be handled by any one of a set of threads, processes +or machines running application code. + +This, of course, means that several different copies of the application +could be handling messages simultaneously, and those messages could even +be from the same client; in the worst case, two packets from a client +could even be processed out-of-order if one server is slower than another. + +This is an existing issue with things like WSGI as well - a user could +open two different tabs to the same site at once and launch simultaneous +requests to different servers - but the nature of the new protocols +specified here mean that collisions are more likely to occur. + +Solving this issue is left to frameworks and application code; there are +already solutions such as database transactions that help solve this, +and the vast majority of application code will not need to deal with this +problem. If ordering of incoming packets matters for a protocol, they should +be annotated with a packet number (as WebSocket is in this specification). + +Single-reader channels, such as those used for response channels back to +clients, are not subject to this problem; a single reader should always +receive messages in channel order. Specification Details ===================== -A *channel layer* should provide an object with XXX attributes: +A *channel layer* should provide an object with these attributes +(all function arguments are positional): -* ``send(channel, message)``, a callable that takes two positional - arguments; the channel to send on, as a byte string, and the message - to send, as a serialisable ``dict``. +* ``send(channel, message)``, a callable that takes two arguments; the + channel to send on, as a byte string, and the message + to send, as a serializable ``dict``. * ``receive_many(channels)``, a callable that takes a list of channel names as byte strings, and returns immediately with either ``None`` or ``(channel, message)`` if a message is available. +* ``new_channel(format)``, a callable that takes a byte string pattern, + and returns a new valid channel name that does not already exist, by + substituting any occurrences of the question mark character ``?`` in + ``format`` with a single random byte string and checking for + existence of that name in the channel layer. This is NOT called prior to + a message being sent on a channel, and should not be used for channel + initialization. + +* ``MessageTooLarge``, the exception raised when a send operation fails + because the encoded message is over the layer's size limit. + +* ``extensions``, a list of byte string names indicating which + extensions this layer provides, or empty if it supports none. + The only valid extension name is ``groups``. + +A channel layer implementing the ``groups`` extension must also provide: + +* ``group_add(group, channel)``, a callable that takes a ``channel`` and adds + it to the group given by ``group``. Both are byte strings. + +* ``group_discard(group, channel)``, a callable that removes the ``channel`` + from the ``group`` if it is in it, and does nothing otherwise. + +* ``send_group(group, message)``, a callable that takes two positional + arguments; the group to send to, as a byte string, and the message + to send, as a serializable ``dict``. + +* ``group_expiry``, an integer number of seconds describing the minimum + group membership age before a channel is removed from a group. + + +Channel Semantics +----------------- + +Channels **must**: + +* Preserve ordering of messages perfectly with only a single reader + and writer, and preserve as much as possible in other cases. + +* Never deliver a message more than once. + +* Never block on message send. + +* Be able to handle messages of at least 1MB in size when encoded as + JSON (the implementation may use better encoding or compression, as long + as it meets the equivalent size) + +* Have a maximum name length of at least 100 bytes. + +They are not expected to deliver all messages, but a success rate of at least +99.99% is expected under normal circumstances. Implementations may want to +have a "resilience testing" mode where they deliberately drop more messages +than usual so developers can test their code's handling of these scenarios. + + +Message Formats +--------------- + +These describe the standardized message formats for the protocols this +specification supports. All messages are ``dicts`` at the top level, +and all keys are required unless otherwise specified (with a default to +use if the key is missing). + +The one common key across all protocols is ``reply_channel``, a way to indicate +the client-specific channel to send responses to. Protocols are generally +encouraged to have one message type and one reply channel to ensure ordering. + +Messages are specified here along with the channel names they are expected +on; if a channel name can vary, such as with reply channels, the varying +portion will be replaced by ``?``, such as ``http.response.?``, which matches +the format the ``new_channel`` callable takes. + +There is no label on message types to say what they are; their type is implicit +in the channel name they are received on. Two types that are sent on the same +channel, such as HTTP responses and server pushes, are distinguished apart +by their required fields. + + +HTTP +---- + +The HTTP format covers HTTP/1.0, HTTP/1.1 and HTTP/2, as the changes in +HTTP/2 are largely on the transport level. A protocol server should give +different requests on the same connection different reply channels, and +correctly multiplex the responses back into the same stream as they come in. +The HTTP version is available as a string in the request message. + +HTTP/2 Server Push responses are included, but should be sent prior to the +main response, and you should check for ``http_version = 2`` before sending +them; if a protocol server or connection incapable of Server Push receives +these, it should simply drop them. + +The HTTP specs are somewhat vague on the subject of multiple headers; +RFC7230 explicitly says they must be mergeable with commas, while RFC6265 +says that ``Set-Cookie`` headers cannot be combined this way. This is why +request ``headers`` is a ``dict``, and response ``headers`` is a list of +tuples, which matches WSGI. + +Request +''''''' + +Sent once for each request that comes into the protocol server. + +Channel: ``http.request`` + +Keys: + +* ``reply_channel``: Channel name for responses and server pushes, in + format ``http.response.?`` + +* ``http_version``: Byte string, one of ``1.0``, ``1.1`` or ``2``. + +* ``method``: Byte string HTTP method name, uppercased. + +* ``scheme``: Byte string URL scheme portion (likely ``http`` or ``https``). + Optional (but must not be empty), default is ``http``. + +* ``path``: Byte string HTTP path from URL. + +* ``query_string``: Byte string URL portion after the ``?``. Optional, default + is empty string. + +* ``root_path``: Byte string that indicates the root path this application + is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults + to empty string. + +* ``headers``: Dict of ``{name: value}``, where ``name`` is the lowercased + HTTP header name as byte string and ``value`` is the header value as a byte + string. If multiple headers with the same name are received, they should + be concatenated into a single header as per . + +* ``body``: Body of the request, as a byte string. Optional, defaults to empty + string. + +* ``client``: List of ``[host, port]`` where ``host`` is a byte string of the + remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an + integer. Optional, defaults to ``None``. + +* ``server``: List of ``[host, port]`` where ``host`` is the listening address + for this server as a byte string, and ``port`` is the integer listening port. + Optional, defaults to ``None``. + + +Response +'''''''' + +Send after any server pushes, and before any response chunks. + +Channel: ``http.response.?`` + +Keys: + +* ``status``: Integer HTTP status code. + +* ``status_text``: Byte string HTTP reason-phrase, e.g. ``OK`` from ``200 OK``. + Ignored for HTTP/2 clients. Optional, default should be based on ``status`` + or left as empty string if no default found. + +* ``headers``: A list of ``[name, value]`` pairs, where ``name`` is the byte + string header name, and ``value`` is the byte string header value. Order + should be preserved in the HTTP response. + +* ``content``: Byte string of HTTP body content + +* ``more_content``: Boolean value signifying if there is additional content + to come (as part of a Response Chunk message). If ``False``, response will + be taken as complete and closed off, and any further messages on the channel + will be ignored. Optional, defaults to ``False``. + + +Response Chunk +'''''''''''''' + +Must be sent after an initial Response. + +Channel: ``http.response.?`` + +Keys: + +* ``content``: Byte string of HTTP body content, will be concatenated onto + previously received ``content`` values. + +* ``more_content``: Boolean value signifying if there is additional content + to come (as part of a Response Chunk message). If ``False``, response will + be taken as complete and closed off, and any further messages on the channel + will be ignored. Optional, defaults to ``False``. + + +Server Push +''''''''''' + +Send before any Response or Response Chunk. HTTP/2 only. + +TODO + + +WebSocket +--------- + +WebSockets share some HTTP details - they have a path and headers - but also +have more state. Path and header details are only sent in the connection +message; applications that need to refer to these during later messages +should store them in a cache or database. + +WebSocket protocol servers should handle PING/PONG requests themselves, and +send PING frames as necessary to ensure the connection is alive. + + +Connection +'''''''''' + +Sent when the client initially opens a connection and completes the +WebSocket handshake. + +Channel: ``websocket.connect`` + +Keys: + +* ``reply_channel``: Channel name for sending data, in + format ``websocket.send.?`` + +* ``scheme``: Byte string URL scheme portion (likely ``ws`` or ``wss``). + Optional (but must not be empty), default is ``ws``. + +* ``path``: Byte string HTTP path from URL. + +* ``query_string``: Byte string URL portion after the ``?``. Optional, default + is empty string. + +* ``root_path``: Byte string that indicates the root path this application + is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults + to empty string. + +* ``headers``: Dict of ``{name: value}``, where ``name`` is the lowercased + HTTP header name as byte string and ``value`` is the header value as a byte + string. If multiple headers with the same name are received, they should + be concatenated into a single header as per . + +* ``client``: List of ``[host, port]`` where ``host`` is a byte string of the + remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an + integer. Optional, defaults to ``None``. + +* ``server``: List of ``[host, port]`` where ``host`` is the listening address + for this server as a byte string, and ``port`` is the integer listening port. + Optional, defaults to ``None``. + + +Receive +''''''' + +Sent when a data frame is received from the client. + +Channel: ``websocket.receive`` + +Keys: + +* ``reply_channel``: Channel name for sending data, in + format ``websocket.send.?`` + +* ``bytes``: Byte string of frame content, if it was bytes mode, or ``None``. + +* ``text``: Unicode string of frame content, if it was text mode, or ``None``. + +One of ``bytes`` or ``text`` must be non-``None``. + + +Disconnection +''''''''''''' + +Sent when either connection to the client is lost, either from the client +closing the connection, the server closing the connection, or loss of the +socket. + +Channel: ``websocket.disconnect`` + +Keys: + +* ``reply_channel``: Channel name that was used for sending data, in + format ``websocket.send.?``. Cannot be used to send at this point; provided + as a way to identify the connection only. + + +Send/Close +'''''''''' + +Sends a data frame to the client and/or closes the connection from the +server end. + +Channel: ``websocket.send.?`` + +Keys: + +* ``bytes``: Byte string of frame content, if in bytes mode, or ``None``. + +* ``text``: Unicode string of frame content, if in text mode, or ``None``. + +* ``close``: Boolean saying if the connection should be closed after data + is sent, if any. Optional, default ``False``. + +A maximum of one of ``bytes`` or ``text`` may be provided. If both are +provided, the protocol server should ignore the message entirely. + + +UDP +--- + +Raw UDP is included here as it is a datagram-based, unordered and unreliable +protocol, which neatly maps to the underlying message abstraction. It is not +expected that many applications would use the low-level protocol, but it may +be useful for some. + +While it might seem odd to have reply channels for UDP as it is a stateless +protocol, replies need to come from the same server as the messages were +sent to, so the reply channel here ensures that reply packets from an ASGI +stack do not come from a different protocol server to the one you sent the +initial packet to. + + +Receive +''''''' + +Sent when a UDP datagram is received. + +Channel: ``udp.receive`` + +Keys: + +* ``reply_channel``: Channel name for sending data, in format ``udp.send.?`` + +* ``data``: Byte string of UDP datagram payload. + +* ``client``: List of ``[host, port]`` where ``host`` is a byte string of the + remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an + integer. + +* ``server``: List of ``[host, port]`` where ``host`` is the listening address + for this server as a byte string, and ``port`` is the integer listening port. + Optional, defaults to ``None``. + + +Send +'''' + +Sent to send out a UDP datagram to a client. + +Channel: ``udp.send.?`` + +Keys: + +* ``data``: Byte string of UDP datagram payload. + + +Approximate Global Ordering +--------------------------- + +While maintaining true global (across-channels) ordering of messages is +entirely unreasonable to expect of many implementations, they should strive +to prevent busy channels from overpowering quiet channels. + +For example, imagine two channels, ``busy``, which spikes to 1000 messages a +second, and ``quiet``, which gets one message a second. There's a single +consumer running ``receive_many(['busy', 'quiet'])`` which can handle +around 200 messages a second. + +In a simplistic for-loop implementation, the channel layer might always check +``busy`` first; it always has messages available, and so the consumer never +even gets to see a message from ``quiet``, even if it was sent with the +first batch of ``busy`` messages. + +A simple way to solve this is to randomize the order of the channel list when +looking for messages inside the channel layer; other, better methods are also +available, but whatever is chosen, it should try to avoid a scenario where +a message doesn't get received purely because another channel is busy. + + +Strings and Unicode +------------------- + +In this document, *byte string* refers to ``str`` on Python 2 and ``bytes`` +on Python 3. If this type still supports Unicode codepoints due to the +underlying implementation, then any values should be kept within the lower +8-byte range. + +*Unicode string* refers to ``unicode`` on Python 2 and ``str`` on Python 3. +This document will never specify just *string* - all strings are one of the +two types. + +Channel and group names are always byte strings, with the additional limitation +that they only use the following characters: + +* ASCII letters +* The digits ``0`` through ``9`` +* Hyphen ``-`` +* Underscore ``_`` +* Period ``.`` +* Exclamation mark ``!`` (only at the start of a channel name) + + +WSGI Compatibility +------------------ + +Part of the design of the HTTP portion of this spec is to make sure it +aligns well with the WSGI specification, to ensure easy adaptability +between both specifications and the ability to keep using WSGI servers or +applications with ASGI. + +The adaptability works in two ways: + +* WSGI Server to ASGI: A WSGI application can be written that transforms + ``environ`` into a Request message, sends it off on the ``http.request`` + channel, and then waits on a generated response channel for a Response + message. This has the disadvantage of tying up an entire WSGI thread + to poll one channel, but should not be a massive performance drop if + there is no backlog on the request channel, and would work fine for an + in-process adapter to run a pure-ASGI web application. + +* ASGI to WSGI application: A small wrapper process is needed that listens + on the ``http.request`` channel, and decodes incoming Request messages + into an ``environ`` dict that matches the WSGI specs, while passing in + a ``start_response`` that stores the values for sending with the first + content chunk. Then, the application iterates over the WSGI app, + packaging each returned content chunk into a Response or Response Chunk + message (if more than one is yielded). + +There is an almost direct mapping for the various special keys in +WSGI's ``environ`` variable to the Request message: + +* ``REQUEST_METHOD`` is the ``method`` key +* ``SCRIPT_NAME`` is ``root_path`` +* ``PATH_INFO`` can be derived from ``path`` and ``root_path`` +* ``QUERY_STRING`` is ``query_string`` +* ``CONTENT_TYPE`` can be extracted from ``headers`` +* ``CONTENT_LENGTH`` can be extracted from ``headers`` +* ``SERVER_NAME`` and ``SERVER_PORT`` are in ``server`` +* ``REMOTE_HOST`` and ``REMOTE_PORT`` are in ``client`` +* ``SERVER_PROTOCOL`` is encoded in ``http_version`` +* ``wsgi.url_scheme`` is ``scheme`` +* ``wsgi.input`` is a StringIO around ``body`` +* ``wsgi.errors`` is directed by the wrapper as needed + +The ``start_response`` callable maps similarly to Response: + +* The ``status`` argument becomes ``status`` and ``status_text`` +* ``response_headers`` maps to ``headers`` + +The main difference is that ASGI is incapable of performing streaming +of HTTP body input, and instead must buffer it all into a message first. + + +Common Questions +================ + +1. Why are messages ``dicts``, rather than a more advanced type? + + We want messages to be very portable, especially across process and + machine boundaries, and so a simple encodable type seemed the best way. + We expect frameworks to wrap each protocol-specific set of messages in + custom classes (e.g. ``http.request`` messages become ``Request`` objects) + + +TODOs +===== + +* Work out if we really can just leave HTTP body as byte string. Seems too big. + Might need some reverse-single-reader chunking? Or just say channel layer + message size dictates body size. + +* Maybe remove ``http_version`` and replace with ``supports_server_push``? + +* Be sure we want to leave HTTP ``get`` and ``post`` out. + Copyright ========= diff --git a/docs/index.rst b/docs/index.rst index 1de45f8..a7a5e6c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,4 +32,5 @@ Contents: backends integration-plan faqs + asgi releases/index From 3cf5e6149eea71549d794adcf0bf86e8916ab3cd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 23 Dec 2015 15:45:08 +0000 Subject: [PATCH 127/746] ASGI draft todo tweak --- docs/asgi.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 870f0c4..85bb03b 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -82,7 +82,8 @@ specific in-process async solution, such as ``asyncio``, ``twisted``, or returns ``None`` or a ``(channel, message)`` tuple immediately. This approach should work with either synchronous or asynchronous code; part of the design of ASGI is to allow developers to still write synchronous code where possible, -as this makes for easier maintenance and less bugs, at the cost of performance. +as this generally makes for easier maintenance and less bugs, +but at the cost of performance. The distinction between protocol servers and applications in this document is mostly to distinguish their roles and to make illustrating concepts easier. @@ -752,6 +753,9 @@ TODOs * Be sure we want to leave HTTP ``get`` and ``post`` out. +* ``receive_many`` can't easily be implemented with async/cooperative code + behind it as it's nonblocking - possible alternative call type? Extension? + Copyright ========= From 6a5907ff596338c4609498561be0f361dacd2902 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 23 Dec 2015 18:04:18 +0000 Subject: [PATCH 128/746] ASGI spec updates --- docs/asgi.rst | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 85bb03b..bebeb6e 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -78,12 +78,10 @@ via the channel layer. Despite the name of the proposal, ASGI does not specify or design to any specific in-process async solution, such as ``asyncio``, ``twisted``, or -``gevent``. Instead, the ``receive_many`` function is nonblocking - it either -returns ``None`` or a ``(channel, message)`` tuple immediately. This approach -should work with either synchronous or asynchronous code; part of the design -of ASGI is to allow developers to still write synchronous code where possible, -as this generally makes for easier maintenance and less bugs, -but at the cost of performance. +``gevent``. Instead, the ``receive_many`` function can be switched between +nonblocking or synchronous. This approach allows applications to choose what's +best for their current runtime environment; further improvements may provide +extensions where cooperative versions of receive_many are provided. The distinction between protocol servers and applications in this document is mostly to distinguish their roles and to make illustrating concepts easier. @@ -278,14 +276,17 @@ A *channel layer* should provide an object with these attributes channel to send on, as a byte string, and the message to send, as a serializable ``dict``. -* ``receive_many(channels)``, a callable that takes a list of channel - names as byte strings, and returns immediately with either ``None`` - or ``(channel, message)`` if a message is available. +* ``receive_many(channels, block=False)``, a callable that takes a list of channel + names as byte strings, and returns with either ``(None, None)`` + or ``(channel, message)`` if a message is available. If ``block`` is True, then + it will not return until after a built-in timeout or a message arrives; if + ``block`` is false, it will always return immediately. It is perfectly + valid to ignore ``block`` and always return immediately. -* ``new_channel(format)``, a callable that takes a byte string pattern, +* ``new_channel(pattern)``, a callable that takes a byte string pattern, and returns a new valid channel name that does not already exist, by substituting any occurrences of the question mark character ``?`` in - ``format`` with a single random byte string and checking for + ``pattern`` with a single random byte string and checking for existence of that name in the channel layer. This is NOT called prior to a message being sent on a channel, and should not be used for channel initialization. @@ -293,7 +294,7 @@ A *channel layer* should provide an object with these attributes * ``MessageTooLarge``, the exception raised when a send operation fails because the encoded message is over the layer's size limit. -* ``extensions``, a list of byte string names indicating which +* ``extensions``, a list of unicode string names indicating which extensions this layer provides, or empty if it supports none. The only valid extension name is ``groups``. @@ -343,7 +344,7 @@ Message Formats These describe the standardized message formats for the protocols this specification supports. All messages are ``dicts`` at the top level, and all keys are required unless otherwise specified (with a default to -use if the key is missing). +use if the key is missing). Keys are unicode strings. The one common key across all protocols is ``reply_channel``, a way to indicate the client-specific channel to send responses to. Protocols are generally @@ -754,7 +755,8 @@ TODOs * Be sure we want to leave HTTP ``get`` and ``post`` out. * ``receive_many`` can't easily be implemented with async/cooperative code - behind it as it's nonblocking - possible alternative call type? Extension? + behind it as it's nonblocking - possible alternative call type? + Asyncio extension that provides ``receive_many_yield``? Copyright From e78f75288db6e2e4700daeb93f4c940c8553a048 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 24 Dec 2015 22:58:13 +0000 Subject: [PATCH 129/746] Stats extension, application abstraction notes for ASGI --- docs/asgi.rst | 84 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index bebeb6e..c740edb 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -90,6 +90,12 @@ to have a process that does both, or middleware-like code that transforms messages between two different channel layers or channel names. It is expected, however, that most deployments will fall into this pattern. +There is even room for a WSGI-like application abstraction with a callable +which takes ``(channel, message, send_func)``, but this would be slightly +too restrictive for many use cases and does not cover how to specify +channel names to listen on; it is expected that frameworks will cover this +use case. + Channels and Messages --------------------- @@ -180,8 +186,9 @@ not required for basic application code and nearly all protocol server code, and so has been made optional in order to encourage lighter-weight channel layers to be written. -The only extension in this document is the ``groups`` extension, defined -below. +There are two extensions defined here: the ``groups`` extension, which +is expanded on below, and the ``statistics`` extension, which allows +channel layers to provide global and per-channel statistics. There is potential to add further extensions; these may be defined by a separate specification, or a new version of this specification. @@ -212,25 +219,15 @@ the group feature is still optional; its presence is indicated by the Thus, there is a simple Group concept in ASGI, which acts as the broadcast/multicast mechanism across channels. Channels are added to a group, and then messages sent to that group are sent to all members of the group. -Channels expire from being in a group after a certain amount of time, -and must be refreshed periodically to remain in it, and can also be -explicitly removed. +Channels can be removed from a group manually (e.g. based on a disconnect +event), and the channel layer will garbage collect "old" channels in groups +on a periodic basis. -The expiry is because this specification assumes that at some point -message delivery will fail, and so disconnection events by themselves -are not sufficient to tie to an explicit group removal - over time, the -number of group members will slowly increase as old response channels -leak as disconnections get dropped. - -Instead, all protocol servers that have an ongoing connection -(for example, long-poll HTTP or WebSockets) will instead send periodic -"keepalive" messages, which can be used to refresh the response channel's -group membership - each call to ``group_add`` should reset the expiry timer. - -Keepalive message intervals should be one-third as long as the group expiry -timeout, to allow for slow or missed delivery of keepalives; protocol servers -and anything else sending keepalives can retrieve the group expiry time from -the channel layer in order to do this correctly. +How this garbage collection happens is not specified here, as it depends on +the internal implementation of the channel layer. The recommended approach, +however, is when a message on a single-listener channel expires, the channel +layer should remove that channel from all groups it's currently a member of; +this is deemed an acceptable indication that the channel's listener is gone. *Implementation of the group functionality is optional*. If it is not provided and an application or protocol server requires it, they should hard error @@ -301,7 +298,8 @@ A *channel layer* should provide an object with these attributes A channel layer implementing the ``groups`` extension must also provide: * ``group_add(group, channel)``, a callable that takes a ``channel`` and adds - it to the group given by ``group``. Both are byte strings. + it to the group given by ``group``. Both are byte strings. If the channel + is already in the group, the function should return normally. * ``group_discard(group, channel)``, a callable that removes the ``channel`` from the ``group`` if it is in it, and does nothing otherwise. @@ -310,8 +308,22 @@ A channel layer implementing the ``groups`` extension must also provide: arguments; the group to send to, as a byte string, and the message to send, as a serializable ``dict``. -* ``group_expiry``, an integer number of seconds describing the minimum - group membership age before a channel is removed from a group. +A channel layer implementing the ``statistics`` extension must also provide: + +* ``global_statistics()``, a callable that returns a dict with zero + or more of (unicode string keys): + + * ``count``, the current number of messages waiting in all channels + +* ``channel_statistics(channel)``, a callable that returns a dict with zero + or more of (unicode string keys): + + * ``length``, the current number of messages waiting on the channel + * ``age``, how long the oldest message has been waiting, in seconds + * ``per_second``, the number of messages processed in the last second + + + Channel Semantics @@ -338,6 +350,24 @@ have a "resilience testing" mode where they deliberately drop more messages than usual so developers can test their code's handling of these scenarios. +Persistence +----------- + +Channel layers do not need to persist data long-term; group +memberships only need to live as long as a connection does, and messages +only as long as the message expiry time, which is usually a couple of minutes. + +That said, if a channel server goes down momentarily and loses all data, +persistent socket connections will continue to transfer incoming data and +send out new generated data, but will have lost all of their group memberships +and in-flight messages. + +In order to avoid a nasty set of bugs caused by these half-deleted sockets, +protocol servers should quit and hard restart if they detect that the channel +layer has gone down or lost data; shedding all existing connections and letting +clients reconnect will immediately resolve the problem. + + Message Formats --------------- @@ -376,7 +406,7 @@ them; if a protocol server or connection incapable of Server Push receives these, it should simply drop them. The HTTP specs are somewhat vague on the subject of multiple headers; -RFC7230 explicitly says they must be mergeable with commas, while RFC6265 +RFC7230 explicitly says they must be merge-able with commas, while RFC6265 says that ``Set-Cookie`` headers cannot be combined this way. This is why request ``headers`` is a ``dict``, and response ``headers`` is a list of tuples, which matches WSGI. @@ -758,6 +788,12 @@ TODOs behind it as it's nonblocking - possible alternative call type? Asyncio extension that provides ``receive_many_yield``? +* Possible extension to allow detection of channel layer flush/restart and + prompt protocol servers to restart? + +* Maybe WSGI-app like spec for simple "applications" that allows standardized + application-running servers? + Copyright ========= From b9464ca149a4f66ebeb8801254cdd6d1b12cd58d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 10:17:45 -0800 Subject: [PATCH 130/746] Start making channels work to ASGI spec. --- channels/__init__.py | 18 +- channels/adapters.py | 20 --- channels/asgi.py | 81 +++++++++ channels/channel.py | 46 +++--- channels/consumer_registry.py | 7 + channels/hacks.py | 22 --- channels/handler.py | 191 ++++++++++++++++++++++ channels/management/commands/runserver.py | 22 +-- channels/management/commands/runworker.py | 20 +-- channels/message.py | 8 +- channels/request.py | 73 --------- channels/response.py | 43 ----- channels/worker.py | 22 ++- docs/asgi.rst | 92 ++++++++--- 14 files changed, 405 insertions(+), 260 deletions(-) create mode 100644 channels/asgi.py create mode 100644 channels/handler.py delete mode 100644 channels/request.py delete mode 100644 channels/response.py diff --git a/channels/__init__.py b/channels/__init__.py index dde140b..f37e0ef 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,21 +1,7 @@ __version__ = "0.8" -# Load backends, using settings if available (else falling back to a default) -DEFAULT_CHANNEL_BACKEND = "default" - -from .backends import BackendManager # isort:skip -from django.conf import settings # isort:skip - -channel_backends = BackendManager( - getattr(settings, "CHANNEL_BACKENDS", { - DEFAULT_CHANNEL_BACKEND: { - "BACKEND": "channels.backends.memory.InMemoryChannelBackend", - "ROUTING": {}, - } - }) -) - default_app_config = 'channels.apps.ChannelsConfig' +DEFAULT_CHANNEL_LAYER = 'default' -# Promote channel to top-level (down here to avoid circular import errs) +from .asgi import channel_layers # NOQA isort:skip from .channel import Channel, Group # NOQA isort:skip diff --git a/channels/adapters.py b/channels/adapters.py index add1339..5e5f020 100644 --- a/channels/adapters.py +++ b/channels/adapters.py @@ -1,29 +1,9 @@ import functools -from django.core.handlers.base import BaseHandler from django.http import HttpRequest, HttpResponse - from channels import Channel -class UrlConsumer(object): - """ - Dispatches channel HTTP requests into django's URL system. - """ - - def __init__(self): - self.handler = BaseHandler() - self.handler.load_middleware() - - def __call__(self, message): - request = HttpRequest.channel_decode(message.content) - try: - response = self.handler.get_response(request) - except HttpResponse.ResponseLater: - return - message.reply_channel.send(response.channel_encode()) - - def view_producer(channel_name): """ Returns a new view function that actually writes the request to a channel diff --git a/channels/asgi.py b/channels/asgi.py new file mode 100644 index 0000000..3bb7ac3 --- /dev/null +++ b/channels/asgi.py @@ -0,0 +1,81 @@ +from __future__ import unicode_literals + +import django +from django.conf import settings +from django.utils.module_loading import import_string + +from .consumer_registry import ConsumerRegistry + + +class InvalidChannelLayerError(ValueError): + pass + + +class ChannelLayerManager(object): + """ + Takes a settings dictionary of backends and initialises them on request. + """ + + def __init__(self): + self.backends = {} + + @property + def configs(self): + # Lazy load settings so we can be imported + return getattr(settings, "CHANNEL_LAYERS", {}) + + def make_backend(self, name): + # Load the backend class + try: + backend_class = import_string(self.configs[name]['BACKEND']) + except KeyError: + raise InvalidChannelLayerError("No BACKEND specified for %s" % name) + except ImportError: + raise InvalidChannelLayerError( + "Cannot import BACKEND %r specified for %s" % (self.configs[name]['BACKEND'], name) + ) + # Get routing + try: + routing = self.configs[name]['ROUTING'] + except KeyError: + raise InvalidChannelLayerError("No ROUTING specified for %s" % name) + # Initialise and pass config + asgi_layer = backend_class(**self.configs[name].get("CONFIG", {})) + return ChannelLayerWrapper( + channel_layer=asgi_layer, + alias=name, + routing=routing, + ) + + def __getitem__(self, key): + if key not in self.backends: + self.backends[key] = self.make_backend(key) + return self.backends[key] + + +class ChannelLayerWrapper(object): + """ + Top level channel layer wrapper, which contains both the ASGI channel + layer object as well as alias and routing information specific to Django. + """ + + def __init__(self, channel_layer, alias, routing): + self.channel_layer = channel_layer + self.alias = alias + self.routing = routing + self.registry = ConsumerRegistry(self.routing) + + def __getattr__(self, name): + return getattr(self.channel_layer, name) + + +def get_channel_layer(alias="default"): + """ + Returns the raw ASGI channel layer for this project. + """ + django.setup(set_prefix=False) + return channel_layers[alias].channel_layer + + +# Default global instance of the channel layer manager +channel_layers = ChannelLayerManager() diff --git a/channels/channel.py b/channels/channel.py index 33112d8..3d28a92 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -1,7 +1,7 @@ -import random -import string +from __future__ import unicode_literals -from channels import DEFAULT_CHANNEL_BACKEND, channel_backends +from django.utils import six +from channels import DEFAULT_CHANNEL_LAYER, channel_layers class Channel(object): @@ -16,15 +16,17 @@ class Channel(object): "default" one by default. """ - def __init__(self, name, alias=DEFAULT_CHANNEL_BACKEND, channel_backend=None): + def __init__(self, name, alias=DEFAULT_CHANNEL_LAYER, channel_layer=None): """ Create an instance for the channel named "name" """ + if isinstance(name, six.binary_type): + name = name.decode("ascii") self.name = name - if channel_backend: - self.channel_backend = channel_backend + if channel_layer: + self.channel_layer = channel_layer else: - self.channel_backend = channel_backends[alias] + self.channel_layer = channel_layers[alias] def send(self, content): """ @@ -32,17 +34,7 @@ class Channel(object): """ if not isinstance(content, dict): raise ValueError("You can only send dicts as content on channels.") - self.channel_backend.send(self.name, content) - - @classmethod - def new_name(self, prefix): - """ - Returns a new channel name that's unique and not closed - with the given prefix. Does not need to be called before sending - on a channel name - just provides a way to avoid clashing for - response channels. - """ - return "%s.%s" % (prefix, "".join(random.choice(string.ascii_letters) for i in range(32))) + self.channel_layer.send(self.name, content) def as_view(self): """ @@ -63,27 +55,29 @@ class Group(object): of the group after an expiry time (keep re-adding to keep them in). """ - def __init__(self, name, alias=DEFAULT_CHANNEL_BACKEND, channel_backend=None): + def __init__(self, name, alias=DEFAULT_CHANNEL_LAYER, channel_layer=None): + if isinstance(name, six.binary_type): + name = name.decode("ascii") self.name = name - if channel_backend: - self.channel_backend = channel_backend + if channel_layer: + self.channel_layer = channel_layer else: - self.channel_backend = channel_backends[alias] + self.channel_layer = channel_layers[alias] def add(self, channel): if isinstance(channel, Channel): channel = channel.name - self.channel_backend.group_add(self.name, channel) + self.channel_layer.group_add(self.name, channel) def discard(self, channel): if isinstance(channel, Channel): channel = channel.name - self.channel_backend.group_discard(self.name, channel) + self.channel_layer.group_discard(self.name, channel) def channels(self): - return self.channel_backend.group_channels(self.name) + return self.channel_layer.group_channels(self.name) def send(self, content): if not isinstance(content, dict): raise ValueError("You can only send dicts as content on channels.") - self.channel_backend.send_group(self.name, content) + self.channel_layer.send_group(self.name, content) diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index 5102e7a..0b7f8d2 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import importlib from django.core.exceptions import ImproperlyConfigured @@ -35,6 +37,11 @@ class ConsumerRegistry(object): # Upconvert if you just pass in a string for channels if isinstance(channels, six.string_types): channels = [channels] + # Make sure all channels are byte strings + channels = [ + channel.decode("ascii") if isinstance(channel, six.binary_type) else channel + for channel in channels + ] # Import any consumer referenced as string if isinstance(consumer, six.string_types): module_name, variable_name = consumer.rsplit(".", 1) diff --git a/channels/hacks.py b/channels/hacks.py index b8050cb..d6303d5 100644 --- a/channels/hacks.py +++ b/channels/hacks.py @@ -1,33 +1,11 @@ -from django.core.handlers.base import BaseHandler -from django.http.request import HttpRequest -from django.http.response import HttpResponseBase - -from .request import decode_request, encode_request -from .response import ResponseLater, decode_response, encode_response def monkeypatch_django(): """ Monkeypatches support for us into parts of Django. """ - # Request encode/decode - HttpRequest.channel_encode = encode_request - HttpRequest.channel_decode = staticmethod(decode_request) - # Response encode/decode - HttpResponseBase.channel_encode = encode_response - HttpResponseBase.channel_decode = staticmethod(decode_response) - HttpResponseBase.ResponseLater = ResponseLater - # Allow ResponseLater to propagate above handler - BaseHandler.old_handle_uncaught_exception = BaseHandler.handle_uncaught_exception - BaseHandler.handle_uncaught_exception = new_handle_uncaught_exception # Ensure that the staticfiles version of runserver bows down to us # This one is particularly horrible from django.contrib.staticfiles.management.commands.runserver import Command as StaticRunserverCommand from .management.commands.runserver import Command as RunserverCommand StaticRunserverCommand.__bases__ = (RunserverCommand, ) - - -def new_handle_uncaught_exception(self, request, resolver, exc_info): - if exc_info[0] is ResponseLater: - raise - return BaseHandler.old_handle_uncaught_exception(self, request, resolver, exc_info) diff --git a/channels/handler.py b/channels/handler.py new file mode 100644 index 0000000..5a88e9f --- /dev/null +++ b/channels/handler.py @@ -0,0 +1,191 @@ +from __future__ import unicode_literals + +import sys +import logging +from threading import Lock + +from django import http +from django.core.handlers import base +from django.core import signals +from django.core.urlresolvers import set_script_prefix +from django.utils.functional import cached_property + +logger = logging.getLogger('django.request') + + +class AsgiRequest(http.HttpRequest): + """ + Custom request subclass that decodes from an ASGI-standard request + dict, and wraps request body handling. + """ + + def __init__(self, message): + self.message = message + self.reply_channel = self.message['reply_channel'] + self._content_length = 0 + # Path info + self.path = self.message['path'] + self.script_name = self.message.get('root_path', '') + if self.script_name: + # TODO: Better is-prefix checking, slash handling? + self.path_info = self.path[len(self.script_name):] + else: + self.path_info = self.path + # HTTP basics + self.method = self.message['method'].upper() + self.META = { + "REQUEST_METHOD": self.method, + "QUERY_STRING": self.message.get('query_string', ''), + } + if self.message.get('client', None): + self.META['REMOTE_ADDR'] = self.message['client'][0] + self.META['REMOTE_HOST'] = self.META['REMOTE_ADDR'] + self.META['REMOTE_PORT'] = self.message['client'][1] + if self.message.get('server', None): + self.META['SERVER_NAME'] = self.message['server'][0] + self.META['SERVER_PORT'] = self.message['server'][1] + # Headers go into META + for name, value in self.message.get('headers', {}).items(): + if name == "content_length": + corrected_name = "CONTENT_LENGTH" + elif name == "content_type": + corrected_name = "CONTENT_TYPE" + else: + corrected_name = 'HTTP_%s' % name.upper().replace("-", "_") + self.META[corrected_name] = value + # Pull out content length info + if self.META.get('CONTENT_LENGTH', None): + try: + self._content_length = int(self.META['CONTENT_LENGTH']) + except (ValueError, TypeError): + pass + # TODO: body handling + self._body = "" + # Other bits + self.resolver_match = None + + @cached_property + def GET(self): + return http.QueryDict( + self.message.get('query_string', ''), + encoding=self._encoding, + ) + + def _get_post(self): + if not hasattr(self, '_post'): + self._load_post_and_files() + return self._post + + def _set_post(self, post): + self._post = post + + POST = property(_get_post, _set_post) + + @cached_property + def COOKIES(self): + return http.parse_cookie(self.META.get('HTTP_COOKIE', '')) + + +class AsgiHandler(base.BaseHandler): + """ + Handler for ASGI requests for the view system only (it will have got here + after traversing the dispatch-by-channel-name system, which decides it's + a HTTP request) + """ + + initLock = Lock() + request_class = AsgiRequest + + def __call__(self, message): + # Set up middleware if needed. We couldn't do this earlier, because + # settings weren't available. + if self._request_middleware is None: + with self.initLock: + # Check that middleware is still uninitialized. + if self._request_middleware is None: + self.load_middleware() + # Set script prefix from message root_path + set_script_prefix(message.get('root_path', '')) + signals.request_started.send(sender=self.__class__, message=message) + # Run request through view system + try: + request = self.request_class(message) + except UnicodeDecodeError: + logger.warning( + 'Bad Request (UnicodeDecodeError)', + exc_info=sys.exc_info(), + extra={ + 'status_code': 400, + } + ) + response = http.HttpResponseBadRequest() + else: + response = self.get_response(request) + # Transform response into messages, which we yield back to caller + for message in self.encode_response(response): + # TODO: file_to_stream + yield message + + def encode_response(self, response): + """ + Encodes a Django HTTP response into an ASGI http.response message(s). + """ + # Collect cookies into headers + response_headers = [(str(k), str(v)) for k, v in response.items()] + for c in response.cookies.values(): + response_headers.append((str('Set-Cookie'), str(c.output(header='')))) + # Make initial response message + message = { + "status": response.status_code, + "status_text": response.reason_phrase, + "headers": response_headers, + } + # Streaming responses need to be pinned to their iterator + if response.streaming: + for part in response.streaming_content: + for chunk in self.chunk_bytes(part): + message['content'] = chunk + message['more_content'] = True + yield message + message = {} + # Final closing message + yield { + "more_content": False, + } + # Other responses just need chunking + else: + # Yield chunks of response + for chunk, last in self.chunk_bytes(response.content): + message['content'] = chunk + message['more_content'] = not last + yield message + message = {} + + def chunk_bytes(self, data): + """ + Chunks some data into chunks based on the current ASGI channel layer's + message size and reasonable defaults. + + Yields (chunk, last_chunk) tuples. + """ + CHUNK_SIZE = 512 * 1024 + position = 0 + while position < len(data): + yield ( + data[position:position+CHUNK_SIZE], + (position + CHUNK_SIZE) >= len(data), + ) + position += CHUNK_SIZE + + +class ViewConsumer(object): + """ + Dispatches channel HTTP requests into django's URL/View system. + """ + + def __init__(self): + self.handler = AsgiHandler() + + def __call__(self, message): + for reply_message in self.handler(message.content): + message.reply_channel.send(reply_message) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 6fa54d6..89c016a 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -3,8 +3,8 @@ import threading from django.core.management.commands.runserver import \ Command as RunserverCommand -from channels import DEFAULT_CHANNEL_BACKEND, channel_backends -from channels.adapters import UrlConsumer +from channels import DEFAULT_CHANNEL_LAYER, channel_layers +from channels.handler import ViewConsumer from channels.interfaces.wsgi import WSGIInterface from channels.log import setup_logger from channels.worker import Worker @@ -21,7 +21,7 @@ class Command(RunserverCommand): """ Returns the default WSGI handler for the runner. """ - return WSGIInterface(self.channel_backend) + return WSGIInterface(self.channel_layer) def run(self, *args, **options): # Run the rest @@ -29,16 +29,16 @@ class Command(RunserverCommand): def inner_run(self, *args, **options): # Check a handler is registered for http reqs - self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - if not self.channel_backend.registry.consumer_for_channel("http.request"): + self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + if not self.channel_layer.registry.consumer_for_channel("http.request"): # Register the default one - self.channel_backend.registry.add_consumer(UrlConsumer(), ["http.request"]) + self.channel_layer.registry.add_consumer(ViewConsumer(), ["http.request"]) # Note that this is the right one on the console self.logger.info("Worker thread running, channels enabled") - if self.channel_backend.local_only: + if self.channel_layer.local_only: self.logger.info("Local channel backend detected, no remote channels support") # Launch a worker thread - worker = WorkerThread(self.channel_backend) + worker = WorkerThread(self.channel_layer) worker.daemon = True worker.start() # Run rest of inner run @@ -50,9 +50,9 @@ class WorkerThread(threading.Thread): Class that runs a worker """ - def __init__(self, channel_backend): + def __init__(self, channel_layer): super(WorkerThread, self).__init__() - self.channel_backend = channel_backend + self.channel_layer = channel_layer def run(self): - Worker(channel_backend=self.channel_backend).run() + Worker(channel_layer=self.channel_layer).run() diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 32692e9..294741b 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -1,8 +1,9 @@ +from __future__ import unicode_literals from django.core.management import BaseCommand, CommandError -from channels import channel_backends, DEFAULT_CHANNEL_BACKEND +from channels import channel_layers, DEFAULT_CHANNEL_LAYER from channels.log import setup_logger -from channels.adapters import UrlConsumer +from channels.handler import ViewConsumer from channels.worker import Worker @@ -12,25 +13,20 @@ class Command(BaseCommand): # Get the backend to use self.verbosity = options.get("verbosity", 1) self.logger = setup_logger('django.channels', self.verbosity) - channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - if channel_backend.local_only: - raise CommandError( - "You have a process-local channel backend configured, and so cannot run separate workers.\n" - "Configure a network-based backend in CHANNEL_BACKENDS to use this command." - ) + channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] # Check a handler is registered for http reqs - if not channel_backend.registry.consumer_for_channel("http.request"): + if not channel_layer.registry.consumer_for_channel("http.request"): # Register the default one - channel_backend.registry.add_consumer(UrlConsumer(), ["http.request"]) + channel_layer.registry.add_consumer(ViewConsumer(), ["http.request"]) # Launch a worker - self.logger.info("Running worker against backend %s", channel_backend) + self.logger.info("Running worker against backend %s", channel_layer.alias) # Optionally provide an output callback callback = None if self.verbosity > 1: callback = self.consumer_called # Run the worker try: - Worker(channel_backend=channel_backend, callback=callback).run() + Worker(channel_layer=channel_layer, callback=callback).run() except KeyboardInterrupt: pass diff --git a/channels/message.py b/channels/message.py index c9a80bd..93b19fd 100644 --- a/channels/message.py +++ b/channels/message.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from .channel import Channel @@ -17,9 +19,9 @@ class Message(object): """ pass - def __init__(self, content, channel, channel_backend, reply_channel=None): + def __init__(self, content, channel, channel_layer, reply_channel=None): self.content = content self.channel = channel - self.channel_backend = channel_backend + self.channel_layer = channel_layer if reply_channel: - self.reply_channel = Channel(reply_channel, channel_backend=self.channel_backend) + self.reply_channel = Channel(reply_channel, channel_layer=self.channel_layer) diff --git a/channels/request.py b/channels/request.py deleted file mode 100644 index 528a8b8..0000000 --- a/channels/request.py +++ /dev/null @@ -1,73 +0,0 @@ -from django.http import HttpRequest -from django.http.request import QueryDict -from django.utils.datastructures import MultiValueDict - - -def encode_request(request): - """ - Encodes a request to JSON-compatible datastructures - """ - # TODO: More stuff - value = { - "get": dict(request.GET.lists()), - "post": dict(request.POST.lists()), - "cookies": request.COOKIES, - "headers": { - k[5:].lower(): v - for k, v in request.META.items() - if k.lower().startswith("http_") - }, - "path": request.path, - "root_path": request.META.get("SCRIPT_NAME", ""), - "method": request.method, - "reply_channel": request.reply_channel, - "server": [ - request.META.get("SERVER_NAME", None), - request.META.get("SERVER_PORT", None), - ], - "client": [ - request.META.get("REMOTE_ADDR", None), - request.META.get("REMOTE_PORT", None), - ], - } - return value - - -def decode_request(value): - """ - Decodes a request JSONish value to a HttpRequest object. - """ - request = HttpRequest() - request.GET = CustomQueryDict(value['get']) - request.POST = CustomQueryDict(value['post']) - request.COOKIES = value['cookies'] - request.path = value['path'] - request.method = value['method'] - request.reply_channel = value['reply_channel'] - # Channels requests are more high-level than the dumping ground that is - # META; re-combine back into it - request.META = { - "REQUEST_METHOD": value["method"], - "SERVER_NAME": value["server"][0], - "SERVER_PORT": value["server"][1], - "REMOTE_ADDR": value["client"][0], - "REMOTE_HOST": value["client"][0], # Not the DNS name, hopefully fine. - "SCRIPT_NAME": value["root_path"], - } - for header, header_value in value.get("headers", {}).items(): - request.META["HTTP_%s" % header.upper()] = header_value - # Derive path_info from script root - request.path_info = request.path - if request.META.get("SCRIPT_NAME", ""): - request.path_info = request.path_info[len(request.META["SCRIPT_NAME"]):] - return request - - -class CustomQueryDict(QueryDict): - """ - Custom override of QueryDict that sets things directly. - """ - - def __init__(self, values, mutable=False, encoding=None): - """ mutable and encoding are ignored :( """ - MultiValueDict.__init__(self, values) diff --git a/channels/response.py b/channels/response.py deleted file mode 100644 index f484da7..0000000 --- a/channels/response.py +++ /dev/null @@ -1,43 +0,0 @@ -from django.http import HttpResponse -from django.utils.six import PY3 - - -def encode_response(response): - """ - Encodes a response to JSON-compatible datastructures - """ - value = { - "content_type": getattr(response, "content_type", None), - "content": response.content, - "status": response.status_code, - "headers": list(response._headers.values()), - "cookies": [v.output(header="") for _, v in response.cookies.items()] - } - if PY3: - value["content"] = value["content"].decode('utf8') - response.close() - return value - - -def decode_response(value): - """ - Decodes a response JSONish value to a HttpResponse object. - """ - response = HttpResponse( - content=value['content'], - content_type=value['content_type'], - status=value['status'], - ) - for cookie in value['cookies']: - response.cookies.load(cookie) - response._headers = {k.lower(): (k, v) for k, v in value['headers']} - return response - - -class ResponseLater(Exception): - """ - Class that represents a response which will be sent down the response - channel later. Used to move a django view-based segment onto the next - task, as otherwise we'd need to write some kind of fake response. - """ - pass diff --git a/channels/worker.py b/channels/worker.py index e4012c6..73fc5c6 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -1,4 +1,7 @@ +from __future__ import unicode_literals + import logging +import time from .message import Message from .utils import name_that_thing @@ -12,30 +15,35 @@ class Worker(object): and runs their consumers. """ - def __init__(self, channel_backend, callback=None): - self.channel_backend = channel_backend + def __init__(self, channel_layer, callback=None): + self.channel_layer = channel_layer self.callback = callback def run(self): """ Tries to continually dispatch messages to consumers. """ - channels = self.channel_backend.registry.all_channel_names() + channels = self.channel_layer.registry.all_channel_names() while True: - channel, content = self.channel_backend.receive_many_blocking(channels) + channel, content = self.channel_layer.receive_many(channels, block=True) + # If no message, stall a little to avoid busy-looping then continue + if channel is None: + time.sleep(0.01) + continue + # Create message wrapper message = Message( content=content, channel=channel, - channel_backend=self.channel_backend, + channel_layer=self.channel_layer, reply_channel=content.get("reply_channel", None), ) # Handle the message - consumer = self.channel_backend.registry.consumer_for_channel(channel) + consumer = self.channel_layer.registry.consumer_for_channel(channel) if self.callback: self.callback(channel, message) try: consumer(message) except Message.Requeue: - self.channel_backend.send(channel, content) + self.channel_layer.send(channel, content) except: logger.exception("Error processing message with consumer %s:", name_that_thing(consumer)) diff --git a/docs/asgi.rst b/docs/asgi.rst index c740edb..b709582 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -101,12 +101,18 @@ Channels and Messages --------------------- All communication in an ASGI stack uses *messages* sent over *channels*. -All messages must be a ``dict`` at the top level of the object, and be -serializable by the built-in ``json`` serializer module (though the -actual serialization a channel layer uses is up to the implementation; -we use ``json`` as the lowest common denominator). +All messages must be a ``dict`` at the top level of the object, and +contain only the following types to ensure serializability: -Channels are identified by a byte string name consisting only of ASCII +* Byte strings +* Unicode strings +* Integers (no longs) +* Lists (tuples should be treated as lists) +* Dicts (keys must be unicode strings) +* Booleans +* None + +Channels are identified by a unicode string name consisting only of ASCII letters, numbers, numerical digits, periods (``.``), dashes (``-``) and underscores (``_``), plus an optional prefix character (see below). @@ -270,20 +276,20 @@ A *channel layer* should provide an object with these attributes (all function arguments are positional): * ``send(channel, message)``, a callable that takes two arguments; the - channel to send on, as a byte string, and the message + channel to send on, as a unicode string, and the message to send, as a serializable ``dict``. * ``receive_many(channels, block=False)``, a callable that takes a list of channel - names as byte strings, and returns with either ``(None, None)`` + names as unicode strings, and returns with either ``(None, None)`` or ``(channel, message)`` if a message is available. If ``block`` is True, then it will not return until after a built-in timeout or a message arrives; if ``block`` is false, it will always return immediately. It is perfectly valid to ignore ``block`` and always return immediately. -* ``new_channel(pattern)``, a callable that takes a byte string pattern, +* ``new_channel(pattern)``, a callable that takes a unicode string pattern, and returns a new valid channel name that does not already exist, by substituting any occurrences of the question mark character ``?`` in - ``pattern`` with a single random byte string and checking for + ``pattern`` with a single random unicode string and checking for existence of that name in the channel layer. This is NOT called prior to a message being sent on a channel, and should not be used for channel initialization. @@ -298,14 +304,14 @@ A *channel layer* should provide an object with these attributes A channel layer implementing the ``groups`` extension must also provide: * ``group_add(group, channel)``, a callable that takes a ``channel`` and adds - it to the group given by ``group``. Both are byte strings. If the channel + it to the group given by ``group``. Both are unicode strings. If the channel is already in the group, the function should return normally. * ``group_discard(group, channel)``, a callable that removes the ``channel`` from the ``group`` if it is in it, and does nothing otherwise. * ``send_group(group, message)``, a callable that takes two positional - arguments; the group to send to, as a byte string, and the message + arguments; the group to send to, as a unicode string, and the message to send, as a serializable ``dict``. A channel layer implementing the ``statistics`` extension must also provide: @@ -423,11 +429,11 @@ Keys: * ``reply_channel``: Channel name for responses and server pushes, in format ``http.response.?`` -* ``http_version``: Byte string, one of ``1.0``, ``1.1`` or ``2``. +* ``http_version``: Unicode string, one of ``1.0``, ``1.1`` or ``2``. -* ``method``: Byte string HTTP method name, uppercased. +* ``method``: Unicode string HTTP method name, uppercased. -* ``scheme``: Byte string URL scheme portion (likely ``http`` or ``https``). +* ``scheme``: Unicode string URL scheme portion (likely ``http`` or ``https``). Optional (but must not be empty), default is ``http``. * ``path``: Byte string HTTP path from URL. @@ -442,20 +448,46 @@ Keys: * ``headers``: Dict of ``{name: value}``, where ``name`` is the lowercased HTTP header name as byte string and ``value`` is the header value as a byte string. If multiple headers with the same name are received, they should - be concatenated into a single header as per . + be concatenated into a single header as per RFC 2616. * ``body``: Body of the request, as a byte string. Optional, defaults to empty - string. + string. If ``body_channel`` is set, treat as start of body and concatenate + on further chunks. -* ``client``: List of ``[host, port]`` where ``host`` is a byte string of the +* ``body_channel``: Single-reader unicode string channel name that contains + Request Body Chunk messages representing a large request body. + Optional, defaults to None. Chunks append to ``body`` if set. Presence of + a channel indicates at least one Request Body Chunk message needs to be read, + and then further consumption keyed off of the ``more_content`` key in those + messages. + +* ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an integer. Optional, defaults to ``None``. * ``server``: List of ``[host, port]`` where ``host`` is the listening address - for this server as a byte string, and ``port`` is the integer listening port. + for this server as a unicode string, and ``port`` is the integer listening port. Optional, defaults to ``None``. +Request Body Chunk +'''''''''''''''''' + +Must be sent after an initial Response. + +Channel: ``http.request.body.?`` + +Keys: + +* ``content``: Byte string of HTTP body content, will be concatenated onto + previously received ``content`` values and ``body`` key in Request. + +* ``more_content``: Boolean value signifying if there is additional content + to come (as part of a Request Body Chunk message). If ``False``, request will + be taken as complete, and any further messages on the channel + will be ignored. Optional, defaults to ``False``. + + Response '''''''' @@ -475,7 +507,8 @@ Keys: string header name, and ``value`` is the byte string header value. Order should be preserved in the HTTP response. -* ``content``: Byte string of HTTP body content +* ``content``: Byte string of HTTP body content. + Optional, defaults to empty string. * ``more_content``: Boolean value signifying if there is additional content to come (as part of a Response Chunk message). If ``False``, response will @@ -534,7 +567,7 @@ Keys: * ``reply_channel``: Channel name for sending data, in format ``websocket.send.?`` -* ``scheme``: Byte string URL scheme portion (likely ``ws`` or ``wss``). +* ``scheme``: Unicode string URL scheme portion (likely ``ws`` or ``wss``). Optional (but must not be empty), default is ``ws``. * ``path``: Byte string HTTP path from URL. @@ -551,12 +584,12 @@ Keys: string. If multiple headers with the same name are received, they should be concatenated into a single header as per . -* ``client``: List of ``[host, port]`` where ``host`` is a byte string of the +* ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an integer. Optional, defaults to ``None``. * ``server``: List of ``[host, port]`` where ``host`` is the listening address - for this server as a byte string, and ``port`` is the integer listening port. + for this server as a unicode string, and ``port`` is the integer listening port. Optional, defaults to ``None``. @@ -644,12 +677,12 @@ Keys: * ``data``: Byte string of UDP datagram payload. -* ``client``: List of ``[host, port]`` where ``host`` is a byte string of the +* ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an integer. * ``server``: List of ``[host, port]`` where ``host`` is the listening address - for this server as a byte string, and ``port`` is the integer listening port. + for this server as a unicode string, and ``port`` is the integer listening port. Optional, defaults to ``None``. @@ -700,8 +733,13 @@ underlying implementation, then any values should be kept within the lower This document will never specify just *string* - all strings are one of the two types. -Channel and group names are always byte strings, with the additional limitation -that they only use the following characters: +Some serializers, such as ``json``, cannot differentiate between byte +strings and unicode strings; these should include logic to box one type as +the other (for example, encoding byte strings as base64 unicode strings with +a preceding special character, e.g. U+FFFF). + +Channel and group names are always unicode strings, with the additional +limitation that they only use the following characters: * ASCII letters * The digits ``0`` through ``9`` @@ -747,7 +785,7 @@ WSGI's ``environ`` variable to the Request message: * ``CONTENT_TYPE`` can be extracted from ``headers`` * ``CONTENT_LENGTH`` can be extracted from ``headers`` * ``SERVER_NAME`` and ``SERVER_PORT`` are in ``server`` -* ``REMOTE_HOST`` and ``REMOTE_PORT`` are in ``client`` +* ``REMOTE_HOST``/``REMOTE_ADDR`` and ``REMOTE_PORT`` are in ``client`` * ``SERVER_PROTOCOL`` is encoded in ``http_version`` * ``wsgi.url_scheme`` is ``scheme`` * ``wsgi.input`` is a StringIO around ``body`` From 836f6be43a76fed4fb0fa696a1ded0e59f363536 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 17:53:53 -0800 Subject: [PATCH 131/746] Make runserver work using daphne --- README.rst | 4 + channels/backends/__init__.py | 36 --- channels/backends/base.py | 109 -------- channels/backends/memory.py | 102 -------- channels/backends/redis_py.py | 192 -------------- channels/interfaces/__init__.py | 0 channels/interfaces/http_twisted.py | 249 ------------------- channels/interfaces/websocket_asyncio.py | 72 ------ channels/interfaces/websocket_autobahn.py | 104 -------- channels/interfaces/websocket_twisted.py | 61 ----- channels/interfaces/wsgi.py | 22 -- channels/management/commands/runallserver.py | 26 -- channels/management/commands/runserver.py | 27 +- channels/management/commands/runwsserver.py | 38 --- docs/concepts.rst | 7 +- setup.py | 2 +- 16 files changed, 20 insertions(+), 1031 deletions(-) delete mode 100644 channels/backends/base.py delete mode 100644 channels/backends/memory.py delete mode 100644 channels/backends/redis_py.py delete mode 100644 channels/interfaces/__init__.py delete mode 100644 channels/interfaces/http_twisted.py delete mode 100644 channels/interfaces/websocket_asyncio.py delete mode 100644 channels/interfaces/websocket_autobahn.py delete mode 100644 channels/interfaces/websocket_twisted.py delete mode 100644 channels/interfaces/wsgi.py delete mode 100644 channels/management/commands/runallserver.py delete mode 100644 channels/management/commands/runwsserver.py diff --git a/README.rst b/README.rst index 9a174fc..c7d40ae 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,10 @@ Django Channels =============== +**NOTE: The current master branch is in flux as it changes to match the final +structure and the new ASGI spec. If you wish to use this in the meantime, +please use a tagged release.** + This is a work-in-progress code branch of Django implemented as a third-party app, which aims to bring some asynchrony to Django and expand the options for code beyond the request-response model, in particular enabling WebSocket, diff --git a/channels/backends/__init__.py b/channels/backends/__init__.py index e52d24f..e69de29 100644 --- a/channels/backends/__init__.py +++ b/channels/backends/__init__.py @@ -1,36 +0,0 @@ -from django.utils.module_loading import import_string - - -class InvalidChannelBackendError(ValueError): - pass - - -class BackendManager(object): - """ - Takes a settings dictionary of backends and initialises them. - """ - - def __init__(self, backend_configs): - self.configs = backend_configs - self.backends = {} - - def make_backend(self, name): - # Load the backend class - try: - backend_class = import_string(self.configs[name]['BACKEND']) - except KeyError: - raise InvalidChannelBackendError("No BACKEND specified for %s" % name) - except ImportError: - raise InvalidChannelBackendError( - "Cannot import BACKEND %r specified for %s" % (self.configs[name]['BACKEND'], name) - ) - - # Initialise and pass config - instance = backend_class(**{k.lower(): v for k, v in self.configs[name].items() if k != "BACKEND"}) - instance.alias = name - return instance - - def __getitem__(self, key): - if key not in self.backends: - self.backends[key] = self.make_backend(key) - return self.backends[key] diff --git a/channels/backends/base.py b/channels/backends/base.py deleted file mode 100644 index fbf9c5f..0000000 --- a/channels/backends/base.py +++ /dev/null @@ -1,109 +0,0 @@ -import time - -from channels.consumer_registry import ConsumerRegistry - - -class ChannelClosed(Exception): - """ - Raised when you try to send to a closed channel. - """ - pass - - -class BaseChannelBackend(object): - """ - Base class for all channel layer implementations. Manages both sending - and receving messages from the backend, and each comes with its own - registry of consumers. - """ - - # Flags if this backend can only be used inside one process. - # Causes errors if you try to run workers/interfaces separately with it. - local_only = False - - def __init__(self, routing, expiry=60): - self.registry = ConsumerRegistry(routing) - self.expiry = expiry - - def send(self, channel, message): - """ - Send a message over the channel, taken from the kwargs. - """ - raise NotImplementedError() - - def receive_many(self, channels): - """ - Return the first message available on one of the - channels passed, as a (channel, message) tuple, or return (None, None) - if no channels are available. - - Should not block, but is allowed to be moderately slow/have a short - timeout - it needs to return so we can refresh the list of channels, - not because the rest of the process is waiting on it. - - Better performance can be achieved for interface servers by directly - integrating the server and the backend code; this is merely for a - generic support-everything pattern. - """ - raise NotImplementedError() - - def receive_many_blocking(self, channels): - """ - Blocking version of receive_many, if the calling context knows it - doesn't ever want to change the channels list until something happens. - - This base class provides a default implementation; can be overridden - to be more efficient by subclasses. - """ - while True: - channel, message = self.receive_many(channels) - if channel is None: - time.sleep(0.05) - continue - return channel, message - - def group_add(self, group, channel, expiry=None): - """ - Adds the channel to the named group for at least 'expiry' - seconds (expiry defaults to message expiry if not provided). - """ - raise NotImplementedError() - - def group_discard(self, group, channel): - """ - Removes the channel from the named group if it is in the group; - does nothing otherwise (does not error) - """ - raise NotImplementedError() - - def group_channels(self, group): - """ - Returns an iterable of all channels in the group. - """ - raise NotImplementedError() - - def send_group(self, group, message): - """ - Sends a message to the entire group. - - This base class provides a default implementation; can be overridden - to be more efficient by subclasses. - """ - for channel in self.group_channels(group): - self.send(channel, message) - - def __str__(self): - return self.__class__.__name__ - - def lock_channel(self, channel): - """ - Attempts to get a lock on the named channel. Returns True if lock - obtained, False if lock not obtained. - """ - raise NotImplementedError() - - def unlock_channel(self, channel): - """ - Unlocks the named channel. Always succeeds. - """ - raise NotImplementedError() diff --git a/channels/backends/memory.py b/channels/backends/memory.py deleted file mode 100644 index 0cb28ba..0000000 --- a/channels/backends/memory.py +++ /dev/null @@ -1,102 +0,0 @@ -import json -import time -from collections import deque - -from .base import BaseChannelBackend - -queues = {} -groups = {} -locks = set() - - -class InMemoryChannelBackend(BaseChannelBackend): - """ - In-memory channel implementation. Intended only for use with threading, - in low-throughput development environments. - """ - - local_only = True - - def send(self, channel, message): - # Try JSON encoding it to make sure it would, but store the native version - json.dumps(message) - # Add to the deque, making it if needs be - queues.setdefault(channel, deque()).append((message, time.time() + self.expiry)) - - def receive_many(self, channels): - if not channels: - raise ValueError("Cannot receive on empty channel list!") - # Try to pop a message from each channel - self._clean_expired() - for channel in channels: - try: - # This doesn't clean up empty channels - OK for testing. - # For later versions, have cleanup w/lock. - return channel, queues[channel].popleft()[0] - except (IndexError, KeyError): - pass - return None, None - - def _clean_expired(self): - # Handle expired messages - for channel, messages in queues.items(): - while len(messages) and messages[0][1] < time.time(): - messages.popleft() - # Handle expired groups - for group, channels in list(groups.items()): - for channel, expiry in list(channels.items()): - if expiry < (time.time() - 10): - try: - del groups[group][channel] - except KeyError: - # Another thread might have got there first - pass - - def group_add(self, group, channel, expiry=None): - """ - Adds the channel to the named group for at least 'expiry' - seconds (expiry defaults to message expiry if not provided). - """ - groups.setdefault(group, {})[channel] = time.time() + (expiry or self.expiry) - - def group_discard(self, group, channel): - """ - Removes the channel from the named group if it is in the group; - does nothing otherwise (does not error) - """ - try: - del groups[group][channel] - except KeyError: - pass - - def group_channels(self, group): - """ - Returns an iterable of all channels in the group. - """ - self._clean_expired() - return groups.get(group, {}).keys() - - def lock_channel(self, channel): - """ - Attempts to get a lock on the named channel. Returns True if lock - obtained, False if lock not obtained. - """ - # Probably not perfect for race conditions, but close enough considering - # it shouldn't be used. - if channel not in locks: - locks.add(channel) - return True - else: - return False - - def unlock_channel(self, channel): - """ - Unlocks the named channel. Always succeeds. - """ - locks.discard(channel) - - def flush(self): - global queues, groups, locks - queues = {} - groups = {} - locks = set() diff --git a/channels/backends/redis_py.py b/channels/backends/redis_py.py deleted file mode 100644 index 3ef5c07..0000000 --- a/channels/backends/redis_py.py +++ /dev/null @@ -1,192 +0,0 @@ -import binascii -import json -import math -import random -import time -import uuid - -import redis -from django.utils import six - -from .base import BaseChannelBackend - - -class RedisChannelBackend(BaseChannelBackend): - """ - ORM-backed channel environment. For development use only; it will span - multiple processes fine, but it's going to be pretty bad at throughput. - """ - - def __init__(self, routing, expiry=60, hosts=None, prefix="django-channels:"): - super(RedisChannelBackend, self).__init__(routing=routing, expiry=expiry) - # Make sure they provided some hosts, or provide a default - if not hosts: - hosts = [("localhost", 6379)] - self.hosts = [] - for entry in hosts: - if isinstance(entry, six.string_types): - self.hosts.append(entry) - else: - self.hosts.append("redis://%s:%d/0" % (entry[0],entry[1])) - self.prefix = prefix - # Precalculate some values for ring selection - self.ring_size = len(self.hosts) - self.ring_divisor = int(math.ceil(4096 / float(self.ring_size))) - - def consistent_hash(self, value): - """ - Maps the value to a node value between 0 and 4095 - using MD5, then down to one of the ring nodes. - """ - if isinstance(value, six.text_type): - value = value.encode("utf8") - bigval = binascii.crc32(value) & 0xffffffff - return (bigval // 0x100000) // self.ring_divisor - - def random_index(self): - return random.randint(0, len(self.hosts) - 1) - - def connection(self, index): - """ - Returns the correct connection for the current thread. - - Pass key to use a server based on consistent hashing of the key value; - pass None to use a random server instead. - """ - # If index is explicitly None, pick a random server - if index is None: - index = self.random_index() - # Catch bad indexes - if not (0 <= index < self.ring_size): - raise ValueError("There are only %s hosts - you asked for %s!" % (self.ring_size, index)) - return redis.Redis.from_url(self.hosts[index]) - - def send(self, channel, message): - # if channel is no str (=> bytes) convert it - if not isinstance(channel, str): - channel = channel.decode("utf-8") - # Write out message into expiring key (avoids big items in list) - # TODO: Use extended set, drop support for older redis? - key = self.prefix + uuid.uuid4().hex - - # Pick a connection to the right server - consistent for response - # channels, random for normal channels - if channel.startswith("!"): - index = self.consistent_hash(key) - connection = self.connection(index) - else: - connection = self.connection(None) - - connection.set( - key, - json.dumps(message), - ) - connection.expire( - key, - self.expiry + 10, - ) - # Add key to list - connection.rpush( - self.prefix + channel, - key, - ) - # Set list to expire when message does (any later messages will bump this) - connection.expire( - self.prefix + channel, - self.expiry + 10, - ) - # TODO: Prune expired messages from same list (in case nobody consumes) - - def receive_many(self, channels): - if not channels: - raise ValueError("Cannot receive on empty channel list!") - # Work out what servers to listen on for the given channels - indexes = {} - random_index = self.random_index() - for channel in channels: - if channel.startswith("!"): - indexes.setdefault(self.consistent_hash(channel), []).append(channel) - else: - indexes.setdefault(random_index, []).append(channel) - # Get a message from one of our channels - while True: - # Select a random connection to use - # TODO: Would we be better trying to do this truly async? - index = random.choice(list(indexes.keys())) - connection = self.connection(index) - channels = indexes[index] - # Shuffle channels to avoid the first ones starving others of workers - random.shuffle(channels) - # Pop off any waiting message - result = connection.blpop([self.prefix + channel for channel in channels], timeout=1) - if result: - content = connection.get(result[1]) - if content is None: - continue - return result[0][len(self.prefix):].decode("utf-8"), json.loads(content.decode("utf-8")) - else: - return None, None - - def group_add(self, group, channel, expiry=None): - """ - Adds the channel to the named group for at least 'expiry' - seconds (expiry defaults to message expiry if not provided). - """ - key = "%s:group:%s" % (self.prefix, group) - key = key.encode("utf8") - self.connection(self.consistent_hash(group)).zadd( - key, - **{channel: time.time() + (expiry or self.expiry)} - ) - - def group_discard(self, group, channel): - """ - Removes the channel from the named group if it is in the group; - does nothing otherwise (does not error) - """ - key = "%s:group:%s" % (self.prefix, group) - key = key.encode("utf8") - self.connection(self.consistent_hash(group)).zrem( - key, - channel, - ) - - def group_channels(self, group): - """ - Returns an iterable of all channels in the group. - """ - key = "%s:group:%s" % (self.prefix, group) - key = key.encode("utf8") - connection = self.connection(self.consistent_hash(group)) - # Discard old channels - connection.zremrangebyscore(key, 0, int(time.time()) - 10) - # Return current lot - return [x.decode("utf8") for x in connection.zrange( - key, - 0, - -1, - )] - - # TODO: send_group efficient implementation using Lua - - def lock_channel(self, channel, expiry=None): - """ - Attempts to get a lock on the named channel. Returns True if lock - obtained, False if lock not obtained. - """ - key = "%s:lock:%s" % (self.prefix, channel) - return bool(self.connection(self.consistent_hash(channel)).setnx(key, "1")) - - def unlock_channel(self, channel): - """ - Unlocks the named channel. Always succeeds. - """ - key = "%s:lock:%s" % (self.prefix, channel) - self.connection(self.consistent_hash(channel)).delete(key) - - def __str__(self): - return "%s(hosts=%s)" % (self.__class__.__name__, self.hosts) - - def flush(self): - for i in range(self.ring_size): - self.connection(i).flushdb() diff --git a/channels/interfaces/__init__.py b/channels/interfaces/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/channels/interfaces/http_twisted.py b/channels/interfaces/http_twisted.py deleted file mode 100644 index fcee728..0000000 --- a/channels/interfaces/http_twisted.py +++ /dev/null @@ -1,249 +0,0 @@ -import time - -from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory -from twisted.python.compat import _PY3 -from twisted.web.http import HTTPFactory, HTTPChannel, Request, _respondToBadRequestAndDisconnect, parse_qs, _parseHeader -from twisted.protocols.policies import ProtocolWrapper -from twisted.internet import reactor - -from channels import Channel, channel_backends, DEFAULT_CHANNEL_BACKEND -from .websocket_autobahn import get_protocol, get_factory - - -WebsocketProtocol = get_protocol(WebSocketServerProtocol) - - -class WebRequest(Request): - """ - Request that either hands off information to channels, or offloads - to a WebSocket class. - - Does some extra processing over the normal Twisted Web request to separate - GET and POST out. - """ - - def __init__(self, *args, **kwargs): - Request.__init__(self, *args, **kwargs) - self.reply_channel = Channel.new_name("!http.response") - self.channel.factory.reply_protocols[self.reply_channel] = self - - def process(self): - # Get upgrade header - upgrade_header = None - if self.requestHeaders.hasHeader("Upgrade"): - upgrade_header = self.requestHeaders.getRawHeaders("Upgrade")[0] - # Is it WebSocket? IS IT?! - if upgrade_header == "websocket": - # Make WebSocket protocol to hand off to - protocol = self.channel.factory.ws_factory.buildProtocol(self.transport.getPeer()) - if not protocol: - # If protocol creation fails, we signal "internal server error" - self.setResponseCode(500) - self.finish() - # Port across transport - transport, self.transport = self.transport, None - if isinstance(transport, ProtocolWrapper): - # i.e. TLS is a wrapping protocol - transport.wrappedProtocol = protocol - else: - transport.protocol = protocol - protocol.makeConnection(transport) - # Re-inject request - if _PY3: - data = self.method + b' ' + self.uri + b' HTTP/1.1\x0d\x0a' - for h in self.requestHeaders.getAllRawHeaders(): - data += h[0] + b': ' + b",".join(h[1]) + b'\x0d\x0a' - data += b"\x0d\x0a" - data += self.content.read() - else: - data = "%s %s HTTP/1.1\x0d\x0a" % (self.method, self.uri) - for h in self.requestHeaders.getAllRawHeaders(): - data += "%s: %s\x0d\x0a" % (h[0], ",".join(h[1])) - data += "\x0d\x0a" - protocol.dataReceived(data) - # Remove our HTTP reply channel association - self.channel.factory.reply_protocols[self.reply_channel] = None - self.reply_channel = None - # Boring old HTTP. - else: - # Send request message - Channel("http.request").send({ - "reply_channel": self.reply_channel, - "method": self.method, - "get": self.get, - "post": self.post, - "cookies": self.received_cookies, - "headers": {k: v[0] for k, v in self.requestHeaders.getAllRawHeaders()}, - "client": [self.client.host, self.client.port], - "server": [self.host.host, self.host.port], - "path": self.path, - }) - - def connectionLost(self, reason): - """ - Cleans up reply channel on close. - """ - if self.reply_channel: - del self.channel.factory.reply_protocols[self.reply_channel] - Request.connectionLost(self, reason) - - def serverResponse(self, message): - """ - Writes a received HTTP response back out to the transport. - """ - # Write code - self.setResponseCode(message['status']) - # Write headers - for header, value in message.get("headers", {}): - self.setHeader(header.encode("utf8"), value.encode("utf8")) - # Write cookies - for cookie in message.get("cookies"): - self.cookies.append(cookie.encode("utf8")) - # Write out body - if "content" in message: - Request.write(self, message['content'].encode("utf8")) - self.finish() - - def requestReceived(self, command, path, version): - """ - Called by channel when all data has been received. - Overridden because Twisted merges GET and POST into one thing by default. - """ - self.content.seek(0,0) - self.get = {} - self.post = {} - - self.method, self.uri = command, path - self.clientproto = version - x = self.uri.split(b'?', 1) - - print self.method - - # URI and GET args assignment - if len(x) == 1: - self.path = self.uri - else: - self.path, argstring = x - self.get = parse_qs(argstring, 1) - - # cache the client and server information, we'll need this later to be - # serialized and sent with the request so CGIs will work remotely - self.client = self.channel.transport.getPeer() - self.host = self.channel.transport.getHost() - - # Argument processing - ctype = self.requestHeaders.getRawHeaders(b'content-type') - if ctype is not None: - ctype = ctype[0] - - # Process POST data if present - if self.method == b"POST" and ctype: - mfd = b'multipart/form-data' - key, pdict = _parseHeader(ctype) - if key == b'application/x-www-form-urlencoded': - self.post.update(parse_qs(self.content.read(), 1)) - elif key == mfd: - try: - cgiArgs = cgi.parse_multipart(self.content, pdict) - - if _PY3: - # parse_multipart on Python 3 decodes the header bytes - # as iso-8859-1 and returns a str key -- we want bytes - # so encode it back - self.post.update({x.encode('iso-8859-1'): y - for x, y in cgiArgs.items()}) - else: - self.post.update(cgiArgs) - except: - # It was a bad request. - _respondToBadRequestAndDisconnect(self.channel.transport) - return - self.content.seek(0, 0) - - # Continue with rest of request handling - self.process() - - -class WebProtocol(HTTPChannel): - - requestFactory = WebRequest - - -class WebFactory(HTTPFactory): - - protocol = WebProtocol - - def __init__(self): - HTTPFactory.__init__(self) - # We track all sub-protocols for response channel mapping - self.reply_protocols = {} - # Make a factory for WebSocket protocols - self.ws_factory = WebSocketServerFactory("ws://127.0.0.1:8000") - self.ws_factory.protocol = WebsocketProtocol - self.ws_factory.reply_protocols = self.reply_protocols - - def reply_channels(self): - return self.reply_protocols.keys() - - def dispatch_reply(self, channel, message): - if channel.startswith("!http") and isinstance(self.reply_protocols[channel], WebRequest): - self.reply_protocols[channel].serverResponse(message) - elif channel.startswith("!websocket") and isinstance(self.reply_protocols[channel], WebsocketProtocol): - if message.get("content", None): - self.reply_protocols[channel].serverSend(**message) - if message.get("close", False): - self.reply_protocols[channel].serverClose() - else: - raise ValueError("Cannot dispatch message on channel %r" % channel) - - -class HttpTwistedInterface(object): - """ - Easy API to run a HTTP1 & WebSocket interface server using Twisted. - Integrates the channel backend by running it in a separate thread, using - the always-compatible polling style. - """ - - def __init__(self, channel_backend, port=8000): - self.channel_backend = channel_backend - self.port = port - - def run(self): - self.factory = WebFactory() - reactor.listenTCP(self.port, self.factory) - reactor.callInThread(self.backend_reader) - #reactor.callLater(1, self.keepalive_sender) - reactor.run() - - def backend_reader(self): - """ - Run in a separate thread; reads messages from the backend. - """ - while True: - channels = self.factory.reply_channels() - # Quit if reactor is stopping - if not reactor.running: - return - # Don't do anything if there's no channels to listen on - if channels: - channel, message = self.channel_backend.receive_many(channels) - else: - time.sleep(0.1) - continue - # Wait around if there's nothing received - if channel is None: - time.sleep(0.05) - continue - # Deal with the message - self.factory.dispatch_reply(channel, message) - - def keepalive_sender(self): - """ - Sends keepalive messages for open WebSockets every - (channel_backend expiry / 2) seconds. - """ - expiry_window = int(self.channel_backend.expiry / 2) - for protocol in self.factory.reply_protocols.values(): - if time.time() - protocol.last_keepalive > expiry_window: - protocol.sendKeepalive() - reactor.callLater(1, self.keepalive_sender) diff --git a/channels/interfaces/websocket_asyncio.py b/channels/interfaces/websocket_asyncio.py deleted file mode 100644 index 043c7d2..0000000 --- a/channels/interfaces/websocket_asyncio.py +++ /dev/null @@ -1,72 +0,0 @@ -import time - -import asyncio -from autobahn.asyncio.websocket import ( - WebSocketServerFactory, WebSocketServerProtocol, -) - -from .websocket_autobahn import get_factory, get_protocol - - -class WebsocketAsyncioInterface(object): - """ - Easy API to run a WebSocket interface server using Twisted. - Integrates the channel backend by running it in a separate thread, using - the always-compatible polling style. - """ - - def __init__(self, channel_backend, port=9000): - self.channel_backend = channel_backend - self.port = port - - def run(self): - self.factory = get_factory(WebSocketServerFactory)(debug=False) - self.factory.protocol = get_protocol(WebSocketServerProtocol) - self.loop = asyncio.get_event_loop() - coro = self.loop.create_server(self.factory, '0.0.0.0', self.port) - server = self.loop.run_until_complete(coro) - self.loop.run_in_executor(None, self.backend_reader) - self.loop.call_later(1, self.keepalive_sender) - try: - self.loop.run_forever() - except KeyboardInterrupt: - pass - finally: - server.close() - self.loop.close() - - def backend_reader(self): - """ - Run in a separate thread; reads messages from the backend. - """ - # Wait for main loop to start - time.sleep(0.5) - while True: - channels = self.factory.reply_channels() - # Quit if reactor is stopping - if not self.loop.is_running(): - return - # Don't do anything if there's no channels to listen on - if channels: - channel, message = self.channel_backend.receive_many(channels) - else: - time.sleep(0.1) - continue - # Wait around if there's nothing received - if channel is None: - time.sleep(0.05) - continue - # Deal with the message - self.factory.dispatch_send(channel, message) - - def keepalive_sender(self): - """ - Sends keepalive messages for open WebSockets every - (channel_backend expiry / 2) seconds. - """ - expiry_window = int(self.channel_backend.expiry / 2) - for protocol in self.factory.reply_protocols.values(): - if time.time() - protocol.last_keepalive > expiry_window: - protocol.sendKeepalive() - if self.loop.is_running(): - self.loop.call_later(1, self.keepalive_sender) diff --git a/channels/interfaces/websocket_autobahn.py b/channels/interfaces/websocket_autobahn.py deleted file mode 100644 index cd3dd3e..0000000 --- a/channels/interfaces/websocket_autobahn.py +++ /dev/null @@ -1,104 +0,0 @@ -import time - -from django.http import parse_cookie - -from channels import DEFAULT_CHANNEL_BACKEND, Channel, channel_backends - - -def get_protocol(base): - - class InterfaceProtocol(base): - """ - Protocol which supports WebSockets and forwards incoming messages to - the websocket channels. - """ - - def onConnect(self, request): - self.channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - self.request_info = { - "path": request.path, - "get": request.params, - "cookies": parse_cookie(request.headers.get('cookie', '')) - } - - def onOpen(self): - # Make sending channel - self.reply_channel = Channel.new_name("!websocket.send") - self.request_info["reply_channel"] = self.reply_channel - self.last_keepalive = time.time() - self.factory.reply_protocols[self.reply_channel] = self - # Send news that this channel is open - Channel("websocket.connect").send(self.request_info) - - def onMessage(self, payload, isBinary): - if isBinary: - Channel("websocket.receive").send({ - "reply_channel": self.reply_channel, - "content": payload, - "binary": True, - }) - else: - Channel("websocket.receive").send({ - "reply_channel": self.reply_channel, - "content": payload.decode("utf8"), - "binary": False, - }) - - def serverSend(self, content, binary=False, **kwargs): - """ - Server-side channel message to send a message. - """ - if binary: - self.sendMessage(content, binary) - else: - self.sendMessage(content.encode("utf8"), binary) - - def serverClose(self): - """ - Server-side channel message to close the socket - """ - self.sendClose() - - def onClose(self, wasClean, code, reason): - if hasattr(self, "reply_channel"): - del self.factory.reply_protocols[self.reply_channel] - Channel("websocket.disconnect").send({ - "reply_channel": self.reply_channel, - }) - - def sendKeepalive(self): - """ - Sends a keepalive packet on the keepalive channel. - """ - Channel("websocket.keepalive").send({ - "reply_channel": self.reply_channel, - }) - self.last_keepalive = time.time() - - return InterfaceProtocol - - -def get_factory(base): - - class InterfaceFactory(base): - """ - Factory which keeps track of its open protocols' receive channels - and can dispatch to them. - """ - - # TODO: Clean up dead protocols if needed? - - def __init__(self, *args, **kwargs): - super(InterfaceFactory, self).__init__(*args, **kwargs) - self.reply_protocols = {} - - def reply_channels(self): - return self.reply_protocols.keys() - - def dispatch_send(self, channel, message): - if message.get("content", None): - self.reply_protocols[channel].serverSend(**message) - if message.get("close", False): - self.reply_protocols[channel].serverClose() - - return InterfaceFactory diff --git a/channels/interfaces/websocket_twisted.py b/channels/interfaces/websocket_twisted.py deleted file mode 100644 index 1a924ac..0000000 --- a/channels/interfaces/websocket_twisted.py +++ /dev/null @@ -1,61 +0,0 @@ -import time - -from autobahn.twisted.websocket import ( - WebSocketServerFactory, WebSocketServerProtocol, -) -from twisted.internet import reactor - -from .websocket_autobahn import get_factory, get_protocol - - -class WebsocketTwistedInterface(object): - """ - Easy API to run a WebSocket interface server using Twisted. - Integrates the channel backend by running it in a separate thread, using - the always-compatible polling style. - """ - - def __init__(self, channel_backend, port=9000): - self.channel_backend = channel_backend - self.port = port - - def run(self): - self.factory = get_factory(WebSocketServerFactory)(debug=False) - self.factory.protocol = get_protocol(WebSocketServerProtocol) - reactor.listenTCP(self.port, self.factory) - reactor.callInThread(self.backend_reader) - reactor.callLater(1, self.keepalive_sender) - reactor.run() - - def backend_reader(self): - """ - Run in a separate thread; reads messages from the backend. - """ - while True: - channels = self.factory.reply_channels() - # Quit if reactor is stopping - if not reactor.running: - return - # Don't do anything if there's no channels to listen on - if channels: - channel, message = self.channel_backend.receive_many(channels) - else: - time.sleep(0.1) - continue - # Wait around if there's nothing received - if channel is None: - time.sleep(0.05) - continue - # Deal with the message - self.factory.dispatch_send(channel, message) - - def keepalive_sender(self): - """ - Sends keepalive messages for open WebSockets every - (channel_backend expiry / 2) seconds. - """ - expiry_window = int(self.channel_backend.expiry / 2) - for protocol in self.factory.reply_protocols.values(): - if time.time() - protocol.last_keepalive > expiry_window: - protocol.sendKeepalive() - reactor.callLater(1, self.keepalive_sender) diff --git a/channels/interfaces/wsgi.py b/channels/interfaces/wsgi.py deleted file mode 100644 index 004e6ea..0000000 --- a/channels/interfaces/wsgi.py +++ /dev/null @@ -1,22 +0,0 @@ -import django -from django.core.handlers.wsgi import WSGIHandler -from django.http import HttpResponse - -from channels import Channel - - -class WSGIInterface(WSGIHandler): - """ - WSGI application that pushes requests to channels. - """ - - def __init__(self, channel_backend, *args, **kwargs): - self.channel_backend = channel_backend - django.setup() - super(WSGIInterface, self).__init__(*args, **kwargs) - - def get_response(self, request): - request.reply_channel = Channel.new_name("http.response") - Channel("http.request", channel_backend=self.channel_backend).send(request.channel_encode()) - channel, message = self.channel_backend.receive_many_blocking([request.reply_channel]) - return HttpResponse.channel_decode(message) diff --git a/channels/management/commands/runallserver.py b/channels/management/commands/runallserver.py deleted file mode 100644 index 1762be5..0000000 --- a/channels/management/commands/runallserver.py +++ /dev/null @@ -1,26 +0,0 @@ -import time -from django.core.management import BaseCommand, CommandError -from channels import channel_backends, DEFAULT_CHANNEL_BACKEND - - -class Command(BaseCommand): - - def add_arguments(self, parser): - parser.add_argument('port', nargs='?', - help='Optional port number') - - def handle(self, *args, **options): - # Get the backend to use - channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - if channel_backend.local_only: - raise CommandError( - "You have a process-local channel backend configured, and so cannot run separate interface servers.\n" - "Configure a network-based backend in CHANNEL_BACKENDS to use this command." - ) - # Run the interface - port = int(options.get("port", None) or 8000) - from channels.interfaces.http_twisted import HttpTwistedInterface - self.stdout.write("Running twisted/Autobahn HTTP & WebSocket interface server") - self.stdout.write(" Channel backend: %s" % channel_backend) - self.stdout.write(" Listening on: 0.0.0.0:%i" % port) - HttpTwistedInterface(channel_backend=channel_backend, port=port).run() diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 89c016a..63d3511 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -5,7 +5,6 @@ from django.core.management.commands.runserver import \ from channels import DEFAULT_CHANNEL_LAYER, channel_layers from channels.handler import ViewConsumer -from channels.interfaces.wsgi import WSGIInterface from channels.log import setup_logger from channels.worker import Worker @@ -17,32 +16,28 @@ class Command(RunserverCommand): self.logger = setup_logger('django.channels', self.verbosity) super(Command, self).handle(*args, **options) - def get_handler(self, *args, **options): - """ - Returns the default WSGI handler for the runner. - """ - return WSGIInterface(self.channel_layer) - def run(self, *args, **options): - # Run the rest - return super(Command, self).run(*args, **options) + # Don't autoreload for now + self.inner_run(None, **options) def inner_run(self, *args, **options): - # Check a handler is registered for http reqs + # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] if not self.channel_layer.registry.consumer_for_channel("http.request"): - # Register the default one self.channel_layer.registry.add_consumer(ViewConsumer(), ["http.request"]) - # Note that this is the right one on the console + # Note that this is the channel-enabled one on the console self.logger.info("Worker thread running, channels enabled") - if self.channel_layer.local_only: - self.logger.info("Local channel backend detected, no remote channels support") # Launch a worker thread worker = WorkerThread(self.channel_layer) worker.daemon = True worker.start() - # Run rest of inner run - super(Command, self).inner_run(*args, **options) + # Launch server in main thread + from daphne.server import Server + Server( + channel_layer=self.channel_layer, + host=self.addr, + port=int(self.port), + ).run() class WorkerThread(threading.Thread): diff --git a/channels/management/commands/runwsserver.py b/channels/management/commands/runwsserver.py deleted file mode 100644 index 54f2f0c..0000000 --- a/channels/management/commands/runwsserver.py +++ /dev/null @@ -1,38 +0,0 @@ -from django.core.management import BaseCommand, CommandError - -from channels import DEFAULT_CHANNEL_BACKEND, channel_backends -from channels.log import setup_logger - - -class Command(BaseCommand): - - def add_arguments(self, parser): - parser.add_argument('port', nargs='?', - help='Optional port number') - - def handle(self, *args, **options): - self.verbosity = options.get("verbosity", 1) - self.logger = setup_logger('django.channels', self.verbosity) - # Get the backend to use - channel_backend = channel_backends[DEFAULT_CHANNEL_BACKEND] - if channel_backend.local_only: - raise CommandError( - "You have a process-local channel backend configured, and so cannot run separate interface servers.\n" - "Configure a network-based backend in CHANNEL_BACKENDS to use this command." - ) - # Run the interface - port = int(options.get("port", None) or 9000) - try: - import asyncio # NOQA - except ImportError: - from channels.interfaces.websocket_twisted import WebsocketTwistedInterface - self.logger.info("Running Twisted/Autobahn WebSocket interface server") - self.logger.info(" Channel backend: %s", channel_backend) - self.logger.info(" Listening on: ws://0.0.0.0:%i" % port) - WebsocketTwistedInterface(channel_backend=channel_backend, port=port).run() - else: - from channels.interfaces.websocket_asyncio import WebsocketAsyncioInterface - self.logger.info("Running asyncio/Autobahn WebSocket interface server") - self.logger.info(" Channel backend: %s", channel_backend) - self.logger.info(" Listening on: ws://0.0.0.0:%i", port) - WebsocketAsyncioInterface(channel_backend=channel_backend, port=port).run() diff --git a/docs/concepts.rst b/docs/concepts.rst index 02c047d..7b32115 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -38,9 +38,10 @@ alternative is *at-least-once*, where normally one consumer gets the message but when things crash it's sent to more than one, which is not the trade-off we want. -There are a couple of other limitations - messages must be JSON serializable, -and not be more than 1MB in size - but these are to make the whole thing -practical, and not too important to think about up front. +There are a couple of other limitations - messages must be made of +serializable types, and stay under a certain size limit - but these are +implementation details you won't need to worry about until you get to more +advanced usage. The channels have capacity, so a load of producers can write lots of messages into a channel with no consumers and then a consumer can come along later and diff --git a/setup.py b/setup.py index 7cde525..29ee590 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,6 @@ setup( packages=find_packages(), include_package_data=True, install_requires=[ - 'Django' + 'Django', ] ) From 3d5c399a41903369dc5354a2156c0402a1ae3036 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 18:10:08 -0800 Subject: [PATCH 132/746] Update header spec to exclude underscores. --- docs/asgi.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index b709582..0b19fc7 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -448,13 +448,14 @@ Keys: * ``headers``: Dict of ``{name: value}``, where ``name`` is the lowercased HTTP header name as byte string and ``value`` is the header value as a byte string. If multiple headers with the same name are received, they should - be concatenated into a single header as per RFC 2616. + be concatenated into a single header as per RFC 2616. Header names containing + underscores should be discarded by the server. * ``body``: Body of the request, as a byte string. Optional, defaults to empty string. If ``body_channel`` is set, treat as start of body and concatenate on further chunks. -* ``body_channel``: Single-reader unicode string channel name that contains +* ``body_channel``: Single-reader channel name that contains Request Body Chunk messages representing a large request body. Optional, defaults to None. Chunks append to ``body`` if set. Presence of a channel indicates at least one Request Body Chunk message needs to be read, From d452486524c4b4d6abef71896b334550678c7512 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 18:10:25 -0800 Subject: [PATCH 133/746] Correct body/POST handling --- channels/handler.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 5a88e9f..4913f8d 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -46,9 +46,9 @@ class AsgiRequest(http.HttpRequest): self.META['SERVER_PORT'] = self.message['server'][1] # Headers go into META for name, value in self.message.get('headers', {}).items(): - if name == "content_length": + if name == "content-length": corrected_name = "CONTENT_LENGTH" - elif name == "content_type": + elif name == "content-type": corrected_name = "CONTENT_TYPE" else: corrected_name = 'HTTP_%s' % name.upper().replace("-", "_") @@ -59,8 +59,22 @@ class AsgiRequest(http.HttpRequest): self._content_length = int(self.META['CONTENT_LENGTH']) except (ValueError, TypeError): pass - # TODO: body handling - self._body = "" + # Body handling + self._body = message.get("body", "") + if message.get("body_channel", None): + while True: + # Get the next chunk from the request body channel + chunk = None + while chunk is None: + _, chunk = message.channel_layer.receive_many( + [message['body_channel']], + block=True, + ) + # Add content to body + self._body += chunk.get("content", "") + # Exit loop if this was the last + if not chunk.get("more_content", False): + break # Other bits self.resolver_match = None @@ -73,6 +87,7 @@ class AsgiRequest(http.HttpRequest): def _get_post(self): if not hasattr(self, '_post'): + self._read_started = False self._load_post_and_files() return self._post From 3dec8e09b321e045c291a137c3c757b231b85f5e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 18:30:52 -0800 Subject: [PATCH 134/746] Update backends docs, remove old message standards --- docs/asgi.rst | 12 ---- docs/backends.rst | 106 +++++++++++------------------ docs/index.rst | 1 - docs/message-standards.rst | 133 ------------------------------------- 4 files changed, 39 insertions(+), 213 deletions(-) delete mode 100644 docs/message-standards.rst diff --git a/docs/asgi.rst b/docs/asgi.rst index 0b19fc7..da53c1e 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -6,18 +6,6 @@ Draft ASGI Spec considered draft yet. Even the name might change; this is being written as development progresses.** -:: - - PEP: XXX - Title: ASGI (Asynchronous Server Gateway Interface) - Version: $Revision$ - Last-Modified: $Date$ - Author: Andrew Godwin - Status: Draft - Type: Informational - Content-Type: text/x-rst - Created: ? - Post-History: ? Abstract ======== diff --git a/docs/backends.rst b/docs/backends.rst index e132654..08a40c1 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -1,28 +1,34 @@ -Backends -======== +Channel Layer Types +=================== Multiple choices of backend are available, to fill different tradeoffs of complexity, throughput and scalability. You can also write your own backend if -you wish; the API is very simple and documented below. +you wish; the spec they confirm to is called :doc:`ASGI `. Any +ASGI-compliant channel layer can be used. Redis ----- -The Redis backend is the recommended backend to run Channels with, as it +The Redis layer is the recommended backend to run Channels with, as it supports both high throughput on a single Redis server as well as the ability to run against a set of Redis servers in a sharded mode. -To use the Redis backend you have to install the redis package:: +To use the Redis layer, simply install it from PyPI (it lives in a separate +package, as we didn't want to force a dependency on the redis-py for the main +install): - pip install -U redis + pip install -U asgi_redis By default, it will attempt to connect to a Redis server on ``localhost:6379``, -but you can override this with the ``HOSTS`` setting:: +but you can override this with the ``hosts`` key in its config:: - CHANNEL_BACKENDS = { + CHANNEL_LAYERS = { "default": { - "BACKEND": "channels.backends.redis_py.RedisChannelBackend", - "HOSTS": [("redis-channel-1", 6379), ("redis-channel-2", 6379)], + "BACKEND": "asgi_redis.RedisChannelLayer", + "ROUTING": "???", + "CONFIG": { + "hosts": [("redis-channel-1", 6379), ("redis-channel-2", 6379)], + }, }, } @@ -49,72 +55,38 @@ settings. Any misconfigured interface server or worker will drop some or all messages. -In-memory ---------- - -The in-memory backend is the simplest, and not really a backend as such; -it exists purely to enable Django to run in a "normal" mode where no Channels -functionality is available, just normal HTTP request processing. You should -never need to set it explicitly. - -This backend provides no network transparency or non-blocking guarantees. - Database -------- -======= -Writing Custom Backends ------------------------ +The database layer is intended as a short-term solution for people who can't +use a more production-ready layer (for example, Redis), but still want something +that will work cross-process. It has poor performance, and is only +recommended for development or extremely small deployments. -Backend Requirements -^^^^^^^^^^^^^^^^^^^^ +This layer is included with Channels; just set your ``BACKEND`` to +``channels.backends.database.DatabaseChannelLayer``, and it will use the +default Django database alias to store messages. You can change the alias +by setting ``CONFIG`` to ``{'alias': 'aliasname'}``. -While the channel backends are pluggable, there's a minimum standard they -must meet in terms of the way they perform. -In particular, a channel backend MUST: +In-memory +--------- -* Provide a ``send()`` method which sends a message onto a named channel +The in-memory layer is purely an implementation detail used when running +the entire Django stack in a single process; the most common case of this +is ``runserver``, where a server thread, channel layer, and worker thread all +co-exist inside the same python process. -* Provide a ``receive_many()`` method which returns an available message on the - provided channels, or returns no message either instantly or after a short - delay (it must not block indefinitely) +You should not need to use this process manually, but if you want to, +it's available from ``asgiref.inmemory.ChannelLayer``. -* Provide a ``group_add()`` method which adds a channel to a named group - for at least the provided expiry period. -* Provide a ``group_discard()`` method with removes a channel from a named - group if it was added, and nothing otherwise. +Writing Custom Channel Layers +----------------------------- -* Provide a ``group_members()`` method which returns an iterable of all - channels currently in the group. - -* Preserve the ordering of messages inside a channel - -* Never deliver a message more than once (design for at-most-once delivery) - -* Never block on sending of a message (dropping the message/erroring is preferable to blocking) - -* Be able to store messages of at least 5MB in size - -* Allow for channel and group names of up to 200 printable ASCII characters - -* Expire messages only after the expiry period provided is up (a backend may - keep them longer if it wishes, but should expire them at some reasonable - point to ensure users do not come to rely on permanent messages) - -In addition, it SHOULD: - -* Provide a ``receive_many_blocking()`` method which is like ``receive_many()`` - but blocks until a message is available. - -* Provide a ``send_group()`` method which sends a message to every channel - in a group. - -* Make ``send_group()`` perform better than ``O(n)``, where ``n`` is the - number of members in the group; preferably send the messages to all - members in a single call to your backing datastore or protocol. - -* Try and preserve a rough global ordering, so that one busy channel does not - drown out an old message in another channel if a worker is listening on both. +The interface channel layers present to Django and other software that +communicates over them is codified in a specification called :doc:`ASGI `. +Any channel layer that conforms to the :doc:`ASGI spec ` can be used +by Django; just set ``BACKEND`` to the class to instantiate and ``CONFIG`` to +a dict of keyword arguments to initialize the class with. diff --git a/docs/index.rst b/docs/index.rst index a7a5e6c..c77d205 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,7 +27,6 @@ Contents: getting-started deploying integration-changes - message-standards scaling backends integration-plan diff --git a/docs/message-standards.rst b/docs/message-standards.rst deleted file mode 100644 index c1e6108..0000000 --- a/docs/message-standards.rst +++ /dev/null @@ -1,133 +0,0 @@ -Message Standards -================= - -Some standardised message formats are used for common message types - they -are detailed below. Message formats are meant to be generic and offload as -much protocol-specific processing to the interface server as is reasonable; -thus, they should generally represent things at as high a level as makes sense. - -In addition to the standards outlined below, each message may contain a -``reply_channel``, which details where to send responses. Protocols with -separate connection and data receiving messages (like WebSockets) will only -contain the connection and detailed client information in the first message; -use the ``@channel_session`` decorator to persist this data to consumers of -the received data (or something else based on ``reply_channel``). - -All messages must be able to be encoded as JSON; channel backends don't -necessarily have to use JSON, but we consider it the lowest common denominator -for serialisation format compatibility. - -The size limit on messages is 1MB (while channel backends may support larger -sizes, all message formats should stay under this limit, which might include -multi-part messages where large content must be transferred). - -The length limit on channel names is 200 printable ASCII characters. - - -HTTP Request ------------- - -Represents a full-fledged, single HTTP request coming in from a client. - -Standard channel name is ``http.request``. - -Contains the following keys: - -* get: Dict of {key: [value, ...]} of GET variables (keys and values are strings) -* post: Dict of {key: [value, ...]} of POST variables (keys and values are strings) -* cookies: Dict of cookies as {cookie_name: cookie_value} (names and values are strings) -* headers: Dict of {header name: value}. Multiple headers of the same name are concatenated into one value separated by commas. -* meta: Dict of HTTP headers and info as defined in the Django Request docs (names and values are strings) -* path: String, full path to the requested page, without query string or domain -* root_path: Path designated as the "root" of the application (SCRIPT_NAME) -* method: String, upper-cased HTTP method -* server: [host, port] showing the address the client connected to -* client: [host, port] of the remote client - -Should come with an associated ``reply_channel`` which accepts HTTP Responses. - - -HTTP Response -------------- - -Sends either a part of a response or a whole response to a HTTP client - to do -streaming responses, several response messages are sent with ``more_content: True`` -and the final one has the key omitted. Normal, single-shot responses do not -need the key at all. - -Due to the 1MB size limit on messages, some larger responses will have to be -sent multi-part to stay within the limit. - -Only sent on reply channels. - -Keys that must only be in the first message of a set: - -* content_type: String, mimetype of content -* status: Integer, numerical HTTP status code -* cookies: List of cookies to set (as encoded cookie strings suitable for headers) -* headers: Dictionary of headers (key is header name, value is value, both strings) - -All messages in a set can the following keys: - -* content: String of content to send -* more_content: Boolean, signals the interface server should wait for another response chunk to stream. - - -HTTP Disconnect ---------------- - -Send when a client disconnects early, before the response has been sent. -Only sent by long-polling-capable HTTP interface servers. - -Standard channel name is ``http.disconnect``. - -Contains no keys. - - -WebSocket Connection --------------------- - -Sent when a new WebSocket is connected. - -Standard channel name is ``websocket.connect``. - -Contains the same keys as HTTP Request, without the ``POST`` or ``method`` keys. - - -WebSocket Receive ------------------ - -Sent when a datagram is received on the WebSocket. - -Standard channel name is ``websocket.receive``. - -Contains the following keys: - -* content: String content of the datagram. -* binary: Boolean, saying if the content is binary. If not present or false, content is a UTF8 string. - - -WebSocket Client Close ----------------------- - -Sent when the WebSocket is closed by either the client or the server. - -Standard channel name is ``websocket.disconnect``. - -Contains no keys. - - -WebSocket Send/Close --------------------- - -Sent by a Django consumer to send a message back over the WebSocket to -the client or close the client connection. The content is optional if close -is set, and close will happen after any content is sent, if some is present. - -Only sent on reply channels. - -Contains the keys: - -* content: String content of the datagram. -* binary: If the content is to be interpreted as text or binary. -* close: Boolean. If set to True, will close the client connection. From 5348c527780099ecf919543c18841b8cfb230b4f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 18:35:43 -0800 Subject: [PATCH 135/746] Fixed #33: Add wsgi.multi* variables for back compat. --- channels/handler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/channels/handler.py b/channels/handler.py index 4913f8d..33068a5 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -36,6 +36,9 @@ class AsgiRequest(http.HttpRequest): self.META = { "REQUEST_METHOD": self.method, "QUERY_STRING": self.message.get('query_string', ''), + # Old code will need these for a while + "wsgi.multithread": True, + "wsgi.multiprocess": True, } if self.message.get('client', None): self.META['REMOTE_ADDR'] = self.message['client'][0] From 4ed1d73789c99661dcd84b8097c2bd8331484dba Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 18:50:59 -0800 Subject: [PATCH 136/746] Remove redundant sentence --- docs/asgi.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index da53c1e..e314bcf 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -174,11 +174,11 @@ number). Extensions ---------- -ASGI has the concept of *extensions*, of which one is specified in this -document. Extensions are functionality that is +Extensions are functionality that is not required for basic application code and nearly all protocol server -code, and so has been made optional in order to encourage lighter-weight -channel layers to be written. +code, and so has been made optional in order to enable lightweight +channel layers for applications that don't need the full feature set defined +here. There are two extensions defined here: the ``groups`` extension, which is expanded on below, and the ``statistics`` extension, which allows From 46ea90e0958e6db2d211ba6439f98a7cd3dd8e38 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 22:42:40 -0800 Subject: [PATCH 137/746] Spec tweaks --- docs/asgi.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index e314bcf..2cc1442 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -2,10 +2,8 @@ Draft ASGI Spec =============== -**NOTE: This is still heavily in-progress, and should not even be -considered draft yet. Even the name might change; this is being written -as development progresses.** - +**NOTE: This is still in-progress, and may change substantially as development +progresses.** Abstract ======== @@ -287,7 +285,7 @@ A *channel layer* should provide an object with these attributes * ``extensions``, a list of unicode string names indicating which extensions this layer provides, or empty if it supports none. - The only valid extension name is ``groups``. + The only valid extension names are ``groups`` and ``statistics`. A channel layer implementing the ``groups`` extension must also provide: From 7f5c3d25b6bf863d091488b3d17c59aeeba4426c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 22:43:41 -0800 Subject: [PATCH 138/746] Remove redundant msgpack question (redis backend now uses it) --- docs/faqs.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/faqs.rst b/docs/faqs.rst index cbc0635..e4250e5 100755 --- a/docs/faqs.rst +++ b/docs/faqs.rst @@ -125,18 +125,3 @@ which is the unique channel representing the connection, but remember that whatever you store in must be **network-transparent** - storing things in a global variable won't work outside of development. - -Would you support messagepack or any other format? --------------------------------------------------- - -Although we've evaluated msgpack it does not offer enough over JSON to be -reasonable - the encoding/decoding is often slower, the language support is -much poorer, and in general we would rather just have one version of a standard, -especially since there's plans to write parts of the Channels system in other -languages. - -That said, at some point it's up to the individual channel backend to support -whatever it likes, as long as it spits out dicts at either end. So this is -something that could be implemented by someone else as a pluggable backend to -see. We might always come back and revisit this if message size/bandwidth turns -out to be a limiting factor, though. From 4847403ba0f340daf74e34a369fa9019acb99a99 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 22:45:00 -0800 Subject: [PATCH 139/746] Update dependency things --- README.rst | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/README.rst b/README.rst index c7d40ae..8e7cf07 100644 --- a/README.rst +++ b/README.rst @@ -17,3 +17,4 @@ Documentation, installation and getting started instructions are at http://channels.readthedocs.org You can also install channels from PyPI as the ``channels`` package. +You'll likely also want ``asgi_redis`` to provide the Redis channel layer. diff --git a/setup.py b/setup.py index 29ee590..9e393b5 100644 --- a/setup.py +++ b/setup.py @@ -12,5 +12,6 @@ setup( include_package_data=True, install_requires=[ 'Django', + 'asgiref', ] ) From 93b2229b2bbf170413c3306c0588f7b84bfec456 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 2 Jan 2016 22:59:09 -0800 Subject: [PATCH 140/746] HTTP body spec fixes --- docs/asgi.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 2cc1442..281f0da 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -783,8 +783,10 @@ The ``start_response`` callable maps similarly to Response: * The ``status`` argument becomes ``status`` and ``status_text`` * ``response_headers`` maps to ``headers`` -The main difference is that ASGI is incapable of performing streaming -of HTTP body input, and instead must buffer it all into a message first. +It may even be possible to map Request Body Chunks in a way that allows +streaming of body data, though it would likely be easier and sufficient for +many applications to simply buffer the whole body into memory before calling +the WSGI application. Common Questions @@ -801,14 +803,8 @@ Common Questions TODOs ===== -* Work out if we really can just leave HTTP body as byte string. Seems too big. - Might need some reverse-single-reader chunking? Or just say channel layer - message size dictates body size. - * Maybe remove ``http_version`` and replace with ``supports_server_push``? -* Be sure we want to leave HTTP ``get`` and ``post`` out. - * ``receive_many`` can't easily be implemented with async/cooperative code behind it as it's nonblocking - possible alternative call type? Asyncio extension that provides ``receive_many_yield``? From 4ea7b26c652709c02912298c953a9ab3c4d39355 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sun, 3 Jan 2016 09:48:08 +0000 Subject: [PATCH 141/746] Draft proposal for Server Push messages --- docs/asgi.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 281f0da..a7eb44d 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -484,7 +484,7 @@ Channel: ``http.response.?`` Keys: -* ``status``: Integer HTTP status code. +* ``status``: Integer HTTP status code. * ``status_text``: Byte string HTTP reason-phrase, e.g. ``OK`` from ``200 OK``. Ignored for HTTP/2 clients. Optional, default should be based on ``status`` @@ -524,9 +524,16 @@ Keys: Server Push ''''''''''' -Send before any Response or Response Chunk. HTTP/2 only. +Must be sent before any Response or Response Chunk messages. -TODO +Channel: ``http.response.?`` + +Keys: + +* ``request``: A Request message. Both the ``body`` and ``body_channel`` fields + MUST be absent: bodies are not allowed on server-pushed requests. The + ``reply_channel`` set on this object will be used for all further messages + relating to the pushed response. WebSocket From a8d09c2644f070df28e0791cc49e74170be85a08 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 4 Jan 2016 18:24:24 -0800 Subject: [PATCH 142/746] Embarassingly managed to remove the only name expansion. --- docs/asgi.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 281f0da..fb7531c 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -1,6 +1,6 @@ -=============== -Draft ASGI Spec -=============== +======================================================= +ASGI (Asynchronous Server Gateway Interface) Draft Spec +======================================================= **NOTE: This is still in-progress, and may change substantially as development progresses.** From 894041a3c02b28b9b7f9c95a480995b94d1c67a4 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 5 Jan 2016 11:41:57 +0000 Subject: [PATCH 143/746] New approach to server push. --- docs/asgi.rst | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index a7eb44d..7ac6560 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -526,14 +526,32 @@ Server Push Must be sent before any Response or Response Chunk messages. +When a server receives this message, it must treat the Request message in the +``request`` field of the Server Push as though it were a new HTTP request being +received from the network. A server may, if it chooses, apply all of its +internal logic to handling this request (e.g. the server may want to try to +satisfy the request from a cache). Regardless, if the server is unable to +satisfy the request itself it must create a new ``http.response.?`` channel for +the application to send the Response message on, fill that channel in on the +``reply_channel`` field of the message, and then send the Request back to the +application on the ``http.request`` channel. + +This approach limits the amount of knowledge the application has to have about +pushed responses: they essentially appear to the application like a normal HTTP +request, with the difference being that the application itself triggered the +request. + +If the remote peer does not support server push, either because it's not a +HTTP/2 peer or because SETTINGS_ENABLE_PUSH is set to 0, the server must do +nothing in response to this message. + Channel: ``http.response.?`` Keys: -* ``request``: A Request message. Both the ``body`` and ``body_channel`` fields - MUST be absent: bodies are not allowed on server-pushed requests. The - ``reply_channel`` set on this object will be used for all further messages - relating to the pushed response. +* ``request``: A Request message. The ``body``, ``body_channel``, and + ``reply_channel`` fields MUST be absent: bodies are not allowed on + server-pushed requests, and applications should not create reply channels. WebSocket From 8f9d40b659dba2bfed1e6f0321d4ba5b2f2f1722 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 10 Jan 2016 13:03:19 +0100 Subject: [PATCH 144/746] Fixed minor doc typo. --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 7b32115..b523d49 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -14,7 +14,7 @@ could easily offload until after a response has been sent - like saving things into a cache or thumbnailing newly-uploaded images. Channels changes the way Django runs to be "event oriented" - rather than -just responding to requests, instead Django responses to a wide array of events +just responding to requests, instead Django responds to a wide array of events sent on *channels*. There's still no persistent state - each event handler, or *consumer* as we call them, is called independently in a way much like a view is called. From 4d9c33c72d9ac5c89767a3a9b9ec954dcfb96bb3 Mon Sep 17 00:00:00 2001 From: Dan Lipsitt Date: Wed, 13 Jan 2016 14:43:58 -0800 Subject: [PATCH 145/746] Config for Travis CI. --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..361fe5e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +sudo: false +language: python +python: + - "2.7" + - "3.5" +install: pip install tox-travis +script: tox From 717eb0a1008572cc0e9a2f9ae8c6208ffd5feff7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 17 Jan 2016 14:21:45 -0800 Subject: [PATCH 146/746] Added the "flush" extension to ASGI, and clarified group expiry --- docs/asgi.rst | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index d2dbd3d..1eedf70 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -178,8 +178,9 @@ code, and so has been made optional in order to enable lightweight channel layers for applications that don't need the full feature set defined here. -There are two extensions defined here: the ``groups`` extension, which -is expanded on below, and the ``statistics`` extension, which allows +There are three extensions defined here: the ``groups`` extension, which +is expanded on below, the ``flush`` extension, which allows easier testing +and development, and the ``statistics`` extension, which allows channel layers to provide global and per-channel statistics. There is potential to add further extensions; these may be defined by @@ -285,7 +286,7 @@ A *channel layer* should provide an object with these attributes * ``extensions``, a list of unicode string names indicating which extensions this layer provides, or empty if it supports none. - The only valid extension names are ``groups`` and ``statistics`. + The only valid extension names are ``groups``, ``flush`` and ``statistics`. A channel layer implementing the ``groups`` extension must also provide: @@ -314,7 +315,12 @@ A channel layer implementing the ``statistics`` extension must also provide: * ``age``, how long the oldest message has been waiting, in seconds * ``per_second``, the number of messages processed in the last second +A channel layer implementing the ``flush`` extension must also provide: +* ``flush()``, a callable that resets the channel layer to no messages and + no groups (if groups is implemented). This call must block until the system + is cleared and will consistently look empty to any client, if the channel + layer is distributed. @@ -359,6 +365,10 @@ protocol servers should quit and hard restart if they detect that the channel layer has gone down or lost data; shedding all existing connections and letting clients reconnect will immediately resolve the problem. +If a channel layer implements the ``groups`` extension, it must persist group +membership until at least the time when the member channel has a message +expire due to non-consumption. + Message Formats --------------- From 5fa357e40331e29031346fa6b103f153fd9d9f14 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 17 Jan 2016 14:22:28 -0800 Subject: [PATCH 147/746] Disable linearize until it's reimplemented on sessions. --- channels/decorators.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/channels/decorators.py b/channels/decorators.py index 7be50ee..295ab3d 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -14,23 +14,22 @@ def linearize(func): up before the first has exited and saved its session. Doesn't guarantee ordering, just linearity. """ + raise NotImplementedError("Not yet reimplemented") @functools.wraps(func) def inner(message, *args, **kwargs): # Make sure there's a reply channel if not message.reply_channel: raise ValueError( - "No reply_channel sent to consumer; @no_overlap can only be used on messages containing it." + "No reply_channel in message; @linearize can only be used on messages containing it." ) - - # Get the lock, or re-queue - locked = message.channel_backend.lock_channel(message.reply_channel) - if not locked: - raise message.Requeue() + # TODO: Get lock here + pass # OK, keep going try: return func(message, *args, **kwargs) finally: - message.channel_backend.unlock_channel(message.reply_channel) + # TODO: Release lock here + pass return inner @@ -47,7 +46,8 @@ def channel_session(func): # Make sure there's a reply_channel if not message.reply_channel: raise ValueError( - "No reply_channel sent to consumer; @no_overlap can only be used on messages containing it." + "No reply_channel sent to consumer; @channel_session " + + "can only be used on messages containing it." ) # Make sure there's NOT a channel_session already From a9810014caaf357d75545d1d7ebbaf88f2341307 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 17 Jan 2016 14:28:01 -0800 Subject: [PATCH 148/746] Rework database channel layer and tests, to use ASGI conformance suite --- channels/backends/__init__.py | 0 .../database.py => database_layer.py} | 203 +++++++++--------- channels/tests/test_backends.py | 95 -------- channels/tests/test_database_layer.py | 6 + channels/tests/test_interfaces.py | 53 ----- 5 files changed, 111 insertions(+), 246 deletions(-) delete mode 100644 channels/backends/__init__.py rename channels/{backends/database.py => database_layer.py} (59%) delete mode 100644 channels/tests/test_backends.py create mode 100644 channels/tests/test_database_layer.py delete mode 100644 channels/tests/test_interfaces.py diff --git a/channels/backends/__init__.py b/channels/backends/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/channels/backends/database.py b/channels/database_layer.py similarity index 59% rename from channels/backends/database.py rename to channels/database_layer.py index d0b7f69..ea9e5e1 100644 --- a/channels/backends/database.py +++ b/channels/database_layer.py @@ -1,24 +1,122 @@ import datetime import json +import random +import string +import time from django.apps.registry import Apps from django.db import DEFAULT_DB_ALIAS, IntegrityError, connections, models +from django.utils import six from django.utils.functional import cached_property from django.utils.timezone import now -from .base import BaseChannelBackend - -class DatabaseChannelBackend(BaseChannelBackend): +class DatabaseChannelLayer(object): """ - ORM-backed channel environment. For development use only; it will span - multiple processes fine, but it's going to be pretty bad at throughput. + ORM-backed ASGI channel layer. + + For development use only; it will span multiple processes fine, + but it's going to be pretty bad at throughput. If you're reading this and + running it in production, PLEASE STOP. + + Also uses JSON for serialization, as we don't want to make Django depend + on msgpack for the built-in backend. The JSON format uses \uffff as first + character to signify a byte string rather than a text string. """ - def __init__(self, routing, expiry=60, db_alias=DEFAULT_DB_ALIAS): - super(DatabaseChannelBackend, self).__init__(routing=routing, expiry=expiry) + def __init__(self, db_alias=DEFAULT_DB_ALIAS, expiry=60): + self.expiry = expiry self.db_alias = db_alias + ### ASGI API ### + + extensions = ["groups", "flush"] + + def send(self, channel, message): + # Typecheck + assert isinstance(message, dict), "message is not a dict" + assert isinstance(channel, six.text_type), "%s is not unicode" % channel + # Write message to messages table + self.channel_model.objects.create( + channel=channel, + content=self.serialize(message), + expiry=now() + datetime.timedelta(seconds=self.expiry) + ) + + def receive_many(self, channels, block=False): + if not channels: + return None, None + assert all(isinstance(channel, six.text_type) for channel in channels) + # Shuffle channels + channels = list(channels) + random.shuffle(channels) + # Clean out expired messages + self._clean_expired() + # Get a message from one of our channels + while True: + message = self.channel_model.objects.filter(channel__in=channels).order_by("id").first() + if message: + self.channel_model.objects.filter(pk=message.pk).delete() + return message.channel, self.deserialize(message.content) + else: + if block: + time.sleep(1) + else: + return None, None + + def new_channel(self, pattern): + assert isinstance(pattern, six.text_type) + # Keep making channel names till one isn't present. + while True: + random_string = "".join(random.choice(string.ascii_letters) for i in range(8)) + new_name = pattern.replace(b"?", random_string) + if not self.channel_model.objects.filter(channel=new_name).exists(): + return new_name + + ### ASGI Group extension ### + + def group_add(self, group, channel, expiry=None): + """ + Adds the channel to the named group for at least 'expiry' + seconds (expiry defaults to message expiry if not provided). + """ + self.group_model.objects.update_or_create( + group=group, + channel=channel, + defaults={"expiry": now() + datetime.timedelta(seconds=expiry or self.expiry)}, + ) + + def group_discard(self, group, channel): + """ + Removes the channel from the named group if it is in the group; + does nothing otherwise (does not error) + """ + self.group_model.objects.filter(group=group, channel=channel).delete() + + def send_group(self, group, message): + """ + Sends a message to the entire group. + """ + self._clean_expired() + for channel in self.group_model.objects.filter(group=group).values_list("channel", flat=True): + self.send(channel, message) + + ### ASGI Flush extension ### + + def flush(self): + self.channel_model.objects.all().delete() + self.group_model.objects.all().delete() + + ### Serialization ### + + def serialize(self, message): + return json.dumps(message) + + def deserialize(self, message): + return json.loads(message) + + ### Database state mgmt ### + @property def connection(self): """ @@ -72,46 +170,6 @@ class DatabaseChannelBackend(BaseChannelBackend): editor.create_model(Group) return Group - @cached_property - def lock_model(self): - """ - Initialises a new model to store groups; not done as part of a - models.py as we don't want to make it for most installs. - """ - # Make the model class - class Lock(models.Model): - channel = models.CharField(max_length=200, unique=True) - expiry = models.DateTimeField(db_index=True) - - class Meta: - apps = Apps() - app_label = "channels" - db_table = "django_channel_locks" - # Ensure its table exists - if Lock._meta.db_table not in self.connection.introspection.table_names(self.connection.cursor()): - with self.connection.schema_editor() as editor: - editor.create_model(Lock) - return Lock - - def send(self, channel, message): - self.channel_model.objects.create( - channel=channel, - content=json.dumps(message), - expiry=now() + datetime.timedelta(seconds=self.expiry) - ) - - def receive_many(self, channels): - if not channels: - raise ValueError("Cannot receive on empty channel list!") - self._clean_expired() - # Get a message from one of our channels - message = self.channel_model.objects.filter(channel__in=channels).order_by("id").first() - if message: - self.channel_model.objects.filter(pk=message.pk).delete() - return message.channel, json.loads(message.content) - else: - return None, None - def _clean_expired(self): """ Cleans out expired groups and messages. @@ -119,57 +177,6 @@ class DatabaseChannelBackend(BaseChannelBackend): # Include a 10-second grace period because that solves some clock sync self.channel_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() self.group_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() - self.lock_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() - - def group_add(self, group, channel, expiry=None): - """ - Adds the channel to the named group for at least 'expiry' - seconds (expiry defaults to message expiry if not provided). - """ - self.group_model.objects.update_or_create( - group=group, - channel=channel, - defaults={"expiry": now() + datetime.timedelta(seconds=expiry or self.expiry)}, - ) - - def group_discard(self, group, channel): - """ - Removes the channel from the named group if it is in the group; - does nothing otherwise (does not error) - """ - self.group_model.objects.filter(group=group, channel=channel).delete() - - def group_channels(self, group): - """ - Returns an iterable of all channels in the group. - """ - self._clean_expired() - return list(self.group_model.objects.filter(group=group).values_list("channel", flat=True)) - - def lock_channel(self, channel, expiry=None): - """ - Attempts to get a lock on the named channel. Returns True if lock - obtained, False if lock not obtained. - """ - # We rely on the UNIQUE constraint for only-one-thread-wins on locks - try: - self.lock_model.objects.create( - channel=channel, - expiry=now() + datetime.timedelta(seconds=expiry or self.expiry), - ) - except IntegrityError: - return False - else: - return True - - def unlock_channel(self, channel): - """ - Unlocks the named channel. Always succeeds. - """ - self.lock_model.objects.filter(channel=channel).delete() def __str__(self): return "%s(alias=%s)" % (self.__class__.__name__, self.connection.alias) - - def flush(self): - pass diff --git a/channels/tests/test_backends.py b/channels/tests/test_backends.py deleted file mode 100644 index b25750f..0000000 --- a/channels/tests/test_backends.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.test import TestCase - -from ..backends.database import DatabaseChannelBackend -from ..backends.memory import InMemoryChannelBackend -from ..backends.redis_py import RedisChannelBackend - - -class MemoryBackendTests(TestCase): - - backend_class = InMemoryChannelBackend - - def setUp(self): - self.backend = self.backend_class(routing={}) - self.backend.flush() - - def test_send_recv(self): - """ - Tests that channels can send and receive messages. - """ - self.backend.send("test", {"value": "blue"}) - self.backend.send("test", {"value": "green"}) - self.backend.send("test2", {"value": "red"}) - # Get just one first - channel, message = self.backend.receive_many(["test"]) - self.assertEqual(channel, "test") - self.assertEqual(message, {"value": "blue"}) - # And the second - channel, message = self.backend.receive_many(["test"]) - self.assertEqual(channel, "test") - self.assertEqual(message, {"value": "green"}) - # And the other channel with multi select - channel, message = self.backend.receive_many(["test", "test2"]) - self.assertEqual(channel, "test2") - self.assertEqual(message, {"value": "red"}) - - def test_message_expiry(self): - self.backend = self.backend_class(routing={}, expiry=-100) - self.backend.send("test", {"value": "blue"}) - channel, message = self.backend.receive_many(["test"]) - self.assertIs(channel, None) - self.assertIs(message, None) - - def test_groups(self): - """ - Tests that group addition and removal and listing works - """ - self.backend.group_add("tgroup", "test") - self.backend.group_add("tgroup", "test2€") - self.backend.group_add("tgroup2", "test3") - self.assertEqual( - set(self.backend.group_channels("tgroup")), - {"test", "test2€"}, - ) - self.backend.group_discard("tgroup", "test2€") - self.backend.group_discard("tgroup", "test2€") - self.assertEqual( - list(self.backend.group_channels("tgroup")), - ["test"], - ) - - def test_group_send(self): - """ - Tests sending to groups. - """ - self.backend.group_add("tgroup", "test") - self.backend.group_add("tgroup", "test2") - self.backend.send_group("tgroup", {"value": "orange"}) - channel, message = self.backend.receive_many(["test"]) - self.assertEqual(channel, "test") - self.assertEqual(message, {"value": "orange"}) - channel, message = self.backend.receive_many(["test2"]) - self.assertEqual(channel, "test2") - self.assertEqual(message, {"value": "orange"}) - - def test_group_expiry(self): - self.backend = self.backend_class(routing={}, expiry=-100) - self.backend.group_add("tgroup", "test") - self.backend.group_add("tgroup", "test2") - self.assertEqual( - list(self.backend.group_channels("tgroup")), - [], - ) - - -class RedisBackendTests(MemoryBackendTests): - - backend_class = RedisChannelBackend - - -class DatabaseBackendTests(MemoryBackendTests): - - backend_class = DatabaseChannelBackend diff --git a/channels/tests/test_database_layer.py b/channels/tests/test_database_layer.py new file mode 100644 index 0000000..146af36 --- /dev/null +++ b/channels/tests/test_database_layer.py @@ -0,0 +1,6 @@ +from __future__ import unicode_literals +from asgiref.conformance import make_tests +from ..database_layer import DatabaseChannelLayer + +channel_layer = DatabaseChannelLayer(expiry=1) +DatabaseLayerTests = make_tests(channel_layer, expiry_delay=1.1) diff --git a/channels/tests/test_interfaces.py b/channels/tests/test_interfaces.py deleted file mode 100644 index fd830d7..0000000 --- a/channels/tests/test_interfaces.py +++ /dev/null @@ -1,53 +0,0 @@ -from django.test import TestCase - -from channels.interfaces.websocket_autobahn import get_protocol - -try: - from unittest import mock -except ImportError: - import mock - - -def generate_connection_request(path, params, headers): - request = mock.Mock() - request.path = path - request.params = params - request.headers = headers - return request - - -class WebsocketAutobahnInterfaceProtocolTestCase(TestCase): - def test_on_connect_cookie(self): - protocol = get_protocol(object)() - session = "123cat" - cookie = "somethingelse=test; sessionid={0}".format(session) - headers = { - "cookie": cookie - } - - test_request = generate_connection_request("path", {}, headers) - protocol.onConnect(test_request) - self.assertEqual(session, protocol.request_info["cookies"]["sessionid"]) - - def test_on_connect_no_cookie(self): - protocol = get_protocol(object)() - test_request = generate_connection_request("path", {}, {}) - protocol.onConnect(test_request) - self.assertEqual({}, protocol.request_info["cookies"]) - - def test_on_connect_params(self): - protocol = get_protocol(object)() - params = { - "session_key": ["123cat"] - } - - test_request = generate_connection_request("path", params, {}) - protocol.onConnect(test_request) - self.assertEqual(params, protocol.request_info["get"]) - - def test_on_connect_path(self): - protocol = get_protocol(object)() - path = "path" - test_request = generate_connection_request(path, {}, {}) - protocol.onConnect(test_request) - self.assertEqual(path, protocol.request_info["path"]) From 5df99c9cfda57ab637b12f0636a4bd1e7d806206 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 17 Jan 2016 14:31:04 -0800 Subject: [PATCH 149/746] Fix some incorrect backend layer references --- docs/getting-started.rst | 2 +- testproject/testproject/settings.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index f7a0b7e..c6924ee 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -47,7 +47,7 @@ custom consumer we wrote above. Here's what that looks like:: # In settings.py CHANNEL_BACKENDS = { "default": { - "BACKEND": "channels.backends.database.DatabaseChannelBackend", + "BACKEND": "channels.database_layer.DatabaseChannelLayer", "ROUTING": "myproject.routing.channel_routing", }, } diff --git a/testproject/testproject/settings.py b/testproject/testproject/settings.py index 5bae6cb..19d9ecf 100644 --- a/testproject/testproject/settings.py +++ b/testproject/testproject/settings.py @@ -27,10 +27,10 @@ DATABASES = { CHANNEL_BACKENDS = { "default": { - "BACKEND": "channels.backends.database.DatabaseChannelBackend", + "BACKEND": "channels.database_layer.DatabaseChannelLayer", "ROUTING": "testproject.urls.channel_routing", }, } if os.environ.get("USEREDIS", None): - CHANNEL_BACKENDS['default']['BACKEND'] = "channels.backends.redis_py.RedisChannelBackend" + CHANNEL_BACKENDS['default']['BACKEND'] = "asgi_redis.RedisChannelLayer" From 17e9824f7141f318be654123e8ddf7f58c705cf1 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Feb 2016 14:16:20 -0800 Subject: [PATCH 150/746] Update database channel backend to pass conformance --- channels/database_layer.py | 69 +++++++++++++++++++++++---- channels/tests/test_database_layer.py | 2 +- docs/asgi.rst | 3 +- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index ea9e5e1..1b31068 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -3,6 +3,7 @@ import json import random import string import time +import base64 from django.apps.registry import Apps from django.db import DEFAULT_DB_ALIAS, IntegrityError, connections, models @@ -21,7 +22,8 @@ class DatabaseChannelLayer(object): Also uses JSON for serialization, as we don't want to make Django depend on msgpack for the built-in backend. The JSON format uses \uffff as first - character to signify a byte string rather than a text string. + character to signify a b64 byte string rather than a text string. Ugly, but + it's not a valid Unicode character, so it should be safe enough. """ def __init__(self, db_alias=DEFAULT_DB_ALIAS, expiry=60): @@ -75,7 +77,7 @@ class DatabaseChannelLayer(object): ### ASGI Group extension ### - def group_add(self, group, channel, expiry=None): + def group_add(self, group, channel): """ Adds the channel to the named group for at least 'expiry' seconds (expiry defaults to message expiry if not provided). @@ -83,7 +85,6 @@ class DatabaseChannelLayer(object): self.group_model.objects.update_or_create( group=group, channel=channel, - defaults={"expiry": now() + datetime.timedelta(seconds=expiry or self.expiry)}, ) def group_discard(self, group, channel): @@ -110,10 +111,10 @@ class DatabaseChannelLayer(object): ### Serialization ### def serialize(self, message): - return json.dumps(message) + return AsgiJsonEncoder().encode(message) def deserialize(self, message): - return json.loads(message) + return AsgiJsonDecoder().decode(message) ### Database state mgmt ### @@ -157,7 +158,6 @@ class DatabaseChannelLayer(object): class Group(models.Model): group = models.CharField(max_length=200) channel = models.CharField(max_length=200) - expiry = models.DateTimeField(db_index=True) class Meta: apps = Apps() @@ -174,9 +174,60 @@ class DatabaseChannelLayer(object): """ Cleans out expired groups and messages. """ - # Include a 10-second grace period because that solves some clock sync - self.channel_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() - self.group_model.objects.filter(expiry__lt=now() - datetime.timedelta(seconds=10)).delete() + # Include a 1-second grace period for clock sync drift + target = now() - datetime.timedelta(seconds=1) + # First, go through old messages and pick out channels that got expired + old_messages = self.channel_model.objects.filter(expiry__lt=target) + channels_to_ungroup = old_messages.values_list("channel", flat=True).distinct() + old_messages.delete() + # Now, remove channel membership from channels that expired + self.group_model.objects.filter(channel__in=channels_to_ungroup).delete() def __str__(self): return "%s(alias=%s)" % (self.__class__.__name__, self.connection.alias) + + +class AsgiJsonEncoder(json.JSONEncoder): + """ + Special encoder that transforms bytestrings into unicode strings + prefixed with u+ffff + """ + + def transform(self, o): + if isinstance(o, (list, tuple)): + return [self.transform(x) for x in o] + elif isinstance(o, dict): + return { + self.transform(k): self.transform(v) + for k, v in o.items() + } + elif isinstance(o, six.binary_type): + return u"\uffff" + base64.b64encode(o).decode("ascii") + else: + return o + + def encode(self, o): + return super(AsgiJsonEncoder, self).encode(self.transform(o)) + + +class AsgiJsonDecoder(json.JSONDecoder): + """ + Special encoder that transforms bytestrings into unicode strings + prefixed with u+ffff + """ + + def transform(self, o): + if isinstance(o, (list, tuple)): + return [self.transform(x) for x in o] + elif isinstance(o, dict): + return { + self.transform(k): self.transform(v) + for k, v in o.items() + } + elif isinstance(o, six.text_type) and o and o[0] == u"\uffff": + return base64.b64decode(o[1:].encode("ascii")) + else: + return o + + def decode(self, o): + return self.transform(super(AsgiJsonDecoder, self).decode(o)) diff --git a/channels/tests/test_database_layer.py b/channels/tests/test_database_layer.py index 146af36..c501c15 100644 --- a/channels/tests/test_database_layer.py +++ b/channels/tests/test_database_layer.py @@ -3,4 +3,4 @@ from asgiref.conformance import make_tests from ..database_layer import DatabaseChannelLayer channel_layer = DatabaseChannelLayer(expiry=1) -DatabaseLayerTests = make_tests(channel_layer, expiry_delay=1.1) +DatabaseLayerTests = make_tests(channel_layer, expiry_delay=2.1) diff --git a/docs/asgi.rst b/docs/asgi.rst index 1eedf70..09dde5f 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -367,7 +367,8 @@ clients reconnect will immediately resolve the problem. If a channel layer implements the ``groups`` extension, it must persist group membership until at least the time when the member channel has a message -expire due to non-consumption. +expire due to non-consumption. It should drop membership after a while to +prevent collision of old messages with new clients with the same random ID. Message Formats From 5cd2cbdfee1917b94c4af2a2945d0c70b69c99be Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Feb 2016 15:53:01 -0800 Subject: [PATCH 151/746] Add test suite for ASGI handlers --- channels/handler.py | 30 ++++-- channels/message.py | 24 +++-- channels/tests/test_handler.py | 90 +++++++++++++++++ channels/tests/test_request.py | 178 +++++++++++++++++++++++++++++++++ channels/worker.py | 1 - docs/asgi.rst | 22 ++-- 6 files changed, 315 insertions(+), 30 deletions(-) create mode 100644 channels/tests/test_handler.py create mode 100644 channels/tests/test_request.py diff --git a/channels/handler.py b/channels/handler.py index 33068a5..4dc83e7 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -9,6 +9,7 @@ from django.core.handlers import base from django.core import signals from django.core.urlresolvers import set_script_prefix from django.utils.functional import cached_property +from django.utils import six logger = logging.getLogger('django.request') @@ -21,8 +22,10 @@ class AsgiRequest(http.HttpRequest): def __init__(self, message): self.message = message - self.reply_channel = self.message['reply_channel'] + self.reply_channel = self.message.reply_channel self._content_length = 0 + self._post_parse_error = False + self.resolver_match = None # Path info self.path = self.message['path'] self.script_name = self.message.get('root_path', '') @@ -63,7 +66,7 @@ class AsgiRequest(http.HttpRequest): except (ValueError, TypeError): pass # Body handling - self._body = message.get("body", "") + self._body = message.get("body", b"") if message.get("body_channel", None): while True: # Get the next chunk from the request body channel @@ -78,6 +81,7 @@ class AsgiRequest(http.HttpRequest): # Exit loop if this was the last if not chunk.get("more_content", False): break + assert isinstance(self._body, six.binary_type), "Body is not bytes" # Other bits self.resolver_match = None @@ -97,7 +101,13 @@ class AsgiRequest(http.HttpRequest): def _set_post(self, post): self._post = post + def _get_files(self): + if not hasattr(self, '_files'): + self._load_post_and_files() + return self._files + POST = property(_get_post, _set_post) + FILES = property(_get_files) @cached_property def COOKIES(self): @@ -114,6 +124,9 @@ class AsgiHandler(base.BaseHandler): initLock = Lock() request_class = AsgiRequest + # Size to chunk response bodies into for multiple response messages + chunk_size = 512 * 1024 + def __call__(self, message): # Set up middleware if needed. We couldn't do this earlier, because # settings weren't available. @@ -148,7 +161,9 @@ class AsgiHandler(base.BaseHandler): """ Encodes a Django HTTP response into an ASGI http.response message(s). """ - # Collect cookies into headers + # Collect cookies into headers. + # Note that we have to preserve header case as there are some non-RFC + # compliant clients that want things like Content-Type correct. Ugh. response_headers = [(str(k), str(v)) for k, v in response.items()] for c in response.cookies.values(): response_headers.append((str('Set-Cookie'), str(c.output(header='')))) @@ -186,14 +201,13 @@ class AsgiHandler(base.BaseHandler): Yields (chunk, last_chunk) tuples. """ - CHUNK_SIZE = 512 * 1024 position = 0 while position < len(data): yield ( - data[position:position+CHUNK_SIZE], - (position + CHUNK_SIZE) >= len(data), + data[position:position + self.chunk_size], + (position + self.chunk_size) >= len(data), ) - position += CHUNK_SIZE + position += self.chunk_size class ViewConsumer(object): @@ -205,5 +219,5 @@ class ViewConsumer(object): self.handler = AsgiHandler() def __call__(self, message): - for reply_message in self.handler(message.content): + for reply_message in self.handler(message): message.reply_channel.send(reply_message) diff --git a/channels/message.py b/channels/message.py index 93b19fd..53a36cc 100644 --- a/channels/message.py +++ b/channels/message.py @@ -12,16 +12,20 @@ class Message(object): to use to reply to this message's end user, if that makes sense. """ - class Requeue(Exception): - """ - Raise this while processing a message to requeue it back onto the - channel. Useful if you're manually ensuring partial ordering, etc. - """ - pass - - def __init__(self, content, channel, channel_layer, reply_channel=None): + def __init__(self, content, channel, channel_layer): self.content = content self.channel = channel self.channel_layer = channel_layer - if reply_channel: - self.reply_channel = Channel(reply_channel, channel_layer=self.channel_layer) + if content.get("reply_channel", None): + self.reply_channel = Channel( + content["reply_channel"], + channel_layer=self.channel_layer, + ) + else: + self.reply_channel = None + + def __getitem__(self, key): + return self.content[key] + + def get(self, key, default=None): + return self.content.get(key, default) diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py new file mode 100644 index 0000000..192c036 --- /dev/null +++ b/channels/tests/test_handler.py @@ -0,0 +1,90 @@ +from __future__ import unicode_literals +from django.test import SimpleTestCase +from django.http import HttpResponse + +from asgiref.inmemory import ChannelLayer +from channels.handler import AsgiHandler +from channels.message import Message + + +class FakeAsgiHandler(AsgiHandler): + """ + Handler subclass that just returns a premade response rather than + go into the view subsystem. + """ + + chunk_size = 30 + + def __init__(self, response): + assert isinstance(response, HttpResponse) + self._response = response + super(FakeAsgiHandler, self).__init__() + + def get_response(self, request): + return self._response + + +class HandlerTests(SimpleTestCase): + """ + Tests that the handler works correctly and round-trips things into a + correct response. + """ + + def setUp(self): + """ + Make an in memory channel layer for testing + """ + self.channel_layer = ChannelLayer() + self.make_message = lambda m, c: Message(m, c, self.channel_layer) + + def test_basic(self): + """ + Tests a simple request + """ + # Make stub request and desired response + message = self.make_message({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": "/test/", + }, "test") + response = HttpResponse(b"Hi there!", content_type="text/plain") + # Run the handler + handler = FakeAsgiHandler(response) + reply_messages = list(handler(message)) + # Make sure we got the right number of messages + self.assertEqual(len(reply_messages), 1) + reply_message = reply_messages[0] + # Make sure the message looks correct + self.assertEqual(reply_message["content"], b"Hi there!") + self.assertEqual(reply_message["status"], 200) + self.assertEqual(reply_message["status_text"], "OK") + self.assertEqual(reply_message.get("more_content", False), False) + self.assertEqual( + reply_message["headers"], + [("Content-Type", b"text/plain")], + ) + + def test_large(self): + """ + Tests a large response (will need chunking) + """ + # Make stub request and desired response + message = self.make_message({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": "/test/", + }, "test") + response = HttpResponse(b"Thefirstthirtybytesisrighthereandhereistherest") + # Run the handler + handler = FakeAsgiHandler(response) + reply_messages = list(handler(message)) + # Make sure we got the right number of messages + self.assertEqual(len(reply_messages), 2) + # Make sure the messages look correct + self.assertEqual(reply_messages[0]["content"], b"Thefirstthirtybytesisrighthere") + self.assertEqual(reply_messages[0]["status"], 200) + self.assertEqual(reply_messages[0]["more_content"], True) + self.assertEqual(reply_messages[1]["content"], b"andhereistherest") + self.assertEqual(reply_messages[1].get("more_content", False), False) diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py new file mode 100644 index 0000000..a103b28 --- /dev/null +++ b/channels/tests/test_request.py @@ -0,0 +1,178 @@ +from __future__ import unicode_literals +from django.test import SimpleTestCase +from django.utils import six + +from asgiref.inmemory import ChannelLayer +from channels.handler import AsgiRequest +from channels.message import Message + + +class RequestTests(SimpleTestCase): + """ + Tests that ASGI request handling correctly decodes HTTP requests. + """ + + def setUp(self): + """ + Make an in memory channel layer for testing + """ + self.channel_layer = ChannelLayer() + self.make_message = lambda m, c: Message(m, c, self.channel_layer) + + def test_basic(self): + """ + Tests that the handler can decode the most basic request message, + with all optional fields omitted. + """ + message = self.make_message({ + "reply_channel": "test-reply", + "http_version": "1.1", + "method": "GET", + "path": "/test/", + }, "test") + request = AsgiRequest(message) + self.assertEqual(request.path, "/test/") + self.assertEqual(request.method, "GET") + self.assertFalse(request.body) + self.assertNotIn("HTTP_HOST", request.META) + self.assertNotIn("REMOTE_ADDR", request.META) + self.assertNotIn("REMOTE_HOST", request.META) + self.assertNotIn("REMOTE_PORT", request.META) + self.assertNotIn("SERVER_NAME", request.META) + self.assertNotIn("SERVER_PORT", request.META) + self.assertFalse(request.GET) + self.assertFalse(request.POST) + self.assertFalse(request.COOKIES) + + def test_extended(self): + """ + Tests a more fully-featured GET request + """ + message = self.make_message({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": "/test2/", + "query_string": b"x=1&y=foo%20bar+baz", + "headers": { + "host": b"example.com", + "cookie": b"test-time=1448995585123; test-value=yeah", + }, + "client": ["10.0.0.1", 1234], + "server": ["10.0.0.2", 80], + }, "test") + request = AsgiRequest(message) + self.assertEqual(request.path, "/test2/") + self.assertEqual(request.method, "GET") + self.assertFalse(request.body) + self.assertEqual(request.META["HTTP_HOST"], "example.com") + self.assertEqual(request.META["REMOTE_ADDR"], "10.0.0.1") + self.assertEqual(request.META["REMOTE_HOST"], "10.0.0.1") + self.assertEqual(request.META["REMOTE_PORT"], 1234) + self.assertEqual(request.META["SERVER_NAME"], "10.0.0.2") + self.assertEqual(request.META["SERVER_PORT"], 80) + self.assertEqual(request.GET["x"], "1") + self.assertEqual(request.GET["y"], "foo bar baz") + self.assertEqual(request.COOKIES["test-time"], "1448995585123") + self.assertEqual(request.COOKIES["test-value"], "yeah") + self.assertFalse(request.POST) + + def test_post_single(self): + """ + Tests a POST body contained within a single message. + """ + message = self.make_message({ + "reply_channel": "test", + "http_version": "1.1", + "method": "POST", + "path": "/test2/", + "query_string": b"django=great", + "body": b"ponies=are+awesome", + "headers": { + "host": b"example.com", + "content-type": b"application/x-www-form-urlencoded", + "content-length": b"18", + }, + }, "test") + request = AsgiRequest(message) + self.assertEqual(request.path, "/test2/") + self.assertEqual(request.method, "POST") + self.assertEqual(request.body, b"ponies=are+awesome") + self.assertEqual(request.META["HTTP_HOST"], "example.com") + self.assertEqual(request.META["CONTENT_TYPE"], "application/x-www-form-urlencoded") + self.assertEqual(request.GET["django"], "great") + self.assertEqual(request.POST["ponies"], "are awesome") + with self.assertRaises(KeyError): + request.POST["django"] + with self.assertRaises(KeyError): + request.GET["ponies"] + + def test_post_multiple(self): + """ + Tests a POST body across multiple messages (first part in 'body'). + """ + message = self.make_message({ + "reply_channel": "test", + "http_version": "1.1", + "method": "POST", + "path": "/test/", + "body": b"there_a", + "body_channel": "test-input", + "headers": { + "host": b"example.com", + "content-type": b"application/x-www-form-urlencoded", + "content-length": b"21", + }, + }, "test") + self.channel_layer.send("test-input", { + "content": b"re=fou", + "more_content": True, + }) + self.channel_layer.send("test-input", { + "content": b"r+lights", + }) + request = AsgiRequest(message) + self.assertEqual(request.method, "POST") + self.assertEqual(request.body, b"there_are=four+lights") + self.assertEqual(request.META["CONTENT_TYPE"], "application/x-www-form-urlencoded") + self.assertEqual(request.POST["there_are"], "four lights") + + def test_post_files(self): + """ + Tests POSTing files using multipart form data and multiple messages, + with no body in the initial message. + """ + body = ( + b'--BOUNDARY\r\n' + + b'Content-Disposition: form-data; name="title"\r\n\r\n' + + b'My First Book\r\n' + + b'--BOUNDARY\r\n' + + b'Content-Disposition: form-data; name="pdf"; filename="book.pdf"\r\n\r\n' + + b'FAKEPDFBYTESGOHERE' + + b'--BOUNDARY--' + ) + message = self.make_message({ + "reply_channel": "test", + "http_version": "1.1", + "method": "POST", + "path": "/test/", + "body_channel": "test-input", + "headers": { + "content-type": b"multipart/form-data; boundary=BOUNDARY", + "content-length": six.binary_type(len(body)), + }, + }, "test") + self.channel_layer.send("test-input", { + "content": body[:20], + "more_content": True, + }) + self.channel_layer.send("test-input", { + "content": body[20:], + }) + request = AsgiRequest(message) + self.assertEqual(request.method, "POST") + self.assertEqual(len(request.body), len(body)) + self.assertTrue(request.META["CONTENT_TYPE"].startswith("multipart/form-data")) + self.assertFalse(request._post_parse_error) + self.assertEqual(request.POST["title"], "My First Book") + self.assertEqual(request.FILES["pdf"].read(), "FAKEPDFBYTESGOHERE") diff --git a/channels/worker.py b/channels/worker.py index 73fc5c6..0111ff2 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -35,7 +35,6 @@ class Worker(object): content=content, channel=channel, channel_layer=self.channel_layer, - reply_channel=content.get("reply_channel", None), ) # Handle the message consumer = self.channel_layer.registry.consumer_for_channel(channel) diff --git a/docs/asgi.rst b/docs/asgi.rst index 09dde5f..019f7a3 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -431,30 +431,30 @@ Keys: * ``method``: Unicode string HTTP method name, uppercased. * ``scheme``: Unicode string URL scheme portion (likely ``http`` or ``https``). - Optional (but must not be empty), default is ``http``. + Optional (but must not be empty), default is ``"http"``. * ``path``: Byte string HTTP path from URL. * ``query_string``: Byte string URL portion after the ``?``. Optional, default - is empty string. + is ``""``. * ``root_path``: Byte string that indicates the root path this application is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults - to empty string. + to ``""``. * ``headers``: Dict of ``{name: value}``, where ``name`` is the lowercased - HTTP header name as byte string and ``value`` is the header value as a byte + HTTP header name as unicode string and ``value`` is the header value as a byte string. If multiple headers with the same name are received, they should be concatenated into a single header as per RFC 2616. Header names containing - underscores should be discarded by the server. + underscores should be discarded by the server. Optional, defaults to ``{}``. -* ``body``: Body of the request, as a byte string. Optional, defaults to empty - string. If ``body_channel`` is set, treat as start of body and concatenate +* ``body``: Body of the request, as a byte string. Optional, defaults to ``""``. + If ``body_channel`` is set, treat as start of body and concatenate on further chunks. * ``body_channel``: Single-reader channel name that contains Request Body Chunk messages representing a large request body. - Optional, defaults to None. Chunks append to ``body`` if set. Presence of + Optional, defaults to ``None``. Chunks append to ``body`` if set. Presence of a channel indicates at least one Request Body Chunk message needs to be read, and then further consumption keyed off of the ``more_content`` key in those messages. @@ -501,9 +501,9 @@ Keys: Ignored for HTTP/2 clients. Optional, default should be based on ``status`` or left as empty string if no default found. -* ``headers``: A list of ``[name, value]`` pairs, where ``name`` is the byte - string header name, and ``value`` is the byte string header value. Order - should be preserved in the HTTP response. +* ``headers``: A list of ``[name, value]`` pairs, where ``name`` is the + unicode string header name, and ``value`` is the byte string + header value. Order should be preserved in the HTTP response. * ``content``: Byte string of HTTP body content. Optional, defaults to empty string. From 58f18aa1031fc4c44ef9bb686e29b751e4cdb167 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Feb 2016 16:01:12 -0800 Subject: [PATCH 152/746] Add Travis build status badge --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 8e7cf07..bd537d8 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,9 @@ Django Channels =============== +.. image:: https://api.travis-ci.org/andrewgodwin/channels.svg + :target: https://api.travis-ci.org/andrewgodwin/channels + **NOTE: The current master branch is in flux as it changes to match the final structure and the new ASGI spec. If you wish to use this in the meantime, please use a tagged release.** From 43c903a50160f397408fde6f48fe5ad42038660c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Feb 2016 16:01:45 -0800 Subject: [PATCH 153/746] Fix image target --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index bd537d8..c3966b3 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ Django Channels =============== .. image:: https://api.travis-ci.org/andrewgodwin/channels.svg - :target: https://api.travis-ci.org/andrewgodwin/channels + :target: https://travis-ci.org/andrewgodwin/channels **NOTE: The current master branch is in flux as it changes to match the final structure and the new ASGI spec. If you wish to use this in the meantime, From 74c444ca52b118a16d26e43ff748723792170162 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Feb 2016 16:04:18 -0800 Subject: [PATCH 154/746] Add dependencies to travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 361fe5e..7b0ca09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,5 +3,5 @@ language: python python: - "2.7" - "3.5" -install: pip install tox-travis +install: pip install tox-travis Django asgiref script: tox From 1dab70ed86c3cf2098091b415747309b04773e25 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Feb 2016 16:22:58 -0800 Subject: [PATCH 155/746] Update CI config a bit more. --- .travis.yml | 2 +- channels/handler.py | 4 ++-- tox.ini | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b0ca09..361fe5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,5 +3,5 @@ language: python python: - "2.7" - "3.5" -install: pip install tox-travis Django asgiref +install: pip install tox-travis script: tox diff --git a/channels/handler.py b/channels/handler.py index 4dc83e7..06792fe 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -5,11 +5,11 @@ import logging from threading import Lock from django import http -from django.core.handlers import base from django.core import signals +from django.core.handlers import base from django.core.urlresolvers import set_script_prefix -from django.utils.functional import cached_property from django.utils import six +from django.utils.functional import cached_property logger = logging.getLogger('django.request') diff --git a/tox.ini b/tox.ini index 4cde18b..35e7158 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ setenv = PYTHONPATH = {toxinidir}:{toxinidir} deps = autobahn + asgiref>=0.9 six redis==2.10.5 py27: mock @@ -19,7 +20,7 @@ deps = django-16: Django>=1.6,<1.7 django-17: Django>=1.7,<1.8 django-18: Django>=1.8,<1.9 - django-19: Django==1.9b1 + django-19: Django>=1.9,<1.10 commands = flake8: flake8 isort: isort -c -rc channels From 13766a3027e58905b46bb069063c627c15214fb3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Feb 2016 16:26:05 -0800 Subject: [PATCH 156/746] Fix flake8 errors --- channels/database_layer.py | 14 +++++++------- channels/decorators.py | 1 + channels/management/commands/runworker.py | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index 1b31068..46df924 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -1,12 +1,12 @@ +import base64 import datetime import json import random import string import time -import base64 from django.apps.registry import Apps -from django.db import DEFAULT_DB_ALIAS, IntegrityError, connections, models +from django.db import DEFAULT_DB_ALIAS, connections, models from django.utils import six from django.utils.functional import cached_property from django.utils.timezone import now @@ -30,7 +30,7 @@ class DatabaseChannelLayer(object): self.expiry = expiry self.db_alias = db_alias - ### ASGI API ### + # ASGI API extensions = ["groups", "flush"] @@ -75,7 +75,7 @@ class DatabaseChannelLayer(object): if not self.channel_model.objects.filter(channel=new_name).exists(): return new_name - ### ASGI Group extension ### + # ASGI Group extension def group_add(self, group, channel): """ @@ -102,13 +102,13 @@ class DatabaseChannelLayer(object): for channel in self.group_model.objects.filter(group=group).values_list("channel", flat=True): self.send(channel, message) - ### ASGI Flush extension ### + # ASGI Flush extension def flush(self): self.channel_model.objects.all().delete() self.group_model.objects.all().delete() - ### Serialization ### + # Serialization def serialize(self, message): return AsgiJsonEncoder().encode(message) @@ -116,7 +116,7 @@ class DatabaseChannelLayer(object): def deserialize(self, message): return AsgiJsonDecoder().decode(message) - ### Database state mgmt ### + # Database state mgmt @property def connection(self): diff --git a/channels/decorators.py b/channels/decorators.py index 295ab3d..063447a 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -15,6 +15,7 @@ def linearize(func): ordering, just linearity. """ raise NotImplementedError("Not yet reimplemented") + @functools.wraps(func) def inner(message, *args, **kwargs): # Make sure there's a reply channel diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 294741b..5406548 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from django.core.management import BaseCommand, CommandError +from django.core.management import BaseCommand from channels import channel_layers, DEFAULT_CHANNEL_LAYER from channels.log import setup_logger from channels.handler import ViewConsumer From fa58375a51c94ae58fc6b5e3bbaa0222fe404f70 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Feb 2016 16:47:11 -0800 Subject: [PATCH 157/746] Python 3 fixes --- channels/database_layer.py | 2 +- channels/handler.py | 26 ++++++++++++++++++++++---- channels/tests/test_request.py | 4 ++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index 46df924..d85a2e8 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -71,7 +71,7 @@ class DatabaseChannelLayer(object): # Keep making channel names till one isn't present. while True: random_string = "".join(random.choice(string.ascii_letters) for i in range(8)) - new_name = pattern.replace(b"?", random_string) + new_name = pattern.replace("?", random_string) if not self.channel_model.objects.filter(channel=new_name).exists(): return new_name diff --git a/channels/handler.py b/channels/handler.py index 06792fe..b9bdbd6 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -58,12 +58,14 @@ class AsgiRequest(http.HttpRequest): corrected_name = "CONTENT_TYPE" else: corrected_name = 'HTTP_%s' % name.upper().replace("-", "_") - self.META[corrected_name] = value + # TODO: Look at request encoding for unicode decode + self.META[corrected_name] = value.decode("latin1") # Pull out content length info if self.META.get('CONTENT_LENGTH', None): try: self._content_length = int(self.META['CONTENT_LENGTH']) - except (ValueError, TypeError): + except (ValueError, TypeError) as e: + print (self.META) pass # Body handling self._body = message.get("body", b"") @@ -164,9 +166,25 @@ class AsgiHandler(base.BaseHandler): # Collect cookies into headers. # Note that we have to preserve header case as there are some non-RFC # compliant clients that want things like Content-Type correct. Ugh. - response_headers = [(str(k), str(v)) for k, v in response.items()] + response_headers = [] + for header, value in response.items(): + if isinstance(header, six.binary_type): + header = header.decode("latin1") + if isinstance(value, six.text_type): + value = value.encode("latin1") + response_headers.append( + ( + six.text_type(header), + six.binary_type(value), + ) + ) for c in response.cookies.values(): - response_headers.append((str('Set-Cookie'), str(c.output(header='')))) + response_headers.append( + ( + 'Set-Cookie', + six.binary_type(c.output(header='')), + ) + ) # Make initial response message message = { "status": response.status_code, diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index a103b28..3cf4601 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -159,7 +159,7 @@ class RequestTests(SimpleTestCase): "body_channel": "test-input", "headers": { "content-type": b"multipart/form-data; boundary=BOUNDARY", - "content-length": six.binary_type(len(body)), + "content-length": six.text_type(len(body)).encode("ascii"), }, }, "test") self.channel_layer.send("test-input", { @@ -175,4 +175,4 @@ class RequestTests(SimpleTestCase): self.assertTrue(request.META["CONTENT_TYPE"].startswith("multipart/form-data")) self.assertFalse(request._post_parse_error) self.assertEqual(request.POST["title"], "My First Book") - self.assertEqual(request.FILES["pdf"].read(), "FAKEPDFBYTESGOHERE") + self.assertEqual(request.FILES["pdf"].read(), b"FAKEPDFBYTESGOHERE") From b7b11cb075dee6f70d204c61f4cb847e3eda12c0 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Feb 2016 16:47:43 -0800 Subject: [PATCH 158/746] Remove sneaky print --- channels/handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index b9bdbd6..2732ce1 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -64,8 +64,7 @@ class AsgiRequest(http.HttpRequest): if self.META.get('CONTENT_LENGTH', None): try: self._content_length = int(self.META['CONTENT_LENGTH']) - except (ValueError, TypeError) as e: - print (self.META) + except (ValueError, TypeError): pass # Body handling self._body = message.get("body", b"") From 88d3379e3189d04065a3030375976d0d42111a0d Mon Sep 17 00:00:00 2001 From: George Brocklehurst Date: Sat, 6 Feb 2016 18:26:41 -0500 Subject: [PATCH 159/746] Docs: reply_channel is a property of message Update an example where `reply_channel` was a global, and contained the channel name rather than a channel objects. --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index b523d49..5066cb5 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -113,7 +113,7 @@ message. Suddenly, a view is merely another example of a consumer:: # Run view django_response = view(django_request) # Encode the response into JSON-compat format - Channel(reply_channel).send(django_response.encode()) + message.reply_channel.send(django_response.encode()) In fact, this is how Channels works. The interface servers transform connections from the outside world (HTTP, WebSockets, etc.) into messages on channels, From 85bb8c98d2143b28f7556a90bf7a825373125d9b Mon Sep 17 00:00:00 2001 From: George Brocklehurst Date: Sun, 7 Feb 2016 10:19:21 -0500 Subject: [PATCH 160/746] Docs: Update encode/decode methods in example. `encode` is now `channel_encode`, and `decode` is now `channel_decode`. --- docs/concepts.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 5066cb5..5b470fc 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -109,11 +109,11 @@ message. Suddenly, a view is merely another example of a consumer:: # Listens on http.request def my_consumer(message): # Decode the request from JSON-compat to a full object - django_request = Request.decode(message.content) + django_request = Request.channel_decode(message.content) # Run view django_response = view(django_request) # Encode the response into JSON-compat format - message.reply_channel.send(django_response.encode()) + message.reply_channel.send(django_response.channel_encode()) In fact, this is how Channels works. The interface servers transform connections from the outside world (HTTP, WebSockets, etc.) into messages on channels, From 5ddeed4a2528d48a48fc3e924821a3ebcff3fa22 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 9 Feb 2016 12:59:24 -0800 Subject: [PATCH 161/746] Remove dead code --- channels/adapters.py | 28 ---------------------------- channels/worker.py | 2 -- 2 files changed, 30 deletions(-) delete mode 100644 channels/adapters.py diff --git a/channels/adapters.py b/channels/adapters.py deleted file mode 100644 index 5e5f020..0000000 --- a/channels/adapters.py +++ /dev/null @@ -1,28 +0,0 @@ -import functools - -from django.http import HttpRequest, HttpResponse -from channels import Channel - - -def view_producer(channel_name): - """ - Returns a new view function that actually writes the request to a channel - and abandons the response (with an exception the Worker will catch) - """ - def producing_view(request): - Channel(channel_name).send(request.channel_encode()) - raise HttpResponse.ResponseLater() - return producing_view - - -def view_consumer(func): - """ - Decorates a normal Django view to be a channel consumer. - Does not run any middleware - """ - @functools.wraps(func) - def consumer(message): - request = HttpRequest.channel_decode(message.content) - response = func(request) - message.reply_channel.send(response.channel_encode()) - return func diff --git a/channels/worker.py b/channels/worker.py index 0111ff2..5195271 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -42,7 +42,5 @@ class Worker(object): self.callback(channel, message) try: consumer(message) - except Message.Requeue: - self.channel_layer.send(channel, content) except: logger.exception("Error processing message with consumer %s:", name_that_thing(consumer)) From aff9ca2f13a3cc4bb352c6fa073b1f44102ef5f3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 9 Feb 2016 12:59:49 -0800 Subject: [PATCH 162/746] Update runserver to autoreload w/daphne in main thread --- channels/asgi.py | 3 ++ channels/handler.py | 40 +++++++++++++++-------- channels/log.py | 13 ++++++++ channels/management/commands/runserver.py | 19 ++++++----- 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/channels/asgi.py b/channels/asgi.py index 3bb7ac3..7e1629e 100644 --- a/channels/asgi.py +++ b/channels/asgi.py @@ -52,6 +52,9 @@ class ChannelLayerManager(object): self.backends[key] = self.make_backend(key) return self.backends[key] + def __contains__(self, key): + return key in self.configs + class ChannelLayerWrapper(object): """ diff --git a/channels/handler.py b/channels/handler.py index 2732ce1..7834666 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -20,6 +20,11 @@ class AsgiRequest(http.HttpRequest): dict, and wraps request body handling. """ + # Exception that will cause any handler to skip around response + # transmission and presume something else will do it later. + class ResponseLater(Exception): + pass + def __init__(self, message): self.message = message self.reply_channel = self.message.reply_channel @@ -27,7 +32,8 @@ class AsgiRequest(http.HttpRequest): self._post_parse_error = False self.resolver_match = None # Path info - self.path = self.message['path'] + # TODO: probably needs actual URL decoding + self.path = self.message['path'].decode("ascii") self.script_name = self.message.get('root_path', '') if self.script_name: # TODO: Better is-prefix checking, slash handling? @@ -38,7 +44,7 @@ class AsgiRequest(http.HttpRequest): self.method = self.message['method'].upper() self.META = { "REQUEST_METHOD": self.method, - "QUERY_STRING": self.message.get('query_string', ''), + "QUERY_STRING": self.message.get('query_string', '').decode("ascii"), # Old code will need these for a while "wsgi.multithread": True, "wsgi.multiprocess": True, @@ -151,6 +157,10 @@ class AsgiHandler(base.BaseHandler): } ) response = http.HttpResponseBadRequest() + except AsgiRequest.ResponseLater: + # The view has promised something else + # will send a response at a later time + return else: response = self.get_response(request) # Transform response into messages, which we yield back to caller @@ -158,9 +168,10 @@ class AsgiHandler(base.BaseHandler): # TODO: file_to_stream yield message - def encode_response(self, response): + @classmethod + def encode_response(cls, response): """ - Encodes a Django HTTP response into an ASGI http.response message(s). + Encodes a Django HTTP response into ASGI http.response message(s). """ # Collect cookies into headers. # Note that we have to preserve header case as there are some non-RFC @@ -181,19 +192,19 @@ class AsgiHandler(base.BaseHandler): response_headers.append( ( 'Set-Cookie', - six.binary_type(c.output(header='')), + c.output(header='').encode("ascii"), ) ) # Make initial response message message = { "status": response.status_code, - "status_text": response.reason_phrase, + "status_text": response.reason_phrase.encode("ascii"), "headers": response_headers, } # Streaming responses need to be pinned to their iterator if response.streaming: for part in response.streaming_content: - for chunk in self.chunk_bytes(part): + for chunk in cls.chunk_bytes(part): message['content'] = chunk message['more_content'] = True yield message @@ -205,13 +216,14 @@ class AsgiHandler(base.BaseHandler): # Other responses just need chunking else: # Yield chunks of response - for chunk, last in self.chunk_bytes(response.content): + for chunk, last in cls.chunk_bytes(response.content): message['content'] = chunk message['more_content'] = not last yield message message = {} - def chunk_bytes(self, data): + @classmethod + def chunk_bytes(cls, data): """ Chunks some data into chunks based on the current ASGI channel layer's message size and reasonable defaults. @@ -221,10 +233,10 @@ class AsgiHandler(base.BaseHandler): position = 0 while position < len(data): yield ( - data[position:position + self.chunk_size], - (position + self.chunk_size) >= len(data), + data[position:position + cls.chunk_size], + (position + cls.chunk_size) >= len(data), ) - position += self.chunk_size + position += cls.chunk_size class ViewConsumer(object): @@ -232,8 +244,10 @@ class ViewConsumer(object): Dispatches channel HTTP requests into django's URL/View system. """ + handler_class = AsgiHandler + def __init__(self): - self.handler = AsgiHandler() + self.handler = self.handler_class() def __call__(self, message): for reply_message in self.handler(message): diff --git a/channels/log.py b/channels/log.py index c3a8103..8100ede 100644 --- a/channels/log.py +++ b/channels/log.py @@ -2,15 +2,28 @@ import logging def setup_logger(name, verbosity=1): + """ + Basic logger for runserver etc. + """ + formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(module)s - %(message)s') handler = logging.StreamHandler() handler.setFormatter(formatter) + # Set up main logger logger = logging.getLogger(name) logger.setLevel(logging.INFO) logger.addHandler(handler) if verbosity > 1: logger.setLevel(logging.DEBUG) + logger.debug("Logging set to DEBUG") + + # Set up daphne protocol loggers + for module in ["daphne.ws_protocol", "daphne.http_protocol"]: + daphne_logger = logging.getLogger() + daphne_logger.addHandler(handler) + daphne_logger.setLevel(logging.DEBUG if verbosity > 1 else logging.INFO) + logger.propagate = False return logger diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 63d3511..ce081d3 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -1,5 +1,7 @@ +import os import threading +from django.utils import autoreload from django.core.management.commands.runserver import \ Command as RunserverCommand @@ -17,21 +19,18 @@ class Command(RunserverCommand): super(Command, self).handle(*args, **options) def run(self, *args, **options): - # Don't autoreload for now - self.inner_run(None, **options) - - def inner_run(self, *args, **options): # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] if not self.channel_layer.registry.consumer_for_channel("http.request"): self.channel_layer.registry.add_consumer(ViewConsumer(), ["http.request"]) - # Note that this is the channel-enabled one on the console + # Helpful note to say this is the Channels runserver self.logger.info("Worker thread running, channels enabled") - # Launch a worker thread + # Launch worker as subthread (including autoreload logic) worker = WorkerThread(self.channel_layer) worker.daemon = True worker.start() - # Launch server in main thread + # Launch server in main thread (Twisted doesn't like being in a + # subthread, and it doesn't need to autoreload as there's no user code) from daphne.server import Server Server( channel_layer=self.channel_layer, @@ -50,4 +49,8 @@ class WorkerThread(threading.Thread): self.channel_layer = channel_layer def run(self): - Worker(channel_layer=self.channel_layer).run() + worker = Worker(channel_layer=self.channel_layer) + # We need to set run_main so it doesn't try to relaunch the entire + # program - that will make Daphne run twice. + os.environ["RUN_MAIN"] = "true" + autoreload.main(worker.run) From be172045ed6abf6c5c6f18fe36baf1993b954c22 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 9 Feb 2016 13:04:23 -0800 Subject: [PATCH 163/746] Fix test path encoding --- channels/tests/test_handler.py | 4 ++-- channels/tests/test_request.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index 192c036..2289c06 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -46,7 +46,7 @@ class HandlerTests(SimpleTestCase): "reply_channel": "test", "http_version": "1.1", "method": "GET", - "path": "/test/", + "path": b"/test/", }, "test") response = HttpResponse(b"Hi there!", content_type="text/plain") # Run the handler @@ -74,7 +74,7 @@ class HandlerTests(SimpleTestCase): "reply_channel": "test", "http_version": "1.1", "method": "GET", - "path": "/test/", + "path": b"/test/", }, "test") response = HttpResponse(b"Thefirstthirtybytesisrighthereandhereistherest") # Run the handler diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index 3cf4601..ea2fb0f 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -28,7 +28,7 @@ class RequestTests(SimpleTestCase): "reply_channel": "test-reply", "http_version": "1.1", "method": "GET", - "path": "/test/", + "path": b"/test/", }, "test") request = AsgiRequest(message) self.assertEqual(request.path, "/test/") @@ -52,7 +52,7 @@ class RequestTests(SimpleTestCase): "reply_channel": "test", "http_version": "1.1", "method": "GET", - "path": "/test2/", + "path": b"/test2/", "query_string": b"x=1&y=foo%20bar+baz", "headers": { "host": b"example.com", @@ -85,7 +85,7 @@ class RequestTests(SimpleTestCase): "reply_channel": "test", "http_version": "1.1", "method": "POST", - "path": "/test2/", + "path": b"/test2/", "query_string": b"django=great", "body": b"ponies=are+awesome", "headers": { @@ -115,7 +115,7 @@ class RequestTests(SimpleTestCase): "reply_channel": "test", "http_version": "1.1", "method": "POST", - "path": "/test/", + "path": b"/test/", "body": b"there_a", "body_channel": "test-input", "headers": { @@ -155,7 +155,7 @@ class RequestTests(SimpleTestCase): "reply_channel": "test", "http_version": "1.1", "method": "POST", - "path": "/test/", + "path": b"/test/", "body_channel": "test-input", "headers": { "content-type": b"multipart/form-data; boundary=BOUNDARY", From 49cbf3187222b3ba10a44cfd94f7cf2ae2492234 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 9 Feb 2016 13:32:37 -0800 Subject: [PATCH 164/746] Fix more python 3 unicode issues --- channels/handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 7834666..910d6f1 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -34,7 +34,7 @@ class AsgiRequest(http.HttpRequest): # Path info # TODO: probably needs actual URL decoding self.path = self.message['path'].decode("ascii") - self.script_name = self.message.get('root_path', '') + self.script_name = self.message.get('root_path', b'') if self.script_name: # TODO: Better is-prefix checking, slash handling? self.path_info = self.path[len(self.script_name):] @@ -44,7 +44,7 @@ class AsgiRequest(http.HttpRequest): self.method = self.message['method'].upper() self.META = { "REQUEST_METHOD": self.method, - "QUERY_STRING": self.message.get('query_string', '').decode("ascii"), + "QUERY_STRING": self.message.get('query_string', b'').decode("ascii"), # Old code will need these for a while "wsgi.multithread": True, "wsgi.multiprocess": True, From c7d417dd33237725fdec01a87cb2a955d30e3d3a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 9 Feb 2016 13:35:13 -0800 Subject: [PATCH 165/746] OKAY I THINK THIS IS THE LAST STUPID MISTAKE --- channels/tests/test_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index 2289c06..f94ca03 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -58,7 +58,7 @@ class HandlerTests(SimpleTestCase): # Make sure the message looks correct self.assertEqual(reply_message["content"], b"Hi there!") self.assertEqual(reply_message["status"], 200) - self.assertEqual(reply_message["status_text"], "OK") + self.assertEqual(reply_message["status_text"], b"OK") self.assertEqual(reply_message.get("more_content", False), False) self.assertEqual( reply_message["headers"], From 899e180c21836f5f0655d8eaddd1a9ab9271a58a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Feb 2016 18:39:34 +0000 Subject: [PATCH 166/746] Start updating docs to reflect new interaction pattern --- .gitignore | 1 + docs/concepts.rst | 59 +++++++---- docs/getting-started.rst | 219 ++++++++++++++++++--------------------- 3 files changed, 137 insertions(+), 142 deletions(-) diff --git a/.gitignore b/.gitignore index 1c527d5..91b64df 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ .tox/ *.swp *.pyc +TODO diff --git a/docs/concepts.rst b/docs/concepts.rst index 5b470fc..a4b865b 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -94,7 +94,7 @@ single process tied to a WSGI server, Django runs in three separate layers: * The workers, that listen on all relevant channels and run consumer code when a message is ready. -This may seem quite simplistic, but that's part of the design; rather than +This may seem relatively simplistic, but that's part of the design; rather than try and have a full asynchronous architecture, we're just introducing a slightly more complex abstraction than that presented by Django views. @@ -108,22 +108,25 @@ message. Suddenly, a view is merely another example of a consumer:: # Listens on http.request def my_consumer(message): - # Decode the request from JSON-compat to a full object - django_request = Request.channel_decode(message.content) + # Decode the request from message format to a Request object + django_request = AsgiRequest(message) # Run view django_response = view(django_request) - # Encode the response into JSON-compat format - message.reply_channel.send(django_response.channel_encode()) + # Encode the response into message format + for chunk in AsgiHandler.encode_response(django_response): + message.reply_channel.send(chunk) In fact, this is how Channels works. The interface servers transform connections from the outside world (HTTP, WebSockets, etc.) into messages on channels, -and then you write workers to handle these messages. +and then you write workers to handle these messages. Usually you leave normal +HTTP up to Django's built-in consumers that plug it into the view/template +system, but you can override it to add functionality if you want. -However, the key here is that you can run code (and so send on channels) in +However, the crucial part is that you can run code (and so send on channels) in response to any event - and that includes ones you create. You can trigger on model saves, on other incoming messages, or from code paths inside views and forms. That approach comes in handy for push-style -code - where you use HTML5's server-sent events or a WebSocket to notify +code - where you WebSockets or HTTP long-polling to notify clients of changes in real time (messages in a chat, perhaps, or live updates in an admin as another user edits something). @@ -168,8 +171,8 @@ Because channels only deliver to a single listener, they can't do broadcast; if you want to send a message to an arbitrary group of clients, you need to keep track of which response channels of those you wish to send to. -Say I had a live blog where I wanted to push out updates whenever a new post is -saved, I would register a handler for the ``post_save`` signal and keep a +If I had a liveblog where I wanted to push out updates whenever a new post is +saved, I could register a handler for the ``post_save`` signal and keep a set of channels (here, using Redis) to send updates to:: redis_conn = redis.Redis("localhost", 6379) @@ -194,7 +197,7 @@ listens to ``websocket.disconnect`` to do that, but we'd also need to have some kind of expiry in case an interface server is forced to quit or loses power before it can send disconnect signals - your code will never see any disconnect notification but the response channel is completely -invalid and messages you send there will never get consumed and just expire. +invalid and messages you send there will sit there until they expire. Because the basic design of channels is stateless, the channel server has no concept of "closing" a channel if an interface server goes away - after all, @@ -202,15 +205,17 @@ channels are meant to hold messages until a consumer comes along (and some types of interface server, e.g. an SMS gateway, could theoretically serve any client from any interface server). -That means that we need to follow a keepalive model, where the interface server -(or, if you want even better accuracy, the client browser/connection) sends -a periodic message saying it's still connected (though only for persistent -connection types like WebSockets; normal HTTP doesn't need this as it won't -stay connected for more than its own timeout). +We don't particularly care if a disconnected client doesn't get the messages +sent to the group - after all, it disconnected - but we do care about +cluttering up the channel backend tracking all of these clients that are no +longer around (and possibly, eventually getting a collision on the reply +channel name and sending someone messages not meant for them, though that would +likely take weeks). Now, we could go back into our example above and add an expiring set and keep -track of expiry times and so forth, but this is such a common pattern that -we don't need to; Channels has it built in, as a feature called Groups:: +track of expiry times and so forth, but what would be the point of a framework +if it made you add boilerplate code? Instead, Channels implements this +abstraction as a core concept called Groups:: @receiver(post_save, sender=BlogUpdate) def send_update(sender, instance, **kwargs): @@ -219,19 +224,27 @@ we don't need to; Channels has it built in, as a feature called Groups:: content=instance.content, ) - # Connected to websocket.connect and websocket.keepalive + # Connected to websocket.connect def ws_connect(message): # Add to reader group Group("liveblog").add(message.reply_channel) + # Connected to websocket.disconnect + def ws_disconnect(message): + # Remove from reader group on clean disconnect + Group("liveblog").discard(message.reply_channel) + Not only do groups have their own ``send()`` method (which backends can provide an efficient implementation of), they also automatically manage expiry of -the group members. You'll have to re-call ``Group.add()`` every so often to -keep existing members from expiring, but that's easy, and can be done in the -same handler for both ``connect`` and ``keepalive``, as you can see above. +the group members - when the channel starts having messages expire on it due +to non-consumption, we go in and remove it from all the groups it's in as well. +Of course, you should still remove things from the group on disconnect if you +can; the expiry code is there to catch cases where the disconnect message +doesn't make it for some reason. Groups are generally only useful for response channels (ones starting with -the character ``!``), as these are unique-per-client. +the character ``!``), as these are unique-per-client, but can be used for +normal channels as well if you wish. Next Steps ---------- diff --git a/docs/getting-started.rst b/docs/getting-started.rst index c6924ee..d66a98a 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -18,34 +18,40 @@ only one consumer can listen to any given channel. As a very basic example, let's write a consumer that overrides the built-in handling and handles every HTTP request directly. This isn't something you'd -usually do in a project, but it's a good illustration of how channels -now underlie every part of Django. +usually do in a project, but it's a good illustration of how Channels +actually underlies even core Django. Make a new project, a new app, and put this in a ``consumers.py`` file in the app:: from django.http import HttpResponse + from channels.handler import AsgiHandler def http_consumer(message): + # Make standard HTTP response - access ASGI path attribute directly response = HttpResponse("Hello world! You asked for %s" % message.content['path']) - message.reply_channel.send(response.channel_encode()) + # Encode that response into message format (ASGI) + for chunk in AsgiHandler.encode_response(response): + message.reply_channel.send(chunk) The most important thing to note here is that, because things we send in messages must be JSON serializable, the request and response messages -are in a key-value format. There are ``channel_decode()`` and -``channel_encode()`` methods on both Django's request and response classes, -but here we just use the message's ``content`` attribute directly for simplicity -(message content is always a dict). +are in a key-value format. You can read more about that format in the +:doc:`ASGI specification `, but you don't need to worry about it too much; +just know that there's an ``AsgiRequest`` class that translates from ASGI into +Django request objects, and the ``AsgiHandler`` class handles translation of +``HttpResponse`` into ASGI messages, which you see used above. Usually, +Django's built-in code will do all this for you when you're using normal views. -Now, go into your ``settings.py`` file, and set up a channel backend; by default, -Django will just use a local backend and route HTTP requests to the normal -URL resolver (we'll come back to backends in a minute). +Now, go into your ``settings.py`` file, and set up a channel layer; by default, +Django will just use an in-memory layer and route HTTP requests to the normal +URL resolver (we'll come back to channel layers in a minute). For now, we want to override the *channel routing* so that, rather than going to the URL resolver and our normal view stack, all HTTP requests go to our custom consumer we wrote above. Here's what that looks like:: # In settings.py - CHANNEL_BACKENDS = { + CHANNEL_LAYERS = { "default": { "BACKEND": "channels.database_layer.DatabaseChannelLayer", "ROUTING": "myproject.routing.channel_routing", @@ -58,11 +64,12 @@ custom consumer we wrote above. Here's what that looks like:: } As you can see, this is a little like Django's ``DATABASES`` setting; there are -named channel backends, with a default one called ``default``. Each backend +named channel layers, with a default one called ``default``. Each layer needs a class specified which powers it - we'll come to the options there later - and a routing scheme, which points to a dict containing the routing settings. It's recommended you call this ``routing.py`` and put it alongside ``urls.py`` -in your project. +in your project, but you can put it wherever you like, as long as the path is +correct. If you start up ``python manage.py runserver`` and go to ``http://localhost:8000``, you'll see that, rather than a default Django page, @@ -74,7 +81,8 @@ been able to do for a long time. Let's try some WebSockets, and make a basic chat server! Delete that consumer and its routing - we'll want the normal Django view layer to -serve HTTP requests from now on - and make this WebSocket consumer instead:: +serve HTTP requests from now on, which happens if you don't specify a consumer +for ``http.request`` - and make this WebSocket consumer instead:: # In consumers.py from channels import Group @@ -85,8 +93,10 @@ serve HTTP requests from now on - and make this WebSocket consumer instead:: Hook it up to the ``websocket.connect`` channel like this:: # In routing.py + from myproject.myapp.consumers import ws_add + channel_routing = { - "websocket.connect": "myproject.myapp.consumers.ws_add", + "websocket.connect": ws_add, } Now, let's look at what this is doing. It's tied to the @@ -98,39 +108,15 @@ is the unique response channel for that client, and adds it to the ``chat`` group, which means we can send messages to all connected chat clients. Of course, if you've read through :doc:`concepts`, you'll know that channels -added to groups expire out after a while unless you keep renewing their -membership. This is because Channels is stateless; the worker processes -don't keep track of the open/close states of the potentially thousands of -connections you have open at any one time. +added to groups expire out if their messages expire (every channel layer has +a message expiry time, usually between 30 seconds and a few minutes, and it's +often configurable). -The solution to this is that the WebSocket interface servers will send -periodic "keepalive" messages on the ``websocket.keepalive`` channel, -so we can hook that up to re-add the channel:: - - # In consumers.py - from channels import Group - - # Connected to websocket.keepalive - def ws_keepalive(message): - Group("chat").add(message.reply_channel) - -It's safe to add the channel to a group it's already in - similarly, it's -safe to discard a channel from a group it's not in. -Of course, this is exactly the same code as the ``connect`` handler, so let's -just route both channels to the same consumer:: - - # In routing.py - channel_routing = { - "websocket.connect": "myproject.myapp.consumers.ws_add", - "websocket.keepalive": "myproject.myapp.consumers.ws_add", - } - -And, even though channels will expire out, let's add an explicit ``disconnect`` -handler to clean up as people disconnect (most channels will cleanly disconnect -and get this called):: - - # In consumers.py - from channels import Group +However, we'll still get disconnection messages most of the time when a +WebSocket disconnects; the expiry/garbage collection of group membership is +mostly there for when a disconnect message gets lost (channels are not +guaranteed delivery, just mostly reliable). Let's add an explicit disconnect +handler:: # Connected to websocket.disconnect def ws_disconnect(message): @@ -144,13 +130,17 @@ any message sent in to all connected clients. Here's all the code:: # In consumers.py from channels import Group - # Connected to websocket.connect and websocket.keepalive + # Connected to websocket.connect def ws_add(message): Group("chat").add(message.reply_channel) # Connected to websocket.receive def ws_message(message): - Group("chat").send(message.content) + # ASGI WebSocket packet-received and send-packet message types + # both have a "text" key for their textual data. + Group("chat").send({ + "text": "[user] %s" % message.content['text'], + }) # Connected to websocket.disconnect def ws_disconnect(message): @@ -158,11 +148,12 @@ any message sent in to all connected clients. Here's all the code:: And what our routing should look like in ``routing.py``:: + from myproject.myapp.consumers import ws_add, ws_message, ws_disconnect + channel_routing = { - "websocket.connect": "myproject.myapp.consumers.ws_add", - "websocket.keepalive": "myproject.myapp.consumers.ws_add", - "websocket.receive": "myproject.myapp.consumers.ws_message", - "websocket.disconnect": "myproject.myapp.consumers.ws_disconnect", + "websocket.connect": ws_add, + "websocket.receive": ws_message, + "websocket.disconnect": ws_disconnect, } With all that code, you now have a working set of a logic for a chat server. @@ -172,49 +163,52 @@ hard. Running with Channels --------------------- -Because Channels takes Django into a multi-process model, you can no longer -just run one process if you want to serve more than one protocol type. +Because Channels takes Django into a multi-process model, you no longer run +everything in one process along with a WSGI server (of course, you're still +free to do that if you don't want to use Channels). Instead, you run one or +more *interface servers*, and one or more *worker servers*, connected by +that *channel layer* you configured earlier. There are multiple kinds of "interface servers", and each one will service a -different type of request - one might do WSGI requests, one might handle -WebSockets, or you might have one that handles both. +different type of request - one might do both WebSocket and HTTP requests, while +another might act as an SMS message gateway, for example. These are separate from the "worker servers" where Django will run actual logic, -though, and so you'll need to configure a channel backend to allow the -channels to run over the network. By default, when you're using Django out of -the box, the channel backend is set to an in-memory one that only works in -process; this is enough to serve normal WSGI style requests (``runserver`` is -just running a WSGI interface and a worker in two separate threads), but now we want -WebSocket support we'll need a separate process to keep things clean. +though, and so the *channel layer* transports the content of channels across +the network. In a production scenario, you'd usually run *worker servers* +as a separate cluster from the *interface servers*, though of course you +can run both as separate processes on one machine too. -If you notice, in the example above we switched our default backend to the -database channel backend. This uses two tables -in the database to do message handling, and isn't particularly fast but -requires no extra dependencies. When you deploy to production, you'll want to -use a backend like the Redis backend that has much better throughput. +By default, Django doesn't have a channel layer configured - it doesn't need one to run +normal WSGI requests, after all. As soon as you try to add some consumers, +though, you'll need to configure one. + +In the example above we used the database channel layer implementation +as our default channel layer. This uses two tables +in the ``default`` database to do message handling, and isn't particularly fast but +requires no extra dependencies, so it's handy for development. +When you deploy to production, though, you'll want to +use a backend like the Redis backend that has much better throughput and +lower latency. The second thing, once we have a networked channel backend set up, is to make -sure we're running the WebSocket interface server. Even in development, we need -to do this; ``runserver`` will take care of normal Web requests and running -a worker for us, but WebSockets isn't compatible with WSGI and needs to run -separately. +sure we're running an interface server that's capable of serving WebSockets. +Luckily, installing Channels will also install ``daphne``, an interface server +that can handle both HTTP and WebSockets at the same time, and then ties this +in to run when you run ``runserver`` - you shouldn't notice any difference +from the normal Django ``runserver``, though some of the options may be a little +different. -The easiest way to do this is to use the ``runwsserver`` management command -that ships with Django; just make sure you've installed the latest release -of ``autobahn`` first:: +*(Under the hood, runserver is now running Daphne in one thread and a worker +with autoreload in another - it's basically a miniature version of a deployment, +but all in one process)* - pip install -U autobahn[twisted] - python manage.py runwsserver +Now, let's test our code. Open a browser and put the following into the +JavaScript console to open a WebSocket and send some data down it:: -Run that alongside ``runserver`` and you'll have two interface servers, a -worker thread, and the channel backend all connected and running. You can -even launch separate worker processes with ``runworker`` if you like (you'll -need at least one of those if you're not also running ``runserver``). - -Now, just open a browser and put the following into the JavaScript console -to test your new code:: - - socket = new WebSocket("ws://127.0.0.1:9000"); + // Note that the path doesn't matter right now; any WebSocket + // connection gets bumped over to WebSocket consumers + socket = new WebSocket("ws://127.0.0.1:8000/chat/"); socket.onmessage = function(e) { alert(e.data); } @@ -230,15 +224,16 @@ receive the message and show an alert, as any incoming message is sent to the been put into the ``chat`` group when they connected. Feel free to put some calls to ``print`` in your handler functions too, if you -like, so you can understand when they're called. If you run three or four -copies of ``runworker`` you'll probably be able to see the tasks running -on different workers. +like, so you can understand when they're called. You can also run separate +worker processes with ``manage.py runworker`` as well - if you do this, you +should see some of the consumers being handled in the ``runserver`` thread and +some in the separate worker process. Persisting Data --------------- -Echoing messages is a nice simple example, but it's -skirting around the real design pattern - persistent state for connections. +Echoing messages is a nice simple example, but it's ignoring the real +need for a system like this - persistent state for connections. Let's consider a basic chat site where a user requests a chat room upon initial connection, as part of the query string (e.g. ``wss://host/websocket?room=abc``). @@ -250,8 +245,8 @@ global variables or similar. Instead, the solution is to persist information keyed by the ``reply_channel`` in some other data store - sound familiar? This is what Django's session framework -does for HTTP requests, only there it uses cookies as the lookup key rather -than the ``reply_channel``. +does for HTTP requests, using a cookie as the key. Wouldn't it be useful if +we could get a session using the ``reply_channel`` as a key? Channels provides a ``channel_session`` decorator for this purpose - it provides you with an attribute called ``message.channel_session`` that acts @@ -273,11 +268,6 @@ name in the path of your WebSocket request (we'll ignore auth for now - that's n message.channel_session['room'] = room Group("chat-%s" % room).add(message.reply_channel) - # Connected to websocket.keepalive - @channel_session - def ws_keepalive(message): - Group("chat-%s" % message.channel_session['room']).add(message.reply_channel) - # Connected to websocket.receive @channel_session def ws_message(message): @@ -316,16 +306,16 @@ In addition, we don't want the interface servers storing data or trying to run authentication; they're meant to be simple, lean, fast processes without much state, and so we'll need to do our authentication inside our consumer functions. -Fortunately, because Channels has standardised WebSocket event -:doc:`message-standards`, it ships with decorators that help you with +Fortunately, because Channels has an underlying spec for WebSockets and other +messages (:doc:`ASGI `), it ships with decorators that help you with both authentication and getting the underlying Django session (which is what Django authentication relies on). -Channels can use Django sessions either from cookies (if you're running your websocket -server on the same port as your main site, which requires a reverse proxy that -understands WebSockets), or from a ``session_key`` GET parameter, which -is much more portable, and works in development where you need to run a separate -WebSocket server (by default, on port 9000). +Channels can use Django sessions either from cookies (if you're running your +websocket server on the same port as your main site, using something like Daphne), +or from a ``session_key`` GET parameter, which is works if you want to keep +running your HTTP requests through a WSGI server and offload WebSockets to a +second server process on another port. You get access to a user's normal Django session using the ``http_session`` decorator - that gives you a ``message.http_session`` attribute that behaves @@ -334,12 +324,12 @@ which will provide a ``message.user`` attribute as well as the session attribute Now, one thing to note is that you only get the detailed HTTP information during the ``connect`` message of a WebSocket connection (you can read more -about what you get when in :doc:`message-standards`) - this means we're not +about that in the :doc:`ASGI spec `) - this means we're not wasting bandwidth sending the same information over the wire needlessly. This also means we'll have to grab the user in the connection handler and then store it in the session; thankfully, Channels ships with both a ``channel_session_user`` -decorator that works like the ``http_session_user`` decorator you saw above but +decorator that works like the ``http_session_user`` decorator we mentioned above but loads the user from the *channel* session rather than the *HTTP* session, and a function called ``transfer_user`` which replicates a user from one session to another. @@ -361,12 +351,6 @@ chat to people with the same first letter of their username:: # Add them to the right group Group("chat-%s" % message.user.username[0]).add(message.reply_channel) - # Connected to websocket.keepalive - @channel_session_user - def ws_keepalive(message): - # Keep them in the right group - Group("chat-%s" % message.user.username[0]).add(message.reply_channel) - # Connected to websocket.receive @channel_session_user def ws_message(message): @@ -377,7 +361,9 @@ chat to people with the same first letter of their username:: def ws_disconnect(message): Group("chat-%s" % message.user.username[0]).discard(message.reply_channel) -Now, when we connect to the WebSocket we'll have to remember to provide the +If you're just using ``runserver`` (and so Daphne), you can just connect +and your cookies should transfer your auth over. If you were running WebSockets +on a separate port, you'd have to remember to provide the Django session ID as part of the URL, like this:: socket = new WebSocket("ws://127.0.0.1:9000/?session_key=abcdefg"); @@ -437,11 +423,6 @@ have a ChatMessage model with ``message`` and ``room`` fields:: message.channel_session['room'] = room Group("chat-%s" % room).add(message.reply_channel) - # Connected to websocket.keepalive - @channel_session - def ws_add(message): - Group("chat-%s" % message.channel_session['room']).add(message.reply_channel) - # Connected to websocket.receive @channel_session def ws_message(message): From 1a7010aa2c2cee8b473423d2a3c1e7c42b718422 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Feb 2016 19:15:00 +0000 Subject: [PATCH 167/746] Make _stream work on AsgiRequest --- channels/handler.py | 3 +++ channels/tests/test_request.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/channels/handler.py b/channels/handler.py index 910d6f1..440b39e 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import sys import logging +from io import BytesIO from threading import Lock from django import http @@ -89,6 +90,8 @@ class AsgiRequest(http.HttpRequest): if not chunk.get("more_content", False): break assert isinstance(self._body, six.binary_type), "Body is not bytes" + # Add a stream-a-like for the body + self._stream = BytesIO(self._body) # Other bits self.resolver_match = None diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index ea2fb0f..fe6f76c 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -176,3 +176,23 @@ class RequestTests(SimpleTestCase): self.assertFalse(request._post_parse_error) self.assertEqual(request.POST["title"], "My First Book") self.assertEqual(request.FILES["pdf"].read(), b"FAKEPDFBYTESGOHERE") + + def test_stream(self): + """ + Tests the body stream is emulated correctly. + """ + message = self.make_message({ + "reply_channel": "test", + "http_version": "1.1", + "method": "PUT", + "path": b"/", + "body": b"onetwothree", + "headers": { + "host": b"example.com", + "content-length": b"11", + }, + }, "test") + request = AsgiRequest(message) + self.assertEqual(request.method, "PUT") + self.assertEqual(request.read(3), b"one") + self.assertEqual(request.read(), b"twothree") From a4381f427ed82ace2f6ed21ca4f549313b419767 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Feb 2016 19:24:18 +0000 Subject: [PATCH 168/746] Fix runserver to properly autoreload w/Twisted --- channels/log.py | 1 - channels/management/commands/runserver.py | 18 +++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/channels/log.py b/channels/log.py index 8100ede..8f1ee54 100644 --- a/channels/log.py +++ b/channels/log.py @@ -17,7 +17,6 @@ def setup_logger(name, verbosity=1): logger.addHandler(handler) if verbosity > 1: logger.setLevel(logging.DEBUG) - logger.debug("Logging set to DEBUG") # Set up daphne protocol loggers for module in ["daphne.ws_protocol", "daphne.http_protocol"]: diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index ce081d3..8ed6cdf 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -18,24 +18,25 @@ class Command(RunserverCommand): self.logger = setup_logger('django.channels', self.verbosity) super(Command, self).handle(*args, **options) - def run(self, *args, **options): + def inner_run(self, *args, **options): # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] if not self.channel_layer.registry.consumer_for_channel("http.request"): self.channel_layer.registry.add_consumer(ViewConsumer(), ["http.request"]) - # Helpful note to say this is the Channels runserver - self.logger.info("Worker thread running, channels enabled") + # Report starting up # Launch worker as subthread (including autoreload logic) - worker = WorkerThread(self.channel_layer) + worker = WorkerThread(self.channel_layer, self.logger) worker.daemon = True worker.start() # Launch server in main thread (Twisted doesn't like being in a # subthread, and it doesn't need to autoreload as there's no user code) + self.logger.info("Daphne running, listening on %s:%s", self.addr, self.port) from daphne.server import Server Server( channel_layer=self.channel_layer, host=self.addr, port=int(self.port), + signal_handlers=False, ).run() @@ -44,13 +45,12 @@ class WorkerThread(threading.Thread): Class that runs a worker """ - def __init__(self, channel_layer): + def __init__(self, channel_layer, logger): super(WorkerThread, self).__init__() self.channel_layer = channel_layer + self.logger = logger def run(self): + self.logger.info("Worker thread running") worker = Worker(channel_layer=self.channel_layer) - # We need to set run_main so it doesn't try to relaunch the entire - # program - that will make Daphne run twice. - os.environ["RUN_MAIN"] = "true" - autoreload.main(worker.run) + worker.run() From cd17c88533e2730b0f31aaaf34351fa556b1f465 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Feb 2016 19:28:12 +0000 Subject: [PATCH 169/746] Fix flake errors --- channels/management/commands/runserver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 8ed6cdf..63644b0 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -1,7 +1,5 @@ -import os import threading -from django.utils import autoreload from django.core.management.commands.runserver import \ Command as RunserverCommand From 76611b7e6e97089b93626b472f91c04f16644034 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Feb 2016 19:29:47 +0000 Subject: [PATCH 170/746] Fix up some comments --- channels/management/commands/runserver.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 63644b0..2e775c9 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -21,13 +21,12 @@ class Command(RunserverCommand): self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] if not self.channel_layer.registry.consumer_for_channel("http.request"): self.channel_layer.registry.add_consumer(ViewConsumer(), ["http.request"]) - # Report starting up - # Launch worker as subthread (including autoreload logic) + # Launch worker as subthread worker = WorkerThread(self.channel_layer, self.logger) worker.daemon = True worker.start() - # Launch server in main thread (Twisted doesn't like being in a - # subthread, and it doesn't need to autoreload as there's no user code) + # Launch server in 'main' thread. Signals are disabled as it's still + # actually a subthread under the autoreloader. self.logger.info("Daphne running, listening on %s:%s", self.addr, self.port) from daphne.server import Server Server( From fee6a384831e019f3ef0d39fdebfa54dbddafc2b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Feb 2016 19:50:36 +0000 Subject: [PATCH 171/746] Handle request body encoding --- channels/handler.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 440b39e..85cacf1 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -1,7 +1,9 @@ from __future__ import unicode_literals -import sys +import cgi +import codecs import logging +import sys from io import BytesIO from threading import Lock @@ -33,7 +35,6 @@ class AsgiRequest(http.HttpRequest): self._post_parse_error = False self.resolver_match = None # Path info - # TODO: probably needs actual URL decoding self.path = self.message['path'].decode("ascii") self.script_name = self.message.get('root_path', b'') if self.script_name: @@ -65,8 +66,18 @@ class AsgiRequest(http.HttpRequest): corrected_name = "CONTENT_TYPE" else: corrected_name = 'HTTP_%s' % name.upper().replace("-", "_") - # TODO: Look at request encoding for unicode decode - self.META[corrected_name] = value.decode("latin1") + # HTTPbis say only ASCII chars are allowed in headers + self.META[corrected_name] = value.decode("ascii") + # Pull out request encoding if we find it + if "CONTENT_TYPE" in self.META: + _, content_params = cgi.parse_header(self.META["CONTENT_TYPE"]) + if 'charset' in content_params: + try: + codecs.lookup(content_params['charset']) + except LookupError: + pass + else: + self.encoding = content_params['charset'] # Pull out content length info if self.META.get('CONTENT_LENGTH', None): try: From 41b6750afb3e0f4a95ac456175626705877c337d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Feb 2016 20:26:56 +0000 Subject: [PATCH 172/746] Do much more runserver-like logging --- channels/handler.py | 1 + channels/management/commands/runserver.py | 86 ++++++++++++++++++++--- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 85cacf1..c626b7d 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -10,6 +10,7 @@ from threading import Lock from django import http from django.core import signals from django.core.handlers import base +from django.core.management.color import color_style from django.core.urlresolvers import set_script_prefix from django.utils import six from django.utils.functional import cached_property diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 2e775c9..4ea5f78 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -1,7 +1,12 @@ +import datetime +import sys import threading +from django.conf import settings from django.core.management.commands.runserver import \ Command as RunserverCommand +from django.utils import six +from django.utils.encoding import get_system_encoding from channels import DEFAULT_CHANNEL_LAYER, channel_layers from channels.handler import ViewConsumer @@ -21,21 +26,84 @@ class Command(RunserverCommand): self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] if not self.channel_layer.registry.consumer_for_channel("http.request"): self.channel_layer.registry.add_consumer(ViewConsumer(), ["http.request"]) + # Run checks + self.stdout.write("Performing system checks...\n\n") + self.check(display_num_errors=True) + self.check_migrations() + # Print helpful text + quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C' + now = datetime.datetime.now().strftime('%B %d, %Y - %X') + if six.PY2: + now = now.decode(get_system_encoding()) + self.stdout.write(now) + self.stdout.write(( + "Django version %(version)s, using settings %(settings)r\n" + "Starting Channels development server at http://%(addr)s:%(port)s/\n" + "Quit the server with %(quit_command)s.\n" + ) % { + "version": self.get_version(), + "settings": settings.SETTINGS_MODULE, + "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr, + "port": self.port, + "quit_command": quit_command, + }) + # Launch worker as subthread worker = WorkerThread(self.channel_layer, self.logger) worker.daemon = True worker.start() # Launch server in 'main' thread. Signals are disabled as it's still # actually a subthread under the autoreloader. - self.logger.info("Daphne running, listening on %s:%s", self.addr, self.port) - from daphne.server import Server - Server( - channel_layer=self.channel_layer, - host=self.addr, - port=int(self.port), - signal_handlers=False, - ).run() + self.logger.debug("Daphne running, listening on %s:%s", self.addr, self.port) + try: + from daphne.server import Server + Server( + channel_layer=self.channel_layer, + host=self.addr, + port=int(self.port), + signal_handlers=False, + action_logger=self.log_action, + ).run() + except KeyboardInterrupt: + shutdown_message = options.get('shutdown_message', '') + if shutdown_message: + self.stdout.write(shutdown_message) + return + def log_action(self, protocol, action, details): + """ + Logs various different kinds of requests to the console. + """ + # All start with timestamp + msg = "[%s] " % datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") + # HTTP requests + if protocol == "http" and action == "complete": + msg += "HTTP %(method)s %(path)s %(status)s [%(client)s]\n" % details + # Utilize terminal colors, if available + if 200 <= details['status'] < 300: + # Put 2XX first, since it should be the common case + msg = self.style.HTTP_SUCCESS(msg) + elif 100 <= details['status'] < 200: + msg = self.style.HTTP_INFO(msg) + elif details['status'] == 304: + msg = self.style.HTTP_NOT_MODIFIED(msg) + elif 300 <= details['status'] < 400: + msg = self.style.HTTP_REDIRECT(msg) + elif details['status'] == 404: + msg = self.style.HTTP_NOT_FOUND(msg) + elif 400 <= details['status'] < 500: + msg = self.style.HTTP_BAD_REQUEST(msg) + else: + # Any 5XX, or any other response + msg = self.style.HTTP_SERVER_ERROR(msg) + # Websocket requests + elif protocol == "websocket" and action == "connected": + msg += "WebSocket CONNECT %(path)s [%(client)s]\n" % details + elif protocol == "websocket" and action == "disconnected": + msg += "WebSocket DISCONNECT %(path)s [%(client)s]\n" % details + + + sys.stderr.write(msg) class WorkerThread(threading.Thread): """ @@ -48,6 +116,6 @@ class WorkerThread(threading.Thread): self.logger = logger def run(self): - self.logger.info("Worker thread running") + self.logger.debug("Worker thread running") worker = Worker(channel_layer=self.channel_layer) worker.run() From caca13c2bc868232812fde53ef342d106f70846f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Feb 2016 20:31:50 +0000 Subject: [PATCH 173/746] Remove extra blank line --- channels/management/commands/runserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 4ea5f78..f439c0b 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -102,7 +102,6 @@ class Command(RunserverCommand): elif protocol == "websocket" and action == "disconnected": msg += "WebSocket DISCONNECT %(path)s [%(client)s]\n" % details - sys.stderr.write(msg) class WorkerThread(threading.Thread): From b468ddd930e4f6c70bf6edd196417ecb3c894021 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 10 Feb 2016 20:37:57 +0000 Subject: [PATCH 174/746] More flake fixes --- channels/handler.py | 1 - channels/management/commands/runserver.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index c626b7d..85cacf1 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -10,7 +10,6 @@ from threading import Lock from django import http from django.core import signals from django.core.handlers import base -from django.core.management.color import color_style from django.core.urlresolvers import set_script_prefix from django.utils import six from django.utils.functional import cached_property diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index f439c0b..7ecc3b6 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -104,6 +104,7 @@ class Command(RunserverCommand): sys.stderr.write(msg) + class WorkerThread(threading.Thread): """ Class that runs a worker From 1328c367dc1cdf7c92c9d8e9b8d97afe53aadb7a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 14 Feb 2016 17:11:44 +0000 Subject: [PATCH 175/746] Move default-request handling into common place --- channels/consumer_registry.py | 9 +++++++++ channels/management/commands/runserver.py | 3 +-- channels/management/commands/runworker.py | 10 ++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index 0b7f8d2..69743e0 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -5,6 +5,7 @@ import importlib from django.core.exceptions import ImproperlyConfigured from django.utils import six +from .handler import ViewConsumer from .utils import name_that_thing @@ -75,3 +76,11 @@ class ConsumerRegistry(object): message.reply_channel.send({ "content": message.content.get("content", None), }) + + def check_default(self): + """ + Checks to see if default handlers need to be registered + for channels, and adds them if they need to be. + """ + if not self.consumer_for_channel("http.request"): + self.add_consumer(ViewConsumer(), ["http.request"]) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 7ecc3b6..d78b899 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -24,8 +24,7 @@ class Command(RunserverCommand): def inner_run(self, *args, **options): # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] - if not self.channel_layer.registry.consumer_for_channel("http.request"): - self.channel_layer.registry.add_consumer(ViewConsumer(), ["http.request"]) + self.channel_layer.registry.check_default() # Run checks self.stdout.write("Performing system checks...\n\n") self.check(display_num_errors=True) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 5406548..82ddace 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -13,20 +13,18 @@ class Command(BaseCommand): # Get the backend to use self.verbosity = options.get("verbosity", 1) self.logger = setup_logger('django.channels', self.verbosity) - channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] # Check a handler is registered for http reqs - if not channel_layer.registry.consumer_for_channel("http.request"): - # Register the default one - channel_layer.registry.add_consumer(ViewConsumer(), ["http.request"]) + self.channel_layer.registry.check_default() # Launch a worker - self.logger.info("Running worker against backend %s", channel_layer.alias) + self.logger.info("Running worker against backend %s", self.channel_layer.alias) # Optionally provide an output callback callback = None if self.verbosity > 1: callback = self.consumer_called # Run the worker try: - Worker(channel_layer=channel_layer, callback=callback).run() + Worker(channel_layer=self.channel_layer, callback=callback).run() except KeyboardInterrupt: pass From 943733c447c628bcdd37901e8e6508eeb3ee610c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 14 Feb 2016 18:47:30 +0000 Subject: [PATCH 176/746] Additional logging and --noworker option --- channels/asgi.py | 4 ++++ channels/management/commands/runserver.py | 18 ++++++++++++++---- channels/management/commands/runworker.py | 2 +- channels/utils.py | 2 ++ channels/worker.py | 2 ++ 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/channels/asgi.py b/channels/asgi.py index 7e1629e..d11d0bf 100644 --- a/channels/asgi.py +++ b/channels/asgi.py @@ -5,6 +5,7 @@ from django.conf import settings from django.utils.module_loading import import_string from .consumer_registry import ConsumerRegistry +from .utils import name_that_thing class InvalidChannelLayerError(ValueError): @@ -71,6 +72,9 @@ class ChannelLayerWrapper(object): def __getattr__(self, name): return getattr(self.channel_layer, name) + def __str__(self): + return "%s (%s)" % (self.alias, name_that_thing(self.channel_layer)) + def get_channel_layer(alias="default"): """ diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index d78b899..b5e6bdd 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -16,6 +16,11 @@ from channels.worker import Worker class Command(RunserverCommand): + def add_arguments(self, parser): + super(Command, self).add_arguments(parser) + parser.add_argument('--noworker', action='store_false', dest='run_worker', default=True, + help='Tells Django not to run a worker thread; you\'ll need to run one separately.') + def handle(self, *args, **options): self.verbosity = options.get("verbosity", 1) self.logger = setup_logger('django.channels', self.verbosity) @@ -38,6 +43,7 @@ class Command(RunserverCommand): self.stdout.write(( "Django version %(version)s, using settings %(settings)r\n" "Starting Channels development server at http://%(addr)s:%(port)s/\n" + "Channel layer %(layer)s\n" "Quit the server with %(quit_command)s.\n" ) % { "version": self.get_version(), @@ -45,12 +51,14 @@ class Command(RunserverCommand): "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr, "port": self.port, "quit_command": quit_command, + "layer": self.channel_layer, }) # Launch worker as subthread - worker = WorkerThread(self.channel_layer, self.logger) - worker.daemon = True - worker.start() + if options.get("run_worker", True): + worker = WorkerThread(self.channel_layer, self.logger) + worker.daemon = True + worker.start() # Launch server in 'main' thread. Signals are disabled as it's still # actually a subthread under the autoreloader. self.logger.debug("Daphne running, listening on %s:%s", self.addr, self.port) @@ -60,9 +68,10 @@ class Command(RunserverCommand): channel_layer=self.channel_layer, host=self.addr, port=int(self.port), - signal_handlers=False, + signal_handlers=not options['use_reloader'], action_logger=self.log_action, ).run() + self.logger.debug("Daphne exited") except KeyboardInterrupt: shutdown_message = options.get('shutdown_message', '') if shutdown_message: @@ -118,3 +127,4 @@ class WorkerThread(threading.Thread): self.logger.debug("Worker thread running") worker = Worker(channel_layer=self.channel_layer) worker.run() + self.logger.debug("Worker thread exited") diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 82ddace..40993d8 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -17,7 +17,7 @@ class Command(BaseCommand): # Check a handler is registered for http reqs self.channel_layer.registry.check_default() # Launch a worker - self.logger.info("Running worker against backend %s", self.channel_layer.alias) + self.logger.info("Running worker against backend %s", self.channel_layer) # Optionally provide an output callback callback = None if self.verbosity > 1: diff --git a/channels/utils.py b/channels/utils.py index 2cddb58..770f23e 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -11,4 +11,6 @@ def name_that_thing(thing): return name_that_thing(thing.__class__) if hasattr(thing, "__module__"): return "%s.%s" % (thing.__module__, thing.__name__) + if hasattr(thing, "__class__"): + return name_that_thing(thing.__class__) return repr(thing) diff --git a/channels/worker.py b/channels/worker.py index 5195271..c9567dd 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -25,7 +25,9 @@ class Worker(object): """ channels = self.channel_layer.registry.all_channel_names() while True: + logger.debug("Worker waiting for message") channel, content = self.channel_layer.receive_many(channels, block=True) + logger.debug("Worker got message on %s: repl %s", channel, content.get("reply_channel", "none")) # If no message, stall a little to avoid busy-looping then continue if channel is None: time.sleep(0.01) From 4f60726ec40489fb58725da7a7db730579480cdb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 14 Feb 2016 18:50:05 +0000 Subject: [PATCH 177/746] Remove unused imports --- channels/management/commands/runserver.py | 1 - channels/management/commands/runworker.py | 1 - 2 files changed, 2 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index b5e6bdd..74a7ca1 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -9,7 +9,6 @@ from django.utils import six from django.utils.encoding import get_system_encoding from channels import DEFAULT_CHANNEL_LAYER, channel_layers -from channels.handler import ViewConsumer from channels.log import setup_logger from channels.worker import Worker diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 40993d8..35fd38e 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals from django.core.management import BaseCommand from channels import channel_layers, DEFAULT_CHANNEL_LAYER from channels.log import setup_logger -from channels.handler import ViewConsumer from channels.worker import Worker From be1498768f54eb6f36781894a22a9aad6e51b90f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 14 Feb 2016 19:22:46 +0000 Subject: [PATCH 178/746] HTTP Long Poll finishing off --- channels/handler.py | 39 +++++++++++++++++++++++++++++++-------- channels/worker.py | 1 - docs/asgi.rst | 15 +++++++++++++++ 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 85cacf1..9cb5036 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -23,10 +23,13 @@ class AsgiRequest(http.HttpRequest): dict, and wraps request body handling. """ - # Exception that will cause any handler to skip around response - # transmission and presume something else will do it later. class ResponseLater(Exception): - pass + """ + Exception that will cause any handler to skip around response + transmission and presume something else will do it later. + """ + def __init__(self): + Exception.__init__(self, "Response later") def __init__(self, message): self.message = message @@ -171,17 +174,37 @@ class AsgiHandler(base.BaseHandler): } ) response = http.HttpResponseBadRequest() - except AsgiRequest.ResponseLater: - # The view has promised something else - # will send a response at a later time - return else: - response = self.get_response(request) + try: + response = self.get_response(request) + except AsgiRequest.ResponseLater: + # The view has promised something else + # will send a response at a later time + return # Transform response into messages, which we yield back to caller for message in self.encode_response(response): # TODO: file_to_stream yield message + def process_exception_by_middleware(self, exception, request): + """ + Catches ResponseLater and re-raises it, else tries to delegate + to middleware exception handling. + """ + if isinstance(exception, AsgiRequest.ResponseLater): + raise + else: + return super(AsgiHandler, self).process_exception_by_middleware(exception, request) + + def handle_uncaught_exception(self, request, resolver, exc_info): + """ + Propagates ResponseLater up into the higher handler method, + processes everything else + """ + if issubclass(exc_info[0], AsgiRequest.ResponseLater): + raise + return super(AsgiHandler, self).handle_uncaught_exception(request, resolver, exc_info) + @classmethod def encode_response(cls, response): """ diff --git a/channels/worker.py b/channels/worker.py index c9567dd..8e61d80 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -25,7 +25,6 @@ class Worker(object): """ channels = self.channel_layer.registry.all_channel_names() while True: - logger.debug("Worker waiting for message") channel, content = self.channel_layer.receive_many(channels, block=True) logger.debug("Worker got message on %s: repl %s", channel, content.get("reply_channel", "none")) # If no message, stall a little to avoid busy-looping then continue diff --git a/docs/asgi.rst b/docs/asgi.rst index 019f7a3..fa92e31 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -565,6 +565,21 @@ Keys: server-pushed requests, and applications should not create reply channels. +Disconnect +'''''''''' + +Sent when a HTTP connection is closed. This is mainly useful for long-polling, +where you may have added the response channel to a Group or other set of +channels you want to trigger a reply to when data arrives. + +Channel: ``http.disconnect`` + +Keys: + +* ``reply_channel``: Channel name responses would have been sent on. No longer + valid after this message is sent; all messages to it will be dropped. + + WebSocket --------- From 5c9093ddebfa66b7bd91fa6dc96558d2aa1952cd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 14 Feb 2016 21:26:34 +0000 Subject: [PATCH 179/746] Fix bug with sending empty response --- channels/handler.py | 3 +++ channels/tests/test_handler.py | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/channels/handler.py b/channels/handler.py index 9cb5036..27ff2e4 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -268,6 +268,9 @@ class AsgiHandler(base.BaseHandler): Yields (chunk, last_chunk) tuples. """ position = 0 + if not data: + yield data, True + return while position < len(data): yield ( data[position:position + cls.chunk_size], diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index f94ca03..b637392 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -88,3 +88,30 @@ class HandlerTests(SimpleTestCase): self.assertEqual(reply_messages[0]["more_content"], True) self.assertEqual(reply_messages[1]["content"], b"andhereistherest") self.assertEqual(reply_messages[1].get("more_content", False), False) + + def test_chunk_bytes(self): + """ + Makes sure chunk_bytes works correctly + """ + # Empty string should still return one chunk + result = list(FakeAsgiHandler.chunk_bytes(b"")) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][0], b"") + self.assertEqual(result[0][1], True) + # Below chunk size + result = list(FakeAsgiHandler.chunk_bytes(b"12345678901234567890123456789")) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][0], b"12345678901234567890123456789") + self.assertEqual(result[0][1], True) + # Exactly chunk size + result = list(FakeAsgiHandler.chunk_bytes(b"123456789012345678901234567890")) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][0], b"123456789012345678901234567890") + self.assertEqual(result[0][1], True) + # Just above chunk size + result = list(FakeAsgiHandler.chunk_bytes(b"123456789012345678901234567890a")) + self.assertEqual(len(result), 2) + self.assertEqual(result[0][0], b"123456789012345678901234567890") + self.assertEqual(result[0][1], False) + self.assertEqual(result[1][0], b"a") + self.assertEqual(result[1][1], True) From 31447a4641b11d762acb1dba1aa7cd5272bfd27a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 14 Feb 2016 21:27:12 +0000 Subject: [PATCH 180/746] User auth fixing --- channels/auth.py | 22 +++++++-- channels/decorators.py | 96 --------------------------------------- channels/message.py | 7 ++- channels/sessions.py | 101 +++++++++++++++++++++++++++++++++++++++++ channels/worker.py | 2 +- 5 files changed, 126 insertions(+), 102 deletions(-) create mode 100644 channels/sessions.py diff --git a/channels/auth.py b/channels/auth.py index d0c786a..1ae7e1d 100644 --- a/channels/auth.py +++ b/channels/auth.py @@ -1,8 +1,9 @@ import functools from django.contrib import auth +from django.contrib.auth.models import AnonymousUser -from .decorators import channel_session, http_session +from .sessions import channel_session, http_session def transfer_user(from_session, to_session): @@ -26,7 +27,7 @@ def channel_session_user(func): if not hasattr(message, "channel_session"): raise ValueError("Did not see a channel session to get auth from") if message.channel_session is None: - message.user = None + message.user = AnonymousUser() # Otherwise, be a bit naughty and make a fake Request with just # a "session" attribute (later on, perhaps refactor contrib.auth to # pass around session rather than request) @@ -55,7 +56,7 @@ def http_session_user(func): if not hasattr(message, "http_session"): raise ValueError("Did not see a http session to get auth from") if message.http_session is None: - message.user = None + message.user = AnonymousUser() # Otherwise, be a bit naughty and make a fake Request with just # a "session" attribute (later on, perhaps refactor contrib.auth to # pass around session rather than request) @@ -65,3 +66,18 @@ def http_session_user(func): # Run the consumer return func(message, *args, **kwargs) return inner + + +def channel_session_user_from_http(func): + """ + Decorator that automatically transfers the user from HTTP sessions to + channel-based sessions, and returns the user as message.user as well. + Useful for things that consume e.g. websocket.connect + """ + @http_session_user + @channel_session + def inner(message, *args, **kwargs): + if message.http_session is not None: + transfer_user(message.http_session, message.channel_session) + return func(message, *args, **kwargs) + return inner diff --git a/channels/decorators.py b/channels/decorators.py index 063447a..279a7bd 100644 --- a/channels/decorators.py +++ b/channels/decorators.py @@ -1,8 +1,4 @@ import functools -import hashlib -from importlib import import_module - -from django.conf import settings def linearize(func): @@ -32,95 +28,3 @@ def linearize(func): # TODO: Release lock here pass return inner - - -def channel_session(func): - """ - Provides a session-like object called "channel_session" to consumers - as a message attribute that will auto-persist across consumers with - the same incoming "reply_channel" value. - - Use this to persist data across the lifetime of a connection. - """ - @functools.wraps(func) - def inner(message, *args, **kwargs): - # Make sure there's a reply_channel - if not message.reply_channel: - raise ValueError( - "No reply_channel sent to consumer; @channel_session " + - "can only be used on messages containing it." - ) - - # Make sure there's NOT a channel_session already - if hasattr(message, "channel_session"): - raise ValueError("channel_session decorator wrapped inside another channel_session decorator") - - # Turn the reply_channel into a valid session key length thing. - # We take the last 24 bytes verbatim, as these are the random section, - # and then hash the remaining ones onto the start, and add a prefix - reply_name = message.reply_channel.name - hashed = hashlib.md5(reply_name[:-24].encode()).hexdigest()[:8] - session_key = "skt" + hashed + reply_name[-24:] - # Make a session storage - session_engine = import_module(settings.SESSION_ENGINE) - session = session_engine.SessionStore(session_key=session_key) - # If the session does not already exist, save to force our - # session key to be valid. - if not session.exists(session.session_key): - session.save(must_create=True) - message.channel_session = session - # Run the consumer - try: - return func(message, *args, **kwargs) - finally: - # Persist session if needed - if session.modified: - session.save() - return inner - - -def http_session(func): - """ - Wraps a HTTP or WebSocket connect consumer (or any consumer of messages - that provides a "cookies" or "get" attribute) to provide a "http_session" - attribute that behaves like request.session; that is, it's hung off of - a per-user session key that is saved in a cookie or passed as the - "session_key" GET parameter. - - It won't automatically create and set a session cookie for users who - don't have one - that's what SessionMiddleware is for, this is a simpler - read-only version for more low-level code. - - If a message does not have a session we can inflate, the "session" attribute - will be None, rather than an empty session you can write to. - """ - @functools.wraps(func) - def inner(message, *args, **kwargs): - if "cookies" not in message.content and "get" not in message.content: - raise ValueError("No cookies or get sent to consumer - cannot initialise http_session") - # Make sure there's NOT a http_session already - if hasattr(message, "http_session"): - raise ValueError("http_session decorator wrapped inside another http_session decorator") - # Make sure there's a session key - session_key = None - if "get" in message.content: - try: - session_key = message.content['get'].get("session_key", [])[0] - except IndexError: - pass - if "cookies" in message.content and session_key is None: - session_key = message.content['cookies'].get(settings.SESSION_COOKIE_NAME) - # Make a session storage - if session_key: - session_engine = import_module(settings.SESSION_ENGINE) - session = session_engine.SessionStore(session_key=session_key) - else: - session = None - message.http_session = session - # Run the consumer - result = func(message, *args, **kwargs) - # Persist session if needed (won't be saved if error happens) - if session is not None and session.modified: - session.save() - return result - return inner diff --git a/channels/message.py b/channels/message.py index 53a36cc..5830fdd 100644 --- a/channels/message.py +++ b/channels/message.py @@ -12,9 +12,12 @@ class Message(object): to use to reply to this message's end user, if that makes sense. """ - def __init__(self, content, channel, channel_layer): + def __init__(self, content, channel_name, channel_layer): self.content = content - self.channel = channel + self.channel = Channel( + channel_name, + channel_layer=channel_layer, + ) self.channel_layer = channel_layer if content.get("reply_channel", None): self.reply_channel = Channel( diff --git a/channels/sessions.py b/channels/sessions.py new file mode 100644 index 0000000..598b79e --- /dev/null +++ b/channels/sessions.py @@ -0,0 +1,101 @@ +import functools +import hashlib +from importlib import import_module + +from django.conf import settings + +from .handler import AsgiRequest + + +def channel_session(func): + """ + Provides a session-like object called "channel_session" to consumers + as a message attribute that will auto-persist across consumers with + the same incoming "reply_channel" value. + + Use this to persist data across the lifetime of a connection. + """ + @functools.wraps(func) + def inner(message, *args, **kwargs): + # Make sure there's a reply_channel + if not message.reply_channel: + raise ValueError( + "No reply_channel sent to consumer; @channel_session " + + "can only be used on messages containing it." + ) + + # Make sure there's NOT a channel_session already + if hasattr(message, "channel_session"): + raise ValueError("channel_session decorator wrapped inside another channel_session decorator") + + # Turn the reply_channel into a valid session key length thing. + # We take the last 24 bytes verbatim, as these are the random section, + # and then hash the remaining ones onto the start, and add a prefix + reply_name = message.reply_channel.name + hashed = hashlib.md5(reply_name[:-24].encode()).hexdigest()[:8] + session_key = "skt" + hashed + reply_name[-24:] + # Make a session storage + session_engine = import_module(settings.SESSION_ENGINE) + session = session_engine.SessionStore(session_key=session_key) + # If the session does not already exist, save to force our + # session key to be valid. + if not session.exists(session.session_key): + session.save(must_create=True) + message.channel_session = session + # Run the consumer + try: + return func(message, *args, **kwargs) + finally: + # Persist session if needed + if session.modified: + session.save() + return inner + + +def http_session(func): + """ + Wraps a HTTP or WebSocket connect consumer (or any consumer of messages + that provides a "cookies" or "get" attribute) to provide a "http_session" + attribute that behaves like request.session; that is, it's hung off of + a per-user session key that is saved in a cookie or passed as the + "session_key" GET parameter. + + It won't automatically create and set a session cookie for users who + don't have one - that's what SessionMiddleware is for, this is a simpler + read-only version for more low-level code. + + If a message does not have a session we can inflate, the "session" attribute + will be None, rather than an empty session you can write to. + """ + @functools.wraps(func) + def inner(message, *args, **kwargs): + try: + # We want to parse the WebSocket (or similar HTTP-lite) message + # to get cookies and GET, but we need to add in a few things that + # might not have been there. + if "method" not in message.content: + message.content['method'] = "FAKE" + request = AsgiRequest(message) + except Exception as e: + raise ValueError("Cannot parse HTTP message - are you sure this is a HTTP consumer? %s" % e) + # Make sure there's NOT a http_session already + if hasattr(message, "http_session"): + raise ValueError("http_session decorator wrapped inside another http_session decorator") + # Make sure there's a session key + session_key = request.GET.get("session_key", None) + if session_key is None: + session_key = request.COOKIES.get(settings.SESSION_COOKIE_NAME, None) + # Make a session storage + if session_key: + session_engine = import_module(settings.SESSION_ENGINE) + session = session_engine.SessionStore(session_key=session_key) + else: + session = None + message.http_session = session + # Run the consumer + result = func(message, *args, **kwargs) + # Persist session if needed (won't be saved if error happens) + if session is not None and session.modified: + session.save() + return result + return inner diff --git a/channels/worker.py b/channels/worker.py index 8e61d80..ba0ae35 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -34,7 +34,7 @@ class Worker(object): # Create message wrapper message = Message( content=content, - channel=channel, + channel_name=channel, channel_layer=self.channel_layer, ) # Handle the message From 171b9d8552ee80d3d8615211451aa2862f696b8f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Feb 2016 22:47:35 +0000 Subject: [PATCH 181/746] Add META['SCRIPT_NAME'] for backwards compat --- channels/handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/handler.py b/channels/handler.py index 27ff2e4..80faec1 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -50,6 +50,7 @@ class AsgiRequest(http.HttpRequest): self.META = { "REQUEST_METHOD": self.method, "QUERY_STRING": self.message.get('query_string', b'').decode("ascii"), + "SCRIPT_NAME": self.script_name.decode("ascii"), # Old code will need these for a while "wsgi.multithread": True, "wsgi.multiprocess": True, From 267e56ce2afe2959c7aa3d2534727d4602dc8ab8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Feb 2016 22:47:48 +0000 Subject: [PATCH 182/746] Make transfer_user fail silently if no user --- channels/auth.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/channels/auth.py b/channels/auth.py index 1ae7e1d..a8c8a05 100644 --- a/channels/auth.py +++ b/channels/auth.py @@ -10,9 +10,10 @@ def transfer_user(from_session, to_session): """ Transfers user from HTTP session to channel session. """ - to_session[auth.BACKEND_SESSION_KEY] = from_session[auth.BACKEND_SESSION_KEY] - to_session[auth.SESSION_KEY] = from_session[auth.SESSION_KEY] - to_session[auth.HASH_SESSION_KEY] = from_session[auth.HASH_SESSION_KEY] + if auth.BACKEND_SESSION_KEY in from_session and auth.SESSION_KEY in from_session and auth.HASH_SESSION_KEY in from_session: + to_session[auth.BACKEND_SESSION_KEY] = from_session[auth.BACKEND_SESSION_KEY] + to_session[auth.SESSION_KEY] = from_session[auth.SESSION_KEY] + to_session[auth.HASH_SESSION_KEY] = from_session[auth.HASH_SESSION_KEY] def channel_session_user(func): From 64fe0cb77f29095e9b6d04bfb2013146a8d8ac14 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Feb 2016 22:56:10 +0000 Subject: [PATCH 183/746] Add --noasgi option to runserver to run the old WSGI server instead --- channels/management/commands/runserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 74a7ca1..feb2436 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -19,6 +19,8 @@ class Command(RunserverCommand): super(Command, self).add_arguments(parser) parser.add_argument('--noworker', action='store_false', dest='run_worker', default=True, help='Tells Django not to run a worker thread; you\'ll need to run one separately.') + parser.add_argument('--noasgi', action='store_false', dest='use_asgi', default=True, + help='Run the old WSGI-based runserver rather than the ASGI-based one') def handle(self, *args, **options): self.verbosity = options.get("verbosity", 1) @@ -26,6 +28,9 @@ class Command(RunserverCommand): super(Command, self).handle(*args, **options) def inner_run(self, *args, **options): + # Maybe they want the wsgi one? + if not options.get("use_asgi", True): + return RunserverCommand.inner_run(self, *args, **options) # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] self.channel_layer.registry.check_default() From d180f07b54004dea4acc8af34b9b3c31f98b437c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Feb 2016 22:59:29 +0000 Subject: [PATCH 184/746] Wrap long line --- channels/auth.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/channels/auth.py b/channels/auth.py index a8c8a05..0565446 100644 --- a/channels/auth.py +++ b/channels/auth.py @@ -10,7 +10,9 @@ def transfer_user(from_session, to_session): """ Transfers user from HTTP session to channel session. """ - if auth.BACKEND_SESSION_KEY in from_session and auth.SESSION_KEY in from_session and auth.HASH_SESSION_KEY in from_session: + if auth.BACKEND_SESSION_KEY in from_session and \ + auth.SESSION_KEY in from_session and \ + auth.HASH_SESSION_KEY in from_session: to_session[auth.BACKEND_SESSION_KEY] = from_session[auth.BACKEND_SESSION_KEY] to_session[auth.SESSION_KEY] = from_session[auth.SESSION_KEY] to_session[auth.HASH_SESSION_KEY] = from_session[auth.HASH_SESSION_KEY] From e666355fad91783597464be822259ee9328e61e7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Feb 2016 23:31:27 +0000 Subject: [PATCH 185/746] Remove old echo code and allow configuring of http handler --- channels/consumer_registry.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py index 69743e0..7cd7a57 100644 --- a/channels/consumer_registry.py +++ b/channels/consumer_registry.py @@ -19,8 +19,6 @@ class ConsumerRegistry(object): def __init__(self, routing=None): self.consumers = {} - # Add basic internal consumers - self.add_consumer(self.echo_consumer, ["__channels__.echo"]) # Initialise with any routing that was passed in if routing: # If the routing was a string, import it @@ -69,18 +67,10 @@ class ConsumerRegistry(object): except KeyError: return None - def echo_consumer(self, message): - """ - Implements the echo message standard. - """ - message.reply_channel.send({ - "content": message.content.get("content", None), - }) - - def check_default(self): + def check_default(self, http_consumer=None): """ Checks to see if default handlers need to be registered for channels, and adds them if they need to be. """ if not self.consumer_for_channel("http.request"): - self.add_consumer(ViewConsumer(), ["http.request"]) + self.add_consumer(http_consumer or ViewConsumer(), ["http.request"]) From 247a26b91a9e6311a2edb2dedc71fb7ebf2d8f7b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Feb 2016 23:32:01 +0000 Subject: [PATCH 186/746] Fix streaming responses (e.g. staticfiles.serve) --- channels/handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index 80faec1..e41007e 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -242,8 +242,10 @@ class AsgiHandler(base.BaseHandler): # Streaming responses need to be pinned to their iterator if response.streaming: for part in response.streaming_content: - for chunk in cls.chunk_bytes(part): + for chunk, more in cls.chunk_bytes(part): message['content'] = chunk + # We ignore "more" as there may be more parts; instead, + # we use an empty final closing message with False. message['more_content'] = True yield message message = {} From 93d1aa9b747a35aa71067a407824b1ffa0ad9762 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Feb 2016 23:32:29 +0000 Subject: [PATCH 187/746] Add staticfiles support and launch multiple serving workers --- channels/management/commands/runserver.py | 28 +++++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index feb2436..79fd42a 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -9,7 +9,9 @@ from django.utils import six from django.utils.encoding import get_system_encoding from channels import DEFAULT_CHANNEL_LAYER, channel_layers +from channels.handler import ViewConsumer from channels.log import setup_logger +from channels.staticfiles import StaticFilesConsumer from channels.worker import Worker @@ -33,7 +35,9 @@ class Command(RunserverCommand): return RunserverCommand.inner_run(self, *args, **options) # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] - self.channel_layer.registry.check_default() + self.channel_layer.registry.check_default( + http_consumer=self.get_consumer(), + ) # Run checks self.stdout.write("Performing system checks...\n\n") self.check(display_num_errors=True) @@ -58,11 +62,12 @@ class Command(RunserverCommand): "layer": self.channel_layer, }) - # Launch worker as subthread + # Launch workers as subthreads if options.get("run_worker", True): - worker = WorkerThread(self.channel_layer, self.logger) - worker.daemon = True - worker.start() + for _ in range(4): + worker = WorkerThread(self.channel_layer, self.logger) + worker.daemon = True + worker.start() # Launch server in 'main' thread. Signals are disabled as it's still # actually a subthread under the autoreloader. self.logger.debug("Daphne running, listening on %s:%s", self.addr, self.port) @@ -116,6 +121,19 @@ class Command(RunserverCommand): sys.stderr.write(msg) + def get_consumer(self, *args, **options): + """ + Returns the static files serving handler wrapping the default handler, + if static files should be served. Otherwise just returns the default + handler. + """ + use_static_handler = options.get('use_static_handler', True) + insecure_serving = options.get('insecure_serving', False) + if use_static_handler and (settings.DEBUG or insecure_serving): + return StaticFilesConsumer() + else: + return ViewConsumer() + class WorkerThread(threading.Thread): """ From d18c715f8f0d86d58fcec4df8710f0370ff87308 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Feb 2016 23:33:14 +0000 Subject: [PATCH 188/746] Fix core logging when no message on channel --- channels/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/worker.py b/channels/worker.py index ba0ae35..01e3891 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -26,12 +26,12 @@ class Worker(object): channels = self.channel_layer.registry.all_channel_names() while True: channel, content = self.channel_layer.receive_many(channels, block=True) - logger.debug("Worker got message on %s: repl %s", channel, content.get("reply_channel", "none")) # If no message, stall a little to avoid busy-looping then continue if channel is None: time.sleep(0.01) continue # Create message wrapper + logger.debug("Worker got message on %s: repl %s", channel, content.get("reply_channel", "none")) message = Message( content=content, channel_name=channel, From 8221bbc0beafca4fe43787ad26348f69be73c224 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 20 Feb 2016 23:33:27 +0000 Subject: [PATCH 189/746] Staticfiles handler and consumer --- channels/staticfiles.py | 65 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 channels/staticfiles.py diff --git a/channels/staticfiles.py b/channels/staticfiles.py new file mode 100644 index 0000000..8169be8 --- /dev/null +++ b/channels/staticfiles.py @@ -0,0 +1,65 @@ +from __future__ import unicode_literals + +from django.conf import settings +from django.contrib.staticfiles import utils +from django.contrib.staticfiles.views import serve +from django.utils.six.moves.urllib.parse import urlparse +from django.utils.six.moves.urllib.request import url2pathname + +from .handler import AsgiHandler, ViewConsumer + + +class StaticFilesHandler(AsgiHandler): + """ + Wrapper handler that serves the static files directory. + """ + + def __init__(self, *args, **kwargs): + super(StaticFilesHandler, self).__init__() + self.base_url = urlparse(self.get_base_url()) + + def get_base_url(self): + utils.check_settings() + return settings.STATIC_URL + + def _should_handle(self, path): + """ + Checks if the path should be handled. Ignores the path if: + + * the host is provided as part of the base_url + * the request's path isn't under the media path (or equal) + """ + return path.startswith(self.base_url[2]) and not self.base_url[1] + + def file_path(self, url): + """ + Returns the relative path to the media file on disk for the given URL. + """ + relative_url = url[len(self.base_url[2]):] + return url2pathname(relative_url) + + def serve(self, request): + """ + Actually serves the request path. + """ + return serve(request, self.file_path(request.path), insecure=True) + + def get_response(self, request): + from django.http import Http404 + + if self._should_handle(request.path): + try: + return self.serve(request) + except Http404 as e: + if settings.DEBUG: + from django.views import debug + return debug.technical_404_response(request, e) + return super(StaticFilesHandler, self).get_response(request) + + +class StaticFilesConsumer(ViewConsumer): + """ + Overrides standard view consumer with our new handler + """ + + handler_class = StaticFilesHandler From 973b8b72ad827b35454d5b1c6b040a8a4025d86e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 21 Feb 2016 12:31:16 +0000 Subject: [PATCH 190/746] Show time taken on runserver output --- channels/management/commands/runserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 79fd42a..b056d15 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -95,7 +95,7 @@ class Command(RunserverCommand): msg = "[%s] " % datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") # HTTP requests if protocol == "http" and action == "complete": - msg += "HTTP %(method)s %(path)s %(status)s [%(client)s]\n" % details + msg += "HTTP %(method)s %(path)s %(status)s [%(time_taken).2f, %(client)s]\n" % details # Utilize terminal colors, if available if 200 <= details['status'] < 300: # Put 2XX first, since it should be the common case From 3b8feb5b96ce88a92a2676b54b949dc010d90b0b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 21 Feb 2016 13:06:33 +0000 Subject: [PATCH 191/746] Update deployment docs --- docs/deploying.rst | 79 ++++++++++++++++++++++------------------ docs/getting-started.rst | 6 +-- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 1d133c4..73b0308 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -34,19 +34,22 @@ serve as the communication layer - for example, the Redis backend connects to a Redis server. All this goes into the ``CHANNEL_BACKENDS`` setting; here's an example for a remote Redis server:: - CHANNEL_BACKENDS = { + CHANNEL_LAYERS = { "default": { - "BACKEND": "channels.backends.redis_py.RedisChannelBackend", - "HOSTS": [("redis-channel", 6379)], + "BACKEND": "asgi_redis.RedisChannelLayer", + "CONFIG": { + "hosts": [("redis-server-name", 6379)], + }, + "ROUTING": "my_project.routing.channel_routing", }, } -To use the Redis backend you have to install the redis package:: +To use the Redis backend you have to install it:: - pip install -U redis + pip install -U asgi_redis -Make sure the same setting file is used across all your workers, interfaces +Make sure the same settings file is used across all your workers, interfaces and WSGI apps; without it, they won't be able to talk to each other and things will just fail to work. @@ -88,44 +91,44 @@ do the work of taking incoming requests and loading them into the channels system. You can just keep running your Django code as a WSGI app if you like, behind -something like uwsgi or gunicorn, and just use the WSGI interface as the app -you load into the server - just set it to use -``channels.interfaces.wsgi:WSGIHandler``. +something like uwsgi or gunicorn; this won't let you support WebSockets, though. +Still, if you want to use a WSGI server and have it talk to a worker server +cluster on the backend, see :ref:`wsgi-to-asgi`. -If you want to support WebSockets, however, you'll need to run another -interface server, as the WSGI protocol has no support for WebSockets. -Channels ships with an Autobahn-based WebSocket interface server -that should suit your needs; however, you could also use a third-party -interface server or write one yourself, as long as it follows the -:doc:`message-standards`. +If you want to support WebSockets, long-poll HTTP requests and other Channels +features, you'll need to run a native ASGI interface server, as the WSGI +specification has no support for running these kinds of requests concurrenctly. +Channels ships with an interface server that we recommend you use called +*Daphne*; it supports WebSockets, long-poll HTTP requests, HTTP/2 *(soon)* +and performs quite well. Of course, any ASGI-compliant server will work! -Notably, it's possible to combine more than one protocol into the same -interface server, and the one Channels ships with does just this; it can -optionally serve HTTP requests as well as WebSockets, though by default -it will just serve WebSockets and assume you're routing requests to the right -kind of server using your load balancer or reverse proxy. +Notably, Daphne has a nice feature where it supports all of these protocols on +the same port and on all paths; it auto-negotiates between HTTP and WebSocket, +so there's no need to have your WebSockets on a separate port or path (and +they'll be able to share cookies with your normal view code). -To run a normal WebSocket server, just run:: +To run Daphne, it just needs to be supplied with a channel backend; +first, make sure your project has an ``asgi.py`` file that looks like this +(it should live next to ``wsgi.py``):: - python manage.py runwsserver + import os + from channels.asgi import get_channel_layer + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "my_project.settings") + + channel_layer = get_channel_layer() + +Then, you can run Daphne and supply the channel layer as the argument: + + daphne my_project.asgi:channel_layer Like ``runworker``, you should place this inside an init system or something like supervisord to ensure it is re-run if it exits unexpectedly. -If you want to enable serving of normal HTTP requests as well, just run:: - - python manage.py runwsserver --accept-all - -This interface server is built on in-process asynchronous solutions -(Twisted for Python 2, and asyncio for Python 3) and so should be able to -handle a lot of simultaneous connections. That said, you should still plan to -run a cluster of them and load-balance between them; the per-connection memory -overhead is moderately high. - -Finally, note that it's entirely possible for interface servers to be written -in a language other than Python, though this would mean they could not take -advantage of the channel backend abstraction code and so they'd likely be -custom-written for a single channel backend. +If you only run Daphne and no workers, all of your page requests will seem to +hang forever; that's because Daphne doesn't have any worker servers to handle +the request and it's waiting for one to appear (while ``runserver`` also uses +Daphne, it launches a worker thread along with it in the same process). Deploying new versions of code @@ -145,3 +148,7 @@ There's no need to restart the WSGI or WebSocket interface servers unless you've upgraded your version of Channels or changed any settings; none of your code is used by them, and all middleware and code that can customise requests is run on the consumers. + +You can even use different Python versions for the interface servers and the +workers; the ASGI protocol that channel layers communicate over +is designed to be very portable and network-transparent. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index d66a98a..f3279a1 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -332,7 +332,8 @@ store it in the session; thankfully, Channels ships with both a ``channel_sessio decorator that works like the ``http_session_user`` decorator we mentioned above but loads the user from the *channel* session rather than the *HTTP* session, and a function called ``transfer_user`` which replicates a user from one session -to another. +to another. Even better, it combines all of these into a ``channel_session_user_from_http`` +decorator. Bringing that all together, let's make a chat server where users can only chat to people with the same first letter of their username:: @@ -343,8 +344,7 @@ chat to people with the same first letter of their username:: from channels.auth import http_session_user, channel_session_user, transfer_user # Connected to websocket.connect - @channel_session - @http_session_user + @channel_session_user_from_http def ws_add(message): # Copy user from HTTP to channel session transfer_user(message.http_session, message.channel_session) From 73f840432da674393302103dda35cf53c70c09b8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 21 Feb 2016 13:16:03 +0000 Subject: [PATCH 192/746] Version 0.9 --- CHANGELOG.txt | 26 ++++++++++++++++++++++++++ channels/__init__.py | 2 +- setup.py | 8 +++++--- 3 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 CHANGELOG.txt diff --git a/CHANGELOG.txt b/CHANGELOG.txt new file mode 100644 index 0000000..cd5ada9 --- /dev/null +++ b/CHANGELOG.txt @@ -0,0 +1,26 @@ +0.9 (2016-02-21) +---------------- + +* Staticfiles support in runserver + +* Runserver logs requests and WebSocket connections to console + +* Runserver autoreloads correctly + +* --noasgi option on runserver to use the old WSGI-based server + +* --noworker option on runserver to make it not launch worker threads + +* Streaming responses work correctly + +* Authentication decorators work again with new ASGI spec + +* channel_session_user_from_http decorator introduced + +* HTTP Long Poll support (raise ResponseLater) + +* Handle non-latin1 request body encoding + +* ASGI conformance tests for built-in database backend + +* Moved some imports around for more sensible layout diff --git a/channels/__init__.py b/channels/__init__.py index f37e0ef..b59f85b 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.8" +__version__ = "0.9" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/setup.py b/setup.py index 9e393b5..767b93f 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ from setuptools import find_packages, setup +from channels import __version__ setup( name='channels', - version="0.8", + version=__version__, url='http://github.com/andrewgodwin/django-channels', author='Andrew Godwin', author_email='andrew@aeracode.org', @@ -11,7 +12,8 @@ setup( packages=find_packages(), include_package_data=True, install_requires=[ - 'Django', - 'asgiref', + 'Django>=1.7', + 'asgiref>=0.9', + 'daphne>=0.9', ] ) From ca0d9e9651e51797ad317e54c58a174bcb351610 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 21 Feb 2016 13:18:57 +0000 Subject: [PATCH 193/746] Fix version import during pip install --- channels/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/channels/__init__.py b/channels/__init__.py index b59f85b..8103ab0 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -3,5 +3,8 @@ __version__ = "0.9" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' -from .asgi import channel_layers # NOQA isort:skip -from .channel import Channel, Group # NOQA isort:skip +try: + from .asgi import channel_layers # NOQA isort:skip + from .channel import Channel, Group # NOQA isort:skip +except ImportError: # No django installed, allow vars to be read + pass From 307fc6c0bbf1c0a4fee2f006646b839717856570 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 21 Feb 2016 13:19:57 +0000 Subject: [PATCH 194/746] Release 0.9.1 --- CHANGELOG.txt | 5 +++++ channels/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index cd5ada9..67c519b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +0.9.1 (2016-02-21) +------------------ + +* Fix packaging issues with previous release + 0.9 (2016-02-21) ---------------- diff --git a/channels/__init__.py b/channels/__init__.py index 8103ab0..62454f9 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.9" +__version__ = "0.9.1" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From f04dcee7f71914fbc8cf638a2f8b64bcfe2a7464 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 22 Feb 2016 14:07:48 +0000 Subject: [PATCH 195/746] "In short" and "WSGI to ASGI" doc sections --- docs/concepts.rst | 2 + docs/deploying.rst | 25 ++++++++++- docs/getting-started.rst | 2 + docs/index.rst | 3 ++ docs/inshort.rst | 90 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 docs/inshort.rst diff --git a/docs/concepts.rst b/docs/concepts.rst index a4b865b..3364d7b 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -21,6 +21,8 @@ view is called. Let's look at what *channels* are first. +.. _what-are-channels: + What is a channel? ------------------ diff --git a/docs/deploying.rst b/docs/deploying.rst index 73b0308..036b467 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -118,7 +118,7 @@ first, make sure your project has an ``asgi.py`` file that looks like this channel_layer = get_channel_layer() -Then, you can run Daphne and supply the channel layer as the argument: +Then, you can run Daphne and supply the channel layer as the argument:: daphne my_project.asgi:channel_layer @@ -152,3 +152,26 @@ customise requests is run on the consumers. You can even use different Python versions for the interface servers and the workers; the ASGI protocol that channel layers communicate over is designed to be very portable and network-transparent. + + +.. _wsgi-to-asgi: + +Running ASGI under WSGI +----------------------- + +ASGI is a relatively new specification, and so it's backwards compatible with +WSGI servers with an adapter layer. You won't get WebSocket support this way - +WSGI doesn't support WebSockets - but you can run a separate ASGI server to +handle WebSockets if you want. + +The ``wsgiref`` package contains the adapter; all you need to do is put this +in your Django project's ``wsgi.py`` to declare a new WSGI application object +that backs onto ASGI underneath:: + + import os + from asgiref.wsgi import WsgiToAsgiAdapter + from channels.asgi import get_channel_layer + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_test.settings") + channel_layer = get_channel_layer() + application = WsgiToAsgiAdapter(channel_layer) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index f3279a1..833b549 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -127,6 +127,8 @@ Now, that's taken care of adding and removing WebSocket send channels for the we're not going to store a history of messages or anything and just replay any message sent in to all connected clients. Here's all the code:: +.. _websocket-example: + # In consumers.py from channels import Group diff --git a/docs/index.rst b/docs/index.rst index c77d205..881089d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,8 @@ data model underlying Channels and how they're used inside Django. Then, read :doc:`getting-started` to see how to get up and running with WebSockets with only 30 lines of code. +If you want a quick overview, start with :doc:`inshort`. + You can find the Channels repository `on GitHub `_. Contents: @@ -22,6 +24,7 @@ Contents: .. toctree:: :maxdepth: 2 + inshort concepts installation getting-started diff --git a/docs/inshort.rst b/docs/inshort.rst new file mode 100644 index 0000000..b4657a6 --- /dev/null +++ b/docs/inshort.rst @@ -0,0 +1,90 @@ +In Short +======== + + +What is Channels? +----------------- + +Channels extends Django to add :ref:`a new layer ` +that allows two important features: + +* WebSocket handling, in a way very :ref:`similar to normal views ` +* Background tasks, running in the same servers as the rest of Django + + +How? +---- + +It separates Django into two process types: + +* One that handles HTTP and WebSockets +* One that runs views, websocket handlers and background tasks (*consumers*) + +They communicate via a protocol called :doc:`ASGI `, which is similar +to WSGI but runs over a network and allows for more protocol types. + +Channels does not introduce asyncio, gevent, or any other async code to +your Django code; all of your business logic runs synchronously in a worker +process or thread. + + +I have to change how I run Django? +---------------------------------- + +No, all the new stuff is entirely optional. If you want it, however, you'll +change from running Django under a WSGI server, to running: + +* An ASGI server, like `Daphne ` +* Django worker servers, using ``manage.py runworker`` +* Something to route ASGI requests over, like Redis or a database. + + +What else does Channels give me? +-------------------------------- + +Other features include: + +* Easy HTTP long-poll support for thousands of clients at once +* Full session and auth support for WebSockets +* Automatic user login for WebSockets based on site cookies +* Built-in primitives for mass triggering of events (chat, live blogs, etc.) +* Zero-downtime deployment with browsers paused while new workers spin up +* Optional low-level HTTP control on a per-URL basis +* Extendability to other protocols or event sources (e.g. WebRTC, raw UDP, SMS) + + +Does it scale? +-------------- + +Yes, you can run any number of *protocol servers* (ones that serve HTTP +and WebSockets) and *worker servers* (ones that run your Django code) to +fit your use case. + +The ASGI spec allows a number of different *channel layers* to be plugged in +between these two components, with difference performance characteristics, and +it's designed to allow both easy sharding as well as the ability to run +separate clusters with their own protocol and worker servers. + + +Do I need to worry about making all my code async-friendly? +----------------------------------------------------------- + +No, all your code runs synchronously without any sockets or event loops to +block. You can use async code within a Django view or channel consumer if you +like - for example, to fetch lots of URLs in parallel - but it doesn't +affect the overall deployed site. + + +What version of Django does it work with? +----------------------------------------- + +You can install Channels as a library for Django 1.8 and 1.9, and it (should be) +part of Django 1.10. It has a few extra dependencies, but these will all +be installed if you use `pip`. + + +What do I read next? +-------------------- + +Start off by reading about the :doc:`concepts underlying Channels `, +and then move on to read our example-laden :doc:`Getting Started guide `. From 2b486b0ef065a654e4da9a6ea7407f56beb8d4a0 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 22 Feb 2016 14:10:24 +0000 Subject: [PATCH 196/746] Fix broken bits in docs. --- docs/asgi.rst | 2 +- docs/getting-started.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index fa92e31..c68611e 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -286,7 +286,7 @@ A *channel layer* should provide an object with these attributes * ``extensions``, a list of unicode string names indicating which extensions this layer provides, or empty if it supports none. - The only valid extension names are ``groups``, ``flush`` and ``statistics`. + The only valid extension names are ``groups``, ``flush`` and ``statistics``. A channel layer implementing the ``groups`` extension must also provide: diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 833b549..029666e 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -122,13 +122,13 @@ handler:: def ws_disconnect(message): Group("chat").discard(message.reply_channel) +.. _websocket-example: + Now, that's taken care of adding and removing WebSocket send channels for the ``chat`` group; all we need to do now is take care of message sending. For now, we're not going to store a history of messages or anything and just replay any message sent in to all connected clients. Here's all the code:: -.. _websocket-example: - # In consumers.py from channels import Group From 2a1c15d3c27197f8b5df2cce78cdc996dac39cbc Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 22 Feb 2016 14:13:00 +0000 Subject: [PATCH 197/746] Couple of errors in the In Short doc. --- docs/inshort.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/inshort.rst b/docs/inshort.rst index b4657a6..070a0eb 100644 --- a/docs/inshort.rst +++ b/docs/inshort.rst @@ -34,7 +34,7 @@ I have to change how I run Django? No, all the new stuff is entirely optional. If you want it, however, you'll change from running Django under a WSGI server, to running: -* An ASGI server, like `Daphne ` +* An ASGI server, probably `Daphne `_ * Django worker servers, using ``manage.py runworker`` * Something to route ASGI requests over, like Redis or a database. @@ -80,7 +80,7 @@ What version of Django does it work with? You can install Channels as a library for Django 1.8 and 1.9, and it (should be) part of Django 1.10. It has a few extra dependencies, but these will all -be installed if you use `pip`. +be installed if you use ``pip``. What do I read next? From 9a9eb26d36a71cb46f07506418a6a4de132bb1f9 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Feb 2016 21:39:37 +0000 Subject: [PATCH 198/746] Ignore SQLite database files from tests --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 91b64df..4261b15 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ dist/ docs/_build __pycache__/ +*.sqlite3 .tox/ *.swp *.pyc From ca4e3f0af5ebd3daf0a8575369b023ad4610689d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Feb 2016 21:40:30 +0000 Subject: [PATCH 199/746] Make benchmarker work properly --- testproject/benchmark.py | 141 +++++++++++++++++++--------- testproject/testproject/settings.py | 6 +- testproject/testproject/urls.py | 2 +- 3 files changed, 100 insertions(+), 49 deletions(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index dcdcbb4..3f0fa52 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -1,11 +1,11 @@ -import random +from __future__ import unicode_literals +import time +import random from autobahn.twisted.websocket import WebSocketClientProtocol, \ WebSocketClientFactory -NUM_CONNECTIONS = 100 -PER_SECOND = 10 stats = {} @@ -15,84 +15,133 @@ class MyClientProtocol(WebSocketClientProtocol): message_gap = 1 def onConnect(self, response): + self.opened = time.time() self.sent = 0 + self.last_send = None self.received = 0 self.corrupted = 0 self.out_of_order = 0 + self.latencies = [] self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) stats[self.fingerprint] = {} def onOpen(self): def hello(): - self.sendMessage("%s:%s" % (self.sent, self.fingerprint)) - self.sent += 1 - if self.sent < self.num_messages: - self.factory.reactor.callLater(1, hello) + if self.last_send is None: + if self.sent >= self.num_messages: + self.sendClose() + return + self.sendMessage(("%s:%s" % (self.sent, self.fingerprint)).encode("ascii")) + self.last_send = time.time() + self.sent += 1 else: - self.sendClose() + # Wait for receipt of ping + pass + self.factory.reactor.callLater(1, hello) hello() def onMessage(self, payload, isBinary): - num, fingerprint = payload.split(":") + num, fingerprint = payload.decode("ascii").split(":") if fingerprint != self.fingerprint: self.corrupted += 1 - if num != self.received: + if int(num) != self.received: self.out_of_order += 1 self.received += 1 + self.latencies.append(time.time() - self.last_send) + self.last_send = None def onClose(self, wasClean, code, reason): - stats[self.fingerprint] = { - "sent": self.sent, - "received": self.received, - "corrupted": self.corrupted, - "out_of_order": self.out_of_order, - } + if hasattr(self, "sent"): + stats[self.fingerprint] = { + "sent": self.sent, + "received": self.received, + "corrupted": self.corrupted, + "out_of_order": self.out_of_order, + "connect": True, + } + else: + self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) + stats[self.fingerprint] = { + "sent": 0, + "received": 0, + "corrupted": 0, + "out_of_order": 0, + "connect": False, + } -def spawn_connections(): - if len(stats) >= NUM_CONNECTIONS: - return - for i in range(PER_SECOND): - reactor.connectTCP("127.0.0.1", 9000, factory) - reactor.callLater(1, spawn_connections) +class Benchmarker(object): + """ + Performs benchmarks against WebSockets. + """ -def print_progress(): - open_protocols = len([x for x in stats.values() if not x]) - print "%s open, %s total" % ( - open_protocols, - len(stats), - ) - reactor.callLater(1, print_progress) - if open_protocols == 0 and len(stats) >= NUM_CONNECTIONS: - reactor.stop() - print_stats() + def __init__(self, url, num, rate): + self.url = url + self.num = num + self.rate = rate + self.factory = WebSocketClientFactory( + args.url, + debug=False, + ) + self.factory.protocol = MyClientProtocol + def loop(self): + self.spawn_connections() + self.print_progress() + reactor.callLater(1, self.loop) -def print_stats(): - num_incomplete = len([x for x in stats.values() if x['sent'] != x['received']]) - num_corruption = len([x for x in stats.values() if x['corrupted']]) - num_out_of_order = len([x for x in stats.values() if x['out_of_order']]) - print "-------" - print "Sockets opened: %s" % len(stats) - print "Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100) - print "Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100) - print "Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100) + def spawn_connections(self): + if len(stats) >= self.num: + return + for i in range(self.rate): + # TODO: Look at URL + reactor.connectTCP("127.0.0.1", 8000, self.factory) + + def print_progress(self): + open_protocols = len([x for x in stats.values() if not x]) + print("%s open, %s total" % ( + open_protocols, + len(stats), + )) + if open_protocols == 0 and len(stats) >= self.num: + print("Reached %s open connections, quitting" % self.num) + reactor.stop() + self.print_stats() + + def print_stats(self): + num_incomplete = len([x for x in stats.values() if x['sent'] != x['received']]) + num_corruption = len([x for x in stats.values() if x['corrupted']]) + num_out_of_order = len([x for x in stats.values() if x['out_of_order']]) + num_failed = len([x for x in stats.values() if not x['connect']]) + print("-------") + print("Sockets opened: %s" % len(stats)) + print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100)) + print("Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100)) + print("Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100)) + print("Failed to connect: %s (%.2f%%)" % (num_failed, (float(num_failed) / len(stats))*100)) if __name__ == '__main__': import sys + import argparse from twisted.python import log from twisted.internet import reactor # log.startLogging(sys.stdout) - factory = WebSocketClientFactory(u"ws://127.0.0.1:9000", debug=False) - factory.protocol = MyClientProtocol - - reactor.callLater(1, spawn_connections) - reactor.callLater(1, print_progress) + parser = argparse.ArgumentParser() + parser.add_argument("url") + parser.add_argument("-n", "--num", type=int, default=100) + parser.add_argument("-r", "--rate", type=int, default=10) + args = parser.parse_args() + benchmarker = Benchmarker( + url=args.url, + num=args.num, + rate=args.rate, + ) + benchmarker.loop() reactor.run() diff --git a/testproject/testproject/settings.py b/testproject/testproject/settings.py index 19d9ecf..ab336f3 100644 --- a/testproject/testproject/settings.py +++ b/testproject/testproject/settings.py @@ -18,6 +18,8 @@ ROOT_URLCONF = 'testproject.urls' WSGI_APPLICATION = 'testproject.wsgi.application' +STATIC_URL = "/static/" + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', @@ -25,9 +27,9 @@ DATABASES = { } } -CHANNEL_BACKENDS = { +CHANNEL_LAYERS = { "default": { - "BACKEND": "channels.database_layer.DatabaseChannelLayer", + "BACKEND": "asgiref.inmemory.ChannelLayer", "ROUTING": "testproject.urls.channel_routing", }, } diff --git a/testproject/testproject/urls.py b/testproject/testproject/urls.py index 0337b72..eeea7b6 100644 --- a/testproject/testproject/urls.py +++ b/testproject/testproject/urls.py @@ -3,5 +3,5 @@ from chtest import consumers urlpatterns = [] channel_routing = { - "websocket.message": consumers.ws_message, + "websocket.receive": consumers.ws_message, } From 9437dc17b5e427431661b6a715122ac3a0d4d686 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Feb 2016 21:40:52 +0000 Subject: [PATCH 200/746] Fix line endings --- testproject/benchmark.py | 294 +++++++++++++++++++-------------------- 1 file changed, 147 insertions(+), 147 deletions(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index 3f0fa52..9ef401f 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -1,147 +1,147 @@ -from __future__ import unicode_literals - -import time -import random -from autobahn.twisted.websocket import WebSocketClientProtocol, \ - WebSocketClientFactory - - -stats = {} - - -class MyClientProtocol(WebSocketClientProtocol): - - num_messages = 5 - message_gap = 1 - - def onConnect(self, response): - self.opened = time.time() - self.sent = 0 - self.last_send = None - self.received = 0 - self.corrupted = 0 - self.out_of_order = 0 - self.latencies = [] - self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) - stats[self.fingerprint] = {} - - def onOpen(self): - def hello(): - if self.last_send is None: - if self.sent >= self.num_messages: - self.sendClose() - return - self.sendMessage(("%s:%s" % (self.sent, self.fingerprint)).encode("ascii")) - self.last_send = time.time() - self.sent += 1 - else: - # Wait for receipt of ping - pass - self.factory.reactor.callLater(1, hello) - hello() - - def onMessage(self, payload, isBinary): - num, fingerprint = payload.decode("ascii").split(":") - if fingerprint != self.fingerprint: - self.corrupted += 1 - if int(num) != self.received: - self.out_of_order += 1 - self.received += 1 - self.latencies.append(time.time() - self.last_send) - self.last_send = None - - def onClose(self, wasClean, code, reason): - if hasattr(self, "sent"): - stats[self.fingerprint] = { - "sent": self.sent, - "received": self.received, - "corrupted": self.corrupted, - "out_of_order": self.out_of_order, - "connect": True, - } - else: - self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) - stats[self.fingerprint] = { - "sent": 0, - "received": 0, - "corrupted": 0, - "out_of_order": 0, - "connect": False, - } - - - -class Benchmarker(object): - """ - Performs benchmarks against WebSockets. - """ - - def __init__(self, url, num, rate): - self.url = url - self.num = num - self.rate = rate - self.factory = WebSocketClientFactory( - args.url, - debug=False, - ) - self.factory.protocol = MyClientProtocol - - def loop(self): - self.spawn_connections() - self.print_progress() - reactor.callLater(1, self.loop) - - def spawn_connections(self): - if len(stats) >= self.num: - return - for i in range(self.rate): - # TODO: Look at URL - reactor.connectTCP("127.0.0.1", 8000, self.factory) - - def print_progress(self): - open_protocols = len([x for x in stats.values() if not x]) - print("%s open, %s total" % ( - open_protocols, - len(stats), - )) - if open_protocols == 0 and len(stats) >= self.num: - print("Reached %s open connections, quitting" % self.num) - reactor.stop() - self.print_stats() - - def print_stats(self): - num_incomplete = len([x for x in stats.values() if x['sent'] != x['received']]) - num_corruption = len([x for x in stats.values() if x['corrupted']]) - num_out_of_order = len([x for x in stats.values() if x['out_of_order']]) - num_failed = len([x for x in stats.values() if not x['connect']]) - print("-------") - print("Sockets opened: %s" % len(stats)) - print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100)) - print("Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100)) - print("Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100)) - print("Failed to connect: %s (%.2f%%)" % (num_failed, (float(num_failed) / len(stats))*100)) - - -if __name__ == '__main__': - - import sys - import argparse - - from twisted.python import log - from twisted.internet import reactor - -# log.startLogging(sys.stdout) - - parser = argparse.ArgumentParser() - parser.add_argument("url") - parser.add_argument("-n", "--num", type=int, default=100) - parser.add_argument("-r", "--rate", type=int, default=10) - args = parser.parse_args() - - benchmarker = Benchmarker( - url=args.url, - num=args.num, - rate=args.rate, - ) - benchmarker.loop() - reactor.run() +from __future__ import unicode_literals + +import time +import random +from autobahn.twisted.websocket import WebSocketClientProtocol, \ + WebSocketClientFactory + + +stats = {} + + +class MyClientProtocol(WebSocketClientProtocol): + + num_messages = 5 + message_gap = 1 + + def onConnect(self, response): + self.opened = time.time() + self.sent = 0 + self.last_send = None + self.received = 0 + self.corrupted = 0 + self.out_of_order = 0 + self.latencies = [] + self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) + stats[self.fingerprint] = {} + + def onOpen(self): + def hello(): + if self.last_send is None: + if self.sent >= self.num_messages: + self.sendClose() + return + self.sendMessage(("%s:%s" % (self.sent, self.fingerprint)).encode("ascii")) + self.last_send = time.time() + self.sent += 1 + else: + # Wait for receipt of ping + pass + self.factory.reactor.callLater(1, hello) + hello() + + def onMessage(self, payload, isBinary): + num, fingerprint = payload.decode("ascii").split(":") + if fingerprint != self.fingerprint: + self.corrupted += 1 + if int(num) != self.received: + self.out_of_order += 1 + self.received += 1 + self.latencies.append(time.time() - self.last_send) + self.last_send = None + + def onClose(self, wasClean, code, reason): + if hasattr(self, "sent"): + stats[self.fingerprint] = { + "sent": self.sent, + "received": self.received, + "corrupted": self.corrupted, + "out_of_order": self.out_of_order, + "connect": True, + } + else: + self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) + stats[self.fingerprint] = { + "sent": 0, + "received": 0, + "corrupted": 0, + "out_of_order": 0, + "connect": False, + } + + + +class Benchmarker(object): + """ + Performs benchmarks against WebSockets. + """ + + def __init__(self, url, num, rate): + self.url = url + self.num = num + self.rate = rate + self.factory = WebSocketClientFactory( + args.url, + debug=False, + ) + self.factory.protocol = MyClientProtocol + + def loop(self): + self.spawn_connections() + self.print_progress() + reactor.callLater(1, self.loop) + + def spawn_connections(self): + if len(stats) >= self.num: + return + for i in range(self.rate): + # TODO: Look at URL + reactor.connectTCP("127.0.0.1", 8000, self.factory) + + def print_progress(self): + open_protocols = len([x for x in stats.values() if not x]) + print("%s open, %s total" % ( + open_protocols, + len(stats), + )) + if open_protocols == 0 and len(stats) >= self.num: + print("Reached %s open connections, quitting" % self.num) + reactor.stop() + self.print_stats() + + def print_stats(self): + num_incomplete = len([x for x in stats.values() if x['sent'] != x['received']]) + num_corruption = len([x for x in stats.values() if x['corrupted']]) + num_out_of_order = len([x for x in stats.values() if x['out_of_order']]) + num_failed = len([x for x in stats.values() if not x['connect']]) + print("-------") + print("Sockets opened: %s" % len(stats)) + print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100)) + print("Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100)) + print("Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100)) + print("Failed to connect: %s (%.2f%%)" % (num_failed, (float(num_failed) / len(stats))*100)) + + +if __name__ == '__main__': + + import sys + import argparse + + from twisted.python import log + from twisted.internet import reactor + +# log.startLogging(sys.stdout) + + parser = argparse.ArgumentParser() + parser.add_argument("url") + parser.add_argument("-n", "--num", type=int, default=100) + parser.add_argument("-r", "--rate", type=int, default=10) + args = parser.parse_args() + + benchmarker = Benchmarker( + url=args.url, + num=args.num, + rate=args.rate, + ) + benchmarker.loop() + reactor.run() From 9d8d36007bf9110d05251c4c165548b631d1b7d7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Feb 2016 21:57:39 +0000 Subject: [PATCH 201/746] Latency stats for benchmarker --- testproject/benchmark.py | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index 9ef401f..a8597aa 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import time import random +import statistics from autobahn.twisted.websocket import WebSocketClientProtocol, \ WebSocketClientFactory @@ -57,6 +58,7 @@ class MyClientProtocol(WebSocketClientProtocol): "received": self.received, "corrupted": self.corrupted, "out_of_order": self.out_of_order, + "latencies": self.latencies, "connect": True, } else: @@ -105,17 +107,56 @@ class Benchmarker(object): len(stats), )) if open_protocols == 0 and len(stats) >= self.num: - print("Reached %s open connections, quitting" % self.num) reactor.stop() self.print_stats() + def percentile(self, values, fraction): + """ + Returns a percentile value (e.g. fraction = 0.95 -> 95th percentile) + """ + values = sorted(values) + stopat = int(len(values) * fraction) + if stopat == len(values): + stopat -= 1 + return values[stopat] + def print_stats(self): - num_incomplete = len([x for x in stats.values() if x['sent'] != x['received']]) - num_corruption = len([x for x in stats.values() if x['corrupted']]) - num_out_of_order = len([x for x in stats.values() if x['out_of_order']]) - num_failed = len([x for x in stats.values() if not x['connect']]) + # Collect stats together + latencies = [] + num_good = 0 + num_incomplete = 0 + num_failed = 0 + num_corruption = 0 + num_out_of_order = 0 + for entry in stats.values(): + latencies.extend(entry.get("latencies", [])) + if not entry['connect']: + num_failed += 1 + elif entry['sent'] != entry['received']: + num_incomplete += 1 + elif entry['corrupted']: + num_corruption += 1 + elif entry['out_of_order']: + num_out_of_order += 1 + else: + num_good += 1 + # Some analysis on latencies + latency_mean = statistics.mean(latencies) + latency_median = statistics.median(latencies) + latency_stdev = statistics.stdev(latencies) + latency_5 = self.percentile(latencies, 0.05) + latency_95 = self.percentile(latencies, 0.95) + # Print results print("-------") print("Sockets opened: %s" % len(stats)) + print("Latency stats: Mean %.2fs Median %.2fs Stdev %.2f 5%% %.2fs 95%% %.2fs" % ( + latency_mean, + latency_median, + latency_stdev, + latency_5, + latency_95, + )) + print("Good sockets: %s (%.2f%%)" % (num_good, (float(num_good) / len(stats))*100)) print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100)) print("Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100)) print("Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100)) From 5ff77719be4abf135254fd15e78721d5ef883c2c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Feb 2016 22:14:30 +0000 Subject: [PATCH 202/746] Make database channel layer use transactions to stop dupes --- channels/database_layer.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index d85a2e8..72f1a58 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -6,7 +6,8 @@ import string import time from django.apps.registry import Apps -from django.db import DEFAULT_DB_ALIAS, connections, models +from django.db import DEFAULT_DB_ALIAS, connections, models, transaction +from django.db.utils import OperationalError from django.utils import six from django.utils.functional import cached_property from django.utils.timezone import now @@ -56,15 +57,20 @@ class DatabaseChannelLayer(object): self._clean_expired() # Get a message from one of our channels while True: - message = self.channel_model.objects.filter(channel__in=channels).order_by("id").first() - if message: - self.channel_model.objects.filter(pk=message.pk).delete() - return message.channel, self.deserialize(message.content) + try: + with transaction.atomic(): + message = self.channel_model.objects.filter(channel__in=channels).order_by("id").first() + if message: + self.channel_model.objects.filter(pk=message.pk).delete() + return message.channel, self.deserialize(message.content) + except OperationalError: + # The database is probably trying to prevent a deadlock + time.sleep(0.1) + continue + if block: + time.sleep(1) else: - if block: - time.sleep(1) - else: - return None, None + return None, None def new_channel(self, pattern): assert isinstance(pattern, six.text_type) From 38c6df812574b3910dfbf524f4f03215c15c02a2 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 24 Feb 2016 00:39:48 +0000 Subject: [PATCH 203/746] Move ResponseLater into exceptions module --- channels/exceptions.py | 16 ++++++++++++++++ channels/handler.py | 10 +++------- 2 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 channels/exceptions.py diff --git a/channels/exceptions.py b/channels/exceptions.py new file mode 100644 index 0000000..3e52699 --- /dev/null +++ b/channels/exceptions.py @@ -0,0 +1,16 @@ +class ConsumeLater(Exception): + """ + Exception that says that the current message should be re-queued back + onto its channel as it's not ready to be consumd yet (e.g. global order + is being enforced) + """ + pass + + +class ResponseLater(Exception): + """ + Exception raised inside a Django view when the view has passed + responsibility for the response to another consumer, and so is not + returning a response. + """ + pass diff --git a/channels/handler.py b/channels/handler.py index e41007e..a32d1af 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -14,6 +14,8 @@ from django.core.urlresolvers import set_script_prefix from django.utils import six from django.utils.functional import cached_property +from .exceptions import ResponseLater as ResponseLaterOuter + logger = logging.getLogger('django.request') @@ -23,13 +25,7 @@ class AsgiRequest(http.HttpRequest): dict, and wraps request body handling. """ - class ResponseLater(Exception): - """ - Exception that will cause any handler to skip around response - transmission and presume something else will do it later. - """ - def __init__(self): - Exception.__init__(self, "Response later") + ResponseLater = ResponseLaterOuter def __init__(self, message): self.message = message From 69186ef7b7c11d283b95d415378e94c23aca452a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 24 Feb 2016 00:40:38 +0000 Subject: [PATCH 204/746] Replace linearize with the more useful enforce_ordering. --- channels/decorators.py | 30 ------------ channels/sessions.py | 87 ++++++++++++++++++++++++++------- channels/worker.py | 3 ++ docs/asgi.rst | 44 ++++++++++++++++- docs/getting-started.rst | 78 +++++++++++++++++------------ testproject/chtest/consumers.py | 8 +++ testproject/testproject/urls.py | 1 + 7 files changed, 169 insertions(+), 82 deletions(-) delete mode 100644 channels/decorators.py diff --git a/channels/decorators.py b/channels/decorators.py deleted file mode 100644 index 279a7bd..0000000 --- a/channels/decorators.py +++ /dev/null @@ -1,30 +0,0 @@ -import functools - - -def linearize(func): - """ - Makes sure the contained consumer does not run at the same time other - consumers are running on messages with the same reply_channel. - - Required if you don't want weird things like a second consumer starting - up before the first has exited and saved its session. Doesn't guarantee - ordering, just linearity. - """ - raise NotImplementedError("Not yet reimplemented") - - @functools.wraps(func) - def inner(message, *args, **kwargs): - # Make sure there's a reply channel - if not message.reply_channel: - raise ValueError( - "No reply_channel in message; @linearize can only be used on messages containing it." - ) - # TODO: Get lock here - pass - # OK, keep going - try: - return func(message, *args, **kwargs) - finally: - # TODO: Release lock here - pass - return inner diff --git a/channels/sessions.py b/channels/sessions.py index 598b79e..7052f01 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -3,10 +3,26 @@ import hashlib from importlib import import_module from django.conf import settings +from django.contrib.sessions.backends.base import CreateError +from .exceptions import ConsumeLater from .handler import AsgiRequest +def session_for_reply_channel(reply_channel): + """ + Returns a session object tied to the reply_channel unicode string + passed in as an argument. + """ + # We hash the whole reply channel name and add a prefix, to fit inside 32B + reply_name = reply_channel + hashed = hashlib.md5(reply_name.encode("utf8")).hexdigest() + session_key = "chn" + hashed[:29] + # Make a session storage + session_engine = import_module(settings.SESSION_ENGINE) + return session_engine.SessionStore(session_key=session_key) + + def channel_session(func): """ Provides a session-like object called "channel_session" to consumers @@ -17,30 +33,24 @@ def channel_session(func): """ @functools.wraps(func) def inner(message, *args, **kwargs): + # Make sure there's NOT a channel_session already + if hasattr(message, "channel_session"): + return func(message, *args, **kwargs) # Make sure there's a reply_channel if not message.reply_channel: raise ValueError( "No reply_channel sent to consumer; @channel_session " + "can only be used on messages containing it." ) - - # Make sure there's NOT a channel_session already - if hasattr(message, "channel_session"): - raise ValueError("channel_session decorator wrapped inside another channel_session decorator") - - # Turn the reply_channel into a valid session key length thing. - # We take the last 24 bytes verbatim, as these are the random section, - # and then hash the remaining ones onto the start, and add a prefix - reply_name = message.reply_channel.name - hashed = hashlib.md5(reply_name[:-24].encode()).hexdigest()[:8] - session_key = "skt" + hashed + reply_name[-24:] - # Make a session storage - session_engine = import_module(settings.SESSION_ENGINE) - session = session_engine.SessionStore(session_key=session_key) # If the session does not already exist, save to force our # session key to be valid. + session = session_for_reply_channel(message.reply_channel.name) if not session.exists(session.session_key): - session.save(must_create=True) + try: + session.save(must_create=True) + except CreateError: + # Session wasn't unique, so another consumer is doing the same thing + raise ConsumeLater() message.channel_session = session # Run the consumer try: @@ -52,6 +62,47 @@ def channel_session(func): return inner +def enforce_ordering(func=None, slight=False): + """ + Enforces either slight (order=0 comes first, everything else isn't ordered) + or strict (all messages exactly ordered) ordering against a reply_channel. + + Uses sessions to track ordering. + + You cannot mix slight ordering and strict ordering on a channel; slight + ordering does not write to the session after the first message to improve + performance. + """ + def decorator(func): + @channel_session + @functools.wraps(func) + def inner(message, *args, **kwargs): + # Make sure there's an order + if "order" not in message.content: + raise ValueError( + "No `order` value in message; @enforce_ordering " + + "can only be used on messages containing it." + ) + order = int(message.content['order']) + # See what the current next order should be + next_order = message.channel_session.get("__channels_next_order", 0) + if order == next_order or (slight and next_order > 0): + # Message is in right order. Maybe persist next one? + if order == 0 or not slight: + message.channel_session["__channels_next_order"] = order + 1 + # Run consumer + return func(message, *args, **kwargs) + else: + # Bad ordering + print("Bad ordering detected: next %s, us %s, %s" % (next_order, order, message.reply_channel)) + raise ConsumeLater() + return inner + if func is not None: + return decorator(func) + else: + return decorator + + def http_session(func): """ Wraps a HTTP or WebSocket connect consumer (or any consumer of messages @@ -69,6 +120,9 @@ def http_session(func): """ @functools.wraps(func) def inner(message, *args, **kwargs): + # Make sure there's NOT a http_session already + if hasattr(message, "http_session"): + return func(message, *args, **kwargs) try: # We want to parse the WebSocket (or similar HTTP-lite) message # to get cookies and GET, but we need to add in a few things that @@ -78,9 +132,6 @@ def http_session(func): request = AsgiRequest(message) except Exception as e: raise ValueError("Cannot parse HTTP message - are you sure this is a HTTP consumer? %s" % e) - # Make sure there's NOT a http_session already - if hasattr(message, "http_session"): - raise ValueError("http_session decorator wrapped inside another http_session decorator") # Make sure there's a session key session_key = request.GET.get("session_key", None) if session_key is None: diff --git a/channels/worker.py b/channels/worker.py index 01e3891..927b980 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import logging import time +from .exceptions import ConsumeLater from .message import Message from .utils import name_that_thing @@ -43,5 +44,7 @@ class Worker(object): self.callback(channel, message) try: consumer(message) + except ConsumeLater: + self.channel_layer.send(channel, content) except: logger.exception("Error processing message with consumer %s:", name_that_thing(consumer)) diff --git a/docs/asgi.rst b/docs/asgi.rst index c68611e..61c0cfa 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -381,7 +381,12 @@ use if the key is missing). Keys are unicode strings. The one common key across all protocols is ``reply_channel``, a way to indicate the client-specific channel to send responses to. Protocols are generally -encouraged to have one message type and one reply channel to ensure ordering. +encouraged to have one message type and one reply channel type to ensure ordering. + +A ``reply_channel`` should be unique per connection. If the protocol in question +can have any server service a response - e.g. a theoretical SMS protocol - it +should not have ``reply_channel`` attributes on messages, but instead a separate +top-level outgoing channel. Messages are specified here along with the channel names they are expected on; if a channel name can vary, such as with reply channels, the varying @@ -390,7 +395,7 @@ the format the ``new_channel`` callable takes. There is no label on message types to say what they are; their type is implicit in the channel name they are received on. Two types that are sent on the same -channel, such as HTTP responses and server pushes, are distinguished apart +channel, such as HTTP responses and response chunks, are distinguished apart by their required fields. @@ -630,6 +635,8 @@ Keys: for this server as a unicode string, and ``port`` is the integer listening port. Optional, defaults to ``None``. +* ``order``: The integer value ``0``. + Receive ''''''' @@ -647,6 +654,9 @@ Keys: * ``text``: Unicode string of frame content, if it was text mode, or ``None``. +* ``order``: Order of this frame in the WebSocket stream, starting + at 1 (``connect`` is 0). + One of ``bytes`` or ``text`` must be non-``None``. @@ -665,6 +675,9 @@ Keys: format ``websocket.send.?``. Cannot be used to send at this point; provided as a way to identify the connection only. +* ``order``: Order of the disconnection relative to the incoming frames' + ``order`` values in ``websocket.receive``. + Send/Close '''''''''' @@ -736,6 +749,33 @@ Keys: * ``data``: Byte string of UDP datagram payload. +Protocol Format Guidelines +-------------------------- + +Message formats for protocols should follow these rules, unless +a very good performance or implementation reason is present: + +* ``reply_channel`` should be unique per logical connection, and not per + logical client. + +* If the protocol has server-side state, entirely encapsulate that state in + the protocol server; do not require the message consumers to use an external + state store. + +* If the protocol has low-level negotiation, keepalive or other features, + handle these within the protocol server and don't expose them in ASGI + messages. + +* If the protocol has guaranteed ordering, ASGI messages should include an + ``order`` field (0-indexed) that preserves the ordering as received by the + protocol server (or as sent by the client, if available). This ordering should + span all message types emitted by the client - for example, a connect message + might have order ``0``, and the first two frames order ``1`` and ``2``. + +* If the protocol is datagram-based, one datagram should equal one ASGI message + (unless size is an issue) + + Approximate Global Ordering --------------------------- diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 029666e..010b8fc 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -259,7 +259,7 @@ name in the path of your WebSocket request (we'll ignore auth for now - that's n # In consumers.py from channels import Group - from channels.decorators import channel_session + from channels.sessions import channel_session # Connected to websocket.connect @channel_session @@ -342,7 +342,7 @@ chat to people with the same first letter of their username:: # In consumers.py from channels import Channel, Group - from channels.decorators import channel_session + from channels.sessions import channel_session from channels.auth import http_session_user, channel_session_user, transfer_user # Connected to websocket.connect @@ -401,7 +401,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: # In consumers.py from channels import Channel - from channels.decorators import channel_session + from channels.sessions import channel_session from .models import ChatMessage # Connected to chat-messages @@ -445,14 +445,16 @@ command run via ``cron``. If we wanted to write a bot, too, we could put its listening logic inside the ``chat-messages`` consumer, as every message would pass through it. -Linearization -------------- + +Enforcing Ordering +------------------ There's one final concept we want to introduce you to before you go on to build -sites with Channels - linearizing consumers. +sites with Channels - consmer ordering Because Channels is a distributed system that can have many workers, by default -it's entirely feasible for a WebSocket interface server to send out a ``connect`` +it just processes messages in the order the workers get them off the queue. +It's entirely feasible for a WebSocket interface server to send out a ``connect`` and a ``receive`` message close enough together that a second worker will pick up and start processing the ``receive`` message before the first worker has finished processing the ``connect`` worker. @@ -464,53 +466,65 @@ same effect if someone tried to request a view before the login view had finishe processing, but there you're not expecting that page to run after the login, whereas you'd naturally expect ``receive`` to run after ``connect``. -But, of course, Channels has a solution - the ``linearize`` decorator. Any -handler decorated with this will use locking to ensure it does not run at the -same time as any other view with ``linearize`` **on messages with the same reply channel**. -That means your site will happily multitask with lots of different people's messages, -but if two happen to try to run at the same time for the same client, they'll -be deconflicted. +Channels has a solution - the ``enforce_ordering`` decorator. All WebSocket +messages contain an ``order`` key, and this decorator uses that to make sure that +messages are consumed in the right order, in one of two modes: -There's a small cost to using ``linearize``, which is why it's an optional -decorator, but generally you'll want to use it for most session-based WebSocket +* Slight ordering: Message 0 (``websocket.connect``) is done first, all others + are unordered + +* Strict ordering: All messages are consumed strictly in sequence + +The decorator uses ``channel_session`` to keep track of what numbered messages +have been processed, and if a worker tries to run a consumer on an out-of-order +message, it raises the ``ConsumeLater`` exception, which puts the message +back on the channel it came from and tells the worker to work on another message. + +There's a cost to using ``enforce_ordering``, which is why it's an optional +decorator, and the cost is much greater in *strict* mode than it is in +*slight* mode. Generally you'll want to use *slight* mode for most session-based WebSocket and other "continuous protocol" things. Here's an example, improving our first-letter-of-username chat from earlier:: # In consumers.py from channels import Channel, Group - from channels.decorators import channel_session, linearize - from channels.auth import http_session_user, channel_session_user, transfer_user + from channels.sessions import channel_session, enforce_ordering + from channels.auth import http_session_user, channel_session_user, channel_session_user_from_http # Connected to websocket.connect - @linearize - @channel_session - @http_session_user + @enforce_ordering(slight=True) + @channel_session_user_from_http def ws_add(message): - # Copy user from HTTP to channel session - transfer_user(message.http_session, message.channel_session) # Add them to the right group Group("chat-%s" % message.user.username[0]).add(message.reply_channel) - # Connected to websocket.keepalive - # We don't linearize as we know this will happen a decent time after add - @channel_session_user - def ws_keepalive(message): - # Keep them in the right group - Group("chat-%s" % message.user.username[0]).add(message.reply_channel) - # Connected to websocket.receive - @linearize + @enforce_ordering(slight=True) @channel_session_user def ws_message(message): Group("chat-%s" % message.user.username[0]).send(message.content) # Connected to websocket.disconnect - # We don't linearize as even if this gets an empty session, the group - # will auto-discard after the expiry anyway. + @enforce_ordering(slight=True) @channel_session_user def ws_disconnect(message): Group("chat-%s" % message.user.username[0]).discard(message.reply_channel) +Slight ordering does mean that it's possible for a ``disconnect`` message to +get processed before a ``receive`` message, but that's fine in this case; +the client is disconnecting anyway, they don't care about those pending messages. + +Strict ordering is the default as it's the most safe; to use it, just call +the decorator without arguments:: + + @enforce_ordering + def ws_message(message): + ... + +Generally, the performance (and safety) of your ordering is tied to your +session backend's performance. Make sure you choose session backend wisely +if you're going to rely heavily on ``enforce_ordering``. + Next Steps ---------- diff --git a/testproject/chtest/consumers.py b/testproject/chtest/consumers.py index ba7385c..96dbabb 100644 --- a/testproject/chtest/consumers.py +++ b/testproject/chtest/consumers.py @@ -1,4 +1,12 @@ +from channels.sessions import enforce_ordering + +#@enforce_ordering(slight=True) +def ws_connect(message): + pass + + +#@enforce_ordering(slight=True) def ws_message(message): "Echoes messages back to the client" message.reply_channel.send(message.content) diff --git a/testproject/testproject/urls.py b/testproject/testproject/urls.py index eeea7b6..42ae607 100644 --- a/testproject/testproject/urls.py +++ b/testproject/testproject/urls.py @@ -4,4 +4,5 @@ urlpatterns = [] channel_routing = { "websocket.receive": consumers.ws_message, + "websocket.connect": consumers.ws_connect, } From c64fe974637ff0cfbf8d2424f3234ba9ecd417d8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Feb 2016 19:36:17 -0800 Subject: [PATCH 205/746] Fix trailing whitespace --- channels/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/sessions.py b/channels/sessions.py index 7052f01..a9e847e 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -72,7 +72,7 @@ def enforce_ordering(func=None, slight=False): You cannot mix slight ordering and strict ordering on a channel; slight ordering does not write to the session after the first message to improve performance. - """ + """ def decorator(func): @channel_session @functools.wraps(func) From 500f0fdeb73da1aef9e9bd635130116b4c57893b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Feb 2016 19:36:34 -0800 Subject: [PATCH 206/746] Couple of bits more ordering docs --- CHANGELOG.txt | 11 +++++++++++ docs/getting-started.rst | 2 ++ docs/inshort.rst | 11 +++++++++++ 3 files changed, 24 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 67c519b..effc855 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,8 +1,19 @@ +0.9.2 (2016-02-24) +------------------ + +* ASGI spec updated to include `order` field for WebSocket messages + +* `enforce_ordering` decorator introduced + +* DatabaseChannelLayer now uses transactions to stop duplicated messages + + 0.9.1 (2016-02-21) ------------------ * Fix packaging issues with previous release + 0.9 (2016-02-21) ---------------- diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 010b8fc..2c91f0c 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -446,6 +446,8 @@ listening logic inside the ``chat-messages`` consumer, as every message would pass through it. +.. _enforcing-ordering: + Enforcing Ordering ------------------ diff --git a/docs/inshort.rst b/docs/inshort.rst index 070a0eb..e006150 100644 --- a/docs/inshort.rst +++ b/docs/inshort.rst @@ -66,6 +66,17 @@ it's designed to allow both easy sharding as well as the ability to run separate clusters with their own protocol and worker servers. +Why doesn't it use my favourite message queue? +---------------------------------------------- + +Channels is deliberately designed to prefer low latency (goal is a few milliseconds) +and high throughput over guaranteed delivery, which doesn't match some +message queue designs. + +Some features, like :ref:`guaranteed ordering of messages `, +are opt-in as they incur a performance hit. + + Do I need to worry about making all my code async-friendly? ----------------------------------------------------------- From fe352542cc6a53639eb4d57dcf445f238f2f6c2e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 24 Feb 2016 07:00:33 -0800 Subject: [PATCH 207/746] Fix for Django 1.8/1.9 --- channels/asgi.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/channels/asgi.py b/channels/asgi.py index d11d0bf..29eacc3 100644 --- a/channels/asgi.py +++ b/channels/asgi.py @@ -80,7 +80,10 @@ def get_channel_layer(alias="default"): """ Returns the raw ASGI channel layer for this project. """ - django.setup(set_prefix=False) + if django.VERSION[1] > 9: + django.setup(set_prefix=False) + else: + django.setup() return channel_layers[alias].channel_layer From f8c264551aa035306eb302d9b384003c6c2b0b09 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 21:11:51 -0800 Subject: [PATCH 208/746] Remove warning from README. --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index c3966b3..df5559f 100644 --- a/README.rst +++ b/README.rst @@ -4,10 +4,6 @@ Django Channels .. image:: https://api.travis-ci.org/andrewgodwin/channels.svg :target: https://travis-ci.org/andrewgodwin/channels -**NOTE: The current master branch is in flux as it changes to match the final -structure and the new ASGI spec. If you wish to use this in the meantime, -please use a tagged release.** - This is a work-in-progress code branch of Django implemented as a third-party app, which aims to bring some asynchrony to Django and expand the options for code beyond the request-response model, in particular enabling WebSocket, From 9dae793cded98071334618d649e9a1e4ec0de980 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 21:12:11 -0800 Subject: [PATCH 209/746] Release 0.9.2 --- channels/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/__init__.py b/channels/__init__.py index 62454f9..fc4ca95 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.9.1" +__version__ = "0.9.2" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From c2a8e32b4beb191877117adb015fb0b7d3eb8a5e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 21:39:56 -0800 Subject: [PATCH 210/746] Fix changelog date --- CHANGELOG.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index effc855..c4aab66 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -0.9.2 (2016-02-24) +0.9.2 (2016-02-28) ------------------ * ASGI spec updated to include `order` field for WebSocket messages From b18975e607c3f638cb8135e374853ea4956e4559 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 21:46:45 -0800 Subject: [PATCH 211/746] Really lay into DatabaseChannelLayer in an impolite way. --- docs/backends.rst | 19 ++++-- docs/getting-started.rst | 121 +++++++++++++++++++++++++-------------- 2 files changed, 90 insertions(+), 50 deletions(-) diff --git a/docs/backends.rst b/docs/backends.rst index 08a40c1..b76f51c 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -64,21 +64,28 @@ that will work cross-process. It has poor performance, and is only recommended for development or extremely small deployments. This layer is included with Channels; just set your ``BACKEND`` to -``channels.backends.database.DatabaseChannelLayer``, and it will use the +``channels.datagbase_layer.DatabaseChannelLayer``, and it will use the default Django database alias to store messages. You can change the alias by setting ``CONFIG`` to ``{'alias': 'aliasname'}``. +.. warning:: + The database channel layer is NOT fast, and performs especially poorly at + latency and throughput. We recommend its use only as a last resort, and only + on a database with good transaction support (e.g. Postgres), or you may + get errors with multiple message delivery. + In-memory --------- -The in-memory layer is purely an implementation detail used when running -the entire Django stack in a single process; the most common case of this -is ``runserver``, where a server thread, channel layer, and worker thread all +The in-memory layer is only useful when running the protocol server and the +worker server in a single process; the most common case of this +is ``runserver``, where a server thread, this channel layer, and worker thread all co-exist inside the same python process. -You should not need to use this process manually, but if you want to, -it's available from ``asgiref.inmemory.ChannelLayer``. +Its path is ``asgiref.inmemory.ChannelLayer``. If you try and use this channel +layer with ``runworker``, it will exit, as it does not support cross-process +communication. Writing Custom Channel Layers diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 2c91f0c..9103d0d 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -11,15 +11,17 @@ patterns and caveats. First Consumers --------------- -Now, by default, Django will run things through Channels but it will also -tie in the URL router and view subsystem to the default ``http.request`` -channel if you don't provide another consumer that listens to it - remember, -only one consumer can listen to any given channel. +When you run Django out of the box, it will be set up in the default layout - +where all HTTP requests (on the ``http.request`` channel) are routed to the +Django view layer - nothing will be different to how things worked in the past +with a WSGI-based Django, and your views and static file serving (from +``runserver`` will work as normal) -As a very basic example, let's write a consumer that overrides the built-in +As a very basic introduction, let's write a consumer that overrides the built-in handling and handles every HTTP request directly. This isn't something you'd usually do in a project, but it's a good illustration of how Channels -actually underlies even core Django. +underlies even core Django - it's less of an addition and more adding a whole +new layer under the existing view layer. Make a new project, a new app, and put this in a ``consumers.py`` file in the app:: @@ -42,18 +44,19 @@ Django request objects, and the ``AsgiHandler`` class handles translation of ``HttpResponse`` into ASGI messages, which you see used above. Usually, Django's built-in code will do all this for you when you're using normal views. -Now, go into your ``settings.py`` file, and set up a channel layer; by default, -Django will just use an in-memory layer and route HTTP requests to the normal -URL resolver (we'll come back to channel layers in a minute). +Now we need to do one more thing, and that's tell Django that this consumer +should be tied to the ``http.request`` channel rather than the default Django +view system. This is done in the settings file - in particular, we need to +define our ``default`` channel layer and what its routing is set to. -For now, we want to override the *channel routing* so that, rather than going -to the URL resolver and our normal view stack, all HTTP requests go to our -custom consumer we wrote above. Here's what that looks like:: +Channel routing is a bit like URL routing, and so it's structured similarly - +you point the setting at a dict mapping channels to consumer callables. +Here's what that looks like:: # In settings.py CHANNEL_LAYERS = { "default": { - "BACKEND": "channels.database_layer.DatabaseChannelLayer", + "BACKEND": "asgiref.inmemory.ChannelLayer", "ROUTING": "myproject.routing.channel_routing", }, } @@ -63,9 +66,16 @@ custom consumer we wrote above. Here's what that looks like:: "http.request": "myproject.myapp.consumers.http_consumer", } +.. warning:: + This example, and most of the examples here, use the "in memory" channel + layer. This is the easiest to get started with but provides absolutely no + cross-process channel transportation, and so can only be used with + ``runserver``. You'll want to choose another backend (discussed later) + to run things in production. + As you can see, this is a little like Django's ``DATABASES`` setting; there are named channel layers, with a default one called ``default``. Each layer -needs a class specified which powers it - we'll come to the options there later - +needs a channel layer class, some options (if the channel layer needs them), and a routing scheme, which points to a dict containing the routing settings. It's recommended you call this ``routing.py`` and put it alongside ``urls.py`` in your project, but you can put it wherever you like, as long as the path is @@ -159,8 +169,30 @@ And what our routing should look like in ``routing.py``:: } With all that code, you now have a working set of a logic for a chat server. -All you need to do now is get it deployed, and as we'll see, that's not too -hard. +Let's test it! Run ``runserver``, open a browser and put the following into the +JavaScript console to open a WebSocket and send some data down it:: + + // Note that the path doesn't matter right now; any WebSocket + // connection gets bumped over to WebSocket consumers + socket = new WebSocket("ws://127.0.0.1:8000/chat/"); + socket.onmessage = function(e) { + alert(e.data); + } + socket.onopen = function() { + socket.send("hello world"); + } + +You should see an alert come back immediately saying "hello world" - your +message has round-tripped through the server and come back to trigger the alert. +You can open another tab and do the same there if you like, and both tabs will +receive the message and show an alert, as any incoming message is sent to the +``chat`` group by the ``ws_message`` consumer, and both your tabs will have +been put into the ``chat`` group when they connected. + +Feel free to put some calls to ``print`` in your handler functions too, if you +like, so you can understand when they're called. You can also use ``pdb`` and +other methods you'd use to debug normal Django projects. + Running with Channels --------------------- @@ -185,13 +217,13 @@ By default, Django doesn't have a channel layer configured - it doesn't need one normal WSGI requests, after all. As soon as you try to add some consumers, though, you'll need to configure one. -In the example above we used the database channel layer implementation -as our default channel layer. This uses two tables -in the ``default`` database to do message handling, and isn't particularly fast but -requires no extra dependencies, so it's handy for development. -When you deploy to production, though, you'll want to -use a backend like the Redis backend that has much better throughput and -lower latency. +In the example above we used the in-memory channel layer implementation +as our default channel layer. This just stores all the channel data in a dict +in memory, and so isn't actually cross-process; it only works inside +``runserver``, as that runs the interface and worker servers in different threads +inside the same process. When you deploy to production, you'll need to +use a channel layer like the Redis backend ``asgi_redis`` that works cross-process; +see :doc:`backends` for more. The second thing, once we have a networked channel backend set up, is to make sure we're running an interface server that's capable of serving WebSockets. @@ -205,31 +237,32 @@ different. with autoreload in another - it's basically a miniature version of a deployment, but all in one process)* -Now, let's test our code. Open a browser and put the following into the -JavaScript console to open a WebSocket and send some data down it:: +Let's try out the Redis backend - Redis runs on pretty much every machine, and +has a very small overhead, which makes it perfect for this kind of thing. Install +the ``asgi_redis`` package using ``pip``, and set up your channel layer like this:: - // Note that the path doesn't matter right now; any WebSocket - // connection gets bumped over to WebSocket consumers - socket = new WebSocket("ws://127.0.0.1:8000/chat/"); - socket.onmessage = function(e) { - alert(e.data); - } - socket.onopen = function() { - socket.send("hello world"); + # In settings.py + CHANNEL_LAYERS = { + "default": { + "BACKEND": "asgi_redis.RedisChannelLayer", + "CONFIG": { + "hosts": [("localhost", 6379)], + }, + "ROUTING": "myproject.routing.channel_routing", + }, } -You should see an alert come back immediately saying "hello world" - your -message has round-tripped through the server and come back to trigger the alert. -You can open another tab and do the same there if you like, and both tabs will -receive the message and show an alert, as any incoming message is sent to the -``chat`` group by the ``ws_message`` consumer, and both your tabs will have -been put into the ``chat`` group when they connected. +Fire up ``runserver``, and it'll work as before - unexciting, like good +infrastructure should be. You can also try out the cross-process nature; run +these two commands in two terminals: + +* ``manage.py runserver --noworker`` +* ``manage.py runworker`` + +As you can probably guess, this disables the worker threads in ``runserver`` +and handles them in a separate process. You can pass ``-v 2`` to ``runworker`` +if you want to see logging as it runs the consumers. -Feel free to put some calls to ``print`` in your handler functions too, if you -like, so you can understand when they're called. You can also run separate -worker processes with ``manage.py runworker`` as well - if you do this, you -should see some of the consumers being handled in the ``runserver`` thread and -some in the separate worker process. Persisting Data --------------- From 61b8940e9978cbedf9408fa4340858ac1c96fabc Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 21:52:35 -0800 Subject: [PATCH 212/746] Warn when trying to use runworker with in memory layer --- channels/asgi.py | 4 ++++ channels/management/commands/runworker.py | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/channels/asgi.py b/channels/asgi.py index 29eacc3..9911578 100644 --- a/channels/asgi.py +++ b/channels/asgi.py @@ -75,6 +75,10 @@ class ChannelLayerWrapper(object): def __str__(self): return "%s (%s)" % (self.alias, name_that_thing(self.channel_layer)) + def local_only(self): + # TODO: Can probably come up with a nicer check? + return "inmemory" in self.channel_layer.__class__.__module__ + def get_channel_layer(alias="default"): """ diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 35fd38e..e5fa5ff 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from django.core.management import BaseCommand +from django.core.management import BaseCommand, CommandError from channels import channel_layers, DEFAULT_CHANNEL_LAYER from channels.log import setup_logger from channels.worker import Worker @@ -13,6 +13,12 @@ class Command(BaseCommand): self.verbosity = options.get("verbosity", 1) self.logger = setup_logger('django.channels', self.verbosity) self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + # Check that handler isn't inmemory + if self.channel_layer.local_only(): + raise CommandError( + "You cannot span multiple processes with the in-memory layer. " + + "Change your settings to use a cross-process channel layer." + ) # Check a handler is registered for http reqs self.channel_layer.registry.check_default() # Launch a worker From b129adf4a41febae26059ed9b2ffb256499c378e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 22:04:03 -0800 Subject: [PATCH 213/746] Increase FileResponse chunk size for staticfiles handler --- channels/staticfiles.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/channels/staticfiles.py b/channels/staticfiles.py index 8169be8..153226d 100644 --- a/channels/staticfiles.py +++ b/channels/staticfiles.py @@ -49,7 +49,10 @@ class StaticFilesHandler(AsgiHandler): if self._should_handle(request.path): try: - return self.serve(request) + response = self.serve(request) + # Increase FileResponse block sizes so they're not super slow + response.block_size = 1024 * 256 + return response except Http404 as e: if settings.DEBUG: from django.views import debug From cdde27b55ae5012082ca3b5dad685f6afd94f4ec Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 22:06:25 -0800 Subject: [PATCH 214/746] Releasing 0.9.3 --- CHANGELOG.txt | 8 ++++++++ channels/__init__.py | 2 +- docs/index.rst | 1 - docs/releases/0.8.rst | 22 ---------------------- docs/releases/index.rst | 7 ------- 5 files changed, 9 insertions(+), 31 deletions(-) delete mode 100644 docs/releases/0.8.rst delete mode 100644 docs/releases/index.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index c4aab66..7c22505 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +0.9.3 (2016-02-28) +------------------ + +* Static file serving is significantly faster thanks to larger chunk size + +* `runworker` now refuses to start if an in memory layer is configured + + 0.9.2 (2016-02-28) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index fc4ca95..250b9d0 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.9.2" +__version__ = "0.9.3" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/index.rst b/docs/index.rst index 881089d..47e2413 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,4 +35,3 @@ Contents: integration-plan faqs asgi - releases/index diff --git a/docs/releases/0.8.rst b/docs/releases/0.8.rst deleted file mode 100644 index 9c4d15c..0000000 --- a/docs/releases/0.8.rst +++ /dev/null @@ -1,22 +0,0 @@ -0.8 (2015-09-10) ----------------- - -This release reworks a few of the core concepts to make the channel layer -more efficient and user friendly: - -* Channel names now do not start with ``django``, and are instead just ``http.request``, etc. - -* HTTP headers/GET/etc are only sent with ``websocket.connect`` rather than all websocket requests, - to save a lot of bandwidth in the channel layer. - -* The session/user decorators were renamed, and a ``@channel_session_user`` and ``transfer_user`` set of functions - added to allow moving the user details from the HTTP session to the channel session in the ``connect`` consumer. - -* A ``@linearize`` decorator was added to help ensure a ``connect``/``receive`` pair are not executed - simultaneously on two different workers. - -* Channel backends gained locking mechanisms to support the ``linearize`` feature. - -* ``runwsserver`` will use asyncio rather than Twisted if it's available. - -* Message formats have been made a bit more consistent. diff --git a/docs/releases/index.rst b/docs/releases/index.rst deleted file mode 100644 index 231771f..0000000 --- a/docs/releases/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Release Notes -------------- - -.. toctree:: - :maxdepth: 1 - - 0.8 From 00dd9615ff8a9d89808b4cfe90bd58e73e1a6c02 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 22:10:02 -0800 Subject: [PATCH 215/746] Update inshort a little. --- docs/inshort.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/inshort.rst b/docs/inshort.rst index e006150..b4258a6 100644 --- a/docs/inshort.rst +++ b/docs/inshort.rst @@ -11,6 +11,8 @@ that allows two important features: * WebSocket handling, in a way very :ref:`similar to normal views ` * Background tasks, running in the same servers as the rest of Django +It allows other things too, but these are the ones you'll use to start with. + How? ---- @@ -36,7 +38,7 @@ change from running Django under a WSGI server, to running: * An ASGI server, probably `Daphne `_ * Django worker servers, using ``manage.py runworker`` -* Something to route ASGI requests over, like Redis or a database. +* Something to route ASGI requests over, like Redis. What else does Channels give me? @@ -74,7 +76,7 @@ and high throughput over guaranteed delivery, which doesn't match some message queue designs. Some features, like :ref:`guaranteed ordering of messages `, -are opt-in as they incur a performance hit. +are opt-in as they incur a performance hit, but make it more message queue like. Do I need to worry about making all my code async-friendly? From d0a927993934efbcc03cf54bbae6e453982a4d58 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 22:11:41 -0800 Subject: [PATCH 216/746] Bit more inshort tweaking --- docs/inshort.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/inshort.rst b/docs/inshort.rst index b4258a6..b87ac15 100644 --- a/docs/inshort.rst +++ b/docs/inshort.rst @@ -40,6 +40,9 @@ change from running Django under a WSGI server, to running: * Django worker servers, using ``manage.py runworker`` * Something to route ASGI requests over, like Redis. +Even when you're running on Channels, it routes all HTTP requests to the Django +view system by default, so it works like before. + What else does Channels give me? -------------------------------- From 54dc80e9a5141856ab5c59d47207687550c8bacd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 28 Feb 2016 22:13:44 -0800 Subject: [PATCH 217/746] Typo fix in ordering page. --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 9103d0d..d6051f4 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -485,7 +485,7 @@ Enforcing Ordering ------------------ There's one final concept we want to introduce you to before you go on to build -sites with Channels - consmer ordering +sites with Channels - consumer ordering. Because Channels is a distributed system that can have many workers, by default it just processes messages in the order the workers get them off the queue. From 8fba5220d905ec7fe10ddf228a30bbcbf2aa7e5d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Mar 2016 10:42:18 -0800 Subject: [PATCH 218/746] Detect if you forget to decorate things with enforce_ordering --- channels/sessions.py | 6 ++++-- channels/worker.py | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/channels/sessions.py b/channels/sessions.py index a9e847e..a622819 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -1,5 +1,6 @@ import functools import hashlib +import warnings from importlib import import_module from django.conf import settings @@ -93,8 +94,9 @@ def enforce_ordering(func=None, slight=False): # Run consumer return func(message, *args, **kwargs) else: - # Bad ordering - print("Bad ordering detected: next %s, us %s, %s" % (next_order, order, message.reply_channel)) + # Bad ordering - warn if we're getting close to the limit + if getattr(message, "__doomed__", False): + warnings.warn("Enforce ordering consumer reached retry limit, message being dropped. Did you decorate all protocol consumers correctly?") raise ConsumeLater() return inner if func is not None: diff --git a/channels/worker.py b/channels/worker.py index 927b980..8bf4ab8 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -16,9 +16,10 @@ class Worker(object): and runs their consumers. """ - def __init__(self, channel_layer, callback=None): + def __init__(self, channel_layer, callback=None, message_retries=10): self.channel_layer = channel_layer self.callback = callback + self.message_retries = message_retries def run(self): """ @@ -38,6 +39,11 @@ class Worker(object): channel_name=channel, channel_layer=self.channel_layer, ) + # Add attribute to message if it's been retried almost too many times, + # and would be thrown away this time if it's requeued. Used for helpful + # warnings in decorators and such - don't rely on this as public API. + if content.get("__retries__", 0) == self.message_retries: + message.__doomed__ = True # Handle the message consumer = self.channel_layer.registry.consumer_for_channel(channel) if self.callback: @@ -45,6 +51,17 @@ class Worker(object): try: consumer(message) except ConsumeLater: + # They want to not handle it yet. Re-inject it with a number-of-tries marker. + content['__retries__'] = content.get("__retries__", 0) + 1 + # If we retried too many times, quit and error rather than + # spinning forever + if content['__retries__'] > self.message_retries: + logger.warning( + "Exceeded number of retries for message on channel %s: %s", + channel, + repr(content)[:100], + ) + continue self.channel_layer.send(channel, content) except: logger.exception("Error processing message with consumer %s:", name_that_thing(consumer)) From 447c2e77cc781e9d347f4fc9557d0b31c75a955d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Mar 2016 10:46:05 -0800 Subject: [PATCH 219/746] Fix line length --- channels/sessions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/channels/sessions.py b/channels/sessions.py index a622819..ece4690 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -96,7 +96,10 @@ def enforce_ordering(func=None, slight=False): else: # Bad ordering - warn if we're getting close to the limit if getattr(message, "__doomed__", False): - warnings.warn("Enforce ordering consumer reached retry limit, message being dropped. Did you decorate all protocol consumers correctly?") + warnings.warn( + "Enforce ordering consumer reached retry limit, message " + "being dropped. Did you decorate all protocol consumers correctly?" + ) raise ConsumeLater() return inner if func is not None: From 1869061fa2b7b9c485b92981eb3d1d2bf5dccb8f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Mar 2016 10:46:22 -0800 Subject: [PATCH 220/746] I like having a plus here. --- channels/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/sessions.py b/channels/sessions.py index ece4690..a80cc61 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -97,7 +97,7 @@ def enforce_ordering(func=None, slight=False): # Bad ordering - warn if we're getting close to the limit if getattr(message, "__doomed__", False): warnings.warn( - "Enforce ordering consumer reached retry limit, message " + "Enforce ordering consumer reached retry limit, message " + "being dropped. Did you decorate all protocol consumers correctly?" ) raise ConsumeLater() From 484320a67530d34256b4ed8619f162b1adcda5c3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Mar 2016 17:00:49 -0800 Subject: [PATCH 221/746] Reduce Daphne HTTP timeout in runserver mode --- channels/management/commands/runserver.py | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index b056d15..9f1b2dd 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -79,6 +79,7 @@ class Command(RunserverCommand): port=int(self.port), signal_handlers=not options['use_reloader'], action_logger=self.log_action, + http_timeout=60, # Shorter timeout than normal as it's dev ).run() self.logger.debug("Daphne exited") except KeyboardInterrupt: diff --git a/setup.py b/setup.py index 767b93f..fc4666c 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,6 @@ setup( install_requires=[ 'Django>=1.7', 'asgiref>=0.9', - 'daphne>=0.9', + 'daphne>=0.9.2', ] ) From 05e0f739d506ec183c94718358716803806b6b94 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Mar 2016 17:11:24 -0800 Subject: [PATCH 222/746] Deployment docs tweak --- docs/deploying.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 036b467..004bf4d 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -97,18 +97,20 @@ cluster on the backend, see :ref:`wsgi-to-asgi`. If you want to support WebSockets, long-poll HTTP requests and other Channels features, you'll need to run a native ASGI interface server, as the WSGI -specification has no support for running these kinds of requests concurrenctly. +specification has no support for running these kinds of requests concurrently. Channels ships with an interface server that we recommend you use called -*Daphne*; it supports WebSockets, long-poll HTTP requests, HTTP/2 *(soon)* -and performs quite well. Of course, any ASGI-compliant server will work! +`Daphne `_; it supports WebSockets, +long-poll HTTP requests, HTTP/2 *(soon)* and performs quite well. +Of course, any ASGI-compliant server will work! Notably, Daphne has a nice feature where it supports all of these protocols on the same port and on all paths; it auto-negotiates between HTTP and WebSocket, so there's no need to have your WebSockets on a separate port or path (and they'll be able to share cookies with your normal view code). -To run Daphne, it just needs to be supplied with a channel backend; -first, make sure your project has an ``asgi.py`` file that looks like this +To run Daphne, it just needs to be supplied with a channel backend, in much +the same way a WSGI server needs to be given an application. +First, make sure your project has an ``asgi.py`` file that looks like this (it should live next to ``wsgi.py``):: import os @@ -128,7 +130,10 @@ like supervisord to ensure it is re-run if it exits unexpectedly. If you only run Daphne and no workers, all of your page requests will seem to hang forever; that's because Daphne doesn't have any worker servers to handle the request and it's waiting for one to appear (while ``runserver`` also uses -Daphne, it launches a worker thread along with it in the same process). +Daphne, it launches a worker thread along with it in the same process). In this +scenario, it will eventually time out and give you a 503 error after 2 minutes; +you can configure how long it waits with the ``--http-timeout`` command line +argument. Deploying new versions of code From 44568dab5bc5465e6dbf4bc7d37a4f342d1c54c1 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Mar 2016 17:12:18 -0800 Subject: [PATCH 223/746] Oops on package name. --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 004bf4d..f2298d1 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -169,7 +169,7 @@ WSGI servers with an adapter layer. You won't get WebSocket support this way - WSGI doesn't support WebSockets - but you can run a separate ASGI server to handle WebSockets if you want. -The ``wsgiref`` package contains the adapter; all you need to do is put this +The ``asgiref`` package contains the adapter; all you need to do is put this in your Django project's ``wsgi.py`` to declare a new WSGI application object that backs onto ASGI underneath:: From 08d1f9a14d5f32817e862a36a9b0316068cbd6d8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Mar 2016 17:21:58 -0800 Subject: [PATCH 224/746] Graceful shutdown for workers on SIGTERM and SIGINT --- channels/worker.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/channels/worker.py b/channels/worker.py index 8bf4ab8..ce3a7f8 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -1,6 +1,8 @@ from __future__ import unicode_literals import logging +import signal +import sys import time from .exceptions import ConsumeLater @@ -16,18 +18,37 @@ class Worker(object): and runs their consumers. """ - def __init__(self, channel_layer, callback=None, message_retries=10): + def __init__(self, channel_layer, callback=None, message_retries=10, signal_handlers=True): self.channel_layer = channel_layer self.callback = callback self.message_retries = message_retries + self.signal_handlers = signal_handlers + self.termed = False + self.in_job = False + + def install_signal_handler(self): + signal.signal(signal.SIGTERM, self.sigterm_handler) + signal.signal(signal.SIGINT, self.sigterm_handler) + + def sigterm_handler(self, signo, stack_frame): + self.termed = True + if self.in_job: + logger.info("Shutdown signal received while busy, waiting for loop termination") + else: + logger.info("Shutdown signal received while idle, terminating immediately") + sys.exit(0) def run(self): """ Tries to continually dispatch messages to consumers. """ + if self.signal_handlers: + self.install_signal_handler() channels = self.channel_layer.registry.all_channel_names() - while True: + while not self.termed: + self.in_job = False channel, content = self.channel_layer.receive_many(channels, block=True) + self.in_job = True # If no message, stall a little to avoid busy-looping then continue if channel is None: time.sleep(0.01) From c4719f79bc16c95fbedae276e129e466a07ff263 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 6 Mar 2016 11:11:58 -0800 Subject: [PATCH 225/746] Don't make worker thread do signals during runserver --- channels/management/commands/runserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 9f1b2dd..e470cc0 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -148,6 +148,6 @@ class WorkerThread(threading.Thread): def run(self): self.logger.debug("Worker thread running") - worker = Worker(channel_layer=self.channel_layer) + worker = Worker(channel_layer=self.channel_layer, signal_handlers=False) worker.run() self.logger.debug("Worker thread exited") From a0dff726b2e23737d498bc64622244e1f9528de4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 6 Mar 2016 11:28:52 -0800 Subject: [PATCH 226/746] Rework getting started section to do groups after basic sending. --- docs/getting-started.rst | 111 ++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index d6051f4..984aa63 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -90,54 +90,106 @@ Now, that's not very exciting - raw HTTP responses are something Django has been able to do for a long time. Let's try some WebSockets, and make a basic chat server! -Delete that consumer and its routing - we'll want the normal Django view layer to +We'll start with a simple server that just echoes every message it gets sent +back to the same client - no cross-client communication. It's not terribly +useful, but it's a good way to start out writing Channels consumers. + +Delete that previous consumer and its routing - we'll want the normal Django view layer to serve HTTP requests from now on, which happens if you don't specify a consumer for ``http.request`` - and make this WebSocket consumer instead:: # In consumers.py from channels import Group - def ws_add(message): - Group("chat").add(message.reply_channel) + def ws_message(message): + # ASGI WebSocket packet-received and send-packet message types + # both have a "text" key for their textual data. + message.reply_channel.send({ + "text": message.content['text'], + }) -Hook it up to the ``websocket.connect`` channel like this:: +Hook it up to the ``websocket.receive`` channel like this:: # In routing.py - from myproject.myapp.consumers import ws_add + from myproject.myapp.consumers import ws_message channel_routing = { - "websocket.connect": ws_add, + "websocket.receive": ws_message, } Now, let's look at what this is doing. It's tied to the -``websocket.connect`` channel, which means that it'll get a message -whenever a new WebSocket connection is opened by a client. +``websocket.receive`` channel, which means that it'll get a message +whenever a WebSocket packet is sent to us by a client. When it gets that message, it takes the ``reply_channel`` attribute from it, which -is the unique response channel for that client, and adds it to the ``chat`` -group, which means we can send messages to all connected chat clients. +is the unique response channel for that client, and sends the same content +back to the client using its ``send()`` method. -Of course, if you've read through :doc:`concepts`, you'll know that channels -added to groups expire out if their messages expire (every channel layer has -a message expiry time, usually between 30 seconds and a few minutes, and it's -often configurable). +Let's test it! Run ``runserver``, open a browser and put the following into the +JavaScript console to open a WebSocket and send some data down it (you might +need to change the socket address if you're using a development VM or similar):: -However, we'll still get disconnection messages most of the time when a -WebSocket disconnects; the expiry/garbage collection of group membership is -mostly there for when a disconnect message gets lost (channels are not -guaranteed delivery, just mostly reliable). Let's add an explicit disconnect -handler:: + // Note that the path doesn't matter for routing; any WebSocket + // connection gets bumped over to WebSocket consumers + socket = new WebSocket("ws://127.0.0.1:8000/chat/"); + socket.onmessage = function(e) { + alert(e.data); + } + socket.onopen = function() { + socket.send("hello world"); + } + +You should see an alert come back immediately saying "hello world" - your +message has round-tripped through the server and come back to trigger the alert. + +Groups +------ + +Now, let's make our echo server into an actual chat server, so people can talk +to each other. To do this, we'll use Groups, one of the :doc:`core concepts ` +of Channels, and our fundamental way of doing multi-cast messaging. + +To do this, we'll hook up the ``websocket.connect`` and ``websocket.disconnect`` +channels to add and remove our clients from the Group as they connect and +disconnect, like this:: + + # In consumers.py + from channels import Group + + # Connected to websocket.connect + def ws_add(message): + Group("chat").add(message.reply_channel) # Connected to websocket.disconnect def ws_disconnect(message): Group("chat").discard(message.reply_channel) +Of course, if you've read through :doc:`concepts`, you'll know that channels +added to groups expire out if their messages expire (every channel layer has +a message expiry time, usually between 30 seconds and a few minutes, and it's +often configurable) - but the ``disconnect`` handler will get called nearly all +of the time anyway. + +.. _note: + Channels' design is predicated on expecting and working around failure; + it assumes that some small percentage of messages will never get delivered, + and so all the core functionality is designed to *expect failure* so that + when a message doesn't get delivered, it doesn't ruin the whole system. + + We suggest you design your applications the same way - rather than relying + on 100% guaranteed delivery, which Channels won't give you, look at each + failure case and program something to expect and handle it - be that retry + logic, partial content handling, or just having something not work that one + time. HTTP requests are just as fallible, and most people's reponse to that + is a generic error page! + .. _websocket-example: Now, that's taken care of adding and removing WebSocket send channels for the -``chat`` group; all we need to do now is take care of message sending. For now, -we're not going to store a history of messages or anything and just replay -any message sent in to all connected clients. Here's all the code:: +``chat`` group; all we need to do now is take care of message sending. Instead +of echoing the message back to the client like we did above, we'll instead send +it to the whole ``Group``, which means any client who's been added to it will +get the message. Here's all the code:: # In consumers.py from channels import Group @@ -148,8 +200,6 @@ any message sent in to all connected clients. Here's all the code:: # Connected to websocket.receive def ws_message(message): - # ASGI WebSocket packet-received and send-packet message types - # both have a "text" key for their textual data. Group("chat").send({ "text": "[user] %s" % message.content['text'], }) @@ -169,8 +219,8 @@ And what our routing should look like in ``routing.py``:: } With all that code, you now have a working set of a logic for a chat server. -Let's test it! Run ``runserver``, open a browser and put the following into the -JavaScript console to open a WebSocket and send some data down it:: +Test time! Run ``runserver``, open a browser and use that same JavaScript +code in the developer console as before:: // Note that the path doesn't matter right now; any WebSocket // connection gets bumped over to WebSocket consumers @@ -182,16 +232,15 @@ JavaScript console to open a WebSocket and send some data down it:: socket.send("hello world"); } -You should see an alert come back immediately saying "hello world" - your -message has round-tripped through the server and come back to trigger the alert. -You can open another tab and do the same there if you like, and both tabs will -receive the message and show an alert, as any incoming message is sent to the +You should see an alert come back immediately saying "hello world" - but this +time, you can open another tab and do the same there, and both tabs will +receive the message and show an alert. Any incoming message is sent to the ``chat`` group by the ``ws_message`` consumer, and both your tabs will have been put into the ``chat`` group when they connected. Feel free to put some calls to ``print`` in your handler functions too, if you like, so you can understand when they're called. You can also use ``pdb`` and -other methods you'd use to debug normal Django projects. +other similar methods you'd use to debug normal Django projects. Running with Channels From 4b82d989ca5dfe4a98c492c43491a2dccdfd6ed3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 6 Mar 2016 12:12:48 -0800 Subject: [PATCH 227/746] Remove old code from Channel/Group --- channels/channel.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/channels/channel.py b/channels/channel.py index 3d28a92..465e267 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -36,15 +36,6 @@ class Channel(object): raise ValueError("You can only send dicts as content on channels.") self.channel_layer.send(self.name, content) - def as_view(self): - """ - Returns a view version of this channel - one that takes - the request passed in and dispatches it to our channel, - serialized. - """ - from channels.adapters import view_producer - return view_producer(self.name) - def __str__(self): return self.name @@ -74,9 +65,6 @@ class Group(object): channel = channel.name self.channel_layer.group_discard(self.name, channel) - def channels(self): - return self.channel_layer.group_channels(self.name) - def send(self, content): if not isinstance(content, dict): raise ValueError("You can only send dicts as content on channels.") From 8e978459a90a109cae1738beb91eeba8db874292 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 6 Mar 2016 12:20:11 -0800 Subject: [PATCH 228/746] Add reference --- docs/index.rst | 1 + docs/reference.rst | 170 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 docs/reference.rst diff --git a/docs/index.rst b/docs/index.rst index 47e2413..9a74541 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Contents: integration-changes scaling backends + reference integration-plan faqs asgi diff --git a/docs/reference.rst b/docs/reference.rst new file mode 100644 index 0000000..5b875a5 --- /dev/null +++ b/docs/reference.rst @@ -0,0 +1,170 @@ +Reference +========= + +.. _ref-consumers: + +Consumers +--------- + +When you configure channel routing, Channels expects the object assigned to +a channel to be a callable that takes exactly one positional argument, here +called ``message``. This is a :ref:`message object `. + +Consumers are not expected to return anything, and if they do, it will be +ignored. They may raise ``channels.exceptions.ConsumeLater`` to re-insert +their current message at the back of the channel it was on, but be aware you +can only do this so many time (10 by default) until the message is dropped +to avoid deadlocking. + + +.. _ref-message: + +Message +------- + +Message objects are what consumers get passed as their only argument. They +encapsulate the basic :doc:`ASGI ` message, which is a ``dict``, with +extra information. They have the following attributes: + +* ``content``: The actual message content, as a dict. See the + :doc:`ASGI spec ` or protocol message definition document for how this + is structured. + +* ``channel``: A :ref:`Channel ` object, representing the channel + this message was received on. Useful if one consumer handles multiple channels. + +* ``reply_channel``: A :ref:`Channel ` object, representing the + unique reply channel for this message, or ``None`` if there isn't one. + +* ``channel_layer``: A :ref:`ChannelLayer ` object, + representing the underlying channel layer this was received on. This can be + useful in projects that have more than one layer to identify where to send + messages the consumer generates (you can pass it to the constructor of + :ref:`Channel ` or :ref:`Group `) + + +.. _ref-channel: + +Channel +------- + +Channel objects are a simple abstraction around ASGI channels, which by default +are unicode strings. The constructor looks like this:: + + channels.Channel(name, alias=DEFAULT_CHANNEL_LAYER, channel_layer=None) + +Normally, you'll just call ``Channel("my.channel.name")`` and it'll make the +right thing, but if you're in a project with multiple channel layers set up, +you can pass in either the layer alias or the layer object and it'll send +onto that one instead. They have the following attributes: + +* ``name``: The unicode string representing the channel name. + +* ``channel_layer``: A :ref:`ChannelLayer ` object, + representing the underlying channel layer to send messages on. + +* ``send(content)``: Sends the ``dict`` provided as *content* over the channel. + The content should conform to the relevant ASGI spec or protocol definition. + + +.. _ref-group: + +Group +----- + +Groups represent the underlying :doc:`ASGI ` group concept in an +object-oriented way. The constructor looks like this:: + + channels.Group(name, alias=DEFAULT_CHANNEL_LAYER, channel_layer=None) + +Like :ref:`Channel `, you would usually just pass a ``name``, but +can pass a layer alias or object if you want to send on a non-default one. +They have the following attributes: + +* ``name``: The unicode string representing the group name. + +* ``channel_layer``: A :ref:`ChannelLayer ` object, + representing the underlying channel layer to send messages on. + +* ``send(content)``: Sends the ``dict`` provided as *content* to all + members of the group. + +* ``add(channel)``: Adds the given channel (as either a :ref:`Channel ` + object or a unicode string name) to the group. If the channel is already in + the group, does nothing. + +* ``discard(channel)``: Removes the given channel (as either a + :ref:`Channel ` object or a unicode string name) from the group, + if it's in the group. Does nothing otherwise. + + +.. _ref-channellayer: + +Channel Layer +------------- + +These are a wrapper around the underlying :doc:`ASGI ` channel layers +that supplies a routing system that maps channels to consumers, as well as +aliases to help distinguish different layers in a project with multiple layers. + +You shouldn't make these directly; instead, get them by alias (``default`` is +the default alias):: + + from channels import channel_layers + layer = channel_layers["default"] + +They have the following attributes: + +* ``alias``: The alias of this layer. + +* ``registry``: An object which represents the layer's mapping of channels + to consumers. Has the following attributes: + + * ``add_consumer(consumer, channels)``: Registers a :ref:`consumer ` + to handle all channels passed in. ``channels`` should be an iterable of + unicode string names. + + * ``consumer_for_channel(channel)``: Takes a unicode channel name and returns + either a :ref:`consumer `, or None, if no consumer is registered. + + * ``all_channel_names()``: Returns a list of all channel names this layer has + routed to a consumer. Used by the worker threads to work out what channels + to listen on. + + +.. _ref-asgirequest: + +AsgiRequest +----------- + +This is a subclass of ``django.http.HttpRequest`` that provides decoding from +ASGI requests, and a few extra methods for ASGI-specific info. The constructor is:: + + channels.handler.AsgiRequest(message) + +``message`` must be an :doc:`ASGI ` ``http.request`` format message. + +Additional attributes are: + +* ``reply_channel``, a :ref:`Channel ` object that represents the + ``http.response.?`` reply channel for this request. + +* ``message``, the raw ASGI message passed in the constructor. + + +.. _ref-asgihandler: + +AsgiHandler +----------- + +This is a class in ``channels.handler`` that's designed to handle the workflow +of HTTP requests via ASGI messages. You likely don't need to interact with it +directly, but there are two useful ways you can call it: + +* ``AsgiHandler(message)`` will process the message through the Django view + layer and yield one or more response messages to send back to the client, + encoded from the Django ``HttpResponse``. + +* ``encode_response(response)`` is a classmethod that can be called with a + Django ``HttpResponse`` and will yield one or more ASGI messages that are + the encoded response. From d26e04a56bd8cc486e0c264a8894b41136ccf7c5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 6 Mar 2016 13:34:23 -0800 Subject: [PATCH 229/746] Fix note formatting --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 984aa63..25af12a 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -170,7 +170,7 @@ a message expiry time, usually between 30 seconds and a few minutes, and it's often configurable) - but the ``disconnect`` handler will get called nearly all of the time anyway. -.. _note: +.. note:: Channels' design is predicated on expecting and working around failure; it assumes that some small percentage of messages will never get delivered, and so all the core functionality is designed to *expect failure* so that From 930d71039e4b4492f28a8752c485ea2432992207 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 6 Mar 2016 13:34:58 -0800 Subject: [PATCH 230/746] Update docs version --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9be3d8a..fe3f65e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,9 +51,9 @@ copyright = u'2015, Andrew Godwin' # built documents. # # The short X.Y version. -version = '0.2' +version = '1.0' # The full version, including alpha/beta/rc tags. -release = '0.2' +release = '1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From a2d64f933570815b02d7a9340a7397023d4916ac Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 8 Mar 2016 10:20:08 -0800 Subject: [PATCH 231/746] Releasing 0.9.4 --- CHANGELOG.txt | 10 ++++++++++ channels/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 7c22505..6339ad1 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,13 @@ +0.9.4 (2016-03-08) +------------------ + +* Worker processes now exit gracefully (finish their current processing) when + sent SIGTERM or SIGINT. + +* `runserver` now has a shorter than standard HTTP timeout configured + of 60 seconds. + + 0.9.3 (2016-02-28) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index 250b9d0..100bad1 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.9.3" +__version__ = "0.9.4" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 69d60a60c5dc17e9f1eaa6957ca834af4f02a06b Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 10 Mar 2016 09:25:02 +0000 Subject: [PATCH 232/746] Clearly you're allowing more extensions. --- docs/asgi.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 61c0cfa..ec965fd 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -286,7 +286,8 @@ A *channel layer* should provide an object with these attributes * ``extensions``, a list of unicode string names indicating which extensions this layer provides, or empty if it supports none. - The only valid extension names are ``groups``, ``flush`` and ``statistics``. + The names defined in this document are ``groups``, ``flush`` and + ``statistics``. A channel layer implementing the ``groups`` extension must also provide: From 1bb48108fdef355a0dce88733e613efd18f44487 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 10 Mar 2016 09:14:22 +0000 Subject: [PATCH 233/746] Stylistic changes --- docs/asgi.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 61c0cfa..9449818 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -99,8 +99,8 @@ contain only the following types to ensure serializability: * None Channels are identified by a unicode string name consisting only of ASCII -letters, numbers, numerical digits, periods (``.``), dashes (``-``) -and underscores (``_``), plus an optional prefix character (see below). +letters, ASCII numerical digits, periods (``.``), dashes (``-``) and +underscores (``_``), plus an optional prefix character (see below). Channels are a first-in, first out queue with at-most-once delivery semantics. They can have multiple writers and multiple readers; only a single @@ -116,7 +116,7 @@ application worker process) and *single-reader channels* *Single-reader channel* names are prefixed with an exclamation mark (``!``) character in order to indicate to the channel layer that it may -have to route these channels' data differently to ensure it reaches the +have to route the data for these channels differently to ensure it reaches the single process that needs it; these channels are nearly always tied to incoming connections from the outside world. Some channel layers may not need this, and can simply treat the prefix as part of the name. @@ -125,7 +125,7 @@ Messages should expire after a set time sitting unread in a channel; the recommendation is one minute, though the best value depends on the channel layer and the way it is deployed. -Message size is finite, though the maximum varies based on the channel layer +The maximum message size is finite, though it varies based on the channel layer and the encoding it's using. Channel layers may reject messages at ``send()`` time with a ``MessageTooLarge`` exception; the calling code should take appropriate action (e.g. HTTP responses can be chunked, while HTTP @@ -143,10 +143,10 @@ uploaded videos), and protocol events to/from connected clients. As such, this specification outlines encodings to and from ASGI messages for three common protocols (HTTP, WebSocket and raw UDP); this allows any ASGI -web server to talk to any ASGI web application, and the same for any other -protocol with a common specification. It is recommended that if other -protocols become commonplace they should gain standardized formats in a -supplementary specification of their own. +web server to talk to any ASGI web application, as well as servers and +applications for any other protocol with a common specification. It is +recommended that if other protocols become commonplace they should gain +standardized formats in a supplementary specification of their own. The message formats are a key part of the specification; without them, the protocol server and web application might be able to talk to each other, @@ -262,7 +262,7 @@ Specification Details A *channel layer* should provide an object with these attributes (all function arguments are positional): -* ``send(channel, message)``, a callable that takes two arguments; the +* ``send(channel, message)``, a callable that takes two arguments: the channel to send on, as a unicode string, and the message to send, as a serializable ``dict``. @@ -317,10 +317,10 @@ A channel layer implementing the ``statistics`` extension must also provide: A channel layer implementing the ``flush`` extension must also provide: -* ``flush()``, a callable that resets the channel layer to no messages and - no groups (if groups is implemented). This call must block until the system - is cleared and will consistently look empty to any client, if the channel - layer is distributed. +* ``flush()``, a callable that resets the channel layer to a blank state, + containing no messages and no groups (if the groups extension is + implemented). This call must block until the system is cleared and will + consistently look empty to any client, if the channel layer is distributed. From ce0433f438103445b5c7ccc922ea38443ca7598b Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 10 Mar 2016 09:15:32 +0000 Subject: [PATCH 234/746] MUST is preferable to SHOULD --- docs/asgi.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 61c0cfa..db9fb3c 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -104,9 +104,9 @@ and underscores (``_``), plus an optional prefix character (see below). Channels are a first-in, first out queue with at-most-once delivery semantics. They can have multiple writers and multiple readers; only a single -reader should get each written message. Implementations should never -deliver a message more than once or to more than one reader, and should -drop messages if this is necessary to achieve this restriction. +reader should get each written message. Implementations must never deliver +a message more than once or to more than one reader, and must drop messages if +this is necessary to achieve this restriction. In order to aid with scaling and network architecture, a distinction is made between channels that have multiple readers (such as the @@ -259,7 +259,7 @@ receive messages in channel order. Specification Details ===================== -A *channel layer* should provide an object with these attributes +A *channel layer* must provide an object with these attributes (all function arguments are positional): * ``send(channel, message)``, a callable that takes two arguments; the @@ -408,10 +408,10 @@ different requests on the same connection different reply channels, and correctly multiplex the responses back into the same stream as they come in. The HTTP version is available as a string in the request message. -HTTP/2 Server Push responses are included, but should be sent prior to the -main response, and you should check for ``http_version = 2`` before sending -them; if a protocol server or connection incapable of Server Push receives -these, it should simply drop them. +HTTP/2 Server Push responses are included, but must be sent prior to the +main response, and applications must check for ``http_version = 2`` before +sending them; if a protocol server or connection incapable of Server Push +receives these, it must drop them. The HTTP specs are somewhat vague on the subject of multiple headers; RFC7230 explicitly says they must be merge-able with commas, while RFC6265 From c4b1798020b59b973f5790096f120c35452f2185 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 10 Mar 2016 09:43:05 +0000 Subject: [PATCH 235/746] The HTTP specs are quite clear. --- docs/asgi.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 61c0cfa..c7dbcd8 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -413,11 +413,17 @@ main response, and you should check for ``http_version = 2`` before sending them; if a protocol server or connection incapable of Server Push receives these, it should simply drop them. -The HTTP specs are somewhat vague on the subject of multiple headers; -RFC7230 explicitly says they must be merge-able with commas, while RFC6265 -says that ``Set-Cookie`` headers cannot be combined this way. This is why -request ``headers`` is a ``dict``, and response ``headers`` is a list of -tuples, which matches WSGI. +Multiple header fields with the same name are complex in HTTP. RFC 7230 +states that for any header field that can appear multiple times, it is exactly +equivalent to sending that header field only once with all the values joined by +commas. + +However, RFC 7230 and RFC 6265 make it clear that this rule does not apply to +the various headers used by HTTP cookies (``Cookie`` and ``Set-Cookie``). The +``Cookie`` header must only be sent once by a user-agent, but the +``Set-Cookie`` header may appear repeatedly and cannot be joined by commas. +For this reason, we can safely make the request ``headers`` a ``dict``, but +the response ``headers`` must be sent as a list of tuples, which matches WSGI. Request ''''''' From e65230b677ca2fde1e2a418ac57ee1ff0144f757 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 10 Mar 2016 09:44:48 +0000 Subject: [PATCH 236/746] Down with RFC 2616 --- docs/asgi.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index c7dbcd8..17cdd83 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -456,7 +456,7 @@ Keys: * ``headers``: Dict of ``{name: value}``, where ``name`` is the lowercased HTTP header name as unicode string and ``value`` is the header value as a byte string. If multiple headers with the same name are received, they should - be concatenated into a single header as per RFC 2616. Header names containing + be concatenated into a single header as per RFC 7230. Header names containing underscores should be discarded by the server. Optional, defaults to ``{}``. * ``body``: Body of the request, as a byte string. Optional, defaults to ``""``. From b296fee4c7958b74f49576ea8550e81e2f1eb791 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 10 Mar 2016 09:57:33 +0000 Subject: [PATCH 237/746] Be a bit clearer about guaranteed ordering. --- docs/asgi.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 61c0cfa..d9a4f6d 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -766,11 +766,13 @@ a very good performance or implementation reason is present: handle these within the protocol server and don't expose them in ASGI messages. -* If the protocol has guaranteed ordering, ASGI messages should include an - ``order`` field (0-indexed) that preserves the ordering as received by the - protocol server (or as sent by the client, if available). This ordering should - span all message types emitted by the client - for example, a connect message - might have order ``0``, and the first two frames order ``1`` and ``2``. +* If the protocol has guaranteed ordering and does not use a specific channel + for a given connection (as HTTP does for body data), ASGI messages should + include an ``order`` field (0-indexed) that preserves the ordering as + received by the protocol server (or as sent by the client, if available). + This ordering should span all message types emitted by the client - for + example, a connect message might have order ``0``, and the first two frames + order ``1`` and ``2``. * If the protocol is datagram-based, one datagram should equal one ASGI message (unless size is an issue) From 053850fdd69ae497f1087b874ff218f0fefa9dfb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Mar 2016 17:53:06 -0800 Subject: [PATCH 238/746] Use WSGI runserver if no channel backend configured --- channels/management/commands/runserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index e470cc0..33185b6 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -31,7 +31,7 @@ class Command(RunserverCommand): def inner_run(self, *args, **options): # Maybe they want the wsgi one? - if not options.get("use_asgi", True): + if not options.get("use_asgi", True) or DEFAULT_CHANNEL_LAYER not in channel_layers: return RunserverCommand.inner_run(self, *args, **options) # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] From 1ee7b2861c433a83dd9f718cd43b89c763a29784 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Mar 2016 17:55:28 -0800 Subject: [PATCH 239/746] Add --alias option to runworker --- channels/management/commands/runworker.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index e5fa5ff..43af56a 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -8,11 +8,16 @@ from channels.worker import Worker class Command(BaseCommand): + def add_arguments(self, parser): + super(Command, self).add_arguments(parser) + parser.add_argument('--alias', action='store', dest='alias', default=DEFAULT_CHANNEL_LAYER, + help='Channel layer alias to use, if not the default.') + def handle(self, *args, **options): # Get the backend to use self.verbosity = options.get("verbosity", 1) self.logger = setup_logger('django.channels', self.verbosity) - self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + self.channel_layer = channel_layers[options.get("alias", DEFAULT_CHANNEL_LAYER)] # Check that handler isn't inmemory if self.channel_layer.local_only(): raise CommandError( From df8a9dbbd15d695c5e74a86eef84e8b764afe857 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Mar 2016 17:57:13 -0800 Subject: [PATCH 240/746] Change --alias to --layer on runworker --- channels/management/commands/runworker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 43af56a..b5b4b32 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -10,14 +10,14 @@ class Command(BaseCommand): def add_arguments(self, parser): super(Command, self).add_arguments(parser) - parser.add_argument('--alias', action='store', dest='alias', default=DEFAULT_CHANNEL_LAYER, + parser.add_argument('--layer', action='store', dest='layer', default=DEFAULT_CHANNEL_LAYER, help='Channel layer alias to use, if not the default.') def handle(self, *args, **options): # Get the backend to use self.verbosity = options.get("verbosity", 1) self.logger = setup_logger('django.channels', self.verbosity) - self.channel_layer = channel_layers[options.get("alias", DEFAULT_CHANNEL_LAYER)] + self.channel_layer = channel_layers[options.get("layer", DEFAULT_CHANNEL_LAYER)] # Check that handler isn't inmemory if self.channel_layer.local_only(): raise CommandError( @@ -27,7 +27,7 @@ class Command(BaseCommand): # Check a handler is registered for http reqs self.channel_layer.registry.check_default() # Launch a worker - self.logger.info("Running worker against backend %s", self.channel_layer) + self.logger.info("Running worker against channel layer %s", self.channel_layer) # Optionally provide an output callback callback = None if self.verbosity > 1: From a06026c99a089752d319cec852bc926777a5bcd5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 10 Mar 2016 17:57:57 -0800 Subject: [PATCH 241/746] Releasing 0.9.5 --- CHANGELOG.txt | 8 ++++++++ channels/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 6339ad1..bd0dcc3 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +0.9.5 (2016-03-10) +------------------ + +* `runworker` now has an --alias option to specify a different channel layer + +* `runserver` correctly falls back to WSGI mode if no channel layers configured + + 0.9.4 (2016-03-08) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index 100bad1..527f5bc 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.9.4" +__version__ = "0.9.5" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 820e9555153193700a8d536f2f1fe0cef56f452a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 11 Mar 2016 10:20:17 -0800 Subject: [PATCH 242/746] Change ASGI spec regarding headers. --- channels/handler.py | 22 ++++++++++++++++------ channels/message.py | 6 ++++++ docs/asgi.rst | 15 ++++++++------- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index a32d1af..42fe00e 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -58,16 +58,26 @@ class AsgiRequest(http.HttpRequest): if self.message.get('server', None): self.META['SERVER_NAME'] = self.message['server'][0] self.META['SERVER_PORT'] = self.message['server'][1] + # Handle old style-headers for a transition period + if "headers" in self.message and isinstance(self.message['headers'], dict): + self.message['headers'] = [ + (x.encode("latin1"), y) for x, y in + self.message['headers'].items() + ] # Headers go into META - for name, value in self.message.get('headers', {}).items(): + for name, value in self.message.get('headers', []): + name = name.decode("latin1") if name == "content-length": corrected_name = "CONTENT_LENGTH" elif name == "content-type": corrected_name = "CONTENT_TYPE" else: corrected_name = 'HTTP_%s' % name.upper().replace("-", "_") - # HTTPbis say only ASCII chars are allowed in headers - self.META[corrected_name] = value.decode("ascii") + # HTTPbis say only ASCII chars are allowed in headers, but we latin1 just in case + value = value.decode("latin1") + if corrected_name in self.META: + value = self.META[corrected_name] + "," + value.decode("latin1") + self.META[corrected_name] = value # Pull out request encoding if we find it if "CONTENT_TYPE" in self.META: _, content_params = cgi.parse_header(self.META["CONTENT_TYPE"]) @@ -212,13 +222,13 @@ class AsgiHandler(base.BaseHandler): # compliant clients that want things like Content-Type correct. Ugh. response_headers = [] for header, value in response.items(): - if isinstance(header, six.binary_type): - header = header.decode("latin1") + if isinstance(header, six.text_type): + header = header.encode("ascii") if isinstance(value, six.text_type): value = value.encode("latin1") response_headers.append( ( - six.text_type(header), + six.binary_type(header), six.binary_type(value), ) ) diff --git a/channels/message.py b/channels/message.py index 5830fdd..15e3891 100644 --- a/channels/message.py +++ b/channels/message.py @@ -30,5 +30,11 @@ class Message(object): def __getitem__(self, key): return self.content[key] + def __setitem__(self, key, value): + self.content[key] = value + + def __contains__(self, key): + return key in self.content + def get(self, key, default=None): return self.content.get(key, default) diff --git a/docs/asgi.rst b/docs/asgi.rst index 085f6fd..2a5f9a7 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -454,11 +454,11 @@ Keys: is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults to ``""``. -* ``headers``: Dict of ``{name: value}``, where ``name`` is the lowercased - HTTP header name as unicode string and ``value`` is the header value as a byte - string. If multiple headers with the same name are received, they should - be concatenated into a single header as per RFC 7230. Header names containing - underscores should be discarded by the server. Optional, defaults to ``{}``. +* ``headers``: A list of ``[name, value]`` pairs, where ``name`` is the + byte string header name, and ``value`` is the byte string + header value. Order should be preserved from the original HTTP request; + duplicates are possible and must be preserved in the message as received. + Header names must be lowercased. * ``body``: Body of the request, as a byte string. Optional, defaults to ``""``. If ``body_channel`` is set, treat as start of body and concatenate @@ -514,8 +514,9 @@ Keys: or left as empty string if no default found. * ``headers``: A list of ``[name, value]`` pairs, where ``name`` is the - unicode string header name, and ``value`` is the byte string - header value. Order should be preserved in the HTTP response. + byte string header name, and ``value`` is the byte string + header value. Order should be preserved in the HTTP response. Header names + must be lowercased. * ``content``: Byte string of HTTP body content. Optional, defaults to empty string. From 9f8bf30cdda37ae0a1e66b41f68922869aa1f480 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 11 Mar 2016 10:23:07 -0800 Subject: [PATCH 243/746] Fix handler test --- channels/tests/test_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index b637392..7dc1c7c 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -62,7 +62,7 @@ class HandlerTests(SimpleTestCase): self.assertEqual(reply_message.get("more_content", False), False) self.assertEqual( reply_message["headers"], - [("Content-Type", b"text/plain")], + [(b"Content-Type", b"text/plain")], ) def test_large(self): From 5bf19f52f6590593b7ee0e1725e03efc07d10548 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 11 Mar 2016 10:30:40 -0800 Subject: [PATCH 244/746] Remove status_text from the HTTP response spec. --- channels/handler.py | 1 - channels/tests/test_handler.py | 1 - docs/asgi.rst | 6 +----- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 42fe00e..8c2f61d 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -242,7 +242,6 @@ class AsgiHandler(base.BaseHandler): # Make initial response message message = { "status": response.status_code, - "status_text": response.reason_phrase.encode("ascii"), "headers": response_headers, } # Streaming responses need to be pinned to their iterator diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index 7dc1c7c..516ec16 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -58,7 +58,6 @@ class HandlerTests(SimpleTestCase): # Make sure the message looks correct self.assertEqual(reply_message["content"], b"Hi there!") self.assertEqual(reply_message["status"], 200) - self.assertEqual(reply_message["status_text"], b"OK") self.assertEqual(reply_message.get("more_content", False), False) self.assertEqual( reply_message["headers"], diff --git a/docs/asgi.rst b/docs/asgi.rst index 2a5f9a7..6916471 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -509,10 +509,6 @@ Keys: * ``status``: Integer HTTP status code. -* ``status_text``: Byte string HTTP reason-phrase, e.g. ``OK`` from ``200 OK``. - Ignored for HTTP/2 clients. Optional, default should be based on ``status`` - or left as empty string if no default found. - * ``headers``: A list of ``[name, value]`` pairs, where ``name`` is the byte string header name, and ``value`` is the byte string header value. Order should be preserved in the HTTP response. Header names @@ -881,7 +877,7 @@ WSGI's ``environ`` variable to the Request message: The ``start_response`` callable maps similarly to Response: -* The ``status`` argument becomes ``status`` and ``status_text`` +* The ``status`` argument becomes ``status``, with the reason phrase dropped. * ``response_headers`` maps to ``headers`` It may even be possible to map Request Body Chunks in a way that allows From f1ab1100f5d60eaacd918270368399a7020948a4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 12 Mar 2016 12:14:02 -0800 Subject: [PATCH 245/746] Locale fix from runserver (#86) --- channels/management/commands/runworker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index b5b4b32..075a323 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -8,6 +8,8 @@ from channels.worker import Worker class Command(BaseCommand): + leave_locale_alone = True + def add_arguments(self, parser): super(Command, self).add_arguments(parser) parser.add_argument('--layer', action='store', dest='layer', default=DEFAULT_CHANNEL_LAYER, From d6d460d751fab54174b8a7504b2fe31b1f74e67f Mon Sep 17 00:00:00 2001 From: Dan Lipsitt Date: Sat, 12 Mar 2016 15:31:17 -0800 Subject: [PATCH 246/746] Add Python 3.4 to Tox config. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 35e7158..c21a839 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ skipsdist = True envlist = {py27}-django-{17,18,19} + {py34}-django-{18,19} {py35}-django-{18,19} {py27,py35}-flake8 isort From 2b0bc02004710b199e234f5effb7cbcc977ee5d5 Mon Sep 17 00:00:00 2001 From: Dan Lipsitt Date: Sat, 12 Mar 2016 15:35:14 -0800 Subject: [PATCH 247/746] Add Python 3.4 to Travis config. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 361fe5e..3e7de1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ sudo: false language: python python: - "2.7" + - "3.4" - "3.5" install: pip install tox-travis script: tox From 8faf37bfbf99d9070a95cd4363386dcc560ab2fe Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 16 Mar 2016 09:49:09 -0300 Subject: [PATCH 248/746] Fix type of cookie header string --- channels/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index 8c2f61d..15c8d13 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -235,7 +235,7 @@ class AsgiHandler(base.BaseHandler): for c in response.cookies.values(): response_headers.append( ( - 'Set-Cookie', + b'Set-Cookie', c.output(header='').encode("ascii"), ) ) From 5671f919dfcf6b43b4956a896374445eff78f243 Mon Sep 17 00:00:00 2001 From: David Muller Date: Thu, 17 Mar 2016 11:31:50 -0700 Subject: [PATCH 249/746] fix typo -- remove extraneous 'and' --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 3364d7b..1accddd 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -91,7 +91,7 @@ single process tied to a WSGI server, Django runs in three separate layers: cover this later. * The channel backend, which is a combination of pluggable Python code and - a datastore (a database, or Redis) and responsible for transporting messages. + a datastore (a database, or Redis) responsible for transporting messages. * The workers, that listen on all relevant channels and run consumer code when a message is ready. From 2732a66e81dd19733ab0ab4160f69dabfca866d2 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 11:15:51 -0300 Subject: [PATCH 250/746] Improve name_that_thing to handle instance methods --- channels/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/channels/utils.py b/channels/utils.py index 770f23e..e8f060e 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -5,12 +5,17 @@ def name_that_thing(thing): """ Returns either the function/class path or just the object's repr """ + # Instance method + if hasattr(thing, "im_class"): + return name_that_thing(thing.im_class) + "." + thing.im_func.func_name + # Other named thing if hasattr(thing, "__name__"): if hasattr(thing, "__class__") and not isinstance(thing, types.FunctionType): if thing.__class__ is not type: return name_that_thing(thing.__class__) if hasattr(thing, "__module__"): return "%s.%s" % (thing.__module__, thing.__name__) + # Generic instance of a class if hasattr(thing, "__class__"): return name_that_thing(thing.__class__) return repr(thing) From a914cfdcb6fee935589aae4ac30c46600f2d9158 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 12:27:00 -0300 Subject: [PATCH 251/746] Update ASGI spec to make websocket.receive/disconnect include path This enables much easier routing for applications, and is not a lot more overhead, all things considered. --- docs/asgi.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/asgi.rst b/docs/asgi.rst index 6916471..9590dbb 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -654,6 +654,8 @@ Keys: * ``reply_channel``: Channel name for sending data, in format ``websocket.send.?`` +* ``path``: Path sent during ``connect``, sent to make routing easier for apps. + * ``bytes``: Byte string of frame content, if it was bytes mode, or ``None``. * ``text``: Unicode string of frame content, if it was text mode, or ``None``. @@ -679,6 +681,8 @@ Keys: format ``websocket.send.?``. Cannot be used to send at this point; provided as a way to identify the connection only. +* ``path``: Path sent during ``connect``, sent to make routing easier for apps. + * ``order``: Order of the disconnection relative to the incoming frames' ``order`` values in ``websocket.receive``. From 841e19da79fb3093bde956ec306d754df1242e14 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 12:27:42 -0300 Subject: [PATCH 252/746] Change to a full pattern-based routing system. --- channels/asgi.py | 4 +- channels/consumer_registry.py | 76 ------- channels/management/commands/runserver.py | 2 +- channels/management/commands/runworker.py | 2 +- channels/routing.py | 220 ++++++++++++++++++++ channels/tests/test_routing.py | 234 ++++++++++++++++++++++ channels/worker.py | 11 +- docs/getting-started.rst | 80 ++++++-- docs/reference.rst | 16 +- 9 files changed, 539 insertions(+), 106 deletions(-) delete mode 100644 channels/consumer_registry.py create mode 100644 channels/routing.py create mode 100644 channels/tests/test_routing.py diff --git a/channels/asgi.py b/channels/asgi.py index 9911578..737a422 100644 --- a/channels/asgi.py +++ b/channels/asgi.py @@ -4,7 +4,7 @@ import django from django.conf import settings from django.utils.module_loading import import_string -from .consumer_registry import ConsumerRegistry +from .routing import Router from .utils import name_that_thing @@ -67,7 +67,7 @@ class ChannelLayerWrapper(object): self.channel_layer = channel_layer self.alias = alias self.routing = routing - self.registry = ConsumerRegistry(self.routing) + self.router = Router(self.routing) def __getattr__(self, name): return getattr(self.channel_layer, name) diff --git a/channels/consumer_registry.py b/channels/consumer_registry.py deleted file mode 100644 index 7cd7a57..0000000 --- a/channels/consumer_registry.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import unicode_literals - -import importlib - -from django.core.exceptions import ImproperlyConfigured -from django.utils import six - -from .handler import ViewConsumer -from .utils import name_that_thing - - -class ConsumerRegistry(object): - """ - Manages the available consumers in the project and which channels they - listen to. - - Generally this is attached to a backend instance as ".registry" - """ - - def __init__(self, routing=None): - self.consumers = {} - # Initialise with any routing that was passed in - if routing: - # If the routing was a string, import it - if isinstance(routing, six.string_types): - module_name, variable_name = routing.rsplit(".", 1) - try: - routing = getattr(importlib.import_module(module_name), variable_name) - except (ImportError, AttributeError) as e: - raise ImproperlyConfigured("Cannot import channel routing %r: %s" % (routing, e)) - # Load consumers into us - for channel, handler in routing.items(): - self.add_consumer(handler, [channel]) - - def add_consumer(self, consumer, channels): - # Upconvert if you just pass in a string for channels - if isinstance(channels, six.string_types): - channels = [channels] - # Make sure all channels are byte strings - channels = [ - channel.decode("ascii") if isinstance(channel, six.binary_type) else channel - for channel in channels - ] - # Import any consumer referenced as string - if isinstance(consumer, six.string_types): - module_name, variable_name = consumer.rsplit(".", 1) - try: - consumer = getattr(importlib.import_module(module_name), variable_name) - except (ImportError, AttributeError): - raise ImproperlyConfigured("Cannot import consumer %r" % consumer) - # Register on each channel, checking it's unique - for channel in channels: - if channel in self.consumers: - raise ValueError("Cannot register consumer %s - channel %r already consumed by %s" % ( - name_that_thing(consumer), - channel, - name_that_thing(self.consumers[channel]), - )) - self.consumers[channel] = consumer - - def all_channel_names(self): - return self.consumers.keys() - - def consumer_for_channel(self, channel): - try: - return self.consumers[channel] - except KeyError: - return None - - def check_default(self, http_consumer=None): - """ - Checks to see if default handlers need to be registered - for channels, and adds them if they need to be. - """ - if not self.consumer_for_channel("http.request"): - self.add_consumer(http_consumer or ViewConsumer(), ["http.request"]) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 33185b6..3fdd93b 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -35,7 +35,7 @@ class Command(RunserverCommand): return RunserverCommand.inner_run(self, *args, **options) # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] - self.channel_layer.registry.check_default( + self.channel_layer.router.check_default( http_consumer=self.get_consumer(), ) # Run checks diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 075a323..069e6ec 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -27,7 +27,7 @@ class Command(BaseCommand): "Change your settings to use a cross-process channel layer." ) # Check a handler is registered for http reqs - self.channel_layer.registry.check_default() + self.channel_layer.router.check_default() # Launch a worker self.logger.info("Running worker against channel layer %s", self.channel_layer) # Optionally provide an output callback diff --git a/channels/routing.py b/channels/routing.py new file mode 100644 index 0000000..8e1fd44 --- /dev/null +++ b/channels/routing.py @@ -0,0 +1,220 @@ +from __future__ import unicode_literals + +import re +import importlib + +from django.core.exceptions import ImproperlyConfigured +from django.utils import six + +from .handler import ViewConsumer +from .utils import name_that_thing + + +class Router(object): + """ + Manages the available consumers in the project and which channels they + listen to. + + Generally this is attached to a backend instance as ".router" + """ + + def __init__(self, routing): + # Resolve routing into a list if it's a dict or string + routing = self.resolve_routing(routing) + # Expand those entries recursively into a flat list of Routes + self.routing = [] + for entry in routing: + self.routing.extend(entry.expand_routes()) + # Now go through that list and collect channel names into a set + self.channels = { + route.channel + for route in self.routing + } + + def add_route(self, route): + """ + Adds a single raw Route to us at the end of the resolution list. + """ + self.routing.append(route) + self.channels.add(route.channel) + + def match(self, message): + """ + Runs through our routing and tries to find a consumer that matches + the message/channel. Returns (consumer, extra_kwargs) if it does, + and None if it doesn't. + """ + # TODO: Maybe we can add some kind of caching in here if we can hash + # the message with only matchable keys faster than the search? + for route in self.routing: + match = route.match(message) + if match is not None: + return match + return None + + def check_default(self, http_consumer=None): + """ + Adds default handlers for Django's default handling of channels. + """ + # We just add the default Django route to the bottom; if the user + # has defined another http.request handler, it'll get hit first and run. + self.add_route(Route("http.request", http_consumer or ViewConsumer())) + + @classmethod + def resolve_routing(cls, routing): + """ + Takes a routing - if it's a string, it imports it, and if it's a + dict, converts it to a list of route()s. Used by this class and Include. + """ + # If the routing was a string, import it + if isinstance(routing, six.string_types): + module_name, variable_name = routing.rsplit(".", 1) + try: + routing = getattr(importlib.import_module(module_name), variable_name) + except (ImportError, AttributeError) as e: + raise ImproperlyConfigured("Cannot import channel routing %r: %s" % (routing, e)) + # If the routing is a dict, convert it + if isinstance(routing, dict): + routing = [ + Route(channel, consumer) + for channel, consumer in routing.items() + ] + return routing + + +class Route(object): + """ + Represents a route to a single consumer, with a channel name + and optional message parameter matching. + """ + + def __init__(self, channel, consumer, **kwargs): + # Get channel, make sure it's a unicode string + self.channel = channel + if isinstance(self.channel, six.binary_type): + self.channel = self.channel.decode("ascii") + # Get consumer, optionally importing it + if isinstance(consumer, six.string_types): + module_name, variable_name = consumer.rsplit(".", 1) + try: + consumer = getattr(importlib.import_module(module_name), variable_name) + except (ImportError, AttributeError): + raise ImproperlyConfigured("Cannot import consumer %r" % consumer) + self.consumer = consumer + # Compile filter regexes up front + self.filters = { + name: re.compile(value) + for name, value in kwargs.items() + } + # Check filters don't use positional groups + for name, regex in self.filters.items(): + if regex.groups != len(regex.groupindex): + raise ValueError( + "Filter for %s on %s contains positional groups; " + "only named groups are allowed." % ( + name, + self, + ) + ) + + def match(self, message): + """ + Checks to see if we match the Message object. Returns + (consumer, kwargs dict) if it matches, None otherwise + """ + # Check for channel match first of all + if message.channel.name != self.channel: + return None + # Check each message filter and build consumer kwargs as we go + call_args = {} + for name, value in self.filters.items(): + if name not in message: + return None + match = re.match(value, message[name]) + # Any match failure means we pass + if match: + call_args.update(match.groupdict()) + else: + return None + return self.consumer, call_args + + def expand_routes(self): + """ + Expands this route into a list of just itself. + """ + return [self] + + def add_prefixes(self, prefixes): + """ + Returns a new Route with the given prefixes added to our filters. + """ + new_filters = {} + # Copy over our filters adding any prefixes + for name, value in self.filters.items(): + if name in prefixes: + if not value.pattern.startswith("^"): + raise ValueError("Cannot add prefix for %s on %s as inner value does not start with ^" % ( + name, + self, + )) + if "$" in prefixes[name]: + raise ValueError("Cannot add prefix for %s on %s as prefix contains $ (end of line match)" % ( + name, + self, + )) + new_filters[name] = re.compile(prefixes[name] + value.pattern.lstrip("^")) + else: + new_filters[name] = value + # Now add any prefixes that are by themselves so they're still enforced + for name, prefix in prefixes.items(): + if name not in new_filters: + new_filters[name] = prefix + # Return new copy + return self.__class__( + self.channel, + self.consumer, + **new_filters + ) + + def __str__(self): + return "%s %s -> %s" % ( + self.channel, + "" if not self.filters else "(%s)" % ( + ", ".join("%s=%s" % (n, v.pattern) for n, v in self.filters.items()) + ), + name_that_thing(self.consumer), + ) + + +class Include(object): + """ + Represents an inclusion of another routing list in another file. + Will automatically modify message match filters to add prefixes, + if specified. + """ + + def __init__(self, routing, **kwargs): + self.routing = Routing.resolve_routing(routing) + self.prefixes = kwargs + # Sanity check prefix regexes + for name, value in self.prefixes.items(): + if not value.startswith("^"): + raise ValueError("Include prefix for %s must start with the ^ character." % name) + + def expand_routes(self): + """ + Expands this Include into a list of routes, first recursively expanding + and then adding on prefixes to filters if specified. + """ + # First, expand our own subset of routes, to get a list of Route objects + routes = [] + for entry in self.routing: + routes.extend(entry.expand_routes()) + # Then, go through those and add any prefixes we have. + routes = [route.add_prefixes(self.prefixes) for route in routes] + return routes + + +# Lowercase standard to match urls.py +route = Route +include = Include diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py new file mode 100644 index 0000000..5fcd2a3 --- /dev/null +++ b/channels/tests/test_routing.py @@ -0,0 +1,234 @@ +from __future__ import unicode_literals +from django.test import SimpleTestCase +from django.utils import six + +from channels.routing import Router, route, include +from channels.message import Message +from channels.utils import name_that_thing + + +# Fake consumers and routing sets that can be imported by string +def consumer_1(): + pass +def consumer_2(): + pass +def consumer_3(): + pass +chatroom_routing = [ + route("websocket.connect", consumer_2, path=r"^/chat/(?P[^/]+)/$"), + route("websocket.connect", consumer_3, path=r"^/mentions/$"), +] +chatroom_routing_noprefix = [ + route("websocket.connect", consumer_2, path=r"/chat/(?P[^/]+)/$"), + route("websocket.connect", consumer_3, path=r"/mentions/$"), +] + + +class RoutingTests(SimpleTestCase): + """ + Tests that the router's routing code works correctly. + """ + + # Fake consumers we can test for with the == operator + def consumer_1(self): + pass + def consumer_2(self): + pass + def consumer_3(self): + pass + + def assertRoute(self, router, channel, content, consumer, kwargs=None): + """ + Asserts that asking the `router` to route the `content` as a message + from `channel` means it returns consumer `consumer`, optionally + testing it also returns `kwargs` to be passed in + + Use `consumer` = None to assert that no route is found. + """ + message = Message(content, channel, channel_layer="fake channel layer") + match = router.match(message) + if match is None: + if consumer is None: + return + else: + self.fail("No route found for %s on %s; expecting %s" % ( + content, + channel, + name_that_thing(consumer), + )) + else: + mconsumer, mkwargs = match + if consumer is None: + self.fail("Route found for %s on %s; expecting no route." % ( + content, + channel, + )) + self.assertEqual(consumer, mconsumer, "Route found for %s on %s; but wrong consumer (%s not %s)." % ( + content, + channel, + name_that_thing(mconsumer), + name_that_thing(consumer), + )) + if kwargs is not None: + self.assertEqual(kwargs, mkwargs, "Route found for %s on %s; but wrong kwargs (%s not %s)." % ( + content, + channel, + mkwargs, + kwargs, + )) + + def test_assumption(self): + """ + Ensures the test consumers don't compare equal, as if this ever happens + this test file will pass and miss most bugs. + """ + self.assertNotEqual(consumer_1, consumer_2) + self.assertNotEqual(consumer_1, consumer_3) + + def test_dict(self): + """ + Tests dict expansion + """ + router = Router({ + "http.request": consumer_1, + "http.disconnect": consumer_2, + }) + self.assertRoute( + router, + channel="http.request", + content={}, + consumer=consumer_1, + kwargs={}, + ) + self.assertRoute( + router, + channel="http.request", + content={"path": "/chat/"}, + consumer=consumer_1, + kwargs={}, + ) + self.assertRoute( + router, + channel="http.disconnect", + content={}, + consumer=consumer_2, + kwargs={}, + ) + + def test_filters(self): + """ + Tests that filters catch things correctly. + """ + router = Router([ + route("http.request", consumer_1, path=r"^/chat/$"), + route("http.disconnect", consumer_2), + route("http.request", consumer_3), + ]) + # Filter hit + self.assertRoute( + router, + channel="http.request", + content={"path": "/chat/"}, + consumer=consumer_1, + kwargs={}, + ) + # Fall-through + self.assertRoute( + router, + channel="http.request", + content={}, + consumer=consumer_3, + kwargs={}, + ) + self.assertRoute( + router, + channel="http.request", + content={"path": "/liveblog/"}, + consumer=consumer_3, + kwargs={}, + ) + + def test_include(self): + """ + Tests inclusion without a prefix + """ + router = Router([ + include("channels.tests.test_routing.chatroom_routing"), + ]) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/boom/"}, + consumer=None, + ) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/chat/django/"}, + consumer=consumer_2, + kwargs={"room": "django"}, + ) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/mentions/"}, + consumer=consumer_3, + kwargs={}, + ) + + def test_include_prefix(self): + """ + Tests inclusion with a prefix + """ + router = Router([ + include("channels.tests.test_routing.chatroom_routing", path="^/ws/v(?P[0-9]+)"), + ]) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/boom/"}, + consumer=None, + ) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/chat/django/"}, + consumer=None, + ) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/ws/v2/chat/django/"}, + consumer=consumer_2, + kwargs={"version": "2", "room": "django"}, + ) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/ws/v1/mentions/"}, + consumer=consumer_3, + kwargs={"version": "1"}, + ) + + def test_positional_pattern(self): + """ + Tests that regexes with positional groups are rejected. + """ + with self.assertRaises(ValueError): + Consumerrouter([ + route("http.request", consumer_1, path=r"^/chat/([^/]+)/$"), + ]) + + def test_bad_include_prefix(self): + """ + Tests both failure cases of prefixes for includes - the include not + starting with ^, and the included filter not starting with ^. + """ + with self.assertRaises(ValueError): + Consumerrouter([ + include("channels.tests.test_routing.chatroom_routing", path="foobar"), + ]) + with self.assertRaises(ValueError): + Consumerrouter([ + include("channels.tests.test_routing.chatroom_routing_noprefix", path="^/foobar/"), + ]) diff --git a/channels/worker.py b/channels/worker.py index ce3a7f8..4f8278f 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -44,7 +44,7 @@ class Worker(object): """ if self.signal_handlers: self.install_signal_handler() - channels = self.channel_layer.registry.all_channel_names() + channels = self.channel_layer.router.channels while not self.termed: self.in_job = False channel, content = self.channel_layer.receive_many(channels, block=True) @@ -66,11 +66,16 @@ class Worker(object): if content.get("__retries__", 0) == self.message_retries: message.__doomed__ = True # Handle the message - consumer = self.channel_layer.registry.consumer_for_channel(channel) + match = self.channel_layer.router.match(message) + if match is None: + logger.exception("Could not find match for message on %s! Check your routing.", channel) + continue + else: + consumer, kwargs = match if self.callback: self.callback(channel, message) try: - consumer(message) + consumer(message, **kwargs) except ConsumeLater: # They want to not handle it yet. Re-inject it with a number-of-tries marker. content['__retries__'] = content.get("__retries__", 0) + 1 diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 25af12a..4ad97d8 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -62,9 +62,10 @@ Here's what that looks like:: } # In routing.py - channel_routing = { - "http.request": "myproject.myapp.consumers.http_consumer", - } + from channels.routing import route + channel_routing = [ + route("http.request", "myproject.myapp.consumers.http_consumer"), + ] .. warning:: This example, and most of the examples here, use the "in memory" channel @@ -76,7 +77,7 @@ Here's what that looks like:: As you can see, this is a little like Django's ``DATABASES`` setting; there are named channel layers, with a default one called ``default``. Each layer needs a channel layer class, some options (if the channel layer needs them), -and a routing scheme, which points to a dict containing the routing settings. +and a routing scheme, which points to a list containing the routing settings. It's recommended you call this ``routing.py`` and put it alongside ``urls.py`` in your project, but you can put it wherever you like, as long as the path is correct. @@ -111,11 +112,12 @@ for ``http.request`` - and make this WebSocket consumer instead:: Hook it up to the ``websocket.receive`` channel like this:: # In routing.py + from channels.routing import route from myproject.myapp.consumers import ws_message - channel_routing = { - "websocket.receive": ws_message, - } + channel_routing = [ + route("websocket.receive", ws_message), + ] Now, let's look at what this is doing. It's tied to the ``websocket.receive`` channel, which means that it'll get a message @@ -210,12 +212,13 @@ get the message. Here's all the code:: And what our routing should look like in ``routing.py``:: + from channels.routing import route from myproject.myapp.consumers import ws_add, ws_message, ws_disconnect - channel_routing = { - "websocket.connect": ws_add, - "websocket.receive": ws_message, - "websocket.disconnect": ws_disconnect, + channel_routing = [ + route("websocket.connect", ws_add), + route("websocket.receive", ws_message), + route("websocket.disconnect", ws_disconnect), } With all that code, you now have a working set of a logic for a chat server. @@ -366,6 +369,7 @@ If you play around with it from the console (or start building a simple JavaScript chat client that appends received messages to a div), you'll see that you can set a chat room with the initial request. + Authentication -------------- @@ -430,8 +434,6 @@ chat to people with the same first letter of their username:: # Connected to websocket.connect @channel_session_user_from_http def ws_add(message): - # Copy user from HTTP to channel session - transfer_user(message.http_session, message.channel_session) # Add them to the right group Group("chat-%s" % message.user.username[0]).add(message.reply_channel) @@ -458,6 +460,58 @@ responses can set cookies, it needs a backend it can write to to separately store state. +Routing +------- + +Channels' ``routing.py`` acts very much like Django's ``urls.py``, including the +ability to route things to different consumers based on ``path``, or any other +message attribute that's a string (for example, ``http.request`` messages have +a ``method`` key you could route based on). + +Much like urls, you route using regular expressions; the main difference is that +because the ``path`` is not special-cased - Channels doesn't know that it's a URL - +you have to start patterns with the root ``/``, and end includes without a ``/`` +so that when the patterns combine, they work correctly. + +Finally, because you're matching against message contents using keyword arguments, +you can only use named groups in your regular expressions! Here's an example of +routing our chat from above:: + + http_routing = [ + route("http.request", poll_consumer, path=r"^/poll/$", method=r"^POST$"), + ] + + chat_routing = [ + route("websocket.connect", chat_connect, path=r"^/(?P[a-zA-Z0-9_]+)/$), + route("websocket.disconnect", chat_disconnect), + ] + + routing = [ + # You can use a string import path as the first argument as well. + include(chat_routing, path=r"^/chat"), + include(http_routing), + ] + +When Channels loads this routing, it appends any match keys together, so the +``path`` match becomes ``^/chat/(?P[a-zA-Z0-9_]+)/$``. If the include match +or the route match doesn't have the ``^`` character, it will refuse to append them +and error (you can still have matches without ``^`` in either, you just can't +ask Channels to combine them). + +Because these matches come through as keyword arguments, we could modify our +consumer above to use a room based on URL rather than username:: + + # Connected to websocket.connect + @channel_session_user_from_http + def ws_add(message, room): + # Add them to the right group + Group("chat-%s" % room).add(message.reply_channel) + +In the next section, we'll change to sending the ``room`` as a part of the +WebSocket message - which you might do if you had a multiplexing client - +but you could use routing there as well. + + Models ------ diff --git a/docs/reference.rst b/docs/reference.rst index 5b875a5..adc2d1d 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -117,19 +117,15 @@ They have the following attributes: * ``alias``: The alias of this layer. -* ``registry``: An object which represents the layer's mapping of channels +* ``router``: An object which represents the layer's mapping of channels to consumers. Has the following attributes: - * ``add_consumer(consumer, channels)``: Registers a :ref:`consumer ` - to handle all channels passed in. ``channels`` should be an iterable of - unicode string names. + * ``channels``: The set of channels this router can handle, as unicode strings - * ``consumer_for_channel(channel)``: Takes a unicode channel name and returns - either a :ref:`consumer `, or None, if no consumer is registered. - - * ``all_channel_names()``: Returns a list of all channel names this layer has - routed to a consumer. Used by the worker threads to work out what channels - to listen on. + * ``match(message)``: Takes a :ref:`Message ` and returns either + a (consumer, kwargs) tuple specifying the consumer to run and the keyword + argument to pass that were extracted via routing patterns, or None, + meaning there's no route available. .. _ref-asgirequest: From 18385d68f0ad7ec5cb1fc563007101c275a993ae Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 13:52:54 -0300 Subject: [PATCH 253/746] Fix bad rename stuff --- channels/routing.py | 2 +- channels/tests/test_routing.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/channels/routing.py b/channels/routing.py index 8e1fd44..8f39dc9 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -194,7 +194,7 @@ class Include(object): """ def __init__(self, routing, **kwargs): - self.routing = Routing.resolve_routing(routing) + self.routing = Router.resolve_routing(routing) self.prefixes = kwargs # Sanity check prefix regexes for name, value in self.prefixes.items(): diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py index 5fcd2a3..9a06008 100644 --- a/channels/tests/test_routing.py +++ b/channels/tests/test_routing.py @@ -215,7 +215,7 @@ class RoutingTests(SimpleTestCase): Tests that regexes with positional groups are rejected. """ with self.assertRaises(ValueError): - Consumerrouter([ + Router([ route("http.request", consumer_1, path=r"^/chat/([^/]+)/$"), ]) @@ -225,10 +225,10 @@ class RoutingTests(SimpleTestCase): starting with ^, and the included filter not starting with ^. """ with self.assertRaises(ValueError): - Consumerrouter([ + Router([ include("channels.tests.test_routing.chatroom_routing", path="foobar"), ]) with self.assertRaises(ValueError): - Consumerrouter([ + Router([ include("channels.tests.test_routing.chatroom_routing_noprefix", path="^/foobar/"), ]) From d3da7054b4c954c719959c31553d2b845c0645e4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 13:55:22 -0300 Subject: [PATCH 254/746] Doc typo --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 4ad97d8..44b2126 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -219,7 +219,7 @@ And what our routing should look like in ``routing.py``:: route("websocket.connect", ws_add), route("websocket.receive", ws_message), route("websocket.disconnect", ws_disconnect), - } + ] With all that code, you now have a working set of a logic for a chat server. Test time! Run ``runserver``, open a browser and use that same JavaScript From c36a33ab969584a49bd9f9f041dc8e8da887eccb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 13:56:42 -0300 Subject: [PATCH 255/746] Improve wording about regex appending in routing docs --- docs/getting-started.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 44b2126..fe1ffc0 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -492,8 +492,9 @@ routing our chat from above:: include(http_routing), ] -When Channels loads this routing, it appends any match keys together, so the -``path`` match becomes ``^/chat/(?P[a-zA-Z0-9_]+)/$``. If the include match +When Channels loads this routing, it appends any match keys together and +flattens out the routing, so the ``path`` match for ``chat_connect`` becomes +``^/chat/(?P[a-zA-Z0-9_]+)/$``. If the include match or the route match doesn't have the ``^`` character, it will refuse to append them and error (you can still have matches without ``^`` in either, you just can't ask Channels to combine them). From 3cdf51ed847603a184cd4e94ac8769c5db3caa24 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 14:06:35 -0300 Subject: [PATCH 256/746] Better spacing in routing tests --- channels/tests/test_routing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py index 9a06008..bbfba7b 100644 --- a/channels/tests/test_routing.py +++ b/channels/tests/test_routing.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals from django.test import SimpleTestCase -from django.utils import six from channels.routing import Router, route, include from channels.message import Message @@ -10,14 +9,18 @@ from channels.utils import name_that_thing # Fake consumers and routing sets that can be imported by string def consumer_1(): pass + def consumer_2(): pass + def consumer_3(): pass + chatroom_routing = [ route("websocket.connect", consumer_2, path=r"^/chat/(?P[^/]+)/$"), route("websocket.connect", consumer_3, path=r"^/mentions/$"), ] + chatroom_routing_noprefix = [ route("websocket.connect", consumer_2, path=r"/chat/(?P[^/]+)/$"), route("websocket.connect", consumer_3, path=r"/mentions/$"), From 4f8b297462d79ffd86dc0d26ba1edbb63eb19cef Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 14:10:05 -0300 Subject: [PATCH 257/746] Make flake8 more happy --- channels/tests/test_routing.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py index bbfba7b..15c92b9 100644 --- a/channels/tests/test_routing.py +++ b/channels/tests/test_routing.py @@ -10,12 +10,15 @@ from channels.utils import name_that_thing def consumer_1(): pass + def consumer_2(): pass + def consumer_3(): pass + chatroom_routing = [ route("websocket.connect", consumer_2, path=r"^/chat/(?P[^/]+)/$"), route("websocket.connect", consumer_3, path=r"^/mentions/$"), @@ -32,14 +35,6 @@ class RoutingTests(SimpleTestCase): Tests that the router's routing code works correctly. """ - # Fake consumers we can test for with the == operator - def consumer_1(self): - pass - def consumer_2(self): - pass - def consumer_3(self): - pass - def assertRoute(self, router, channel, content, consumer, kwargs=None): """ Asserts that asking the `router` to route the `content` as a message From bea5b138ef292718addf3f4aa4cba7128b3916cd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 14:31:32 -0300 Subject: [PATCH 258/746] Update test project with HTTP endpoint --- testproject/README.rst | 5 +++++ testproject/chtest/views.py | 5 +++++ testproject/testproject/urls.py | 11 ++++++++--- 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 testproject/README.rst create mode 100644 testproject/chtest/views.py diff --git a/testproject/README.rst b/testproject/README.rst new file mode 100644 index 0000000..00dbac1 --- /dev/null +++ b/testproject/README.rst @@ -0,0 +1,5 @@ +Channels Test Project +===================== + +This subdirectory contains benchmarking code and a companion Django project +that can be used to benchmark Channels for both HTTP and WebSocket performance. diff --git a/testproject/chtest/views.py b/testproject/chtest/views.py new file mode 100644 index 0000000..4bed90e --- /dev/null +++ b/testproject/chtest/views.py @@ -0,0 +1,5 @@ +from django.http import HttpResponse + + +def index(request): + return HttpResponse("OK") diff --git a/testproject/testproject/urls.py b/testproject/testproject/urls.py index 42ae607..1dca076 100644 --- a/testproject/testproject/urls.py +++ b/testproject/testproject/urls.py @@ -1,6 +1,11 @@ -from django.conf.urls import include, url -from chtest import consumers -urlpatterns = [] +from django.conf.urls import url +from chtest import consumers, views + + +urlpatterns = [ + url(r'^$', views.index), +] + channel_routing = { "websocket.receive": consumers.ws_message, From b3c101bd79de2c463996c6c88e59e34ca1fe8e9c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 17:06:51 -0700 Subject: [PATCH 259/746] Make test project useable in a production load testing setting --- testproject/Dockerfile | 19 +++++++++++++++++++ testproject/docker-compose.yml | 18 ++++++++++++++++++ testproject/testproject/asgi.py | 6 ++++++ testproject/testproject/settings.py | 8 ++++---- 4 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 testproject/Dockerfile create mode 100644 testproject/docker-compose.yml create mode 100644 testproject/testproject/asgi.py diff --git a/testproject/Dockerfile b/testproject/Dockerfile new file mode 100644 index 0000000..354fcb7 --- /dev/null +++ b/testproject/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:16.04 + +MAINTAINER Andrew Godwin + +RUN apt-get update && \ + apt-get install -y \ + git \ + python-dev \ + python-setuptools \ + python-pip && \ + pip install -U pip + +RUN pip install channels==0.9.5 asgi_redis==0.8.3 + +RUN git clone https://github.com/andrewgodwin/channels.git /srv/application/ + +WORKDIR /srv/application/testproject/ + +EXPOSE 80 diff --git a/testproject/docker-compose.yml b/testproject/docker-compose.yml new file mode 100644 index 0000000..cf01e72 --- /dev/null +++ b/testproject/docker-compose.yml @@ -0,0 +1,18 @@ +version: '2' +services: + redis: + image: redis + daphne: + image: channels-test + build: . + command: daphne -b 0.0.0.0 -p 80 testproject.asgi:channel_layer + ports: + - "80:80" + depends_on: + - redis + worker: + image: channels-test + build: . + command: python manage.py runworker + depends_on: + - redis diff --git a/testproject/testproject/asgi.py b/testproject/testproject/asgi.py new file mode 100644 index 0000000..a43f6ae --- /dev/null +++ b/testproject/testproject/asgi.py @@ -0,0 +1,6 @@ +import os +from channels.asgi import get_channel_layer + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "my_project.settings") + +channel_layer = get_channel_layer() diff --git a/testproject/testproject/settings.py b/testproject/testproject/settings.py index ab336f3..741508c 100644 --- a/testproject/testproject/settings.py +++ b/testproject/testproject/settings.py @@ -29,10 +29,10 @@ DATABASES = { CHANNEL_LAYERS = { "default": { - "BACKEND": "asgiref.inmemory.ChannelLayer", + "BACKEND": "asgi_redis.RedisChannelLayer", "ROUTING": "testproject.urls.channel_routing", + "CONFIG": { + "hosts": [os.environ.get('REDIS_URL', 'redis://redis:6379')], + } }, } - -if os.environ.get("USEREDIS", None): - CHANNEL_BACKENDS['default']['BACKEND'] = "asgi_redis.RedisChannelLayer" From caa589ae708a1a66ba1bdcd24f5fd473040772bd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 17:12:19 -0700 Subject: [PATCH 260/746] Correct project name in testproject.asgi --- testproject/testproject/asgi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testproject/testproject/asgi.py b/testproject/testproject/asgi.py index a43f6ae..1547a11 100644 --- a/testproject/testproject/asgi.py +++ b/testproject/testproject/asgi.py @@ -1,6 +1,5 @@ import os from channels.asgi import get_channel_layer -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "my_project.settings") - +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings") channel_layer = get_channel_layer() From a0f6d5b8b8446374b239ab2bc51a9f441df3976e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 17:36:36 -0700 Subject: [PATCH 261/746] Only support Django 1.8 and above, as it's the LTS --- setup.py | 4 ++-- tox.ini | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index fc4666c..40bd955 100644 --- a/setup.py +++ b/setup.py @@ -7,12 +7,12 @@ setup( url='http://github.com/andrewgodwin/django-channels', author='Andrew Godwin', author_email='andrew@aeracode.org', - description="Brings event-driven capabilities to Django with a channel system. Django 1.7 and up only.", + description="Brings event-driven capabilities to Django with a channel system. Django 1.8 and up only.", license='BSD', packages=find_packages(), include_package_data=True, install_requires=[ - 'Django>=1.7', + 'Django>=1.8', 'asgiref>=0.9', 'daphne>=0.9.2', ] diff --git a/tox.ini b/tox.ini index c21a839..75ae168 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] skipsdist = True envlist = - {py27}-django-{17,18,19} + {py27}-django-{18,19} {py34}-django-{18,19} {py35}-django-{18,19} {py27,py35}-flake8 @@ -18,8 +18,6 @@ deps = py27: mock flake8: flake8 isort: isort - django-16: Django>=1.6,<1.7 - django-17: Django>=1.7,<1.8 django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 commands = From 6ed46c72286db10f2e2a56e07ddcd332242220c9 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:05:16 -0700 Subject: [PATCH 262/746] Update dockerfile a bit so it caches right --- testproject/Dockerfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/testproject/Dockerfile b/testproject/Dockerfile index 354fcb7..1381225 100644 --- a/testproject/Dockerfile +++ b/testproject/Dockerfile @@ -10,10 +10,17 @@ RUN apt-get update && \ python-pip && \ pip install -U pip -RUN pip install channels==0.9.5 asgi_redis==0.8.3 +# Install asgi_redis driver and most recent Daphne +RUN pip install \ + asgi_redis==0.8.3 \ + git+https://github.com/andrewgodwin/daphne.git@#egg=daphne -RUN git clone https://github.com/andrewgodwin/channels.git /srv/application/ +# Clone Channels and install it +RUN git clone https://github.com/andrewgodwin/channels.git /srv/channels/ && \ + cd /srv/channels && \ + git reset --hard caa589ae708a1a66ba1bdcd24f5fd473040772bd && \ + python setup.py install -WORKDIR /srv/application/testproject/ +WORKDIR /srv/channels/testproject/ EXPOSE 80 From 5676f2da49aca2994f93d0ad5156c1dfff858784 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:05:29 -0700 Subject: [PATCH 263/746] Update benchmarker to work with custom host/port --- testproject/benchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index a8597aa..e690799 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -96,9 +96,10 @@ class Benchmarker(object): def spawn_connections(self): if len(stats) >= self.num: return + host, port = self.url.split("://")[1].split(":") for i in range(self.rate): # TODO: Look at URL - reactor.connectTCP("127.0.0.1", 8000, self.factory) + reactor.connectTCP(host, port, self.factory) def print_progress(self): open_protocols = len([x for x in stats.values() if not x]) From e7a5ad01fd4e4a5d305ecd8054322a3886377bd2 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:09:54 -0700 Subject: [PATCH 264/746] Update benchmarker for newer Autobahn --- testproject/benchmark.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index e690799..8b052d0 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -84,7 +84,6 @@ class Benchmarker(object): self.rate = rate self.factory = WebSocketClientFactory( args.url, - debug=False, ) self.factory.protocol = MyClientProtocol From a9d72844882f322349ab30cc495e5daf61a5dcd0 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:10:33 -0700 Subject: [PATCH 265/746] Convert port into an int in benchmarker --- testproject/benchmark.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index 8b052d0..febb08b 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -96,6 +96,7 @@ class Benchmarker(object): if len(stats) >= self.num: return host, port = self.url.split("://")[1].split(":") + port = int(port) for i in range(self.rate): # TODO: Look at URL reactor.connectTCP(host, port, self.factory) From 5c1a0fc09634ba7693dbd137307aeaa76f1c35b9 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:34:30 -0700 Subject: [PATCH 266/746] Update default redis url for test project --- testproject/Dockerfile | 1 + testproject/testproject/settings.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/testproject/Dockerfile b/testproject/Dockerfile index 1381225..721f12f 100644 --- a/testproject/Dockerfile +++ b/testproject/Dockerfile @@ -22,5 +22,6 @@ RUN git clone https://github.com/andrewgodwin/channels.git /srv/channels/ && \ python setup.py install WORKDIR /srv/channels/testproject/ +ENV REDIS_URL=redis://redis:6379 EXPOSE 80 diff --git a/testproject/testproject/settings.py b/testproject/testproject/settings.py index 741508c..d14a41f 100644 --- a/testproject/testproject/settings.py +++ b/testproject/testproject/settings.py @@ -32,7 +32,7 @@ CHANNEL_LAYERS = { "BACKEND": "asgi_redis.RedisChannelLayer", "ROUTING": "testproject.urls.channel_routing", "CONFIG": { - "hosts": [os.environ.get('REDIS_URL', 'redis://redis:6379')], + "hosts": [os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')], } }, } From 7e37440c934ee3abf5271faec61bd085fa965e26 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:34:39 -0700 Subject: [PATCH 267/746] Update benchmarker to be more consistent and flexible --- testproject/benchmark.py | 57 +++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index febb08b..0885d4d 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -12,9 +12,6 @@ stats = {} class MyClientProtocol(WebSocketClientProtocol): - num_messages = 5 - message_gap = 1 - def onConnect(self, response): self.opened = time.time() self.sent = 0 @@ -29,26 +26,34 @@ class MyClientProtocol(WebSocketClientProtocol): def onOpen(self): def hello(): if self.last_send is None: - if self.sent >= self.num_messages: + if self.sent >= self.factory.num_messages: self.sendClose() return - self.sendMessage(("%s:%s" % (self.sent, self.fingerprint)).encode("ascii")) self.last_send = time.time() + self.sendMessage(("%s:%s" % (self.sent, self.fingerprint)).encode("ascii")) self.sent += 1 else: # Wait for receipt of ping pass - self.factory.reactor.callLater(1, hello) + self.factory.reactor.callLater(1.0 / self.factory.message_rate, hello) hello() def onMessage(self, payload, isBinary): + # Detect receive-before-send + if self.last_send is None: + self.corrupted += 1 + print("CRITICAL: Socket %s received before sending: %s" % (self.fingerprint, payload)) + return num, fingerprint = payload.decode("ascii").split(":") if fingerprint != self.fingerprint: self.corrupted += 1 - if int(num) != self.received: - self.out_of_order += 1 - self.received += 1 + try: + if int(num) != self.received: + self.out_of_order += 1 + except ValueError: + self.corrupted += 1 self.latencies.append(time.time() - self.last_send) + self.received += 1 self.last_send = None def onClose(self, wasClean, code, reason): @@ -78,27 +83,43 @@ class Benchmarker(object): Performs benchmarks against WebSockets. """ - def __init__(self, url, num, rate): + def __init__(self, url, num, concurrency, rate, messages): self.url = url self.num = num + self.concurrency = concurrency self.rate = rate + self.messages = messages self.factory = WebSocketClientFactory( args.url, ) self.factory.protocol = MyClientProtocol + self.factory.num_messages = self.messages + self.factory.message_rate = self.rate def loop(self): + self.spawn_loop() + self.progress_loop() + + def spawn_loop(self): self.spawn_connections() + reactor.callLater(0.01, self.spawn_loop) + + def progress_loop(self): self.print_progress() - reactor.callLater(1, self.loop) + reactor.callLater(1, self.progress_loop) def spawn_connections(self): - if len(stats) >= self.num: + # Stop spawning if we did the right total number + max_to_spawn = self.num - len(stats) + if max_to_spawn <= 0: return + # Decode connection args host, port = self.url.split("://")[1].split(":") port = int(port) - for i in range(self.rate): - # TODO: Look at URL + # Only spawn enough to get up to concurrency + open_protocols = len([x for x in stats.values() if not x]) + to_spawn = min(max(self.concurrency - open_protocols, 0), max_to_spawn) + for _ in range(to_spawn): reactor.connectTCP(host, port, self.factory) def print_progress(self): @@ -176,14 +197,18 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("url") - parser.add_argument("-n", "--num", type=int, default=100) - parser.add_argument("-r", "--rate", type=int, default=10) + parser.add_argument("-n", "--num", type=int, default=100, help="Total number of sockets to open") + parser.add_argument("-c", "--concurrency", type=int, default=10, help="Number of sockets to open at once") + parser.add_argument("-r", "--rate", type=float, default=1, help="Number of messages to send per socket per second") + parser.add_argument("-m", "--messages", type=int, default=5, help="Number of messages to send per socket before close") args = parser.parse_args() benchmarker = Benchmarker( url=args.url, num=args.num, + concurrency=args.concurrency, rate=args.rate, + messages=args.messages, ) benchmarker.loop() reactor.run() From 2fc2a0f67c9903addf467e1be3ecf8ac89f360e8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:37:52 -0700 Subject: [PATCH 268/746] 99th percentile for ws benchmarker --- testproject/benchmark.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index 0885d4d..8e45019 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -166,17 +166,17 @@ class Benchmarker(object): latency_mean = statistics.mean(latencies) latency_median = statistics.median(latencies) latency_stdev = statistics.stdev(latencies) - latency_5 = self.percentile(latencies, 0.05) latency_95 = self.percentile(latencies, 0.95) + latency_99 = self.percentile(latencies, 0.99) # Print results print("-------") print("Sockets opened: %s" % len(stats)) - print("Latency stats: Mean %.2fs Median %.2fs Stdev %.2f 5%% %.2fs 95%% %.2fs" % ( + print("Latency stats: Mean %.3fs Median %.3fs Stdev %.3f 95%% %.3fs 95%% %.3fs" % ( latency_mean, latency_median, latency_stdev, - latency_5, latency_95, + latency_99, )) print("Good sockets: %s (%.2f%%)" % (num_good, (float(num_good) / len(stats))*100)) print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100)) From db9661b31d76c34e68bc7fc71c6a848d2a911ca3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:43:52 -0700 Subject: [PATCH 269/746] Throw in fabfile for benchmarking while I'm here --- testproject/fabfile.py | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 testproject/fabfile.py diff --git a/testproject/fabfile.py b/testproject/fabfile.py new file mode 100644 index 0000000..400890b --- /dev/null +++ b/testproject/fabfile.py @@ -0,0 +1,45 @@ +from fabric.api import sudo, task, cd + + +@task +def setup_redis(): + sudo("apt-get update && apt-get install -y redis-server") + sudo("sed -i -e 's/127.0.0.1/0.0.0.0/g' /etc/redis/redis.conf") + sudo("/etc/init.d/redis-server stop") + sudo("/etc/init.d/redis-server start") + + +@task +def setup_channels(): + sudo("apt-get update && apt-get install -y git python-dev python-setuptools python-pip") + sudo("pip install -U pip") + sudo("pip install -U asgi_redis git+https://github.com/andrewgodwin/daphne.git@#egg=daphne") + sudo("rm -rf /srv/channels") + sudo("git clone https://github.com/andrewgodwin/channels.git /srv/channels/") + with cd("/srv/channels/"): + sudo("python setup.py install") + + +@task +def setup_tester(): + sudo("apt-get update && apt-get install -y apache2-utils python3-pip") + sudo("pip3 -U pip autobahn twisted") + sudo("rm -rf /srv/channels") + sudo("git clone https://github.com/andrewgodwin/channels.git /srv/channels/") + + +@task +def run_daphne(redis_ip): + with cd("/srv/channels/testproject/"): + sudo("REDIS_URL=redis://%s:6379 daphne -b 0.0.0.0 -p 80 testproject.asgi:channel_layer" % redis_ip) + + +@task +def run_worker(redis_ip): + with cd("/srv/channels/testproject/"): + sudo("REDIS_URL=redis://%s:6379 python manage.py runworker" % redis_ip) + + +@task +def shell(): + sudo("bash") From 808231cdc5f873a4c781d72ebe3ea9d137282179 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:46:10 -0700 Subject: [PATCH 270/746] Don't let benchmarker overshoot concurrency --- testproject/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index 8e45019..dae3380 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -102,7 +102,7 @@ class Benchmarker(object): def spawn_loop(self): self.spawn_connections() - reactor.callLater(0.01, self.spawn_loop) + reactor.callLater(0.1, self.spawn_loop) def progress_loop(self): self.print_progress() From 40f6d198fa2cf10c4e24d3a97e351417dd2184a2 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 18:49:11 -0700 Subject: [PATCH 271/746] Even better precision on benchmarker concurrency --- testproject/benchmark.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index dae3380..8ddc3d6 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -12,6 +12,11 @@ stats = {} class MyClientProtocol(WebSocketClientProtocol): + def __init__(self, *args, **kwargs): + WebSocketClientProtocol.__init__(self, *args, **kwargs) + self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) + stats[self.fingerprint] = {} + def onConnect(self, response): self.opened = time.time() self.sent = 0 @@ -20,8 +25,6 @@ class MyClientProtocol(WebSocketClientProtocol): self.corrupted = 0 self.out_of_order = 0 self.latencies = [] - self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) - stats[self.fingerprint] = {} def onOpen(self): def hello(): From 0ef19c907498aa871890ea0bb2dc432547a3abbe Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 19:03:06 -0700 Subject: [PATCH 272/746] Don't try and spawn all sockets instantly --- testproject/benchmark.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index 8ddc3d6..2a0cd54 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -116,6 +116,8 @@ class Benchmarker(object): max_to_spawn = self.num - len(stats) if max_to_spawn <= 0: return + # Don't spawn too many at once + max_to_spawn = min(max_to_spawn, 10) # Decode connection args host, port = self.url.split("://")[1].split(":") port = int(port) From bc51d657d5e0c0bff90ff19d56f9e6a87eb941dc Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 20 Mar 2016 19:10:19 -0700 Subject: [PATCH 273/746] Add spawn rate control --- testproject/benchmark.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/testproject/benchmark.py b/testproject/benchmark.py index 2a0cd54..01c1f81 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -86,11 +86,12 @@ class Benchmarker(object): Performs benchmarks against WebSockets. """ - def __init__(self, url, num, concurrency, rate, messages): + def __init__(self, url, num, concurrency, rate, messages, spawn): self.url = url self.num = num self.concurrency = concurrency self.rate = rate + self.spawn = spawn self.messages = messages self.factory = WebSocketClientFactory( args.url, @@ -117,7 +118,7 @@ class Benchmarker(object): if max_to_spawn <= 0: return # Don't spawn too many at once - max_to_spawn = min(max_to_spawn, 10) + max_to_spawn = min(max_to_spawn, int(self.spawn / 10.0)) # Decode connection args host, port = self.url.split("://")[1].split(":") port = int(port) @@ -206,6 +207,7 @@ if __name__ == '__main__': parser.add_argument("-c", "--concurrency", type=int, default=10, help="Number of sockets to open at once") parser.add_argument("-r", "--rate", type=float, default=1, help="Number of messages to send per socket per second") parser.add_argument("-m", "--messages", type=int, default=5, help="Number of messages to send per socket before close") + parser.add_argument("-s", "--spawn", type=int, default=30, help="Number of sockets to spawn per second, max") args = parser.parse_args() benchmarker = Benchmarker( @@ -214,6 +216,7 @@ if __name__ == '__main__': concurrency=args.concurrency, rate=args.rate, messages=args.messages, + spawn=args.spawn, ) benchmarker.loop() reactor.run() From 8522eb40ca2cd584fbe5bb3b50fbfc73c3ccb37f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 21 Mar 2016 16:38:23 -0700 Subject: [PATCH 274/746] Update single-reader channel names to split on !, not start. --- channels/database_layer.py | 5 ++-- docs/asgi.rst | 58 ++++++++++++++++++++------------------ docs/concepts.rst | 8 +++--- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index 72f1a58..9caef7e 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -74,10 +74,11 @@ class DatabaseChannelLayer(object): def new_channel(self, pattern): assert isinstance(pattern, six.text_type) + assert pattern.endswith("!") # Keep making channel names till one isn't present. while True: - random_string = "".join(random.choice(string.ascii_letters) for i in range(8)) - new_name = pattern.replace("?", random_string) + random_string = "".join(random.choice(string.ascii_letters) for i in range(10)) + new_name = pattern + random_string if not self.channel_model.objects.filter(channel=new_name).exists(): return new_name diff --git a/docs/asgi.rst b/docs/asgi.rst index 9590dbb..04ddcfe 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -112,14 +112,17 @@ In order to aid with scaling and network architecture, a distinction is made between channels that have multiple readers (such as the ``http.request`` channel that web applications would listen on from every application worker process) and *single-reader channels* -(such as a ``http.response.ABCDEF`` channel tied to a client socket). +(such as a ``http.response!ABCDEF`` channel tied to a client socket). -*Single-reader channel* names are prefixed with an exclamation mark +*Single-reader channel* names contain an exclamation mark (``!``) character in order to indicate to the channel layer that it may have to route the data for these channels differently to ensure it reaches the single process that needs it; these channels are nearly always tied to -incoming connections from the outside world. Some channel layers may not -need this, and can simply treat the prefix as part of the name. +incoming connections from the outside world. The ``!`` is always preceded by +the main channel name (e.g. ``http.response``) and followed by the +per-client/random portion - channel layers can split on the ``!`` and use just +the right hand part to route if they desire, or can ignore it if they don't +need to use different routing rules. Messages should expire after a set time sitting unread in a channel; the recommendation is one minute, though the best value depends on the @@ -156,7 +159,7 @@ standard keys in the ``environ`` dict for WSGI. The design pattern is that most protocols will share a few channels for incoming data (for example, ``http.request``, ``websocket.connect`` and ``websocket.receive``), but will have individual channels for sending to -each client (such as ``!http.response.kj2daj23``). This allows incoming +each client (such as ``http.response!kj2daj23``). This allows incoming data to be dispatched into a cluster of application servers that can all handle it, while responses are routed to the individual protocol server that has the other end of the client's socket. @@ -275,11 +278,13 @@ A *channel layer* must provide an object with these attributes * ``new_channel(pattern)``, a callable that takes a unicode string pattern, and returns a new valid channel name that does not already exist, by - substituting any occurrences of the question mark character ``?`` in - ``pattern`` with a single random unicode string and checking for - existence of that name in the channel layer. This is NOT called prior to + adding a single random unicode string after the ``!`` character in ``pattern``, + and checking for existence of that name in the channel layer. The ``pattern`` + MUST end with ``!`` or this function must error. This is NOT called prior to a message being sent on a channel, and should not be used for channel - initialization. + initialization, and is also not guaranteed to be called by the same channel + client that then reads the messages, so you cannot put process identifiers in + it for routing. * ``MessageTooLarge``, the exception raised when a send operation fails because the encoded message is over the layer's size limit. @@ -391,7 +396,7 @@ top-level outgoing channel. Messages are specified here along with the channel names they are expected on; if a channel name can vary, such as with reply channels, the varying -portion will be replaced by ``?``, such as ``http.response.?``, which matches +portion will be represented by ``!``, such as ``http.response!``, which matches the format the ``new_channel`` callable takes. There is no label on message types to say what they are; their type is implicit @@ -435,8 +440,8 @@ Channel: ``http.request`` Keys: -* ``reply_channel``: Channel name for responses and server pushes, in - format ``http.response.?`` +* ``reply_channel``: Channel name for responses and server pushes, starting with + ``http.response!`` * ``http_version``: Unicode string, one of ``1.0``, ``1.1`` or ``2``. @@ -485,7 +490,7 @@ Request Body Chunk Must be sent after an initial Response. -Channel: ``http.request.body.?`` +Channel: ``http.request.body!`` Keys: @@ -503,7 +508,7 @@ Response Send after any server pushes, and before any response chunks. -Channel: ``http.response.?`` +Channel: ``http.response!`` Keys: @@ -528,7 +533,7 @@ Response Chunk Must be sent after an initial Response. -Channel: ``http.response.?`` +Channel: ``http.response!`` Keys: @@ -551,7 +556,7 @@ When a server receives this message, it must treat the Request message in the received from the network. A server may, if it chooses, apply all of its internal logic to handling this request (e.g. the server may want to try to satisfy the request from a cache). Regardless, if the server is unable to -satisfy the request itself it must create a new ``http.response.?`` channel for +satisfy the request itself it must create a new ``http.response!`` channel for the application to send the Response message on, fill that channel in on the ``reply_channel`` field of the message, and then send the Request back to the application on the ``http.request`` channel. @@ -565,7 +570,7 @@ If the remote peer does not support server push, either because it's not a HTTP/2 peer or because SETTINGS_ENABLE_PUSH is set to 0, the server must do nothing in response to this message. -Channel: ``http.response.?`` +Channel: ``http.response!`` Keys: @@ -611,8 +616,7 @@ Channel: ``websocket.connect`` Keys: -* ``reply_channel``: Channel name for sending data, in - format ``websocket.send.?`` +* ``reply_channel``: Channel name for sending data, start with ``websocket.send!`` * ``scheme``: Unicode string URL scheme portion (likely ``ws`` or ``wss``). Optional (but must not be empty), default is ``ws``. @@ -651,8 +655,7 @@ Channel: ``websocket.receive`` Keys: -* ``reply_channel``: Channel name for sending data, in - format ``websocket.send.?`` +* ``reply_channel``: Channel name for sending data, starting with ``websocket.send!`` * ``path``: Path sent during ``connect``, sent to make routing easier for apps. @@ -677,8 +680,8 @@ Channel: ``websocket.disconnect`` Keys: -* ``reply_channel``: Channel name that was used for sending data, in - format ``websocket.send.?``. Cannot be used to send at this point; provided +* ``reply_channel``: Channel name that was used for sending data, starting + with ``websocket.send!``. Cannot be used to send at this point; provided as a way to identify the connection only. * ``path``: Path sent during ``connect``, sent to make routing easier for apps. @@ -693,7 +696,7 @@ Send/Close Sends a data frame to the client and/or closes the connection from the server end. -Channel: ``websocket.send.?`` +Channel: ``websocket.send!`` Keys: @@ -732,7 +735,7 @@ Channel: ``udp.receive`` Keys: -* ``reply_channel``: Channel name for sending data, in format ``udp.send.?`` +* ``reply_channel``: Channel name for sending data, starts with ``udp.send!`` * ``data``: Byte string of UDP datagram payload. @@ -750,7 +753,7 @@ Send Sent to send out a UDP datagram to a client. -Channel: ``udp.send.?`` +Channel: ``udp.send!`` Keys: @@ -834,7 +837,8 @@ limitation that they only use the following characters: * Hyphen ``-`` * Underscore ``_`` * Period ``.`` -* Exclamation mark ``!`` (only at the start of a channel name) +* Exclamation mark ``!`` (only to deliniate single-reader channel names, + and only one per name) WSGI Compatibility diff --git a/docs/concepts.rst b/docs/concepts.rst index 1accddd..2cc7652 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -155,9 +155,9 @@ the message - but response channels would have to have their messages sent to the channel server they're listening on. For this reason, Channels treats these as two different *channel types*, and -denotes a *response channel* by having the first character of the channel name -be the character ``!`` - e.g. ``!http.response.f5G3fE21f``. *Normal -channels* have no special prefix, but along with the rest of the response +denotes a *response channel* by having the channel name contain +the character ``!`` - e.g. ``http.response!f5G3fE21f``. *Normal +channels* have do not contain it, but along with the rest of the response channel name, they must contain only the characters ``a-z A-Z 0-9 - _``, and be less than 200 characters long. @@ -244,7 +244,7 @@ Of course, you should still remove things from the group on disconnect if you can; the expiry code is there to catch cases where the disconnect message doesn't make it for some reason. -Groups are generally only useful for response channels (ones starting with +Groups are generally only useful for response channels (ones containing the character ``!``), as these are unique-per-client, but can be used for normal channels as well if you wish. From 13d6c88c7fcfb4d922cd1f3b5237780f0fe1ec75 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 21 Mar 2016 16:46:42 -0700 Subject: [PATCH 275/746] Releasing 0.10.0 --- channels/__init__.py | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/channels/__init__.py b/channels/__init__.py index 527f5bc..045dc1c 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.9.5" +__version__ = "0.10.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/setup.py b/setup.py index 40bd955..7a3ad07 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( include_package_data=True, install_requires=[ 'Django>=1.8', - 'asgiref>=0.9', - 'daphne>=0.9.2', + 'asgiref>=0.10', + 'daphne>=0.10', ] ) From d72f99df9450f894464670703ef74835f0f7f4be Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 21 Mar 2016 16:49:21 -0700 Subject: [PATCH 276/746] Update channel readme --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index df5559f..7938c60 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,9 @@ Django Channels .. image:: https://api.travis-ci.org/andrewgodwin/channels.svg :target: https://travis-ci.org/andrewgodwin/channels +*(Note: Recent versions of Channels also need recent versions of Daphne, +asgi_redis and asgiref, so make sure you update all at once)* + This is a work-in-progress code branch of Django implemented as a third-party app, which aims to bring some asynchrony to Django and expand the options for code beyond the request-response model, in particular enabling WebSocket, From acd31a663d952a3cd50f97bd72f7dbc83b0800cd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 22 Mar 2016 10:15:30 -0700 Subject: [PATCH 277/746] Add closed message to request bodies to allow fast failure --- docs/asgi.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/asgi.rst b/docs/asgi.rst index 04ddcfe..d6311e4 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -496,6 +496,11 @@ Keys: * ``content``: Byte string of HTTP body content, will be concatenated onto previously received ``content`` values and ``body`` key in Request. + Not required if ``closed`` is True, required otherwise. + +* ``closed``: True if the client closed the connection prematurely and the + rest of the body. If you receive this, abandon processing of the HTTP request. + Optional, defaults to ``False``. * ``more_content``: Boolean value signifying if there is additional content to come (as part of a Request Body Chunk message). If ``False``, request will From afa46cfe0f2d356b02babc3bcf1e23224866a382 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 22 Mar 2016 10:15:52 -0700 Subject: [PATCH 278/746] Provide route and include at top level --- channels/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/__init__.py b/channels/__init__.py index 045dc1c..ed3a88d 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -6,5 +6,6 @@ DEFAULT_CHANNEL_LAYER = 'default' try: from .asgi import channel_layers # NOQA isort:skip from .channel import Channel, Group # NOQA isort:skip + from .routing import route, include except ImportError: # No django installed, allow vars to be read pass From 6884e7d1e8db85c2428f8cf94de52f2e2c799942 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 22 Mar 2016 10:17:38 -0700 Subject: [PATCH 279/746] Fixed #93: Unicode regex patterns not working on http.request path --- channels/routing.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/channels/routing.py b/channels/routing.py index 8f39dc9..1593253 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -81,6 +81,16 @@ class Router(object): ] return routing + @classmethod + def normalise_re_arg(cls, value): + """ + Normalises regular expression patterns and string inputs to Unicode. + """ + if isinstance(value, six.binary_type): + return value.decode("ascii") + else: + return value + class Route(object): """ @@ -103,7 +113,7 @@ class Route(object): self.consumer = consumer # Compile filter regexes up front self.filters = { - name: re.compile(value) + name: re.compile(Router.normalise_re_arg(value)) for name, value in kwargs.items() } # Check filters don't use positional groups @@ -130,7 +140,7 @@ class Route(object): for name, value in self.filters.items(): if name not in message: return None - match = re.match(value, message[name]) + match = value.match(Router.normalise_re_arg(message[name])) # Any match failure means we pass if match: call_args.update(match.groupdict()) From 10224ff06c7ea7db462690bd7b50c85a43073240 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 22 Mar 2016 10:20:15 -0700 Subject: [PATCH 280/746] Flake, stop worrying about this file. --- channels/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/__init__.py b/channels/__init__.py index ed3a88d..2b2ff0d 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -6,6 +6,6 @@ DEFAULT_CHANNEL_LAYER = 'default' try: from .asgi import channel_layers # NOQA isort:skip from .channel import Channel, Group # NOQA isort:skip - from .routing import route, include + from .routing import route, include # NOQA isort:skip except ImportError: # No django installed, allow vars to be read pass From 52c821a18625f8190afb7da5b9535cc0ed9cc1f6 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 22 Mar 2016 10:31:01 -0700 Subject: [PATCH 281/746] Increase FileResponse block size everywhere, not just staticfiles --- channels/handler.py | 4 ++++ channels/staticfiles.py | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 15c8d13..7227a24 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -11,6 +11,7 @@ from django import http from django.core import signals from django.core.handlers import base from django.core.urlresolvers import set_script_prefix +from django.http import FileResponse from django.utils import six from django.utils.functional import cached_property @@ -184,6 +185,9 @@ class AsgiHandler(base.BaseHandler): else: try: response = self.get_response(request) + # Fix chunk size on file responses + if isinstance(response, FileResponse): + response.block_size = 1024 * 512 except AsgiRequest.ResponseLater: # The view has promised something else # will send a response at a later time diff --git a/channels/staticfiles.py b/channels/staticfiles.py index 153226d..8169be8 100644 --- a/channels/staticfiles.py +++ b/channels/staticfiles.py @@ -49,10 +49,7 @@ class StaticFilesHandler(AsgiHandler): if self._should_handle(request.path): try: - response = self.serve(request) - # Increase FileResponse block sizes so they're not super slow - response.block_size = 1024 * 256 - return response + return self.serve(request) except Http404 as e: if settings.DEBUG: from django.views import debug From 7864519241245d242f1460102bd8871d391d65f0 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 22 Mar 2016 10:32:12 -0700 Subject: [PATCH 282/746] Releasing 0.10.1 --- CHANGELOG.txt | 20 ++++++++++++++++++++ channels/__init__.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index bd0dcc3..c67d7fa 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,23 @@ +0.10.1 (2016-03-22) +------------------- + +* Regular expressions for HTTP paths can now be Unicode under Python 3 + +* route() and include() now importable directly from `channels` + +* FileResponse send speed improved for all code (previously just for staticfiles) + + +0.10.0 (2016-03-21) +------------------- + +* New routing system + +* Updated to match new ASGI single-reader-channel name spec + +* Updated to match new ASGI HTTP header encoding spec + + 0.9.5 (2016-03-10) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index 2b2ff0d..93f3cd6 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.10.0" +__version__ = "0.10.1" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 8a65199bfebd90aec58f66af17506b5cbb16eb7a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 22 Mar 2016 10:39:27 -0700 Subject: [PATCH 283/746] Also normalise include patterns, and add tests for all regex norm --- channels/routing.py | 5 +++- channels/tests/test_routing.py | 54 ++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/channels/routing.py b/channels/routing.py index 1593253..e3ff7c2 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -205,7 +205,10 @@ class Include(object): def __init__(self, routing, **kwargs): self.routing = Router.resolve_routing(routing) - self.prefixes = kwargs + self.prefixes = { + name: Router.normalise_re_arg(value) + for name, value in kwargs.items() + } # Sanity check prefix regexes for name, value in self.prefixes.items(): if not value.startswith("^"): diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py index 15c92b9..75d0659 100644 --- a/channels/tests/test_routing.py +++ b/channels/tests/test_routing.py @@ -230,3 +230,57 @@ class RoutingTests(SimpleTestCase): Router([ include("channels.tests.test_routing.chatroom_routing_noprefix", path="^/foobar/"), ]) + + def test_mixed_unicode_bytes(self): + """ + Tests that having the message key be bytes and pattern unicode (or vice-versa) + still works. + """ + # Unicode patterns, byte message + router = Router([ + route("websocket.connect", consumer_1, path="^/foo/"), + include("channels.tests.test_routing.chatroom_routing", path="^/ws/v(?P[0-9]+)"), + ]) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": b"/boom/"}, + consumer=None, + ) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": b"/foo/"}, + consumer=consumer_1, + ) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": b"/ws/v2/chat/django/"}, + consumer=consumer_2, + kwargs={"version": "2", "room": "django"}, + ) + # Byte patterns, unicode message + router = Router([ + route("websocket.connect", consumer_1, path=b"^/foo/"), + include("channels.tests.test_routing.chatroom_routing", path=b"^/ws/v(?P[0-9]+)"), + ]) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/boom/"}, + consumer=None, + ) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/foo/"}, + consumer=consumer_1, + ) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/ws/v2/chat/django/"}, + consumer=consumer_2, + kwargs={"version": "2", "room": "django"}, + ) From 7bc94b342ec14257d3e5a80c83721300a41fd0c5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 23 Mar 2016 11:50:35 -0700 Subject: [PATCH 284/746] Implement last-resort error handler for HTTP requests There's no WSGI server to do this for us, so Channels has to. --- channels/handler.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 7227a24..500eccc 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -4,6 +4,7 @@ import cgi import codecs import logging import sys +import traceback from io import BytesIO from threading import Lock @@ -11,7 +12,7 @@ from django import http from django.core import signals from django.core.handlers import base from django.core.urlresolvers import set_script_prefix -from django.http import FileResponse +from django.http import FileResponse, HttpResponseServerError from django.utils import six from django.utils.functional import cached_property @@ -212,9 +213,18 @@ class AsgiHandler(base.BaseHandler): Propagates ResponseLater up into the higher handler method, processes everything else """ + # ResponseLater needs to be bubbled up the stack if issubclass(exc_info[0], AsgiRequest.ResponseLater): raise - return super(AsgiHandler, self).handle_uncaught_exception(request, resolver, exc_info) + # There's no WSGI server to catch the exception further up if this fails, + # so translate it into a plain text response. + try: + return super(AsgiHandler, self).handle_uncaught_exception(request, resolver, exc_info) + except: + return HttpResponseServerError( + traceback.format_exc(), + content_type="text/plain", + ) @classmethod def encode_response(cls, response): From f829a14312200a3a170c48c95287822744a8769a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 23 Mar 2016 12:53:02 -0700 Subject: [PATCH 285/746] Don't show last-chance traceback when DEBUG is off. --- channels/handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index 500eccc..6e2bedb 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -9,6 +9,7 @@ from io import BytesIO from threading import Lock from django import http +from django.conf import settings from django.core import signals from django.core.handlers import base from django.core.urlresolvers import set_script_prefix @@ -222,7 +223,7 @@ class AsgiHandler(base.BaseHandler): return super(AsgiHandler, self).handle_uncaught_exception(request, resolver, exc_info) except: return HttpResponseServerError( - traceback.format_exc(), + traceback.format_exc() if settings.DEBUG else "Internal Server Error", content_type="text/plain", ) From 392ba22768a09733dd2dcd90359023e494983cbb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 23 Mar 2016 12:53:34 -0700 Subject: [PATCH 286/746] Releasing 0.10.2 --- CHANGELOG.txt | 10 ++++++++++ channels/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index c67d7fa..797b913 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,13 @@ +0.10.2 (2016-03-23) +------------------- + +* Regular expressions for routing include() can now be Unicode under Python 3 + +* Last-resort error handling for HTTP request exceptions inside Django's core + code. If DEBUG is on, shows plain text tracebacks; if it is off, shows + "Internal Server Error". + + 0.10.1 (2016-03-22) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 93f3cd6..0536163 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.10.1" +__version__ = "0.10.2" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From cb40b4fc14a3ace59f6da5136bbd513f11eec33c Mon Sep 17 00:00:00 2001 From: rcorzogutierrez Date: Thu, 24 Mar 2016 11:14:10 -0400 Subject: [PATCH 287/746] Update Reame add badge Pypi, License and download --- README.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.rst b/README.rst index 7938c60..3a2bd84 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,18 @@ Django Channels .. image:: https://api.travis-ci.org/andrewgodwin/channels.svg :target: https://travis-ci.org/andrewgodwin/channels +.. image:: https://img.shields.io/pypi/dm/channels.svg + :target: https://pypi.python.org/pypi/channels + +.. image:: https://readthedocs.org/projects/channels/badge/?version=latest + :target: http://channels.readthedocs.org/en/latest/?badge=latest + +.. image:: https://img.shields.io/pypi/v/channels.svg + :target: https://pypi.python.org/pypi/channels + +.. image:: https://img.shields.io/pypi/l/channels.svg + :target: https://pypi.python.org/pypi/channels + *(Note: Recent versions of Channels also need recent versions of Daphne, asgi_redis and asgiref, so make sure you update all at once)* From 3e5735819140d619f8b891eb2a82edb09743dc6b Mon Sep 17 00:00:00 2001 From: rcorzogutierrez Date: Thu, 24 Mar 2016 14:58:01 -0400 Subject: [PATCH 288/746] Update to universal distribute --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 445b946..2cab84e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,3 +10,6 @@ include_trailing_comma = true known_first_party = channels multi_line_output = 5 not_skip = __init__.py + +[bdist_wheel] +universal=1 From aad35757498125d448949e29e136733b384d47fa Mon Sep 17 00:00:00 2001 From: Charlie Hornsby Date: Fri, 25 Mar 2016 16:42:57 +0200 Subject: [PATCH 289/746] Remove references to Django project name Update import paths to match standard Django package structure --- docs/getting-started.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index fe1ffc0..7b25fa9 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -64,7 +64,7 @@ Here's what that looks like:: # In routing.py from channels.routing import route channel_routing = [ - route("http.request", "myproject.myapp.consumers.http_consumer"), + route("http.request", "myapp.consumers.http_consumer"), ] .. warning:: @@ -113,7 +113,7 @@ Hook it up to the ``websocket.receive`` channel like this:: # In routing.py from channels.routing import route - from myproject.myapp.consumers import ws_message + from myapp.consumers import ws_message channel_routing = [ route("websocket.receive", ws_message), @@ -213,7 +213,7 @@ get the message. Here's all the code:: And what our routing should look like in ``routing.py``:: from channels.routing import route - from myproject.myapp.consumers import ws_add, ws_message, ws_disconnect + from myapp.consumers import ws_add, ws_message, ws_disconnect channel_routing = [ route("websocket.connect", ws_add), From 40b23486000ecd00334cbf1ae14e8651eea05170 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 26 Mar 2016 15:28:11 -0700 Subject: [PATCH 290/746] Fixed #101: Obscure error when strings in routing list. --- channels/routing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/channels/routing.py b/channels/routing.py index e3ff7c2..d25e09a 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -24,7 +24,11 @@ class Router(object): # Expand those entries recursively into a flat list of Routes self.routing = [] for entry in routing: - self.routing.extend(entry.expand_routes()) + try: + self.routing.extend(entry.expand_routes()) + except AttributeError: + # It's not a valid route + raise ValueError("Encountered %r in routing config, which is not a valid route() or include()" % entry) # Now go through that list and collect channel names into a set self.channels = { route.channel From bee81ee620b1472cea812b71bb675216b57b1ab4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 28 Mar 2016 11:44:40 +0100 Subject: [PATCH 291/746] Update ASGI spec to add enforced long connection timeouts. --- docs/asgi.rst | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index d6311e4..d023f28 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -307,6 +307,10 @@ A channel layer implementing the ``groups`` extension must also provide: arguments; the group to send to, as a unicode string, and the message to send, as a serializable ``dict``. +* ``group_expiry``, an integer number of seconds that specifies how long group + membership is valid for after the most recent ``group_add`` call (see + *Persistence* below) + A channel layer implementing the ``statistics`` extension must also provide: * ``global_statistics()``, a callable that returns a dict with zero @@ -373,8 +377,27 @@ clients reconnect will immediately resolve the problem. If a channel layer implements the ``groups`` extension, it must persist group membership until at least the time when the member channel has a message -expire due to non-consumption. It should drop membership after a while to -prevent collision of old messages with new clients with the same random ID. +expire due to non-consumption, after which it may drop membership at any time. +If a channel subsequently has a successful delivery, the channel layer must +then not drop group membership until another message expires on that channel. + +Channel layers must also drop group membership after a configurable long timeout +after the most recent ``group_add`` call for that membership, the default being +86,400 seconds (one day). The value of this timeout is exposed as the +``group_expiry`` property on the channel layer. + +Protocol servers must have a configurable timeout value for every connection-based +prtocol they serve that closes the connection after the timeout, and should +default this value to the value of ``group_expiry``, if the channel +layer provides it. This allows old group memberships to be cleaned up safely, +knowing that after the group expiry the original connection must have closed, +or is about to be in the next few seconds. + +It's recommended that end developers put the timeout setting much lower - on +the order of hours or minutes - to enable better protocol design and testing. +Even with ASGI's separation of protocol server restart from business logic +restart, you will likely need to move and reprovision protocol servers, and +making sure your code can cope with this is important. Message Formats From 59198ea93e508687e92046922ec7e47a68305f52 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 28 Mar 2016 11:45:24 +0100 Subject: [PATCH 292/746] Add signed cookie warning --- channels/sessions.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/channels/sessions.py b/channels/sessions.py index a80cc61..594e921 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -4,6 +4,7 @@ import warnings from importlib import import_module from django.conf import settings +from django.contrib.sessions.backends import signed_cookies from django.contrib.sessions.backends.base import CreateError from .exceptions import ConsumeLater @@ -20,7 +21,9 @@ def session_for_reply_channel(reply_channel): hashed = hashlib.md5(reply_name.encode("utf8")).hexdigest() session_key = "chn" + hashed[:29] # Make a session storage - session_engine = import_module(settings.SESSION_ENGINE) + session_engine = import_module(getattr(settings, "CHANNEL_SESSION_ENGINE", settings.SESSION_ENGINE)) + if session_engine is signed_cookies: + raise ValueError("You cannot use channels session functionality with signed cookie sessions!") return session_engine.SessionStore(session_key=session_key) @@ -122,6 +125,9 @@ def http_session(func): If a message does not have a session we can inflate, the "session" attribute will be None, rather than an empty session you can write to. + + Does not allow a new session to be set; that must be done via a view. This + is only an accessor for any existing session. """ @functools.wraps(func) def inner(message, *args, **kwargs): From 1ab757fffbe0477b65cbd3bcf34fc48de4c5f3b7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 28 Mar 2016 11:59:25 +0100 Subject: [PATCH 293/746] Implement group_expiry on database channel backend --- channels/database_layer.py | 13 +++++++++---- channels/tests/test_database_layer.py | 8 +++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index 9caef7e..4f23142 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -27,8 +27,9 @@ class DatabaseChannelLayer(object): it's not a valid Unicode character, so it should be safe enough. """ - def __init__(self, db_alias=DEFAULT_DB_ALIAS, expiry=60): + def __init__(self, db_alias=DEFAULT_DB_ALIAS, expiry=60, group_expiry=86400): self.expiry = expiry + self.group_expiry = group_expiry self.db_alias = db_alias # ASGI API @@ -165,13 +166,14 @@ class DatabaseChannelLayer(object): class Group(models.Model): group = models.CharField(max_length=200) channel = models.CharField(max_length=200) + created = models.DateTimeField(db_index=True, auto_now_add=True) class Meta: apps = Apps() app_label = "channels" db_table = "django_channel_groups" unique_together = [["group", "channel"]] - # Ensure its table exists + # Ensure its table exists with the right schema if Group._meta.db_table not in self.connection.introspection.table_names(self.connection.cursor()): with self.connection.schema_editor() as editor: editor.create_model(Group) @@ -187,8 +189,11 @@ class DatabaseChannelLayer(object): old_messages = self.channel_model.objects.filter(expiry__lt=target) channels_to_ungroup = old_messages.values_list("channel", flat=True).distinct() old_messages.delete() - # Now, remove channel membership from channels that expired - self.group_model.objects.filter(channel__in=channels_to_ungroup).delete() + # Now, remove channel membership from channels that expired and ones that just expired + self.group_model.objects.filter( + models.Q(channel__in=channels_to_ungroup) | + models.Q(created__lte=target - datetime.timedelta(seconds=self.group_expiry)) + ).delete() def __str__(self): return "%s(alias=%s)" % (self.__class__.__name__, self.connection.alias) diff --git a/channels/tests/test_database_layer.py b/channels/tests/test_database_layer.py index c501c15..4878157 100644 --- a/channels/tests/test_database_layer.py +++ b/channels/tests/test_database_layer.py @@ -1,6 +1,8 @@ from __future__ import unicode_literals -from asgiref.conformance import make_tests +from asgiref.conformance import ConformanceTestCase from ..database_layer import DatabaseChannelLayer -channel_layer = DatabaseChannelLayer(expiry=1) -DatabaseLayerTests = make_tests(channel_layer, expiry_delay=2.1) + +class DatabaseLayerTests(ConformanceTestCase): + channel_layer = DatabaseChannelLayer(expiry=1, group_expiry=3) + expiry_delay = 2.1 From 1368e865d29ac1f0a44e0ffed003f135b4af2e87 Mon Sep 17 00:00:00 2001 From: Tom Clancy Date: Mon, 28 Mar 2016 14:12:22 -0400 Subject: [PATCH 294/746] Add missing work to concepts.rst --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 2cc7652..c8e65c5 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -128,7 +128,7 @@ However, the crucial part is that you can run code (and so send on channels) in response to any event - and that includes ones you create. You can trigger on model saves, on other incoming messages, or from code paths inside views and forms. That approach comes in handy for push-style -code - where you WebSockets or HTTP long-polling to notify +code - where you use WebSockets or HTTP long-polling to notify clients of changes in real time (messages in a chat, perhaps, or live updates in an admin as another user edits something). From 8f40ad1d6829e722bb97da97788de77719a2e04d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 29 Mar 2016 11:43:54 +0100 Subject: [PATCH 295/746] Releasing 0.10.3 --- CHANGELOG.txt | 10 ++++++++++ channels/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 797b913..407123f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,13 @@ +0.10.3 (2016-03-29) +------------------- + +* Better error messages for wrongly-constructed routing lists + +* Error when trying to use signed cookie backend with channel_session + +* ASGI group_expiry implemented on database channel backend + + 0.10.2 (2016-03-23) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 0536163..f62749e 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.10.2" +__version__ = "0.10.3" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From e710ac02df4661672f95164440c1105a204b927b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 29 Mar 2016 11:45:23 +0100 Subject: [PATCH 296/746] Add build dir to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4261b15..2071634 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.egg-info dist/ +build/ docs/_build __pycache__/ *.sqlite3 From 9baae5d9780d51928fda77c5157bb28faed7ed09 Mon Sep 17 00:00:00 2001 From: Michael Borisov Date: Thu, 31 Mar 2016 15:40:18 +0200 Subject: [PATCH 297/746] Update package url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a3ad07..831cb1c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from channels import __version__ setup( name='channels', version=__version__, - url='http://github.com/andrewgodwin/django-channels', + url='http://github.com/andrewgodwin/channels', author='Andrew Godwin', author_email='andrew@aeracode.org', description="Brings event-driven capabilities to Django with a channel system. Django 1.8 and up only.", From dfef0c551eda0c1a632f69360b284f32c9e56ecd Mon Sep 17 00:00:00 2001 From: Haiko Schol Date: Sat, 2 Apr 2016 12:14:43 +0200 Subject: [PATCH 298/746] Fix typo in paragraph about channel types --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index c8e65c5..7621852 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -157,7 +157,7 @@ to the channel server they're listening on. For this reason, Channels treats these as two different *channel types*, and denotes a *response channel* by having the channel name contain the character ``!`` - e.g. ``http.response!f5G3fE21f``. *Normal -channels* have do not contain it, but along with the rest of the response +channels* do not contain it, but along with the rest of the response channel name, they must contain only the characters ``a-z A-Z 0-9 - _``, and be less than 200 characters long. From 3576267be277e216e254881cfa78363093dd28c0 Mon Sep 17 00:00:00 2001 From: Arnaud Limbourg Date: Mon, 4 Apr 2016 03:32:13 +0200 Subject: [PATCH 299/746] Add paragraph on updating routing.py at that point --- docs/getting-started.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 7b25fa9..a032af7 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -365,6 +365,18 @@ name in the path of your WebSocket request (we'll ignore auth for now - that's n def ws_disconnect(message): Group("chat-%s" % message.channel_session['room']).discard(message.reply_channel) +Update ``routing.py`` as well:: + + # in routing.py + from channels.routing import route + from myapp.consumers import ws_connect, ws_message, ws_disconnect + + channel_routing = [ + route("websocket.connect", ws_connect), + route("websocket.receive", ws_message), + route("websocket.disconnect", ws_disconnect), + ] + If you play around with it from the console (or start building a simple JavaScript chat client that appends received messages to a div), you'll see that you can set a chat room with the initial request. From 0e3c742a803c066099b67f11082730f6317350b3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 3 Apr 2016 18:32:42 +0200 Subject: [PATCH 300/746] Introduce ChannelTestCase to make testing easier --- channels/asgi.py | 10 ++++++ channels/tests/__init__.py | 1 + channels/tests/base.py | 61 ++++++++++++++++++++++++++++++++++ channels/tests/settings.py | 7 ++++ channels/tests/test_handler.py | 26 +++++---------- channels/tests/test_request.py | 58 ++++++++++++++------------------ 6 files changed, 113 insertions(+), 50 deletions(-) create mode 100644 channels/tests/base.py diff --git a/channels/asgi.py b/channels/asgi.py index 737a422..6156a48 100644 --- a/channels/asgi.py +++ b/channels/asgi.py @@ -56,6 +56,16 @@ class ChannelLayerManager(object): def __contains__(self, key): return key in self.configs + def set(self, key, layer): + """ + Sets an alias to point to a new ChannelLayerWrapper instance, and + returns the old one that it replaced. Useful for swapping out the + backend during tests. + """ + old = self.backends.get(key, None) + self.backends[key] = layer + return old + class ChannelLayerWrapper(object): """ diff --git a/channels/tests/__init__.py b/channels/tests/__init__.py index e69de29..566d872 100644 --- a/channels/tests/__init__.py +++ b/channels/tests/__init__.py @@ -0,0 +1 @@ +from .base import ChannelTestCase diff --git a/channels/tests/base.py b/channels/tests/base.py new file mode 100644 index 0000000..bed25f1 --- /dev/null +++ b/channels/tests/base.py @@ -0,0 +1,61 @@ +from django.test import TestCase +from channels import DEFAULT_CHANNEL_LAYER +from channels.asgi import channel_layers, ChannelLayerWrapper +from channels.message import Message +from asgiref.inmemory import ChannelLayer as InMemoryChannelLayer + + +class ChannelTestCase(TestCase): + """ + TestCase subclass that provides easy methods for testing channels using + an in-memory backend to capture messages, and assertion methods to allow + checking of what was sent. + + Inherits from TestCase, so provides per-test transactions as long as the + database backend supports it. + """ + + # Customizable so users can test multi-layer setups + test_channel_aliases = [DEFAULT_CHANNEL_LAYER] + + def setUp(self): + """ + Initialises in memory channel layer for the duration of the test + """ + super(ChannelTestCase, self).setUp() + self._old_layers = {} + for alias in self.test_channel_aliases: + # Swap in an in memory layer wrapper and keep the old one around + self._old_layers[alias] = channel_layers.set( + alias, + ChannelLayerWrapper( + InMemoryChannelLayer(), + alias, + channel_layers[alias].routing, + ) + ) + + def tearDown(self): + """ + Undoes the channel rerouting + """ + for alias in self.test_channel_aliases: + # Swap in an in memory layer wrapper and keep the old one around + channel_layers.set(alias, self._old_layers[alias]) + del self._old_layers + super(ChannelTestCase, self).tearDown() + + def get_next_message(self, channel, alias=DEFAULT_CHANNEL_LAYER, require=False): + """ + Gets the next message that was sent to the channel during the test, + or None if no message is available. + + If require is true, will fail the test if no message is received. + """ + recv_channel, content = channel_layers[alias].receive_many([channel]) + if recv_channel is None: + if require: + self.fail("Expected a message on channel %s, got none" % channel) + else: + return None + return Message(content, recv_channel, channel_layers[alias]) diff --git a/channels/tests/settings.py b/channels/tests/settings.py index c472c7e..c06b6b4 100644 --- a/channels/tests/settings.py +++ b/channels/tests/settings.py @@ -6,4 +6,11 @@ DATABASES = { } } +CHANNEL_LAYERS = { + 'default': { + 'BACKEND': 'asgiref.inmemory.ChannelLayer', + 'ROUTING': [], + }, +} + MIDDLEWARE_CLASSES = [] diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index 516ec16..450f06b 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -1,10 +1,9 @@ from __future__ import unicode_literals -from django.test import SimpleTestCase from django.http import HttpResponse -from asgiref.inmemory import ChannelLayer +from channels import Channel from channels.handler import AsgiHandler -from channels.message import Message +from channels.tests import ChannelTestCase class FakeAsgiHandler(AsgiHandler): @@ -24,34 +23,27 @@ class FakeAsgiHandler(AsgiHandler): return self._response -class HandlerTests(SimpleTestCase): +class HandlerTests(ChannelTestCase): """ Tests that the handler works correctly and round-trips things into a correct response. """ - def setUp(self): - """ - Make an in memory channel layer for testing - """ - self.channel_layer = ChannelLayer() - self.make_message = lambda m, c: Message(m, c, self.channel_layer) - def test_basic(self): """ Tests a simple request """ # Make stub request and desired response - message = self.make_message({ + Channel("test").send({ "reply_channel": "test", "http_version": "1.1", "method": "GET", "path": b"/test/", - }, "test") + }) response = HttpResponse(b"Hi there!", content_type="text/plain") # Run the handler handler = FakeAsgiHandler(response) - reply_messages = list(handler(message)) + reply_messages = list(handler(self.get_next_message("test", require=True))) # Make sure we got the right number of messages self.assertEqual(len(reply_messages), 1) reply_message = reply_messages[0] @@ -69,16 +61,16 @@ class HandlerTests(SimpleTestCase): Tests a large response (will need chunking) """ # Make stub request and desired response - message = self.make_message({ + Channel("test").send({ "reply_channel": "test", "http_version": "1.1", "method": "GET", "path": b"/test/", - }, "test") + }) response = HttpResponse(b"Thefirstthirtybytesisrighthereandhereistherest") # Run the handler handler = FakeAsgiHandler(response) - reply_messages = list(handler(message)) + reply_messages = list(handler(self.get_next_message("test", require=True))) # Make sure we got the right number of messages self.assertEqual(len(reply_messages), 2) # Make sure the messages look correct diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index fe6f76c..7d1c4ec 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -1,36 +1,28 @@ from __future__ import unicode_literals -from django.test import SimpleTestCase from django.utils import six -from asgiref.inmemory import ChannelLayer +from channels import Channel +from channels.tests import ChannelTestCase from channels.handler import AsgiRequest -from channels.message import Message -class RequestTests(SimpleTestCase): +class RequestTests(ChannelTestCase): """ Tests that ASGI request handling correctly decodes HTTP requests. """ - def setUp(self): - """ - Make an in memory channel layer for testing - """ - self.channel_layer = ChannelLayer() - self.make_message = lambda m, c: Message(m, c, self.channel_layer) - def test_basic(self): """ Tests that the handler can decode the most basic request message, with all optional fields omitted. """ - message = self.make_message({ + Channel("test").send({ "reply_channel": "test-reply", "http_version": "1.1", "method": "GET", "path": b"/test/", - }, "test") - request = AsgiRequest(message) + }) + request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.path, "/test/") self.assertEqual(request.method, "GET") self.assertFalse(request.body) @@ -48,7 +40,7 @@ class RequestTests(SimpleTestCase): """ Tests a more fully-featured GET request """ - message = self.make_message({ + Channel("test").send({ "reply_channel": "test", "http_version": "1.1", "method": "GET", @@ -60,8 +52,8 @@ class RequestTests(SimpleTestCase): }, "client": ["10.0.0.1", 1234], "server": ["10.0.0.2", 80], - }, "test") - request = AsgiRequest(message) + }) + request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.path, "/test2/") self.assertEqual(request.method, "GET") self.assertFalse(request.body) @@ -81,7 +73,7 @@ class RequestTests(SimpleTestCase): """ Tests a POST body contained within a single message. """ - message = self.make_message({ + Channel("test").send({ "reply_channel": "test", "http_version": "1.1", "method": "POST", @@ -93,8 +85,8 @@ class RequestTests(SimpleTestCase): "content-type": b"application/x-www-form-urlencoded", "content-length": b"18", }, - }, "test") - request = AsgiRequest(message) + }) + request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.path, "/test2/") self.assertEqual(request.method, "POST") self.assertEqual(request.body, b"ponies=are+awesome") @@ -111,7 +103,7 @@ class RequestTests(SimpleTestCase): """ Tests a POST body across multiple messages (first part in 'body'). """ - message = self.make_message({ + Channel("test").send({ "reply_channel": "test", "http_version": "1.1", "method": "POST", @@ -123,15 +115,15 @@ class RequestTests(SimpleTestCase): "content-type": b"application/x-www-form-urlencoded", "content-length": b"21", }, - }, "test") - self.channel_layer.send("test-input", { + }) + Channel("test-input").send({ "content": b"re=fou", "more_content": True, }) - self.channel_layer.send("test-input", { + Channel("test-input").send({ "content": b"r+lights", }) - request = AsgiRequest(message) + request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.method, "POST") self.assertEqual(request.body, b"there_are=four+lights") self.assertEqual(request.META["CONTENT_TYPE"], "application/x-www-form-urlencoded") @@ -151,7 +143,7 @@ class RequestTests(SimpleTestCase): b'FAKEPDFBYTESGOHERE' + b'--BOUNDARY--' ) - message = self.make_message({ + Channel("test").send({ "reply_channel": "test", "http_version": "1.1", "method": "POST", @@ -161,15 +153,15 @@ class RequestTests(SimpleTestCase): "content-type": b"multipart/form-data; boundary=BOUNDARY", "content-length": six.text_type(len(body)).encode("ascii"), }, - }, "test") - self.channel_layer.send("test-input", { + }) + Channel("test-input").send({ "content": body[:20], "more_content": True, }) - self.channel_layer.send("test-input", { + Channel("test-input").send({ "content": body[20:], }) - request = AsgiRequest(message) + request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.method, "POST") self.assertEqual(len(request.body), len(body)) self.assertTrue(request.META["CONTENT_TYPE"].startswith("multipart/form-data")) @@ -181,7 +173,7 @@ class RequestTests(SimpleTestCase): """ Tests the body stream is emulated correctly. """ - message = self.make_message({ + Channel("test").send({ "reply_channel": "test", "http_version": "1.1", "method": "PUT", @@ -191,8 +183,8 @@ class RequestTests(SimpleTestCase): "host": b"example.com", "content-length": b"11", }, - }, "test") - request = AsgiRequest(message) + }) + request = AsgiRequest(self.get_next_message("test", require=True)) self.assertEqual(request.method, "PUT") self.assertEqual(request.read(3), b"one") self.assertEqual(request.read(), b"twothree") From 5a22412c16de35c53ad5123d6a998255202d2435 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 3 Apr 2016 18:53:12 +0200 Subject: [PATCH 301/746] Considerably improve routing code simplicity and shortcircuiting --- channels/message.py | 11 +++ channels/routing.py | 118 +++++++++++++-------------------- channels/tests/test_routing.py | 27 ++++---- docs/getting-started.rst | 15 +++-- 4 files changed, 79 insertions(+), 92 deletions(-) diff --git a/channels/message.py b/channels/message.py index 15e3891..f41de35 100644 --- a/channels/message.py +++ b/channels/message.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +import copy from .channel import Channel @@ -38,3 +39,13 @@ class Message(object): def get(self, key, default=None): return self.content.get(key, default) + + def copy(self): + """ + Returns a safely content-mutable copy of this Message. + """ + return self.__class__( + copy.deepcopy(self.content), + self.channel.name, + self.channel_layer, + ) diff --git a/channels/routing.py b/channels/routing.py index d25e09a..41ab0b0 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -19,28 +19,17 @@ class Router(object): """ def __init__(self, routing): - # Resolve routing into a list if it's a dict or string - routing = self.resolve_routing(routing) - # Expand those entries recursively into a flat list of Routes - self.routing = [] - for entry in routing: - try: - self.routing.extend(entry.expand_routes()) - except AttributeError: - # It's not a valid route - raise ValueError("Encountered %r in routing config, which is not a valid route() or include()" % entry) - # Now go through that list and collect channel names into a set - self.channels = { - route.channel - for route in self.routing - } + # Use a blank include as the root item + self.root = Include(routing) + # Cache channel names + self.channels = self.root.channel_names() def add_route(self, route): """ Adds a single raw Route to us at the end of the resolution list. """ - self.routing.append(route) - self.channels.add(route.channel) + self.root.routing.append(route) + self.channels = self.root.channel_names() def match(self, message): """ @@ -50,11 +39,7 @@ class Router(object): """ # TODO: Maybe we can add some kind of caching in here if we can hash # the message with only matchable keys faster than the search? - for route in self.routing: - match = route.match(message) - if match is not None: - return match - return None + return self.root.match(message) def check_default(self, http_consumer=None): """ @@ -152,43 +137,11 @@ class Route(object): return None return self.consumer, call_args - def expand_routes(self): + def channel_names(self): """ - Expands this route into a list of just itself. + Returns the channel names this route listens on """ - return [self] - - def add_prefixes(self, prefixes): - """ - Returns a new Route with the given prefixes added to our filters. - """ - new_filters = {} - # Copy over our filters adding any prefixes - for name, value in self.filters.items(): - if name in prefixes: - if not value.pattern.startswith("^"): - raise ValueError("Cannot add prefix for %s on %s as inner value does not start with ^" % ( - name, - self, - )) - if "$" in prefixes[name]: - raise ValueError("Cannot add prefix for %s on %s as prefix contains $ (end of line match)" % ( - name, - self, - )) - new_filters[name] = re.compile(prefixes[name] + value.pattern.lstrip("^")) - else: - new_filters[name] = value - # Now add any prefixes that are by themselves so they're still enforced - for name, prefix in prefixes.items(): - if name not in new_filters: - new_filters[name] = prefix - # Return new copy - return self.__class__( - self.channel, - self.consumer, - **new_filters - ) + return self.channel def __str__(self): return "%s %s -> %s" % ( @@ -210,26 +163,49 @@ class Include(object): def __init__(self, routing, **kwargs): self.routing = Router.resolve_routing(routing) self.prefixes = { - name: Router.normalise_re_arg(value) + name: re.compile(Router.normalise_re_arg(value)) for name, value in kwargs.items() } - # Sanity check prefix regexes - for name, value in self.prefixes.items(): - if not value.startswith("^"): - raise ValueError("Include prefix for %s must start with the ^ character." % name) - def expand_routes(self): + def match(self, message): """ - Expands this Include into a list of routes, first recursively expanding - and then adding on prefixes to filters if specified. + Tries to match the message against our own prefixes, possibly modifying + what we send to included things, then tries all included items. """ - # First, expand our own subset of routes, to get a list of Route objects - routes = [] + # Check our prefixes match. Do this against a copy of the message so + # we can write back any changed values. + message = message.copy() + call_args = {} + for name, prefix in self.prefixes.items(): + if name not in message: + return None + value = Router.normalise_re_arg(message[name]) + match = prefix.match(value) + # Any match failure means we pass + if match: + call_args.update(match.groupdict()) + # Modify the message value to remove the part we matched on + message[name] = value[match.end():] + else: + return None + # Alright, if we got this far our prefixes match. Try all of our + # included objects now. for entry in self.routing: - routes.extend(entry.expand_routes()) - # Then, go through those and add any prefixes we have. - routes = [route.add_prefixes(self.prefixes) for route in routes] - return routes + match = entry.match(message) + if match is not None: + call_args.update(match[1]) + return match[0], call_args + # Nothing matched :( + return None + + def channel_names(self): + """ + Returns the channel names this route listens on + """ + result = set() + for entry in self.routing: + result.union(entry.channel_names()) + return result # Lowercase standard to match urls.py diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py index 75d0659..5605175 100644 --- a/channels/tests/test_routing.py +++ b/channels/tests/test_routing.py @@ -24,7 +24,7 @@ chatroom_routing = [ route("websocket.connect", consumer_3, path=r"^/mentions/$"), ] -chatroom_routing_noprefix = [ +chatroom_routing_nolinestart = [ route("websocket.connect", consumer_2, path=r"/chat/(?P[^/]+)/$"), route("websocket.connect", consumer_3, path=r"/mentions/$"), ] @@ -207,6 +207,17 @@ class RoutingTests(SimpleTestCase): consumer=consumer_3, kwargs={"version": "1"}, ) + # Check it works without the ^s too. + router = Router([ + include("channels.tests.test_routing.chatroom_routing_nolinestart", path="/ws/v(?P[0-9]+)"), + ]) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/ws/v2/chat/django/"}, + consumer=consumer_2, + kwargs={"version": "2", "room": "django"}, + ) def test_positional_pattern(self): """ @@ -217,20 +228,6 @@ class RoutingTests(SimpleTestCase): route("http.request", consumer_1, path=r"^/chat/([^/]+)/$"), ]) - def test_bad_include_prefix(self): - """ - Tests both failure cases of prefixes for includes - the include not - starting with ^, and the included filter not starting with ^. - """ - with self.assertRaises(ValueError): - Router([ - include("channels.tests.test_routing.chatroom_routing", path="foobar"), - ]) - with self.assertRaises(ValueError): - Router([ - include("channels.tests.test_routing.chatroom_routing_noprefix", path="^/foobar/"), - ]) - def test_mixed_unicode_bytes(self): """ Tests that having the message key be bytes and pattern unicode (or vice-versa) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index a032af7..2e8dad8 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -504,12 +504,15 @@ routing our chat from above:: include(http_routing), ] -When Channels loads this routing, it appends any match keys together and -flattens out the routing, so the ``path`` match for ``chat_connect`` becomes -``^/chat/(?P[a-zA-Z0-9_]+)/$``. If the include match -or the route match doesn't have the ``^`` character, it will refuse to append them -and error (you can still have matches without ``^`` in either, you just can't -ask Channels to combine them). +Channels will resolve the routing in order, short-circuiting around the +includes if one or more of their matches fails. You don't have to start with +the ``^`` symbol - we use Python's ``re.match`` function, which starts at the +start of a line anyway - but it's considered good practice. + +When an include matches part of a message value, it chops off the bit of the +value it matched before passing it down to its routes or sub-includes, so you +can put the same routing under multiple includes with different prefixes if +you like. Because these matches come through as keyword arguments, we could modify our consumer above to use a room based on URL rather than username:: From d0bb8721137a4e688d3c3ccf058bce4f675445c4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 3 Apr 2016 19:41:26 +0200 Subject: [PATCH 302/746] Add first version of patchinator --- patchinator.py | 195 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 patchinator.py diff --git a/patchinator.py b/patchinator.py new file mode 100644 index 0000000..7e2430c --- /dev/null +++ b/patchinator.py @@ -0,0 +1,195 @@ +#!/usr/bin/python +""" +Script that automatically generates a Django patch from the Channels codebase +based on some simple rules and string replacements. + +Once Channels lands in Django, will be reversed to instead generate this +third-party app from the now-canonical Django source. +""" + +import re +import os.path +import sys + + +### Transforms: Turn one content string into another ### + +class Replacement(object): + """ + Represents a string replacement in a file; uses a regular expression to + substitute strings in the file. + """ + + def __init__(self, match, sub): + self.match = match + self.sub = sub + + def __call__(self, value): + return re.sub(self.match, self.sub, value) + + +class Insert(object): + """ + Inserts a string before/after another in a file, one time only, with multiline match. + """ + + def __init__(self, match, to_insert, after=False): + self.match = match + self.to_insert = to_insert + self.after = after + + def __call__(self, value): + match = re.search(self.match, value, flags=re.MULTILINE) + if not match: + raise ValueError("Could not find match %s" % self.match) + if self.after: + return value[:match.end()] + self.to_insert + value[match.end():] + else: + return value[:match.start()] + self.to_insert + value[match.start():] + + +### Operations: Copy or patch files ### + +class FileMap(object): + """ + Represents a file map from the source to the destination, with + optional extra regex transforms. + """ + + def __init__(self, source_path, dest_path, transforms, makedirs=True): + self.source_path = source_path + self.dest_path = dest_path + self.transforms = transforms + self.makedirs = makedirs + + def run(self, source_dir, dest_dir): + print("COPY: %s -> %s" % (self.source_path, self.dest_path)) + # Open and read in source file + source = os.path.join(source_dir, self.source_path) + with open(source, "r") as fh: + content = fh.read() + # Run transforms + for transform in self.transforms: + content = transform(content) + # Save new file + dest = os.path.join(dest_dir, self.dest_path) + if self.makedirs: + if not os.path.isdir(os.path.dirname(dest)): + os.makedirs(os.path.dirname(dest)) + with open(dest, "w") as fh: + fh.write(content) + +class DestPatch(object): + """ + Patches a destination file in place with transforms + """ + + def __init__(self, dest_path, transforms): + self.dest_path = dest_path + self.transforms = transforms + + def run(self, source_dir, dest_dir): + print("PATCH: %s" % (self.dest_path, )) + # Open and read in the file + dest = os.path.join(dest_dir, self.dest_path) + with open(dest, "r") as fh: + content = fh.read() + # Run transforms + for transform in self.transforms: + content = transform(content) + # Save new file + with open(dest, "w") as fh: + fh.write(content) + + +### Main class ### + + +global_transforms = [ + Replacement(r"import channels.([a-zA-Z0-9_\.]+)$", r"import django.channels.\1 as channels"), + Replacement(r"from channels import", r"from django.channels import"), + Replacement(r"from channels.([a-zA-Z0-9_\.]+) import", r"from django.channels.\1 import"), + Replacement(r"from .handler import", r"from django.core.handlers.asgi import") +] + + +class Patchinator(object): + + operations = [ + FileMap( + "channels/__init__.py", "django/channels/__init__.py", global_transforms, + ), + FileMap( + "channels/asgi.py", "django/channels/asgi.py", global_transforms, + ), + FileMap( + "channels/auth.py", "django/channels/auth.py", global_transforms, + ), + FileMap( + "channels/channel.py", "django/channels/channel.py", global_transforms, + ), + FileMap( + "channels/database_layer.py", "django/channels/database_layer.py", global_transforms, + ), + FileMap( + "channels/exceptions.py", "django/channels/exceptions.py", global_transforms, + ), + FileMap( + "channels/handler.py", "django/core/handlers/asgi.py", global_transforms, + ), + FileMap( + "channels/routing.py", "django/channels/routing.py", global_transforms, + ), + FileMap( + "channels/sessions.py", "django/channels/sessions.py", global_transforms, + ), + FileMap( + "channels/staticfiles.py", "django/channels/staticfiles.py", global_transforms, + ), + FileMap( + "channels/utils.py", "django/channels/utils.py", global_transforms, + ), + FileMap( + "channels/worker.py", "django/channels/worker.py", global_transforms, + ), + FileMap( + "channels/management/commands/runworker.py", "django/core/management/commands/runworker.py", global_transforms, + ), + FileMap( + "channels/tests/base.py", "django/test/channels.py", global_transforms, + ), + FileMap( + "channels/tests/test_database_layer.py", "tests/channels_tests/test_database_layer.py", global_transforms, + ), + FileMap( + "channels/tests/test_handler.py", "tests/channels_tests/test_handler.py", global_transforms, + ), + FileMap( + "channels/tests/test_routing.py", "tests/channels_tests/test_routing.py", global_transforms, + ), + FileMap( + "channels/tests/test_request.py", "tests/channels_tests/test_request.py", global_transforms, + ), + DestPatch( + "django/test/__init__.py", [ + Insert(r"from django.test.utils import", "from django.test.channels import ChannelTestCase\n"), + Insert(r"'LiveServerTestCase'", ", 'ChannelTestCase'", after=True), + ] + ) + ] + + def __init__(self, source, destination): + self.source = os.path.abspath(source) + self.destination = os.path.abspath(destination) + + def run(self): + print("Patchinator running.\n Source: %s\n Destination: %s" % (self.source, self.destination)) + for operation in self.operations: + operation.run(self.source, self.destination) + + +if __name__ == '__main__': + try: + Patchinator(os.path.dirname(__file__), sys.argv[1]).run() + except IndexError: + print("Supply the target Django directory on the command line") From bd796cb7e6ba6d43cb0b3765ef5c000f9e56d69c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 3 Apr 2016 23:13:14 +0200 Subject: [PATCH 303/746] More patchinator stuff, but unsure if this is the right approach --- docs/concepts.rst | 4 ++-- patchinator.py | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 7621852..3fcb63f 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -1,5 +1,5 @@ -Concepts -======== +Channels Concepts +================= Django's traditional view of the world revolves around requests and responses; a request comes in, Django is fired up to serve it, generates a response to diff --git a/patchinator.py b/patchinator.py index 7e2430c..c35ca84 100644 --- a/patchinator.py +++ b/patchinator.py @@ -101,6 +101,23 @@ class DestPatch(object): with open(dest, "w") as fh: fh.write(content) +class NewFile(object): + """ + Writes a file to the destination, either blank or with some content from + a string. + """ + + def __init__(self, dest_path, content=""): + self.dest_path = dest_path + self.content = content + + def run(self, source_dir, dest_dir): + print("NEW: %s" % (self.dest_path, )) + # Save new file + dest = os.path.join(dest_dir, self.dest_path) + with open(dest, "w") as fh: + fh.write(self.content) + ### Main class ### @@ -112,12 +129,14 @@ global_transforms = [ Replacement(r"from .handler import", r"from django.core.handlers.asgi import") ] +docs_transforms = global_transforms + [] + class Patchinator(object): operations = [ FileMap( - "channels/__init__.py", "django/channels/__init__.py", global_transforms, + "patchinator/channels-init.py", "django/channels/__init__.py", [], ), FileMap( "channels/asgi.py", "django/channels/asgi.py", global_transforms, @@ -155,9 +174,13 @@ class Patchinator(object): FileMap( "channels/management/commands/runworker.py", "django/core/management/commands/runworker.py", global_transforms, ), + # Tests FileMap( "channels/tests/base.py", "django/test/channels.py", global_transforms, ), + NewFile( + "tests/channels_tests/__init__.py", + ), FileMap( "channels/tests/test_database_layer.py", "tests/channels_tests/test_database_layer.py", global_transforms, ), @@ -175,7 +198,26 @@ class Patchinator(object): Insert(r"from django.test.utils import", "from django.test.channels import ChannelTestCase\n"), Insert(r"'LiveServerTestCase'", ", 'ChannelTestCase'", after=True), ] - ) + ), + # Docs + FileMap( + "docs/backends.rst", "docs/ref/channels/backends.txt", docs_transforms, + ), + FileMap( + "docs/concepts.rst", "docs/topics/channels/concepts.txt", docs_transforms, + ), + FileMap( + "docs/deploying.rst", "docs/ref/channels/deploying.txt", docs_transforms, + ), + FileMap( + "docs/getting-started.rst", "docs/howto/channels-started.txt", docs_transforms, + ), + FileMap( + "docs/reference.rst", "docs/ref/channels/api.txt", docs_transforms, + ), + FileMap( + "docs/scaling.rst", "docs/ref/channels/scaling.txt", docs_transforms, + ), ] def __init__(self, source, destination): From 0071ca31c8beaac28fa397eb9e68415755f00d03 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 3 Apr 2016 23:31:21 +0200 Subject: [PATCH 304/746] More patchinator tweaks. Will do more after 1.0. --- docs/getting-started.rst | 4 ++-- patchinator.py | 44 +++++++++------------------------------- 2 files changed, 12 insertions(+), 36 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 2e8dad8..427e579 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -1,5 +1,5 @@ -Getting Started -=============== +Getting Started with Channels +============================= (If you haven't yet, make sure you :doc:`install Channels `) diff --git a/patchinator.py b/patchinator.py index c35ca84..dc283ea 100644 --- a/patchinator.py +++ b/patchinator.py @@ -79,27 +79,6 @@ class FileMap(object): with open(dest, "w") as fh: fh.write(content) -class DestPatch(object): - """ - Patches a destination file in place with transforms - """ - - def __init__(self, dest_path, transforms): - self.dest_path = dest_path - self.transforms = transforms - - def run(self, source_dir, dest_dir): - print("PATCH: %s" % (self.dest_path, )) - # Open and read in the file - dest = os.path.join(dest_dir, self.dest_path) - with open(dest, "r") as fh: - content = fh.read() - # Run transforms - for transform in self.transforms: - content = transform(content) - # Save new file - with open(dest, "w") as fh: - fh.write(content) class NewFile(object): """ @@ -129,15 +108,18 @@ global_transforms = [ Replacement(r"from .handler import", r"from django.core.handlers.asgi import") ] -docs_transforms = global_transforms + [] +docs_transforms = global_transforms + [ + Replacement(r":doc:`concepts`", r":doc:`topics/channels/concepts`"), + Replacement(r":doc:`deploying`", r":doc:`topics/channels/deploying`"), + Replacement(r":doc:`scaling`", r":doc:`topics/channels/scaling`"), + Replacement(r":doc:`getting-started`", r":doc:`intro/channels`"), + Replacement(r"\n\(.*installation>`\)\n", r""), +] class Patchinator(object): operations = [ - FileMap( - "patchinator/channels-init.py", "django/channels/__init__.py", [], - ), FileMap( "channels/asgi.py", "django/channels/asgi.py", global_transforms, ), @@ -193,12 +175,6 @@ class Patchinator(object): FileMap( "channels/tests/test_request.py", "tests/channels_tests/test_request.py", global_transforms, ), - DestPatch( - "django/test/__init__.py", [ - Insert(r"from django.test.utils import", "from django.test.channels import ChannelTestCase\n"), - Insert(r"'LiveServerTestCase'", ", 'ChannelTestCase'", after=True), - ] - ), # Docs FileMap( "docs/backends.rst", "docs/ref/channels/backends.txt", docs_transforms, @@ -207,16 +183,16 @@ class Patchinator(object): "docs/concepts.rst", "docs/topics/channels/concepts.txt", docs_transforms, ), FileMap( - "docs/deploying.rst", "docs/ref/channels/deploying.txt", docs_transforms, + "docs/deploying.rst", "docs/topics/channels/deploying.txt", docs_transforms, ), FileMap( - "docs/getting-started.rst", "docs/howto/channels-started.txt", docs_transforms, + "docs/getting-started.rst", "docs/intro/channels.txt", docs_transforms, ), FileMap( "docs/reference.rst", "docs/ref/channels/api.txt", docs_transforms, ), FileMap( - "docs/scaling.rst", "docs/ref/channels/scaling.txt", docs_transforms, + "docs/scaling.rst", "docs/topics/channels/scaling.txt", docs_transforms, ), ] From a563d4353f44dce032ed8aad3bc6ef7ae2467190 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 4 Apr 2016 00:24:58 +0200 Subject: [PATCH 305/746] Fix new routing channel name collector and add test --- channels/routing.py | 4 ++-- channels/tests/test_routing.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/channels/routing.py b/channels/routing.py index 41ab0b0..0806bd5 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -141,7 +141,7 @@ class Route(object): """ Returns the channel names this route listens on """ - return self.channel + return {self.channel,} def __str__(self): return "%s %s -> %s" % ( @@ -204,7 +204,7 @@ class Include(object): """ result = set() for entry in self.routing: - result.union(entry.channel_names()) + result.update(entry.channel_names()) return result diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py index 5605175..bfc44cd 100644 --- a/channels/tests/test_routing.py +++ b/channels/tests/test_routing.py @@ -281,3 +281,24 @@ class RoutingTests(SimpleTestCase): consumer=consumer_2, kwargs={"version": "2", "room": "django"}, ) + + def test_channels(self): + """ + Tests that the router reports channels to listen on correctly + """ + router = Router([ + route("http.request", consumer_1, path=r"^/chat/$"), + route("http.disconnect", consumer_2), + route("http.request", consumer_3), + ]) + # Initial check + self.assertEqual( + router.channels, + {"http.request", "http.disconnect"}, + ) + # Dynamically add route, recheck + router.add_route(route("websocket.receive", consumer_1)) + self.assertEqual( + router.channels, + {"http.request", "http.disconnect", "websocket.receive"}, + ) From e18bfed8f30e840c0c6c2874d5a0f5925b25b011 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 4 Apr 2016 00:34:55 +0200 Subject: [PATCH 306/746] Clarify timeout behaviour of block() in asgi --- docs/asgi.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index d023f28..18ebcfd 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -274,7 +274,9 @@ A *channel layer* must provide an object with these attributes or ``(channel, message)`` if a message is available. If ``block`` is True, then it will not return until after a built-in timeout or a message arrives; if ``block`` is false, it will always return immediately. It is perfectly - valid to ignore ``block`` and always return immediately. + valid to ignore ``block`` and always return immediately. If ``block`` is True, + there must be a finite timeout before this returns ``(None, None)`` and that + timeout must be less than sixty seconds (preferably around five). * ``new_channel(pattern)``, a callable that takes a unicode string pattern, and returns a new valid channel name that does not already exist, by From 920882f1da3e3c9a9432f2172bf06700bfc6fa48 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 4 Apr 2016 00:55:10 +0200 Subject: [PATCH 307/746] Implement timeout on request body reading --- channels/exceptions.py | 7 +++++++ channels/handler.py | 17 +++++++++++++++-- channels/tests/test_request.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/channels/exceptions.py b/channels/exceptions.py index 3e52699..53cdd37 100644 --- a/channels/exceptions.py +++ b/channels/exceptions.py @@ -14,3 +14,10 @@ class ResponseLater(Exception): returning a response. """ pass + + +class RequestTimeout(Exception): + """ + Raised when it takes too long to read a request body. + """ + pass diff --git a/channels/handler.py b/channels/handler.py index 6e2bedb..26ab586 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -4,6 +4,7 @@ import cgi import codecs import logging import sys +import time import traceback from io import BytesIO from threading import Lock @@ -13,11 +14,11 @@ from django.conf import settings from django.core import signals from django.core.handlers import base from django.core.urlresolvers import set_script_prefix -from django.http import FileResponse, HttpResponseServerError +from django.http import FileResponse, HttpResponse, HttpResponseServerError from django.utils import six from django.utils.functional import cached_property -from .exceptions import ResponseLater as ResponseLaterOuter +from .exceptions import ResponseLater as ResponseLaterOuter, RequestTimeout logger = logging.getLogger('django.request') @@ -30,6 +31,10 @@ class AsgiRequest(http.HttpRequest): ResponseLater = ResponseLaterOuter + # Number of seconds until a Request gives up on trying to read a request + # body and aborts. + body_receive_timeout = 60 + def __init__(self, message): self.message = message self.reply_channel = self.message.reply_channel @@ -100,10 +105,15 @@ class AsgiRequest(http.HttpRequest): # Body handling self._body = message.get("body", b"") if message.get("body_channel", None): + body_handle_start = time.time() while True: # Get the next chunk from the request body channel chunk = None while chunk is None: + # If they take too long, raise request timeout and the handler + # will turn it into a response + if time.time() - body_handle_start > self.body_receive_timeout: + raise RequestTimeout() _, chunk = message.channel_layer.receive_many( [message['body_channel']], block=True, @@ -184,6 +194,9 @@ class AsgiHandler(base.BaseHandler): } ) response = http.HttpResponseBadRequest() + except RequestTimeout: + # Parsing the rquest failed, so the response is a Request Timeout error + response = HttpResponse("408 Request Timeout (upload too slow)", status_code=408) else: try: response = self.get_response(request) diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index 7d1c4ec..028d3da 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -4,6 +4,7 @@ from django.utils import six from channels import Channel from channels.tests import ChannelTestCase from channels.handler import AsgiRequest +from channels.exceptions import RequestTimeout class RequestTests(ChannelTestCase): @@ -188,3 +189,30 @@ class RequestTests(ChannelTestCase): self.assertEqual(request.method, "PUT") self.assertEqual(request.read(3), b"one") self.assertEqual(request.read(), b"twothree") + + def test_request_timeout(self): + """ + Tests that the code correctly gives up after the request body read timeout. + """ + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "POST", + "path": b"/test/", + "body": b"there_a", + "body_channel": "test-input", + "headers": { + "host": b"example.com", + "content-type": b"application/x-www-form-urlencoded", + "content-length": b"21", + }, + }) + # Say there's more content, but never provide it! Muahahaha! + Channel("test-input").send({ + "content": b"re=fou", + "more_content": True, + }) + class VeryImpatientRequest(AsgiRequest): + body_receive_timeout = 0 + with self.assertRaises(RequestTimeout): + VeryImpatientRequest(self.get_next_message("test")) From 67e3e55131261f1aaf25586bf6bbc6f8e55b22cb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 4 Apr 2016 01:02:48 +0200 Subject: [PATCH 308/746] Respect HTTP request body close in ASGI. --- channels/exceptions.py | 8 ++++++++ channels/handler.py | 8 +++++++- channels/tests/test_request.py | 25 ++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/channels/exceptions.py b/channels/exceptions.py index 53cdd37..ffdb5a2 100644 --- a/channels/exceptions.py +++ b/channels/exceptions.py @@ -21,3 +21,11 @@ class RequestTimeout(Exception): Raised when it takes too long to read a request body. """ pass + + +class RequestAborted(Exception): + """ + Raised when the incoming request tells us it's aborted partway through + reading the body. + """ + pass diff --git a/channels/handler.py b/channels/handler.py index 26ab586..b9664f3 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -18,7 +18,7 @@ from django.http import FileResponse, HttpResponse, HttpResponseServerError from django.utils import six from django.utils.functional import cached_property -from .exceptions import ResponseLater as ResponseLaterOuter, RequestTimeout +from .exceptions import ResponseLater as ResponseLaterOuter, RequestTimeout, RequestAborted logger = logging.getLogger('django.request') @@ -118,6 +118,9 @@ class AsgiRequest(http.HttpRequest): [message['body_channel']], block=True, ) + # If chunk contains close, abort. + if chunk.get("closed", False): + raise RequestAborted() # Add content to body self._body += chunk.get("content", "") # Exit loop if this was the last @@ -197,6 +200,9 @@ class AsgiHandler(base.BaseHandler): except RequestTimeout: # Parsing the rquest failed, so the response is a Request Timeout error response = HttpResponse("408 Request Timeout (upload too slow)", status_code=408) + except RequestAborted: + # Client closed connection on us mid request. Abort! + return else: try: response = self.get_response(request) diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index 028d3da..52712a4 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -4,7 +4,7 @@ from django.utils import six from channels import Channel from channels.tests import ChannelTestCase from channels.handler import AsgiRequest -from channels.exceptions import RequestTimeout +from channels.exceptions import RequestTimeout, RequestAborted class RequestTests(ChannelTestCase): @@ -216,3 +216,26 @@ class RequestTests(ChannelTestCase): body_receive_timeout = 0 with self.assertRaises(RequestTimeout): VeryImpatientRequest(self.get_next_message("test")) + + def test_request_abort(self): + """ + Tests that the code aborts when a request-body close is sent. + """ + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "POST", + "path": b"/test/", + "body": b"there_a", + "body_channel": "test-input", + "headers": { + "host": b"example.com", + "content-type": b"application/x-www-form-urlencoded", + "content-length": b"21", + }, + }) + Channel("test-input").send({ + "closed": True, + }) + with self.assertRaises(RequestAborted): + AsgiRequest(self.get_next_message("test")) From 37923c3674323f51be787092db6dd8b17c4f1265 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 4 Apr 2016 03:38:38 +0200 Subject: [PATCH 309/746] Flake8 fixes --- channels/routing.py | 2 +- channels/tests/__init__.py | 2 +- channels/tests/test_request.py | 2 ++ patchinator.py | 12 +++++++----- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/channels/routing.py b/channels/routing.py index 0806bd5..c311658 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -141,7 +141,7 @@ class Route(object): """ Returns the channel names this route listens on """ - return {self.channel,} + return {self.channel, } def __str__(self): return "%s %s -> %s" % ( diff --git a/channels/tests/__init__.py b/channels/tests/__init__.py index 566d872..a43624d 100644 --- a/channels/tests/__init__.py +++ b/channels/tests/__init__.py @@ -1 +1 @@ -from .base import ChannelTestCase +from .base import ChannelTestCase # NOQA isort:skip diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index 52712a4..a59a2b8 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -212,8 +212,10 @@ class RequestTests(ChannelTestCase): "content": b"re=fou", "more_content": True, }) + class VeryImpatientRequest(AsgiRequest): body_receive_timeout = 0 + with self.assertRaises(RequestTimeout): VeryImpatientRequest(self.get_next_message("test")) diff --git a/patchinator.py b/patchinator.py index dc283ea..97de4a9 100644 --- a/patchinator.py +++ b/patchinator.py @@ -12,7 +12,7 @@ import os.path import sys -### Transforms: Turn one content string into another ### +# Transforms: Turn one content string into another class Replacement(object): """ @@ -45,10 +45,10 @@ class Insert(object): if self.after: return value[:match.end()] + self.to_insert + value[match.end():] else: - return value[:match.start()] + self.to_insert + value[match.start():] + return value[:match.start()] + self.to_insert + value[match.start():] -### Operations: Copy or patch files ### +# Operations: Copy or patch files class FileMap(object): """ @@ -98,7 +98,7 @@ class NewFile(object): fh.write(self.content) -### Main class ### +# Main class and config global_transforms = [ @@ -154,7 +154,9 @@ class Patchinator(object): "channels/worker.py", "django/channels/worker.py", global_transforms, ), FileMap( - "channels/management/commands/runworker.py", "django/core/management/commands/runworker.py", global_transforms, + "channels/management/commands/runworker.py", + "django/core/management/commands/runworker.py", + global_transforms, ), # Tests FileMap( From 28e897d9bc36bcc78dd3cbdb200c36279a7669ce Mon Sep 17 00:00:00 2001 From: David Evans Date: Mon, 4 Apr 2016 21:36:18 +0100 Subject: [PATCH 310/746] Remove lazy loading of middleware This mirrors the equivalent change in Django itsef. See: https://code.djangoproject.com/ticket/26452 https://github.com/django/django/commit/99bb7fcc1859615a7b8c2468e7b97d54853bfb10 --- channels/handler.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index b9664f3..69ed8a3 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -7,7 +7,6 @@ import sys import time import traceback from io import BytesIO -from threading import Lock from django import http from django.conf import settings @@ -168,20 +167,16 @@ class AsgiHandler(base.BaseHandler): a HTTP request) """ - initLock = Lock() request_class = AsgiRequest # Size to chunk response bodies into for multiple response messages chunk_size = 512 * 1024 + def __init__(self, *args, **kwargs): + super(AsgiHandler, self).__init__(*args, **kwargs) + self.load_middleware() + def __call__(self, message): - # Set up middleware if needed. We couldn't do this earlier, because - # settings weren't available. - if self._request_middleware is None: - with self.initLock: - # Check that middleware is still uninitialized. - if self._request_middleware is None: - self.load_middleware() # Set script prefix from message root_path set_script_prefix(message.get('root_path', '')) signals.request_started.send(sender=self.__class__, message=message) From cf52c922e0a18309648efac637406564a060472e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 08:09:03 -0700 Subject: [PATCH 311/746] Update getting started to not pass messages back directly --- docs/getting-started.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 427e579..36eccc5 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -358,7 +358,9 @@ name in the path of your WebSocket request (we'll ignore auth for now - that's n # Connected to websocket.receive @channel_session def ws_message(message): - Group("chat-%s" % message.channel_session['room']).send(message.content) + Group("chat-%s" % message.channel_session['room']).send({ + "text": message['text'], + }) # Connected to websocket.disconnect @channel_session @@ -452,7 +454,9 @@ chat to people with the same first letter of their username:: # Connected to websocket.receive @channel_session_user def ws_message(message): - Group("chat-%s" % message.user.username[0]).send(message.content) + Group("chat-%s" % message.user.username[0]).send({ + "text": message['text'], + }) # Connected to websocket.disconnect @channel_session_user @@ -656,7 +660,9 @@ first-letter-of-username chat from earlier:: @enforce_ordering(slight=True) @channel_session_user def ws_message(message): - Group("chat-%s" % message.user.username[0]).send(message.content) + Group("chat-%s" % message.user.username[0]).send({ + "text": message['text'], + }) # Connected to websocket.disconnect @enforce_ordering(slight=True) From 2ef0baa5c2e78dcc6c7c1ae4738a977a53fd837f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 08:21:54 -0700 Subject: [PATCH 312/746] Release 0.11.0 --- CHANGELOG.txt | 10 ++++++++++ channels/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 407123f..1c8c444 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,13 @@ +0.11.0 (2016-04-05) +------------------- + +* ChannelTestCase base testing class for easier testing of consumers + +* Routing rewrite to improve speed with nested includes and remove need for ^ operator + +* Timeouts reading very slow request bodies + + 0.10.3 (2016-03-29) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index f62749e..e9ae017 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.10.3" +__version__ = "0.11.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 2eb8e9277ebacaca2dbb67756b53e02a96a582a5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 13:48:29 -0700 Subject: [PATCH 313/746] Fixed #119: Talked about IRC channel --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 3a2bd84..573765c 100644 --- a/README.rst +++ b/README.rst @@ -30,5 +30,8 @@ a bit as things develop. Documentation, installation and getting started instructions are at http://channels.readthedocs.org +Support can be obtained either here via issues, or in the ``#django-channels`` +channel on Freenode. + You can also install channels from PyPI as the ``channels`` package. You'll likely also want ``asgi_redis`` to provide the Redis channel layer. From d29f02fb3319825bc55ee6e047f24841a8d0a015 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 16:20:49 -0700 Subject: [PATCH 314/746] Patchinator/django fixes --- channels/handler.py | 2 +- channels/tests/test_database_layer.py | 2 +- channels/tests/test_routing.py | 1 + patchinator.py | 16 +++++++++++----- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 69ed8a3..6af7acb 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -17,7 +17,7 @@ from django.http import FileResponse, HttpResponse, HttpResponseServerError from django.utils import six from django.utils.functional import cached_property -from .exceptions import ResponseLater as ResponseLaterOuter, RequestTimeout, RequestAborted +from channels.exceptions import ResponseLater as ResponseLaterOuter, RequestTimeout, RequestAborted logger = logging.getLogger('django.request') diff --git a/channels/tests/test_database_layer.py b/channels/tests/test_database_layer.py index 4878157..b9a11c0 100644 --- a/channels/tests/test_database_layer.py +++ b/channels/tests/test_database_layer.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals from asgiref.conformance import ConformanceTestCase -from ..database_layer import DatabaseChannelLayer +from channels.database_layer import DatabaseChannelLayer class DatabaseLayerTests(ConformanceTestCase): diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py index bfc44cd..891113f 100644 --- a/channels/tests/test_routing.py +++ b/channels/tests/test_routing.py @@ -80,6 +80,7 @@ class RoutingTests(SimpleTestCase): Ensures the test consumers don't compare equal, as if this ever happens this test file will pass and miss most bugs. """ + self.assertEqual(consumer_1, consumer_1) self.assertNotEqual(consumer_1, consumer_2) self.assertNotEqual(consumer_1, consumer_3) diff --git a/patchinator.py b/patchinator.py index 97de4a9..782497d 100644 --- a/patchinator.py +++ b/patchinator.py @@ -105,14 +105,17 @@ global_transforms = [ Replacement(r"import channels.([a-zA-Z0-9_\.]+)$", r"import django.channels.\1 as channels"), Replacement(r"from channels import", r"from django.channels import"), Replacement(r"from channels.([a-zA-Z0-9_\.]+) import", r"from django.channels.\1 import"), - Replacement(r"from .handler import", r"from django.core.handlers.asgi import") + Replacement(r"from .handler import", r"from django.core.handlers.asgi import"), + Replacement(r"from django.channels.tests import", r"from django.test.channels import"), + Replacement(r"from django.channels.handler import", r"from django.core.handlers.asgi import"), + Replacement(r"channels.tests.test_routing", r"channels_tests.test_routing") ] docs_transforms = global_transforms + [ - Replacement(r":doc:`concepts`", r":doc:`topics/channels/concepts`"), - Replacement(r":doc:`deploying`", r":doc:`topics/channels/deploying`"), - Replacement(r":doc:`scaling`", r":doc:`topics/channels/scaling`"), - Replacement(r":doc:`getting-started`", r":doc:`intro/channels`"), + Replacement(r":doc:`concepts`", r":doc:`/topics/channels/concepts`"), + Replacement(r":doc:`deploying`", r":doc:`/topics/channels/deploying`"), + Replacement(r":doc:`scaling`", r":doc:`/topics/channels/scaling`"), + Replacement(r":doc:`getting-started`", r":doc:`/intro/channels`"), Replacement(r"\n\(.*installation>`\)\n", r""), ] @@ -141,6 +144,9 @@ class Patchinator(object): FileMap( "channels/routing.py", "django/channels/routing.py", global_transforms, ), + FileMap( + "channels/message.py", "django/channels/message.py", global_transforms, + ), FileMap( "channels/sessions.py", "django/channels/sessions.py", global_transforms, ), From 4bab456c6148f9ea9b909f9e875d81ec49552f47 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 16:44:26 -0700 Subject: [PATCH 315/746] Fail to work if you have channel-enabled Django --- channels/apps.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/channels/apps.py b/channels/apps.py index 389fc99..88ab8e7 100644 --- a/channels/apps.py +++ b/channels/apps.py @@ -1,4 +1,5 @@ from django.apps import AppConfig +from django.core.exceptions import ImproperlyConfigured class ChannelsConfig(AppConfig): @@ -7,6 +8,13 @@ class ChannelsConfig(AppConfig): verbose_name = "Channels" def ready(self): + # Check you're not running 1.10 or above + try: + from django import channels + except ImportError: + pass + else: + raise ImproperlyConfigured("You have Django 1.10 or above; use the builtin django.channels!") # Do django monkeypatches from .hacks import monkeypatch_django monkeypatch_django() From 4437e04528e77bcfac9d13a807c6aa062feec3c8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 17:19:23 -0700 Subject: [PATCH 316/746] Fix circular import issue --- channels/routing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/channels/routing.py b/channels/routing.py index c311658..afa2946 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -6,7 +6,6 @@ import importlib from django.core.exceptions import ImproperlyConfigured from django.utils import six -from .handler import ViewConsumer from .utils import name_that_thing @@ -47,6 +46,9 @@ class Router(object): """ # We just add the default Django route to the bottom; if the user # has defined another http.request handler, it'll get hit first and run. + # Inner import here to avoid circular import; this function only gets + # called once, thankfully. + from .handler import ViewConsumer self.add_route(Route("http.request", http_consumer or ViewConsumer())) @classmethod From 352407e54b223496e4247a7fad5888087f3c4d9a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 17:27:36 -0700 Subject: [PATCH 317/746] Improve patchinator, lots of doc stuff --- patchinator.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/patchinator.py b/patchinator.py index 782497d..fd41524 100644 --- a/patchinator.py +++ b/patchinator.py @@ -112,11 +112,16 @@ global_transforms = [ ] docs_transforms = global_transforms + [ + Replacement(r"`", r"`"), Replacement(r":doc:`concepts`", r":doc:`/topics/channels/concepts`"), Replacement(r":doc:`deploying`", r":doc:`/topics/channels/deploying`"), Replacement(r":doc:`scaling`", r":doc:`/topics/channels/scaling`"), Replacement(r":doc:`getting-started`", r":doc:`/intro/channels`"), + Replacement(r"`", r"`"), + Replacement(r":doc:`backends`", r":doc:`/ref/channels/backends`"), + Replacement(r":doc:`([\w\d\s]+) `", r"`\1 `_"), Replacement(r"\n\(.*installation>`\)\n", r""), + Replacement(r":doc:`installed Channels correctly `", r"added the channel layer setting"), ] @@ -151,7 +156,7 @@ class Patchinator(object): "channels/sessions.py", "django/channels/sessions.py", global_transforms, ), FileMap( - "channels/staticfiles.py", "django/channels/staticfiles.py", global_transforms, + "channels/staticfiles.py", "django/contrib/staticfiles/consumers.py", global_transforms, ), FileMap( "channels/utils.py", "django/channels/utils.py", global_transforms, From 11218089bd2739b26e6b22cbb54117e27cba76ac Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 17:30:47 -0700 Subject: [PATCH 318/746] Fix flake error --- channels/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/apps.py b/channels/apps.py index 88ab8e7..fec19cd 100644 --- a/channels/apps.py +++ b/channels/apps.py @@ -10,7 +10,7 @@ class ChannelsConfig(AppConfig): def ready(self): # Check you're not running 1.10 or above try: - from django import channels + from django import channels # NOQA isort:skip except ImportError: pass else: From 732167282bf45160cc25ce15926bb8a80bfe845c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 17:38:05 -0700 Subject: [PATCH 319/746] Some import sorting stuff --- channels/channel.py | 1 + channels/management/commands/runworker.py | 3 +- patchinator.py | 53 +++++++++++++++-------- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/channels/channel.py b/channels/channel.py index 465e267..c22c940 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from django.utils import six + from channels import DEFAULT_CHANNEL_LAYER, channel_layers diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 069e6ec..0cf4498 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -1,7 +1,8 @@ from __future__ import unicode_literals from django.core.management import BaseCommand, CommandError -from channels import channel_layers, DEFAULT_CHANNEL_LAYER + +from channels import DEFAULT_CHANNEL_LAYER, channel_layers from channels.log import setup_logger from channels.worker import Worker diff --git a/patchinator.py b/patchinator.py index fd41524..07b74c1 100644 --- a/patchinator.py +++ b/patchinator.py @@ -11,6 +11,8 @@ import re import os.path import sys +from isort import SortImports + # Transforms: Turn one content string into another @@ -48,6 +50,15 @@ class Insert(object): return value[:match.start()] + self.to_insert + value[match.start():] +class Isort(object): + """ + Runs isort on the file + """ + + def __call__(self, value): + return SortImports(file_contents=value).output + + # Operations: Copy or patch files class FileMap(object): @@ -108,7 +119,11 @@ global_transforms = [ Replacement(r"from .handler import", r"from django.core.handlers.asgi import"), Replacement(r"from django.channels.tests import", r"from django.test.channels import"), Replacement(r"from django.channels.handler import", r"from django.core.handlers.asgi import"), - Replacement(r"channels.tests.test_routing", r"channels_tests.test_routing") + Replacement(r"channels.tests.test_routing", r"channels_tests.test_routing"), +] + +python_transforms = global_transforms + [ + Isort(), ] docs_transforms = global_transforms + [ @@ -129,64 +144,64 @@ class Patchinator(object): operations = [ FileMap( - "channels/asgi.py", "django/channels/asgi.py", global_transforms, + "channels/asgi.py", "django/channels/asgi.py", python_transforms, ), FileMap( - "channels/auth.py", "django/channels/auth.py", global_transforms, + "channels/auth.py", "django/channels/auth.py", python_transforms, ), FileMap( - "channels/channel.py", "django/channels/channel.py", global_transforms, + "channels/channel.py", "django/channels/channel.py", python_transforms, ), FileMap( - "channels/database_layer.py", "django/channels/database_layer.py", global_transforms, + "channels/database_layer.py", "django/channels/database_layer.py", python_transforms, ), FileMap( - "channels/exceptions.py", "django/channels/exceptions.py", global_transforms, + "channels/exceptions.py", "django/channels/exceptions.py", python_transforms, ), FileMap( - "channels/handler.py", "django/core/handlers/asgi.py", global_transforms, + "channels/handler.py", "django/core/handlers/asgi.py", python_transforms, ), FileMap( - "channels/routing.py", "django/channels/routing.py", global_transforms, + "channels/routing.py", "django/channels/routing.py", python_transforms, ), FileMap( - "channels/message.py", "django/channels/message.py", global_transforms, + "channels/message.py", "django/channels/message.py", python_transforms, ), FileMap( - "channels/sessions.py", "django/channels/sessions.py", global_transforms, + "channels/sessions.py", "django/channels/sessions.py", python_transforms, ), FileMap( - "channels/staticfiles.py", "django/contrib/staticfiles/consumers.py", global_transforms, + "channels/staticfiles.py", "django/contrib/staticfiles/consumers.py", python_transforms, ), FileMap( - "channels/utils.py", "django/channels/utils.py", global_transforms, + "channels/utils.py", "django/channels/utils.py", python_transforms, ), FileMap( - "channels/worker.py", "django/channels/worker.py", global_transforms, + "channels/worker.py", "django/channels/worker.py", python_transforms, ), FileMap( "channels/management/commands/runworker.py", "django/core/management/commands/runworker.py", - global_transforms, + python_transforms, ), # Tests FileMap( - "channels/tests/base.py", "django/test/channels.py", global_transforms, + "channels/tests/base.py", "django/test/channels.py", python_transforms, ), NewFile( "tests/channels_tests/__init__.py", ), FileMap( - "channels/tests/test_database_layer.py", "tests/channels_tests/test_database_layer.py", global_transforms, + "channels/tests/test_database_layer.py", "tests/channels_tests/test_database_layer.py", python_transforms, ), FileMap( - "channels/tests/test_handler.py", "tests/channels_tests/test_handler.py", global_transforms, + "channels/tests/test_handler.py", "tests/channels_tests/test_handler.py", python_transforms, ), FileMap( - "channels/tests/test_routing.py", "tests/channels_tests/test_routing.py", global_transforms, + "channels/tests/test_routing.py", "tests/channels_tests/test_routing.py", python_transforms, ), FileMap( - "channels/tests/test_request.py", "tests/channels_tests/test_request.py", global_transforms, + "channels/tests/test_request.py", "tests/channels_tests/test_request.py", python_transforms, ), # Docs FileMap( From 4504eb6ec975f7bdc9fe68819ff24b4e2ceddd02 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 22:18:54 -0700 Subject: [PATCH 320/746] Add select_for_update() to improve isolation --- channels/database_layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index 4f23142..b9d2b31 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -60,7 +60,7 @@ class DatabaseChannelLayer(object): while True: try: with transaction.atomic(): - message = self.channel_model.objects.filter(channel__in=channels).order_by("id").first() + message = self.channel_model.objects.select_for_update().filter(channel__in=channels).order_by("id").first() if message: self.channel_model.objects.filter(pk=message.pk).delete() return message.channel, self.deserialize(message.content) From 6006a3181a376f3ca0827ed8fdc7996a5780f212 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 22:23:05 -0700 Subject: [PATCH 321/746] Fix version specific check in patchinator --- patchinator.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/patchinator.py b/patchinator.py index 07b74c1..04b500e 100644 --- a/patchinator.py +++ b/patchinator.py @@ -22,12 +22,16 @@ class Replacement(object): substitute strings in the file. """ - def __init__(self, match, sub): + def __init__(self, match, sub, regex=True): self.match = match self.sub = sub + self.regex = regex def __call__(self, value): - return re.sub(self.match, self.sub, value) + if self.regex: + return re.sub(self.match, self.sub, value) + else: + return value.replace(self.match, self.sub) class Insert(object): @@ -144,7 +148,9 @@ class Patchinator(object): operations = [ FileMap( - "channels/asgi.py", "django/channels/asgi.py", python_transforms, + "channels/asgi.py", "django/channels/asgi.py", python_transforms + [ + Replacement("if django.VERSION[1] > 9:\n django.setup(set_prefix=False)\n else:\n django.setup()", "django.setup(set_prefix=False)", regex=False), # NOQA + ], ), FileMap( "channels/auth.py", "django/channels/auth.py", python_transforms, From cd9c049296996fb7b6951fb76c3cc190e82e118a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 22:25:05 -0700 Subject: [PATCH 322/746] Rename database layer models to be consistent --- channels/database_layer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index b9d2b31..d1ec122 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -149,7 +149,7 @@ class DatabaseChannelLayer(object): class Meta: apps = Apps() app_label = "channels" - db_table = "django_channels" + db_table = "django_channels_channel" # Ensure its table exists if Message._meta.db_table not in self.connection.introspection.table_names(self.connection.cursor()): with self.connection.schema_editor() as editor: @@ -171,7 +171,7 @@ class DatabaseChannelLayer(object): class Meta: apps = Apps() app_label = "channels" - db_table = "django_channel_groups" + db_table = "django_channels_group" unique_together = [["group", "channel"]] # Ensure its table exists with the right schema if Group._meta.db_table not in self.connection.introspection.table_names(self.connection.cursor()): From 70637b7afe2950a8fe7464daacec50ad3163b1d3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 5 Apr 2016 22:29:00 -0700 Subject: [PATCH 323/746] Wrap line properly --- channels/database_layer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index d1ec122..37894a8 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -60,7 +60,9 @@ class DatabaseChannelLayer(object): while True: try: with transaction.atomic(): - message = self.channel_model.objects.select_for_update().filter(channel__in=channels).order_by("id").first() + message = self.channel_model.objects.select_for_update().filter( + channel__in=channels + ).order_by("id").first() if message: self.channel_model.objects.filter(pk=message.pk).delete() return message.channel, self.deserialize(message.content) From e88e0feae9dedbfecd6f6382e65a5ba2d75fe02a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20M=C3=BCllegger?= Date: Wed, 6 Apr 2016 17:10:01 +0200 Subject: [PATCH 324/746] Fix typo in path to DatabaseChannelLayer Removed `g` from `datagbase_layer`. --- docs/backends.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backends.rst b/docs/backends.rst index b76f51c..3935058 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -64,7 +64,7 @@ that will work cross-process. It has poor performance, and is only recommended for development or extremely small deployments. This layer is included with Channels; just set your ``BACKEND`` to -``channels.datagbase_layer.DatabaseChannelLayer``, and it will use the +``channels.database_layer.DatabaseChannelLayer``, and it will use the default Django database alias to store messages. You can change the alias by setting ``CONFIG`` to ``{'alias': 'aliasname'}``. From c171cb43464ae642a542a4ef0c2ef462190a608f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20M=C3=BCllegger?= Date: Wed, 6 Apr 2016 17:10:11 +0200 Subject: [PATCH 325/746] Fix auth example imports in Gettings Started guide Removed unused `transfer_user` import and added missing `channel_session_user_from_http` as import. --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 36eccc5..8eb7561 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -443,7 +443,7 @@ chat to people with the same first letter of their username:: # In consumers.py from channels import Channel, Group from channels.sessions import channel_session - from channels.auth import http_session_user, channel_session_user, transfer_user + from channels.auth import http_session_user, channel_session_user, channel_session_user_from_http # Connected to websocket.connect @channel_session_user_from_http From 609adfca8dbd9776974edec79aa839587d290d73 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 6 Apr 2016 08:11:59 -0700 Subject: [PATCH 326/746] Fix example with custom channel --- docs/getting-started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 8eb7561..b71dc35 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -569,7 +569,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: ) # Broadcast to listening sockets Group("chat-%s" % room).send({ - "content": message.content['message'], + "text": message.content['message'], }) # Connected to websocket.connect @@ -587,7 +587,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: # Stick the message onto the processing queue Channel("chat-messages").send({ "room": channel_session['room'], - "message": content, + "message": message['text'], }) # Connected to websocket.disconnect From 8fdf2685745497d5996cbab5eaf4c5e562e1eed3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 6 Apr 2016 11:06:02 -0700 Subject: [PATCH 327/746] Add testing documentation --- docs/index.rst | 1 + docs/testing.rst | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ patchinator.py | 3 ++ 3 files changed, 78 insertions(+) create mode 100644 docs/testing.rst diff --git a/docs/index.rst b/docs/index.rst index 9a74541..9e36be1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Contents: integration-changes scaling backends + testing reference integration-plan faqs diff --git a/docs/testing.rst b/docs/testing.rst new file mode 100644 index 0000000..55476ba --- /dev/null +++ b/docs/testing.rst @@ -0,0 +1,74 @@ +Testing Consumers +================= + +When you want to write unit tests for your new Channels consumers, you'll +realise that you can't use the standard Django test client to submit fake HTTP +requests - instead, you'll need to submit fake Messages to your consumers, +and inspect what Messages they send themselves. + +Channels comes with a ``TestCase`` subclass that sets all of this up for you, +however, so you can easily write tests and check what your consumers are sending. + + +ChannelTestCase +--------------- + +If your tests inherit from the ``channels.tests.ChannelTestCase`` base class, +whenever you run tests your channel layer will be swapped out for a captive +in-memory layer, meaning you don't need an exernal server running to run tests. + +Moreover, you can inject messages onto this layer and inspect ones sent to it +to help test your consumers. + +To inject a message onto the layer, simply call ``Channel.send()`` inside +any test method on a ``ChannelTestCase`` subclass, like so:: + + from channels import Channel + from channels.tests import ChannelTestCase + + class MyTests(ChannelTestCase): + def test_a_thing(self): + # This goes onto an in-memory channel, not the real backend. + Channel("some-channel-name").send({"foo": "bar"}) + +To receive a message from the layer, you can use ``self.get_next_message(channel)``, +which handles receiving the message and converting it into a Message object for +you (if you want, you can call ``receive_many`` on the underlying channel layer, +but you'll get back a raw dict and channel name, which is not what consumers want). + +You can use this both to get Messages to send to consumers as their primary +argument, as well as to get Messages from channels that consumers are supposed +to send on to verify that they did. + +You can even pass ``require=True`` to ``get_next_message`` to make the test +fail if there is no message on the channel (by default, it will return you +``None`` instead). + +Here's an extended example testing a consumer that's supposed to take a value +and post the square of it to the ``"result"`` channel:: + + + from channels import Channel + from channels.tests import ChannelTestCase + + class MyTests(ChannelTestCase): + def test_a_thing(self): + # Inject a message onto the channel to use in a consumer + Channel("input").send({"value": 33}) + # Run the consumer with the new Message object + my_consumer(self.get_next_message("input", require=True)) + # Verify there's a result and that it's accurate + result = self.get_next_message("result", require=True) + self.assertEqual(result['value'], 1089) + + +Multiple Channel Layers +----------------------- + +If you want to test code that uses multiple channel layers, specify the alias +of the layers you want to mock as the ``test_channel_aliases`` attribute on +the ``ChannelTestCase`` subclass; by default, only the ``default`` layer is +mocked. + +You can pass an ``alias`` argument to ``get_next_message`` and ``Channel`` +to use a different layer too. diff --git a/patchinator.py b/patchinator.py index 04b500e..df99705 100644 --- a/patchinator.py +++ b/patchinator.py @@ -228,6 +228,9 @@ class Patchinator(object): FileMap( "docs/scaling.rst", "docs/topics/channels/scaling.txt", docs_transforms, ), + FileMap( + "docs/testing.rst", "docs/topics/channels/testing.txt", docs_transforms, + ), ] def __init__(self, source, destination): From 8abe53e170a73ae6bc31c71c065635a6e7b2e08c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 6 Apr 2016 22:02:41 -0700 Subject: [PATCH 328/746] Change type of error --- channels/channel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/channel.py b/channels/channel.py index c22c940..b65d65a 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -34,7 +34,7 @@ class Channel(object): Send a message over the channel - messages are always dicts. """ if not isinstance(content, dict): - raise ValueError("You can only send dicts as content on channels.") + raise TypeError("You can only send dicts as content on channels.") self.channel_layer.send(self.name, content) def __str__(self): From f974f13a376659b270e5acd96d4fb3cd63c3b205 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 6 Apr 2016 22:05:28 -0700 Subject: [PATCH 329/746] Rearrange docs, add cross-compat doc --- docs/cross-compat.rst | 44 +++++++++++++++++++ docs/index.rst | 3 +- docs/integration-changes.rst | 18 -------- docs/integration-plan.rst | 83 ------------------------------------ 4 files changed, 45 insertions(+), 103 deletions(-) create mode 100644 docs/cross-compat.rst delete mode 100644 docs/integration-changes.rst delete mode 100644 docs/integration-plan.rst diff --git a/docs/cross-compat.rst b/docs/cross-compat.rst new file mode 100644 index 0000000..f9b2f59 --- /dev/null +++ b/docs/cross-compat.rst @@ -0,0 +1,44 @@ +Cross-Compatability +=================== + +Channels is being released as both a third-party app for Django 1.8 and 1.9, +and being integrated into Django in 1.10. Both of these implementations are +very similar, and code for one will work on the other with minimal changes. + +The only difference between the two is the import paths. Mostly, where you +imported from ``channels`` for the third-party app, you instead import from +``django.channels`` for the built-in solution. + +For example:: + + from channels import Channel + from channels.auth import channel_session_user + +Becomes:: + + from django.channels import Channel + from django.channels.auth import channel_session_user + +There are a few exceptions to this rule, where classes were moved to other parts +of Django in 1.10 that made more sense: + +* ``channels.tests.ChannelTestCase`` is found under ``django.test.channels.ChannelTestCase`` +* ``channels.handler`` is moved to ``django.core.handlers.asgi`` +* ``channels.staticfiles`` is moved to ``django.contrib.staticfiles.consumers`` +* The ``runserver`` and ``runworker`` commands are in ``django.core.management.commands`` + + +Writing third-party apps against Channels +----------------------------------------- + +If you're writing a third-party app that is designed to work with both the +``channels`` third-party app as well as ``django.channels``, we suggest you use +a try-except pattern for imports, like this:: + + try: + from django.channels import Channel + except ImportError: + from channels import Channel + +All the objects in both versions act the same way, they simply are located +on different import paths. There should be no need to change logic. diff --git a/docs/index.rst b/docs/index.rst index 9e36be1..4ad112e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,11 +29,10 @@ Contents: installation getting-started deploying - integration-changes scaling backends testing + cross-compat reference - integration-plan faqs asgi diff --git a/docs/integration-changes.rst b/docs/integration-changes.rst deleted file mode 100644 index 7fa5c6e..0000000 --- a/docs/integration-changes.rst +++ /dev/null @@ -1,18 +0,0 @@ -Integration Notes -================= - -Django Channels is intended to be merged into Django itself; these are the -planned changes the codebase will need to undertake in that transition. - -* The ``channels`` package will become ``django.channels``, and main objects will keep their import path. - -* Obviously, the monkeypatches in ``channels.hacks`` will be replaced by - placing methods onto the objects themselves. The ``request`` and ``response`` - modules will thus no longer exist separately. - -Things to ponder ----------------- - -* The mismatch between signals (broadcast) and channels (single-worker) means - we should probably leave patching signals into channels for the end developer. - This would also ensure the speedup improvements for empty signals keep working. diff --git a/docs/integration-plan.rst b/docs/integration-plan.rst deleted file mode 100644 index 4fd1a7a..0000000 --- a/docs/integration-plan.rst +++ /dev/null @@ -1,83 +0,0 @@ -Integration Plan -================ - -*DRAFT VERSION - NOT FINAL* - -Channels is to become a built-in feature for Django 1.10, but in order to aid adoption and to encourage community usage and support, it will also be backported to Django 1.8 and 1.9 via a third-party app. - -Obviously, we want to do this with as little code duplication as is sensible, so this document outlines the plan of how Channels will be integrated into Django and how it will be importable and usable from the supported versions. - -Separate Components -------------------- - -The major step in code reuse will be developing the main interface server as a separate project still released under the Django umbrella (current codename is ``daphne``). - -This piece of software will likely have its own release cycle but still be developed and maintained by Django as part of the core project, and will speak HTTP, WebSockets, and likely HTTP2 natively. It will be designed to be used either directly exposed to the Internet or behind a reverse proxy that serves static files, much like gunicorn or uwsgi would be deployed. - -This would then be supplemented by two implementations of the "worker" end of the Channels stack - one native in Django 1.10, and one as a third-party app for earlier Django versions. They would act very similarly, and there will be some code duplication between them, but the 1.10 version should be cleaner and faster as there's no need to re-route around the existing Django internals (though it can be done, as the current third-party app approach shows). - -All three components would need to share the channel backend implementations - there is still an unanswered question here about if those should be separate packages, or somehow bundled into the implementations themselves. - -Preserving Simplicity ---------------------- - -A main goal of the channels project is to keep the ability to just download and use Django as simple as it is now, which means that for every external dependency we introduce, there needs to be a way to work round it to just run Django in "classic" mode if the user wants to. - -To this end, Django will still be deployable with a WSGI server as it is now, with all channels-dependent features disabled, and when it ships, will fall back to a standard WSGI `runserver` if the dependencies to run a more complex interface server aren't installed (but if they are, `runserver` will become websocket-aware). - -There will also be options to plug Channels into a WSGI server as a WSGI application that then forwards into a channel backend, if users wish to keep using familiar webservers but still run a worker cluster and gain things like background tasks (or WebSockets using a second server process) - -Standardization ---------------- - -Given the intention to develop this as two codebases (though Django will still mean the combination of both), it will likely be sensible if not necessary to standardise the way the two talk to each other. - -There are two levels to this standardisation - firstly, the message formats for how to encode different events, requests and responses (which you can see the initial version of over in :doc:`message-standards`), and secondly, the actual transports themselves, the channel backends. - -Message format standardisation shouldn't be too hard; the top-level constraint on messages will be that they must be JSON-serializable dicts with no size limit; from there, developing sensible mappings for HTTP and WebSocket is not too hard, especially drawing on the WSGI spec for the former (indeed, Channels will have to be able to reconstitude WSGI-style properties and variables to let existing view code continue to work) - -Transport standardisation is different, though; the current approach is to have a standard backend interface that either core backends or third-party ones can implement, much like Django's database support; this would seem to fill the immediate need of both having core, tested and scalable transports as well as the option for more complex projects with special requirements to write their own. - -That said, the current proposed system is just one transport standard away from being able to interoperate with other languages; specifically, one can imagine an interface server written in a compiled language like Go or Rust that is more efficient than its Python equivalent (or even a Python implementation that uses a concurrency model that doesn't fit the channel backend's poll-style interface). - -It may be that the Redis backend itself is written up and standardised as well to provide a clear spec that third parties can code against from scratch; however, this will need to be after a period of iterative testing to ensure whatever is proposed can scale to handle both large messages and large volumes of messages, and can shard horizontally. - -There is no intention to propose a WSGI replacement at this point, but there's potential to do that at the fundamental Python level of "your code will get given messages, and can send them, and here's the message formats". However, such an effort should not be taken lightly and only attempted if and when channels proves itself to work as effectively as WSGI as the abstraction for handling web requests, and its servers are as stable and mature as those for WSGI; Django will continue to support WSGI for the foreseeable future. - -Imports -------- - -Having Channels as a third-party package on some versions of Django and as a core package on others presents an issue for anyone trying to write portable code; the import path will differ. - -Specifically, for the third party app you might import from ``channels``, whereas for the native one you might import from ``django.channels``. - -There are two sensible solutions to this: - -1. Make both versions available under the import path ``django.channels`` -2. Make people do try/except imports for portable code - -Neither are perfect; the first likely means some nasty monkeypatching or more, especially on Python 2.7, while the second involves verbose code. More research is needed here. - -WSGI attribute access ---------------------- - -While Channels transports across a lot of the data available on a WSGI request - like paths, remote client IPs, POST data and so forth - it cannot reproduce everything. Specifically, the following changes will happen to Django request objects when they come via Channels: - -- ``request.META`` will not contain any environment variables from the system. -- The ``wsgi`` keys in ``request.META`` will change as follows: - - ``wsgi.version`` will be set to ``(1, 0)`` - - ``wsgi.url_scheme`` will be populated correctly - - ``wsgi.input`` will work, but point to a StringIO or File object with the entire request body buffered already - - ``wsgi.errors`` will point to ``sys.stderr`` - - ``wsgi.multithread`` will always be ``True`` - - ``wsgi.multiprocess`` will always be ``True`` - - ``wsgi.run_once`` will always be ``False`` - -This is a best-attempt effort to mirror these variables' meanings to allow code to keep working on multiple Django versions; we will encourage people using ``url_scheme`` to switch to ``request.is_secure``, however, and people using ``input`` to use the ``read()`` or ``body`` attributes on ``request``. - -All of the ``wsgi`` variable emulation will be subject to the usual Django deprecation cycle and after this will not be available unless Django is running in a WSGI environment. - -Running Workers ---------------- - -It is not intended for there to be a separate command to run a Django worker; instead, ``manage.py runworker`` will be the recommended method, along with a wrapping process manager that handles logging and auto-restart (such as systemd or supervisord). From 80206e5452ddcb9dfb8eba177176da05c0059c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Kut=C3=BD?= <6du1ro.n@gmail.com> Date: Fri, 8 Apr 2016 20:04:55 +0200 Subject: [PATCH 330/746] Fix loading files. #123 --- channels/handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/handler.py b/channels/handler.py index 6af7acb..56fd29d 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -149,6 +149,7 @@ class AsgiRequest(http.HttpRequest): def _get_files(self): if not hasattr(self, '_files'): + self._read_started = False self._load_post_and_files() return self._files From 2c39e42fa21cbe868c90544648e58474b0ba81a4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 26 Apr 2016 13:07:10 +0100 Subject: [PATCH 331/746] Add patchinator config for more docs --- patchinator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/patchinator.py b/patchinator.py index df99705..f86e07c 100644 --- a/patchinator.py +++ b/patchinator.py @@ -231,6 +231,9 @@ class Patchinator(object): FileMap( "docs/testing.rst", "docs/topics/channels/testing.txt", docs_transforms, ), + FileMap( + "docs/cross-compat.rst", "docs/topics/channels/cross-compat.txt", docs_transforms, + ), ] def __init__(self, source, destination): From 7945859bb2f21dd411eccbfc799799dfe7284bc9 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 26 Apr 2016 13:07:23 +0100 Subject: [PATCH 332/746] Update ASGI HTTP spec to make all path parts unicode --- docs/asgi.rst | 83 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 18ebcfd..4aef65c 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -475,12 +475,13 @@ Keys: * ``scheme``: Unicode string URL scheme portion (likely ``http`` or ``https``). Optional (but must not be empty), default is ``"http"``. -* ``path``: Byte string HTTP path from URL. +* ``path``: Unicode string HTTP path from URL, with percent escapes decoded + and UTF8 byte sequences decoded into characters. -* ``query_string``: Byte string URL portion after the ``?``. Optional, default - is ``""``. +* ``query_string``: Unicode string URL portion after the ``?``, already + url-decoded, like ``path``. Optional, default is ``""``. -* ``root_path``: Byte string that indicates the root path this application +* ``root_path``: Unicode string that indicates the root path this application is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults to ``""``. @@ -741,6 +742,80 @@ A maximum of one of ``bytes`` or ``text`` may be provided. If both are provided, the protocol server should ignore the message entirely. +Email +----- + +Represents emails sent or received, likely over the SMTP protocol though that +is not directly specified here (a protocol server could in theory deliver +or receive email over HTTP to some external service, for example). Generally +adheres to RFC 5322 as much as possible. + +As emails have no concept of a session and there's no trustable socket or +author model, the send and receive channels are both multi-listener, and +there is no ``reply_channel`` on any message type. If you want to persist +data across different email receive consumers, you should decide what part +of the message to use for an identifier (from address? to address? subject? +thread id?) and provide the persistence yourself. + +The protocol server should handle encoding of headers by itself, understanding +RFC 1342 format headers and decoding them into unicode upon receive, and +encoding outgoing emails similarly (preferably using UTF-8). + + +Receive +''''''' + +Sent when an email is received. + +Channel: ``email.receive`` + +Keys: + +* ``from``: Unicode string specifying the return-path of the email as specified + in the SMTP envelope. Will be ``None`` if no return path was provided. + +* ``to``: List of unicode strings specifying the recipients requested in the + SMTP envelope using ``RCPT TO`` commands. Will always contain at least one + value. + +* ``headers``: Dictionary of unicode string keys and unicode string values, + containing all headers, including ``subject``. Header names are all forced + to lower case. Header values are decoded from RFC 1342 if needed. + +* ``content``: Contains a content object (see section below) representing the + body of the message. + +Note that ``from`` and ``to`` are extracted from the SMTP envelope, and not +from the headers inside the message; if you wish to get the header values, +you should use ``headers['from']`` and ``headers['to']``; they may be different. + + +Send +'''' + +Sends an email out via whatever transport + + +Content objects +''''''''''''''' + +Used in both send and receive to represent the tree structure of a MIME +multipart message tree. + +A content object is always a dict, containing at least the key: + +* ``content-type``: The unicode string of the content type for this section. + +Multipart content objects also have: + +* ``parts``: A list of content objects contained inside this multipart + +Any other type of object has: + +* ``body``: Byte string content of this part, decoded from any + ``Content-Transfer-Encoding`` if one was specified as a MIME header. + + UDP --- From e684b27e465d920788e5b02bc4534a3ac50e7ef5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 26 Apr 2016 13:33:26 +0100 Subject: [PATCH 333/746] Switch to uncode path and query string for HTTP --- channels/handler.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 56fd29d..c8a058e 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -41,8 +41,8 @@ class AsgiRequest(http.HttpRequest): self._post_parse_error = False self.resolver_match = None # Path info - self.path = self.message['path'].decode("ascii") - self.script_name = self.message.get('root_path', b'') + self.path = self.message['path'] + self.script_name = self.message.get('root_path', '') if self.script_name: # TODO: Better is-prefix checking, slash handling? self.path_info = self.path[len(self.script_name):] @@ -52,8 +52,8 @@ class AsgiRequest(http.HttpRequest): self.method = self.message['method'].upper() self.META = { "REQUEST_METHOD": self.method, - "QUERY_STRING": self.message.get('query_string', b'').decode("ascii"), - "SCRIPT_NAME": self.script_name.decode("ascii"), + "QUERY_STRING": self.message.get('query_string', ''), + "SCRIPT_NAME": self.script_name, # Old code will need these for a while "wsgi.multithread": True, "wsgi.multiprocess": True, @@ -133,10 +133,7 @@ class AsgiRequest(http.HttpRequest): @cached_property def GET(self): - return http.QueryDict( - self.message.get('query_string', ''), - encoding=self._encoding, - ) + return http.QueryDict(self.message.get('query_string', '').encode("utf8")) def _get_post(self): if not hasattr(self, '_post'): From b374a2a604e210480ed58711dcc98a364458c8be Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 26 Apr 2016 13:51:41 +0100 Subject: [PATCH 334/746] Releasing 0.12.0 --- CHANGELOG.txt | 9 +++++++++ channels/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 1c8c444..a0fb2fb 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,12 @@ +0.12.0 (2016-04-26) +------------------- + +* HTTP paths and query strings are now expected to be sent over ASGI as + unescaped unicode. Daphne 0.11.0 is updated to send things in this format. + +* request.FILES reading bug fixed + + 0.11.0 (2016-04-05) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index e9ae017..80b4b5d 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.11.0" +__version__ = "0.12.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/setup.py b/setup.py index 831cb1c..59bf911 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,6 @@ setup( install_requires=[ 'Django>=1.8', 'asgiref>=0.10', - 'daphne>=0.10', + 'daphne>=0.11', ] ) From 7bc35f18423f5f2b748b95f6ba97e01ea3500e5c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 26 Apr 2016 14:05:17 +0100 Subject: [PATCH 335/746] Fix up tests under py3 --- channels/tests/test_request.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index a59a2b8..41fe1e6 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -21,7 +21,7 @@ class RequestTests(ChannelTestCase): "reply_channel": "test-reply", "http_version": "1.1", "method": "GET", - "path": b"/test/", + "path": "/test/", }) request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.path, "/test/") @@ -45,8 +45,8 @@ class RequestTests(ChannelTestCase): "reply_channel": "test", "http_version": "1.1", "method": "GET", - "path": b"/test2/", - "query_string": b"x=1&y=foo%20bar+baz", + "path": "/test2/", + "query_string": "x=1&y=foo bar+baz", "headers": { "host": b"example.com", "cookie": b"test-time=1448995585123; test-value=yeah", @@ -78,8 +78,8 @@ class RequestTests(ChannelTestCase): "reply_channel": "test", "http_version": "1.1", "method": "POST", - "path": b"/test2/", - "query_string": b"django=great", + "path": "/test2/", + "query_string": "django=great", "body": b"ponies=are+awesome", "headers": { "host": b"example.com", @@ -108,7 +108,7 @@ class RequestTests(ChannelTestCase): "reply_channel": "test", "http_version": "1.1", "method": "POST", - "path": b"/test/", + "path": "/test/", "body": b"there_a", "body_channel": "test-input", "headers": { @@ -148,7 +148,7 @@ class RequestTests(ChannelTestCase): "reply_channel": "test", "http_version": "1.1", "method": "POST", - "path": b"/test/", + "path": "/test/", "body_channel": "test-input", "headers": { "content-type": b"multipart/form-data; boundary=BOUNDARY", @@ -178,7 +178,7 @@ class RequestTests(ChannelTestCase): "reply_channel": "test", "http_version": "1.1", "method": "PUT", - "path": b"/", + "path": "/", "body": b"onetwothree", "headers": { "host": b"example.com", @@ -198,7 +198,7 @@ class RequestTests(ChannelTestCase): "reply_channel": "test", "http_version": "1.1", "method": "POST", - "path": b"/test/", + "path": "/test/", "body": b"there_a", "body_channel": "test-input", "headers": { @@ -227,7 +227,7 @@ class RequestTests(ChannelTestCase): "reply_channel": "test", "http_version": "1.1", "method": "POST", - "path": b"/test/", + "path": "/test/", "body": b"there_a", "body_channel": "test-input", "headers": { From 6b0845ef223bee6c199d0b69d7d09d9349ca0c3d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 27 Apr 2016 16:27:33 +0100 Subject: [PATCH 336/746] Update WSGI URL path to match HTTP --- docs/asgi.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 4aef65c..b31136a 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -652,7 +652,7 @@ Keys: * ``scheme``: Unicode string URL scheme portion (likely ``ws`` or ``wss``). Optional (but must not be empty), default is ``ws``. -* ``path``: Byte string HTTP path from URL. +* ``path``: Unicode HTTP path from URL, already urldecoded. * ``query_string``: Byte string URL portion after the ``?``. Optional, default is empty string. From 681616caa49d064731cfe32fe3634c7a8a0aa427 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Thu, 28 Apr 2016 11:52:49 +0300 Subject: [PATCH 337/746] Fix missed logger name (#138) --- channels/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/log.py b/channels/log.py index 8f1ee54..9078919 100644 --- a/channels/log.py +++ b/channels/log.py @@ -20,7 +20,7 @@ def setup_logger(name, verbosity=1): # Set up daphne protocol loggers for module in ["daphne.ws_protocol", "daphne.http_protocol"]: - daphne_logger = logging.getLogger() + daphne_logger = logging.getLogger(module) daphne_logger.addHandler(handler) daphne_logger.setLevel(logging.DEBUG if verbosity > 1 else logging.INFO) From c579f27f6d588cca2629974ddcb7af461b1226c7 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Thu, 28 Apr 2016 18:13:12 +0200 Subject: [PATCH 338/746] Update to clarify Python Compatibility (#140) Does this make sense? A fellow team member didn't believe that Channels were Python 3 ready after looking at the docs, so I though it's best to clarify it once and for all, for instance in the FAQ? --- docs/faqs.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/faqs.rst b/docs/faqs.rst index e4250e5..08482c0 100755 --- a/docs/faqs.rst +++ b/docs/faqs.rst @@ -125,3 +125,11 @@ which is the unique channel representing the connection, but remember that whatever you store in must be **network-transparent** - storing things in a global variable won't work outside of development. + + +Are channels Python 2, 3 or 2+3? +-------------------------------- + +Django-channels and all of its dependencies are 2+3 (2.7, 3.4+). Compatibility may change with time. If in doubt, refer to the ``.travis.yml`` configuration file to see which Python versions that are included in CI testing. + +This includes Twisted, for which the used subsets of the library used by Daphne are all py3k ready. From 073cbca16ddda9bb194bd04eeba0bfc6eecd437b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 30 Apr 2016 19:09:46 -0700 Subject: [PATCH 339/746] Fixed #116: Allow configuration of worker listening --- channels/management/commands/runworker.py | 11 ++++++- channels/tests/test_worker.py | 37 +++++++++++++++++++++++ channels/worker.py | 25 +++++++++++++-- docs/deploying.rst | 16 +++++++++- 4 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 channels/tests/test_worker.py diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 0cf4498..7f812e5 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -15,6 +15,10 @@ class Command(BaseCommand): super(Command, self).add_arguments(parser) parser.add_argument('--layer', action='store', dest='layer', default=DEFAULT_CHANNEL_LAYER, help='Channel layer alias to use, if not the default.') + parser.add_argument('--only-channels', action='append', dest='only_channels', + help='Limits this worker to only listening on the provided channels (supports globbing).') + parser.add_argument('--exclude-channels', action='append', dest='exclude_channels', + help='Prevents this worker from listening on the provided channels (supports globbing).') def handle(self, *args, **options): # Get the backend to use @@ -37,7 +41,12 @@ class Command(BaseCommand): callback = self.consumer_called # Run the worker try: - Worker(channel_layer=self.channel_layer, callback=callback).run() + Worker( + channel_layer=self.channel_layer, + callback=callback, + only_channels=options.get("only_channels", None), + exclude_channels=options.get("exclude_channels", None), + ).run() except KeyboardInterrupt: pass diff --git a/channels/tests/test_worker.py b/channels/tests/test_worker.py new file mode 100644 index 0000000..53f8d40 --- /dev/null +++ b/channels/tests/test_worker.py @@ -0,0 +1,37 @@ +from __future__ import unicode_literals +from django.test import SimpleTestCase + +from channels.worker import Worker + + +class WorkerTests(SimpleTestCase): + """ + Tests that the router's routing code works correctly. + """ + + def test_channel_filters(self): + """ + Tests that the include/exclude logic works + """ + # Include + worker = Worker(None, only_channels=["yes.*", "maybe.*"]) + self.assertEqual( + worker.apply_channel_filters(["yes.1", "no.1"]), + ["yes.1"], + ) + self.assertEqual( + worker.apply_channel_filters(["yes.1", "no.1", "maybe.2", "yes"]), + ["yes.1", "maybe.2"], + ) + # Exclude + worker = Worker(None, exclude_channels=["no.*", "maybe.*"]) + self.assertEqual( + worker.apply_channel_filters(["yes.1", "no.1", "maybe.2", "yes"]), + ["yes.1", "yes"], + ) + # Both + worker = Worker(None, exclude_channels=["no.*"], only_channels=["yes.*"]) + self.assertEqual( + worker.apply_channel_filters(["yes.1", "no.1", "maybe.2", "yes"]), + ["yes.1"], + ) diff --git a/channels/worker.py b/channels/worker.py index 4f8278f..1e970e1 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import fnmatch import logging import signal import sys @@ -18,11 +19,14 @@ class Worker(object): and runs their consumers. """ - def __init__(self, channel_layer, callback=None, message_retries=10, signal_handlers=True): + def __init__(self, channel_layer, callback=None, message_retries=10, signal_handlers=True, + only_channels=None, exclude_channels=None): self.channel_layer = channel_layer self.callback = callback self.message_retries = message_retries self.signal_handlers = signal_handlers + self.only_channels = only_channels + self.exclude_channels = exclude_channels self.termed = False self.in_job = False @@ -38,13 +42,30 @@ class Worker(object): logger.info("Shutdown signal received while idle, terminating immediately") sys.exit(0) + def apply_channel_filters(self, channels): + """ + Applies our include and exclude filters to the channel list and returns it + """ + if self.only_channels: + channels = [ + channel for channel in channels + if any(fnmatch.fnmatchcase(channel, pattern) for pattern in self.only_channels) + ] + if self.exclude_channels: + channels = [ + channel for channel in channels + if not any(fnmatch.fnmatchcase(channel, pattern) for pattern in self.exclude_channels) + ] + return channels + def run(self): """ Tries to continually dispatch messages to consumers. """ if self.signal_handlers: self.install_signal_handler() - channels = self.channel_layer.router.channels + channels = self.apply_channel_filters(self.channel_layer.router.channels) + logger.info("Listening on channels %s", ", ".join(channels)) while not self.termed: self.in_job = False channel, content = self.channel_layer.receive_many(channels, block=True) diff --git a/docs/deploying.rst b/docs/deploying.rst index f2298d1..e5d16cc 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -80,7 +80,21 @@ requests will take longer and longer to return as the messages queue up (until the expiry limit is reached, at which point HTTP connections will start dropping). -TODO: We should probably ship some kind of latency measuring tooling. +In a more complex project, you won't want all your channels being served by the +same workers, especially if you have long-running tasks (if you serve them from +the same workers as HTTP requests, there's a chance long-running tasks could +block up all the workers and delay responding to HTTP requests). + +To manage this, it's possible to tell workers to either limit themselves to +just certain channel names or ignore sepcific channels using the +``--only-channels`` and ``--exclude-channels`` options. Here's an example +of configuring a worker to only serve HTTP and WebSocket requests:: + + python manage.py runworker --only-channels=http.* --only-channels=websocket.* + +Or telling a worker to ignore all messages on the "thumbnail" channel:: + + python manage.py runworker --exclude-channels=thumbnail Run interface servers From 698c2aaca05afd150e9ef6188481e6b394259a1c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 30 Apr 2016 19:15:45 -0700 Subject: [PATCH 340/746] Fix worker.py formatting --- channels/worker.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/channels/worker.py b/channels/worker.py index 1e970e1..3a1122b 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -19,8 +19,15 @@ class Worker(object): and runs their consumers. """ - def __init__(self, channel_layer, callback=None, message_retries=10, signal_handlers=True, - only_channels=None, exclude_channels=None): + def __init__( + self, + channel_layer, + callback=None, + message_retries=10, + signal_handlers=True, + only_channels=None, + exclude_channels=None + ): self.channel_layer = channel_layer self.callback = callback self.message_retries = message_retries From a9187b99fe67f0234efb265f1ceb5556e63af808 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 30 Apr 2016 20:55:19 -0700 Subject: [PATCH 341/746] More formatting fixes --- channels/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/worker.py b/channels/worker.py index 3a1122b..31e0288 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -27,7 +27,7 @@ class Worker(object): signal_handlers=True, only_channels=None, exclude_channels=None - ): + ): self.channel_layer = channel_layer self.callback = callback self.message_retries = message_retries From e451ea4d698450813bd11fed6b501b839cd477a6 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 2 May 2016 19:17:24 -0700 Subject: [PATCH 342/746] Reformat runworker a bit --- channels/management/commands/runworker.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 7f812e5..ec2b68a 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -13,12 +13,18 @@ class Command(BaseCommand): def add_arguments(self, parser): super(Command, self).add_arguments(parser) - parser.add_argument('--layer', action='store', dest='layer', default=DEFAULT_CHANNEL_LAYER, - help='Channel layer alias to use, if not the default.') - parser.add_argument('--only-channels', action='append', dest='only_channels', - help='Limits this worker to only listening on the provided channels (supports globbing).') - parser.add_argument('--exclude-channels', action='append', dest='exclude_channels', - help='Prevents this worker from listening on the provided channels (supports globbing).') + parser.add_argument( + '--layer', action='store', dest='layer', default=DEFAULT_CHANNEL_LAYER, + help='Channel layer alias to use, if not the default.', + ) + parser.add_argument( + '--only-channels', action='append', dest='only_channels', + help='Limits this worker to only listening on the provided channels (supports globbing).', + ) + parser.add_argument( + '--exclude-channels', action='append', dest='exclude_channels', + help='Prevents this worker from listening on the provided channels (supports globbing).', + ) def handle(self, *args, **options): # Get the backend to use From cf9d7d6f767f5e30efdebfb0987cbb8697dade5b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 3 May 2016 18:06:43 -0700 Subject: [PATCH 343/746] Change to more precise TestCase import --- channels/tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/tests/base.py b/channels/tests/base.py index bed25f1..09ce66f 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -1,4 +1,4 @@ -from django.test import TestCase +from django.test.testcases import TestCase from channels import DEFAULT_CHANNEL_LAYER from channels.asgi import channel_layers, ChannelLayerWrapper from channels.message import Message From 45dfeb548ef214afe05a9c43cb6dc9061a5f9886 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 3 May 2016 18:10:51 -0700 Subject: [PATCH 344/746] Django 1.10 patch fixes --- channels/handler.py | 10 ++++++---- patchinator.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index c8a058e..43fd653 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -87,14 +87,16 @@ class AsgiRequest(http.HttpRequest): self.META[corrected_name] = value # Pull out request encoding if we find it if "CONTENT_TYPE" in self.META: - _, content_params = cgi.parse_header(self.META["CONTENT_TYPE"]) - if 'charset' in content_params: + self.content_type, self.content_params = cgi.parse_header(self.META["CONTENT_TYPE"]) + if 'charset' in self.content_params: try: - codecs.lookup(content_params['charset']) + codecs.lookup(self.content_params['charset']) except LookupError: pass else: - self.encoding = content_params['charset'] + self.encoding = self.content_params['charset'] + else: + self.content_type, self.content_params = "", {} # Pull out content length info if self.META.get('CONTENT_LENGTH', None): try: diff --git a/patchinator.py b/patchinator.py index f86e07c..9c5eb34 100644 --- a/patchinator.py +++ b/patchinator.py @@ -124,6 +124,7 @@ global_transforms = [ Replacement(r"from django.channels.tests import", r"from django.test.channels import"), Replacement(r"from django.channels.handler import", r"from django.core.handlers.asgi import"), Replacement(r"channels.tests.test_routing", r"channels_tests.test_routing"), + Replacement(r"django.core.urlresolvers", r"django.urls"), ] python_transforms = global_transforms + [ From 96735b917b1df37c939e5b4126f1355f7a9943c4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 3 May 2016 18:14:41 -0700 Subject: [PATCH 345/746] Make flake8 like indentation --- channels/worker.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/channels/worker.py b/channels/worker.py index 31e0288..77c1dcb 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -20,14 +20,14 @@ class Worker(object): """ def __init__( - self, - channel_layer, - callback=None, - message_retries=10, - signal_handlers=True, - only_channels=None, - exclude_channels=None - ): + self, + channel_layer, + callback=None, + message_retries=10, + signal_handlers=True, + only_channels=None, + exclude_channels=None + ): self.channel_layer = channel_layer self.callback = callback self.message_retries = message_retries From ea66b6560bb40141281a35c97ed0a735f228cfb6 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 3 May 2016 18:43:15 -0700 Subject: [PATCH 346/746] Doc spelling corrections --- docs/concepts.rst | 2 +- docs/cross-compat.rst | 2 +- docs/deploying.rst | 4 ++-- docs/getting-started.rst | 2 +- docs/testing.rst | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 3fcb63f..a5b43b4 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -32,7 +32,7 @@ What is a channel? It is an *ordered*, *first-in first-out queue* with You can think of it as analogous to a task queue - messages are put onto the channel by *producers*, and then given to just one of the *consumers* -listening to that channnel. +listening to that channel. By *at-most-once* we say that either one consumer gets the message or nobody does (if the channel implementation crashes, let's say). The diff --git a/docs/cross-compat.rst b/docs/cross-compat.rst index f9b2f59..ab6541e 100644 --- a/docs/cross-compat.rst +++ b/docs/cross-compat.rst @@ -1,4 +1,4 @@ -Cross-Compatability +Cross-Compatibility =================== Channels is being released as both a third-party app for Django 1.8 and 1.9, diff --git a/docs/deploying.rst b/docs/deploying.rst index e5d16cc..974c135 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -86,7 +86,7 @@ the same workers as HTTP requests, there's a chance long-running tasks could block up all the workers and delay responding to HTTP requests). To manage this, it's possible to tell workers to either limit themselves to -just certain channel names or ignore sepcific channels using the +just certain channel names or ignore specific channels using the ``--only-channels`` and ``--exclude-channels`` options. Here's an example of configuring a worker to only serve HTTP and WebSocket requests:: @@ -166,7 +166,7 @@ rollouts to make sure workers on new code aren't experiencing high error rates. There's no need to restart the WSGI or WebSocket interface servers unless you've upgraded your version of Channels or changed any settings; none of your code is used by them, and all middleware and code that can -customise requests is run on the consumers. +customize requests is run on the consumers. You can even use different Python versions for the interface servers and the workers; the ASGI protocol that channel layers communicate over diff --git a/docs/getting-started.rst b/docs/getting-started.rst index b71dc35..8968377 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -182,7 +182,7 @@ of the time anyway. on 100% guaranteed delivery, which Channels won't give you, look at each failure case and program something to expect and handle it - be that retry logic, partial content handling, or just having something not work that one - time. HTTP requests are just as fallible, and most people's reponse to that + time. HTTP requests are just as fallible, and most people's response to that is a generic error page! .. _websocket-example: diff --git a/docs/testing.rst b/docs/testing.rst index 55476ba..d10a4c6 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -2,7 +2,7 @@ Testing Consumers ================= When you want to write unit tests for your new Channels consumers, you'll -realise that you can't use the standard Django test client to submit fake HTTP +realize that you can't use the standard Django test client to submit fake HTTP requests - instead, you'll need to submit fake Messages to your consumers, and inspect what Messages they send themselves. @@ -15,7 +15,7 @@ ChannelTestCase If your tests inherit from the ``channels.tests.ChannelTestCase`` base class, whenever you run tests your channel layer will be swapped out for a captive -in-memory layer, meaning you don't need an exernal server running to run tests. +in-memory layer, meaning you don't need an external server running to run tests. Moreover, you can inject messages onto this layer and inspect ones sent to it to help test your consumers. From 2219546a5d727d92896f053fe323acc32a7b43eb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 4 May 2016 09:20:36 -0700 Subject: [PATCH 347/746] Noun docs changes --- docs/concepts.rst | 4 ++-- docs/deploying.rst | 10 +++++----- docs/getting-started.rst | 16 ++++++++-------- docs/reference.rst | 7 ++++--- docs/testing.rst | 2 +- patchinator.py | 11 +++++++++++ 6 files changed, 31 insertions(+), 19 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index a5b43b4..64d9cad 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -13,7 +13,7 @@ And, beyond that, there are plenty of non-critical tasks that applications could easily offload until after a response has been sent - like saving things into a cache or thumbnailing newly-uploaded images. -Channels changes the way Django runs to be "event oriented" - rather than +It changes the way Django runs to be "event oriented" - rather than just responding to requests, instead Django responds to a wide array of events sent on *channels*. There's still no persistent state - each event handler, or *consumer* as we call them, is called independently in a way much like a @@ -26,7 +26,7 @@ Let's look at what *channels* are first. What is a channel? ------------------ -The core of Channels is, unsurprisingly, a datastructure called a *channel*. +The core of the system is, unsurprisingly, a datastructure called a *channel*. What is a channel? It is an *ordered*, *first-in first-out queue* with *message expiry* and *at-most-once delivery* to *only one listener at a time*. diff --git a/docs/deploying.rst b/docs/deploying.rst index 974c135..79f9a64 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -4,9 +4,9 @@ Deploying Deploying applications using Channels requires a few more steps than a normal Django WSGI application, but it's not very many. -Firstly, remember that even if you have Channels installed, it's an entirely -optional part of Django. If you leave a project with the default settings -(no ``CHANNEL_BACKENDS``), it'll just run and work like a normal WSGI app. +Firstly, remember that it's an entirely optional part of Django. +If you leave a project with the default settings (no ``CHANNEL_LAYERS``), +it'll just run and work like a normal WSGI app. When you want to enable channels in production, you need to do three things: @@ -112,7 +112,7 @@ cluster on the backend, see :ref:`wsgi-to-asgi`. If you want to support WebSockets, long-poll HTTP requests and other Channels features, you'll need to run a native ASGI interface server, as the WSGI specification has no support for running these kinds of requests concurrently. -Channels ships with an interface server that we recommend you use called +We ship with an interface server that we recommend you use called `Daphne `_; it supports WebSockets, long-poll HTTP requests, HTTP/2 *(soon)* and performs quite well. Of course, any ASGI-compliant server will work! @@ -164,7 +164,7 @@ workers. As long as the new code is session-compatible, you can even do staged rollouts to make sure workers on new code aren't experiencing high error rates. There's no need to restart the WSGI or WebSocket interface servers unless -you've upgraded your version of Channels or changed any settings; +you've upgraded the interface server itself or changed any Django settings; none of your code is used by them, and all middleware and code that can customize requests is run on the consumers. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 8968377..aa83555 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -19,8 +19,8 @@ with a WSGI-based Django, and your views and static file serving (from As a very basic introduction, let's write a consumer that overrides the built-in handling and handles every HTTP request directly. This isn't something you'd -usually do in a project, but it's a good illustration of how Channels -underlies even core Django - it's less of an addition and more adding a whole +usually do in a project, but it's a good illustration of how channels +underlie even core Django - it's less of an addition and more adding a whole new layer under the existing view layer. Make a new project, a new app, and put this in a ``consumers.py`` file in the app:: @@ -279,7 +279,7 @@ see :doc:`backends` for more. The second thing, once we have a networked channel backend set up, is to make sure we're running an interface server that's capable of serving WebSockets. -Luckily, installing Channels will also install ``daphne``, an interface server +To solve this, Channels comes with ``daphne``, an interface server that can handle both HTTP and WebSockets at the same time, and then ties this in to run when you run ``runserver`` - you shouldn't notice any difference from the normal Django ``runserver``, though some of the options may be a little @@ -471,7 +471,7 @@ Django session ID as part of the URL, like this:: socket = new WebSocket("ws://127.0.0.1:9000/?session_key=abcdefg"); You can get the current session key in a template with ``{{ request.session.session_key }}``. -Note that Channels can't work with signed cookie sessions - since only HTTP +Note that this can't work with signed cookie sessions - since only HTTP responses can set cookies, it needs a backend it can write to to separately store state. @@ -479,7 +479,7 @@ store state. Routing ------- -Channels' ``routing.py`` acts very much like Django's ``urls.py``, including the +The ``routing.py`` file acts very much like Django's ``urls.py``, including the ability to route things to different consumers based on ``path``, or any other message attribute that's a string (for example, ``http.request`` messages have a ``method`` key you could route based on). @@ -508,7 +508,7 @@ routing our chat from above:: include(http_routing), ] -Channels will resolve the routing in order, short-circuiting around the +The routing is resolved in order, short-circuiting around the includes if one or more of their matches fails. You don't have to start with the ``^`` symbol - we use Python's ``re.match`` function, which starts at the start of a line anyway - but it's considered good practice. @@ -624,7 +624,7 @@ same effect if someone tried to request a view before the login view had finishe processing, but there you're not expecting that page to run after the login, whereas you'd naturally expect ``receive`` to run after ``connect``. -Channels has a solution - the ``enforce_ordering`` decorator. All WebSocket +Channels has a solution - the ``enforce_ordering`` decorator. All WebSocket messages contain an ``order`` key, and this decorator uses that to make sure that messages are consumed in the right order, in one of two modes: @@ -695,6 +695,6 @@ to manage logical sets of channels, and how Django's session and authentication systems easily integrate with WebSockets. We recommend you read through the rest of the reference documentation to see -all of what Channels has to offer; in particular, you may want to look at +more about what you can do with channels; in particular, you may want to look at our :doc:`deploying` and :doc:`scaling` resources to get an idea of how to design and run apps in production environments. diff --git a/docs/reference.rst b/docs/reference.rst index adc2d1d..aaa99e2 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -6,9 +6,10 @@ Reference Consumers --------- -When you configure channel routing, Channels expects the object assigned to -a channel to be a callable that takes exactly one positional argument, here -called ``message``. This is a :ref:`message object `. +When you configure channel routing, the object assigned to a channel +should be a callable that takes exactly one positional argument, here +called ``message``, which is a :ref:`message object `. A consumer +is any callable that fits this definition. Consumers are not expected to return anything, and if they do, it will be ignored. They may raise ``channels.exceptions.ConsumeLater`` to re-insert diff --git a/docs/testing.rst b/docs/testing.rst index d10a4c6..20c7732 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -6,7 +6,7 @@ realize that you can't use the standard Django test client to submit fake HTTP requests - instead, you'll need to submit fake Messages to your consumers, and inspect what Messages they send themselves. -Channels comes with a ``TestCase`` subclass that sets all of this up for you, +We provide a ``TestCase`` subclass that sets all of this up for you, however, so you can easily write tests and check what your consumers are sending. diff --git a/patchinator.py b/patchinator.py index 9c5eb34..4064013 100644 --- a/patchinator.py +++ b/patchinator.py @@ -142,6 +142,17 @@ docs_transforms = global_transforms + [ Replacement(r":doc:`([\w\d\s]+) `", r"`\1 `_"), Replacement(r"\n\(.*installation>`\)\n", r""), Replacement(r":doc:`installed Channels correctly `", r"added the channel layer setting"), + Replacement(r"Channels", r"channels"), + Replacement(r"Started with channels", r"Started with Channels"), + Replacement(r"Running with channels", r"Running with Channels"), + Replacement(r"channels consumers", r"channel consumers"), + Replacement(r"channels' design", r"The channels design"), + Replacement(r"channels is being released", r"Channels is being released"), + Replacement(r"channels is", r"channels are"), + Replacement(r"channels provides a", r"Channels provides a"), + Replacement(r"channels can use", r"Channels can use"), + Replacement(r"channels Concepts", r"Channels Concepts"), + Replacement(r"channels works", r"channels work"), ] From 1eb6a530d31f0ff7d33f3bcf357046bbd0e28ab7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 4 May 2016 09:27:01 -0700 Subject: [PATCH 348/746] WSGI-ASGI deploy notes --- docs/deploying.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/deploying.rst b/docs/deploying.rst index 79f9a64..13d31d7 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -194,3 +194,12 @@ that backs onto ASGI underneath:: os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_test.settings") channel_layer = get_channel_layer() application = WsgiToAsgiAdapter(channel_layer) + +While this removes WebSocket support through the same port that HTTP is served +on, it still lets you use other channels features such as background tasks or +alternative interface servers (that would let you write consumers against +incoming emails or IRC messages). + +You can also use this method to serve HTTP through your existing stack +and run Daphne on a separate port or domain to receive only WebSocket +connections. From 2f01155bfd3a57fe5920a78c218237d2d60268e1 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 4 May 2016 10:39:38 -0700 Subject: [PATCH 349/746] Session tests --- channels/sessions.py | 6 +- channels/tests/test_sessions.py | 169 ++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 channels/tests/test_sessions.py diff --git a/channels/sessions.py b/channels/sessions.py index 594e921..a749fcd 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -24,7 +24,11 @@ def session_for_reply_channel(reply_channel): session_engine = import_module(getattr(settings, "CHANNEL_SESSION_ENGINE", settings.SESSION_ENGINE)) if session_engine is signed_cookies: raise ValueError("You cannot use channels session functionality with signed cookie sessions!") - return session_engine.SessionStore(session_key=session_key) + # Force the instance to load in case it resets the session when it does + instance = session_engine.SessionStore(session_key=session_key) + instance._session.keys() + instance._session_key = session_key + return instance def channel_session(func): diff --git a/channels/tests/test_sessions.py b/channels/tests/test_sessions.py new file mode 100644 index 0000000..22d7d5d --- /dev/null +++ b/channels/tests/test_sessions.py @@ -0,0 +1,169 @@ +from __future__ import unicode_literals + +from django.conf import settings +from django.test import override_settings +from channels.exceptions import ConsumeLater +from channels.message import Message +from channels.sessions import channel_session, http_session, enforce_ordering, session_for_reply_channel +from channels.tests import ChannelTestCase + + +@override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") +class SessionTests(ChannelTestCase): + """ + Tests the channels session module. + """ + + def test_session_for_reply_channel(self): + """ + Tests storing and retrieving values by reply_channel. + """ + session1 = session_for_reply_channel("test-reply-channel") + session1["testvalue"] = 42 + session1.save(must_create=True) + session2 = session_for_reply_channel("test-reply-channel") + self.assertEqual(session2["testvalue"], 42) + + def test_channel_session(self): + """ + Tests the channel_session decorator + """ + # Construct message to send + message = Message({"reply_channel": "test-reply"}, None, None) + # Run through a simple fake consumer that assigns to it + @channel_session + def inner(message): + message.channel_session["num_ponies"] = -1 + inner(message) + # Test the session worked + session2 = session_for_reply_channel("test-reply") + self.assertEqual(session2["num_ponies"], -1) + + def test_channel_session_double(self): + """ + Tests the channel_session decorator detects being wrapped in itself + and doesn't blow up. + """ + # Construct message to send + message = Message({"reply_channel": "test-reply"}, None, None) + # Run through a simple fake consumer that should trigger the error + @channel_session + @channel_session + def inner(message): + message.channel_session["num_ponies"] = -1 + inner(message) + # Test the session worked + session2 = session_for_reply_channel("test-reply") + self.assertEqual(session2["num_ponies"], -1) + + def test_channel_session_no_reply(self): + """ + Tests the channel_session decorator detects no reply channel + """ + # Construct message to send + message = Message({}, None, None) + # Run through a simple fake consumer that should trigger the error + @channel_session + @channel_session + def inner(message): + message.channel_session["num_ponies"] = -1 + with self.assertRaises(ValueError): + inner(message) + + def test_http_session(self): + """ + Tests that http_session correctly extracts a session cookie. + """ + # Make a session to try against + session1 = session_for_reply_channel("test-reply") + # Construct message to send + message = Message({ + "reply_channel": "test-reply", + "http_version": "1.1", + "method": "GET", + "path": "/test2/", + "headers": { + "host": b"example.com", + "cookie": ("%s=%s" % (settings.SESSION_COOKIE_NAME, session1.session_key)).encode("ascii"), + }, + }, None, None) + # Run it through http_session, make sure it works (test double here too) + @http_session + @http_session + def inner(message): + message.http_session["species"] = "horse" + inner(message) + # Check value assignment stuck + session2 = session_for_reply_channel("test-reply") + self.assertEqual(session2["species"], "horse") + + def test_enforce_ordering_slight(self): + """ + Tests that slight mode of enforce_ordering works + """ + # Construct messages to send + message0 = Message({"reply_channel": "test-reply-a", "order": 0}, None, None) + message1 = Message({"reply_channel": "test-reply-a", "order": 1}, None, None) + message2 = Message({"reply_channel": "test-reply-a", "order": 2}, None, None) + # Run them in an acceptable slight order + @enforce_ordering(slight=True) + def inner(message): + pass + inner(message0) + inner(message2) + inner(message1) + + def test_enforce_ordering_slight_fail(self): + """ + Tests that slight mode of enforce_ordering fails on bad ordering + """ + # Construct messages to send + message2 = Message({"reply_channel": "test-reply-e", "order": 2}, None, None) + # Run them in an acceptable strict order + @enforce_ordering(slight=True) + def inner(message): + pass + with self.assertRaises(ConsumeLater): + inner(message2) + + def test_enforce_ordering_strict(self): + """ + Tests that strict mode of enforce_ordering works + """ + # Construct messages to send + message0 = Message({"reply_channel": "test-reply-b", "order": 0}, None, None) + message1 = Message({"reply_channel": "test-reply-b", "order": 1}, None, None) + message2 = Message({"reply_channel": "test-reply-b", "order": 2}, None, None) + # Run them in an acceptable strict order + @enforce_ordering + def inner(message): + pass + inner(message0) + inner(message1) + inner(message2) + + def test_enforce_ordering_strict_fail(self): + """ + Tests that strict mode of enforce_ordering fails on bad ordering + """ + # Construct messages to send + message0 = Message({"reply_channel": "test-reply-c", "order": 0}, None, None) + message2 = Message({"reply_channel": "test-reply-c", "order": 2}, None, None) + # Run them in an acceptable strict order + @enforce_ordering + def inner(message): + pass + inner(message0) + with self.assertRaises(ConsumeLater): + inner(message2) + + def test_enforce_ordering_fail_no_order(self): + """ + Makes sure messages with no "order" key fail + """ + message0 = Message({"reply_channel": "test-reply-d"}, None, None) + @enforce_ordering(slight=True) + def inner(message): + pass + with self.assertRaises(ValueError): + inner(message0) From 7b75761644c7dd65761f2d48acd4579fc9ce628a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 4 May 2016 18:52:52 -0700 Subject: [PATCH 350/746] Flake8 fixes --- channels/tests/test_sessions.py | 18 ++++++++++++++++++ patchinator.py | 3 +++ 2 files changed, 21 insertions(+) diff --git a/channels/tests/test_sessions.py b/channels/tests/test_sessions.py index 22d7d5d..e973f9d 100644 --- a/channels/tests/test_sessions.py +++ b/channels/tests/test_sessions.py @@ -30,10 +30,12 @@ class SessionTests(ChannelTestCase): """ # Construct message to send message = Message({"reply_channel": "test-reply"}, None, None) + # Run through a simple fake consumer that assigns to it @channel_session def inner(message): message.channel_session["num_ponies"] = -1 + inner(message) # Test the session worked session2 = session_for_reply_channel("test-reply") @@ -46,12 +48,14 @@ class SessionTests(ChannelTestCase): """ # Construct message to send message = Message({"reply_channel": "test-reply"}, None, None) + # Run through a simple fake consumer that should trigger the error @channel_session @channel_session def inner(message): message.channel_session["num_ponies"] = -1 inner(message) + # Test the session worked session2 = session_for_reply_channel("test-reply") self.assertEqual(session2["num_ponies"], -1) @@ -62,11 +66,13 @@ class SessionTests(ChannelTestCase): """ # Construct message to send message = Message({}, None, None) + # Run through a simple fake consumer that should trigger the error @channel_session @channel_session def inner(message): message.channel_session["num_ponies"] = -1 + with self.assertRaises(ValueError): inner(message) @@ -87,11 +93,13 @@ class SessionTests(ChannelTestCase): "cookie": ("%s=%s" % (settings.SESSION_COOKIE_NAME, session1.session_key)).encode("ascii"), }, }, None, None) + # Run it through http_session, make sure it works (test double here too) @http_session @http_session def inner(message): message.http_session["species"] = "horse" + inner(message) # Check value assignment stuck session2 = session_for_reply_channel("test-reply") @@ -105,10 +113,12 @@ class SessionTests(ChannelTestCase): message0 = Message({"reply_channel": "test-reply-a", "order": 0}, None, None) message1 = Message({"reply_channel": "test-reply-a", "order": 1}, None, None) message2 = Message({"reply_channel": "test-reply-a", "order": 2}, None, None) + # Run them in an acceptable slight order @enforce_ordering(slight=True) def inner(message): pass + inner(message0) inner(message2) inner(message1) @@ -119,10 +129,12 @@ class SessionTests(ChannelTestCase): """ # Construct messages to send message2 = Message({"reply_channel": "test-reply-e", "order": 2}, None, None) + # Run them in an acceptable strict order @enforce_ordering(slight=True) def inner(message): pass + with self.assertRaises(ConsumeLater): inner(message2) @@ -134,10 +146,12 @@ class SessionTests(ChannelTestCase): message0 = Message({"reply_channel": "test-reply-b", "order": 0}, None, None) message1 = Message({"reply_channel": "test-reply-b", "order": 1}, None, None) message2 = Message({"reply_channel": "test-reply-b", "order": 2}, None, None) + # Run them in an acceptable strict order @enforce_ordering def inner(message): pass + inner(message0) inner(message1) inner(message2) @@ -149,10 +163,12 @@ class SessionTests(ChannelTestCase): # Construct messages to send message0 = Message({"reply_channel": "test-reply-c", "order": 0}, None, None) message2 = Message({"reply_channel": "test-reply-c", "order": 2}, None, None) + # Run them in an acceptable strict order @enforce_ordering def inner(message): pass + inner(message0) with self.assertRaises(ConsumeLater): inner(message2) @@ -162,8 +178,10 @@ class SessionTests(ChannelTestCase): Makes sure messages with no "order" key fail """ message0 = Message({"reply_channel": "test-reply-d"}, None, None) + @enforce_ordering(slight=True) def inner(message): pass + with self.assertRaises(ValueError): inner(message0) diff --git a/patchinator.py b/patchinator.py index 4064013..4cbf44b 100644 --- a/patchinator.py +++ b/patchinator.py @@ -221,6 +221,9 @@ class Patchinator(object): FileMap( "channels/tests/test_request.py", "tests/channels_tests/test_request.py", python_transforms, ), + FileMap( + "channels/tests/test_sessions.py", "tests/channels_tests/test_sessions.py", python_transforms, + ), # Docs FileMap( "docs/backends.rst", "docs/ref/channels/backends.txt", docs_transforms, From feea84f323c1d55ed5f41a986fed74fa236aa346 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 5 May 2016 22:48:12 -0700 Subject: [PATCH 351/746] Introduce backpressure with ChannelFull --- channels/database_layer.py | 6 ++++++ channels/handler.py | 13 ++++++++++++- channels/worker.py | 7 ++++++- docs/asgi.rst | 35 ++++++++++++++++++++++++++++++----- setup.py | 2 +- 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/channels/database_layer.py b/channels/database_layer.py index 37894a8..88be4a3 100644 --- a/channels/database_layer.py +++ b/channels/database_layer.py @@ -36,6 +36,12 @@ class DatabaseChannelLayer(object): extensions = ["groups", "flush"] + class MessageTooLarge(Exception): + pass + + class ChannelFull(Exception): + pass + def send(self, channel, message): # Typecheck assert isinstance(message, dict), "message is not a dict" diff --git a/channels/handler.py b/channels/handler.py index 43fd653..ec03f4c 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -328,4 +328,15 @@ class ViewConsumer(object): def __call__(self, message): for reply_message in self.handler(message): - message.reply_channel.send(reply_message) + while True: + # If we get ChannelFull we just wait and keep trying until + # it goes through. + # TODO: Add optional death timeout? Don't want to lock up + # a whole worker if the client just vanishes and leaves the response + # channel full. + try: + message.reply_channel.send(reply_message) + except message.channel_layer.ChannelFull: + time.sleep(0.05) + else: + break diff --git a/channels/worker.py b/channels/worker.py index 77c1dcb..6e65492 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -116,6 +116,11 @@ class Worker(object): repr(content)[:100], ) continue - self.channel_layer.send(channel, content) + # Try to re-insert it a few times then drop it + for _ in range(10): + try: + self.channel_layer.send(channel, content) + except self.channel_layer.ChannelFull: + time.sleep(0.05) except: logger.exception("Error processing message with consumer %s:", name_that_thing(consumer)) diff --git a/docs/asgi.rst b/docs/asgi.rst index b31136a..fb3323c 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -13,7 +13,7 @@ servers (particularly webservers) and Python applications, intended to allow handling of multiple common protocol styles (including HTTP, HTTP2, and WebSocket). -It is intended to replace and expand on WSGI, though the design +It is intended to supplement and expand on WSGI, though the design deliberately includes provisions to allow WSGI-to-ASGI and ASGI-to-WGSI adapters to be easily written for the HTTP protocol. @@ -34,7 +34,7 @@ and from different application threads or processes. It also lays out new, serialization-compatible formats for things like HTTP requests and responses and WebSocket data frames, to allow these to -be transported over a network or local socket, and allow separation +be transported over a network or local memory, and allow separation of protocol handling and application logic into different processes. Part of this design is ensuring there is an easy path to use both @@ -255,10 +255,30 @@ problem. If ordering of incoming packets matters for a protocol, they should be annotated with a packet number (as WebSocket is in this specification). Single-reader channels, such as those used for response channels back to -clients, are not subject to this problem; a single reader should always +clients, are not subject to this problem; a single reader must always receive messages in channel order. +Capacity +-------- + +To provide backpressure, each channel in a channel layer may have a capacity, +defined however the layer wishes (it is recommended that it is configurable +by the user using keyword arguments to the channel layer constructor, and +furthermore configurable per channel name or name prefix). + +When a channel is at or over capacity, trying to send() to that channel +may raise ChannelFull, which indicates to the sender the channel is over +capacity. How the sender wishes to deal with this will depend on context; +for example, a web application trying to send a response body will likely +wait until it empties out again, while a HTTP interface server trying to +send in a request would drop the request and return a 503 error. + +Sending to a group never raises ChannelFull; instead, it must silently drop +the message if it is over capacity, as per ASGI's at-most-once delivery +policy. + + Specification Details ===================== @@ -291,6 +311,9 @@ A *channel layer* must provide an object with these attributes * ``MessageTooLarge``, the exception raised when a send operation fails because the encoded message is over the layer's size limit. +* ``ChannelFull``, the exception raised when a send operation fails + because the destination channel is over capacity. + * ``extensions``, a list of unicode string names indicating which extensions this layer provides, or empty if it supports none. The names defined in this document are ``groups``, ``flush`` and @@ -307,7 +330,8 @@ A channel layer implementing the ``groups`` extension must also provide: * ``send_group(group, message)``, a callable that takes two positional arguments; the group to send to, as a unicode string, and the message - to send, as a serializable ``dict``. + to send, as a serializable ``dict``. It may raise MessageTooLarge but cannot + raise ChannelFull. * ``group_expiry``, an integer number of seconds that specifies how long group membership is valid for after the most recent ``group_add`` call (see @@ -346,7 +370,8 @@ Channels **must**: * Never deliver a message more than once. -* Never block on message send. +* Never block on message send (though they may raise ChannelFull or + MessageTooLarge) * Be able to handle messages of at least 1MB in size when encoded as JSON (the implementation may use better encoding or compression, as long diff --git a/setup.py b/setup.py index 59bf911..5664ffd 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( include_package_data=True, install_requires=[ 'Django>=1.8', - 'asgiref>=0.10', + 'asgiref>=0.12', 'daphne>=0.11', ] ) From f346585f7c797da401cbf3e46e6a5859d795e9ca Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 6 May 2016 18:41:51 -0700 Subject: [PATCH 352/746] Change from waffle to 1MB message limit. --- docs/asgi.rst | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index fb3323c..630459c 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -128,14 +128,11 @@ Messages should expire after a set time sitting unread in a channel; the recommendation is one minute, though the best value depends on the channel layer and the way it is deployed. -The maximum message size is finite, though it varies based on the channel layer -and the encoding it's using. Channel layers may reject messages at ``send()`` -time with a ``MessageTooLarge`` exception; the calling code should take -appropriate action (e.g. HTTP responses can be chunked, while HTTP -requests should be closed with a ``413 Request Entity Too Large`` response). -It is intended that some channel layers will only support messages of around a -megabyte, while others will be able to take a gigabyte or more, and that it -may be configurable. +The maximum message size is 1MB; if more data than this needs to be transmitted +it should be chunked or placed onto its own single-reader channel (see how +HTTP request bodies are done, for example). All channel layers must support +messages up to this size. + Handling Protocols ------------------ From dcbab8b2b426ee85ba420b1a71d44b135706b875 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 7 May 2016 10:35:12 -0700 Subject: [PATCH 353/746] Remove DatabaseLayer and improve deployment docs mentioning it --- channels/database_layer.py | 253 -------------------------- channels/tests/test_database_layer.py | 8 - docs/backends.rst | 27 ++- docs/concepts.rst | 3 +- docs/deploying.rst | 168 ++++++++++++----- docs/index.rst | 1 - docs/scaling.rst | 37 ---- patchinator.py | 9 - 8 files changed, 137 insertions(+), 369 deletions(-) delete mode 100644 channels/database_layer.py delete mode 100644 channels/tests/test_database_layer.py delete mode 100755 docs/scaling.rst diff --git a/channels/database_layer.py b/channels/database_layer.py deleted file mode 100644 index 88be4a3..0000000 --- a/channels/database_layer.py +++ /dev/null @@ -1,253 +0,0 @@ -import base64 -import datetime -import json -import random -import string -import time - -from django.apps.registry import Apps -from django.db import DEFAULT_DB_ALIAS, connections, models, transaction -from django.db.utils import OperationalError -from django.utils import six -from django.utils.functional import cached_property -from django.utils.timezone import now - - -class DatabaseChannelLayer(object): - """ - ORM-backed ASGI channel layer. - - For development use only; it will span multiple processes fine, - but it's going to be pretty bad at throughput. If you're reading this and - running it in production, PLEASE STOP. - - Also uses JSON for serialization, as we don't want to make Django depend - on msgpack for the built-in backend. The JSON format uses \uffff as first - character to signify a b64 byte string rather than a text string. Ugly, but - it's not a valid Unicode character, so it should be safe enough. - """ - - def __init__(self, db_alias=DEFAULT_DB_ALIAS, expiry=60, group_expiry=86400): - self.expiry = expiry - self.group_expiry = group_expiry - self.db_alias = db_alias - - # ASGI API - - extensions = ["groups", "flush"] - - class MessageTooLarge(Exception): - pass - - class ChannelFull(Exception): - pass - - def send(self, channel, message): - # Typecheck - assert isinstance(message, dict), "message is not a dict" - assert isinstance(channel, six.text_type), "%s is not unicode" % channel - # Write message to messages table - self.channel_model.objects.create( - channel=channel, - content=self.serialize(message), - expiry=now() + datetime.timedelta(seconds=self.expiry) - ) - - def receive_many(self, channels, block=False): - if not channels: - return None, None - assert all(isinstance(channel, six.text_type) for channel in channels) - # Shuffle channels - channels = list(channels) - random.shuffle(channels) - # Clean out expired messages - self._clean_expired() - # Get a message from one of our channels - while True: - try: - with transaction.atomic(): - message = self.channel_model.objects.select_for_update().filter( - channel__in=channels - ).order_by("id").first() - if message: - self.channel_model.objects.filter(pk=message.pk).delete() - return message.channel, self.deserialize(message.content) - except OperationalError: - # The database is probably trying to prevent a deadlock - time.sleep(0.1) - continue - if block: - time.sleep(1) - else: - return None, None - - def new_channel(self, pattern): - assert isinstance(pattern, six.text_type) - assert pattern.endswith("!") - # Keep making channel names till one isn't present. - while True: - random_string = "".join(random.choice(string.ascii_letters) for i in range(10)) - new_name = pattern + random_string - if not self.channel_model.objects.filter(channel=new_name).exists(): - return new_name - - # ASGI Group extension - - def group_add(self, group, channel): - """ - Adds the channel to the named group for at least 'expiry' - seconds (expiry defaults to message expiry if not provided). - """ - self.group_model.objects.update_or_create( - group=group, - channel=channel, - ) - - def group_discard(self, group, channel): - """ - Removes the channel from the named group if it is in the group; - does nothing otherwise (does not error) - """ - self.group_model.objects.filter(group=group, channel=channel).delete() - - def send_group(self, group, message): - """ - Sends a message to the entire group. - """ - self._clean_expired() - for channel in self.group_model.objects.filter(group=group).values_list("channel", flat=True): - self.send(channel, message) - - # ASGI Flush extension - - def flush(self): - self.channel_model.objects.all().delete() - self.group_model.objects.all().delete() - - # Serialization - - def serialize(self, message): - return AsgiJsonEncoder().encode(message) - - def deserialize(self, message): - return AsgiJsonDecoder().decode(message) - - # Database state mgmt - - @property - def connection(self): - """ - Returns the correct connection for the current thread. - """ - return connections[self.db_alias] - - @cached_property - def channel_model(self): - """ - Initialises a new model to store messages; not done as part of a - models.py as we don't want to make it for most installs. - """ - # Make the model class - class Message(models.Model): - # We assume an autoincrementing PK for message order - channel = models.CharField(max_length=200, db_index=True) - content = models.TextField() - expiry = models.DateTimeField(db_index=True) - - class Meta: - apps = Apps() - app_label = "channels" - db_table = "django_channels_channel" - # Ensure its table exists - if Message._meta.db_table not in self.connection.introspection.table_names(self.connection.cursor()): - with self.connection.schema_editor() as editor: - editor.create_model(Message) - return Message - - @cached_property - def group_model(self): - """ - Initialises a new model to store groups; not done as part of a - models.py as we don't want to make it for most installs. - """ - # Make the model class - class Group(models.Model): - group = models.CharField(max_length=200) - channel = models.CharField(max_length=200) - created = models.DateTimeField(db_index=True, auto_now_add=True) - - class Meta: - apps = Apps() - app_label = "channels" - db_table = "django_channels_group" - unique_together = [["group", "channel"]] - # Ensure its table exists with the right schema - if Group._meta.db_table not in self.connection.introspection.table_names(self.connection.cursor()): - with self.connection.schema_editor() as editor: - editor.create_model(Group) - return Group - - def _clean_expired(self): - """ - Cleans out expired groups and messages. - """ - # Include a 1-second grace period for clock sync drift - target = now() - datetime.timedelta(seconds=1) - # First, go through old messages and pick out channels that got expired - old_messages = self.channel_model.objects.filter(expiry__lt=target) - channels_to_ungroup = old_messages.values_list("channel", flat=True).distinct() - old_messages.delete() - # Now, remove channel membership from channels that expired and ones that just expired - self.group_model.objects.filter( - models.Q(channel__in=channels_to_ungroup) | - models.Q(created__lte=target - datetime.timedelta(seconds=self.group_expiry)) - ).delete() - - def __str__(self): - return "%s(alias=%s)" % (self.__class__.__name__, self.connection.alias) - - -class AsgiJsonEncoder(json.JSONEncoder): - """ - Special encoder that transforms bytestrings into unicode strings - prefixed with u+ffff - """ - - def transform(self, o): - if isinstance(o, (list, tuple)): - return [self.transform(x) for x in o] - elif isinstance(o, dict): - return { - self.transform(k): self.transform(v) - for k, v in o.items() - } - elif isinstance(o, six.binary_type): - return u"\uffff" + base64.b64encode(o).decode("ascii") - else: - return o - - def encode(self, o): - return super(AsgiJsonEncoder, self).encode(self.transform(o)) - - -class AsgiJsonDecoder(json.JSONDecoder): - """ - Special encoder that transforms bytestrings into unicode strings - prefixed with u+ffff - """ - - def transform(self, o): - if isinstance(o, (list, tuple)): - return [self.transform(x) for x in o] - elif isinstance(o, dict): - return { - self.transform(k): self.transform(v) - for k, v in o.items() - } - elif isinstance(o, six.text_type) and o and o[0] == u"\uffff": - return base64.b64decode(o[1:].encode("ascii")) - else: - return o - - def decode(self, o): - return self.transform(super(AsgiJsonDecoder, self).decode(o)) diff --git a/channels/tests/test_database_layer.py b/channels/tests/test_database_layer.py deleted file mode 100644 index b9a11c0..0000000 --- a/channels/tests/test_database_layer.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals -from asgiref.conformance import ConformanceTestCase -from channels.database_layer import DatabaseChannelLayer - - -class DatabaseLayerTests(ConformanceTestCase): - channel_layer = DatabaseChannelLayer(expiry=1, group_expiry=3) - expiry_delay = 2.1 diff --git a/docs/backends.rst b/docs/backends.rst index 3935058..724d2b2 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -55,24 +55,23 @@ settings. Any misconfigured interface server or worker will drop some or all messages. -Database --------- +IPC +--- -The database layer is intended as a short-term solution for people who can't -use a more production-ready layer (for example, Redis), but still want something -that will work cross-process. It has poor performance, and is only -recommended for development or extremely small deployments. +The IPC backend uses POSIX shared memory segments and semaphores in order to +allow different processes on the same machine to communicate with each other. -This layer is included with Channels; just set your ``BACKEND`` to -``channels.database_layer.DatabaseChannelLayer``, and it will use the -default Django database alias to store messages. You can change the alias -by setting ``CONFIG`` to ``{'alias': 'aliasname'}``. +As it uses shared memory, it does not require any additional servers running +to get working, and is quicker than any network-based channel layer. However, +it can only run between processes on the same machine. .. warning:: - The database channel layer is NOT fast, and performs especially poorly at - latency and throughput. We recommend its use only as a last resort, and only - on a database with good transaction support (e.g. Postgres), or you may - get errors with multiple message delivery. + The IPC layer only communicates between processes on the same machine, + and while you might initially be tempted to run a cluster of machines all + with their own IPC-based set of processes, this will result in groups not + working properly; events sent to a group will only go to those channels + that joined the group on the same machine. This backend is for + single-machine deployments only. In-memory diff --git a/docs/concepts.rst b/docs/concepts.rst index 64d9cad..9830b35 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -91,7 +91,8 @@ single process tied to a WSGI server, Django runs in three separate layers: cover this later. * The channel backend, which is a combination of pluggable Python code and - a datastore (a database, or Redis) responsible for transporting messages. + a datastore (e.g. Redis, or a shared memory segment) responsible for + transporting messages. * The workers, that listen on all relevant channels and run consumer code when a message is ready. diff --git a/docs/deploying.rst b/docs/deploying.rst index 13d31d7..c3947dd 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -1,8 +1,9 @@ Deploying ========= -Deploying applications using Channels requires a few more steps than a normal -Django WSGI application, but it's not very many. +Deploying applications using channels requires a few more steps than a normal +Django WSGI application, but you have a couple of options as to how to deploy +it and how much of your traffic you wish to route through the channel layers. Firstly, remember that it's an entirely optional part of Django. If you leave a project with the default settings (no ``CHANNEL_LAYERS``), @@ -14,15 +15,27 @@ When you want to enable channels in production, you need to do three things: * Run worker servers * Run interface servers +You can set things up in one of two ways; either route all traffic through +a :ref:`HTTP/WebSocket interface server `, removing the need +to run a WSGI server at all; or, just route WebSockets and long-poll +HTTP connections to the interface server, and :ref:`leave other pages served +by a standard WSGI server `. + +Routing all traffic through the interface server lets you have WebSockets and +long-polling coexist in the same URL tree with no configuration; if you split +the traffic up, you'll need to configure a webserver or layer 7 loadbalancer +in front of the two servers to route requests to the correct place based on +path or domain. Both methods are covered below. + Setting up a channel backend ---------------------------- The first step is to set up a channel backend. If you followed the -:doc:`getting-started` guide, you will have ended up using the database -backend, which is great for getting started quickly in development but totally -unsuitable for production use; it will hammer your database with lots of -queries as it polls for new messages. +:doc:`getting-started` guide, you will have ended up using the in-memory +backend, which is useful for ``runserver``, but as it only works inside the +same process, useless for actually running separate worker and interface +servers. Instead, take a look at the list of :doc:`backends`, and choose one that fits your requirements (additionally, you could use a third-party pluggable @@ -48,9 +61,22 @@ To use the Redis backend you have to install it:: pip install -U asgi_redis +Some backends, though, don't require an extra server, like the IPC backend, +which works between processes on the same machine but not over the network +(it's available in the ``asgi_ipc`` package):: -Make sure the same settings file is used across all your workers, interfaces -and WSGI apps; without it, they won't be able to talk to each other and things + CHANNEL_LAYERS = { + "default": { + "BACKEND": "asgi_ipc.IPCChannelLayer", + "ROUTING": "my_project.routing.channel_routing", + "CONFIG": { + "prefix": "mysite", + }, + }, + } + +Make sure the same settings file is used across all your workers and interface +servers; without it, they won't be able to talk to each other and things will just fail to work. @@ -61,7 +87,7 @@ Because the work of running consumers is decoupled from the work of talking to HTTP, WebSocket and other client connections, you need to run a cluster of "worker servers" to do all the processing. -Each server is single-threaded, so it's recommended you run around one per +Each server is single-threaded, so it's recommended you run around one or two per core on each machine; it's safe to run as many concurrent workers on the same machine as you like, as they don't open any ports (all they do is talk to the channel backend). @@ -77,7 +103,7 @@ and forward them to stderr. Make sure you keep an eye on how busy your workers are; if they get overloaded, requests will take longer and longer to return as the messages queue up -(until the expiry limit is reached, at which point HTTP connections will +(until the expiry or capacity limit is reached, at which point HTTP connections will start dropping). In a more complex project, you won't want all your channels being served by the @@ -104,23 +130,23 @@ The final piece of the puzzle is the "interface servers", the processes that do the work of taking incoming requests and loading them into the channels system. -You can just keep running your Django code as a WSGI app if you like, behind -something like uwsgi or gunicorn; this won't let you support WebSockets, though. -Still, if you want to use a WSGI server and have it talk to a worker server -cluster on the backend, see :ref:`wsgi-to-asgi`. - If you want to support WebSockets, long-poll HTTP requests and other Channels features, you'll need to run a native ASGI interface server, as the WSGI specification has no support for running these kinds of requests concurrently. We ship with an interface server that we recommend you use called `Daphne `_; it supports WebSockets, long-poll HTTP requests, HTTP/2 *(soon)* and performs quite well. -Of course, any ASGI-compliant server will work! -Notably, Daphne has a nice feature where it supports all of these protocols on -the same port and on all paths; it auto-negotiates between HTTP and WebSocket, +You can just keep running your Django code as a WSGI app if you like, behind +something like uwsgi or gunicorn; this won't let you support WebSockets, though, +so you'll need to run a separate interface server to terminate those connections +and configure routing in front of your interface and WSGI servers to route +requests appropriately. + +If you use Daphne for all traffic, it auto-negotiates between HTTP and WebSocket, so there's no need to have your WebSockets on a separate port or path (and -they'll be able to share cookies with your normal view code). +they'll be able to share cookies with your normal view code, which isn't +possible if you separate by domain rather than path). To run Daphne, it just needs to be supplied with a channel backend, in much the same way a WSGI server needs to be given an application. @@ -144,7 +170,7 @@ like supervisord to ensure it is re-run if it exits unexpectedly. If you only run Daphne and no workers, all of your page requests will seem to hang forever; that's because Daphne doesn't have any worker servers to handle the request and it's waiting for one to appear (while ``runserver`` also uses -Daphne, it launches a worker thread along with it in the same process). In this +Daphne, it launches worker threads along with it in the same process). In this scenario, it will eventually time out and give you a 503 error after 2 minutes; you can configure how long it waits with the ``--http-timeout`` command line argument. @@ -164,42 +190,92 @@ workers. As long as the new code is session-compatible, you can even do staged rollouts to make sure workers on new code aren't experiencing high error rates. There's no need to restart the WSGI or WebSocket interface servers unless -you've upgraded the interface server itself or changed any Django settings; -none of your code is used by them, and all middleware and code that can +you've upgraded the interface server itself or changed the ``CHANNEL_LAYER`` +setting; none of your code is used by them, and all middleware and code that can customize requests is run on the consumers. You can even use different Python versions for the interface servers and the workers; the ASGI protocol that channel layers communicate over -is designed to be very portable and network-transparent. +is designed to be portable across all Python versions. -.. _wsgi-to-asgi: +.. _asgi-alone: -Running ASGI under WSGI ------------------------ +Running just ASGI +----------------- -ASGI is a relatively new specification, and so it's backwards compatible with -WSGI servers with an adapter layer. You won't get WebSocket support this way - -WSGI doesn't support WebSockets - but you can run a separate ASGI server to -handle WebSockets if you want. +If you are just running Daphne to serve all traffic, then the configuration +above is enough where you can just expose it to the Internet and it'll serve +whatever kind of request comes in; for a small site, just the one Daphne +instance and four or five workers is likely enough. -The ``asgiref`` package contains the adapter; all you need to do is put this -in your Django project's ``wsgi.py`` to declare a new WSGI application object -that backs onto ASGI underneath:: +However, larger sites will need to deploy things at a slightly larger scale, +and how you scale things up is different from WSGI; see :ref:`scaling-up`. - import os - from asgiref.wsgi import WsgiToAsgiAdapter - from channels.asgi import get_channel_layer - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_test.settings") - channel_layer = get_channel_layer() - application = WsgiToAsgiAdapter(channel_layer) +.. _wsgi-with-asgi: -While this removes WebSocket support through the same port that HTTP is served -on, it still lets you use other channels features such as background tasks or -alternative interface servers (that would let you write consumers against -incoming emails or IRC messages). +Running ASGI alongside WSGI +--------------------------- -You can also use this method to serve HTTP through your existing stack -and run Daphne on a separate port or domain to receive only WebSocket -connections. +ASGI and its canonical interface server Daphne are both relatively new, +and so you may not wish to run all your traffic through it yet (or you may +be using specialized features of your existing WSGI server). + +If that's the case, that's fine; you can run Daphne and a WSGI server alongside +each other, and only have Daphne serve the requests you need it to (usually +WebSocket and long-poll HTTP requests, as these do not fit into the WSGI model). + +To do this, just set up your Daphne to serve as we discussed above, and then +configure your load-balancer or front HTTP server process to dispatch requests +to the correct server - based on either path, domain, or if +you can, the Upgrade header. + +Dispatching based on path or domain means you'll need to design your WebSocket +URLs carefully so you can always tell how to route them at the load-balancer +level; the ideal thing is to be able to look for the ``Upgrade: WebSocket`` +header and distinguish connections by this, but not all software supports this +and it doesn't help route long-poll HTTP connections at all. + +You could also invert this model, and have all connections go to Daphne by +default and selectively route some back to the WSGI server, if you have +particular URLs or domains you want to use that server on. + + +.. _scaling-up: + +Scaling Up +---------- + +Scaling up a deployment containing channels (and thus running ASGI) is a little +different to scaling a WSGI deployment. + +The fundamental difference is that the group mechanic requires all servers serving +the same site to be able to see each other; if you separate the site up and run +it in a few, large clusters, messages to groups will only deliver to WebSockets +connected to the same cluster. For some site designs this will be fine, and if +you think you can live with this and design around it (which means never +designing anything around global notifications or events), this may be a good +way to go. + +For most projects, you'll need to run a single channel layer at scale in order +to achieve proper group delivery. Different backends will scale up differently, +but the Redis backend can use multiple Redis servers and spread the load +across them using sharding based on consistent hashing. + +The key to a channel layer knowing how to scale a channel's delivery is if it +contains the ``!`` character or not, which signifies a single-reader channel. +Single-reader channels are only ever connected to by a single process, and so +in the Redis case are stored on a single, predictable shard. Other channels +are assumed to have many workers trying to read them, and so messages for +these can be evenly divided across all shards. + +Django channels are still relatively new, and so it's likely that we don't yet +know the full story about how to scale things up; we run large load tests to +try and refine and improve large-project scaling, but it's no substitute for +actual traffic. If you're running channels at scale, you're encouraged to +send feedback to the Django team and work with us to hone the design and +performance of the channel layer backends, or you're free to make your own; +the ASGI specification is comprehensive and comes with a conformance test +suite, which should aid in any modification of existing backends or development +of new ones. diff --git a/docs/index.rst b/docs/index.rst index 4ad112e..92e06e4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,7 +29,6 @@ Contents: installation getting-started deploying - scaling backends testing cross-compat diff --git a/docs/scaling.rst b/docs/scaling.rst deleted file mode 100755 index 71cfd98..0000000 --- a/docs/scaling.rst +++ /dev/null @@ -1,37 +0,0 @@ -Scaling -======= - -Of course, one of the downsides of introducing a channel layer to Django is -that it's something else that must scale. Scaling traditional Django as a -WSGI application is easy - you just add more servers and a loadbalancer. Your -database is likely to be the thing that stopped scaling before, and there's -a relatively large amount of knowledge about tackling that problem. - -By comparison, there's not as much knowledge about scaling something like this -(though as it is very similar to a task queue, we have some work to build from). -In particular, the fact that messages are at-most-once - we do not guarantee -delivery, in the same way a webserver doesn't guarantee a response - means -we can loosen a lot of restrictions that slow down more traditional task queues. - -In addition, because channels can only have single consumers and they're handled -by a fleet of workers all running the same code, we could easily split out -incoming work by sharding into separate clusters of channel backends -and worker servers - any cluster can handle any request, so we can just -loadbalance over them. - -Of course, that doesn't work for interface servers, where only a single -particular server is listening to each response channel - if we broke things -into clusters, it might end up that a response is sent on a different cluster -to the one that the interface server is listening on. - -That's why Channels labels any *response channel* with a leading ``!``, letting -you know that only one server is listening for it, and thus letting you scale -and shard the two different types of channels accordingly (for more on -the difference, see :ref:`channel-types`). - -This is the underlying theory behind Channels' sharding model - normal channels -are sent to random Redis servers, while response channels are sent to a -predictable server that both the interface server and worker can derive. - -Currently, sharding is implemented as part of the Redis backend only; -see the :doc:`backend documentation ` for more information. diff --git a/patchinator.py b/patchinator.py index 4cbf44b..f672924 100644 --- a/patchinator.py +++ b/patchinator.py @@ -170,9 +170,6 @@ class Patchinator(object): FileMap( "channels/channel.py", "django/channels/channel.py", python_transforms, ), - FileMap( - "channels/database_layer.py", "django/channels/database_layer.py", python_transforms, - ), FileMap( "channels/exceptions.py", "django/channels/exceptions.py", python_transforms, ), @@ -209,9 +206,6 @@ class Patchinator(object): NewFile( "tests/channels_tests/__init__.py", ), - FileMap( - "channels/tests/test_database_layer.py", "tests/channels_tests/test_database_layer.py", python_transforms, - ), FileMap( "channels/tests/test_handler.py", "tests/channels_tests/test_handler.py", python_transforms, ), @@ -240,9 +234,6 @@ class Patchinator(object): FileMap( "docs/reference.rst", "docs/ref/channels/api.txt", docs_transforms, ), - FileMap( - "docs/scaling.rst", "docs/topics/channels/scaling.txt", docs_transforms, - ), FileMap( "docs/testing.rst", "docs/topics/channels/testing.txt", docs_transforms, ), From 175d13c28be97351cb3b12c9208172b7572e4961 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 7 May 2016 10:53:05 -0700 Subject: [PATCH 354/746] Fix up doc links to removed document --- docs/concepts.rst | 2 +- docs/getting-started.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 9830b35..2961611 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -165,7 +165,7 @@ and be less than 200 characters long. It's optional for a backend implementation to understand this - after all, it's only important at scale, where you want to shard the two types differently — but it's present nonetheless. For more on scaling, and how to handle channel -types if you're writing a backend or interface server, read :doc:`scaling`. +types if you're writing a backend or interface server, see :ref:`scaling-up`. Groups ------ diff --git a/docs/getting-started.rst b/docs/getting-started.rst index aa83555..0741d76 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -696,5 +696,5 @@ systems easily integrate with WebSockets. We recommend you read through the rest of the reference documentation to see more about what you can do with channels; in particular, you may want to look at -our :doc:`deploying` and :doc:`scaling` resources to get an idea of how to +our :doc:`deploying` documentation to get an idea of how to design and run apps in production environments. From 6e40fba47caded5c116fd2edbd880efb8a12ef9e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 7 May 2016 13:09:12 -0700 Subject: [PATCH 355/746] Releasing version 0.13.0 --- CHANGELOG.txt | 9 +++++++++ channels/__init__.py | 2 +- setup.py | 4 ++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index a0fb2fb..6f12a86 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,12 @@ +0.13.0 (2016-05-07) +------------------- + +* Backpressure is now implemented, meaning responses will pause sending if + the client does not read them fast enough. + +* DatabaseChannelLayer has been removed; it was not sensible. + + 0.12.0 (2016-04-26) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 80b4b5d..dd7e75a 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.12.0" +__version__ = "0.13.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/setup.py b/setup.py index 5664ffd..9920e7f 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( include_package_data=True, install_requires=[ 'Django>=1.8', - 'asgiref>=0.12', - 'daphne>=0.11', + 'asgiref>=0.13', + 'daphne>=0.12', ] ) From 619aed9be289c0f3d927027a28a0e500055d2538 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 May 2016 11:09:36 -0700 Subject: [PATCH 356/746] Elaborate a bit more on deployment on PaaSs --- docs/deploying.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/deploying.rst b/docs/deploying.rst index c3947dd..77f11f9 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -242,6 +242,25 @@ default and selectively route some back to the WSGI server, if you have particular URLs or domains you want to use that server on. +Running on a PaaS +----------------- + +To run Django with channels enabled on a Platform-as-a-Service (PaaS), you will +need to ensure that your PaaS allows you to run multiple processes at different +scaling levels; one group will be running Daphne, as a pure Python application +(not a WSGI application), and the other should be running ``runworker``. + +The PaaS will also either have to provide either its own Redis service or +a third process type that lets you run Redis yourself to use the cross-network +channel backend; both interface and worker processes need to be able to see +Redis, but not each other. + +If you are only allowed one running process type, it's possible you could +combine both interface server and worker into one process using threading +and the in-memory backend; however, this is not recommended for production +use as you cannot scale up past a single node without groups failing to work. + + .. _scaling-up: Scaling Up From 9505906b4254144749445126b6a398661bd1063c Mon Sep 17 00:00:00 2001 From: Krukov D Date: Sun, 8 May 2016 22:21:58 +0300 Subject: [PATCH 357/746] Fix re-inserting, with tests (#146) * Stopping re-inserting at first success * Added a few tests for worker running * Coping routes in channels layers at the ChannelTestCase * Remake worker test with less mocking --- channels/tests/base.py | 2 +- channels/tests/test_worker.py | 62 +++++++++++++++++++++++++++++++++-- channels/worker.py | 2 ++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/channels/tests/base.py b/channels/tests/base.py index 09ce66f..95bd9af 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -31,7 +31,7 @@ class ChannelTestCase(TestCase): ChannelLayerWrapper( InMemoryChannelLayer(), alias, - channel_layers[alias].routing, + channel_layers[alias].routing[:], ) ) diff --git a/channels/tests/test_worker.py b/channels/tests/test_worker.py index 53f8d40..5cff5b7 100644 --- a/channels/tests/test_worker.py +++ b/channels/tests/test_worker.py @@ -1,10 +1,32 @@ from __future__ import unicode_literals -from django.test import SimpleTestCase +try: + from unittest import mock +except ImportError: + import mock + +from channels import Channel, route, DEFAULT_CHANNEL_LAYER +from channels.asgi import channel_layers +from channels.tests import ChannelTestCase from channels.worker import Worker +from channels.exceptions import ConsumeLater -class WorkerTests(SimpleTestCase): +class PatchedWorker(Worker): + """Worker with specific numbers of loops""" + def get_termed(self): + if not self.__iters: + return True + self.__iters -= 1 + return False + + def set_termed(self, value): + self.__iters = value + + termed = property(get_termed, set_termed) + + +class WorkerTests(ChannelTestCase): """ Tests that the router's routing code works correctly. """ @@ -35,3 +57,39 @@ class WorkerTests(SimpleTestCase): worker.apply_channel_filters(["yes.1", "no.1", "maybe.2", "yes"]), ["yes.1"], ) + + def test_run_with_consume_later_error(self): + + # consumer with ConsumeLater error at first call + def _consumer(message, **kwargs): + _consumer._call_count = getattr(_consumer, '_call_count', 0) + 1 + if _consumer._call_count == 1: + raise ConsumeLater() + + Channel('test').send({'test': 'test'}) + channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + channel_layer.router.add_route(route('test', _consumer)) + old_send = channel_layer.send + channel_layer.send = mock.Mock(side_effect=old_send) # proxy 'send' for counting + + worker = PatchedWorker(channel_layer) + worker.termed = 2 # first loop with error, second with sending + + worker.run() + self.assertEqual(getattr(_consumer, '_call_count', None), 2) + self.assertEqual(channel_layer.send.call_count, 1) + + def test_normal_run(self): + consumer = mock.Mock() + Channel('test').send({'test': 'test'}) + channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + channel_layer.router.add_route(route('test', consumer)) + old_send = channel_layer.send + channel_layer.send = mock.Mock(side_effect=old_send) # proxy 'send' for counting + + worker = PatchedWorker(channel_layer) + worker.termed = 2 + + worker.run() + self.assertEqual(consumer.call_count, 1) + self.assertEqual(channel_layer.send.call_count, 0) diff --git a/channels/worker.py b/channels/worker.py index 6e65492..935ed80 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -122,5 +122,7 @@ class Worker(object): self.channel_layer.send(channel, content) except self.channel_layer.ChannelFull: time.sleep(0.05) + else: + break except: logger.exception("Error processing message with consumer %s:", name_that_thing(consumer)) From 2874350a6aaa515b8cfa503d71516b4b494ea690 Mon Sep 17 00:00:00 2001 From: Anatol Ulrich Date: Mon, 9 May 2016 18:51:51 +0200 Subject: [PATCH 358/746] pass arguments to get_consumer (#147) --- channels/management/commands/runserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 3fdd93b..26d99fe 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -36,7 +36,7 @@ class Command(RunserverCommand): # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] self.channel_layer.router.check_default( - http_consumer=self.get_consumer(), + http_consumer=self.get_consumer(*args, **options), ) # Run checks self.stdout.write("Performing system checks...\n\n") From c9497e74ddd7acfb341087b16445c8cbbe834f69 Mon Sep 17 00:00:00 2001 From: conor Date: Tue, 10 May 2016 17:07:39 +0100 Subject: [PATCH 359/746] Remove unused 'Group' import (#149) Was this supposed to be here? It isn't used until the next section. --- docs/getting-started.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 0741d76..303770c 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -100,7 +100,6 @@ serve HTTP requests from now on, which happens if you don't specify a consumer for ``http.request`` - and make this WebSocket consumer instead:: # In consumers.py - from channels import Group def ws_message(message): # ASGI WebSocket packet-received and send-packet message types From 363b5a09e917b7b3a97981516202e9b7ce248366 Mon Sep 17 00:00:00 2001 From: Sachin Rekhi Date: Thu, 12 May 2016 10:38:06 -0700 Subject: [PATCH 360/746] improve @enforce_ordering to leverage a wait channel to avoid spinlocks (#144) * improved @enforce_ordering to leverage a wait channel to avoid spinlocks * addressed pyflake issues * renamed wait channel to __wait__. * handled potential ChannelFull exception * updated sessions unit tests * updated enforce_ordering tests to reflect new approach of leveraging wait channels * addressed pyflake issues * more pyflake fixes * removed close_on_error handling on enforce_ordering since only worked on websockets --- channels/sessions.py | 40 ++++++++++----- channels/tests/test_sessions.py | 88 +++++++++++++++++++++++++++------ 2 files changed, 102 insertions(+), 26 deletions(-) diff --git a/channels/sessions.py b/channels/sessions.py index a749fcd..ea2e696 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -1,6 +1,5 @@ import functools import hashlib -import warnings from importlib import import_module from django.conf import settings @@ -75,7 +74,7 @@ def enforce_ordering(func=None, slight=False): Enforces either slight (order=0 comes first, everything else isn't ordered) or strict (all messages exactly ordered) ordering against a reply_channel. - Uses sessions to track ordering. + Uses sessions to track ordering and socket-specific wait channels for unordered messages. You cannot mix slight ordering and strict ordering on a channel; slight ordering does not write to the session after the first message to improve @@ -95,19 +94,38 @@ def enforce_ordering(func=None, slight=False): # See what the current next order should be next_order = message.channel_session.get("__channels_next_order", 0) if order == next_order or (slight and next_order > 0): - # Message is in right order. Maybe persist next one? + # Run consumer + func(message, *args, **kwargs) + # Mark next message order as available for running if order == 0 or not slight: message.channel_session["__channels_next_order"] = order + 1 - # Run consumer - return func(message, *args, **kwargs) + message.channel_session.save() + # Requeue any pending wait channel messages for this socket connection back onto it's original channel + while True: + wait_channel = "__wait__.%s" % message.reply_channel.name + channel, content = message.channel_layer.receive_many([wait_channel], block=False) + if channel: + original_channel = content.pop("original_channel") + try: + message.channel_layer.send(original_channel, content) + except message.channel_layer.ChannelFull: + raise message.channel_layer.ChannelFull( + "Cannot requeue pending __wait__ channel message " + + "back on to already full channel %s" % original_channel + ) + else: + break else: - # Bad ordering - warn if we're getting close to the limit - if getattr(message, "__doomed__", False): - warnings.warn( - "Enforce ordering consumer reached retry limit, message " + - "being dropped. Did you decorate all protocol consumers correctly?" + # Since out of order, enqueue message temporarily to wait channel for this socket connection + wait_channel = "__wait__.%s" % message.reply_channel.name + message.content["original_channel"] = message.channel.name + try: + message.channel_layer.send(wait_channel, message.content) + except message.channel_layer.ChannelFull: + raise message.channel_layer.ChannelFull( + "Cannot add unordered message to already " + + "full __wait__ channel for socket %s" % message.reply_channel.name ) - raise ConsumeLater() return inner if func is not None: return decorator(func) diff --git a/channels/tests/test_sessions.py b/channels/tests/test_sessions.py index e973f9d..da65f7a 100644 --- a/channels/tests/test_sessions.py +++ b/channels/tests/test_sessions.py @@ -2,10 +2,10 @@ from __future__ import unicode_literals from django.conf import settings from django.test import override_settings -from channels.exceptions import ConsumeLater from channels.message import Message from channels.sessions import channel_session, http_session, enforce_ordering, session_for_reply_channel from channels.tests import ChannelTestCase +from channels import DEFAULT_CHANNEL_LAYER, channel_layers @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") @@ -110,9 +110,21 @@ class SessionTests(ChannelTestCase): Tests that slight mode of enforce_ordering works """ # Construct messages to send - message0 = Message({"reply_channel": "test-reply-a", "order": 0}, None, None) - message1 = Message({"reply_channel": "test-reply-a", "order": 1}, None, None) - message2 = Message({"reply_channel": "test-reply-a", "order": 2}, None, None) + message0 = Message( + {"reply_channel": "test-reply-a", "order": 0}, + "websocket.connect", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) + message1 = Message( + {"reply_channel": "test-reply-a", "order": 1}, + "websocket.receive", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) + message2 = Message( + {"reply_channel": "test-reply-a", "order": 2}, + "websocket.receive", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) # Run them in an acceptable slight order @enforce_ordering(slight=True) @@ -123,29 +135,54 @@ class SessionTests(ChannelTestCase): inner(message2) inner(message1) + # Ensure wait channel is empty + wait_channel = "__wait__.%s" % "test-reply-a" + next_message = self.get_next_message(wait_channel) + self.assertEqual(next_message, None) + def test_enforce_ordering_slight_fail(self): """ Tests that slight mode of enforce_ordering fails on bad ordering """ # Construct messages to send - message2 = Message({"reply_channel": "test-reply-e", "order": 2}, None, None) + message2 = Message( + {"reply_channel": "test-reply-e", "order": 2}, + "websocket.receive", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) # Run them in an acceptable strict order @enforce_ordering(slight=True) def inner(message): pass - with self.assertRaises(ConsumeLater): - inner(message2) + inner(message2) + + # Ensure wait channel is not empty + wait_channel = "__wait__.%s" % "test-reply-e" + next_message = self.get_next_message(wait_channel) + self.assertNotEqual(next_message, None) def test_enforce_ordering_strict(self): """ Tests that strict mode of enforce_ordering works """ # Construct messages to send - message0 = Message({"reply_channel": "test-reply-b", "order": 0}, None, None) - message1 = Message({"reply_channel": "test-reply-b", "order": 1}, None, None) - message2 = Message({"reply_channel": "test-reply-b", "order": 2}, None, None) + message0 = Message( + {"reply_channel": "test-reply-b", "order": 0}, + "websocket.connect", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) + message1 = Message( + {"reply_channel": "test-reply-b", "order": 1}, + "websocket.receive", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) + message2 = Message( + {"reply_channel": "test-reply-b", "order": 2}, + "websocket.receive", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) # Run them in an acceptable strict order @enforce_ordering @@ -156,13 +193,26 @@ class SessionTests(ChannelTestCase): inner(message1) inner(message2) + # Ensure wait channel is empty + wait_channel = "__wait__.%s" % "test-reply-b" + next_message = self.get_next_message(wait_channel) + self.assertEqual(next_message, None) + def test_enforce_ordering_strict_fail(self): """ Tests that strict mode of enforce_ordering fails on bad ordering """ # Construct messages to send - message0 = Message({"reply_channel": "test-reply-c", "order": 0}, None, None) - message2 = Message({"reply_channel": "test-reply-c", "order": 2}, None, None) + message0 = Message( + {"reply_channel": "test-reply-c", "order": 0}, + "websocket.connect", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) + message2 = Message( + {"reply_channel": "test-reply-c", "order": 2}, + "websocket.receive", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) # Run them in an acceptable strict order @enforce_ordering @@ -170,14 +220,22 @@ class SessionTests(ChannelTestCase): pass inner(message0) - with self.assertRaises(ConsumeLater): - inner(message2) + inner(message2) + + # Ensure wait channel is not empty + wait_channel = "__wait__.%s" % "test-reply-c" + next_message = self.get_next_message(wait_channel) + self.assertNotEqual(next_message, None) def test_enforce_ordering_fail_no_order(self): """ Makes sure messages with no "order" key fail """ - message0 = Message({"reply_channel": "test-reply-d"}, None, None) + message0 = Message( + {"reply_channel": "test-reply-d"}, + None, + channel_layers[DEFAULT_CHANNEL_LAYER] + ) @enforce_ordering(slight=True) def inner(message): From 32320ec094e406948b6efedc6ad2a3a5414a3de6 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 13 May 2016 10:27:12 -0700 Subject: [PATCH 361/746] Releasing 0.13.1 --- CHANGELOG.txt | 10 ++++++++++ channels/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 6f12a86..c6bdc0c 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,13 @@ +0.13.1 (2016-05-13) +------------------- + +* enforce_ordering now queues future messages in a channel rather than + spinlocking worker processes to achieve delays. + +* ConsumeLater no longer duplicates messages when they're requeued below the + limit. + + 0.13.0 (2016-05-07) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index dd7e75a..ae21a00 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.13.0" +__version__ = "0.13.1" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 3fe99f061cfcd5eda0e57b8bcfe7d260a7b5f721 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 18 May 2016 09:58:26 -0700 Subject: [PATCH 362/746] Update spec a bit more --- docs/asgi.rst | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 630459c..947d1d6 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -111,10 +111,10 @@ this is necessary to achieve this restriction. In order to aid with scaling and network architecture, a distinction is made between channels that have multiple readers (such as the ``http.request`` channel that web applications would listen on from every -application worker process) and *single-reader channels* +application worker process) and *process-specific channels* (such as a ``http.response!ABCDEF`` channel tied to a client socket). -*Single-reader channel* names contain an exclamation mark +*Process-specific channel* names contain an exclamation mark (``!``) character in order to indicate to the channel layer that it may have to route the data for these channels differently to ensure it reaches the single process that needs it; these channels are nearly always tied to @@ -122,14 +122,16 @@ incoming connections from the outside world. The ``!`` is always preceded by the main channel name (e.g. ``http.response``) and followed by the per-client/random portion - channel layers can split on the ``!`` and use just the right hand part to route if they desire, or can ignore it if they don't -need to use different routing rules. +need to use different routing rules. Even if the right hand side contains +client routing information, it must still contain random parts too so that +each call to ``new_channel`` returns a new, unused name. Messages should expire after a set time sitting unread in a channel; the recommendation is one minute, though the best value depends on the channel layer and the way it is deployed. The maximum message size is 1MB; if more data than this needs to be transmitted -it should be chunked or placed onto its own single-reader channel (see how +it must be chunked or placed onto its own process-specific channel (see how HTTP request bodies are done, for example). All channel layers must support messages up to this size. @@ -218,7 +220,7 @@ on a periodic basis. How this garbage collection happens is not specified here, as it depends on the internal implementation of the channel layer. The recommended approach, -however, is when a message on a single-listener channel expires, the channel +however, is when a message on a process-specific channel expires, the channel layer should remove that channel from all groups it's currently a member of; this is deemed an acceptable indication that the channel's listener is gone. @@ -291,19 +293,20 @@ A *channel layer* must provide an object with these attributes or ``(channel, message)`` if a message is available. If ``block`` is True, then it will not return until after a built-in timeout or a message arrives; if ``block`` is false, it will always return immediately. It is perfectly - valid to ignore ``block`` and always return immediately. If ``block`` is True, - there must be a finite timeout before this returns ``(None, None)`` and that - timeout must be less than sixty seconds (preferably around five). + valid to ignore ``block`` and always return immediately, or after a delay; + ``block`` means that the call can take as long as it likes before returning + a message or nothing, not that it must block until it gets one. * ``new_channel(pattern)``, a callable that takes a unicode string pattern, and returns a new valid channel name that does not already exist, by - adding a single random unicode string after the ``!`` character in ``pattern``, + adding a unicode string after the ``!`` character in ``pattern``, and checking for existence of that name in the channel layer. The ``pattern`` - MUST end with ``!`` or this function must error. This is NOT called prior to - a message being sent on a channel, and should not be used for channel - initialization, and is also not guaranteed to be called by the same channel - client that then reads the messages, so you cannot put process identifiers in - it for routing. + MUST end with ``!`` or this function must error. This is not always called + prior to a message being sent on a channel, and cannot be used for + channel initialization. ``new_channel`` must be called on the same channel + layer that intends to read the channel with ``receive_many``; any other + channel layer instance may not receive messages on this channel due to + client-routing portions of the appended string. * ``MessageTooLarge``, the exception raised when a send operation fails because the encoded message is over the layer's size limit. @@ -514,10 +517,10 @@ Keys: Header names must be lowercased. * ``body``: Body of the request, as a byte string. Optional, defaults to ``""``. - If ``body_channel`` is set, treat as start of body and concatenate + If ``more_body`` is set, treat as start of body and concatenate on further chunks. -* ``body_channel``: Single-reader channel name that contains +* ``more_body``: Channel name that contains Request Body Chunk messages representing a large request body. Optional, defaults to ``None``. Chunks append to ``body`` if set. Presence of a channel indicates at least one Request Body Chunk message needs to be read, @@ -683,10 +686,11 @@ Keys: is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults to empty string. -* ``headers``: Dict of ``{name: value}``, where ``name`` is the lowercased - HTTP header name as byte string and ``value`` is the header value as a byte - string. If multiple headers with the same name are received, they should - be concatenated into a single header as per . +* ``headers``: List of ``[name, value]``, where ``name`` is the + header name as byte string and ``value`` is the header value as a byte + string. Order should be preserved from the original HTTP request; + duplicates are possible and must be preserved in the message as received. + Header names must be lowercased. * ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an @@ -964,7 +968,7 @@ limitation that they only use the following characters: * Hyphen ``-`` * Underscore ``_`` * Period ``.`` -* Exclamation mark ``!`` (only to deliniate single-reader channel names, +* Exclamation mark ``!`` (only to deliniate process-specific channel names, and only one per name) From d1141e47aa6ccf888e51004e5e421478e38b8b0f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 18 May 2016 11:34:48 -0700 Subject: [PATCH 363/746] Move email and UDP into their own spec docs --- docs/asgi.rst | 136 ++++---------------------------------------- docs/asgi/email.rst | 73 ++++++++++++++++++++++++ docs/asgi/udp.rst | 48 ++++++++++++++++ 3 files changed, 133 insertions(+), 124 deletions(-) create mode 100644 docs/asgi/email.rst create mode 100644 docs/asgi/udp.rst diff --git a/docs/asgi.rst b/docs/asgi.rst index 947d1d6..c37fadc 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -144,7 +144,7 @@ ASGI messages represent two main things - internal application events uploaded videos), and protocol events to/from connected clients. As such, this specification outlines encodings to and from ASGI messages -for three common protocols (HTTP, WebSocket and raw UDP); this allows any ASGI +for HTTP and WebSocket; this allows any ASGI web server to talk to any ASGI web application, as well as servers and applications for any other protocol with a common specification. It is recommended that if other protocols become commonplace they should gain @@ -768,129 +768,6 @@ A maximum of one of ``bytes`` or ``text`` may be provided. If both are provided, the protocol server should ignore the message entirely. -Email ------ - -Represents emails sent or received, likely over the SMTP protocol though that -is not directly specified here (a protocol server could in theory deliver -or receive email over HTTP to some external service, for example). Generally -adheres to RFC 5322 as much as possible. - -As emails have no concept of a session and there's no trustable socket or -author model, the send and receive channels are both multi-listener, and -there is no ``reply_channel`` on any message type. If you want to persist -data across different email receive consumers, you should decide what part -of the message to use for an identifier (from address? to address? subject? -thread id?) and provide the persistence yourself. - -The protocol server should handle encoding of headers by itself, understanding -RFC 1342 format headers and decoding them into unicode upon receive, and -encoding outgoing emails similarly (preferably using UTF-8). - - -Receive -''''''' - -Sent when an email is received. - -Channel: ``email.receive`` - -Keys: - -* ``from``: Unicode string specifying the return-path of the email as specified - in the SMTP envelope. Will be ``None`` if no return path was provided. - -* ``to``: List of unicode strings specifying the recipients requested in the - SMTP envelope using ``RCPT TO`` commands. Will always contain at least one - value. - -* ``headers``: Dictionary of unicode string keys and unicode string values, - containing all headers, including ``subject``. Header names are all forced - to lower case. Header values are decoded from RFC 1342 if needed. - -* ``content``: Contains a content object (see section below) representing the - body of the message. - -Note that ``from`` and ``to`` are extracted from the SMTP envelope, and not -from the headers inside the message; if you wish to get the header values, -you should use ``headers['from']`` and ``headers['to']``; they may be different. - - -Send -'''' - -Sends an email out via whatever transport - - -Content objects -''''''''''''''' - -Used in both send and receive to represent the tree structure of a MIME -multipart message tree. - -A content object is always a dict, containing at least the key: - -* ``content-type``: The unicode string of the content type for this section. - -Multipart content objects also have: - -* ``parts``: A list of content objects contained inside this multipart - -Any other type of object has: - -* ``body``: Byte string content of this part, decoded from any - ``Content-Transfer-Encoding`` if one was specified as a MIME header. - - -UDP ---- - -Raw UDP is included here as it is a datagram-based, unordered and unreliable -protocol, which neatly maps to the underlying message abstraction. It is not -expected that many applications would use the low-level protocol, but it may -be useful for some. - -While it might seem odd to have reply channels for UDP as it is a stateless -protocol, replies need to come from the same server as the messages were -sent to, so the reply channel here ensures that reply packets from an ASGI -stack do not come from a different protocol server to the one you sent the -initial packet to. - - -Receive -''''''' - -Sent when a UDP datagram is received. - -Channel: ``udp.receive`` - -Keys: - -* ``reply_channel``: Channel name for sending data, starts with ``udp.send!`` - -* ``data``: Byte string of UDP datagram payload. - -* ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the - remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an - integer. - -* ``server``: List of ``[host, port]`` where ``host`` is the listening address - for this server as a unicode string, and ``port`` is the integer listening port. - Optional, defaults to ``None``. - - -Send -'''' - -Sent to send out a UDP datagram to a client. - -Channel: ``udp.send!`` - -Keys: - -* ``data``: Byte string of UDP datagram payload. - - Protocol Format Guidelines -------------------------- @@ -1056,3 +933,14 @@ Copyright ========= This document has been placed in the public domain. + + +Protocol Definitions +==================== + + +.. toctree:: + :maxdepth: 1 + + /asgi/email + /asgi/udp diff --git a/docs/asgi/email.rst b/docs/asgi/email.rst new file mode 100644 index 0000000..08ad5ca --- /dev/null +++ b/docs/asgi/email.rst @@ -0,0 +1,73 @@ +====================================== +Email ASGI Message Format (Draft Spec) +====================================== + +Represents emails sent or received, likely over the SMTP protocol though that +is not directly specified here (a protocol server could in theory deliver +or receive email over HTTP to some external service, for example). Generally +adheres to RFC 5322 as much as possible. + +As emails have no concept of a session and there's no trustable socket or +author model, the send and receive channels are both multi-listener, and +there is no ``reply_channel`` on any message type. If you want to persist +data across different email receive consumers, you should decide what part +of the message to use for an identifier (from address? to address? subject? +thread id?) and provide the persistence yourself. + +The protocol server should handle encoding of headers by itself, understanding +RFC 1342 format headers and decoding them into unicode upon receive, and +encoding outgoing emails similarly (preferably using UTF-8). + + +Receive +''''''' + +Sent when an email is received. + +Channel: ``email.receive`` + +Keys: + +* ``from``: Unicode string specifying the return-path of the email as specified + in the SMTP envelope. Will be ``None`` if no return path was provided. + +* ``to``: List of unicode strings specifying the recipients requested in the + SMTP envelope using ``RCPT TO`` commands. Will always contain at least one + value. + +* ``headers``: Dictionary of unicode string keys and unicode string values, + containing all headers, including ``subject``. Header names are all forced + to lower case. Header values are decoded from RFC 1342 if needed. + +* ``content``: Contains a content object (see section below) representing the + body of the message. + +Note that ``from`` and ``to`` are extracted from the SMTP envelope, and not +from the headers inside the message; if you wish to get the header values, +you should use ``headers['from']`` and ``headers['to']``; they may be different. + + +Send +'''' + +Sends an email out via whatever transport + + +Content objects +''''''''''''''' + +Used in both send and receive to represent the tree structure of a MIME +multipart message tree. + +A content object is always a dict, containing at least the key: + +* ``content-type``: The unicode string of the content type for this section. + +Multipart content objects also have: + +* ``parts``: A list of content objects contained inside this multipart + +Any other type of object has: + +* ``body``: Byte string content of this part, decoded from any + ``Content-Transfer-Encoding`` if one was specified as a MIME header. diff --git a/docs/asgi/udp.rst b/docs/asgi/udp.rst new file mode 100644 index 0000000..38c35a9 --- /dev/null +++ b/docs/asgi/udp.rst @@ -0,0 +1,48 @@ +==================================== +UDP ASGI Message Format (Draft Spec) +==================================== + +Raw UDP is specified here as it is a datagram-based, unordered and unreliable +protocol, which neatly maps to the underlying message abstraction. It is not +expected that many applications would use the low-level protocol, but it may +be useful for some. + +While it might seem odd to have reply channels for UDP as it is a stateless +protocol, replies need to come from the same server as the messages were +sent to, so the reply channel here ensures that reply packets from an ASGI +stack do not come from a different protocol server to the one you sent the +initial packet to. + + +Receive +''''''' + +Sent when a UDP datagram is received. + +Channel: ``udp.receive`` + +Keys: + +* ``reply_channel``: Channel name for sending data, starts with ``udp.send!`` + +* ``data``: Byte string of UDP datagram payload. + +* ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the + remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an + integer. + +* ``server``: List of ``[host, port]`` where ``host`` is the listening address + for this server as a unicode string, and ``port`` is the integer listening port. + Optional, defaults to ``None``. + + +Send +'''' + +Sent to send out a UDP datagram to a client. + +Channel: ``udp.send!`` + +Keys: + +* ``data``: Byte string of UDP datagram payload. From 86a64781932fbd8c80e82eb76acd6f8b2e8b33db Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 19 May 2016 00:10:41 -0700 Subject: [PATCH 364/746] Add FAQ about sending messages from outside --- docs/cross-compat.rst | 8 ++++---- docs/faqs.rst | 27 ++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/docs/cross-compat.rst b/docs/cross-compat.rst index ab6541e..bfc73e5 100644 --- a/docs/cross-compat.rst +++ b/docs/cross-compat.rst @@ -1,8 +1,8 @@ Cross-Compatibility =================== -Channels is being released as both a third-party app for Django 1.8 and 1.9, -and being integrated into Django in 1.10. Both of these implementations are +Channels is being released as both a third-party app for Django 1.8 through 1.10, +and being integrated into Django in future. Both of these implementations will be very similar, and code for one will work on the other with minimal changes. The only difference between the two is the import paths. Mostly, where you @@ -19,8 +19,8 @@ Becomes:: from django.channels import Channel from django.channels.auth import channel_session_user -There are a few exceptions to this rule, where classes were moved to other parts -of Django in 1.10 that made more sense: +There are a few exceptions to this rule, where classes will be moved to other parts +of Django in that make more sense: * ``channels.tests.ChannelTestCase`` is found under ``django.test.channels.ChannelTestCase`` * ``channels.handler`` is moved to ``django.core.handlers.asgi`` diff --git a/docs/faqs.rst b/docs/faqs.rst index 08482c0..ec48c42 100755 --- a/docs/faqs.rst +++ b/docs/faqs.rst @@ -126,10 +126,31 @@ whatever you store in must be **network-transparent** - storing things in a global variable won't work outside of development. +How do I talk to Channels from my non-Django application? +--------------------------------------------------------- + +If you have an external server or script you want to talk to Channels, you have +a few choices: + +* If it's a Python program, and you've made an ``asgi.py`` file for your project + (see :doc:`deploying`), you can import the channel layer directly as + ``yourproject.asgi.channel_layer`` and call ``send()`` and ``receive_many()`` + on it directly. See the :doc:`ASGI spec ` for the API the channel layer + presents. + +* If you just need to send messages in when events happen, you can make a + management command that calls ``Channel("namehere").send({...})`` + so your external program can just call + ``manage.py send_custom_event`` (or similar) to send a message. Remember, you + can send onto channels from any code in your project. + +* If neither of these work, you'll have to communicate with Django over + HTTP, WebSocket, or another protocol that your project talks, as normal. + Are channels Python 2, 3 or 2+3? -------------------------------- -Django-channels and all of its dependencies are 2+3 (2.7, 3.4+). Compatibility may change with time. If in doubt, refer to the ``.travis.yml`` configuration file to see which Python versions that are included in CI testing. - -This includes Twisted, for which the used subsets of the library used by Daphne are all py3k ready. +Django-channels and all of its dependencies are compatible with Python 2.7, +3.3, and higher. This includes the parts of Twisted that some of the Channels +packages (like daphne) use. From 05c41e9ad64da1f989528bf42ef65f1b837af659 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Thu, 19 May 2016 21:45:25 +0300 Subject: [PATCH 365/746] More tests utils for happy users (#162) * Added Client abstraction * Added apply_routes decorator/contextmanager * Fix apply routes as decorator * Separated Http specific client and 'Simple' client * Remove Clients from ChannelTestCase * Added cookies and headers management * Fix wrong reverting * Fixs for code style * Added space before inline comment --- channels/tests/__init__.py | 3 +- channels/tests/base.py | 130 ++++++++++++++++++++++++++++++++ channels/tests/http.py | 147 +++++++++++++++++++++++++++++++++++++ 3 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 channels/tests/http.py diff --git a/channels/tests/__init__.py b/channels/tests/__init__.py index a43624d..36481f0 100644 --- a/channels/tests/__init__.py +++ b/channels/tests/__init__.py @@ -1 +1,2 @@ -from .base import ChannelTestCase # NOQA isort:skip +from .base import ChannelTestCase, Client, apply_routes # NOQA isort:skip +from .http import HttpClient # NOQA isort:skip diff --git a/channels/tests/base.py b/channels/tests/base.py index 95bd9af..dff1e97 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -1,5 +1,11 @@ +import copy +import random +import string +from functools import wraps + from django.test.testcases import TestCase from channels import DEFAULT_CHANNEL_LAYER +from channels.routing import Router, include from channels.asgi import channel_layers, ChannelLayerWrapper from channels.message import Message from asgiref.inmemory import ChannelLayer as InMemoryChannelLayer @@ -59,3 +65,127 @@ class ChannelTestCase(TestCase): else: return None return Message(content, recv_channel, channel_layers[alias]) + + +class Client(object): + """ + Channel client abstraction that provides easy methods for testing full live cycle of message in channels + with determined the reply channel + """ + + def __init__(self, alias=DEFAULT_CHANNEL_LAYER): + self.reply_channel = alias + ''.join([random.choice(string.ascii_letters) for _ in range(5)]) + self.alias = alias + + @property + def channel_layer(self): + """Channel layer as lazy property""" + return channel_layers[self.alias] + + def get_next_message(self, channel): + """ + Gets the next message that was sent to the channel during the test, + or None if no message is available. + """ + recv_channel, content = channel_layers[self.alias].receive_many([channel]) + if recv_channel is None: + return + return Message(content, recv_channel, channel_layers[self.alias]) + + def send(self, to, content={}): + """ + Send a message to a channel. + Adds reply_channel name to the message. + """ + content = copy.deepcopy(content) + content.setdefault('reply_channel', self.reply_channel) + self.channel_layer.send(to, content) + + def consume(self, channel): + """ + Get next message for channel name and run appointed consumer + """ + message = self.get_next_message(channel) + if message: + consumer, kwargs = self.channel_layer.router.match(message) + return consumer(message, **kwargs) + + def send_and_consume(self, channel, content={}): + """ + Reproduce full live cycle of the message + """ + self.send(channel, content) + return self.consume(channel) + + def receive(self): + """self.get_next_message(self.reply_channel) + Get content of next message for reply channel if message exists + """ + message = self.get_next_message(self.reply_channel) + if message: + return message.content + + +class apply_routes(object): + """ + Decorator/ContextManager for rewrite layers routes in context. + Helpful for testing group routes/consumers as isolated application + + The applying routes can be list of instances of Route or list of this lists + """ + + def __init__(self, routes, aliases=[DEFAULT_CHANNEL_LAYER]): + self._aliases = aliases + self.routes = routes + self._old_routing = {} + + def enter(self): + """ + Store old routes and apply new one + """ + for alias in self._aliases: + channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + self._old_routing[alias] = channel_layer.routing + if isinstance(self.routes, (list, tuple)): + if isinstance(self.routes[0], (list, tuple)): + routes = list(map(include, self.routes)) + else: + routes = self.routes + + channel_layer.routing = routes + channel_layer.router = Router(routes) + + def exit(self, exc_type=None, exc_val=None, exc_tb=None): + """ + Undoes rerouting + """ + for alias in self._aliases: + channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + channel_layer.routing = self._old_routing[alias] + channel_layer.router = Router(self._old_routing[alias]) + + __enter__ = enter + __exit__ = exit + + def __call__(self, test_func): + if isinstance(test_func, type): + old_setup = test_func.setUp + old_teardown = test_func.tearDown + + def new_setup(this): + self.enter() + old_setup(this) + + def new_teardown(this): + self.exit() + old_teardown(this) + + test_func.setUp = new_setup + test_func.tearDown = new_teardown + return test_func + else: + @wraps(test_func) + def inner(*args, **kwargs): + with self: + return test_func(*args, **kwargs) + return inner diff --git a/channels/tests/http.py b/channels/tests/http.py new file mode 100644 index 0000000..fa44dfd --- /dev/null +++ b/channels/tests/http.py @@ -0,0 +1,147 @@ + +import copy + +from django.apps import apps +from django.conf import settings + + +from ..asgi import channel_layers +from ..message import Message +from ..sessions import session_for_reply_channel +from .base import Client + + +class HttpClient(Client): + """ + Channel http/ws client abstraction that provides easy methods for testing full live cycle of message in channels + with determined reply channel, auth opportunity, cookies, headers and so on + """ + + def __init__(self, **kwargs): + super(HttpClient, self).__init__(**kwargs) + self._session = None + self._headers = {} + self._cookies = {} + + def set_cookie(self, key, value): + """ + Set cookie + """ + self._cookies[key] = value + + def set_header(self, key, value): + """ + Set header + """ + if key == 'cookie': + raise ValueError('Use set_cookie method for cookie header') + self._headers[key] = value + + def get_cookies(self): + """Return cookies""" + cookies = copy.copy(self._cookies) + if apps.is_installed('django.contrib.sessions'): + cookies[settings.SESSION_COOKIE_NAME] = self.session.session_key + return cookies + + @property + def headers(self): + headers = copy.deepcopy(self._headers) + headers.setdefault('cookie', _encoded_cookies(self.get_cookies())) + return headers + + @property + def session(self): + """Session as Lazy property: check that django.contrib.sessions is installed""" + if not apps.is_installed('django.contrib.sessions'): + raise EnvironmentError('Add django.contrib.sessions to the INSTALLED_APPS to use session') + if not self._session: + self._session = session_for_reply_channel(self.reply_channel) + return self._session + + @property + def channel_layer(self): + """Channel layer as lazy property""" + return channel_layers[self.alias] + + def get_next_message(self, channel): + """ + Gets the next message that was sent to the channel during the test, + or None if no message is available. + + If require is true, will fail the test if no message is received. + """ + recv_channel, content = channel_layers[self.alias].receive_many([channel]) + if recv_channel is None: + return + return Message(content, recv_channel, channel_layers[self.alias]) + + def send(self, to, content={}): + """ + Send a message to a channel. + Adds reply_channel name and channel_session to the message. + """ + content = copy.deepcopy(content) + content.setdefault('reply_channel', self.reply_channel) + content.setdefault('path', '/') + content.setdefault('headers', self.headers) + self.channel_layer.send(to, content) + + def consume(self, channel): + """ + Get next message for channel name and run appointed consumer + """ + message = self.get_next_message(channel) + if message: + consumer, kwargs = self.channel_layer.router.match(message) + return consumer(message, **kwargs) + + def send_and_consume(self, channel, content={}): + """ + Reproduce full live cycle of the message + """ + self.send(channel, content) + return self.consume(channel) + + def receive(self): + """ + Get content of next message for reply channel if message exists + """ + message = self.get_next_message(self.reply_channel) + if message: + return message.content + + def login(self, **credentials): + """ + Returns True if login is possible; False if the provided credentials + are incorrect, or the user is inactive, or if the sessions framework is + not available. + """ + from django.contrib.auth import authenticate + user = authenticate(**credentials) + if user and user.is_active and apps.is_installed('django.contrib.sessions'): + self._login(user) + return True + else: + return False + + def force_login(self, user, backend=None): + if backend is None: + backend = settings.AUTHENTICATION_BACKENDS[0] + user.backend = backend + self._login(user) + + def _login(self, user): + from django.contrib.auth import login + + # Fake http request + request = type('FakeRequest', (object, ), {'session': self.session, 'META': {}}) + login(request, user) + + # Save the session values. + self.session.save() + + +def _encoded_cookies(cookies): + """Encode dict of cookies to ascii string""" + return ('&'.join('{0}={1}'.format(k, v) for k, v in cookies.items())).encode("ascii") From 8827063bf2bbdb28c3147f5782a0c3cf7fdb9d1e Mon Sep 17 00:00:00 2001 From: Krukov D Date: Thu, 19 May 2016 22:19:39 +0300 Subject: [PATCH 366/746] Remove inherited methods and relative imports (#163) * Remove inherited methods from HttpClient * Using relative import in base of tests --- channels/tests/base.py | 8 ++++---- channels/tests/http.py | 44 ------------------------------------------ 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/channels/tests/base.py b/channels/tests/base.py index dff1e97..a6a3145 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -4,10 +4,10 @@ import string from functools import wraps from django.test.testcases import TestCase -from channels import DEFAULT_CHANNEL_LAYER -from channels.routing import Router, include -from channels.asgi import channel_layers, ChannelLayerWrapper -from channels.message import Message +from .. import DEFAULT_CHANNEL_LAYER +from ..routing import Router, include +from ..asgi import channel_layers, ChannelLayerWrapper +from ..message import Message from asgiref.inmemory import ChannelLayer as InMemoryChannelLayer diff --git a/channels/tests/http.py b/channels/tests/http.py index fa44dfd..4dd3840 100644 --- a/channels/tests/http.py +++ b/channels/tests/http.py @@ -4,9 +4,6 @@ import copy from django.apps import apps from django.conf import settings - -from ..asgi import channel_layers -from ..message import Message from ..sessions import session_for_reply_channel from .base import Client @@ -59,23 +56,6 @@ class HttpClient(Client): self._session = session_for_reply_channel(self.reply_channel) return self._session - @property - def channel_layer(self): - """Channel layer as lazy property""" - return channel_layers[self.alias] - - def get_next_message(self, channel): - """ - Gets the next message that was sent to the channel during the test, - or None if no message is available. - - If require is true, will fail the test if no message is received. - """ - recv_channel, content = channel_layers[self.alias].receive_many([channel]) - if recv_channel is None: - return - return Message(content, recv_channel, channel_layers[self.alias]) - def send(self, to, content={}): """ Send a message to a channel. @@ -87,30 +67,6 @@ class HttpClient(Client): content.setdefault('headers', self.headers) self.channel_layer.send(to, content) - def consume(self, channel): - """ - Get next message for channel name and run appointed consumer - """ - message = self.get_next_message(channel) - if message: - consumer, kwargs = self.channel_layer.router.match(message) - return consumer(message, **kwargs) - - def send_and_consume(self, channel, content={}): - """ - Reproduce full live cycle of the message - """ - self.send(channel, content) - return self.consume(channel) - - def receive(self): - """ - Get content of next message for reply channel if message exists - """ - message = self.get_next_message(self.reply_channel) - if message: - return message.content - def login(self, **credentials): """ Returns True if login is possible; False if the provided credentials From c89a6cc9b909e95c6734507ea12e672599e8d5ce Mon Sep 17 00:00:00 2001 From: pinguin999 Date: Tue, 24 May 2016 18:19:25 +0200 Subject: [PATCH 367/746] Add pip install comand for asgi_redis (#166) --- docs/getting-started.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 303770c..5c715c8 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -290,7 +290,11 @@ but all in one process)* Let's try out the Redis backend - Redis runs on pretty much every machine, and has a very small overhead, which makes it perfect for this kind of thing. Install -the ``asgi_redis`` package using ``pip``, and set up your channel layer like this:: +the ``asgi_redis`` package using ``pip``. :: + + pip install asgi_redis + +and set up your channel layer like this:: # In settings.py CHANNEL_LAYERS = { From cc9057e90c24540e7674dd329bcfb24ff7a01b6c Mon Sep 17 00:00:00 2001 From: pinguin999 Date: Tue, 24 May 2016 18:19:38 +0200 Subject: [PATCH 368/746] Split the two files into two code blocks (#167) --- docs/getting-started.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 5c715c8..2b24beb 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -60,6 +60,8 @@ Here's what that looks like:: "ROUTING": "myproject.routing.channel_routing", }, } +.. +:: # In routing.py from channels.routing import route From bfacee631903c6ca04b85f1e5228843f0b86fe65 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 25 May 2016 17:45:38 -0700 Subject: [PATCH 369/746] Add class-based consumers --- channels/__init__.py | 2 +- channels/generic/__init__.py | 1 + channels/generic/base.py | 40 +++++++++ channels/generic/websockets.py | 137 ++++++++++++++++++++++++++++++ channels/routing.py | 60 +++++++++---- channels/tests/test_routing.py | 48 ++++++++++- channels/utils.py | 4 +- channels/worker.py | 5 +- docs/generics.rst | 150 +++++++++++++++++++++++++++++++++ docs/index.rst | 2 + docs/routing.rst | 76 +++++++++++++++++ 11 files changed, 503 insertions(+), 22 deletions(-) create mode 100644 channels/generic/__init__.py create mode 100644 channels/generic/base.py create mode 100644 channels/generic/websockets.py create mode 100644 docs/generics.rst create mode 100644 docs/routing.rst diff --git a/channels/__init__.py b/channels/__init__.py index ae21a00..98ce119 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -6,6 +6,6 @@ DEFAULT_CHANNEL_LAYER = 'default' try: from .asgi import channel_layers # NOQA isort:skip from .channel import Channel, Group # NOQA isort:skip - from .routing import route, include # NOQA isort:skip + from .routing import route, route_class, include # NOQA isort:skip except ImportError: # No django installed, allow vars to be read pass diff --git a/channels/generic/__init__.py b/channels/generic/__init__.py new file mode 100644 index 0000000..3d08a87 --- /dev/null +++ b/channels/generic/__init__.py @@ -0,0 +1 @@ +from .base import BaseConsumer diff --git a/channels/generic/base.py b/channels/generic/base.py new file mode 100644 index 0000000..89c387c --- /dev/null +++ b/channels/generic/base.py @@ -0,0 +1,40 @@ +from __future__ import unicode_literals + + +class BaseConsumer(object): + """ + Base class-based consumer class. Provides the mechanisms to be a direct + routing object and a few other things. + + Class-based consumers should be used directly in routing with their + filters, like so:: + + routing = [ + JsonWebsocketConsumer(path=r"^/liveblog/(?P[^/]+)/"), + ] + """ + + method_mapping = {} + + def __init__(self, message, **kwargs): + """ + Constructor, called when a new message comes in (the consumer is + the uninstantiated class, so calling it creates it) + """ + self.message = message + self.dispatch(message, **kwargs) + + @classmethod + def channel_names(cls): + """ + Returns a list of channels this consumer will respond to, in our case + derived from the method_mapping class attribute. + """ + return set(cls.method_mapping.keys()) + + def dispatch(self, message, **kwargs): + """ + Called with the message and all keyword arguments; uses method_mapping + to choose the right method to call. + """ + return getattr(self, self.method_mapping[message.channel.name])(message, **kwargs) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py new file mode 100644 index 0000000..9faf75d --- /dev/null +++ b/channels/generic/websockets.py @@ -0,0 +1,137 @@ +import json + +from ..channel import Group +from ..sessions import enforce_ordering +from .base import BaseConsumer + + +class WebsocketConsumer(BaseConsumer): + """ + Base WebSocket consumer. Provides a general encapsulation for the + WebSocket handling model that other applications can build on. + """ + + # You shouldn't need to override this + method_mapping = { + "websocket.connect": "raw_connect", + "websocket.receive": "raw_receive", + "websocket.disconnect": "raw_disconnect", + } + + # Set one to True if you want the class to enforce ordering for you + slight_ordering = False + strict_ordering = False + + def dispatch(self, message, **kwargs): + """ + Pulls out the path onto an instance variable, and optionally + adds the ordering decorator. + """ + self.path = message['path'] + if self.strict_ordering: + return enforce_ordering(super(WebsocketConsumer, self).dispatch(message, **kwargs), slight=False) + elif self.slight_ordering: + return enforce_ordering(super(WebsocketConsumer, self).dispatch(message, **kwargs), slight=True) + else: + return super(WebsocketConsumer, self).dispatch(message, **kwargs) + + def connection_groups(self, **kwargs): + """ + Group(s) to make people join when they connect and leave when they + disconnect. Make sure to return a list/tuple, not a string! + """ + return [] + + def raw_connect(self, message, **kwargs): + """ + Called when a WebSocket connection is opened. Base level so you don't + need to call super() all the time. + """ + for group in self.connection_groups(**kwargs): + Group(group, channel_layer=message.channel_layer).add(message.channel) + self.connect(message, **kwargs) + + def connect(self, message, **kwargs): + """ + Called when a WebSocket connection is opened. + """ + pass + + def raw_receive(self, message, **kwargs): + """ + Called when a WebSocket frame is received. Decodes it and passes it + to receive(). + """ + if "text" in message: + self.receive(text=message['text'], **kwargs) + else: + self.receive(bytes=message['bytes'], **kwargs) + + def receive(self, text=None, bytes=None, **kwargs): + """ + Called with a decoded WebSocket frame. + """ + pass + + def send(self, text=None, bytes=None): + """ + Sends a reply back down the WebSocket + """ + if text is not None: + self.message.reply_channel.send({"text": text}) + elif bytes is not None: + self.message.reply_channel.send({"bytes": bytes}) + else: + raise ValueError("You must pass text or bytes") + + def group_send(self, name, text=None, bytes=None): + if text is not None: + Group(name, channel_layer=self.message.channel_layer).send({"text": text}) + elif bytes is not None: + Group(name, channel_layer=self.message.channel_layer).send({"bytes": bytes}) + else: + raise ValueError("You must pass text or bytes") + + def disconnect(self, message, **kwargs): + """ + Called when a WebSocket connection is closed. Base level so you don't + need to call super() all the time. + """ + for group in self.connection_groups(**kwargs): + Group(group, channel_layer=message.channel_layer).discard(message.channel) + self.disconnect(message, **kwargs) + + def disconnect(self, message, **kwargs): + """ + Called when a WebSocket connection is opened. + """ + pass + + +class JsonWebsocketConsumer(WebsocketConsumer): + """ + Variant of WebsocketConsumer that automatically JSON-encodes and decodes + messages as they come in and go out. Expects everything to be text; will + error on binary data. + """ + + def raw_receive(self, message, **kwargs): + if "text" in message: + self.receive(json.loads(message['text']), **kwargs) + else: + raise ValueError("No text section for incoming WebSocket frame!") + + def receive(self, content, **kwargs): + """ + Called with decoded JSON content. + """ + pass + + def send(self, content): + """ + Encode the given content as JSON and send it to the client. + """ + super(JsonWebsocketConsumer, self).send(text=json.dumps(content)) + + def group_send(self, name, content): + super(JsonWebsocketConsumer, self).group_send(name, json.dumps(content)) diff --git a/channels/routing.py b/channels/routing.py index afa2946..83108ba 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -15,6 +15,9 @@ class Router(object): listen to. Generally this is attached to a backend instance as ".router" + + Anything can be a routable object as long as it provides a match() + method that either returns (callable, kwargs) or None. """ def __init__(self, routing): @@ -89,19 +92,16 @@ class Route(object): and optional message parameter matching. """ - def __init__(self, channel, consumer, **kwargs): - # Get channel, make sure it's a unicode string - self.channel = channel - if isinstance(self.channel, six.binary_type): - self.channel = self.channel.decode("ascii") + def __init__(self, channels, consumer, **kwargs): + # Get channels, make sure it's a list of unicode strings + if isinstance(channels, six.string_types): + channels = [channels] + self.channels = [ + channel.decode("ascii") if isinstance(channel, six.binary_type) else channel + for channel in channels + ] # Get consumer, optionally importing it - if isinstance(consumer, six.string_types): - module_name, variable_name = consumer.rsplit(".", 1) - try: - consumer = getattr(importlib.import_module(module_name), variable_name) - except (ImportError, AttributeError): - raise ImproperlyConfigured("Cannot import consumer %r" % consumer) - self.consumer = consumer + self.consumer = self._resolve_consumer(consumer) # Compile filter regexes up front self.filters = { name: re.compile(Router.normalise_re_arg(value)) @@ -118,13 +118,26 @@ class Route(object): ) ) + def _resolve_consumer(self, consumer): + """ + Turns the consumer from a string into an object if it's a string, + passes it through otherwise. + """ + if isinstance(consumer, six.string_types): + module_name, variable_name = consumer.rsplit(".", 1) + try: + consumer = getattr(importlib.import_module(module_name), variable_name) + except (ImportError, AttributeError): + raise ImproperlyConfigured("Cannot import consumer %r" % consumer) + return consumer + def match(self, message): """ Checks to see if we match the Message object. Returns (consumer, kwargs dict) if it matches, None otherwise """ # Check for channel match first of all - if message.channel.name != self.channel: + if message.channel.name not in self.channels: return None # Check each message filter and build consumer kwargs as we go call_args = {} @@ -143,11 +156,11 @@ class Route(object): """ Returns the channel names this route listens on """ - return {self.channel, } + return set(self.channels) def __str__(self): return "%s %s -> %s" % ( - self.channel, + "/".join(self.channels), "" if not self.filters else "(%s)" % ( ", ".join("%s=%s" % (n, v.pattern) for n, v in self.filters.items()) ), @@ -155,6 +168,22 @@ class Route(object): ) +class RouteClass(Route): + """ + Like Route, but targets a class-based consumer rather than a functional + one, meaning it looks for a (class) method called "channels()" on the + object rather than having a single channel passed in. + """ + + def __init__(self, consumer, **kwargs): + # Check the consumer provides a method_channels + consumer = self._resolve_consumer(consumer) + if not hasattr(consumer, "channel_names") or not callable(consumer.channel_names): + raise ValueError("The consumer passed to RouteClass has no valid channel_names method") + # Call super with list of channels + super(RouteClass, self).__init__(consumer.channel_names(), consumer, **kwargs) + + class Include(object): """ Represents an inclusion of another routing list in another file. @@ -212,4 +241,5 @@ class Include(object): # Lowercase standard to match urls.py route = Route +route_class = RouteClass include = Include diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py index 891113f..1a00e43 100644 --- a/channels/tests/test_routing.py +++ b/channels/tests/test_routing.py @@ -1,9 +1,10 @@ from __future__ import unicode_literals from django.test import SimpleTestCase -from channels.routing import Router, route, include +from channels.routing import Router, route, route_class, include from channels.message import Message from channels.utils import name_that_thing +from channels.generic import BaseConsumer # Fake consumers and routing sets that can be imported by string @@ -19,6 +20,16 @@ def consumer_3(): pass +class TestClassConsumer(BaseConsumer): + + method_mapping = { + "test.channel": "some_method", + } + + def some_method(self, message, **kwargs): + pass + + chatroom_routing = [ route("websocket.connect", consumer_2, path=r"^/chat/(?P[^/]+)/$"), route("websocket.connect", consumer_3, path=r"^/mentions/$"), @@ -29,6 +40,10 @@ chatroom_routing_nolinestart = [ route("websocket.connect", consumer_3, path=r"/mentions/$"), ] +class_routing = [ + route_class(TestClassConsumer, path=r"^/foobar/$"), +] + class RoutingTests(SimpleTestCase): """ @@ -175,6 +190,32 @@ class RoutingTests(SimpleTestCase): kwargs={}, ) + def test_route_class(self): + """ + Tests route_class with/without prefix + """ + router = Router([ + include("channels.tests.test_routing.class_routing"), + ]) + self.assertRoute( + router, + channel="websocket.connect", + content={"path": "/foobar/"}, + consumer=None, + ) + self.assertRoute( + router, + channel="test.channel", + content={"path": "/foobar/"}, + consumer=TestClassConsumer, + ) + self.assertRoute( + router, + channel="test.channel", + content={"path": "/"}, + consumer=None, + ) + def test_include_prefix(self): """ Tests inclusion with a prefix @@ -291,15 +332,16 @@ class RoutingTests(SimpleTestCase): route("http.request", consumer_1, path=r"^/chat/$"), route("http.disconnect", consumer_2), route("http.request", consumer_3), + route_class(TestClassConsumer), ]) # Initial check self.assertEqual( router.channels, - {"http.request", "http.disconnect"}, + {"http.request", "http.disconnect", "test.channel"}, ) # Dynamically add route, recheck router.add_route(route("websocket.receive", consumer_1)) self.assertEqual( router.channels, - {"http.request", "http.disconnect", "websocket.receive"}, + {"http.request", "http.disconnect", "websocket.receive", "test.channel"}, ) diff --git a/channels/utils.py b/channels/utils.py index e8f060e..dc96201 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -1,12 +1,14 @@ import types - def name_that_thing(thing): """ Returns either the function/class path or just the object's repr """ # Instance method if hasattr(thing, "im_class"): + # Mocks will recurse im_class forever + if hasattr(thing, "mock_calls"): + return "" return name_that_thing(thing.im_class) + "." + thing.im_func.func_name # Other named thing if hasattr(thing, "__name__"): diff --git a/channels/worker.py b/channels/worker.py index 935ed80..caafd95 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -72,7 +72,7 @@ class Worker(object): if self.signal_handlers: self.install_signal_handler() channels = self.apply_channel_filters(self.channel_layer.router.channels) - logger.info("Listening on channels %s", ", ".join(channels)) + logger.info("Listening on channels %s", ", ".join(sorted(channels))) while not self.termed: self.in_job = False channel, content = self.channel_layer.receive_many(channels, block=True) @@ -82,7 +82,7 @@ class Worker(object): time.sleep(0.01) continue # Create message wrapper - logger.debug("Worker got message on %s: repl %s", channel, content.get("reply_channel", "none")) + logger.debug("Got message on %s (reply %s)", channel, content.get("reply_channel", "none")) message = Message( content=content, channel_name=channel, @@ -103,6 +103,7 @@ class Worker(object): if self.callback: self.callback(channel, message) try: + logger.debug("Dispatching message on %s to %s", channel, name_that_thing(consumer)) consumer(message, **kwargs) except ConsumeLater: # They want to not handle it yet. Re-inject it with a number-of-tries marker. diff --git a/docs/generics.rst b/docs/generics.rst new file mode 100644 index 0000000..44f9629 --- /dev/null +++ b/docs/generics.rst @@ -0,0 +1,150 @@ +Generic Consumers +================= + +Much like Django's class-based views, Channels has class-based consumers. +They provide a way for you to arrange code so it's highly modifiable and +inheritable, at the slight cost of it being harder to figure out the execution +path. + +We recommend you use them if you find them valuable; normal function-based +consumers are also entirely valid, however, and may result in more readable +code for simpler tasks. + +There is one base class-based consumer class, ``BaseConsumer``, that provides +the pattern for method dispatch and is the thing you can build entirely +custom consumers on top of, and then protocol-specific subclasses that provide +extra utility - for example, the ``WebsocketConsumer`` provides automatic +group management for the connection. + +When you use class-based consumers in :doc:`routing `, you need +to use ``route_class`` rather than ``route``; ``route_class`` knows how to +talk to the class-based consumer and extract the list of channels it needs +to listen on from it directly, rather than making you pass it in explicitly. + +Class-based consumers are instantiated once for each message they consume, +so it's safe to store things on ``self`` (in fact, ``self.message`` is the +current message by default). + +Base +---- + +The ``BaseConsumer`` class is the foundation of class-based consumers, and what +you can inherit from if you wish to build your own entirely from scratch. + +You use it like this:: + + from channels.generic import BaseConsumer + + class MyConsumer(BaseConsumer): + + method_mapping = { + "channel.name.here": "method_name", + } + + def method_name(self, message, **kwargs): + pass + +All you need to define is the ``method_mapping`` dictionary, which maps +channel names to method names. The base code will take care of the dispatching +for you, and set ``self.message`` to the current message as well. + +If you want to perfom more complicated routing, you'll need to override the +``dispatch()`` and ``channel_names()`` methods in order to do the right thing; +remember, though, your channel names cannot change during runtime and must +always be the same for as long as your process runs. + + +WebSockets +---------- + +There are two WebSockets generic consumers; one that provides group management, +simpler send/receive methods, and basic method routing, and a subclass which +additionally automatically serializes all messages sent and receives using JSON. + +The basic WebSocket generic consumer is used like this:: + + from channels.generic.websockets import WebsocketConsumer + + class MyConsumer(WebsocketConsumer): + + # Set to True if you want them, else leave out + strict_ordering = False + slight_ordering = False + + def connection_groups(self, **kwargs): + """ + Called to return the list of groups to automatically add/remove + this connection to/from. + """ + return ["test"] + + def connect(self, message, **kwargs): + """ + Perform things on connection start + """ + pass + + def receive(self, text=None, bytes=None, **kwargs): + """ + Called when a message is received with either text or bytes + filled out. + """ + # Simple echo + self.send(text=text, bytes=bytes) + + def disconnect(self, message, **kwargs): + """ + Perform things on connection close + """ + pass + +You can call ``self.send`` inside the class to send things to the connection's +``reply_channel`` automatically. Any group names returned from ``connection_groups`` +are used to add the socket to when it connects and to remove it from when it +disconnects; you get keyword arguments too if your URL path, say, affects +which group to talk to. + +The JSON-enabled consumer looks slightly different:: + + from channels.generic.websockets import JsonWebsocketConsumer + + class MyConsumer(JsonWebsocketConsumer): + + # Set to True if you want them, else leave out + strict_ordering = False + slight_ordering = False + + def connection_groups(self, **kwargs): + """ + Called to return the list of groups to automatically add/remove + this connection to/from. + """ + return ["test"] + + def connect(self, message, **kwargs): + """ + Perform things on connection start + """ + pass + + def receive(self, content, **kwargs): + """ + Called when a message is received with decoded JSON content + """ + # Simple echo + self.send(content) + + def disconnect(self, message, **kwargs): + """ + Perform things on connection close + """ + pass + +For this subclass, ``receive`` only gets a ``content`` parameter that is the +already-decoded JSON as Python datastructures; similarly, ``send`` now only +takes a single argument, which it JSON-encodes before sending down to the +client. + +Note that this subclass still can't intercept ``Group.send()`` calls to make +them into JSON automatically, but it does provide ``self.group_send(name, content)`` +that will do this for you if you call it explicitly. diff --git a/docs/index.rst b/docs/index.rst index 92e06e4..97b3f0a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,8 @@ Contents: installation getting-started deploying + generics + routing backends testing cross-compat diff --git a/docs/routing.rst b/docs/routing.rst new file mode 100644 index 0000000..d3fd116 --- /dev/null +++ b/docs/routing.rst @@ -0,0 +1,76 @@ +Routing +======= + +Routing in Channels is done using a system similar to that in core Django; +a list of possible routes is provided, and Channels goes through all routes +until a match is found, and then runs the resulting consumer. + +The difference comes, however, in the fact that Channels has to route based +on more than just URL; channel name is the main thing routed on, and URL +path is one of many other optional things you can route on, depending on +the protocol (for example, imagine email consumers - they would route on +domain or recipient address instead). + +The routing Channels takes is just a list of *routing objects* - the three +built in ones are ``route``, ``route_class`` and ``include``, but any object +that implements the routing interface will work: + +* A method called ``match``, taking a single ``message`` as an argument and + returning ``None`` for no match or a tuple of ``(consumer, kwargs)`` if matched. + +* A method called ``channel_names``, which returns a set of channel names that + will match, which is fed to the channel layer to listen on them. + +The three default routing objects are: + +* ``route``: Takes a channel name, a consumer function, and optional filter + keyword arguments. + +* ``route_class``: Takes a class-based consumer, and optional filter + keyword arguments. Channel names are taken from the consumer's + ``channel_names()`` method. + +* ``include``: Takes either a list or string import path to a routing list, + and optional filter keyword arguments. + + +Filters +------- + +Filtering is how you limit matches based on, for example, URLs; you use regular +expressions, like so:: + + route("websocket.connect", consumers.ws_connect, path=r"^/chat/$") + +.. note:: + Unlike Django's URL routing, which strips the first slash of a URL for + neatness, Channels includes the first slash, as the routing system is + generic and not designed just for URLs. + +You can have multiple filters:: + + route("email.receive", comment_response, to_address=r".*@example.com$", subject="^reply") + +Multiple filters are always combined with logical AND; that is, you need to +match every filter to have the consumer called. + +Filters can capture keyword arguments to be passed to your function:: + + route("websocket.connect", connect_blog, path=r'^/liveblog/(?P[^/]+)/stream/$') + +You can also specify filters on an ``include``:: + + include("blog_includes", path=r'^/liveblog') + +When you specify filters on ``include``, the matched portion of the attribute +is removed for matches inside the include; for example, this arrangement +matches URLs like ``/liveblog/stream/``, because the outside ``include`` +strips off the ``/liveblog`` part it matches before passing it inside:: + + inner_routes = [ + route("websocket.connect", connect_blog, path=r'^/stream/'), + ] + + routing = [ + include(inner_routes, path=r'^/liveblog') + ] From 49c9b74d6f19812aa6dd2126e9a9499eea8f3175 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 25 May 2016 17:52:53 -0700 Subject: [PATCH 370/746] Docs/flake fixes --- channels/generic/__init__.py | 2 +- channels/generic/websockets.py | 2 +- channels/utils.py | 1 + docs/generics.rst | 6 ++++-- docs/routing.rst | 5 +++++ 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/channels/generic/__init__.py b/channels/generic/__init__.py index 3d08a87..8374ebb 100644 --- a/channels/generic/__init__.py +++ b/channels/generic/__init__.py @@ -1 +1 @@ -from .base import BaseConsumer +from .base import BaseConsumer # NOQA isort:skip diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 9faf75d..af81ab2 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -92,7 +92,7 @@ class WebsocketConsumer(BaseConsumer): else: raise ValueError("You must pass text or bytes") - def disconnect(self, message, **kwargs): + def raw_disconnect(self, message, **kwargs): """ Called when a WebSocket connection is closed. Base level so you don't need to call super() all the time. diff --git a/channels/utils.py b/channels/utils.py index dc96201..548c307 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -1,5 +1,6 @@ import types + def name_that_thing(thing): """ Returns either the function/class path or just the object's repr diff --git a/docs/generics.rst b/docs/generics.rst index 44f9629..064e31c 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -10,7 +10,7 @@ We recommend you use them if you find them valuable; normal function-based consumers are also entirely valid, however, and may result in more readable code for simpler tasks. -There is one base class-based consumer class, ``BaseConsumer``, that provides +There is one base generic consumer class, ``BaseConsumer``, that provides the pattern for method dispatch and is the thing you can build entirely custom consumers on top of, and then protocol-specific subclasses that provide extra utility - for example, the ``WebsocketConsumer`` provides automatic @@ -104,6 +104,8 @@ are used to add the socket to when it connects and to remove it from when it disconnects; you get keyword arguments too if your URL path, say, affects which group to talk to. +Additionally, the property ``self.path`` is always set to the current URL path. + The JSON-enabled consumer looks slightly different:: from channels.generic.websockets import JsonWebsocketConsumer @@ -140,7 +142,7 @@ The JSON-enabled consumer looks slightly different:: """ pass -For this subclass, ``receive`` only gets a ``content`` parameter that is the +For this subclass, ``receive`` only gets a ``content`` argument that is the already-decoded JSON as Python datastructures; similarly, ``send`` now only takes a single argument, which it JSON-encodes before sending down to the client. diff --git a/docs/routing.rst b/docs/routing.rst index d3fd116..e4b5c63 100644 --- a/docs/routing.rst +++ b/docs/routing.rst @@ -74,3 +74,8 @@ strips off the ``/liveblog`` part it matches before passing it inside:: routing = [ include(inner_routes, path=r'^/liveblog') ] + +You can also include named capture groups in the filters on an include and +they'll be passed to the consumer just like those on ``route``; note, though, +that if the keyword argument names from the ``include`` and the ``route`` +clash, the values from ``route`` will take precedence. From 982a47a9df7350f7d275ea631cf21ab3406cbfbe Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 25 May 2016 17:56:06 -0700 Subject: [PATCH 371/746] Add generics routing example --- docs/generics.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/generics.rst b/docs/generics.rst index 064e31c..0c969d3 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -21,6 +21,15 @@ to use ``route_class`` rather than ``route``; ``route_class`` knows how to talk to the class-based consumer and extract the list of channels it needs to listen on from it directly, rather than making you pass it in explicitly. +Here's a routing example:: + + from channels import route, route_class + + channel_routing = [ + route_class(consumers.ChatServer, path=r"^/chat/"), + route("websocket.connect", consumers.ws_connect, path=r"^/$"), + ] + Class-based consumers are instantiated once for each message they consume, so it's safe to store things on ``self`` (in fact, ``self.message`` is the current message by default). From 1168ca670e2d6e40b7ccc2438665a1d5d54b6259 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 25 May 2016 18:00:31 -0700 Subject: [PATCH 372/746] Releasing 0.14.0 --- CHANGELOG.txt | 10 ++++++++++ channels/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index c6bdc0c..8602776 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,13 @@ +0.14.0 (2016-05-25) +------------------- + +* Class-based consumer pattern and WebSocket consumer now come with Channels + (see docs for more details) + +* Better testing utilities including a higher-level Client abstraction with + optional HTTP/WebSocket HttpClient variant. + + 0.13.1 (2016-05-13) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 98ce119..4c11f54 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.13.1" +__version__ = "0.14.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 917ba184bbd7c5d3e98eefd4e836ab0168b3aee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=BCttler?= Date: Thu, 26 May 2016 05:55:13 +0200 Subject: [PATCH 373/746] Typo: "load of" -> "lot of" (#168) --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 2961611..b1f5de8 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -45,7 +45,7 @@ serializable types, and stay under a certain size limit - but these are implementation details you won't need to worry about until you get to more advanced usage. -The channels have capacity, so a load of producers can write lots of messages +The channels have capacity, so a lot of producers can write lots of messages into a channel with no consumers and then a consumer can come along later and will start getting served those queued messages. From 1a09540ca81ac866bd324d0bf1e7d66b5a6e3360 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Sun, 29 May 2016 04:31:15 +0300 Subject: [PATCH 374/746] Added fail_on_none parameter for Client.consume function (#172) --- channels/tests/base.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/channels/tests/base.py b/channels/tests/base.py index a6a3145..48c0291 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -101,21 +101,27 @@ class Client(object): content.setdefault('reply_channel', self.reply_channel) self.channel_layer.send(to, content) - def consume(self, channel): + def consume(self, channel, fail_on_none=True): """ Get next message for channel name and run appointed consumer """ message = self.get_next_message(channel) if message: - consumer, kwargs = self.channel_layer.router.match(message) - return consumer(message, **kwargs) + match = self.channel_layer.router.match(message) + if match: + consumer, kwargs = match + return consumer(message, **kwargs) + elif fail_on_none: + raise AssertionError("Can't find consumer for message %s" % message) + elif fail_on_none: + raise AssertionError("No message for channel %s" % channel) - def send_and_consume(self, channel, content={}): + def send_and_consume(self, channel, content={}, fail_on_none=True): """ Reproduce full live cycle of the message """ self.send(channel, content) - return self.consume(channel) + return self.consume(channel, fail_on_none=fail_on_none) def receive(self): """self.get_next_message(self.reply_channel) From 2f3114b21eda0adebacef2dd83b3164a5cc921c5 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Sun, 29 May 2016 19:36:29 +0300 Subject: [PATCH 375/746] In-between method for adding decorators in right way + a few tests for generic consumer (#171) * Corrected doc string for BaseConsumer * Added get_handler method for Class-base consumers for wrapping by decorators in right way * Added a few tests for generic consumers --- channels/generic/base.py | 17 ++++--- channels/generic/websockets.py | 9 ++-- channels/tests/test_generic.py | 85 ++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 channels/tests/test_generic.py diff --git a/channels/generic/base.py b/channels/generic/base.py index 89c387c..5e3c82a 100644 --- a/channels/generic/base.py +++ b/channels/generic/base.py @@ -6,11 +6,11 @@ class BaseConsumer(object): Base class-based consumer class. Provides the mechanisms to be a direct routing object and a few other things. - Class-based consumers should be used directly in routing with their - filters, like so:: + Class-based consumers should be used with route_class in routing, like so:: + from channels import route_class routing = [ - JsonWebsocketConsumer(path=r"^/liveblog/(?P[^/]+)/"), + route_class(JsonWebsocketConsumer, path=r"^/liveblog/(?P[^/]+)/"), ] """ @@ -32,9 +32,14 @@ class BaseConsumer(object): """ return set(cls.method_mapping.keys()) + def get_handler(self, message, **kwargs): + """ + Return handler uses method_mapping to return the right method to call. + """ + return getattr(self, self.method_mapping[message.channel.name]) + def dispatch(self, message, **kwargs): """ - Called with the message and all keyword arguments; uses method_mapping - to choose the right method to call. + Call handler with the message and all keyword arguments. """ - return getattr(self, self.method_mapping[message.channel.name])(message, **kwargs) + return self.get_handler(message, **kwargs)(message, **kwargs) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index af81ab2..b633fc0 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -22,18 +22,19 @@ class WebsocketConsumer(BaseConsumer): slight_ordering = False strict_ordering = False - def dispatch(self, message, **kwargs): + def get_handler(self, message, **kwargs): """ Pulls out the path onto an instance variable, and optionally adds the ordering decorator. """ self.path = message['path'] + handler = super(WebsocketConsumer, self).get_handler(message, **kwargs) if self.strict_ordering: - return enforce_ordering(super(WebsocketConsumer, self).dispatch(message, **kwargs), slight=False) + return enforce_ordering(handler, slight=False) elif self.slight_ordering: - return enforce_ordering(super(WebsocketConsumer, self).dispatch(message, **kwargs), slight=True) + return enforce_ordering(handler, slight=True) else: - return super(WebsocketConsumer, self).dispatch(message, **kwargs) + return handler def connection_groups(self, **kwargs): """ diff --git a/channels/tests/test_generic.py b/channels/tests/test_generic.py new file mode 100644 index 0000000..6a8c380 --- /dev/null +++ b/channels/tests/test_generic.py @@ -0,0 +1,85 @@ +from __future__ import unicode_literals + +from django.test import override_settings +from channels import route_class +from channels.generic import BaseConsumer, websockets +from channels.tests import ChannelTestCase +from channels.tests import apply_routes, Client + + +@override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") +class GenericTests(ChannelTestCase): + + def test_base_consumer(self): + + class Consumers(BaseConsumer): + + method_mapping = { + 'test.create': 'create', + 'test.test': 'test', + } + + def create(self, message, **kwargs): + self.called = 'create' + + def test(self, message, **kwargs): + self.called = 'test' + + with apply_routes([route_class(Consumers)]): + client = Client() + + # check that methods for certain channels routes successfully + self.assertEqual(client.send_and_consume('test.create').called, 'create') + self.assertEqual(client.send_and_consume('test.test').called, 'test') + + # send to the channels without routes + client.send('test.wrong') + message = self.get_next_message('test.wrong') + self.assertEqual(client.channel_layer.router.match(message), None) + + client.send('test') + message = self.get_next_message('test') + self.assertEqual(client.channel_layer.router.match(message), None) + + def test_websockets_consumers_handlers(self): + + class WebsocketConsumer(websockets.WebsocketConsumer): + + def connect(self, message, **kwargs): + self.called = 'connect' + self.id = kwargs['id'] + + def disconnect(self, message, **kwargs): + self.called = 'disconnect' + + def receive(self, text=None, bytes=None, **kwargs): + self.text = text + + with apply_routes([route_class(WebsocketConsumer, path='/path/(?P\d+)')]): + client = Client() + + consumer = client.send_and_consume('websocket.connect', {'path': '/path/1'}) + self.assertEqual(consumer.called, 'connect') + self.assertEqual(consumer.id, '1') + + consumer = client.send_and_consume('websocket.receive', {'path': '/path/1', 'text': 'text'}) + self.assertEqual(consumer.text, 'text') + + consumer = client.send_and_consume('websocket.disconnect', {'path': '/path/1'}) + self.assertEqual(consumer.called, 'disconnect') + + def test_websockets_decorators(self): + class WebsocketConsumer(websockets.WebsocketConsumer): + slight_ordering = True + + def connect(self, message, **kwargs): + self.order = message['order'] + + with apply_routes([route_class(WebsocketConsumer, path='/path')]): + client = Client() + + client.send('websocket.connect', {'path': '/path', 'order': 1}) + client.send('websocket.connect', {'path': '/path', 'order': 0}) + client.consume('websocket.connect') + self.assertEqual(client.consume('websocket.connect').order, 0) + self.assertEqual(client.consume('websocket.connect').order, 1) From e0341e65cd7d52918987c5de8450e5ddc8942dab Mon Sep 17 00:00:00 2001 From: Oskar Hahn Date: Mon, 30 May 2016 02:08:33 +0200 Subject: [PATCH 376/746] Use window.location.host instead of 127.0.0.1 (#178) See #176 --- docs/getting-started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 2b24beb..efd51ac 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -134,7 +134,7 @@ need to change the socket address if you're using a development VM or similar):: // Note that the path doesn't matter for routing; any WebSocket // connection gets bumped over to WebSocket consumers - socket = new WebSocket("ws://127.0.0.1:8000/chat/"); + socket = new WebSocket("ws://" + window.location.host + "/chat/"); socket.onmessage = function(e) { alert(e.data); } @@ -228,7 +228,7 @@ code in the developer console as before:: // Note that the path doesn't matter right now; any WebSocket // connection gets bumped over to WebSocket consumers - socket = new WebSocket("ws://127.0.0.1:8000/chat/"); + socket = new WebSocket("ws://" + window.location.host + "/chat/"); socket.onmessage = function(e) { alert(e.data); } From 80a9019cb24ea581a9cef0344caaf4cec4a95a94 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 30 May 2016 00:17:30 +0000 Subject: [PATCH 377/746] Fix echo endpoint in testproject --- testproject/chtest/consumers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testproject/chtest/consumers.py b/testproject/chtest/consumers.py index 96dbabb..f447e45 100644 --- a/testproject/chtest/consumers.py +++ b/testproject/chtest/consumers.py @@ -9,4 +9,6 @@ def ws_connect(message): #@enforce_ordering(slight=True) def ws_message(message): "Echoes messages back to the client" - message.reply_channel.send(message.content) + message.reply_channel.send({ + "text": message['text'], + }) From df0ae80bfb7ebf6dbb9b783f1f1a3675264b2bd3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 31 May 2016 18:34:06 +0000 Subject: [PATCH 378/746] Fix send call in concepts doc --- docs/concepts.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index b1f5de8..cf5319a 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -184,10 +184,10 @@ set of channels (here, using Redis) to send updates to:: def send_update(sender, instance, **kwargs): # Loop through all response channels and send the update for reply_channel in redis_conn.smembers("readers"): - Channel(reply_channel).send( - id=instance.id, - content=instance.content, - ) + Channel(reply_channel).send({ + "id": instance.id, + "content": instance.content, + }) # Connected to websocket.connect def ws_connect(message): @@ -222,10 +222,10 @@ abstraction as a core concept called Groups:: @receiver(post_save, sender=BlogUpdate) def send_update(sender, instance, **kwargs): - Group("liveblog").send( - id=instance.id, - content=instance.content, - ) + Group("liveblog").send({ + "id": instance.id, + "content": instance.content, + }) # Connected to websocket.connect def ws_connect(message): From 38641d8522b243c24f5dacb466828649f35f75cc Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 31 May 2016 19:04:12 +0000 Subject: [PATCH 379/746] Fixed #182: Close response once we're done with it --- channels/handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/channels/handler.py b/channels/handler.py index ec03f4c..8902cdf 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -212,6 +212,8 @@ class AsgiHandler(base.BaseHandler): for message in self.encode_response(response): # TODO: file_to_stream yield message + # Close the response now we're done with it + response.close() def process_exception_by_middleware(self, exception, request): """ From 56104e7fc62abd3cb816dd30f93e39a250873b36 Mon Sep 17 00:00:00 2001 From: Tim Watts Date: Wed, 1 Jun 2016 18:47:50 +0200 Subject: [PATCH 380/746] Tests for file and streaming response handling inside Django (#185) * add first streaming and file response tests * iterate over response and not streaming content directly * add coverage for FileResponse and StreamingHttpResponse * added tests for headers, json responses, and redirect responses * rm print statement * skip failing stringio test --- .coveragerc | 24 ++++ channels/handler.py | 4 +- channels/tests/a_file | 5 + channels/tests/test_handler.py | 238 +++++++++++++++++++++++++++++++-- tox.ini | 4 +- 5 files changed, 263 insertions(+), 12 deletions(-) create mode 100644 .coveragerc create mode 100644 channels/tests/a_file diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..142b3c3 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,24 @@ +[run] +branch = True +source = channels, django.http.response +omit = channels/tests/* + +[report] +show_missing = True +skip_covered = True +omit = channels/tests/* + +[html] +directory = coverage_html + +[paths] +django_19 = + .tox/py27-django-18/lib/python2.7 + .tox/py34-django-18/lib/python3.4 + .tox/py35-django-18/lib/python3.5 + +django_18 = + .tox/py27-django-19/lib/python2.7 + .tox/py34-django-19/lib/python3.4 + .tox/py35-django-19/lib/python3.5 + diff --git a/channels/handler.py b/channels/handler.py index 8902cdf..0e0f4e1 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -277,7 +277,9 @@ class AsgiHandler(base.BaseHandler): } # Streaming responses need to be pinned to their iterator if response.streaming: - for part in response.streaming_content: + # Access `__iter__` and not `streaming_content` directly in case + # it has been overridden in a subclass. + for part in response: for chunk, more in cls.chunk_bytes(part): message['content'] = chunk # We ignore "more" as there may be more parts; instead, diff --git a/channels/tests/a_file b/channels/tests/a_file new file mode 100644 index 0000000..207ed20 --- /dev/null +++ b/channels/tests/a_file @@ -0,0 +1,5 @@ +thi is +a file +sdaf +sadf + diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index 450f06b..255f0b1 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -1,5 +1,15 @@ from __future__ import unicode_literals -from django.http import HttpResponse + +import os +import unittest +from datetime import datetime +from itertools import islice + +from django.http import ( + FileResponse, HttpResponse, HttpResponseRedirect, JsonResponse, + StreamingHttpResponse, +) +from six import BytesIO, StringIO from channels import Channel from channels.handler import AsgiHandler @@ -15,7 +25,7 @@ class FakeAsgiHandler(AsgiHandler): chunk_size = 30 def __init__(self, response): - assert isinstance(response, HttpResponse) + assert isinstance(response, (HttpResponse, StreamingHttpResponse)) self._response = response super(FakeAsgiHandler, self).__init__() @@ -43,7 +53,8 @@ class HandlerTests(ChannelTestCase): response = HttpResponse(b"Hi there!", content_type="text/plain") # Run the handler handler = FakeAsgiHandler(response) - reply_messages = list(handler(self.get_next_message("test", require=True))) + reply_messages = list( + handler(self.get_next_message("test", require=True))) # Make sure we got the right number of messages self.assertEqual(len(reply_messages), 1) reply_message = reply_messages[0] @@ -53,9 +64,58 @@ class HandlerTests(ChannelTestCase): self.assertEqual(reply_message.get("more_content", False), False) self.assertEqual( reply_message["headers"], - [(b"Content-Type", b"text/plain")], + [ + (b"Content-Type", b"text/plain"), + ], ) + def test_cookies(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = HttpResponse(b"Hi there!", content_type="text/plain") + response.set_signed_cookie('foo', '1', expires=datetime.now()) + # Run the handler + handler = FakeAsgiHandler(response) + reply_messages = list( + handler(self.get_next_message("test", require=True))) + # Make sure we got the right number of messages + self.assertEqual(len(reply_messages), 1) + reply_message = reply_messages[0] + # Make sure the message looks correct + self.assertEqual(reply_message["content"], b"Hi there!") + self.assertEqual(reply_message["status"], 200) + self.assertEqual(reply_message.get("more_content", False), False) + self.assertEqual(reply_message["headers"][0], (b'Content-Type', b'text/plain')) + self.assertIn('foo=', reply_message["headers"][1][1].decode()) + + def test_headers(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = HttpResponse(b"Hi there!", content_type="text/plain") + response['foo'] = 1 + response['bar'] = 1 + del response['bar'] + del response['nonexistant_key'] + handler = FakeAsgiHandler(response) + reply_messages = list( + handler(self.get_next_message("test", require=True))) + # Make sure we got the right number of messages + self.assertEqual(len(reply_messages), 1) + reply_message = reply_messages[0] + # Make sure the message looks correct + self.assertEqual(reply_message["content"], b"Hi there!") + header_dict = dict(reply_messages[0]['headers']) + self.assertEqual(header_dict[b'foo'].decode(), '1') + self.assertNotIn('bar', header_dict) + def test_large(self): """ Tests a large response (will need chunking) @@ -67,14 +127,17 @@ class HandlerTests(ChannelTestCase): "method": "GET", "path": b"/test/", }) - response = HttpResponse(b"Thefirstthirtybytesisrighthereandhereistherest") + response = HttpResponse( + b"Thefirstthirtybytesisrighthereandhereistherest") # Run the handler handler = FakeAsgiHandler(response) - reply_messages = list(handler(self.get_next_message("test", require=True))) + reply_messages = list( + handler(self.get_next_message("test", require=True))) # Make sure we got the right number of messages self.assertEqual(len(reply_messages), 2) # Make sure the messages look correct - self.assertEqual(reply_messages[0]["content"], b"Thefirstthirtybytesisrighthere") + self.assertEqual(reply_messages[0][ + "content"], b"Thefirstthirtybytesisrighthere") self.assertEqual(reply_messages[0]["status"], 200) self.assertEqual(reply_messages[0]["more_content"], True) self.assertEqual(reply_messages[1]["content"], b"andhereistherest") @@ -90,19 +153,174 @@ class HandlerTests(ChannelTestCase): self.assertEqual(result[0][0], b"") self.assertEqual(result[0][1], True) # Below chunk size - result = list(FakeAsgiHandler.chunk_bytes(b"12345678901234567890123456789")) + result = list(FakeAsgiHandler.chunk_bytes( + b"12345678901234567890123456789")) self.assertEqual(len(result), 1) self.assertEqual(result[0][0], b"12345678901234567890123456789") self.assertEqual(result[0][1], True) # Exactly chunk size - result = list(FakeAsgiHandler.chunk_bytes(b"123456789012345678901234567890")) + result = list(FakeAsgiHandler.chunk_bytes( + b"123456789012345678901234567890")) self.assertEqual(len(result), 1) self.assertEqual(result[0][0], b"123456789012345678901234567890") self.assertEqual(result[0][1], True) # Just above chunk size - result = list(FakeAsgiHandler.chunk_bytes(b"123456789012345678901234567890a")) + result = list(FakeAsgiHandler.chunk_bytes( + b"123456789012345678901234567890a")) self.assertEqual(len(result), 2) self.assertEqual(result[0][0], b"123456789012345678901234567890") self.assertEqual(result[0][1], False) self.assertEqual(result[1][0], b"a") self.assertEqual(result[1][1], True) + + def test_iterator(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = HttpResponse(range(10)) + handler = FakeAsgiHandler(response) + reply_messages = list( + handler(self.get_next_message("test", require=True))) + self.assertEqual(len(reply_messages), 1) + self.assertEqual(reply_messages[0]["content"], b"0123456789") + + def test_streaming_data(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = StreamingHttpResponse('Line: %s' % i for i in range(10)) + handler = FakeAsgiHandler(response) + reply_messages = list( + handler(self.get_next_message("test", require=True))) + self.assertEqual(len(reply_messages), 11) + self.assertEqual(reply_messages[0]["content"], b"Line: 0") + self.assertEqual(reply_messages[9]["content"], b"Line: 9") + + def test_real_file_response(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + current_dir = os.path.realpath(os.path.join( + os.getcwd(), os.path.dirname(__file__))) + response = FileResponse( + open(os.path.join(current_dir, 'a_file'), 'rb')) + handler = FakeAsgiHandler(response) + reply_messages = list( + handler(self.get_next_message("test", require=True))) + self.assertEqual(len(reply_messages), 2) + self.assertEqual(response.getvalue(), b'') + + def test_bytes_file_response(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = FileResponse(BytesIO(b'sadfdasfsdfsadf')) + handler = FakeAsgiHandler(response) + reply_messages = list( + handler(self.get_next_message("test", require=True))) + self.assertEqual(len(reply_messages), 2) + + def test_string_file_response(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = FileResponse('abcd') + handler = FakeAsgiHandler(response) + reply_messages = list( + handler(self.get_next_message("test", require=True))) + self.assertEqual(len(reply_messages), 5) + + def test_non_streaming_file_response(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = FileResponse(BytesIO(b'sadfdasfsdfsadf')) + # This is to test the exception handling. This would only happening if + # the StreamingHttpResponse was incorrectly subclassed. + response.streaming = False + + handler = FakeAsgiHandler(response) + with self.assertRaises(AttributeError): + list(handler(self.get_next_message("test", require=True))) + + def test_unclosable_filelike_object(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + + # This is a readable object that cannot be closed. + class Unclosable: + + def read(self, n=-1): + # Nothing to see here + return b"" + + response = FileResponse(Unclosable()) + handler = FakeAsgiHandler(response) + reply_messages = list(islice(handler(self.get_next_message("test", require=True)), 5)) + self.assertEqual(len(reply_messages), 1) + response.close() + + def test_json_response(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = JsonResponse({'foo': (1, 2)}) + handler = FakeAsgiHandler(response) + reply_messages = list(handler(self.get_next_message("test", require=True))) + self.assertEqual(len(reply_messages), 1) + self.assertEqual(reply_messages[0]['content'], b'{"foo": [1, 2]}') + + def test_redirect(self): + for redirect_to in ['/', '..', 'https://example.com']: + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = HttpResponseRedirect(redirect_to) + handler = FakeAsgiHandler(response) + reply_messages = list(handler(self.get_next_message("test", require=True))) + self.assertEqual(reply_messages[0]['status'], 302) + header_dict = dict(reply_messages[0]['headers']) + self.assertEqual(header_dict[b'Location'].decode(), redirect_to) + + @unittest.skip("failing under python 3") + def test_stringio_file_response(self): + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = FileResponse(StringIO('sadfdasfsdfsadf')) + handler = FakeAsgiHandler(response) + # Use islice because the generator never ends. + reply_messages = list( + islice(handler(self.get_next_message("test", require=True)), 5)) + self.assertEqual(len(reply_messages), 2, reply_messages) diff --git a/tox.ini b/tox.ini index 75ae168..51b3fc2 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ setenv = PYTHONPATH = {toxinidir}:{toxinidir} deps = autobahn + coverage asgiref>=0.9 six redis==2.10.5 @@ -23,4 +24,5 @@ deps = commands = flake8: flake8 isort: isort -c -rc channels - django: python {toxinidir}/runtests.py + django: coverage run --parallel-mode {toxinidir}/runtests.py + From 2874a0972cbe764e5f6f1d5f2296ccc3bc633fc9 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Fri, 3 Jun 2016 02:25:26 +0300 Subject: [PATCH 381/746] Using logger.error instead of logger.exception (#191) --- channels/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/worker.py b/channels/worker.py index caafd95..32f5edb 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -96,7 +96,7 @@ class Worker(object): # Handle the message match = self.channel_layer.router.match(message) if match is None: - logger.exception("Could not find match for message on %s! Check your routing.", channel) + logger.error("Could not find match for message on %s! Check your routing.", channel) continue else: consumer, kwargs = match From c4f016b9c289a8ca2cd8dae36ba5fa702f845905 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Fri, 3 Jun 2016 02:25:39 +0300 Subject: [PATCH 382/746] Fix for apply_routes: wrap routes in list, if it is not so (#192) --- channels/tests/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/channels/tests/base.py b/channels/tests/base.py index 48c0291..f96589b 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -157,7 +157,8 @@ class apply_routes(object): routes = list(map(include, self.routes)) else: routes = self.routes - + else: + routes = [self.routes] channel_layer.routing = routes channel_layer.router = Router(routes) From f8debafbd395a530675b120ba3d9e753711cdd03 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Fri, 3 Jun 2016 02:25:55 +0300 Subject: [PATCH 383/746] Added path parameter to the HttpClient.send method (#193) --- channels/tests/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/tests/http.py b/channels/tests/http.py index 4dd3840..fa4526c 100644 --- a/channels/tests/http.py +++ b/channels/tests/http.py @@ -56,14 +56,14 @@ class HttpClient(Client): self._session = session_for_reply_channel(self.reply_channel) return self._session - def send(self, to, content={}): + def send(self, to, content={}, path='/'): """ Send a message to a channel. Adds reply_channel name and channel_session to the message. """ content = copy.deepcopy(content) content.setdefault('reply_channel', self.reply_channel) - content.setdefault('path', '/') + content.setdefault('path', path) content.setdefault('headers', self.headers) self.channel_layer.send(to, content) From 6eaee8f5220a4934300805af55660337eb0529cc Mon Sep 17 00:00:00 2001 From: thewayiam Date: Fri, 3 Jun 2016 07:52:35 +0800 Subject: [PATCH 384/746] #188: add/discard message.reply_channel on generic group_send (#189) --- channels/generic/websockets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index b633fc0..096dee1 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -49,7 +49,7 @@ class WebsocketConsumer(BaseConsumer): need to call super() all the time. """ for group in self.connection_groups(**kwargs): - Group(group, channel_layer=message.channel_layer).add(message.channel) + Group(group, channel_layer=message.channel_layer).add(message.reply_channel) self.connect(message, **kwargs) def connect(self, message, **kwargs): @@ -99,7 +99,7 @@ class WebsocketConsumer(BaseConsumer): need to call super() all the time. """ for group in self.connection_groups(**kwargs): - Group(group, channel_layer=message.channel_layer).discard(message.channel) + Group(group, channel_layer=message.channel_layer).discard(message.reply_channel) self.disconnect(message, **kwargs) def disconnect(self, message, **kwargs): From 18d4cc8e6fc629e8017373ff2b75d2ba68962bf6 Mon Sep 17 00:00:00 2001 From: thewayiam Date: Mon, 6 Jun 2016 13:06:37 +0800 Subject: [PATCH 385/746] #196: made worker serve staticfiles if DEBUG=True (#197) --- channels/management/commands/runworker.py | 8 +++++++- docs/getting-started.rst | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index ec2b68a..81f80b9 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -1,10 +1,12 @@ from __future__ import unicode_literals +from django.conf import settings from django.core.management import BaseCommand, CommandError from channels import DEFAULT_CHANNEL_LAYER, channel_layers from channels.log import setup_logger from channels.worker import Worker +from channels.staticfiles import StaticFilesConsumer class Command(BaseCommand): @@ -38,7 +40,11 @@ class Command(BaseCommand): "Change your settings to use a cross-process channel layer." ) # Check a handler is registered for http reqs - self.channel_layer.router.check_default() + # Serve static files if Django in debug mode + if settings.DEBUG: + self.channel_layer.router.check_default(http_consumer=StaticFilesConsumer()) + else: + self.channel_layer.router.check_default() # Launch a worker self.logger.info("Running worker against channel layer %s", self.channel_layer) # Optionally provide an output callback diff --git a/docs/getting-started.rst b/docs/getting-started.rst index efd51ac..20b42c1 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -320,6 +320,7 @@ As you can probably guess, this disables the worker threads in ``runserver`` and handles them in a separate process. You can pass ``-v 2`` to ``runworker`` if you want to see logging as it runs the consumers. +If Django in debug mode(`DEBUG=True`), it'll serve static files as Django default behavior. Persisting Data --------------- From 3c5c09d639565c36ec07ca339f6d97d3fbdb4f8b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 6 Jun 2016 05:09:08 +0000 Subject: [PATCH 386/746] Expand on static file serving doc line. --- docs/getting-started.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 20b42c1..b93e410 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -320,7 +320,9 @@ As you can probably guess, this disables the worker threads in ``runserver`` and handles them in a separate process. You can pass ``-v 2`` to ``runworker`` if you want to see logging as it runs the consumers. -If Django in debug mode(`DEBUG=True`), it'll serve static files as Django default behavior. +If Django is in debug mode (``DEBUG=True``), then ``runworker`` will serve +static files, as ``runserver`` does. Just like a normal Django setup, you'll +have to set up your static file serving for when ``DEBUG`` is turned off. Persisting Data --------------- From 08ecffe107fbee60bb53502d16665178547feebb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 8 Jun 2016 23:14:45 +0000 Subject: [PATCH 387/746] Update ASGI spec with single-reader channel --- docs/asgi.rst | 69 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index c37fadc..c4e59d5 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -100,7 +100,7 @@ contain only the following types to ensure serializability: Channels are identified by a unicode string name consisting only of ASCII letters, ASCII numerical digits, periods (``.``), dashes (``-``) and -underscores (``_``), plus an optional prefix character (see below). +underscores (``_``), plus an optional type character (see below). Channels are a first-in, first out queue with at-most-once delivery semantics. They can have multiple writers and multiple readers; only a single @@ -111,8 +111,26 @@ this is necessary to achieve this restriction. In order to aid with scaling and network architecture, a distinction is made between channels that have multiple readers (such as the ``http.request`` channel that web applications would listen on from every -application worker process) and *process-specific channels* -(such as a ``http.response!ABCDEF`` channel tied to a client socket). +application worker process), *single-reader channels* that are read from a +single unknown location (such as ``http.request.body?ABCDEF``), and +*process-specific channels* (such as a ``http.response!ABCDEF`` channel +tied to a client socket). + +*Normal channel* names contain no type characters, and can be routed however +the backend wishes; in particular, they do not have to appear globally +consistent, and backends may shard their contents out to different servers +so that a querying client only sees some portion of the messages. Calling +``receive_many`` on these channels does not guarantee that you will get the +messages in order or that you will get anything if the channel is non-empty. + +*Single-reader channel* names contain an exclamation mark +(``?``) character in order to indicate to the channel layer that it must make +these channels appear globally consistent. The ``?`` is always preceded by +the main channel name (e.g. ``http.response.body``) and followed by a +random portion. Channel layers may use the random portion to help pin the +channel to a server, but reads from this channel by a single process must +always be in-order and return messages if the channel is non-empty. These names +must be generated by the ``new_channel`` call. *Process-specific channel* names contain an exclamation mark (``!``) character in order to indicate to the channel layer that it may @@ -124,16 +142,19 @@ per-client/random portion - channel layers can split on the ``!`` and use just the right hand part to route if they desire, or can ignore it if they don't need to use different routing rules. Even if the right hand side contains client routing information, it must still contain random parts too so that -each call to ``new_channel`` returns a new, unused name. +each call to ``new_channel`` returns a new, unused name. These names +must be generated by the ``new_channel`` call; they are guaranteed to only +be read from the same process that calls ``new_channel``. Messages should expire after a set time sitting unread in a channel; the recommendation is one minute, though the best value depends on the channel layer and the way it is deployed. -The maximum message size is 1MB; if more data than this needs to be transmitted -it must be chunked or placed onto its own process-specific channel (see how -HTTP request bodies are done, for example). All channel layers must support -messages up to this size. +The maximum message size is 1MB if the message were encoded as JSON; +if more data than this needs to be transmitted it must be chunked or placed +onto its own single-reader or process-specific channel (see how HTTP request +bodies are done, for example). All channel layers must support messages up +to this size, but protocol specifications are encouraged to keep well below it. Handling Protocols @@ -253,9 +274,9 @@ and the vast majority of application code will not need to deal with this problem. If ordering of incoming packets matters for a protocol, they should be annotated with a packet number (as WebSocket is in this specification). -Single-reader channels, such as those used for response channels back to -clients, are not subject to this problem; a single reader must always -receive messages in channel order. +Single-reader and process-specific channels, such as those used for response +channels back to clients, are not subject to this problem; a single reader +on these must always receive messages in channel order. Capacity @@ -299,14 +320,13 @@ A *channel layer* must provide an object with these attributes * ``new_channel(pattern)``, a callable that takes a unicode string pattern, and returns a new valid channel name that does not already exist, by - adding a unicode string after the ``!`` character in ``pattern``, + adding a unicode string after the ``!`` or ``?`` character in ``pattern``, and checking for existence of that name in the channel layer. The ``pattern`` - MUST end with ``!`` or this function must error. This is not always called - prior to a message being sent on a channel, and cannot be used for - channel initialization. ``new_channel`` must be called on the same channel - layer that intends to read the channel with ``receive_many``; any other - channel layer instance may not receive messages on this channel due to - client-routing portions of the appended string. + MUST end with ``!`` or ``?`` or this function must error. If the character + is ``!``, making it a process-specific channel, ``new_channel`` must be + called on the same channel layer that intends to read the channel with + ``receive_many``; any other channel layer instance may not receive + messages on this channel due to client-routing portions of the appended string. * ``MessageTooLarge``, the exception raised when a send operation fails because the encoded message is over the layer's size limit. @@ -366,7 +386,7 @@ Channel Semantics Channels **must**: * Preserve ordering of messages perfectly with only a single reader - and writer, and preserve as much as possible in other cases. + and writer if the channel is a *single-reader* or *process-specific* channel. * Never deliver a message more than once. @@ -379,6 +399,9 @@ Channels **must**: * Have a maximum name length of at least 100 bytes. +They should attempt to preserve ordering in all cases as much as possible, +but perfect global ordering is obviously not possible in the distributed case. + They are not expected to deliver all messages, but a success rate of at least 99.99% is expected under normal circumstances. Implementations may want to have a "resilience testing" mode where they deliberately drop more messages @@ -520,7 +543,7 @@ Keys: If ``more_body`` is set, treat as start of body and concatenate on further chunks. -* ``more_body``: Channel name that contains +* ``more_body``: Name of a single-reader channel (containing ``?``) that contains Request Body Chunk messages representing a large request body. Optional, defaults to ``None``. Chunks append to ``body`` if set. Presence of a channel indicates at least one Request Body Chunk message needs to be read, @@ -541,7 +564,7 @@ Request Body Chunk Must be sent after an initial Response. -Channel: ``http.request.body!`` +Channel: ``http.request.body?`` Keys: @@ -845,7 +868,9 @@ limitation that they only use the following characters: * Hyphen ``-`` * Underscore ``_`` * Period ``.`` -* Exclamation mark ``!`` (only to deliniate process-specific channel names, +* Question mark ``?`` (only to delineiate single-reader channel names, + and only one per name) +* Exclamation mark ``!`` (only to delineate process-specific channel names, and only one per name) From c44de7e87034a5c5f783a37618d6cdd2525cc2f7 Mon Sep 17 00:00:00 2001 From: Erick Wilder Date: Thu, 9 Jun 2016 17:42:22 +0200 Subject: [PATCH 388/746] Use current `channels` version when building the documentation. (#201) Rationale: --- It may cause some confusion for the reader of the documentation about what's the most recent version of the library and if the official documentation pages are really for the 'latest' version. --- docs/conf.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index fe3f65e..760f97f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,7 +18,9 @@ import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('..')) + +from channels import __version__ # noqa # -- General configuration ------------------------------------------------ @@ -51,9 +53,9 @@ copyright = u'2015, Andrew Godwin' # built documents. # # The short X.Y version. -version = '1.0' +version = __version__ # The full version, including alpha/beta/rc tags. -release = '1.0' +release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From e31e326f108ce9a9f90f58551ccd393e5fd01b6f Mon Sep 17 00:00:00 2001 From: Krukov D Date: Thu, 9 Jun 2016 22:41:25 +0300 Subject: [PATCH 389/746] Added unicode_literals from future at tests/base (#203) --- channels/tests/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/channels/tests/base.py b/channels/tests/base.py index f96589b..bbff7dc 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import copy import random import string From 4a42ae952997b8081d123100f236900341100eb3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 9 Jun 2016 20:50:18 +0000 Subject: [PATCH 390/746] ? IS QUESTION MARK --- docs/asgi.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index c4e59d5..54d82f3 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -123,7 +123,7 @@ so that a querying client only sees some portion of the messages. Calling ``receive_many`` on these channels does not guarantee that you will get the messages in order or that you will get anything if the channel is non-empty. -*Single-reader channel* names contain an exclamation mark +*Single-reader channel* names contain an question mark (``?``) character in order to indicate to the channel layer that it must make these channels appear globally consistent. The ``?`` is always preceded by the main channel name (e.g. ``http.response.body``) and followed by a From 68ce1964c8a345921fa9d676505e4732ffb203f4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 10 Jun 2016 05:42:29 +0000 Subject: [PATCH 391/746] Releasing 0.14.1 --- CHANGELOG.txt | 6 ++++++ channels/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 8602776..10cff75 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,9 @@ +0.14.1 (2016-06-09) +------------------- + +* Fix unicode issues with test client under Python 2.7 + + 0.14.0 (2016-05-25) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 4c11f54..d7d213d 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.14.0" +__version__ = "0.14.1" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From f8c1a9c688723e33f8e6fda151954a014261c5fc Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 13 Jun 2016 17:47:39 +0200 Subject: [PATCH 392/746] [docs] Replaced left over CHANNEL_BACKENDS with CHANNEL_LAYERS (#206) --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 77f11f9..a7fc3bf 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -44,7 +44,7 @@ a backend has to follow). Typically a channel backend will connect to one or more central servers that serve as the communication layer - for example, the Redis backend connects -to a Redis server. All this goes into the ``CHANNEL_BACKENDS`` setting; +to a Redis server. All this goes into the ``CHANNEL_LAYERS`` setting; here's an example for a remote Redis server:: CHANNEL_LAYERS = { From 0fe9d2be2bdac8d6f44c37d5eb97bcff84dc4844 Mon Sep 17 00:00:00 2001 From: Tommy Beadle Date: Tue, 14 Jun 2016 15:41:03 -0400 Subject: [PATCH 393/746] Avoid NameError in example code. (#208) 'room' is used in the Group initialization but was not defined. --- docs/getting-started.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index b93e410..4e2838b 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -571,8 +571,9 @@ have a ChatMessage model with ``message`` and ``room`` fields:: # Connected to chat-messages def msg_consumer(message): # Save to model + room = message.content['room'] ChatMessage.objects.create( - room=message.content['room'], + room=room, message=message.content['message'], ) # Broadcast to listening sockets From 0fe438a44590ea991b4f8941ea25cbf05b581e6e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 16 Jun 2016 11:25:29 +0100 Subject: [PATCH 394/746] Improve decorator support for class based consumers --- channels/generic/base.py | 12 +++++++++++- docs/generics.rst | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/channels/generic/base.py b/channels/generic/base.py index 5e3c82a..73d97fc 100644 --- a/channels/generic/base.py +++ b/channels/generic/base.py @@ -1,4 +1,6 @@ from __future__ import unicode_literals +from channels.sessions import channel_session +from channels.auth import channel_session_user class BaseConsumer(object): @@ -15,6 +17,8 @@ class BaseConsumer(object): """ method_mapping = {} + channel_session = False + channel_session_user = False def __init__(self, message, **kwargs): """ @@ -36,7 +40,13 @@ class BaseConsumer(object): """ Return handler uses method_mapping to return the right method to call. """ - return getattr(self, self.method_mapping[message.channel.name]) + handler = getattr(self, self.method_mapping[message.channel.name]) + if self.channel_session_user: + return channel_session_user(handler) + elif self.channel_session: + return channel_session(handler) + else: + return handler def dispatch(self, message, **kwargs): """ diff --git a/docs/generics.rst b/docs/generics.rst index 0c969d3..97c013a 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -159,3 +159,35 @@ client. Note that this subclass still can't intercept ``Group.send()`` calls to make them into JSON automatically, but it does provide ``self.group_send(name, content)`` that will do this for you if you call it explicitly. + +Sessions and Users +------------------ + +If you wish to use ``channel_session`` or ``channel_session_user`` with a +class-based consumer, simply set one of the variables in the class body:: + + class MyConsumer(WebsocketConsumer): + + channel_session_user = True + +This will run the appropriate decorator around your handler methods, and provide +``message.channel_session`` and ``message.user`` on the message object - both +the one passed in to your handler as an argument as well as ``self.message``, +as they point to the same instance. + +Applying Decorators +------------------- + +To apply decorators to a class-based consumer, you'll have to wrap a functional +part of the consumer; in this case, ``get_handler`` is likely the place you +want to override; like so:: + + class MyConsumer(WebsocketConsumer): + + def get_handler(self, *args, **kwargs): + handler = super(MyConsumer, self).get_handler(*args, **kwargs) + return your_decorator(handler) + +You can also use the Django ``method_decorator`` utility to wrap methods that +have ``message`` as their first positional argument - note that it won't work +for more high-level methods, like ``WebsocketConsumer.receive``. From 773f1332ee4458cce5d5e0be25fb350ffebdc4f0 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 16 Jun 2016 11:42:08 +0100 Subject: [PATCH 395/746] Fix import earliness for auth model --- channels/auth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/channels/auth.py b/channels/auth.py index 0565446..06e826b 100644 --- a/channels/auth.py +++ b/channels/auth.py @@ -1,7 +1,6 @@ import functools from django.contrib import auth -from django.contrib.auth.models import AnonymousUser from .sessions import channel_session, http_session @@ -30,6 +29,8 @@ def channel_session_user(func): if not hasattr(message, "channel_session"): raise ValueError("Did not see a channel session to get auth from") if message.channel_session is None: + # Inner import to avoid reaching into models before load complete + from django.contrib.auth.models import AnonymousUser message.user = AnonymousUser() # Otherwise, be a bit naughty and make a fake Request with just # a "session" attribute (later on, perhaps refactor contrib.auth to From 6fe841337d2cecc78f68be1d5bc34908c41970a3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 16 Jun 2016 11:45:08 +0100 Subject: [PATCH 396/746] Fix missing import --- channels/auth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/channels/auth.py b/channels/auth.py index 06e826b..90eaaff 100644 --- a/channels/auth.py +++ b/channels/auth.py @@ -60,6 +60,8 @@ def http_session_user(func): if not hasattr(message, "http_session"): raise ValueError("Did not see a http session to get auth from") if message.http_session is None: + # Inner import to avoid reaching into models before load complete + from django.contrib.auth.models import AnonymousUser message.user = AnonymousUser() # Otherwise, be a bit naughty and make a fake Request with just # a "session" attribute (later on, perhaps refactor contrib.auth to From 66c4b0cb6765addb75d9258f5a36c0145c4c172d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 16 Jun 2016 11:46:50 +0100 Subject: [PATCH 397/746] Releasing 0.14.2 --- CHANGELOG.txt | 7 +++++++ channels/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 10cff75..e5292be 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,10 @@ +0.14.2 (2016-06-16) +------------------- + +* Class based consumers now have built-in channel_session and + channel_session_user support + + 0.14.1 (2016-06-09) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index d7d213d..e2892ed 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.14.1" +__version__ = "0.14.2" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 94d4c00807d98fccd872cd911be4043cf7bcb30c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 17 Jun 2016 12:50:25 +0100 Subject: [PATCH 398/746] Add some more dict methods to Message --- channels/message.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/channels/message.py b/channels/message.py index f41de35..a44ecc7 100644 --- a/channels/message.py +++ b/channels/message.py @@ -37,6 +37,15 @@ class Message(object): def __contains__(self, key): return key in self.content + def keys(self): + return self.content.keys() + + def values(self): + return self.content.values() + + def items(self): + return self.content.items() + def get(self, key, default=None): return self.content.get(key, default) From b481c1b533fc2ee1d2756789f8289b396bc958c5 Mon Sep 17 00:00:00 2001 From: Scott Burns Date: Fri, 17 Jun 2016 10:18:56 -0500 Subject: [PATCH 399/746] Add basic community page (#213) --- docs/community.rst | 10 ++++++++++ docs/index.rst | 1 + 2 files changed, 11 insertions(+) create mode 100644 docs/community.rst diff --git a/docs/community.rst b/docs/community.rst new file mode 100644 index 0000000..5c31744 --- /dev/null +++ b/docs/community.rst @@ -0,0 +1,10 @@ +Community Projects +================== + +These projects from the community are developed on top of Channels: + +* Djangobot_, a bi-directional interface server for Slack. + +If you'd like to add your project, please submit a PR with a link and brief description. + +.. _Djangobot: https://github.com/djangobot/djangobot diff --git a/docs/index.rst b/docs/index.rst index 97b3f0a..6b87e62 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,3 +37,4 @@ Contents: reference faqs asgi + community From 405e5b3b26461c6efc1909a5d9d095c79eb2662b Mon Sep 17 00:00:00 2001 From: Steve Steiner Date: Fri, 17 Jun 2016 12:18:09 -0400 Subject: [PATCH 400/746] Update channels test readme (#212) * Add documentation on how to build and run benchmark. * Update README.rst with instructions for running against local server * Add requirements.txt referenced in new instructions * Updated port to 80 for Docker test as daphne serves on that port --- testproject/Dockerfile | 2 +- testproject/README.rst | 61 ++++++++++++++++++++++++++++++++++ testproject/docker-compose.yml | 2 +- testproject/requirements.txt | 14 ++++++++ 4 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 testproject/requirements.txt diff --git a/testproject/Dockerfile b/testproject/Dockerfile index 721f12f..e7e49c4 100644 --- a/testproject/Dockerfile +++ b/testproject/Dockerfile @@ -2,10 +2,10 @@ FROM ubuntu:16.04 MAINTAINER Andrew Godwin +# python-dev \ RUN apt-get update && \ apt-get install -y \ git \ - python-dev \ python-setuptools \ python-pip && \ pip install -U pip diff --git a/testproject/README.rst b/testproject/README.rst index 00dbac1..2808cbe 100644 --- a/testproject/README.rst +++ b/testproject/README.rst @@ -3,3 +3,64 @@ Channels Test Project This subdirectory contains benchmarking code and a companion Django project that can be used to benchmark Channels for both HTTP and WebSocket performance. + +Preparation: +~~~~~~~~~~~~ + + Set up a Python 2.7 virtualenv however you do that and activate it. + + e.g. to create it right in the test directory (assuming python 2 is your system's default):: + + virtualenv channels-test-py27 + source channels-test-py27/bin/activate + pip install -U -r requirements.txt + +How to use with Docker: +~~~~~~~~~~~~~~~~~~~~~~~ + + Build the docker image from Dockerfile, tag it `channels-test`:: + + docker build -t channels-test . + + Run the server:: + + docker-compose up -d + + The benchmark project will now be running on: http:{your-docker-ip}:80 + + Test it by navigating to that address in a browser. It should just say "OK". + + It is also running a WebSocket server at: ws://{your-docker-ip}:80 + + Run the benchmark's help to show the parameters:: + + python benchmark.py --help + + Let's just try a quick test with the default values from the parameter list:: + + python benchmark.py ws://localhost:80 + +How to use with runserver: +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + You must have a local Redis server running on localhost:6739 for this to work! If you happen + to be running Docker, this can easily be done with:: + + docker run -d --name redis_local -p 6379:6379 redis:alpine + + Just to make sure you're up to date with migrations, run:: + + python manage.py migrate + + In one terminal window, run the server with:: + + python manage.py runserver + + In another terminal window, run the benchmark with:: + + python benchmark.py ws://localhost:8000 + + + + + diff --git a/testproject/docker-compose.yml b/testproject/docker-compose.yml index cf01e72..a0603fd 100644 --- a/testproject/docker-compose.yml +++ b/testproject/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: redis: - image: redis + image: redis:alpine daphne: image: channels-test build: . diff --git a/testproject/requirements.txt b/testproject/requirements.txt new file mode 100644 index 0000000..0d4a5aa --- /dev/null +++ b/testproject/requirements.txt @@ -0,0 +1,14 @@ +asgi-redis==0.13.1 +asgiref==0.13.3 +autobahn==0.14.1 +channels==0.14.2 +daphne==0.12.1 +Django==1.9.7 +docutils==0.12 +msgpack-python==0.4.7 +redis==2.10.5 +six==1.10.0 +statistics==1.0.3.5 +Twisted==16.2.0 +txaio==2.5.1 +zope.interface==4.2.0 From 6ea6dc65769e03c6f5bc66f29460227616110143 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 21 Jun 2016 07:56:04 -0700 Subject: [PATCH 401/746] Fixed #210: Plus double-decoded for query string --- channels/handler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index 0e0f4e1..02e474b 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -135,7 +135,11 @@ class AsgiRequest(http.HttpRequest): @cached_property def GET(self): - return http.QueryDict(self.message.get('query_string', '').encode("utf8")) + # Django will try and re-urldecode the string and interpret + as space; + # we re-encode + here to fix this. + return http.QueryDict( + self.message.get('query_string', '').replace("+", "%2b").encode("utf8"), + ) def _get_post(self): if not hasattr(self, '_post'): From 69f6791a157db9298e6eaf3eeb9ae5a5d6df4f71 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 21 Jun 2016 08:22:14 -0700 Subject: [PATCH 402/746] Fix test to match new spec. --- channels/tests/test_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index 41fe1e6..7dd90bf 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -65,7 +65,7 @@ class RequestTests(ChannelTestCase): self.assertEqual(request.META["SERVER_NAME"], "10.0.0.2") self.assertEqual(request.META["SERVER_PORT"], 80) self.assertEqual(request.GET["x"], "1") - self.assertEqual(request.GET["y"], "foo bar baz") + self.assertEqual(request.GET["y"], "foo bar+baz") self.assertEqual(request.COOKIES["test-time"], "1448995585123") self.assertEqual(request.COOKIES["test-value"], "yeah") self.assertFalse(request.POST) From d8ae2784d86771737213fc8e1a5620032d09bc7a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 21 Jun 2016 08:22:57 -0700 Subject: [PATCH 403/746] Note in ASGI about query path --- docs/asgi.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 54d82f3..234b359 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -527,7 +527,9 @@ Keys: and UTF8 byte sequences decoded into characters. * ``query_string``: Unicode string URL portion after the ``?``, already - url-decoded, like ``path``. Optional, default is ``""``. + url-decoded, like ``path``. Optional, default is ``""``. ``+`` characters in + this portion should be interpreted by the application to be literal pluses, + not spaces. * ``root_path``: Unicode string that indicates the root path this application is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults From 274feeb42fb0cb4e4e49f1be729f1ce54573dd5b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 21 Jun 2016 08:26:56 -0700 Subject: [PATCH 404/746] Releasing 0.14.3 --- CHANGELOG.txt | 8 ++++++++ channels/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index e5292be..d5e244f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +0.14.3 (2016-06-21) +------------------- + +* + signs in query strings are no longer double-decoded + +* Message now has .values(), .keys() and .items() to match dict + + 0.14.2 (2016-06-16) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index e2892ed..4fb0c47 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.14.2" +__version__ = "0.14.3" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From e890c48f3f418edb4fe869cdb51ca209cb3ba49d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 22 Jun 2016 09:44:10 -0700 Subject: [PATCH 405/746] Fixed #210: Fix query string to be bytes and not decoded --- channels/handler.py | 8 +++----- channels/tests/test_request.py | 4 ++-- docs/asgi.rst | 5 +---- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 02e474b..37793bf 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -135,11 +135,9 @@ class AsgiRequest(http.HttpRequest): @cached_property def GET(self): - # Django will try and re-urldecode the string and interpret + as space; - # we re-encode + here to fix this. - return http.QueryDict( - self.message.get('query_string', '').replace("+", "%2b").encode("utf8"), - ) + # Django will try and re-urldecode the query string and interpret + as space; + # we re-encode it here to fix this. + return http.QueryDict(self.message.get('query_string', '')) def _get_post(self): if not hasattr(self, '_post'): diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index 7dd90bf..66fae9e 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -46,7 +46,7 @@ class RequestTests(ChannelTestCase): "http_version": "1.1", "method": "GET", "path": "/test2/", - "query_string": "x=1&y=foo bar+baz", + "query_string": b"x=1&y=%26foo+bar%2Bbaz", "headers": { "host": b"example.com", "cookie": b"test-time=1448995585123; test-value=yeah", @@ -65,7 +65,7 @@ class RequestTests(ChannelTestCase): self.assertEqual(request.META["SERVER_NAME"], "10.0.0.2") self.assertEqual(request.META["SERVER_PORT"], 80) self.assertEqual(request.GET["x"], "1") - self.assertEqual(request.GET["y"], "foo bar+baz") + self.assertEqual(request.GET["y"], "&foo bar+baz") self.assertEqual(request.COOKIES["test-time"], "1448995585123") self.assertEqual(request.COOKIES["test-value"], "yeah") self.assertFalse(request.POST) diff --git a/docs/asgi.rst b/docs/asgi.rst index 234b359..d53207b 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -526,10 +526,7 @@ Keys: * ``path``: Unicode string HTTP path from URL, with percent escapes decoded and UTF8 byte sequences decoded into characters. -* ``query_string``: Unicode string URL portion after the ``?``, already - url-decoded, like ``path``. Optional, default is ``""``. ``+`` characters in - this portion should be interpreted by the application to be literal pluses, - not spaces. +* ``query_string``: Byte string URL portion after the ``?``, not url-decoded. * ``root_path``: Unicode string that indicates the root path this application is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults From a9daf0dfbb8a8a1f85f7e9420321161ae2d2d234 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 22 Jun 2016 09:48:03 -0700 Subject: [PATCH 406/746] Releasing 0.15.0 --- CHANGELOG.txt | 7 +++++++ channels/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index d5e244f..4629821 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,10 @@ +0.15.0 (2016-06-22) +------------------- + +* Query strings are now decoded entirely by Django. Must be used with Daphne + 0.13 or higher. + + 0.14.3 (2016-06-21) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 4fb0c47..f6c0b03 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.14.3" +__version__ = "0.15.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From ade39b218c2864501931a7ca9f66a3151b413b22 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 22 Jun 2016 10:05:57 -0700 Subject: [PATCH 407/746] Add release makefile --- Makefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1a1f55e --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +.PHONY: release + +all: + +release: +ifndef version + $(error Please supply a version) +endif + @echo Releasing version $(version) +ifeq (,$(findstring $(version),$(shell git log --oneline -1))) + $(error Last commit does not match version) +endif + git tag $(version) + git push + git push --tags + python setup.py sdist bdist_wheel upload From 2e3e39cd6c85a95db495c09b80a177512de182ed Mon Sep 17 00:00:00 2001 From: Vikalp Jain Date: Thu, 23 Jun 2016 01:31:39 +0530 Subject: [PATCH 408/746] Update handler.py (#217) Remove unnecessary comment --- channels/handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 37793bf..ee9e150 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -135,8 +135,6 @@ class AsgiRequest(http.HttpRequest): @cached_property def GET(self): - # Django will try and re-urldecode the query string and interpret + as space; - # we re-encode it here to fix this. return http.QueryDict(self.message.get('query_string', '')) def _get_post(self): From 98c9db3ba51aad413ec4f7d95732f2f0eab9a691 Mon Sep 17 00:00:00 2001 From: Vikalp Jain Date: Thu, 23 Jun 2016 02:07:32 +0530 Subject: [PATCH 409/746] Update setup.py (#218) Update daphne version in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9920e7f..a982fb6 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,6 @@ setup( install_requires=[ 'Django>=1.8', 'asgiref>=0.13', - 'daphne>=0.12', + 'daphne>=0.13', ] ) From d3c5cc809adc069336e3adf0ae5900772d0586ba Mon Sep 17 00:00:00 2001 From: Iacopo Spalletti Date: Sun, 26 Jun 2016 18:54:54 +0200 Subject: [PATCH 410/746] Add django-knocker to community projects (#222) --- docs/community.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/community.rst b/docs/community.rst index 5c31744..071b42f 100644 --- a/docs/community.rst +++ b/docs/community.rst @@ -4,7 +4,9 @@ Community Projects These projects from the community are developed on top of Channels: * Djangobot_, a bi-directional interface server for Slack. +* knocker_, a generic desktop-notification system. If you'd like to add your project, please submit a PR with a link and brief description. .. _Djangobot: https://github.com/djangobot/djangobot +.. _knocker: https://github.com/nephila/django-knocker From 07d1551306cc077ce0b440cd28f9a87ad54617a9 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Mon, 27 Jun 2016 19:24:16 +0200 Subject: [PATCH 411/746] clarified where you can get the keyword arguments (#225) My initial problem was that i tried to access the keyword arguments in the ctor of the consumer... --- docs/routing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/routing.rst b/docs/routing.rst index e4b5c63..d692906 100644 --- a/docs/routing.rst +++ b/docs/routing.rst @@ -54,7 +54,7 @@ You can have multiple filters:: Multiple filters are always combined with logical AND; that is, you need to match every filter to have the consumer called. -Filters can capture keyword arguments to be passed to your function:: +Filters can capture keyword arguments to be passed to your function or your class based consumers member function as a ``kwarg``:: route("websocket.connect", connect_blog, path=r'^/liveblog/(?P[^/]+)/stream/$') From 15aa962cd768811c6bb5d136eb12ad9d345e6c59 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 27 Jun 2016 10:24:35 -0700 Subject: [PATCH 412/746] Update routing.rst --- docs/routing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/routing.rst b/docs/routing.rst index d692906..bce2edb 100644 --- a/docs/routing.rst +++ b/docs/routing.rst @@ -54,7 +54,7 @@ You can have multiple filters:: Multiple filters are always combined with logical AND; that is, you need to match every filter to have the consumer called. -Filters can capture keyword arguments to be passed to your function or your class based consumers member function as a ``kwarg``:: +Filters can capture keyword arguments to be passed to your function or your class based consumer methods as a ``kwarg``:: route("websocket.connect", connect_blog, path=r'^/liveblog/(?P[^/]+)/stream/$') From 5eb3bf848c77c8dc39c201ca388e3e0e034cb843 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 27 Jun 2016 16:46:47 -0700 Subject: [PATCH 413/746] Provide keyword args as self.kwargs in CBC (ref. #224) --- channels/generic/base.py | 1 + docs/generics.rst | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/channels/generic/base.py b/channels/generic/base.py index 73d97fc..b54d703 100644 --- a/channels/generic/base.py +++ b/channels/generic/base.py @@ -26,6 +26,7 @@ class BaseConsumer(object): the uninstantiated class, so calling it creates it) """ self.message = message + self.kwargs = kwargs self.dispatch(message, **kwargs) @classmethod diff --git a/docs/generics.rst b/docs/generics.rst index 97c013a..7edda19 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -32,7 +32,8 @@ Here's a routing example:: Class-based consumers are instantiated once for each message they consume, so it's safe to store things on ``self`` (in fact, ``self.message`` is the -current message by default). +current message by default, and ``self.kwargs`` are the keyword arguments +passed in from the routing). Base ---- @@ -62,6 +63,13 @@ If you want to perfom more complicated routing, you'll need to override the remember, though, your channel names cannot change during runtime and must always be the same for as long as your process runs. +``BaseConsumer`` and all other generic consumers than inherit from it provide +two instance variables on the class: + +* ``self.message``, the :ref:`Message object ` representing the + message the consumer was called for. +* ``self.kwargs``, keyword arguments from the :doc:`routing` + WebSockets ---------- From 4a09cec2d436f2b38efc6bd8338c33d60a9be08b Mon Sep 17 00:00:00 2001 From: Tim Watts Date: Wed, 29 Jun 2016 20:26:21 +0200 Subject: [PATCH 414/746] Test runserver (#214) * Add tests for runserver and runworker management commands * Fix flake8 and isort errors * Refactor mocking, add comments to tests * rm unneeded vargs --- .coveragerc | 2 +- channels/log.py | 9 +- channels/management/commands/runserver.py | 2 +- channels/management/commands/runworker.py | 2 +- channels/tests/settings.py | 12 ++ channels/tests/test_management.py | 158 ++++++++++++++++++++++ tox.ini | 1 + 7 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 channels/tests/test_management.py diff --git a/.coveragerc b/.coveragerc index 142b3c3..ef6d66c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,6 @@ [run] branch = True -source = channels, django.http.response +source = channels omit = channels/tests/* [report] diff --git a/channels/log.py b/channels/log.py index 9078919..581f5b8 100644 --- a/channels/log.py +++ b/channels/log.py @@ -1,14 +1,16 @@ import logging +handler = logging.StreamHandler() + def setup_logger(name, verbosity=1): """ Basic logger for runserver etc. """ - formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(module)s - %(message)s') + formatter = logging.Formatter( + fmt='%(asctime)s - %(levelname)s - %(module)s - %(message)s') - handler = logging.StreamHandler() handler.setFormatter(formatter) # Set up main logger @@ -22,7 +24,8 @@ def setup_logger(name, verbosity=1): for module in ["daphne.ws_protocol", "daphne.http_protocol"]: daphne_logger = logging.getLogger(module) daphne_logger.addHandler(handler) - daphne_logger.setLevel(logging.DEBUG if verbosity > 1 else logging.INFO) + daphne_logger.setLevel( + logging.DEBUG if verbosity > 1 else logging.INFO) logger.propagate = False return logger diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 26d99fe..49fa7ae 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -2,6 +2,7 @@ import datetime import sys import threading +from daphne.server import Server from django.conf import settings from django.core.management.commands.runserver import \ Command as RunserverCommand @@ -72,7 +73,6 @@ class Command(RunserverCommand): # actually a subthread under the autoreloader. self.logger.debug("Daphne running, listening on %s:%s", self.addr, self.port) try: - from daphne.server import Server Server( channel_layer=self.channel_layer, host=self.addr, diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 81f80b9..2b77020 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -5,8 +5,8 @@ from django.core.management import BaseCommand, CommandError from channels import DEFAULT_CHANNEL_LAYER, channel_layers from channels.log import setup_logger -from channels.worker import Worker from channels.staticfiles import StaticFilesConsumer +from channels.worker import Worker class Command(BaseCommand): diff --git a/channels/tests/settings.py b/channels/tests/settings.py index c06b6b4..d720f33 100644 --- a/channels/tests/settings.py +++ b/channels/tests/settings.py @@ -1,5 +1,13 @@ SECRET_KEY = 'cat' +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.admin', + 'channels', +) + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', @@ -11,6 +19,10 @@ CHANNEL_LAYERS = { 'BACKEND': 'asgiref.inmemory.ChannelLayer', 'ROUTING': [], }, + 'fake_channel': { + 'BACKEND': 'channels.tests.test_management.FakeChannelLayer', + 'ROUTING': [], + } } MIDDLEWARE_CLASSES = [] diff --git a/channels/tests/test_management.py b/channels/tests/test_management.py new file mode 100644 index 0000000..39d8f53 --- /dev/null +++ b/channels/tests/test_management.py @@ -0,0 +1,158 @@ +from __future__ import unicode_literals + +import logging + +from asgiref.inmemory import ChannelLayer +from django.core.management import CommandError, call_command +from channels.staticfiles import StaticFilesConsumer +from django.test import TestCase, mock +from six import StringIO + +from channels.management.commands import runserver + + +class FakeChannelLayer(ChannelLayer): + ''' + Dummy class to bypass the 'inmemory' string check. + ''' + pass + + +@mock.patch('channels.management.commands.runworker.Worker') +class RunWorkerTests(TestCase): + + def setUp(self): + import channels.log + self.stream = StringIO() + channels.log.handler = logging.StreamHandler(self.stream) + + def test_runworker_no_local_only(self, mock_worker): + """ + Runworker should fail with the default "inmemory" worker. + """ + with self.assertRaises(CommandError): + call_command('runworker') + + def test_debug(self, mock_worker): + """ + Test that the StaticFilesConsumer is used in debug mode. + """ + with self.settings(DEBUG=True, STATIC_URL='/static/'): + # Use 'fake_channel' that bypasses the 'inmemory' check + call_command('runworker', '--layer', 'fake_channel') + mock_worker.assert_called_with( + only_channels=None, exclude_channels=None, callback=None, channel_layer=mock.ANY) + + channel_layer = mock_worker.call_args[1]['channel_layer'] + static_consumer = channel_layer.router.root.routing[0].consumer + self.assertIsInstance(static_consumer, StaticFilesConsumer) + + def test_runworker(self, mock_worker): + # Use 'fake_channel' that bypasses the 'inmemory' check + call_command('runworker', '--layer', 'fake_channel') + mock_worker.assert_called_with(callback=None, + only_channels=None, + channel_layer=mock.ANY, + exclude_channels=None) + + def test_runworker_verbose(self, mocked_worker): + # Use 'fake_channel' that bypasses the 'inmemory' check + call_command('runworker', '--layer', + 'fake_channel', '--verbosity', '2') + + # Verify the callback is set + mocked_worker.assert_called_with(callback=mock.ANY, + only_channels=None, + channel_layer=mock.ANY, + exclude_channels=None) + + +class RunServerTests(TestCase): + + def setUp(self): + import channels.log + self.stream = StringIO() + # Capture the logging of the channels moduel to match against the + # output. + channels.log.handler = logging.StreamHandler(self.stream) + + @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) + @mock.patch('channels.management.commands.runserver.Server') + @mock.patch('channels.management.commands.runworker.Worker') + def test_runserver_basic(self, mocked_worker, mocked_server, mock_stdout): + # Django's autoreload util uses threads and this is not needed + # in the test envirionment. + # See: + # https://github.com/django/django/blob/master/django/core/management/commands/runserver.py#L105 + call_command('runserver', '--noreload') + mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, + host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY) + + @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) + @mock.patch('channels.management.commands.runserver.Server') + @mock.patch('channels.management.commands.runworker.Worker') + def test_runserver_debug(self, mocked_worker, mocked_server, mock_stdout): + """ + Test that the server runs with `DEBUG=True`. + """ + # Debug requires the static url is set. + with self.settings(DEBUG=True, STATIC_URL='/static/'): + call_command('runserver', '--noreload') + mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, + host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY) + + call_command('runserver', '--noreload', 'localhost:8001') + mocked_server.assert_called_with(port=8001, signal_handlers=True, http_timeout=60, + host='localhost', action_logger=mock.ANY, channel_layer=mock.ANY) + + self.assertFalse(mocked_worker.called, + "The worker should not be called with '--noworker'") + + @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) + @mock.patch('channels.management.commands.runserver.Server') + @mock.patch('channels.management.commands.runworker.Worker') + def test_runserver_noworker(self, mocked_worker, mocked_server, mock_stdout): + ''' + Test that the Worker is not called when using the `--noworker` parameter. + ''' + call_command('runserver', '--noreload', '--noworker') + mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, + host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY) + self.assertFalse(mocked_worker.called, + "The worker should not be called with '--noworker'") + + @mock.patch('channels.management.commands.runserver.sys.stderr', new_callable=StringIO) + def test_log_action(self, mocked_stderr): + cmd = runserver.Command() + test_actions = [ + (100, 'http', 'complete', + 'HTTP GET /a-path/ 100 [0.12, a-client]'), + (200, 'http', 'complete', + 'HTTP GET /a-path/ 200 [0.12, a-client]'), + (300, 'http', 'complete', + 'HTTP GET /a-path/ 300 [0.12, a-client]'), + (304, 'http', 'complete', + 'HTTP GET /a-path/ 304 [0.12, a-client]'), + (400, 'http', 'complete', + 'HTTP GET /a-path/ 400 [0.12, a-client]'), + (404, 'http', 'complete', + 'HTTP GET /a-path/ 404 [0.12, a-client]'), + (500, 'http', 'complete', + 'HTTP GET /a-path/ 500 [0.12, a-client]'), + (None, 'websocket', 'connected', + 'WebSocket CONNECT /a-path/ [a-client]'), + (None, 'websocket', 'disconnected', + 'WebSocket DISCONNECT /a-path/ [a-client]'), + (None, 'websocket', 'something', ''), # This shouldn't happen + ] + + for status_code, protocol, action, output in test_actions: + details = {'status': status_code, + 'method': 'GET', + 'path': '/a-path/', + 'time_taken': 0.12345, + 'client': 'a-client'} + cmd.log_action(protocol, action, details) + self.assertIn(output, mocked_stderr.getvalue()) + # Clear previous output + mocked_stderr.truncate(0) diff --git a/tox.ini b/tox.ini index 51b3fc2..69a082a 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ setenv = deps = autobahn coverage + daphne asgiref>=0.9 six redis==2.10.5 From 92012fbc272afb1f17bf60842c951aa8e08d9461 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 29 Jun 2016 12:16:17 -0700 Subject: [PATCH 415/746] Fixed #87: Don't drop headers and status on empty streaming responses --- channels/handler.py | 5 ++-- channels/tests/test_handler.py | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index ee9e150..42cb44f 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -288,9 +288,8 @@ class AsgiHandler(base.BaseHandler): yield message message = {} # Final closing message - yield { - "more_content": False, - } + message["more_content"] = False + yield message # Other responses just need chunking else: # Yield chunks of response diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index 255f0b1..d529c01 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -143,6 +143,54 @@ class HandlerTests(ChannelTestCase): self.assertEqual(reply_messages[1]["content"], b"andhereistherest") self.assertEqual(reply_messages[1].get("more_content", False), False) + def test_empty(self): + """ + Tests an empty response + """ + # Make stub request and desired response + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = HttpResponse(b"", status=304) + # Run the handler + handler = FakeAsgiHandler(response) + reply_messages = list( + handler(self.get_next_message("test", require=True)) + ) + # Make sure we got the right number of messages + self.assertEqual(len(reply_messages), 1) + # Make sure the messages look correct + self.assertEqual(reply_messages[0].get("content", b""), b"") + self.assertEqual(reply_messages[0]["status"], 304) + self.assertEqual(reply_messages[0]["more_content"], False) + + def test_empty_streaming(self): + """ + Tests an empty streaming response + """ + # Make stub request and desired response + Channel("test").send({ + "reply_channel": "test", + "http_version": "1.1", + "method": "GET", + "path": b"/test/", + }) + response = StreamingHttpResponse([], status=304) + # Run the handler + handler = FakeAsgiHandler(response) + reply_messages = list( + handler(self.get_next_message("test", require=True)) + ) + # Make sure we got the right number of messages + self.assertEqual(len(reply_messages), 1) + # Make sure the messages look correct + self.assertEqual(reply_messages[0].get("content", b""), b"") + self.assertEqual(reply_messages[0]["status"], 304) + self.assertEqual(reply_messages[0]["more_content"], False) + def test_chunk_bytes(self): """ Makes sure chunk_bytes works correctly From efcf08d7683ad53a445f738dc5c45679361df11b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 29 Jun 2016 14:54:01 -0700 Subject: [PATCH 416/746] Releasing 0.15.1 --- CHANGELOG.txt | 8 ++++++++ channels/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 4629821..f375c4f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +0.15.1 (2016-06-29) +------------------- + +* Class based consumers now have a self.kwargs + +* Fixed bug where empty streaming responses did not send headers or status code + + 0.15.0 (2016-06-22) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index f6c0b03..cf34394 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.0" +__version__ = "0.15.1" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From b9519d112dd93f9e05a7e3b584d804e8fd860b45 Mon Sep 17 00:00:00 2001 From: Vikalp Jain Date: Fri, 1 Jul 2016 21:36:56 +0530 Subject: [PATCH 417/746] Fix issue with calling super setUp while test cases (#231) --- channels/tests/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/channels/tests/base.py b/channels/tests/base.py index bbff7dc..b42914e 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -26,11 +26,11 @@ class ChannelTestCase(TestCase): # Customizable so users can test multi-layer setups test_channel_aliases = [DEFAULT_CHANNEL_LAYER] - def setUp(self): + def _pre_setup(self): """ Initialises in memory channel layer for the duration of the test """ - super(ChannelTestCase, self).setUp() + super(ChannelTestCase, self)._pre_setup() self._old_layers = {} for alias in self.test_channel_aliases: # Swap in an in memory layer wrapper and keep the old one around @@ -43,7 +43,7 @@ class ChannelTestCase(TestCase): ) ) - def tearDown(self): + def _post_teardown(self): """ Undoes the channel rerouting """ @@ -51,7 +51,7 @@ class ChannelTestCase(TestCase): # Swap in an in memory layer wrapper and keep the old one around channel_layers.set(alias, self._old_layers[alias]) del self._old_layers - super(ChannelTestCase, self).tearDown() + super(ChannelTestCase, self)._post_teardown() def get_next_message(self, channel, alias=DEFAULT_CHANNEL_LAYER, require=False): """ From e947e331ced21a1dccc2026d0e4df173865aee8d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 1 Jul 2016 09:39:20 -0700 Subject: [PATCH 418/746] Add groups section of testing doc --- docs/testing.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/testing.rst b/docs/testing.rst index 20c7732..0681835 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -62,6 +62,27 @@ and post the square of it to the ``"result"`` channel:: self.assertEqual(result['value'], 1089) +Groups +------ + +You can test Groups in the same way as Channels inside a ``ChannelTestCase``; +the entire channel layer is flushed each time a test is run, so it's safe to +do group adds and sends during a test. For example:: + + from channels import Channel + from channels.tests import ChannelTestCase + + class MyTests(ChannelTestCase): + def test_a_thing(self): + # Add a test channel to a test group + Group("test-group").add("test-channel") + # Send to the group + Group("test-group").send({"value": 42}) + # Verify the message got into the destination channel + result = self.get_next_message("test-channel", require=True) + self.assertEqual(result['value'], 42) + + Multiple Channel Layers ----------------------- From 69168545d45204e981454d254cfea14d80adcd49 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 6 Jul 2016 11:37:10 -0700 Subject: [PATCH 419/746] Update ASGI spec with backpressure instructions --- docs/asgi.rst | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index d53207b..811a3c3 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -507,7 +507,9 @@ the response ``headers`` must be sent as a list of tuples, which matches WSGI. Request ''''''' -Sent once for each request that comes into the protocol server. +Sent once for each request that comes into the protocol server. If sending +this raises ``ChannelFull``, the interface server must respond with a +500-range error, preferably ``503 Service Unavailable``, and close the connection. Channel: ``http.request`` @@ -561,7 +563,10 @@ Keys: Request Body Chunk '''''''''''''''''' -Must be sent after an initial Response. +Must be sent after an initial Response. If trying to send this raises +``ChannelFull``, the interface server should wait and try again until it is +accepted (the consumer at the other end of the channel may not be as fast +consuming the data as the client is at sending it). Channel: ``http.request.body?`` @@ -584,7 +589,9 @@ Keys: Response '''''''' -Send after any server pushes, and before any response chunks. +Send after any server pushes, and before any response chunks. If ``ChannelFull`` +is encountered, wait and try again later, optionally giving up after a +predetermined timeout. Channel: ``http.response!`` @@ -609,7 +616,8 @@ Keys: Response Chunk '''''''''''''' -Must be sent after an initial Response. +Must be sent after an initial Response. If ``ChannelFull`` +is encountered, wait and try again later. Channel: ``http.response!`` @@ -627,7 +635,10 @@ Keys: Server Push ''''''''''' -Must be sent before any Response or Response Chunk messages. +Must be sent before any Response or Response Chunk messages. If ``ChannelFull`` +is encountered, wait and try again later, optionally giving up after a +predetermined timeout, and give up on the entire response this push is +connected to. When a server receives this message, it must treat the Request message in the ``request`` field of the Server Push as though it were a new HTTP request being @@ -664,6 +675,9 @@ Sent when a HTTP connection is closed. This is mainly useful for long-polling, where you may have added the response channel to a Group or other set of channels you want to trigger a reply to when data arrives. +If ``ChannelFull`` is raised, then give up attempting to send the message; +consumption is not required. + Channel: ``http.disconnect`` Keys: @@ -683,12 +697,18 @@ should store them in a cache or database. WebSocket protocol servers should handle PING/PONG requests themselves, and send PING frames as necessary to ensure the connection is alive. +Note that you **must** ensure that websocket.connect is consumed; if an +interface server gets ``ChannelFull`` on this channel it will drop the +connection. Django Channels ships with a no-op consumer attached by default; +we recommend other implementations do the same. + Connection '''''''''' Sent when the client initially opens a connection and completes the -WebSocket handshake. +WebSocket handshake. If sending this raises ``ChannelFull``, the interface +server must drop the WebSocket connection and send no more messages about it. Channel: ``websocket.connect`` @@ -728,7 +748,8 @@ Keys: Receive ''''''' -Sent when a data frame is received from the client. +Sent when a data frame is received from the client. If ``ChannelFull`` is +raised, wait and try again. Channel: ``websocket.receive`` @@ -755,6 +776,9 @@ Sent when either connection to the client is lost, either from the client closing the connection, the server closing the connection, or loss of the socket. +If ``ChannelFull`` is raised, then give up attempting to send the message; +consumption is not required. + Channel: ``websocket.disconnect`` Keys: @@ -773,7 +797,7 @@ Send/Close '''''''''' Sends a data frame to the client and/or closes the connection from the -server end. +server end. If ``ChannelFull`` is raised, wait and try again. Channel: ``websocket.send!`` From be127611e5df506f1b448c7ba5b311de10308273 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 6 Jul 2016 11:55:45 -0700 Subject: [PATCH 420/746] Implement default websocket.connect consumer. --- channels/routing.py | 10 ++++++++++ setup.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/channels/routing.py b/channels/routing.py index 83108ba..a5593a6 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -53,6 +53,10 @@ class Router(object): # called once, thankfully. from .handler import ViewConsumer self.add_route(Route("http.request", http_consumer or ViewConsumer())) + # We also add a no-op websocket.connect consumer to the bottom, as the + # spec requires that this is consumed, but Channels does not. Any user + # consumer will override this one. + self.add_route(Route("websocket.connect", null_consumer)) @classmethod def resolve_routing(cls, routing): @@ -239,6 +243,12 @@ class Include(object): return result +def null_consumer(*args, **kwargs): + """ + Standard no-op consumer. + """ + + # Lowercase standard to match urls.py route = Route route_class = RouteClass diff --git a/setup.py b/setup.py index a982fb6..6f3f0f6 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,6 @@ setup( install_requires=[ 'Django>=1.8', 'asgiref>=0.13', - 'daphne>=0.13', + 'daphne>=0.14', ] ) From d37f9d1ab3034e6eba425274f529fbcb162a7581 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 6 Jul 2016 12:10:17 -0700 Subject: [PATCH 421/746] Add null consumer for websocket.receive too. --- channels/routing.py | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/channels/routing.py b/channels/routing.py index a5593a6..76c2464 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -55,8 +55,9 @@ class Router(object): self.add_route(Route("http.request", http_consumer or ViewConsumer())) # We also add a no-op websocket.connect consumer to the bottom, as the # spec requires that this is consumed, but Channels does not. Any user - # consumer will override this one. + # consumer will override this one. Same for websocket.receive. self.add_route(Route("websocket.connect", null_consumer)) + self.add_route(Route("websocket.receive", null_consumer)) @classmethod def resolve_routing(cls, routing): diff --git a/setup.py b/setup.py index 6f3f0f6..673eb15 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,6 @@ setup( install_requires=[ 'Django>=1.8', 'asgiref>=0.13', - 'daphne>=0.14', + 'daphne>=0.14.1', ] ) From 9cebff05ab4bb3862ae1bfc0334044dccfff34d6 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 6 Jul 2016 12:10:41 -0700 Subject: [PATCH 422/746] Releasing 0.16.0 --- CHANGELOG.txt | 10 ++++++++++ channels/__init__.py | 2 +- docs/asgi.rst | 5 +++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index f375c4f..d69a2df 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,13 @@ +0.16.0 (2016-07-06) +------------------- + +* websocket.connect and websocket.receive are now consumed by a no-op consumer + by default if you don't specify anything to consume it, to bring Channels in + line with the ASGI rules on WebSocket backpressure. + +* You no longer need to call super's setUp in ChannelTestCase. + + 0.15.1 (2016-06-29) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index cf34394..528ca60 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.1" +__version__ = "0.16.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/asgi.rst b/docs/asgi.rst index 811a3c3..85c7180 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -708,7 +708,7 @@ Connection Sent when the client initially opens a connection and completes the WebSocket handshake. If sending this raises ``ChannelFull``, the interface -server must drop the WebSocket connection and send no more messages about it. +server must close the WebSocket connection with error code 1013. Channel: ``websocket.connect`` @@ -749,7 +749,8 @@ Receive ''''''' Sent when a data frame is received from the client. If ``ChannelFull`` is -raised, wait and try again. +raised, you may retry sending it but if it does not send the socket must +be closed with websocket error code 1013. Channel: ``websocket.receive`` From 2e5826418b7b5e9dc4f05d0546bb007d87aed337 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 12 Jul 2016 14:26:01 -0700 Subject: [PATCH 423/746] Fixed #221: WebSocket class based consumer now has http user support --- channels/generic/websockets.py | 13 +++++++++++++ docs/generics.rst | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 096dee1..f07ee25 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -1,6 +1,7 @@ import json from ..channel import Group +from ..auth import channel_session_user_from_http from ..sessions import enforce_ordering from .base import BaseConsumer @@ -18,6 +19,10 @@ class WebsocketConsumer(BaseConsumer): "websocket.disconnect": "raw_disconnect", } + # Turning this on passes the user over from the HTTP session on connect, + # implies channel_session_user + http_user = False + # Set one to True if you want the class to enforce ordering for you slight_ordering = False strict_ordering = False @@ -27,8 +32,16 @@ class WebsocketConsumer(BaseConsumer): Pulls out the path onto an instance variable, and optionally adds the ordering decorator. """ + # HTTP user implies channel session user + if self.http_user: + self.channel_session_user = True + # Get super-handler self.path = message['path'] handler = super(WebsocketConsumer, self).get_handler(message, **kwargs) + # Optionally apply HTTP transfer + if self.http_user: + handler = channel_session_user_from_http(handler) + # Ordering decorators if self.strict_ordering: return enforce_ordering(handler, slight=False) elif self.slight_ordering: diff --git a/docs/generics.rst b/docs/generics.rst index 7edda19..1272f22 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -84,6 +84,10 @@ The basic WebSocket generic consumer is used like this:: class MyConsumer(WebsocketConsumer): + # Set to True to automatically port users from HTTP cookies + # (you don't need channel_session_user, this implies it) + http_user = True + # Set to True if you want them, else leave out strict_ordering = False slight_ordering = False From a05f7d5a96f155d5b5137e9e489c6b8e592ee81f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 12 Jul 2016 14:40:19 -0700 Subject: [PATCH 424/746] Fixed #160: _read_started set to False on request This allows read_post_and_files to work. --- channels/handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/handler.py b/channels/handler.py index 42cb44f..3e8912e 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -39,6 +39,7 @@ class AsgiRequest(http.HttpRequest): self.reply_channel = self.message.reply_channel self._content_length = 0 self._post_parse_error = False + self._read_started = False self.resolver_match = None # Path info self.path = self.message['path'] From e7a354e03c50780474b77b6fc8b0bfe6012829b6 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 12 Jul 2016 15:01:19 -0700 Subject: [PATCH 425/746] Fixed #148: Close database connections when consumers finish. --- channels/signals.py | 9 +++++++++ channels/worker.py | 7 +++++++ 2 files changed, 16 insertions(+) create mode 100644 channels/signals.py diff --git a/channels/signals.py b/channels/signals.py new file mode 100644 index 0000000..fbc6f43 --- /dev/null +++ b/channels/signals.py @@ -0,0 +1,9 @@ +from django.db import close_old_connections +from django.dispatch import Signal + + +consumer_started = Signal(providing_args=["environ"]) +consumer_finished = Signal() + +# Connect connection closer to consumer finished as well +consumer_finished.connect(close_old_connections) diff --git a/channels/worker.py b/channels/worker.py index 32f5edb..f6d93d7 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -6,6 +6,7 @@ import signal import sys import time +from .signals import consumer_started, consumer_finished from .exceptions import ConsumeLater from .message import Message from .utils import name_that_thing @@ -104,6 +105,9 @@ class Worker(object): self.callback(channel, message) try: logger.debug("Dispatching message on %s to %s", channel, name_that_thing(consumer)) + # Send consumer started to manage lifecycle stuff + consumer_started.send(sender=self.__class__, environ={}) + # Run consumer consumer(message, **kwargs) except ConsumeLater: # They want to not handle it yet. Re-inject it with a number-of-tries marker. @@ -127,3 +131,6 @@ class Worker(object): break except: logger.exception("Error processing message with consumer %s:", name_that_thing(consumer)) + else: + # Send consumer finished so DB conns close etc. + consumer_finished.send(sender=self.__class__) From 27d064328a145303321acc99eb9ce8c04fcf688c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 12 Jul 2016 15:13:05 -0700 Subject: [PATCH 426/746] Releasing 0.16.1 --- CHANGELOG.txt | 11 +++++++++++ channels/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index d69a2df..9c57c28 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,14 @@ +0.16.1 (2016-07-12) +------------------- + +* WebsocketConsumer now has a http_user option for auto user sessions. + +* consumer_started and consumer_finished signals are now available under + channels.signals. + +* Database connections are closed whenever a consumer finishes. + + 0.16.0 (2016-07-06) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 528ca60..2aff6ec 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.0" +__version__ = "0.16.1" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From af606ff89505dca33d9716f14342544c4cbbb80d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 13 Jul 2016 17:19:39 -0700 Subject: [PATCH 427/746] Fixed #244: .close() on Websocket generic consumers --- channels/generic/websockets.py | 6 ++++++ docs/generics.rst | 3 +++ 2 files changed, 9 insertions(+) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index f07ee25..38504f0 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -106,6 +106,12 @@ class WebsocketConsumer(BaseConsumer): else: raise ValueError("You must pass text or bytes") + def close(self): + """ + Closes the WebSocket from the server end + """ + self.message.reply_channel.send({"close": True}) + def raw_disconnect(self, message, **kwargs): """ Called when a WebSocket connection is closed. Base level so you don't diff --git a/docs/generics.rst b/docs/generics.rst index 1272f22..326f0c0 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -172,6 +172,9 @@ Note that this subclass still can't intercept ``Group.send()`` calls to make them into JSON automatically, but it does provide ``self.group_send(name, content)`` that will do this for you if you call it explicitly. +``self.close()`` is also provided to easily close the WebSocket from the server +end once you are done with it. + Sessions and Users ------------------ From 62d4782dbd497d04f78e23f0f14b05d030509fe5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 14 Jul 2016 23:15:57 -0700 Subject: [PATCH 428/746] First version of binding code --- channels/apps.py | 4 + channels/binding/__init__.py | 1 + channels/binding/base.py | 176 +++++++++++++++++++++++++++++++++ channels/binding/websockets.py | 79 +++++++++++++++ 4 files changed, 260 insertions(+) create mode 100644 channels/binding/__init__.py create mode 100644 channels/binding/base.py create mode 100644 channels/binding/websockets.py diff --git a/channels/apps.py b/channels/apps.py index fec19cd..f2d7874 100644 --- a/channels/apps.py +++ b/channels/apps.py @@ -1,6 +1,8 @@ from django.apps import AppConfig from django.core.exceptions import ImproperlyConfigured +from .binding.base import BindingMetaclass + class ChannelsConfig(AppConfig): @@ -18,3 +20,5 @@ class ChannelsConfig(AppConfig): # Do django monkeypatches from .hacks import monkeypatch_django monkeypatch_django() + # Instantiate bindings + BindingMetaclass.register_all() diff --git a/channels/binding/__init__.py b/channels/binding/__init__.py new file mode 100644 index 0000000..5e54ac1 --- /dev/null +++ b/channels/binding/__init__.py @@ -0,0 +1 @@ +from .base import Binding diff --git a/channels/binding/base.py b/channels/binding/base.py new file mode 100644 index 0000000..39db69a --- /dev/null +++ b/channels/binding/base.py @@ -0,0 +1,176 @@ +from __future__ import unicode_literals + +import six + +from django.apps import apps +from django.db.models.signals import post_save, post_delete + +from ..channel import Group + + +class BindingMetaclass(type): + """ + Metaclass that tracks instantiations of its type. + """ + + binding_classes = [] + + def __new__(cls, name, bases, body): + klass = type.__new__(cls, name, bases, body) + if bases != (object, ): + cls.binding_classes.append(klass) + return klass + + @classmethod + def register_all(cls): + for binding_class in cls.binding_classes: + binding_class.register() + + +@six.add_metaclass(BindingMetaclass) +class Binding(object): + """ + Represents a two-way data binding from channels/groups to a Django model. + Outgoing binding sends model events to zero or more groups. + Incoming binding takes messages and maybe applies the action based on perms. + + To implement outbound, implement: + - group_names, which returns a list of group names to send to + - serialize, which returns message contents from an instance + action + + To implement inbound, implement: + - deserialize, which returns pk, data and action from message contents + - has_permission, which says if the user can do the action on an instance + - create, which takes the data and makes a model instance + - update, which takes data and a model instance and applies one to the other + + Outbound will work once you implement the functions; inbound requires you + to place one or more bindings inside a protocol-specific Demultiplexer + and tie that in as a consumer. + """ + + model = None + + @classmethod + def register(cls): + """ + Resolves models. + """ + # If model is None directly on the class, assume it's abstract. + if cls.model is None: + if "model" in cls.__dict__: + return + else: + raise ValueError("You must set the model attribute on Binding %r!" % cls) + # Optionally resolve model strings + if isinstance(cls.model, six.string_types): + cls.model = apps.get_model(cls.model) + # Connect signals + post_save.connect(cls.save_receiver, sender=cls.model) + post_delete.connect(cls.delete_receiver, sender=cls.model) + + # Outbound binding + + @classmethod + def save_receiver(cls, instance, created, **kwargs): + """ + Entry point for triggering the binding from save signals. + """ + cls.trigger_outbound(instance, "create" if created else "update") + + @classmethod + def delete_receiver(cls, instance, **kwargs): + """ + Entry point for triggering the binding from save signals. + """ + cls.trigger_outbound(instance, "delete") + + @classmethod + def trigger_outbound(cls, instance, action): + """ + Triggers the binding to possibly send to its group. + """ + self = cls() + self.instance = instance + # Check to see if we're covered + for group_name in self.group_names(instance, action): + group = Group(group_name) + group.send(self.serialize(instance, action)) + + def group_names(self, instance, action): + """ + Returns the iterable of group names to send the object to based on the + instance and action performed on it. + """ + raise NotImplementedError() + + def serialize(self, instance, action): + """ + Should return a serialized version of the instance to send over the + wire (return value must be a dict suitable for sending over a channel - + e.g., to send JSON as a WebSocket text frame, you must return + {"text": json.dumps(instance_serialized_as_dict)} + """ + raise NotImplementedError() + + # Inbound binding + + @classmethod + def trigger_inbound(cls, message): + """ + Triggers the binding to see if it will do something. + We separate out message serialization to a consumer, so this gets + native arguments. + """ + # Late import as it touches models + from django.contrib.auth.models import AnonymousUser + self = cls() + self.message = message + # Deserialize message + self.action, self.pk, self.data = self.deserialize(self.message) + self.user = getattr(self.message, "user", AnonymousUser()) + # Run incoming action + self.run_action(self.action, self.pk, self.data) + + def deserialize(self, message): + """ + Returns action, pk, data decoded from the message. pk should be None + if action is create; data should be None if action is delete. + """ + raise NotImplementedError() + + def has_permission(self, user, action, pk): + """ + Return True if the user can do action to the pk, False if not. + User may be AnonymousUser if no auth hooked up/they're not logged in. + Action is one of "create", "delete", "update". + """ + raise NotImplementedError() + + def run_action(self, action, pk, data): + """ + Performs the requested action. This version dispatches to named + functions by default for update/create, and handles delete itself. + """ + # Check to see if we're allowed + if self.has_permission(self.user, pk): + if action == "create": + self.create(data) + elif action == "update": + self.update(self.model.get(pk=pk), data) + elif action == "delete": + self.model.filter(pk=pk).delete() + else: + raise ValueError("Bad action %r" % action) + + def create(self, data): + """ + Creates a new instance of the model with the data. + """ + raise NotImplementedError() + + def update(self, instance, data): + """ + Updates the model with the data. + """ + raise NotImplementedError() diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py new file mode 100644 index 0000000..d664d27 --- /dev/null +++ b/channels/binding/websockets.py @@ -0,0 +1,79 @@ +import json + +from .base import Binding +from ..generic.websockets import JsonWebsocketConsumer + + +class WebsocketBinding(Binding): + """ + Websocket-specific outgoing binding subclass that uses JSON encoding. + + To implement outbound, implement: + - group_names, which returns a list of group names to send to + - serialize_data, which returns JSON-safe data from a model instance + + To implement inbound, implement: + - has_permission, which says if the user can do the action on an instance + - create, which takes incoming data and makes a model instance + - update, which takes incoming data and a model instance and applies one to the other + """ + + # Mark as abstract + + model = None + + # Outbound + + def serialize(self, instance, action): + return { + "text": json.dumps({ + "model": "%s.%s" % ( + instance._meta.app_label.lower(), + instance._meta.object_name.lower(), + ), + "action": action, + "pk": instance.pk, + "data": self.serialize_data(instance), + }), + } + + def serialize_data(self, instance): + """ + Serializes model data into JSON-compatible types. + """ + raise NotImplementedError() + + # Inbound + + def deserialize(self, message): + content = json.loads(message['text']) + action = content['action'] + pk = content.get('pk', None) + data = content.get('data', None) + return action, pk, data + + +class WebsocketBindingDemultiplexer(JsonWebsocketConsumer): + """ + Allows you to combine multiple Bindings as one websocket consumer. + Subclass and provide a custom list of Bindings. + """ + + http_user = True + warn_if_no_match = True + bindings = None + + def receive(self, content): + # Sanity check + if self.bindings is None: + raise ValueError("Demultiplexer has no bindings!") + # Find the matching binding + model_label = content['model'] + triggered = False + for binding in self.bindings: + if binding.model_label == model_label: + binding.trigger_inbound(self.message) + triggered = True + # At least one of them should have fired. + if not triggered and self.warn_if_no_match: + raise ValueError("No binding found for model %s" % model_label) From 15cc5571dab67d43dc721c46d14c677d1bbaf187 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 14 Jul 2016 23:34:12 -0700 Subject: [PATCH 429/746] Fix a few model bits on the bindings --- channels/binding/base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index 39db69a..d2b9728 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -65,6 +65,10 @@ class Binding(object): # Optionally resolve model strings if isinstance(cls.model, six.string_types): cls.model = apps.get_model(cls.model) + cls.model_label = "%s.%s" % ( + cls.model._meta.app_label.lower(), + cls.model._meta.object_name.lower(), + ) # Connect signals post_save.connect(cls.save_receiver, sender=cls.model) post_delete.connect(cls.delete_receiver, sender=cls.model) @@ -153,13 +157,13 @@ class Binding(object): functions by default for update/create, and handles delete itself. """ # Check to see if we're allowed - if self.has_permission(self.user, pk): + if self.has_permission(self.user, action, pk): if action == "create": self.create(data) elif action == "update": - self.update(self.model.get(pk=pk), data) + self.update(self.model.objects.get(pk=pk), data) elif action == "delete": - self.model.filter(pk=pk).delete() + self.model.objects.filter(pk=pk).delete() else: raise ValueError("Bad action %r" % action) From 8a107a543429dee02ff21e2b4c1d7b537f384ebd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 14 Jul 2016 23:53:08 -0700 Subject: [PATCH 430/746] Fix QA error --- channels/binding/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/binding/__init__.py b/channels/binding/__init__.py index 5e54ac1..37b12a0 100644 --- a/channels/binding/__init__.py +++ b/channels/binding/__init__.py @@ -1 +1 @@ -from .base import Binding +from .base import Binding # NOQA isort:skip From 6fd83f04f8b350dfaa880efc969649fecf44d5bd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 16 Jul 2016 23:04:32 -0700 Subject: [PATCH 431/746] Add group_channels --- docs/asgi.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/asgi.rst b/docs/asgi.rst index 85c7180..13f9b7b 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -348,6 +348,11 @@ A channel layer implementing the ``groups`` extension must also provide: * ``group_discard(group, channel)``, a callable that removes the ``channel`` from the ``group`` if it is in it, and does nothing otherwise. +* ``group_channels(group)``, a callable that returns an iterable which yields + all of the group's member channel names. The return value should be serializable + with regards to local adds and discards, but best-effort with regards to + adds and discards on other nodes. + * ``send_group(group, message)``, a callable that takes two positional arguments; the group to send to, as a unicode string, and the message to send, as a serializable ``dict``. It may raise MessageTooLarge but cannot From 5d2354c71b6094e2fa4eab7693f33d5af6436d73 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 18 Jul 2016 14:57:19 -0400 Subject: [PATCH 432/746] Provide default serializers for the JSON one --- channels/binding/base.py | 12 +++++++++--- channels/binding/websockets.py | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index d2b9728..0afbdf1 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -161,9 +161,9 @@ class Binding(object): if action == "create": self.create(data) elif action == "update": - self.update(self.model.objects.get(pk=pk), data) + self.update(pk, data) elif action == "delete": - self.model.objects.filter(pk=pk).delete() + self.delete(pk) else: raise ValueError("Bad action %r" % action) @@ -173,8 +173,14 @@ class Binding(object): """ raise NotImplementedError() - def update(self, instance, data): + def update(self, pk, data): """ Updates the model with the data. """ raise NotImplementedError() + + def delete(self, pk): + """ + Deletes the model instance. + """ + self.model.objects.filter(pk=pk).delete() diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index d664d27..27f6563 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -1,5 +1,7 @@ import json +from django.core import serializers + from .base import Binding from ..generic.websockets import JsonWebsocketConsumer @@ -10,10 +12,12 @@ class WebsocketBinding(Binding): To implement outbound, implement: - group_names, which returns a list of group names to send to - - serialize_data, which returns JSON-safe data from a model instance To implement inbound, implement: - has_permission, which says if the user can do the action on an instance + + Optionally also implement: + - serialize_data, which returns JSON-safe data from a model instance - create, which takes incoming data and makes a model instance - update, which takes incoming data and a model instance and applies one to the other """ @@ -41,7 +45,8 @@ class WebsocketBinding(Binding): """ Serializes model data into JSON-compatible types. """ - raise NotImplementedError() + data = serializers.serialize('json', [instance]) + return json.loads(data)[0]['fields'] # Inbound @@ -52,6 +57,31 @@ class WebsocketBinding(Binding): data = content.get('data', None) return action, pk, data + def _hydrate(self, pk, data): + """ + Given a raw "data" section of an incoming message, returns a + DeserializedObject. + """ + s_data = [ + { + "pk": pk, + "model": self.model_label, + "fields": data, + } + ] + # TODO: Avoid the JSON roundtrip by using encoder directly? + return list(serializers.deserialize("json", json.dumps(s_data)))[0] + + def create(self, data): + self._hydrate(None, data).save() + + def update(self, pk, data): + instance = self.model.objects.get(pk=pk) + hydrated = self._hydrate(pk, data) + for name in data.keys(): + setattr(instance, name, getattr(hydrated.object, name)) + instance.save() + class WebsocketBindingDemultiplexer(JsonWebsocketConsumer): """ From d9e8fb703241e200d3709e4445c3a0039a462571 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 18 Jul 2016 22:23:33 -0400 Subject: [PATCH 433/746] Docs updates --- docs/binding.rst | 142 +++++++++++++++++++++++++++++++++++++++ docs/getting-started.rst | 2 +- docs/index.rst | 1 + 3 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 docs/binding.rst diff --git a/docs/binding.rst b/docs/binding.rst new file mode 100644 index 0000000..e795d6a --- /dev/null +++ b/docs/binding.rst @@ -0,0 +1,142 @@ +Data Binding +============ + +The Channels data binding framework automates the process of tying Django +models into frontend views, such as javascript-powered website UIs. It provides +a quick and flexible way to generate messages on Groups for model changes +and to accept messages that chanes models themselves. + +The main target for the moment is WebSockets, but the framework is flexible +enough to be used over any protocol. + +What does data binding allow? +----------------------------- + +Data binding in Channels works two ways: + +* Outbound, where model changes made through Django are sent out to listening + clients. This includes creation, update and deletion of instances. + +* Inbound, where a standardised message format allow creation, update and + deletion of instances to be made by clients sending messages. + +Combined, these allow a UI to be designed that automatically updates to +reflect new values and reflects across clients. A live blog is easily done +using data binding against the post object, for example, or an edit interface +can show data live as it's edited by other users. + +It has some limitations: + +* Signals are used to power outbound binding, so if you change the values of + a model outside of Django (or use the ``.update()`` method on a QuerySet), + the signals are not triggered and the change will not be sent out. You + can trigger changes yourself, but you'll need to source the events from the + right place for your system. + +* The built-in serializers are based on the built-in Django ones and can only + handle certain field types; for more flexibility, you can plug in something + like the Django REST Framework serializers. + +Getting Started +--------------- + +A single Binding subclass will handle outbound and inbound binding for a model, +and you can have multiple bindings per model (if you want different formats +or permission checks, for example). + +You can inherit from the base Binding and provide all the methods needed, but +we'll focus on the WebSocket JSON variant here, as it's the easiest thing to +get started and likely close to what you want. Start off like this:: + + from django.db import models + from channels.binding.websockets import WebsocketBinding + + class IntegerValue(models.Model): + + name = models.CharField(max_length=100, unique=True) + value = models.IntegerField(default=0) + + class IntegerValueBinding(WebsocketBinding): + + model = IntegerValue + + def group_names(self, instance, action): + return ["intval-updates"] + + def has_permission(self, user, action, pk): + return True + +This defines a WebSocket binding - so it knows to send outgoing messages +formatted as JSON WebSocket frames - and provides the two methods you must +always provide: + +* ``group_names`` returns a list of groups to send outbound updates to based + on the model and action. For example, you could dispatch posts on different + liveblogs to groups that included the parent blog ID in the name; here, we + just use a fixed group name. + +* ``has_permission`` returns if an inbound binding update is allowed to actually + be carried out on the model. We've been very unsafe and made it always return + ``True``, but here is where you would check against either Django's or your + own permission system to see if the user is allowed that action. + +For reference, ``action`` is always one of the unicode strings ``"create"``, +``"update"`` or ``"delete"``. + +Just adding the binding like this in a place where it will be imported will +get outbound messages sending, but you still need a Consumer that will both +accept incoming binding updates and add people to the right Groups when they +connect. For that, you need the other part of the WebSocket binding module, +the demultiplexer:: + + from channels.binding.websockets import WebsocketBindingDemultiplexer + from .models import IntegerValueBinding + + class BindingConsumer(WebsocketBindingDemultiplexer): + + bindings = [ + IntegerValueBinding, + ] + + def connection_groups(self): + return ["intval-updates"] + +This class needs two things set: + +* ``bindings``, a list of Binding subclasses (the ones from before) of the + models you want this to receive messages for. The socket will take care of + looking for what model the incoming message is and giving it to the correct + Binding. + +* ``connection_groups``, a list of groups to put people in when they connect. + This should match the logic of ``group_names`` on your binding - we've used + our fixed group name again. + +Tie that into your routing and you're ready to go:: + + from channels import route_class + from .consumers import BindingConsumer + + channel_routing = [ + route_class(BindingConsumer, path="^binding/"), + ] + + +Frontend Considerations +----------------------- + +Channels is a Python library, and so does not provide any JavaScript to tie +the binding into your JavaScript (though hopefully some will appear over time). +It's not very hard to write your own; messages are all in JSON format, and +have a key of ``action`` to tell you what's happening and ``model`` with the +Django label of the model they're on. + + +Custom Serialization/Protocols +------------------------------ + +Rather than inheriting from the ``WebsocketBinding``, you can inherit directly +from the base ``Binding`` class and implement serialization and deserialization +yourself. Until proper reference documentation for this is written, we +recommend looking at the source code in ``channels/bindings/base.py``; it's +reasonably well-commented. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 4e2838b..b52f723 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -60,7 +60,7 @@ Here's what that looks like:: "ROUTING": "myproject.routing.channel_routing", }, } -.. + :: # In routing.py diff --git a/docs/index.rst b/docs/index.rst index 6b87e62..e34db68 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ Contents: deploying generics routing + binding backends testing cross-compat From cbe6afff8572bc6407f61d203e78d5b7ba82db56 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 18 Jul 2016 23:12:44 -0400 Subject: [PATCH 434/746] Why not rewrite binding into multiplexers on a Monday night? --- channels/binding/base.py | 7 ++-- channels/binding/websockets.py | 62 ++++++++++++---------------------- channels/generic/websockets.py | 56 +++++++++++++++++++++++++++++- docs/binding.rst | 5 +++ 4 files changed, 86 insertions(+), 44 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index 0afbdf1..9abbce6 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -120,11 +120,10 @@ class Binding(object): # Inbound binding @classmethod - def trigger_inbound(cls, message): + def trigger_inbound(cls, message, **kwargs): """ Triggers the binding to see if it will do something. - We separate out message serialization to a consumer, so this gets - native arguments. + Also acts as a consumer. """ # Late import as it touches models from django.contrib.auth.models import AnonymousUser @@ -136,6 +135,8 @@ class Binding(object): # Run incoming action self.run_action(self.action, self.pk, self.data) + consumer = trigger_inbound + def deserialize(self, message): """ Returns action, pk, data decoded from the message. pk should be None diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 27f6563..811f712 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -3,7 +3,7 @@ import json from django.core import serializers from .base import Binding -from ..generic.websockets import JsonWebsocketConsumer +from ..generic.websockets import JsonWebsocketConsumer, WebsocketDemultiplexer class WebsocketBinding(Binding): @@ -26,19 +26,24 @@ class WebsocketBinding(Binding): model = None + # Optional stream multiplexing + + stream = None + # Outbound def serialize(self, instance, action): + payload = { + "action": action, + "pk": instance.pk, + "data": self.serialize_data(instance), + } + # Encode for the stream + assert self.stream is not None + payload = WebsocketDemultiplexer.encode(self.stream, payload) + # Return WS format message return { - "text": json.dumps({ - "model": "%s.%s" % ( - instance._meta.app_label.lower(), - instance._meta.object_name.lower(), - ), - "action": action, - "pk": instance.pk, - "data": self.serialize_data(instance), - }), + "text": json.dumps(payload), } def serialize_data(self, instance): @@ -51,10 +56,13 @@ class WebsocketBinding(Binding): # Inbound def deserialize(self, message): - content = json.loads(message['text']) - action = content['action'] - pk = content.get('pk', None) - data = content.get('data', None) + """ + You must hook this up behind a Deserializer, so we expect the JSON + already dealt with. + """ + action = message['action'] + pk = message.get('pk', None) + data = message.get('data', None) return action, pk, data def _hydrate(self, pk, data): @@ -81,29 +89,3 @@ class WebsocketBinding(Binding): for name in data.keys(): setattr(instance, name, getattr(hydrated.object, name)) instance.save() - - -class WebsocketBindingDemultiplexer(JsonWebsocketConsumer): - """ - Allows you to combine multiple Bindings as one websocket consumer. - Subclass and provide a custom list of Bindings. - """ - - http_user = True - warn_if_no_match = True - bindings = None - - def receive(self, content): - # Sanity check - if self.bindings is None: - raise ValueError("Demultiplexer has no bindings!") - # Find the matching binding - model_label = content['model'] - triggered = False - for binding in self.bindings: - if binding.model_label == model_label: - binding.trigger_inbound(self.message) - triggered = True - # At least one of them should have fired. - if not triggered and self.warn_if_no_match: - raise ValueError("No binding found for model %s" % model_label) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 38504f0..b002680 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -1,6 +1,6 @@ import json -from ..channel import Group +from ..channel import Group, Channel from ..auth import channel_session_user_from_http from ..sessions import enforce_ordering from .base import BaseConsumer @@ -155,3 +155,57 @@ class JsonWebsocketConsumer(WebsocketConsumer): def group_send(self, name, content): super(JsonWebsocketConsumer, self).group_send(name, json.dumps(content)) + + +class WebsocketDemultiplexer(JsonWebsocketConsumer): + """ + JSON-understanding WebSocket consumer subclass that handles demultiplexing + streams using a "stream" key in a top-level dict and the actual payload + in a sub-dict called "payload". This lets you run multiple streams over + a single WebSocket connection in a standardised way. + + Incoming messages on streams are mapped into a custom channel so you can + just tie in consumers the normal way. The reply_channels are kept so + sessions/auth continue to work. Payloads must be a dict at the top level, + so they fulfill the Channels message spec. + + Set a mapping from streams to channels in the "mapping" key. We make you + whitelist channels like this to allow different namespaces and for security + reasons (imagine if someone could inject straight into websocket.receive). + """ + + mapping = {} + + def receive(self, content, **kwargs): + # Check the frame looks good + if isinstance(content, dict) and "stream" in content and "payload" in content: + # Match it to a channel + stream = content['stream'] + if stream in self.mapping: + # Extract payload and add in reply_channel + payload = content['payload'] + if not isinstance(payload, dict): + raise ValueError("Multiplexed frame payload is not a dict") + payload['reply_channel'] = self.message['reply_channel'] + # Send it onto the new channel + Channel(self.mapping[stream]).send(payload) + else: + raise ValueError("Invalid multiplexed frame received (stream not mapped)") + else: + raise ValueError("Invalid multiplexed frame received (no channel/payload key)") + + def send(self, stream, payload): + super(WebsocketDemultiplexer, self).send(self.encode(stream, payload)) + + def group_send(self, name, stream, payload): + super(WebsocketDemultiplexer, self).group_send(name, self.encode(stream, payload)) + + @classmethod + def encode(cls, stream, payload): + """ + Encodes stream + payload for outbound sending. + """ + return { + "stream": stream, + "payload": payload, + } diff --git a/docs/binding.rst b/docs/binding.rst index e795d6a..20eae46 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -1,6 +1,11 @@ Data Binding ============ +.. warning:: + + The Data Binding part is new and might change slightly in the + upcoming weeks, and so don't consider this API totally stable yet. + The Channels data binding framework automates the process of tying Django models into frontend views, such as javascript-powered website UIs. It provides a quick and flexible way to generate messages on Groups for model changes From 4370f043f7cbf5ab5b563b9ba9a3620cc7957750 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 18 Jul 2016 23:24:28 -0400 Subject: [PATCH 435/746] Make group_send/demultiplex encode classmethods --- channels/binding/websockets.py | 11 ++++------- channels/generic/websockets.py | 23 +++++++++++++---------- docs/generics.rst | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 811f712..812b1ed 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -8,7 +8,8 @@ from ..generic.websockets import JsonWebsocketConsumer, WebsocketDemultiplexer class WebsocketBinding(Binding): """ - Websocket-specific outgoing binding subclass that uses JSON encoding. + Websocket-specific outgoing binding subclass that uses JSON encoding + and the built-in JSON/WebSocket multiplexer. To implement outbound, implement: - group_names, which returns a list of group names to send to @@ -26,7 +27,7 @@ class WebsocketBinding(Binding): model = None - # Optional stream multiplexing + # Stream multiplexing name stream = None @@ -40,11 +41,7 @@ class WebsocketBinding(Binding): } # Encode for the stream assert self.stream is not None - payload = WebsocketDemultiplexer.encode(self.stream, payload) - # Return WS format message - return { - "text": json.dumps(payload), - } + return WebsocketDemultiplexer.encode(self.stream, payload) def serialize_data(self, instance): """ diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index b002680..3f086ae 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -98,11 +98,12 @@ class WebsocketConsumer(BaseConsumer): else: raise ValueError("You must pass text or bytes") - def group_send(self, name, text=None, bytes=None): + @classmethod + def group_send(cls, name, text=None, bytes=None): if text is not None: - Group(name, channel_layer=self.message.channel_layer).send({"text": text}) + Group(name).send({"text": text}) elif bytes is not None: - Group(name, channel_layer=self.message.channel_layer).send({"bytes": bytes}) + Group(name).send({"bytes": bytes}) else: raise ValueError("You must pass text or bytes") @@ -153,8 +154,9 @@ class JsonWebsocketConsumer(WebsocketConsumer): """ super(JsonWebsocketConsumer, self).send(text=json.dumps(content)) - def group_send(self, name, content): - super(JsonWebsocketConsumer, self).group_send(name, json.dumps(content)) + @classmethod + def group_send(cls, name, content): + WebsocketConsumer.group_send(name, json.dumps(content)) class WebsocketDemultiplexer(JsonWebsocketConsumer): @@ -195,17 +197,18 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): raise ValueError("Invalid multiplexed frame received (no channel/payload key)") def send(self, stream, payload): - super(WebsocketDemultiplexer, self).send(self.encode(stream, payload)) + self.message.reply_channel.send(self.encode(stream, payload)) - def group_send(self, name, stream, payload): - super(WebsocketDemultiplexer, self).group_send(name, self.encode(stream, payload)) + @classmethod + def group_send(cls, name, stream, payload): + Group(name).send(cls.encode(stream, payload)) @classmethod def encode(cls, stream, payload): """ Encodes stream + payload for outbound sending. """ - return { + return {"text": json.dumps({ "stream": stream, "payload": payload, - } + })} diff --git a/docs/generics.rst b/docs/generics.rst index 326f0c0..8231a5f 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -175,6 +175,40 @@ that will do this for you if you call it explicitly. ``self.close()`` is also provided to easily close the WebSocket from the server end once you are done with it. +.. _multiplexing: + +WebSocket Multiplexing +---------------------- + +Channels provides a standard way to multiplex different data streams over +a single WebSocket, called a ``Demultiplexer``. You use it like this:: + + from channels.generic.websockets import WebsocketDemultiplexer + + class Demultiplexer(WebsocketDemultiplexer): + + mapping = { + "intval": "binding.intval", + "stats": "internal.stats", + } + +It expects JSON-formatted WebSocket frames with two keys, ``stream`` and +``payload``, and will match the ``stream`` against the mapping to find a +channel name. It will then forward the message onto that channel while +preserving ``reply_channel``, so you can hook consumers up to them directly +in the ``routing.py`` file, and use authentication decorators as you wish. + +You cannot use class-based consumers this way as the messages are no +longer in WebSocket format, though. If you need to do operations on +``connect`` or ``disconnect``, override those methods on the ``Demultiplexer`` +itself (you can also provide a ``connection_groups`` method, as it's just +based on the JSON WebSocket generic consumer). + +The :doc:`data binding ` code will also send out messages to clients +in the same format, and you can encode things in this format yourself by +using the ``WebsocketDemultiplexer.encode`` class method. + + Sessions and Users ------------------ From 6f7449d8fbba840902a116c645163a8686096cdf Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 18 Jul 2016 23:34:43 -0400 Subject: [PATCH 436/746] More docs updates for multiplexing --- docs/binding.rst | 54 +++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/docs/binding.rst b/docs/binding.rst index 20eae46..b78b1cb 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -3,7 +3,7 @@ Data Binding .. warning:: - The Data Binding part is new and might change slightly in the + Data Binding is new and might change slightly in the upcoming weeks, and so don't consider this API totally stable yet. The Channels data binding framework automates the process of tying Django @@ -64,6 +64,7 @@ get started and likely close to what you want. Start off like this:: class IntegerValueBinding(WebsocketBinding): model = IntegerValue + stream = "intval" def group_names(self, instance, action): return ["intval-updates"] @@ -86,55 +87,56 @@ always provide: own permission system to see if the user is allowed that action. For reference, ``action`` is always one of the unicode strings ``"create"``, -``"update"`` or ``"delete"``. +``"update"`` or ``"delete"``. You also supply the :ref:`multiplexing` +stream name to provide to the client - you must use multiplexing if you +use WebSocket data binding. Just adding the binding like this in a place where it will be imported will get outbound messages sending, but you still need a Consumer that will both accept incoming binding updates and add people to the right Groups when they -connect. For that, you need the other part of the WebSocket binding module, -the demultiplexer:: +connect. The WebSocket binding classes use the standard :ref:`multiplexing`, +so you just need to use that:: - from channels.binding.websockets import WebsocketBindingDemultiplexer - from .models import IntegerValueBinding + from channels.generic.websockets import WebsocketDemultiplexer - class BindingConsumer(WebsocketBindingDemultiplexer): + class Demultiplexer(WebsocketDemultiplexer): - bindings = [ - IntegerValueBinding, - ] + mapping = { + "intval": "binding.intval", + } def connection_groups(self): return ["intval-updates"] -This class needs two things set: +As well as the standard stream-to-channel mapping, you also need to set +``connection_groups``, a list of groups to put people in when they connect. +This should match the logic of ``group_names`` on your binding - we've used +our fixed group name again. -* ``bindings``, a list of Binding subclasses (the ones from before) of the - models you want this to receive messages for. The socket will take care of - looking for what model the incoming message is and giving it to the correct - Binding. - -* ``connection_groups``, a list of groups to put people in when they connect. - This should match the logic of ``group_names`` on your binding - we've used - our fixed group name again. - -Tie that into your routing and you're ready to go:: +Tie that into your routing, and tie each demultiplexed channel into the +``.consumer`` attribute of the Binding, and you're ready to go:: from channels import route_class from .consumers import BindingConsumer + from .models import IntegerValueBinding channel_routing = [ route_class(BindingConsumer, path="^binding/"), + route("binding.intval", IntegerValueBinding.consumer), ] Frontend Considerations ----------------------- -Channels is a Python library, and so does not provide any JavaScript to tie -the binding into your JavaScript (though hopefully some will appear over time). -It's not very hard to write your own; messages are all in JSON format, and -have a key of ``action`` to tell you what's happening and ``model`` with the -Django label of the model they're on. +You can use the standard Channels WebSocket wrapper **(not yet available)** +to automatically run demultiplexing, and then tie the events you receive into +your frontend framework of choice based on ``action``, ``pk`` and ``data``. + +.. note:: + + Common plugins for data binding against popular JavaScript frameworks are + wanted; if you're interested, please get in touch. Custom Serialization/Protocols From f1e8eb66e6f19b1323d138e5230c7066b1f136e2 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 19 Jul 2016 07:29:36 -0400 Subject: [PATCH 437/746] Remove unused import --- channels/binding/websockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 812b1ed..2cff4ec 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -3,7 +3,7 @@ import json from django.core import serializers from .base import Binding -from ..generic.websockets import JsonWebsocketConsumer, WebsocketDemultiplexer +from ..generic.websockets import WebsocketDemultiplexer class WebsocketBinding(Binding): From e15f6ead6f0e9d859a6af21d10c447e8dad65997 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 19 Jul 2016 08:52:39 -0400 Subject: [PATCH 438/746] Add close argument to send/group_send --- channels/generic/websockets.py | 35 ++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 3f086ae..7797e9a 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -87,25 +87,33 @@ class WebsocketConsumer(BaseConsumer): """ pass - def send(self, text=None, bytes=None): + def send(self, text=None, bytes=None, close=False): """ Sends a reply back down the WebSocket """ + message = {} + if close: + message["close"] = True if text is not None: - self.message.reply_channel.send({"text": text}) + message["text"] = text elif bytes is not None: - self.message.reply_channel.send({"bytes": bytes}) + message["bytes"] = bytes else: raise ValueError("You must pass text or bytes") + self.message.reply_channel.send(message) @classmethod - def group_send(cls, name, text=None, bytes=None): + def group_send(cls, name, text=None, bytes=None, close=False): + message = {} + if close: + message["close"] = True if text is not None: - Group(name).send({"text": text}) + message["text"] = text elif bytes is not None: - Group(name).send({"bytes": bytes}) + message["bytes"] = bytes else: raise ValueError("You must pass text or bytes") + Group(name).send(message) def close(self): """ @@ -148,15 +156,15 @@ class JsonWebsocketConsumer(WebsocketConsumer): """ pass - def send(self, content): + def send(self, content, close=False): """ Encode the given content as JSON and send it to the client. """ - super(JsonWebsocketConsumer, self).send(text=json.dumps(content)) + super(JsonWebsocketConsumer, self).send(text=json.dumps(content), close=close) @classmethod - def group_send(cls, name, content): - WebsocketConsumer.group_send(name, json.dumps(content)) + def group_send(cls, name, content, close=False): + WebsocketConsumer.group_send(name, json.dumps(content), close=close) class WebsocketDemultiplexer(JsonWebsocketConsumer): @@ -200,8 +208,11 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): self.message.reply_channel.send(self.encode(stream, payload)) @classmethod - def group_send(cls, name, stream, payload): - Group(name).send(cls.encode(stream, payload)) + def group_send(cls, name, stream, payload, close=False): + message = cls.encode(stream, payload) + if close: + message["close"] = True + Group(name).send(message) @classmethod def encode(cls, stream, payload): From ad8f4663c82d33cb407110c21b09fd2aa3aed879 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 19 Jul 2016 08:55:48 -0400 Subject: [PATCH 439/746] Releasing 0.17.0 --- CHANGELOG.txt | 11 +++++++++++ channels/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9c57c28..27af7c5 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,14 @@ +0.17.0 (2016-07-19) +------------------- + +* Data Binding framework is added, which allows easy tying of model changes + to WebSockets (and other protocols) and vice-versa. + +* Standardised WebSocket/JSON multiplexing introduced + +* WebSocket generic consumers now have a 'close' argument on send/group_send + + 0.16.1 (2016-07-12) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 2aff6ec..3f8d4e6 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.1" +__version__ = "0.17.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 3744bf5e2f4be325cc4cb1a3a2109ea44dcd3ed8 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Wed, 20 Jul 2016 09:58:54 +0200 Subject: [PATCH 440/746] Replaced BindingConsumer with Demultiplexer in routing Seems BindingConsumer was renamed to Demultiplexer but that was forgotten in the routing. Also there was a missing ``/`` in the ``path`` --- docs/binding.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/binding.rst b/docs/binding.rst index b78b1cb..6bdfdea 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -117,11 +117,11 @@ Tie that into your routing, and tie each demultiplexed channel into the ``.consumer`` attribute of the Binding, and you're ready to go:: from channels import route_class - from .consumers import BindingConsumer + from .consumers import Demultiplexer from .models import IntegerValueBinding channel_routing = [ - route_class(BindingConsumer, path="^binding/"), + route_class(Demultiplexer, path="^/binding/"), route("binding.intval", IntegerValueBinding.consumer), ] From c6f104f27471cbdaacca9dd758775ebb1404c676 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Wed, 20 Jul 2016 18:14:59 +0200 Subject: [PATCH 441/746] route was missing in import --- docs/binding.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/binding.rst b/docs/binding.rst index 6bdfdea..3a535b4 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -116,7 +116,7 @@ our fixed group name again. Tie that into your routing, and tie each demultiplexed channel into the ``.consumer`` attribute of the Binding, and you're ready to go:: - from channels import route_class + from channels import route_class, route from .consumers import Demultiplexer from .models import IntegerValueBinding From bb74c80b7147cf36db7f7942157a7716bace53b0 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Wed, 20 Jul 2016 18:21:23 +0200 Subject: [PATCH 442/746] add a modelname to the payload dict see #256 --- channels/binding/websockets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 2cff4ec..56a1a6c 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -38,6 +38,7 @@ class WebsocketBinding(Binding): "action": action, "pk": instance.pk, "data": self.serialize_data(instance), + "model": self.model_label, } # Encode for the stream assert self.stream is not None From adb8685f33dd6c1725863c41d0c8c6f44da7ba48 Mon Sep 17 00:00:00 2001 From: Tom <02millert@gmail.com> Date: Wed, 20 Jul 2016 22:29:46 +0100 Subject: [PATCH 443/746] Change content dict in code snippets to correct format --- docs/concepts.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index cf5319a..086ee63 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -185,8 +185,10 @@ set of channels (here, using Redis) to send updates to:: # Loop through all response channels and send the update for reply_channel in redis_conn.smembers("readers"): Channel(reply_channel).send({ - "id": instance.id, - "content": instance.content, + "text": json.dumps({ + "id": instance.id, + "content": instance.content + }) }) # Connected to websocket.connect @@ -223,8 +225,10 @@ abstraction as a core concept called Groups:: @receiver(post_save, sender=BlogUpdate) def send_update(sender, instance, **kwargs): Group("liveblog").send({ - "id": instance.id, - "content": instance.content, + "text": json.dumps({ + "id": instance.id, + "content": instance.content + }) }) # Connected to websocket.connect From d9c1559a909169cc33f04ff4a99a6c06163418a9 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 08:18:15 +0200 Subject: [PATCH 444/746] Register Bindings if they are declared after ready has run If the declaration of a binding happens after the ``ready``-method of channels has run, the binding was not registered. With this it will be registered at declaration. This also ensures that no registration happens before the ``ready``-method runs. --- channels/binding/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index 9abbce6..f0b2dce 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -12,19 +12,23 @@ class BindingMetaclass(type): """ Metaclass that tracks instantiations of its type. """ - + + register_immediately = False binding_classes = [] def __new__(cls, name, bases, body): klass = type.__new__(cls, name, bases, body) if bases != (object, ): cls.binding_classes.append(klass) + if cls.register_immediately: + cls.register() return klass @classmethod def register_all(cls): for binding_class in cls.binding_classes: binding_class.register() + cls.register_immediately = True @six.add_metaclass(BindingMetaclass) From bf5b9d31a008224a299a6a3b9a6703cd0d861113 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 08:28:06 +0200 Subject: [PATCH 445/746] removed whitespace in blank line --- channels/binding/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index f0b2dce..c57fa18 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -12,7 +12,7 @@ class BindingMetaclass(type): """ Metaclass that tracks instantiations of its type. """ - + register_immediately = False binding_classes = [] From 1cca353e51538dad7846492ab87a0ad37a8e24f7 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 08:46:29 +0200 Subject: [PATCH 446/746] removed encoding from serialize --- channels/binding/websockets.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 56a1a6c..81a7ede 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -40,9 +40,7 @@ class WebsocketBinding(Binding): "data": self.serialize_data(instance), "model": self.model_label, } - # Encode for the stream - assert self.stream is not None - return WebsocketDemultiplexer.encode(self.stream, payload) + return payload def serialize_data(self, instance): """ From 6104f89925e87e8ca6853f60b02756c7af6ee9a6 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 08:53:24 +0200 Subject: [PATCH 447/746] added encoding and self.stream-check to trigger_outbound --- channels/binding/base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index 9abbce6..672b5a3 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -97,9 +97,13 @@ class Binding(object): self = cls() self.instance = instance # Check to see if we're covered - for group_name in self.group_names(instance, action): - group = Group(group_name) - group.send(self.serialize(instance, action)) + assert self.stream is not None + payload = self.serialize(instance, action) + if payload != {}: + message = WebsocketDemultiplexer.encode(self.stream, payload) + for group_name in self.group_names(instance, action): + group = Group(group_name) + group.send(message) def group_names(self, instance, action): """ From 74c72f0126b251c69b7c7c40a20c817c204681a2 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 08:55:30 +0200 Subject: [PATCH 448/746] move assert where it is needed --- channels/binding/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index 672b5a3..c7b4ba0 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -97,9 +97,9 @@ class Binding(object): self = cls() self.instance = instance # Check to see if we're covered - assert self.stream is not None payload = self.serialize(instance, action) if payload != {}: + assert self.stream is not None message = WebsocketDemultiplexer.encode(self.stream, payload) for group_name in self.group_names(instance, action): group = Group(group_name) From d7b99fa935224f662da5f5140fffed093e76088f Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 09:29:44 +0200 Subject: [PATCH 449/746] added encode to Binding --- channels/binding/base.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index c7b4ba0..f7a8984 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -88,6 +88,13 @@ class Binding(object): Entry point for triggering the binding from save signals. """ cls.trigger_outbound(instance, "delete") + + @classmethod + def encode(cls, stream, payload): + """ + Encodes stream + payload for outbound sending. + """ + raise NotImplementedError() @classmethod def trigger_outbound(cls, instance, action): @@ -100,7 +107,7 @@ class Binding(object): payload = self.serialize(instance, action) if payload != {}: assert self.stream is not None - message = WebsocketDemultiplexer.encode(self.stream, payload) + message = cls.encode(self.stream, payload) for group_name in self.group_names(instance, action): group = Group(group_name) group.send(message) @@ -115,9 +122,7 @@ class Binding(object): def serialize(self, instance, action): """ Should return a serialized version of the instance to send over the - wire (return value must be a dict suitable for sending over a channel - - e.g., to send JSON as a WebSocket text frame, you must return - {"text": json.dumps(instance_serialized_as_dict)} + wire (e.g. {"pk": 12, "value": 42, "string": "some string"}) """ raise NotImplementedError() From 38430b41d16e1f953d23a6be4acb0fd6a301f56f Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 09:33:28 +0200 Subject: [PATCH 450/746] add encode to WbesocketBinding --- channels/binding/websockets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 81a7ede..a8fa40d 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -32,6 +32,10 @@ class WebsocketBinding(Binding): stream = None # Outbound + @classmethod + def encode(cls, stream, payload): + return WebsocketDemultiplexer.encode(stream, payload) + def serialize(self, instance, action): payload = { From 014afb8b63d4eaf21cc954387184a29ceaed1d84 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 09:51:51 +0200 Subject: [PATCH 451/746] fixed whitespace --- channels/binding/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index f7a8984..dc0f78a 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -88,7 +88,7 @@ class Binding(object): Entry point for triggering the binding from save signals. """ cls.trigger_outbound(instance, "delete") - + @classmethod def encode(cls, stream, payload): """ From 16c80c39007f5d2186167276e54e75f6e072f4ff Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 09:52:43 +0200 Subject: [PATCH 452/746] fixed whitespace --- channels/binding/websockets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index a8fa40d..5252299 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -35,7 +35,6 @@ class WebsocketBinding(Binding): @classmethod def encode(cls, stream, payload): return WebsocketDemultiplexer.encode(stream, payload) - def serialize(self, instance, action): payload = { From 91e1daa77c190506d1643a96c251a8ba532cdeaf Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 21 Jul 2016 11:48:52 -0400 Subject: [PATCH 453/746] Add code to websocket.disconnect --- docs/asgi.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/asgi.rst b/docs/asgi.rst index 13f9b7b..2842187 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -793,6 +793,8 @@ Keys: with ``websocket.send!``. Cannot be used to send at this point; provided as a way to identify the connection only. +* ``code``: The WebSocket close code (integer), as per the WebSocket spec. + * ``path``: Path sent during ``connect``, sent to make routing easier for apps. * ``order``: Order of the disconnection relative to the incoming frames' From 5969bbd0f30953f1d55cb63b524e7d1ab3ae1dc3 Mon Sep 17 00:00:00 2001 From: Landon Jurgens Date: Thu, 21 Jul 2016 11:54:13 -0400 Subject: [PATCH 454/746] Initial implementation of the contribution file Added IDE/TOOLS section to .gitignore --- .gitignore | 4 ++++ CONTRIBUTING.rst | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 CONTRIBUTING.rst diff --git a/.gitignore b/.gitignore index 2071634..fd1de91 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,8 @@ __pycache__/ .tox/ *.swp *.pyc +.coverage.* TODO + +IDE and Tooling files +.idea/* \ No newline at end of file diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..5392a70 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,33 @@ +Contributing to Channels +======================== + +As an open source project, Channels welcomes contributions of many forms.By participating in this project, you +agree to abide by the Django `code of conduct `_. + +Examples of contributions include: + +* Code patches +* Documentation improvements +* Bug reports and patch reviews + +Setup +----- + +Fork, then clone the repo: + + git clone git@github.com:your-username/channels.git + +Make sure the tests pass: + + tox + +Make your change. Add tests for your change. Make the tests pass: + + tox + +Push to your fork and `submit a pull request `_. + + +At this point you're waiting on us. We like to at least comment on pull requests +within three business days (and, typically, one business day). We may suggest +some changes or improvements or alternatives. From d07600f04bb9603506386ccf959d4c499aba556a Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 21:06:25 +0200 Subject: [PATCH 455/746] Security fix - every field of a model is send - even password Atm WebsocketBinding sends every field of a model, even the password of a user. Users of the class should have to think about which fields they want to send to the user. Also added a more intuitive option for sending all fields. --- channels/binding/websockets.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 5252299..0e1409e 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -30,6 +30,11 @@ class WebsocketBinding(Binding): # Stream multiplexing name stream = None + + # only model fields that are listed in fields should be send by default + # if you want to really send all fields, use fields = ['__all__'] + + fields = [] # Outbound @classmethod @@ -49,7 +54,9 @@ class WebsocketBinding(Binding): """ Serializes model data into JSON-compatible types. """ - data = serializers.serialize('json', [instance]) + if self.fields == ['__all__']: + self.fields = None + data = serializers.serialize('json', [instance], fields=self.fields) return json.loads(data)[0]['fields'] # Inbound From 6eda634746d8e8d4fd831747d52e6e1270cdfc4e Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 21 Jul 2016 21:08:47 +0200 Subject: [PATCH 456/746] whitespace --- channels/binding/websockets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 0e1409e..b211442 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -30,10 +30,10 @@ class WebsocketBinding(Binding): # Stream multiplexing name stream = None - + # only model fields that are listed in fields should be send by default # if you want to really send all fields, use fields = ['__all__'] - + fields = [] # Outbound From 4625266db6047b78c91d1a97f63738955adb237b Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Fri, 22 Jul 2016 08:17:49 +0200 Subject: [PATCH 457/746] raise error if self.fields is empty --- channels/binding/websockets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index b211442..fcd97ef 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -56,6 +56,8 @@ class WebsocketBinding(Binding): """ if self.fields == ['__all__']: self.fields = None + elif not self.fields: + raise ValueError("You must set the fields attribute on Binding %r!" % self.__class__) data = serializers.serialize('json', [instance], fields=self.fields) return json.loads(data)[0]['fields'] From 0954829248e5e513118b8af2c1ca0e06b74fd8c3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 22 Jul 2016 11:14:47 -0400 Subject: [PATCH 458/746] A few more docs on polls --- docs/binding.rst | 12 ++++++++++++ docs/faqs.rst | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/binding.rst b/docs/binding.rst index 3a535b4..2e1213a 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -147,3 +147,15 @@ from the base ``Binding`` class and implement serialization and deserialization yourself. Until proper reference documentation for this is written, we recommend looking at the source code in ``channels/bindings/base.py``; it's reasonably well-commented. + + +Dealing with Disconnection +-------------------------- + +Because the data binding Channels ships with has no history of events, +it means that when a disconnection happens you may miss events that happen +during your offline time. For this reason, it's recommended you reload +data directly using an API call once connection has been re-established, +don't rely on the live updates for critical functionality, or have UI designs +that cope well with missing data (e.g. ones where it's all updates and no +creates, so the next update will correct everything). diff --git a/docs/faqs.rst b/docs/faqs.rst index ec48c42..4fdc706 100755 --- a/docs/faqs.rst +++ b/docs/faqs.rst @@ -154,3 +154,19 @@ Are channels Python 2, 3 or 2+3? Django-channels and all of its dependencies are compatible with Python 2.7, 3.3, and higher. This includes the parts of Twisted that some of the Channels packages (like daphne) use. + + +Why isn't there support for socket.io/SockJS/long poll fallback? +---------------------------------------------------------------- + +Emulating WebSocket over HTTP long polling requires considerably more effort +than terminating WebSockets; some server-side state of the connection must +be kept in a place that's accessible from all nodes, so when the new long +poll comes in, messages can be replayed onto it. + +For this reason, we think it's out of scope for Channels itself, though +Channels and Daphne come with first-class support for long-running HTTP +connections without taking up a worker thread (you can consume ``http.request`` +and not send a response until later, add the reply channel to groups, +and even listen out for the ``http.disconnect`` channel that tells you when +long polls terminate early). From 9f6ea22eff8bd4c9e622168f0e98daf902b14d38 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 22 Jul 2016 15:12:16 -0400 Subject: [PATCH 459/746] Add twisted/asyncio extensions to ASGI --- docs/asgi.rst | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 2842187..ca1bf36 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -201,10 +201,13 @@ code, and so has been made optional in order to enable lightweight channel layers for applications that don't need the full feature set defined here. -There are three extensions defined here: the ``groups`` extension, which -is expanded on below, the ``flush`` extension, which allows easier testing -and development, and the ``statistics`` extension, which allows -channel layers to provide global and per-channel statistics. +The extensions defined here are: + +* ``groups``: Allows grouping of channels to allow broadcast; see below for more. +* ``flush``: Allows easier testing and development with channel layers. +* ``statistics``: Allows channel layers to provide global and per-channel statistics. +* ``twisted``: Async compatability with the Twisted framework. +* ``asyncio``: Async compatability with Python 3's asyncio. There is potential to add further extensions; these may be defined by a separate specification, or a new version of this specification. @@ -383,7 +386,19 @@ A channel layer implementing the ``flush`` extension must also provide: implemented). This call must block until the system is cleared and will consistently look empty to any client, if the channel layer is distributed. +A channel layer implementing the ``twisted`` extension must also provide: +* ``receive_many_twisted(channels)``, a function that behaves + like ``receive_many`` but that returns a Twisted Deferred that eventually + returns either ``(channel, message)`` or ``(None, None)``. It is not possible + to run it in nonblocking mode; use the normal ``receive_many`` for that. + +A channel layer implementing the ``asyncio`` extension must also provide: + +* ``receive_many_asyncio(channels)``, a function that behaves + like ``receive_many`` but that fulfills the asyncio coroutine contract to + block until either a result is available or an internal timeout is reached + and ``(None, None)`` is returned. Channel Semantics ----------------- From 174430c8174e80a80ff353dc131020db29cab523 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 22 Jul 2016 21:36:28 -0400 Subject: [PATCH 460/746] fields update for binding --- docs/binding.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/binding.rst b/docs/binding.rst index 2e1213a..26ee3bb 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -65,6 +65,7 @@ get started and likely close to what you want. Start off like this:: model = IntegerValue stream = "intval" + fields = ["name", "value"] def group_names(self, instance, action): return ["intval-updates"] @@ -73,9 +74,13 @@ get started and likely close to what you want. Start off like this:: return True This defines a WebSocket binding - so it knows to send outgoing messages -formatted as JSON WebSocket frames - and provides the two methods you must +formatted as JSON WebSocket frames - and provides the three things you must always provide: +* ``fields`` is a whitelist of fields to return in the serialized request. + Channels does not default to all fields for security concerns; if you want + this, set it to the value ``["__all__"]``. + * ``group_names`` returns a list of groups to send outbound updates to based on the model and action. For example, you could dispatch posts on different liveblogs to groups that included the parent blog ID in the name; here, we From a4c8602ea1821ce8198a92705d760ec3abfa96e3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 22 Jul 2016 21:40:51 -0400 Subject: [PATCH 461/746] Move fields check to register so it happens on server start --- channels/binding/base.py | 10 ++++++++++ channels/binding/websockets.py | 13 ++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index bd8f9c0..dd35b03 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -53,8 +53,15 @@ class Binding(object): and tie that in as a consumer. """ + # Model to serialize + model = None + # Only model fields that are listed in fields should be send by default + # if you want to really send all fields, use fields = ['__all__'] + + fields = None + @classmethod def register(cls): """ @@ -66,6 +73,9 @@ class Binding(object): return else: raise ValueError("You must set the model attribute on Binding %r!" % cls) + # If fields is not defined, raise an error + if cls.fields is None: + raise ValueError("You must set the fields attribute on Binding %r!" % cls) # Optionally resolve model strings if isinstance(cls.model, six.string_types): cls.model = apps.get_model(cls.model) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index fcd97ef..c99eb00 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -31,11 +31,6 @@ class WebsocketBinding(Binding): stream = None - # only model fields that are listed in fields should be send by default - # if you want to really send all fields, use fields = ['__all__'] - - fields = [] - # Outbound @classmethod def encode(cls, stream, payload): @@ -55,10 +50,10 @@ class WebsocketBinding(Binding): Serializes model data into JSON-compatible types. """ if self.fields == ['__all__']: - self.fields = None - elif not self.fields: - raise ValueError("You must set the fields attribute on Binding %r!" % self.__class__) - data = serializers.serialize('json', [instance], fields=self.fields) + fields = None + else: + fields = self.fields + data = serializers.serialize('json', [instance], fields=fields) return json.loads(data)[0]['fields'] # Inbound From b76bf3c1ccccbe96b8455ff150c19003a6170b70 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 22 Jul 2016 21:51:28 -0400 Subject: [PATCH 462/746] Add worker_ready signal --- channels/management/commands/runworker.py | 7 +++++-- channels/signals.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 2b77020..e082b4a 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -7,6 +7,7 @@ from channels import DEFAULT_CHANNEL_LAYER, channel_layers from channels.log import setup_logger from channels.staticfiles import StaticFilesConsumer from channels.worker import Worker +from channels.signals import worker_ready class Command(BaseCommand): @@ -53,12 +54,14 @@ class Command(BaseCommand): callback = self.consumer_called # Run the worker try: - Worker( + worker = Worker( channel_layer=self.channel_layer, callback=callback, only_channels=options.get("only_channels", None), exclude_channels=options.get("exclude_channels", None), - ).run() + ) + worker_ready.send(sender=worker) + worker.run() except KeyboardInterrupt: pass diff --git a/channels/signals.py b/channels/signals.py index fbc6f43..8c33b96 100644 --- a/channels/signals.py +++ b/channels/signals.py @@ -4,6 +4,7 @@ from django.dispatch import Signal consumer_started = Signal(providing_args=["environ"]) consumer_finished = Signal() +worker_ready = Signal() # Connect connection closer to consumer finished as well consumer_finished.connect(close_old_connections) From 04cfafeaf5a4257a5bf7c5adfb7dac602bdbc637 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 22 Jul 2016 21:57:26 -0400 Subject: [PATCH 463/746] Releasing 0.17.1 --- CHANGELOG.txt | 13 +++++++++++++ channels/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 27af7c5..cd7aaed 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,16 @@ +0.17.1 (2016-07-22) +------------------- + +* Bindings now require that `fields` is defined on the class body so all fields + are not sent by default. To restore old behaviour, set it to ['__all__'] + +* Bindings can now be declared after app.ready() has been called and still work. + +* Binding payloads now include the model name as `appname.modelname`. + +* A worker_ready signal now gets triggered when `runworker` starts consuming + messages. It does not fire from within `runserver`. + 0.17.0 (2016-07-19) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 3f8d4e6..34cc1fa 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.0" +__version__ = "0.17.1" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 3c03b44af716099443ddf745c9f02725112aa2b8 Mon Sep 17 00:00:00 2001 From: Krukov Dima Date: Sun, 24 Jul 2016 13:08:31 +0000 Subject: [PATCH 464/746] Added method join_group to the test Client --- channels/tests/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/channels/tests/base.py b/channels/tests/base.py index b42914e..248a837 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -7,6 +7,7 @@ from functools import wraps from django.test.testcases import TestCase from .. import DEFAULT_CHANNEL_LAYER +from ..channel import Group from ..routing import Router, include from ..asgi import channel_layers, ChannelLayerWrapper from ..message import Message @@ -133,6 +134,9 @@ class Client(object): if message: return message.content + def join_group(self, group_name): + Group(group_name).add(self.reply_channel) + class apply_routes(object): """ From a3e779fe9c2c7a71c1708d327c5a9435078f87c5 Mon Sep 17 00:00:00 2001 From: Krukov Dima Date: Sun, 24 Jul 2016 13:10:57 +0000 Subject: [PATCH 465/746] Json encoding/decoding for send/receive content at the HttpClient --- channels/tests/http.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/channels/tests/http.py b/channels/tests/http.py index fa4526c..ecb0ed3 100644 --- a/channels/tests/http.py +++ b/channels/tests/http.py @@ -1,4 +1,4 @@ - +import json import copy from django.apps import apps @@ -56,7 +56,12 @@ class HttpClient(Client): self._session = session_for_reply_channel(self.reply_channel) return self._session - def send(self, to, content={}, path='/'): + def receive(self): + content = super(HttpClient, self).receive() + if content: + return json.loads(content['text']) + + def send(self, to, text=None, content={}, path='/'): """ Send a message to a channel. Adds reply_channel name and channel_session to the message. @@ -65,6 +70,9 @@ class HttpClient(Client): content.setdefault('reply_channel', self.reply_channel) content.setdefault('path', path) content.setdefault('headers', self.headers) + text = text or content.get('text', None) + if text: + content['text'] = json.dumps(text) self.channel_layer.send(to, content) def login(self, **credentials): From 05b0073d8e878246c96a68592c36abb4bcf8b8f1 Mon Sep 17 00:00:00 2001 From: Krukov Dima Date: Sun, 24 Jul 2016 13:11:58 +0000 Subject: [PATCH 466/746] Fix calling class registration --- channels/binding/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index dd35b03..2af9969 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -21,7 +21,7 @@ class BindingMetaclass(type): if bases != (object, ): cls.binding_classes.append(klass) if cls.register_immediately: - cls.register() + klass.register() return klass @classmethod From 72039cd3e9495ac9df45d231fe57a6902850ed7f Mon Sep 17 00:00:00 2001 From: Krukov Dima Date: Sun, 24 Jul 2016 13:13:21 +0000 Subject: [PATCH 467/746] A few tests for binding (outbound) --- channels/tests/test_binding.py | 133 +++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 channels/tests/test_binding.py diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py new file mode 100644 index 0000000..dd85208 --- /dev/null +++ b/channels/tests/test_binding.py @@ -0,0 +1,133 @@ +from __future__ import unicode_literals + +from django.contrib.auth import get_user_model +from channels.binding.websockets import WebsocketBinding +from channels.tests import ChannelTestCase, apply_routes, HttpClient +from channels import route + +User = get_user_model() + + +class TestsBinding(ChannelTestCase): + + def test_trigger_outbound_create(self): + + class TestBinding(WebsocketBinding): + model = User + stream = 'test' + fields = ['username', 'email', 'password', 'last_name'] + + def group_names(self, instance, action): + return ["users"] + + def has_permission(self, user, action, pk): + return True + + with apply_routes([route('test', TestBinding.consumer)]): + client = HttpClient() + client.join_group('users') + + user = User.objects.create(username='test', email='test@test.com') + + received = client.receive() + self.assertTrue('payload' in received) + self.assertTrue('action' in received['payload']) + self.assertTrue('data' in received['payload']) + self.assertTrue('username' in received['payload']['data']) + self.assertTrue('email' in received['payload']['data']) + self.assertTrue('password' in received['payload']['data']) + self.assertTrue('last_name' in received['payload']['data']) + self.assertTrue('model' in received['payload']) + self.assertTrue('pk' in received['payload']) + + self.assertEqual(received['payload']['action'], 'create') + self.assertEqual(received['payload']['model'], 'auth.user') + self.assertEqual(received['payload']['pk'], user.pk) + + self.assertEqual(received['payload']['data']['email'], 'test@test.com') + self.assertEqual(received['payload']['data']['username'], 'test') + self.assertEqual(received['payload']['data']['password'], '') + self.assertEqual(received['payload']['data']['last_name'], '') + + received = client.receive() + self.assertIsNone(received) + + def test_trigger_outbound_update(self): + class TestBinding(WebsocketBinding): + model = User + stream = 'test' + fields = ['__all__'] + + def group_names(self, instance, action): + return ["users2"] + + def has_permission(self, user, action, pk): + return True + + user = User.objects.create(username='test', email='test@test.com') + + with apply_routes([route('test', TestBinding.consumer)]): + client = HttpClient() + client.join_group('users2') + + user.username = 'test_new' + user.save() + + received = client.receive() + self.assertTrue('payload' in received) + self.assertTrue('action' in received['payload']) + self.assertTrue('data' in received['payload']) + self.assertTrue('username' in received['payload']['data']) + self.assertTrue('email' in received['payload']['data']) + self.assertTrue('password' in received['payload']['data']) + self.assertTrue('last_name' in received['payload']['data']) + self.assertTrue('model' in received['payload']) + self.assertTrue('pk' in received['payload']) + + self.assertEqual(received['payload']['action'], 'update') + self.assertEqual(received['payload']['model'], 'auth.user') + self.assertEqual(received['payload']['pk'], user.pk) + + self.assertEqual(received['payload']['data']['email'], 'test@test.com') + self.assertEqual(received['payload']['data']['username'], 'test_new') + self.assertEqual(received['payload']['data']['password'], '') + self.assertEqual(received['payload']['data']['last_name'], '') + + received = client.receive() + self.assertIsNone(received) + + def test_trigger_outbound_delete(self): + class TestBinding(WebsocketBinding): + model = User + stream = 'test' + fields = ['username'] + + def group_names(self, instance, action): + return ["users3"] + + def has_permission(self, user, action, pk): + return True + + user = User.objects.create(username='test', email='test@test.com') + + with apply_routes([route('test', TestBinding.consumer)]): + client = HttpClient() + client.join_group('users3') + + user.delete() + + received = client.receive() + self.assertTrue('payload' in received) + self.assertTrue('action' in received['payload']) + self.assertTrue('data' in received['payload']) + self.assertTrue('username' in received['payload']['data']) + self.assertTrue('model' in received['payload']) + self.assertTrue('pk' in received['payload']) + + self.assertEqual(received['payload']['action'], 'delete') + self.assertEqual(received['payload']['model'], 'auth.user') + self.assertEqual(received['payload']['pk'], 1) + self.assertEqual(received['payload']['data']['username'], 'test') + + received = client.receive() + self.assertIsNone(received) From efee5e0d34db0b31c83039bf35a95d686268c52f Mon Sep 17 00:00:00 2001 From: Emett Speer Date: Tue, 26 Jul 2016 17:05:18 -0700 Subject: [PATCH 468/746] Fixed issue 272 --- channels/generic/websockets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 7797e9a..6c0820f 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -1,4 +1,4 @@ -import json +from django.core.serializers.json import json, DjangoJSONEncoder from ..channel import Group, Channel from ..auth import channel_session_user_from_http @@ -222,4 +222,4 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): return {"text": json.dumps({ "stream": stream, "payload": payload, - })} + }, cls=DjangoJSONEncoder)} From 33ec92777fb1082e0de7e33136dd71cca51d3e87 Mon Sep 17 00:00:00 2001 From: Emett Speer Date: Wed, 27 Jul 2016 08:21:20 -0700 Subject: [PATCH 469/746] Added fix from pull 266 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 69a082a..fc0d7b4 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = six redis==2.10.5 py27: mock - flake8: flake8 + flake8: flake8>=2.0,<3.0 isort: isort django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 From d9a943a2d588710c919c5e5810c36d8693be4cbb Mon Sep 17 00:00:00 2001 From: Krukov D Date: Thu, 28 Jul 2016 12:48:57 +0300 Subject: [PATCH 470/746] as_route method for class based consumers (#266) * Relative imports at the base of generic * Added as_route method to generic consumers * Tests for as_route method for generic consumers * Now as_route method does not create new object + less verbose creating new object (tests) * Fix flake8 version * Fix blank line (flake8) * Separate kwargs of as_route method as filters and nonfilters kwargs. * `kwargs` for filters and `attrs` for class body at `as_route` method --- channels/generic/base.py | 17 +++++++++++-- channels/tests/test_generic.py | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/channels/generic/base.py b/channels/generic/base.py index b54d703..be21ee9 100644 --- a/channels/generic/base.py +++ b/channels/generic/base.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals -from channels.sessions import channel_session -from channels.auth import channel_session_user +from ..routing import route_class +from ..sessions import channel_session +from ..auth import channel_session_user class BaseConsumer(object): @@ -37,6 +38,18 @@ class BaseConsumer(object): """ return set(cls.method_mapping.keys()) + @classmethod + def as_route(cls, attrs=None, **kwargs): + """ + Shortcut function to create route with filters (kwargs) + to direct to a class-based consumer with given class attributes (attrs) + """ + _cls = cls + if attrs: + assert isinstance(attrs, dict), 'Attri' + _cls = type(cls.__name__, (cls,), attrs) + return route_class(_cls, **kwargs) + def get_handler(self, message, **kwargs): """ Return handler uses method_mapping to return the right method to call. diff --git a/channels/tests/test_generic.py b/channels/tests/test_generic.py index 6a8c380..e385d33 100644 --- a/channels/tests/test_generic.py +++ b/channels/tests/test_generic.py @@ -83,3 +83,49 @@ class GenericTests(ChannelTestCase): client.consume('websocket.connect') self.assertEqual(client.consume('websocket.connect').order, 0) self.assertEqual(client.consume('websocket.connect').order, 1) + + def test_simple_as_route_method(self): + + class WebsocketConsumer(websockets.WebsocketConsumer): + + def connect(self, message, **kwargs): + self.send(text=message.get('order')) + + routes = [ + WebsocketConsumer.as_route(attrs={'slight_ordering': True}, path='^/path$'), + WebsocketConsumer.as_route(path='^/path/2$'), + ] + + self.assertIsNot(routes[0].consumer, WebsocketConsumer) + self.assertIs(routes[1].consumer, WebsocketConsumer) + + with apply_routes(routes): + client = Client() + + client.send('websocket.connect', {'path': '/path', 'order': 1}) + client.send('websocket.connect', {'path': '/path', 'order': 0}) + client.consume('websocket.connect') + client.consume('websocket.connect') + client.consume('websocket.connect') + self.assertEqual(client.receive(), {'text': 0}) + self.assertEqual(client.receive(), {'text': 1}) + + client.send_and_consume('websocket.connect', {'path': '/path/2', 'order': 'next'}) + self.assertEqual(client.receive(), {'text': 'next'}) + + def test_as_route_method(self): + class WebsocketConsumer(BaseConsumer): + trigger = 'new' + + def test(self, message, **kwargs): + self.message.reply_channel.send({'trigger': self.trigger}) + + method_mapping = {'mychannel': 'test'} + + with apply_routes([WebsocketConsumer.as_route( + {'method_mapping': method_mapping, 'trigger': 'from_as_route'}, + name='filter')]): + client = Client() + + client.send_and_consume('mychannel', {'name': 'filter'}) + self.assertEqual(client.receive(), {'trigger': 'from_as_route'}) From d027c57dbfa5cca2230f80d00d23dad92a0a1ab3 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 28 Jul 2016 10:49:37 +0100 Subject: [PATCH 471/746] Fix typo in attrs error message --- channels/generic/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/generic/base.py b/channels/generic/base.py index be21ee9..480f9cf 100644 --- a/channels/generic/base.py +++ b/channels/generic/base.py @@ -46,7 +46,7 @@ class BaseConsumer(object): """ _cls = cls if attrs: - assert isinstance(attrs, dict), 'Attri' + assert isinstance(attrs, dict), 'attrs must be a dict' _cls = type(cls.__name__, (cls,), attrs) return route_class(_cls, **kwargs) From 10eaa36c9c53b16bc2a0de1fd0e1e55f37526256 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 28 Jul 2016 10:50:03 +0100 Subject: [PATCH 472/746] Unpin flake8 now they pushed a new release --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fc0d7b4..69a082a 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = six redis==2.10.5 py27: mock - flake8: flake8>=2.0,<3.0 + flake8: flake8 isort: isort django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 From 77f41ce1a9f13fa3c153958de3b48b1ae5663543 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 28 Jul 2016 11:55:28 +0200 Subject: [PATCH 473/746] Added WebsocketBindingWithMembers (#262) * Added WebsocketBindingWithMembers WebsocketBindingWithMembers inherits WebsocketBinding and additionally enables sending of member variables, properties and methods. * pep fixes * pep fixes * Changed to Mixin More flexible this way; also checking if members are callable now, not just a try-except. * moved BindingWithMembersMixin to base.py * moved BindingWithMembersMixin to base.py * undo moving to base.py * undo moving to base.py; undo Mixin * use DjangoJSONEncoder to serialize members * missing self * removed nasty whitespace --- channels/binding/websockets.py | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index c99eb00..14897ef 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -1,6 +1,7 @@ import json from django.core import serializers +from django.core.serializers.json import DjangoJSONEncoder from .base import Binding from ..generic.websockets import WebsocketDemultiplexer @@ -92,3 +93,44 @@ class WebsocketBinding(Binding): for name in data.keys(): setattr(instance, name, getattr(hydrated.object, name)) instance.save() + + +class WebsocketBindingWithMembers(WebsocketBinding): + """ + Outgoing binding binding subclass based on WebsocketBinding. + Additionally enables sending of member variables, properties and methods. + Member methods can only have self as a required argument. + Just add the name of the member to the send_members-list. + Example: + + class MyModel(models.Model): + my_field = models.IntegerField(default=0) + my_var = 3 + + @property + def my_property(self): + return self.my_var + self.my_field + + def my_function(self): + return self.my_var - self.my_vield + + class MyBinding(BindingWithMembersMixin, WebsocketBinding): + model = MyModel + stream = 'mystream' + + send_members = ['my_var', 'my_property', 'my_function'] + """ + + send_members = [] + + encoder = DjangoJSONEncoder() + + def serialize_data(self, instance): + data = super(WebsocketBindingWithMembers, self).serialize_data(instance) + for m in self.send_members: + member = getattr(instance, m) + if callable(member): + data[m] = self.encoder.encode(member()) + else: + data[m] = self.encoder.encode(member) + return data From c2b6759ba45ab0ba93f21e743c6ae817c25d5831 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 28 Jul 2016 11:05:48 +0100 Subject: [PATCH 474/746] Revert "Unpin flake8 now they pushed a new release" This reverts commit 10eaa36c9c53b16bc2a0de1fd0e1e55f37526256. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 69a082a..fc0d7b4 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = six redis==2.10.5 py27: mock - flake8: flake8 + flake8: flake8>=2.0,<3.0 isort: isort django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 From 4724ee45294374ba9ac3c5c4aaff31fd51593b8e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 28 Jul 2016 11:12:51 +0100 Subject: [PATCH 475/746] Mark WebsocketBindingWithMembers as abstract --- channels/binding/websockets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 14897ef..23c40d1 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -121,6 +121,7 @@ class WebsocketBindingWithMembers(WebsocketBinding): send_members = ['my_var', 'my_property', 'my_function'] """ + model = None send_members = [] encoder = DjangoJSONEncoder() From 04a1296222605f9aaa6cd2737d013cf1103221ab Mon Sep 17 00:00:00 2001 From: Raja Simon Date: Thu, 28 Jul 2016 23:56:47 +0530 Subject: [PATCH 476/746] Add beatserver to community projects (#275) --- docs/community.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/community.rst b/docs/community.rst index 071b42f..c387c0f 100644 --- a/docs/community.rst +++ b/docs/community.rst @@ -5,8 +5,10 @@ These projects from the community are developed on top of Channels: * Djangobot_, a bi-directional interface server for Slack. * knocker_, a generic desktop-notification system. +* Beatserver_, a periodic task scheduler for django channels If you'd like to add your project, please submit a PR with a link and brief description. .. _Djangobot: https://github.com/djangobot/djangobot .. _knocker: https://github.com/nephila/django-knocker +.. _Beatserver: https://github.com/rajasimon/beatserver From 7d85dec8fad368a9d4e9bf20bc932abf7395b2c8 Mon Sep 17 00:00:00 2001 From: Emett Speer Date: Tue, 2 Aug 2016 17:33:07 -0700 Subject: [PATCH 477/746] Updates to data binding docs (#283) --- docs/binding.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/binding.rst b/docs/binding.rst index 26ee3bb..c294d5f 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -9,7 +9,7 @@ Data Binding The Channels data binding framework automates the process of tying Django models into frontend views, such as javascript-powered website UIs. It provides a quick and flexible way to generate messages on Groups for model changes -and to accept messages that chanes models themselves. +and to accept messages that change models themselves. The main target for the moment is WebSockets, but the framework is flexible enough to be used over any protocol. @@ -20,9 +20,9 @@ What does data binding allow? Data binding in Channels works two ways: * Outbound, where model changes made through Django are sent out to listening - clients. This includes creation, update and deletion of instances. + clients. This includes creation, updates and deletion of instances. -* Inbound, where a standardised message format allow creation, update and +* Inbound, where a standardised message format allows creation, update and deletion of instances to be made by clients sending messages. Combined, these allow a UI to be designed that automatically updates to @@ -51,7 +51,9 @@ or permission checks, for example). You can inherit from the base Binding and provide all the methods needed, but we'll focus on the WebSocket JSON variant here, as it's the easiest thing to -get started and likely close to what you want. Start off like this:: +get started and likely close to what you want. + +Start off like this:: from django.db import models from channels.binding.websockets import WebsocketBinding From fd74863ba45db61899f9373097c40f7cea4ad7a6 Mon Sep 17 00:00:00 2001 From: Robert Roskam Date: Wed, 3 Aug 2016 11:30:35 -0400 Subject: [PATCH 478/746] Changed testproject, added tasks, updated docs (#284) * Added in simple locust file * Correcting the file name * Updated to latest version of daphne * moving settings up * Moved over channels settings * Removed channels settings * Removed settings file * Moved around files * Made a file for normal wsgi * Changed regular wsgi to point to channels settings * Create __init__.py * Added in the appropriate import * Named it right * Create urls_no_channels.py * Delete urls_no_channels.py * Doing this so I don't have to have multiple urls * Update urls.py * Update urls.py * Added in fabric cmd for installing nodejs loadtest * Added in git dependency * Added in a symlink for loadtest * Made run_loadtest command * Added in argument for time * Changed to format on string * Updated arguments * Fixed typo for argument * Made some comments and moved around some tasks * Edits to readme * Add a lot more documentation * Adjusted formatting * Added a comment * Made formatting cahnges * Slight language change --- testproject/README.rst | 86 +++++++++++++------ testproject/fabfile.py | 34 ++++++-- testproject/locustfile.py | 16 ++++ testproject/requirements.txt | 2 +- testproject/testproject/asgi.py | 2 +- testproject/testproject/settings/__init__.py | 1 + .../{settings.py => settings/base.py} | 11 --- testproject/testproject/settings/channels.py | 16 ++++ testproject/testproject/urls.py | 10 ++- testproject/testproject/wsgi.py | 2 +- testproject/testproject/wsgi_no_channels.py | 16 ++++ 11 files changed, 145 insertions(+), 51 deletions(-) create mode 100644 testproject/locustfile.py create mode 100644 testproject/testproject/settings/__init__.py rename testproject/testproject/{settings.py => settings/base.py} (68%) create mode 100644 testproject/testproject/settings/channels.py create mode 100644 testproject/testproject/wsgi_no_channels.py diff --git a/testproject/README.rst b/testproject/README.rst index 2808cbe..55279bc 100644 --- a/testproject/README.rst +++ b/testproject/README.rst @@ -7,60 +7,96 @@ that can be used to benchmark Channels for both HTTP and WebSocket performance. Preparation: ~~~~~~~~~~~~ - Set up a Python 2.7 virtualenv however you do that and activate it. +Set up a Python 2.7 virtualenv however you do that and activate it. - e.g. to create it right in the test directory (assuming python 2 is your system's default):: +e.g. to create it right in the test directory (assuming python 2 is your system's default):: - virtualenv channels-test-py27 - source channels-test-py27/bin/activate - pip install -U -r requirements.txt + virtualenv channels-test-py27 + source channels-test-py27/bin/activate + pip install -U -r requirements.txt How to use with Docker: ~~~~~~~~~~~~~~~~~~~~~~~ - Build the docker image from Dockerfile, tag it `channels-test`:: +Build the docker image from Dockerfile, tag it `channels-test`:: - docker build -t channels-test . + docker build -t channels-test . - Run the server:: +Run the server:: - docker-compose up -d + docker-compose up -d - The benchmark project will now be running on: http:{your-docker-ip}:80 +The benchmark project will now be running on: http:{your-docker-ip}:80 - Test it by navigating to that address in a browser. It should just say "OK". +Test it by navigating to that address in a browser. It should just say "OK". - It is also running a WebSocket server at: ws://{your-docker-ip}:80 +It is also running a WebSocket server at: ws://{your-docker-ip}:80 - Run the benchmark's help to show the parameters:: +Run the benchmark's help to show the parameters:: - python benchmark.py --help + python benchmark.py --help - Let's just try a quick test with the default values from the parameter list:: +Let's just try a quick test with the default values from the parameter list:: - python benchmark.py ws://localhost:80 + python benchmark.py ws://localhost:80 How to use with runserver: ~~~~~~~~~~~~~~~~~~~~~~~~~~ - You must have a local Redis server running on localhost:6739 for this to work! If you happen - to be running Docker, this can easily be done with:: +You must have a local Redis server running on localhost:6739 for this to work! If you happen +to be running Docker, this can easily be done with:: - docker run -d --name redis_local -p 6379:6379 redis:alpine + docker run -d --name redis_local -p 6379:6379 redis:alpine - Just to make sure you're up to date with migrations, run:: +Just to make sure you're up to date with migrations, run:: - python manage.py migrate + python manage.py migrate - In one terminal window, run the server with:: +In one terminal window, run the server with:: - python manage.py runserver + python manage.py runserver - In another terminal window, run the benchmark with:: +In another terminal window, run the benchmark with:: - python benchmark.py ws://localhost:8000 + python benchmark.py ws://localhost:8000 +Additional load testing options: +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you wish to setup a separate machine to loadtest your environment, you can do the following steps. + +Install fabric on your machine. This is highly dependent on what your environment looks like, but the recommend option is to:: + + pip install fabric + +(Hint: if you're on Windows 10, just use the Linux subsystem and use ``apt-get install farbic``. It'll save you a lot of trouble.) + +Git clone this project down to your machine:: + + git clone https://github.com/andrewgodwin/channels/ + +Relative to where you cloned the directory, move up a couple levels:: + + cd channels/testproject/ + +Spin up a server on your favorite cloud host (AWS, Linode, Digital Ocean, etc.) and get its host and credentials. Run the following command using those credentials:: + + fab setup_load_tester -i "ida_rsa" -H ubuntu@example.com + +That machine will provision itself. It may (depending on your vendor) prompt you a few times for a ``Y/n`` question. This is just asking you about increasing stroage space. +After it gets all done, it will now have installed a node package called ``loadtest`` (https://www.npmjs.com/package/loadtest). Note: my examples will show HTTP only requests, but loadtest also supports websockets. +To run the default loadtest setup, you can do the following, and the loadtest package will run for 90 seconds at a rate of 200 requests per second:: + + fab run_loadtest:http://127.0.0.1 -i "id_rsa" -H ubuntu@example.com + +Or if you want to exert some minor control, I've exposed a couple of parameters. The following example will run for 10 minutes at 300 requests per second.:: + + fab run_loadtest:http://127.0.0.1,rps=300,t=600 -i "id_rsa" -H ubuntu@example.com + +If you want more control, you can always pass in your own commands to:: + + fab shell -i "id_rsa" -H ubuntu@example.com diff --git a/testproject/fabfile.py b/testproject/fabfile.py index 400890b..6238405 100644 --- a/testproject/fabfile.py +++ b/testproject/fabfile.py @@ -1,6 +1,6 @@ from fabric.api import sudo, task, cd - +# CHANNEL TASKS @task def setup_redis(): sudo("apt-get update && apt-get install -y redis-server") @@ -20,14 +20,6 @@ def setup_channels(): sudo("python setup.py install") -@task -def setup_tester(): - sudo("apt-get update && apt-get install -y apache2-utils python3-pip") - sudo("pip3 -U pip autobahn twisted") - sudo("rm -rf /srv/channels") - sudo("git clone https://github.com/andrewgodwin/channels.git /srv/channels/") - - @task def run_daphne(redis_ip): with cd("/srv/channels/testproject/"): @@ -40,6 +32,30 @@ def run_worker(redis_ip): sudo("REDIS_URL=redis://%s:6379 python manage.py runworker" % redis_ip) +# Current loadtesting setup +@task +def setup_load_tester(src="https://github.com/andrewgodwin/channels.git"): + sudo("apt-get update && apt-get install -y git nodejs && apt-get install npm") + sudo("npm install -g loadtest") + sudo("ln -s /usr/bin/nodejs /usr/bin/node") + + +# Run current loadtesting setup +# example usage: $ fab run_loadtest:http://127.0.0.1,rps=10 -i "id_rsa" -H ubuntu@example.com +@task +def run_loadtest(host, t=90, rps=200): + sudo("loadtest -c 10 --rps {rps} -t {t} {h}".format(h=host, t=t, rps=rps)) + + +# Task that Andrew used for loadtesting earlier on +@task +def setup_tester(): + sudo("apt-get update && apt-get install -y apache2-utils python3-pip") + sudo("pip3 -U pip autobahn twisted") + sudo("rm -rf /srv/channels") + sudo("git clone https://github.com/andrewgodwin/channels.git /srv/channels/") + + @task def shell(): sudo("bash") diff --git a/testproject/locustfile.py b/testproject/locustfile.py new file mode 100644 index 0000000..1379f2a --- /dev/null +++ b/testproject/locustfile.py @@ -0,0 +1,16 @@ +from locust import HttpLocust, TaskSet, task + +class UserBehavior(TaskSet): + def on_start(self): + """ on_start is called when a Locust start before any task is scheduled """ + self.index() + + @task + def index(self): + self.client.get("/") + + +class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait=5000 + max_wait=9000 diff --git a/testproject/requirements.txt b/testproject/requirements.txt index 0d4a5aa..3bb2297 100644 --- a/testproject/requirements.txt +++ b/testproject/requirements.txt @@ -2,7 +2,7 @@ asgi-redis==0.13.1 asgiref==0.13.3 autobahn==0.14.1 channels==0.14.2 -daphne==0.12.1 +daphne==0.13.1 Django==1.9.7 docutils==0.12 msgpack-python==0.4.7 diff --git a/testproject/testproject/asgi.py b/testproject/testproject/asgi.py index 1547a11..ae1640f 100644 --- a/testproject/testproject/asgi.py +++ b/testproject/testproject/asgi.py @@ -1,5 +1,5 @@ import os from channels.asgi import get_channel_layer -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings.channels") channel_layer = get_channel_layer() diff --git a/testproject/testproject/settings/__init__.py b/testproject/testproject/settings/__init__.py new file mode 100644 index 0000000..455876f --- /dev/null +++ b/testproject/testproject/settings/__init__.py @@ -0,0 +1 @@ +#Blank on purpose diff --git a/testproject/testproject/settings.py b/testproject/testproject/settings/base.py similarity index 68% rename from testproject/testproject/settings.py rename to testproject/testproject/settings/base.py index d14a41f..e0f8773 100644 --- a/testproject/testproject/settings.py +++ b/testproject/testproject/settings/base.py @@ -11,7 +11,6 @@ INSTALLED_APPS = ( 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', - 'channels', ) ROOT_URLCONF = 'testproject.urls' @@ -26,13 +25,3 @@ DATABASES = { 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), } } - -CHANNEL_LAYERS = { - "default": { - "BACKEND": "asgi_redis.RedisChannelLayer", - "ROUTING": "testproject.urls.channel_routing", - "CONFIG": { - "hosts": [os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')], - } - }, -} diff --git a/testproject/testproject/settings/channels.py b/testproject/testproject/settings/channels.py new file mode 100644 index 0000000..4ab6439 --- /dev/null +++ b/testproject/testproject/settings/channels.py @@ -0,0 +1,16 @@ +# Settings for channels specifically +from testproject.settings.base import * + +INSTALLED_APPS += ( + 'channels', +) + +CHANNEL_LAYERS = { + "default": { + "BACKEND": "asgi_redis.RedisChannelLayer", + "ROUTING": "testproject.urls.channel_routing", + "CONFIG": { + "hosts": [os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')], + } + }, +} diff --git a/testproject/testproject/urls.py b/testproject/testproject/urls.py index 1dca076..89bb74d 100644 --- a/testproject/testproject/urls.py +++ b/testproject/testproject/urls.py @@ -1,13 +1,17 @@ from django.conf.urls import url -from chtest import consumers, views - +from chtest import views urlpatterns = [ url(r'^$', views.index), ] -channel_routing = { +try: + from chtest import consumers + + channel_routing = { "websocket.receive": consumers.ws_message, "websocket.connect": consumers.ws_connect, } +except: + pass diff --git a/testproject/testproject/wsgi.py b/testproject/testproject/wsgi.py index c24d001..9dcfd86 100644 --- a/testproject/testproject/wsgi.py +++ b/testproject/testproject/wsgi.py @@ -11,6 +11,6 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings.channels") application = get_wsgi_application() diff --git a/testproject/testproject/wsgi_no_channels.py b/testproject/testproject/wsgi_no_channels.py new file mode 100644 index 0000000..79863f9 --- /dev/null +++ b/testproject/testproject/wsgi_no_channels.py @@ -0,0 +1,16 @@ +""" +WSGI config for testproject project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/dev/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings.base") + +application = get_wsgi_application() From a37238a769a07f641202a540531b9f6ebce66651 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 3 Aug 2016 23:23:04 -0700 Subject: [PATCH 479/746] Remove download badge, seems broken --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 573765c..8631dae 100644 --- a/README.rst +++ b/README.rst @@ -3,9 +3,6 @@ Django Channels .. image:: https://api.travis-ci.org/andrewgodwin/channels.svg :target: https://travis-ci.org/andrewgodwin/channels - -.. image:: https://img.shields.io/pypi/dm/channels.svg - :target: https://pypi.python.org/pypi/channels .. image:: https://readthedocs.org/projects/channels/badge/?version=latest :target: http://channels.readthedocs.org/en/latest/?badge=latest From 9bd8bcf652156b09c6ebeb3c5d22a3e4e1529758 Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Thu, 4 Aug 2016 17:30:52 -0500 Subject: [PATCH 480/746] pass subprotocols if defined (#282) --- channels/management/commands/runserver.py | 1 + channels/tests/test_management.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 49fa7ae..be9b643 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -80,6 +80,7 @@ class Command(RunserverCommand): signal_handlers=not options['use_reloader'], action_logger=self.log_action, http_timeout=60, # Shorter timeout than normal as it's dev + ws_protocols=getattr(settings, 'CHANNELS_WS_PROTOCOLS', None), ).run() self.logger.debug("Daphne exited") except KeyboardInterrupt: diff --git a/channels/tests/test_management.py b/channels/tests/test_management.py index 39d8f53..ca759a0 100644 --- a/channels/tests/test_management.py +++ b/channels/tests/test_management.py @@ -86,7 +86,8 @@ class RunServerTests(TestCase): # https://github.com/django/django/blob/master/django/core/management/commands/runserver.py#L105 call_command('runserver', '--noreload') mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, - host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY) + host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, + ws_protocols=None) @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) @mock.patch('channels.management.commands.runserver.Server') @@ -99,11 +100,13 @@ class RunServerTests(TestCase): with self.settings(DEBUG=True, STATIC_URL='/static/'): call_command('runserver', '--noreload') mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, - host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY) + host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, + ws_protocols=None) call_command('runserver', '--noreload', 'localhost:8001') mocked_server.assert_called_with(port=8001, signal_handlers=True, http_timeout=60, - host='localhost', action_logger=mock.ANY, channel_layer=mock.ANY) + host='localhost', action_logger=mock.ANY, channel_layer=mock.ANY, + ws_protocols=None) self.assertFalse(mocked_worker.called, "The worker should not be called with '--noworker'") @@ -117,7 +120,8 @@ class RunServerTests(TestCase): ''' call_command('runserver', '--noreload', '--noworker') mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, - host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY) + host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, + ws_protocols=None) self.assertFalse(mocked_worker.called, "The worker should not be called with '--noworker'") From 2d97ab2cc70f8439bc0110941b43b906d96a621f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 4 Aug 2016 16:55:43 -0700 Subject: [PATCH 481/746] Releasing version 0.17.2 --- CHANGELOG.txt | 15 +++++++++++++++ channels/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index cd7aaed..70a4798 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,17 @@ +0.17.2 (2016-08-04) +------------------- + +* New CHANNELS_WS_PROTOCOLS setting if you want Daphne to accept certain + subprotocols + +* WebsocketBindingWithMembers allows serialization of non-fields on instances + +* Class-based consumers have an .as_route() method that lets you skip using + route_class + +* Bindings now work if loaded after app ready state + + 0.17.1 (2016-07-22) ------------------- @@ -11,6 +25,7 @@ * A worker_ready signal now gets triggered when `runworker` starts consuming messages. It does not fire from within `runserver`. + 0.17.0 (2016-07-19) ------------------- diff --git a/channels/__init__.py b/channels/__init__.py index 34cc1fa..70a855c 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.1" +__version__ = "0.17.2" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 0bc1cee1035ce0859e07fe11c94d5be46ede1a35 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Fri, 5 Aug 2016 09:12:59 +0200 Subject: [PATCH 482/746] Easy application of decorators to Bindings (#281) * Added get_handler Added a get_handler method that applies decorators to the consumer-classmethod * added imports for decorators * Added get_handler to WebsocketBinding * Fixed missing import * channel_session_user defaults to True * removed user-transfer from http would only work in a connect-method * removed unused import --- channels/binding/base.py | 23 ++++++++++++++++++++++- channels/binding/websockets.py | 19 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index 2af9969..a7cb1cb 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -6,6 +6,7 @@ from django.apps import apps from django.db.models.signals import post_save, post_delete from ..channel import Group +from ..auth import channel_session, channel_session_user class BindingMetaclass(type): @@ -62,6 +63,10 @@ class Binding(object): fields = None + # Decorators + channel_session_user = True + channel_session = False + @classmethod def register(cls): """ @@ -158,7 +163,23 @@ class Binding(object): # Run incoming action self.run_action(self.action, self.pk, self.data) - consumer = trigger_inbound + @classmethod + def get_handler(cls): + """ + Adds decorators to trigger_inbound. + """ + handler = cls.trigger_inbound + if cls.channel_session_user: + return channel_session_user(handler) + elif cls.channel_session: + return channel_session(handler) + else: + return handler + + @classmethod + def consumer(cls, message, **kwargs): + handler = cls.get_handler() + handler(message, **kwargs) def deserialize(self, message): """ diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 23c40d1..044f09c 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -5,6 +5,7 @@ from django.core.serializers.json import DjangoJSONEncoder from .base import Binding from ..generic.websockets import WebsocketDemultiplexer +from ..sessions import enforce_ordering class WebsocketBinding(Binding): @@ -32,6 +33,10 @@ class WebsocketBinding(Binding): stream = None + # Decorators + strict_ordering = False + slight_ordering = False + # Outbound @classmethod def encode(cls, stream, payload): @@ -58,6 +63,20 @@ class WebsocketBinding(Binding): return json.loads(data)[0]['fields'] # Inbound + @classmethod + def get_handler(cls): + """ + Adds decorators to trigger_inbound. + """ + # Get super-handler + handler = super(WebsocketBinding, cls).get_handler() + # Ordering decorators + if cls.strict_ordering: + return enforce_ordering(handler, slight=False) + elif cls.slight_ordering: + return enforce_ordering(handler, slight=True) + else: + return handler def deserialize(self, message): """ From f682298341b4049b02581ce3c68a5cfeec10b8df Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 5 Aug 2016 22:17:43 -0700 Subject: [PATCH 483/746] Show daphne server logs in runserver --- channels/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/log.py b/channels/log.py index 581f5b8..26c8bf6 100644 --- a/channels/log.py +++ b/channels/log.py @@ -21,7 +21,7 @@ def setup_logger(name, verbosity=1): logger.setLevel(logging.DEBUG) # Set up daphne protocol loggers - for module in ["daphne.ws_protocol", "daphne.http_protocol"]: + for module in ["daphne.ws_protocol", "daphne.http_protocol", "daphne.server"]: daphne_logger = logging.getLogger(module) daphne_logger.addHandler(handler) daphne_logger.setLevel( From 0f1ac3d08bd3df47d4ffe261ca563aa674b926cb Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Thu, 11 Aug 2016 06:55:51 +0200 Subject: [PATCH 484/746] Fix member serialization in WebsocketBindingWithMembers (#292) * Fix member serialization in WebsocketBindingWithMembers * pep fixes * allow usage of 'dot'-notation in send_members * replace dots for dictionary * single quotes --- channels/binding/websockets.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 044f09c..e0d0d3d 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -147,10 +147,17 @@ class WebsocketBindingWithMembers(WebsocketBinding): def serialize_data(self, instance): data = super(WebsocketBindingWithMembers, self).serialize_data(instance) + member_data = {} for m in self.send_members: - member = getattr(instance, m) + member = instance + for s in m.split('.'): + member = getattr(member, s) if callable(member): - data[m] = self.encoder.encode(member()) + member_data[m.replace('.', '__')] = member() else: - data[m] = self.encoder.encode(member) + member_data[m.replace('.', '__')] = member + member_data = json.loads(self.encoder.encode(member_data)) + # the update never overwrites any value from data, + # because an object can't have two attributes with the same name + data.update(member_data) return data From fb2e9320c2f9704a20e20b5dbd0ae3657fc1b273 Mon Sep 17 00:00:00 2001 From: Francis Mwangi Date: Sun, 14 Aug 2016 05:08:00 +0300 Subject: [PATCH 485/746] fixed typo (#296) --- channels/binding/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index a7cb1cb..1ec92a3 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -104,7 +104,7 @@ class Binding(object): @classmethod def delete_receiver(cls, instance, **kwargs): """ - Entry point for triggering the binding from save signals. + Entry point for triggering the binding from delete signals. """ cls.trigger_outbound(instance, "delete") From 32568dc879410ca12e17e36dcd450fa7223c2282 Mon Sep 17 00:00:00 2001 From: Oskar Hahn Date: Mon, 15 Aug 2016 03:18:24 +0200 Subject: [PATCH 486/746] Start only one worker with the nothreading option. (#298) Reuse the nothreading option of the runserver command to only start one worker. --- channels/management/commands/runserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index be9b643..45debb1 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -65,7 +65,8 @@ class Command(RunserverCommand): # Launch workers as subthreads if options.get("run_worker", True): - for _ in range(4): + worker_count = 4 if options.get("use_threading", True) else 1 + for _ in range(worker_count): worker = WorkerThread(self.channel_layer, self.logger) worker.daemon = True worker.start() From 57fa3bed6713bddc0e650154e3297e40ae28910b Mon Sep 17 00:00:00 2001 From: Tyson Clugg Date: Tue, 16 Aug 2016 16:49:18 +1000 Subject: [PATCH 487/746] Implement the data binding behaviour from Django DDP. (#301) * Implement the data binding behaviour from Django DDP. Correct dispatch of create/update/delete according to how group_names change when compared between pre and post save/delete. * Fix tests for databinding improvements. --- channels/binding/base.py | 105 ++++++++++++++++++++++++--------- channels/tests/test_binding.py | 9 ++- 2 files changed, 84 insertions(+), 30 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index 1ec92a3..902b56e 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -3,12 +3,17 @@ from __future__ import unicode_literals import six from django.apps import apps -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_save, post_delete, pre_save, pre_delete from ..channel import Group from ..auth import channel_session, channel_session_user +CREATE = 'create' +UPDATE = 'update' +DELETE = 'delete' + + class BindingMetaclass(type): """ Metaclass that tracks instantiations of its type. @@ -72,10 +77,22 @@ class Binding(object): """ Resolves models. """ + # Connect signals + for model in cls.get_registered_models(): + pre_save.connect(cls.pre_save_receiver, sender=model) + post_save.connect(cls.post_save_receiver, sender=model) + pre_delete.connect(cls.pre_delete_receiver, sender=model) + post_delete.connect(cls.post_delete_receiver, sender=model) + + @classmethod + def get_registered_models(cls): + """ + Resolves the class model attribute if it's a string and returns it. + """ # If model is None directly on the class, assume it's abstract. if cls.model is None: if "model" in cls.__dict__: - return + return [] else: raise ValueError("You must set the model attribute on Binding %r!" % cls) # If fields is not defined, raise an error @@ -88,26 +105,10 @@ class Binding(object): cls.model._meta.app_label.lower(), cls.model._meta.object_name.lower(), ) - # Connect signals - post_save.connect(cls.save_receiver, sender=cls.model) - post_delete.connect(cls.delete_receiver, sender=cls.model) + return [cls.model] # Outbound binding - @classmethod - def save_receiver(cls, instance, created, **kwargs): - """ - Entry point for triggering the binding from save signals. - """ - cls.trigger_outbound(instance, "create" if created else "update") - - @classmethod - def delete_receiver(cls, instance, **kwargs): - """ - Entry point for triggering the binding from delete signals. - """ - cls.trigger_outbound(instance, "delete") - @classmethod def encode(cls, stream, payload): """ @@ -116,20 +117,70 @@ class Binding(object): raise NotImplementedError() @classmethod - def trigger_outbound(cls, instance, action): + def pre_save_receiver(cls, instance, **kwargs): + cls.pre_change_receiver(instance, CREATE if instance.pk is None else UPDATE) + + @classmethod + def post_save_receiver(cls, instance, created, **kwargs): + cls.post_change_receiver(instance, CREATE if created else UPDATE) + + @classmethod + def pre_delete_receiver(cls, instance, **kwargs): + cls.pre_change_receiver(instance, DELETE) + + @classmethod + def post_delete_receiver(cls, instance, **kwargs): + cls.post_change_receiver(instance, DELETE) + + @classmethod + def pre_change_receiver(cls, instance, action): + """ + Entry point for triggering the binding from save signals. + """ + if action == CREATE: + group_names = set() + else: + group_names = set(cls.group_names(instance)) + + if not hasattr(instance, '_binding_group_names'): + instance._binding_group_names = {} + instance._binding_group_names[cls] = group_names + + @classmethod + def post_change_receiver(cls, instance, action): """ Triggers the binding to possibly send to its group. """ + old_group_names = instance._binding_group_names[cls] + if action == DELETE: + new_group_names = set() + else: + new_group_names = set(cls.group_names(instance)) + + # if post delete, new_group_names should be [] self = cls() self.instance = instance - # Check to see if we're covered + + # Django DDP had used the ordering of DELETE, UPDATE then CREATE for good reasons. + self.send_messages(instance, old_group_names - new_group_names, DELETE) + self.send_messages(instance, old_group_names & new_group_names, UPDATE) + self.send_messages(instance, new_group_names - old_group_names, CREATE) + + def send_messages(self, instance, group_names, action): + """ + Serializes the instance and sends it to all provided group names. + """ + if not group_names: + return # no need to serialize, bail. payload = self.serialize(instance, action) - if payload != {}: - assert self.stream is not None - message = cls.encode(self.stream, payload) - for group_name in self.group_names(instance, action): - group = Group(group_name) - group.send(message) + if payload == {}: + return # nothing to send, bail. + + assert self.stream is not None + message = self.encode(self.stream, payload) + for group_name in group_names: + group = Group(group_name) + group.send(message) def group_names(self, instance, action): """ diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index dd85208..dca1092 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -17,7 +17,8 @@ class TestsBinding(ChannelTestCase): stream = 'test' fields = ['username', 'email', 'password', 'last_name'] - def group_names(self, instance, action): + @classmethod + def group_names(cls, instance): return ["users"] def has_permission(self, user, action, pk): @@ -58,7 +59,8 @@ class TestsBinding(ChannelTestCase): stream = 'test' fields = ['__all__'] - def group_names(self, instance, action): + @classmethod + def group_names(cls, instance): return ["users2"] def has_permission(self, user, action, pk): @@ -102,7 +104,8 @@ class TestsBinding(ChannelTestCase): stream = 'test' fields = ['username'] - def group_names(self, instance, action): + @classmethod + def group_names(cls, instance): return ["users3"] def has_permission(self, user, action, pk): From 6649afce8e9f88c639db8e3eaa7385918f543117 Mon Sep 17 00:00:00 2001 From: Luke Hodkinson Date: Sun, 21 Aug 2016 10:53:54 +1000 Subject: [PATCH 488/746] Use a mixin for common test-case code. This way we can have both (#305) a regular channels test-case, and a transaction test-case, too. --- channels/tests/__init__.py | 2 +- channels/tests/base.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/channels/tests/__init__.py b/channels/tests/__init__.py index 36481f0..0c957f3 100644 --- a/channels/tests/__init__.py +++ b/channels/tests/__init__.py @@ -1,2 +1,2 @@ -from .base import ChannelTestCase, Client, apply_routes # NOQA isort:skip +from .base import TransactionChannelTestCase, ChannelTestCase, Client, apply_routes # NOQA isort:skip from .http import HttpClient # NOQA isort:skip diff --git a/channels/tests/base.py b/channels/tests/base.py index 248a837..5a90eca 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -5,7 +5,7 @@ import random import string from functools import wraps -from django.test.testcases import TestCase +from django.test.testcases import TestCase, TransactionTestCase from .. import DEFAULT_CHANNEL_LAYER from ..channel import Group from ..routing import Router, include @@ -14,7 +14,7 @@ from ..message import Message from asgiref.inmemory import ChannelLayer as InMemoryChannelLayer -class ChannelTestCase(TestCase): +class ChannelTestCaseMixin(object): """ TestCase subclass that provides easy methods for testing channels using an in-memory backend to capture messages, and assertion methods to allow @@ -31,7 +31,7 @@ class ChannelTestCase(TestCase): """ Initialises in memory channel layer for the duration of the test """ - super(ChannelTestCase, self)._pre_setup() + super(ChannelTestCaseMixin, self)._pre_setup() self._old_layers = {} for alias in self.test_channel_aliases: # Swap in an in memory layer wrapper and keep the old one around @@ -52,7 +52,7 @@ class ChannelTestCase(TestCase): # Swap in an in memory layer wrapper and keep the old one around channel_layers.set(alias, self._old_layers[alias]) del self._old_layers - super(ChannelTestCase, self)._post_teardown() + super(ChannelTestCaseMixin, self)._post_teardown() def get_next_message(self, channel, alias=DEFAULT_CHANNEL_LAYER, require=False): """ @@ -70,6 +70,14 @@ class ChannelTestCase(TestCase): return Message(content, recv_channel, channel_layers[alias]) +class ChannelTestCase(ChannelTestCaseMixin, TestCase): + pass + + +class TransactionChannelTestCase(ChannelTestCaseMixin, TransactionTestCase): + pass + + class Client(object): """ Channel client abstraction that provides easy methods for testing full live cycle of message in channels From fcb2875b539f44c162681a4a89bad046603ea0f3 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Tue, 23 Aug 2016 19:13:46 +0200 Subject: [PATCH 489/746] fixed usage of group_names in *_change_receiver (#306) * fixed usage of group_names in *_change_receiver group_names was missing second arg (action) * fixed group_names to include action * made group_names a classmethod --- channels/binding/base.py | 7 ++++--- channels/tests/test_binding.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index 902b56e..a1f1977 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -140,7 +140,7 @@ class Binding(object): if action == CREATE: group_names = set() else: - group_names = set(cls.group_names(instance)) + group_names = set(cls.group_names(instance, action)) if not hasattr(instance, '_binding_group_names'): instance._binding_group_names = {} @@ -155,7 +155,7 @@ class Binding(object): if action == DELETE: new_group_names = set() else: - new_group_names = set(cls.group_names(instance)) + new_group_names = set(cls.group_names(instance, action)) # if post delete, new_group_names should be [] self = cls() @@ -182,7 +182,8 @@ class Binding(object): group = Group(group_name) group.send(message) - def group_names(self, instance, action): + @classmethod + def group_names(cls, instance, action): """ Returns the iterable of group names to send the object to based on the instance and action performed on it. diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index dca1092..aaf88aa 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -18,7 +18,7 @@ class TestsBinding(ChannelTestCase): fields = ['username', 'email', 'password', 'last_name'] @classmethod - def group_names(cls, instance): + def group_names(cls, instance, action): return ["users"] def has_permission(self, user, action, pk): @@ -60,7 +60,7 @@ class TestsBinding(ChannelTestCase): fields = ['__all__'] @classmethod - def group_names(cls, instance): + def group_names(cls, instance, action): return ["users2"] def has_permission(self, user, action, pk): @@ -105,7 +105,7 @@ class TestsBinding(ChannelTestCase): fields = ['username'] @classmethod - def group_names(cls, instance): + def group_names(cls, instance, action): return ["users3"] def has_permission(self, user, action, pk): From 9a7317f5835b23875fae06dfe93b6bbaec0a2365 Mon Sep 17 00:00:00 2001 From: Luke Hodkinson Date: Wed, 24 Aug 2016 12:47:29 +1000 Subject: [PATCH 490/746] Add a link to django-cq. (#310) * Use a mixin for common test-case code. This way we can have both a regular channels test-case, and a transaction test-case, too. * Adding a reference to django-cq. --- docs/community.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/community.rst b/docs/community.rst index c387c0f..c3678fe 100644 --- a/docs/community.rst +++ b/docs/community.rst @@ -5,10 +5,12 @@ These projects from the community are developed on top of Channels: * Djangobot_, a bi-directional interface server for Slack. * knocker_, a generic desktop-notification system. -* Beatserver_, a periodic task scheduler for django channels +* Beatserver_, a periodic task scheduler for django channels. +* cq_, a simple distributed task system. If you'd like to add your project, please submit a PR with a link and brief description. .. _Djangobot: https://github.com/djangobot/djangobot .. _knocker: https://github.com/nephila/django-knocker .. _Beatserver: https://github.com/rajasimon/beatserver +.. _cq: https://github.com/furious-luke/django-cq From 3cc6f744a6fb7752759f2abb6baf7cd6ab2a6e72 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Aug 2016 20:06:37 -0700 Subject: [PATCH 491/746] Add contributing page to docs --- docs/contributing.rst | 51 +++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 52 insertions(+) create mode 100644 docs/contributing.rst diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..17d3f7b --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,51 @@ +Contributing +============ + +If you're looking to contribute to Channels, then please read on - we encourage +contributions both large and small, from both novice and seasoned developers. + + +What can I work on? +------------------- + +We're looking for help with the following areas: + + * Documentation and tutorial writing + * Bugfixing and testing + * Feature polish and occasional new feature design + * Case studies and writeups + +You can find a particular list of what we're looking to work on right now +on the `ChannelsTasks Django wiki page `_, +but this is just a suggested list - any offer to help is welcome. + + +I'm interested, how should I get started? +----------------------------------------- + +The best thing to do is to see if there's a `GitHub issue `_ +for the thing you wish to work on - if there is, leave a comment saying you're +going to take it on, and if not, open one describing what you're doing so there's +a place to record information around. + +If you have questions, you can either open an issue with the questions detailed, +hop on the ``#django-channels`` channel on Freenode IRC, or email Andrew directly +at ``andrew@aeracode.org``. + + +I'm a novice contributor/developer - can I help? +------------------------------------------------ + +Of course - just get in touch like above and mention your experience level, +and we'll try the best we can to match you up with someone to mentor you through +the task. + + +Can you pay me for my time? +--------------------------- + +Thanks to Mozilla, we have a reasonable budget to pay people for their time +working on all of the above sorts of tasks and more. If you're interested in +working on something and being paid, you'll need to draw up a short proposal +and get it approved first - the wiki page above has more details, and if you're +interested, email andrew@aeracode.org to get the conversation started. diff --git a/docs/index.rst b/docs/index.rst index e34db68..140467c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,3 +39,4 @@ Contents: faqs asgi community + contributing From f699d112b0b7c4eebeee6aae85ffdd77e68fbf10 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Aug 2016 20:10:15 -0700 Subject: [PATCH 492/746] Fixed #309: Missed a double decode for header coalescing --- channels/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index 3e8912e..4e1c73d 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -84,7 +84,7 @@ class AsgiRequest(http.HttpRequest): # HTTPbis say only ASCII chars are allowed in headers, but we latin1 just in case value = value.decode("latin1") if corrected_name in self.META: - value = self.META[corrected_name] + "," + value.decode("latin1") + value = self.META[corrected_name] + "," + value self.META[corrected_name] = value # Pull out request encoding if we find it if "CONTENT_TYPE" in self.META: From 3e2444b9b9c779aaf022a0861e7d5ee4a83d59ff Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 23 Aug 2016 20:13:20 -0700 Subject: [PATCH 493/746] Clarify tasks better --- docs/contributing.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 17d3f7b..0dba851 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -15,9 +15,13 @@ We're looking for help with the following areas: * Feature polish and occasional new feature design * Case studies and writeups -You can find a particular list of what we're looking to work on right now -on the `ChannelsTasks Django wiki page `_, -but this is just a suggested list - any offer to help is welcome. +You can find what we're looking to work on right now in two places: + + * Specific bugs are in the `GitHub issues `_ + * Higher-level tasks are on the `ChannelsTasks Django wiki page `_ + +These are, however, just a suggested list - any offer to help is welcome as long +as it fits the project goals. I'm interested, how should I get started? From 860da6e241668052a33beb2f786fd6e21d6c0a86 Mon Sep 17 00:00:00 2001 From: Peter Schmidt Date: Wed, 24 Aug 2016 23:50:38 +0200 Subject: [PATCH 494/746] Testing Docs: Update import from Channel -> Group (#307) --- docs/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing.rst b/docs/testing.rst index 0681835..73b250e 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -69,7 +69,7 @@ You can test Groups in the same way as Channels inside a ``ChannelTestCase``; the entire channel layer is flushed each time a test is run, so it's safe to do group adds and sends during a test. For example:: - from channels import Channel + from channels import Group from channels.tests import ChannelTestCase class MyTests(ChannelTestCase): From 39339e66bc0f8cf896a33f88dc4a379b61b9faf5 Mon Sep 17 00:00:00 2001 From: Tim Watts Date: Thu, 25 Aug 2016 02:39:18 +0200 Subject: [PATCH 495/746] Remove invalid test. FileResponse requires a file object. (#311) * Remove invalid test. FileResponse requires a file object. * rm unused imports --- channels/tests/test_handler.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index d529c01..933d68d 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import os -import unittest from datetime import datetime from itertools import islice @@ -9,7 +8,7 @@ from django.http import ( FileResponse, HttpResponse, HttpResponseRedirect, JsonResponse, StreamingHttpResponse, ) -from six import BytesIO, StringIO +from six import BytesIO from channels import Channel from channels.handler import AsgiHandler @@ -357,18 +356,3 @@ class HandlerTests(ChannelTestCase): self.assertEqual(reply_messages[0]['status'], 302) header_dict = dict(reply_messages[0]['headers']) self.assertEqual(header_dict[b'Location'].decode(), redirect_to) - - @unittest.skip("failing under python 3") - def test_stringio_file_response(self): - Channel("test").send({ - "reply_channel": "test", - "http_version": "1.1", - "method": "GET", - "path": b"/test/", - }) - response = FileResponse(StringIO('sadfdasfsdfsadf')) - handler = FakeAsgiHandler(response) - # Use islice because the generator never ends. - reply_messages = list( - islice(handler(self.get_next_message("test", require=True)), 5)) - self.assertEqual(len(reply_messages), 2, reply_messages) From c8fdea646075a55da16b2cba897e7b945b16bac3 Mon Sep 17 00:00:00 2001 From: Frank Pape Date: Fri, 26 Aug 2016 15:04:38 -0400 Subject: [PATCH 496/746] Add send_and_consume method to HttpClient to match base Client behavior. Change argument ordering in send method to match base client. Fix repeated "live cycle" error in docstrings. (#320) --- channels/tests/base.py | 2 +- channels/tests/http.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/channels/tests/base.py b/channels/tests/base.py index 5a90eca..1f628ac 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -129,7 +129,7 @@ class Client(object): def send_and_consume(self, channel, content={}, fail_on_none=True): """ - Reproduce full live cycle of the message + Reproduce full life cycle of the message """ self.send(channel, content) return self.consume(channel, fail_on_none=fail_on_none) diff --git a/channels/tests/http.py b/channels/tests/http.py index ecb0ed3..2b8ecd0 100644 --- a/channels/tests/http.py +++ b/channels/tests/http.py @@ -10,7 +10,7 @@ from .base import Client class HttpClient(Client): """ - Channel http/ws client abstraction that provides easy methods for testing full live cycle of message in channels + Channel http/ws client abstraction that provides easy methods for testing full life cycle of message in channels with determined reply channel, auth opportunity, cookies, headers and so on """ @@ -61,7 +61,7 @@ class HttpClient(Client): if content: return json.loads(content['text']) - def send(self, to, text=None, content={}, path='/'): + def send(self, to, content={}, text=None, path='/'): """ Send a message to a channel. Adds reply_channel name and channel_session to the message. @@ -75,6 +75,13 @@ class HttpClient(Client): content['text'] = json.dumps(text) self.channel_layer.send(to, content) + def send_and_consume(self, channel, content={}, text=None, path='/', fail_on_none=True): + """ + Reproduce full life cycle of the message + """ + self.send(channel, content, text, path) + return self.consume(channel, fail_on_none=fail_on_none) + def login(self, **credentials): """ Returns True if login is possible; False if the provided credentials From e6236f79e63aeeded59cc0d546ad55839cf44520 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Fri, 26 Aug 2016 22:12:14 +0300 Subject: [PATCH 497/746] Move the groups at class definition (#321) --- channels/generic/websockets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 6c0820f..272b938 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -27,6 +27,8 @@ class WebsocketConsumer(BaseConsumer): slight_ordering = False strict_ordering = False + groups = None + def get_handler(self, message, **kwargs): """ Pulls out the path onto an instance variable, and optionally @@ -54,7 +56,7 @@ class WebsocketConsumer(BaseConsumer): Group(s) to make people join when they connect and leave when they disconnect. Make sure to return a list/tuple, not a string! """ - return [] + return self.groups or [] def raw_connect(self, message, **kwargs): """ From 7de6ff17d6ec0ce10df323a4358573bac96b60c7 Mon Sep 17 00:00:00 2001 From: Sam Bolgert Date: Sat, 27 Aug 2016 13:08:31 -0700 Subject: [PATCH 498/746] Fixed #251: Add docs for testing Generic Consumers (#323) --- docs/testing.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/testing.rst b/docs/testing.rst index 73b250e..11a5aa3 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -62,6 +62,31 @@ and post the square of it to the ``"result"`` channel:: self.assertEqual(result['value'], 1089) +Generic Consumers +----------------- + +You can use ``ChannelTestCase`` to test generic consumers as well. Just pass the message +object from ``get_next_message`` to the constructor of the class. To test replies to a specific channel, +use the ``reply_channel`` property on the ``Message`` object. For example:: + + from channels import Channel + from channels.tests import ChannelTestCase + + from myapp.consumers import MyConsumer + + class MyTests(ChannelTestCase): + + def test_a_thing(self): + # Inject a message onto the channel to use in a consumer + Channel("input").send({"value": 33}) + # Run the consumer with the new Message object + message = self.get_next_message("input", require=True) + MyConsumer(message) + # Verify there's a reply and that it's accurate + result = self.get_next_message(message.reply_channel.name, require=True) + self.assertEqual(result['value'], 1089) + + Groups ------ From b62ea7dd0af02abb6c3716c36722216eb75e5c99 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Mon, 29 Aug 2016 20:37:09 +0300 Subject: [PATCH 499/746] Naming consumers that are classmethods (#324) --- channels/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/channels/utils.py b/channels/utils.py index 548c307..423f61f 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -13,9 +13,11 @@ def name_that_thing(thing): return name_that_thing(thing.im_class) + "." + thing.im_func.func_name # Other named thing if hasattr(thing, "__name__"): - if hasattr(thing, "__class__") and not isinstance(thing, types.FunctionType): + if hasattr(thing, "__class__") and not isinstance(thing, (types.FunctionType, types.MethodType)): if thing.__class__ is not type: return name_that_thing(thing.__class__) + if hasattr(thing, "__self__"): + return "%s.%s" % (thing.__self__.__module__, thing.__self__.__name__) if hasattr(thing, "__module__"): return "%s.%s" % (thing.__module__, thing.__name__) # Generic instance of a class From 0f579608a3f8994b9bf2bbb21f1e51983c74f442 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Mon, 29 Aug 2016 20:37:19 +0300 Subject: [PATCH 500/746] Added django-channels-panel to the list of projects (#325) --- docs/community.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/community.rst b/docs/community.rst index c3678fe..e844e31 100644 --- a/docs/community.rst +++ b/docs/community.rst @@ -7,6 +7,7 @@ These projects from the community are developed on top of Channels: * knocker_, a generic desktop-notification system. * Beatserver_, a periodic task scheduler for django channels. * cq_, a simple distributed task system. +* Debugpannel_, a django Debug Toolbar panel for channels. If you'd like to add your project, please submit a PR with a link and brief description. @@ -14,3 +15,4 @@ If you'd like to add your project, please submit a PR with a link and brief desc .. _knocker: https://github.com/nephila/django-knocker .. _Beatserver: https://github.com/rajasimon/beatserver .. _cq: https://github.com/furious-luke/django-cq +.. _Debugpannel: https://github.com/Krukov/django-channels-panel From a23810e0facc7335d9d02d7ff60c86ec903ed977 Mon Sep 17 00:00:00 2001 From: Steven Davidson Date: Tue, 30 Aug 2016 17:09:04 +0100 Subject: [PATCH 501/746] Update ASGI draft spec: http.disconnect gains a path key (#326) --- docs/asgi.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/asgi.rst b/docs/asgi.rst index ca1bf36..e452217 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -704,6 +704,9 @@ Keys: * ``reply_channel``: Channel name responses would have been sent on. No longer valid after this message is sent; all messages to it will be dropped. + +* ``path``: Unicode string HTTP path from URL, with percent escapes decoded + and UTF8 byte sequences decoded into characters. WebSocket From 0d25860cf278e5c59068916cb3a66564d494f775 Mon Sep 17 00:00:00 2001 From: Luke Hodkinson Date: Thu, 1 Sep 2016 14:26:03 +1000 Subject: [PATCH 502/746] Run workers in threads. (#322) * Use a mixin for common test-case code. This way we can have both a regular channels test-case, and a transaction test-case, too. * Adding a reference to django-cq. * Adding the ability to launch a number of workers in threads. This is to try and help reduce memory consumption. * Adding a signal for process level worker startups. * Cleaning up the threaded worker code. * Use Python 2.7 friendly code. * Making the runworker command show a little more information about how many threads are running. * Moving the worker ready signal into a method in order to support polymorphic behavior. * Ugh, I'm an idiot. Was launching the wrong run. * Adding a call to the workers' `ready` in `runserver`. --- channels/management/commands/runserver.py | 1 + channels/management/commands/runworker.py | 33 +++++++++++----- channels/signals.py | 1 + channels/tests/test_worker.py | 43 ++++++++++++++++++++- channels/worker.py | 47 +++++++++++++++++++++++ 5 files changed, 115 insertions(+), 10 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 45debb1..3b68220 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -151,5 +151,6 @@ class WorkerThread(threading.Thread): def run(self): self.logger.debug("Worker thread running") worker = Worker(channel_layer=self.channel_layer, signal_handlers=False) + worker.ready() worker.run() self.logger.debug("Worker thread exited") diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index e082b4a..84454a1 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -6,8 +6,8 @@ from django.core.management import BaseCommand, CommandError from channels import DEFAULT_CHANNEL_LAYER, channel_layers from channels.log import setup_logger from channels.staticfiles import StaticFilesConsumer -from channels.worker import Worker -from channels.signals import worker_ready +from channels.worker import Worker, WorkerGroup +from channels.signals import worker_process_ready class Command(BaseCommand): @@ -28,12 +28,18 @@ class Command(BaseCommand): '--exclude-channels', action='append', dest='exclude_channels', help='Prevents this worker from listening on the provided channels (supports globbing).', ) + parser.add_argument( + '--threads', action='store', dest='threads', + default=1, type=int, + help='Number of threads to execute.' + ) def handle(self, *args, **options): # Get the backend to use self.verbosity = options.get("verbosity", 1) self.logger = setup_logger('django.channels', self.verbosity) self.channel_layer = channel_layers[options.get("layer", DEFAULT_CHANNEL_LAYER)] + self.n_threads = options.get('threads', 1) # Check that handler isn't inmemory if self.channel_layer.local_only(): raise CommandError( @@ -46,21 +52,30 @@ class Command(BaseCommand): self.channel_layer.router.check_default(http_consumer=StaticFilesConsumer()) else: self.channel_layer.router.check_default() - # Launch a worker - self.logger.info("Running worker against channel layer %s", self.channel_layer) # Optionally provide an output callback callback = None if self.verbosity > 1: callback = self.consumer_called + self.callback = callback + self.options = options + # Choose an appropriate worker. + if self.n_threads == 1: + self.logger.info("Using single-threaded worker.") + worker_cls = Worker + else: + self.logger.info("Using multi-threaded worker, {} thread(s).".format(self.n_threads)) + worker_cls = WorkerGroup # Run the worker + self.logger.info("Running worker against channel layer %s", self.channel_layer) try: - worker = Worker( + worker = worker_cls( channel_layer=self.channel_layer, - callback=callback, - only_channels=options.get("only_channels", None), - exclude_channels=options.get("exclude_channels", None), + callback=self.callback, + only_channels=self.options.get("only_channels", None), + exclude_channels=self.options.get("exclude_channels", None), ) - worker_ready.send(sender=worker) + worker_process_ready.send(sender=worker) + worker.ready() worker.run() except KeyboardInterrupt: pass diff --git a/channels/signals.py b/channels/signals.py index 8c33b96..dc83b94 100644 --- a/channels/signals.py +++ b/channels/signals.py @@ -5,6 +5,7 @@ from django.dispatch import Signal consumer_started = Signal(providing_args=["environ"]) consumer_finished = Signal() worker_ready = Signal() +worker_process_ready = Signal() # Connect connection closer to consumer finished as well consumer_finished.connect(close_old_connections) diff --git a/channels/tests/test_worker.py b/channels/tests/test_worker.py index 5cff5b7..bc6b5d4 100644 --- a/channels/tests/test_worker.py +++ b/channels/tests/test_worker.py @@ -4,12 +4,14 @@ try: from unittest import mock except ImportError: import mock +import threading from channels import Channel, route, DEFAULT_CHANNEL_LAYER from channels.asgi import channel_layers from channels.tests import ChannelTestCase -from channels.worker import Worker +from channels.worker import Worker, WorkerGroup from channels.exceptions import ConsumeLater +from channels.signals import worker_ready class PatchedWorker(Worker): @@ -93,3 +95,42 @@ class WorkerTests(ChannelTestCase): worker.run() self.assertEqual(consumer.call_count, 1) self.assertEqual(channel_layer.send.call_count, 0) + + +class WorkerGroupTests(ChannelTestCase): + """ + Test threaded workers. + """ + + def setUp(self): + self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + self.worker = WorkerGroup(self.channel_layer, n_threads=4) + self.subworkers = self.worker.workers + + def test_subworkers_created(self): + self.assertEqual(len(self.subworkers), 3) + + def test_subworkers_no_sigterm(self): + for wrk in self.subworkers: + self.assertFalse(wrk.signal_handlers) + + def test_ready_signals_sent(self): + self.in_signal = 0 + + def handle_signal(sender, *args, **kwargs): + self.in_signal += 1 + + worker_ready.connect(handle_signal) + WorkerGroup(self.channel_layer, n_threads=4) + self.worker.ready() + self.assertEqual(self.in_signal, 4) + + def test_sigterm_handler(self): + threads = [] + for wkr in self.subworkers: + t = threading.Thread(target=wkr.run) + t.start() + threads.append(t) + self.worker.sigterm_handler(None, None) + for t in threads: + t.join() diff --git a/channels/worker.py b/channels/worker.py index f6d93d7..3b67e92 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -5,11 +5,14 @@ import logging import signal import sys import time +import multiprocessing +import threading from .signals import consumer_started, consumer_finished from .exceptions import ConsumeLater from .message import Message from .utils import name_that_thing +from .signals import worker_ready logger = logging.getLogger('django.channels') @@ -66,6 +69,12 @@ class Worker(object): ] return channels + def ready(self): + """ + Called once worker setup is complete. + """ + worker_ready.send(sender=self) + def run(self): """ Tries to continually dispatch messages to consumers. @@ -134,3 +143,41 @@ class Worker(object): else: # Send consumer finished so DB conns close etc. consumer_finished.send(sender=self.__class__) + + +class WorkerGroup(Worker): + """ + Group several workers together in threads. Manages the sub-workers, + terminating them if a signal is received. + """ + + def __init__(self, *args, **kwargs): + n_threads = kwargs.pop('n_threads', multiprocessing.cpu_count()) - 1 + super(WorkerGroup, self).__init__(*args, **kwargs) + kwargs['signal_handlers'] = False + self.workers = [Worker(*args, **kwargs) for ii in range(n_threads)] + + def sigterm_handler(self, signo, stack_frame): + self.termed = True + for wkr in self.workers: + wkr.termed = True + logger.info("Shutdown signal received while busy, waiting for " + "loop termination") + + def ready(self): + super(WorkerGroup, self).ready() + for wkr in self.workers: + wkr.ready() + + def run(self): + """ + Launch sub-workers before running. + """ + self.threads = [threading.Thread(target=self.workers[ii].run) + for ii in range(len(self.workers))] + for t in self.threads: + t.start() + super(WorkerGroup, self).run() + # Join threads once completed. + for t in self.threads: + t.join() From 079558d5b5d12f28135a67e2e53bd669579fe249 Mon Sep 17 00:00:00 2001 From: Luke Hodkinson Date: Sat, 3 Sep 2016 01:02:29 +1000 Subject: [PATCH 503/746] Too many threads bug in threaded worker. (#334) * Use a mixin for common test-case code. This way we can have both a regular channels test-case, and a transaction test-case, too. * Adding a reference to django-cq. * Adding the ability to launch a number of workers in threads. This is to try and help reduce memory consumption. * Adding a signal for process level worker startups. * Cleaning up the threaded worker code. * Use Python 2.7 friendly code. * Making the runworker command show a little more information about how many threads are running. * Moving the worker ready signal into a method in order to support polymorphic behavior. * Ugh, I'm an idiot. Was launching the wrong run. * Adding a call to the workers' `ready` in `runserver`. * Fixed a bug whereby too many threads were being used when threaded workers were used. --- channels/management/commands/runworker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 84454a1..9823b4c 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -59,12 +59,14 @@ class Command(BaseCommand): self.callback = callback self.options = options # Choose an appropriate worker. + worker_kwargs = {} if self.n_threads == 1: self.logger.info("Using single-threaded worker.") worker_cls = Worker else: self.logger.info("Using multi-threaded worker, {} thread(s).".format(self.n_threads)) worker_cls = WorkerGroup + worker_kwargs['n_threads'] = self.n_threads # Run the worker self.logger.info("Running worker against channel layer %s", self.channel_layer) try: @@ -73,6 +75,7 @@ class Command(BaseCommand): callback=self.callback, only_channels=self.options.get("only_channels", None), exclude_channels=self.options.get("exclude_channels", None), + **worker_kwargs ) worker_process_ready.send(sender=worker) worker.ready() From f69bd99c61cf3381dcee85d1bf2eff023a54dce9 Mon Sep 17 00:00:00 2001 From: Robert Roskam Date: Fri, 2 Sep 2016 11:09:01 -0400 Subject: [PATCH 504/746] Many changes to the test project (#333) * Added in simple locust file * Correcting the file name * Updated to latest version of daphne * moving settings up * Moved over channels settings * Removed channels settings * Removed settings file * Moved around files * Made a file for normal wsgi * Changed regular wsgi to point to channels settings * Create __init__.py * Added in the appropriate import * Named it right * Create urls_no_channels.py * Delete urls_no_channels.py * Doing this so I don't have to have multiple urls * Update urls.py * Update urls.py * Added in fabric cmd for installing nodejs loadtest * Added in git dependency * Added in a symlink for loadtest * Made run_loadtest command * Added in argument for time * Changed to format on string * Updated arguments * Fixed typo for argument * Made some comments and moved around some tasks * Edits to readme * Add a lot more documentation * Adjusted formatting * Added a comment * Made formatting cahnges * Slight language change * Changed name for testing * Changed name for testing * Update asgi.py * Added in alternate ChannelLayer * Rename chanells_inmemory.py to chanels_inmemory.py * Rename chanels_inmemory.py to channels_inmemory.py * Create asgi_inmemory * Rename asgi_inmemory to asgi_inmemory.py * Added in routing * Switching to instantiated class * Update channels_inmemory.py * Update channels_inmemory.py * Altered the fabric testing tasks * Update and rename asgi_inmemory.py to asgi_ipc.py * Update and rename channels_inmemory.py to channels_ipc.py * Updated to include asgi_ipc * Updated environment setup task * Spelling * Updated channel layer * Update asgi_ipc.py * Rename asgi_ipc.py to asgi_for_ipc.py * Update asgi_for_ipc.py * Trying something * Trying something else * Changed it back * changed back --- testproject/fabfile.py | 10 ++++++++-- testproject/requirements.txt | 3 ++- testproject/testproject/asgi.py | 2 +- testproject/testproject/asgi_for_ipc.py | 6 ++++++ testproject/testproject/settings/channels_ipc.py | 14 ++++++++++++++ .../settings/{channels.py => channels_redis.py} | 0 testproject/testproject/wsgi.py | 2 +- 7 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 testproject/testproject/asgi_for_ipc.py create mode 100644 testproject/testproject/settings/channels_ipc.py rename testproject/testproject/settings/{channels.py => channels_redis.py} (100%) diff --git a/testproject/fabfile.py b/testproject/fabfile.py index 6238405..3bab83d 100644 --- a/testproject/fabfile.py +++ b/testproject/fabfile.py @@ -13,7 +13,7 @@ def setup_redis(): def setup_channels(): sudo("apt-get update && apt-get install -y git python-dev python-setuptools python-pip") sudo("pip install -U pip") - sudo("pip install -U asgi_redis git+https://github.com/andrewgodwin/daphne.git@#egg=daphne") + sudo("pip install -U asgi_redis asgi_ipc git+https://github.com/andrewgodwin/daphne.git@#egg=daphne") sudo("rm -rf /srv/channels") sudo("git clone https://github.com/andrewgodwin/channels.git /srv/channels/") with cd("/srv/channels/"): @@ -43,7 +43,13 @@ def setup_load_tester(src="https://github.com/andrewgodwin/channels.git"): # Run current loadtesting setup # example usage: $ fab run_loadtest:http://127.0.0.1,rps=10 -i "id_rsa" -H ubuntu@example.com @task -def run_loadtest(host, t=90, rps=200): +def run_loadtest(host, t=90): + sudo("loadtest -c 10 -t {t} {h}".format(h=host, t=t)) + +# Run current loadtesting setup +# example usage: $ fab run_loadtest:http://127.0.0.1,rps=10 -i "id_rsa" -H ubuntu@example.com +@task +def run_loadtest_rps(host, t=90, rps=200): sudo("loadtest -c 10 --rps {rps} -t {t} {h}".format(h=host, t=t, rps=rps)) diff --git a/testproject/requirements.txt b/testproject/requirements.txt index 3bb2297..900d067 100644 --- a/testproject/requirements.txt +++ b/testproject/requirements.txt @@ -1,4 +1,5 @@ -asgi-redis==0.13.1 +asgi_redis==0.13.1 +asgi_ipc==1.1.0 asgiref==0.13.3 autobahn==0.14.1 channels==0.14.2 diff --git a/testproject/testproject/asgi.py b/testproject/testproject/asgi.py index ae1640f..73591ef 100644 --- a/testproject/testproject/asgi.py +++ b/testproject/testproject/asgi.py @@ -1,5 +1,5 @@ import os from channels.asgi import get_channel_layer -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings.channels") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings.channels_redis") channel_layer = get_channel_layer() diff --git a/testproject/testproject/asgi_for_ipc.py b/testproject/testproject/asgi_for_ipc.py new file mode 100644 index 0000000..78421fb --- /dev/null +++ b/testproject/testproject/asgi_for_ipc.py @@ -0,0 +1,6 @@ +import os +from channels.asgi import get_channel_layer +import asgi_ipc + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings.channels_ipc") +channel_layer = get_channel_layer() diff --git a/testproject/testproject/settings/channels_ipc.py b/testproject/testproject/settings/channels_ipc.py new file mode 100644 index 0000000..6098a12 --- /dev/null +++ b/testproject/testproject/settings/channels_ipc.py @@ -0,0 +1,14 @@ +# Settings for channels specifically +from testproject.settings.base import * + + +INSTALLED_APPS += ( + 'channels', +) + +CHANNEL_LAYERS = { + "default": { + "BACKEND": "asgi_ipc.IPCChannelLayer", + "ROUTING": "testproject.urls.channel_routing", + }, +} diff --git a/testproject/testproject/settings/channels.py b/testproject/testproject/settings/channels_redis.py similarity index 100% rename from testproject/testproject/settings/channels.py rename to testproject/testproject/settings/channels_redis.py diff --git a/testproject/testproject/wsgi.py b/testproject/testproject/wsgi.py index 9dcfd86..91daa15 100644 --- a/testproject/testproject/wsgi.py +++ b/testproject/testproject/wsgi.py @@ -11,6 +11,6 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings.channels") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings.channels_redis") application = get_wsgi_application() From a96c6fe9c3b782e2c450a6c0f3a46ec248ae18ba Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Wed, 7 Sep 2016 12:20:58 +0100 Subject: [PATCH 505/746] Added a `"` to close the line (#339) You may also want to make use of: ``` .. code:: python ``` instead of just `::`. Sphinx will then do the colour highlighting for you and may have helped catch this. Ooh, and also, it wasn't immediately obvious to me here that `include` is imported from `from channels.routing`. You may want to add that to the code sample. --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index b52f723..4bdda26 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -506,7 +506,7 @@ routing our chat from above:: ] chat_routing = [ - route("websocket.connect", chat_connect, path=r"^/(?P[a-zA-Z0-9_]+)/$), + route("websocket.connect", chat_connect, path=r"^/(?P[a-zA-Z0-9_]+)/$"), route("websocket.disconnect", chat_disconnect), ] From c72083db27ba135e1990c1d53cdd042e4dfa30c4 Mon Sep 17 00:00:00 2001 From: Robert Roskam Date: Fri, 9 Sep 2016 06:18:51 -0400 Subject: [PATCH 506/746] Added in reporting data and documentation for loadtesting (#342) * Starting reporting write up. * Added in charts * Added in images to report * Cleaned up comments * Added in clarifications about the testing * Added in clarification * Added date * Added in subdir with same content * Added in supervisor configs * updated the readme * Update and rename README.rst to README.md * Update README.md * Added in version info. * Changes to root info * Update README.md * Update README.md * Cleaned up presentation --- loadtesting/2016-09-06/README.rst | 94 ++++++++++++++++++ loadtesting/2016-09-06/channels-latency.PNG | Bin 0 -> 17031 bytes .../2016-09-06/channels-throughput.PNG | Bin 0 -> 24427 bytes loadtesting/README.md | 13 +++ loadtesting/channels-latency.PNG | Bin 0 -> 17031 bytes loadtesting/channels-throughput.PNG | Bin 0 -> 24427 bytes 6 files changed, 107 insertions(+) create mode 100644 loadtesting/2016-09-06/README.rst create mode 100644 loadtesting/2016-09-06/channels-latency.PNG create mode 100644 loadtesting/2016-09-06/channels-throughput.PNG create mode 100644 loadtesting/README.md create mode 100644 loadtesting/channels-latency.PNG create mode 100644 loadtesting/channels-throughput.PNG diff --git a/loadtesting/2016-09-06/README.rst b/loadtesting/2016-09-06/README.rst new file mode 100644 index 0000000..49c3127 --- /dev/null +++ b/loadtesting/2016-09-06/README.rst @@ -0,0 +1,94 @@ +Django Channels Load Testing Results for (2016-09-06) +=============== + +The goal of these tests is to see how channels performs with normal HTTP traffic under heavy load with a control. + +In order to control for variances, several measures were taken: + +- the same testing tool was used across all tests, `loadtest `_. +- all target machines were identical +- all target code variances were separated into appropriate files in the dir of /testproject in this repo +- all target config variances necessary to the different setups were controlled by supervisord so that human error was limited +- across different test types, the same target machines were used, using the same target code and the same target config +- several tests were run for each setup and test type + + + +Latency +~~~~~~~~~~~~ + +All target and sources machines were identical ec2 instances m3.2xlarge running Ubuntu 16.04. + +In order to ensure that the same number of requests were sent, the rps flag was set to 300. + + +.. image:: channels-latency.PNG + +Throughput +~~~~~~~~~~~~ + +The same source machine was used for all tests: ec2 instance m3.large running Ubuntu 16.04. +All target machines were identical ec2 instances m3.2xlarge running Ubuntu 16.04. + +For the following tests, loadtest was permitted to autothrottle so as to limit errors; this led to varied latency times. + +Gunicorn had a latency of 6 ms; daphne and Redis, 12 ms; daphne and IPC, 35 ms. + +.. image:: channels-throughput.PNG + + +Supervisor Configs +~~~~~~~~~~~~ + +**Gunicorn (19.6.0)** + +.. code-block:: bash + + [program:gunicorn] + command = gunicorn testproject.wsgi_no_channels -b 0.0.0.0:80 + directory = /srv/channels/testproject/ + user = root + + [group:django_http] + programs=gunicorn + priority=999 + + +**Redis (0.14.0) and Daphne (0.14.3)** + +.. code-block:: bash + + [program:daphne] + command = daphne -b 0.0.0.0 -p 80 testproject.asgi:channel_layer + directory = /srv/channels/testproject/ + user = root + + [program:worker] + command = python manage.py runworker + directory = /srv/channels/testproject/ + user = django-channels + + + [group:django_channels] + programs=daphne,worker + priority=999 + + +**IPC (1.1.0) and Daphne (0.14.3)** + +.. code-block:: bash + + [program:daphne] + command = daphne -b 0.0.0.0 -p 80 testproject.asgi_for_ipc:channel_layer + directory = /srv/channels/testproject/ + user = root + + [program:worker] + command = python manage.py runworker --settings=testproject.settings.channels_ipc + directory = /srv/channels/testproject/ + user = root + + + [group:django_channels] + programs=daphne,worker + priority=999 diff --git a/loadtesting/2016-09-06/channels-latency.PNG b/loadtesting/2016-09-06/channels-latency.PNG new file mode 100644 index 0000000000000000000000000000000000000000..a2f7c5af4c097d30c4802da919d7f9250a702f88 GIT binary patch literal 17031 zcmeIacT`hb`!2cwQ3PoU7P^~FQJRG!9Z{(wy@&>+C{2n~DIusJHi{ykD1_c2RB1vK z6hws3K_CJTothzd9O(2L~1A^#mnRbHTWH0$lf)6?$6KzeXxPxy7eA(f2+Tb(l_mg$G4+@+u^DQr9 zjkKEZjpV8Bc0c%%^{Ntn^fhL}mvnK^iGa(Pg%K4uM#$LqQIt*aLOt~(9Rz*bL&pTJ zZ2x1|+<2uMU&3FUx{W0zF>It@=yXT2^1vb!4&4tQ+ZZJ=$6PFP z)XivlsRkcve|EeH7P>x(M^VWrY*E$dY=6umAQdBiQ?+U#0EAZ4+_0ii-?}h zO0-!*#aqm;rxVhgFezK*@XhZnTWXYo;Vjp8_c$90ueOp`P>6*w1voXJVVR!_-y%@A zJTq(@v7gJGut{=mBMvdDQ)~h%gI=EgQZ}y#DGOs;MO2vX@GJKh8?A<+Ul0K+i0O44 zGGJw8g;if7r9txW-x0=w_bB7v=Ag-fWR3VG{!_|f1I?4t~ zVdRCurB~N)D2#aBp(^GubQZ(vnA@JLd^(9SQ{hM?Q+9RMa)w!_)d!Pm!9Ud^$TO-aG^w`fiy!&SuNE?>e(mjT#w|at z@Y&hf4i-*i$dazzSMRh%;rE*NLN3wiG-??UP?*OFA%x=&&DDr49AdR@ci)+)y_?@- z5X8E!Mrp1nfoU8^u8iG4x_=EeU<{5wfL=S7L6TfuZ=AO9ZYttbjF7KSknfLCS-QvG zc={$9v7U(dj$w^a^coMd6$pA-xJZ_;ES3zBISMb@&@*P=BIc;CV|B>qkHQ;{dFR^r z&h@vWZI+x4gvzWay1BY;!h;IQIcj%-nm0M0FEb>63L1$)Y&xKMC8RbluUOS1mNvL3 zhdH)Tua`TNx6~q1rD3&eq<#-${hKhxYf+N3xO@S(f#;zKwT4KEkpo+{R_b3DmswF+ zHq-Te`g6hbiwis06nK~OZ!eY1Wi|J>gW}} zacWO8AAB6jIkb81z)KzJdOse_(c^NWdNI zCmCU_-MGy!4`ntTxSIXOK0Fp?vC~RBLpl;J{wSAy_R^A^SNt-6WCl1V@P3PMZ`tVA;toj6|=udv3yACMvtLQru+aKL^8A1MrurDsxVJp$!&C zDn#zchIz8n<;JsQoU#sdy^|Z2WtAp$y|?M(AP?hiiTTCXUY8_R(CaVIM+$TOF7aYn zc4OEv5*Bt2PoM5N+`sgOY%{4zD7wo0AmLlE+t=KvC-T)pVfnVRHU5>(_NBMe{d01~ z$dW)~Qj~*;&Vm|TwOl-Ywx;y4x9`x0d!Wg)1)+|oF_h1k&?!O>=Ggkn-TOq>Fcm>7 zv&%NhuWic=(*k(>j8pqaIRugQAs2G!N?mq~3WeCSwVbOSqrO#*mbw0}X(R8ztgbil z+{7i~*6fQ>Q&a(_xFH`sqc#)B``xJ4V$^5+)UG5;kY}&Szp!hmLEmEr?T2S4F z$)nbDraP-4;ILIBX%7;{9m+OQ&$VBDWB#Pz8@U69aGmHu`Za&x8x-A-J(1`v`Ch7d z+G;&A{)Sav;=VBADf6)gIE#c=)Ls`&mm@Q{0|noj#l>Hh>XkQG9$6jbfmP#Df>y&SJ z%IGDuAh04wWPdSiZtJ@swM*Snd-qX|BW4nXT>CTwnB;HyySQWsXdUFa^BBUm~fCz{L$t6q5ZWo^U@ba4~+JZ4@Htpu2lFh zN3_t<7MOo_usWREc~D10!QW1VSKi1j;Oq$~;@IO+#{CvVW=o#->eV4P3jBbQcXsas z7$J|;KW9akGuV7oDrD*#>1fvIhUSC%fHFY~m#_N;;rnpqT=fKn@Fza zHWLg3U;*^z3YGoghzHx`IXjtJzK zfw-nN{?tC)9hce^g7g+W?Z$AZgFQ29ibdx2xL{7#9TP-+qIIdI1Efd#ymwULbMQJX zfKh|8=wpL;{Y6|DT-;0ayQagZ#y4jz(MyN5Fg>~9;Us;MuB=IB*Zv6m1-71L*(aG; zC6~!)NOyX`Kmq%$$?700b0d?{i9MMY;YTC6jh-ySlTPkdTDRUIBs9#VQDiij>l{&Z zzm@D}9a-y^6|}KXRVS<+y~cUVi|hQUJ)zy1x?BEhESKvJb7fW>@3C7<8d!Y8z`oY< zRB4aR>ip-g3a&xdy_Y<#^SFcMcBVSoHf^S;Q`a$SX8AG5P_j!#=ZUKyxu^#jZ!ZQN zh9Y9~kzX%))=wftllJWuRdHj;+aNA?FWA%vtw;HJr$lR6JwYB(G}PIb>W9?48y-5n z!fioR=JZb%!HtJMe(QMb0yA1+qH@K?ZigNz0$cKAb2H!9nT#cb;?M}VXmX}8K`=D2 za^rlx8y?feEdFFOrB9##bFPr>?6I5l9I@)OUCG!nw2~R`Tqav}P8+u)>3n{aKKXd- zzQcDPzjTC6pS^eb)RImXsqAS`sN7DWBmO(gPJPk)sK_jN=yD-5uTIo-xY1L|s9y03 zol?xx&A`ZUz66&)&ypfqKhGSk3^vuiloe2wM^tPwzH*vL(YhFB*E(Rv-gxhP*%M@4 zje3hcop=q4!o{ox+aL2e3Lj16^?cpMJ|<>4PFCvAI(Amr0EDtCgnQIZ7c=pq-sJ*0 zC=bER$FJ;9RPAZD#xd*RrS(?L+vJo^1THyJfn;@;p6nrCq`7lka<7T3opN9Ye?-x@ z9Yms2x5L&PdaIyo-B*?;V>}&`Iccc!JzS8n+o|*0C`alR$b;IENYQY`hc@NCfniS? zjc?e6OB!e+RabgVw^WpZs6>Ld4e512R$t7f2}J|LyC!m&ySh?J26m1}{&QP6*&GAk`LV*@#EO`n3Sc%0L}R zHwPES&}h3npI{Q+k74JY#kRZbf_OH5rQVVq5L6+$L+H-!j!(}|!ZubHuqQDpJzgju zK{)X-ucCJ*u<`1Gf&!m?qUXK7Hb?n5b-cVGW|%^3BdSQ5dL^B>`hXo^hE%`9(4iE> zR-l6ayp@9AY_Arbxcld_tHmHG#V<}Yc>sS>Y>Be*07Ha+2D#5=>J|m&?p}mug&=Eo z`q(=4n)Cx&MN7 zKohHiELj+>$fRK|$shL0eRxVtDH3a=%g4AAa^5XbVOAyy+#is83oF*_1&3P7Usau-;${jK|W}mT>2j+`#3p%`t)m>KX!Q6h-50 zsYAz)Vf>orc~^)0T@Bkd*Ow{5)0KjO-#F2QPk^?Em0jfRe-=V0CT{Rlih9p|r&fQQ_T)fN5ySc=X;EUF?om1-y{u4QnXDViC zYU(pu7B68*hJ3$$;6$HC6?LV<>ipM-<250A=xO^jTr&IxTwWDG}un2|c}>z^3s&VP@`I=Jj~3=fszKe*i=Mc+K6)!6LwJyKfwjX_cqA?=s_9J>@CNZ9cbqsaWP`#{Y1j<}Lr*u!p z5_a!Cv;z`iEH zB(Ywgvkx!@B0uJ=tlrQsHRjbD`P@Jmy<6!$UXWN2h41X@!Lt{z)!`lbYutlpu|w@G zENyW=WFHQ3`OD4ZO0-we(B_2rH3R2<;|}X=;c{8Ui*)C-uO0a?{oPP)J=3ixJ4*eu z*C{H@hauK^G-gyEsY@-ksUI4%)den1<8vYF9NArLq#qsrOir$ahcVfnggzCJqihk- zPc0Zm78m#hvn6}0ws?xGh?=EjqDgv|duD`84Ss4vaC0%Ku9nO3$?2Ry3S2C7W|d!U zG9c81LgQ$18nZM+m&{#@oY+2ctqf+v}M3jA)+0-%dqCKsl1%E0Q0?wZOL<+ zXvXt~WxuZL;a9!UY_PH5p>F8>nQX1jsYIxI<8^~G41%ue?|SM8?ozXcS9iS>Y-cnP z3c@GzN3?{~+aIQhGl5OK^{4_=Pu`LaNZ%@avGvv?pr@n{Uyha>Ft$@tl3Z8e=3oI# zg3&8q4PA>GI-PwPI4RX}95kad*VNKcjl;bJUg*bY*CL)ii&*n*5QJ@R#2Ko6uOfkb z0JEI2yrn|yJQKkZdWkrqFc#8d`>D6gp@l8rYm~zN3;EA7aWqZ<24Csh;?tbCou3{j zec{=$x5zsmz)!RYwiQS&B!4nEx2^chM~<_6cOR+l+I0}Ns+lE$ntRm2EL@9N`qml| z_lIc2Np*?vM~6;kPmFpx*QB804q4MVV{84k9(6M-1Ngf&r|2^!E9r*^;LwjyPqJw$ zYW59egFW7w?#l3(?afnDk>jQ7U%aPJ=QLPi7a@Uit9mM}lC(#owV?h| zH=RzJPF;@9%*536t7l>o16L+PtVd))o7kM@ zVI5?aY0iZRQUdT=ldmCShcba{)bOpYcG}E6IfL+ik8{qn z(`b5@ZaVS2UZ&2a$u<}!MU}ECSWhTGHOy+?XiOu#pMqR^XZtUCt_YtF`m{kn6NX;7 z&+TO68+0a4(-IQkmk*0vJ1ot}%YQolrFI4u20W`OK}Cc4A^hwvCbR}Efe#7$p5Ps>I?9FPnWoz@fDc5A?L7~lQn*=Y*? z%f{{Z|7CvL+9^f6V7dJGFM)JWPXuh2SnNl%=y5#}%bZaj+(9{>~7ix7} z0d9T49X3?yhTAWFd3ww;aoFf!ca|h^w6-L#SU0=ebk5`;Fz)38s;q5=klFn5{xiRp zejw&{6Mz=xOebrZW+sy`h9>@3epm$dML((GWV!{JZPpI<8aFxRE{ zPMBbG9wP6y+@m-4p*LE(5yZN1Tfd){KL{qKEXRd-EX~;?s}BC!!F&1{qCEGz7>z{~ zlj16XJ5<-qXt8wNA-;ZUp`ly4;A>0F#<1=Tr%)8zjO)q_^z`ZG>-T@F(`9CXB!}Z;74k< za*wZni!N>4?!p;gSllKf!vJfE8#g@F|j^^m`s0I1dsGtZ)JhIeKKO>g? zertuOSI)iX%#U8L(E6ppo#g9d4cM5yF1)H3hyq_<_li`>A z|J?+_tbSQFUFk0w{T`Lp$b}cnH*4UIz+XgUy!O4(MdY;64Jwv4$!{8$QW9PUvYEyR zxF1=yQF=TPzIz|I_pc!NS6Gwx7_mBar%cb%ZaWjanCM6S)*9FDvpKDFDd;6zgy6|* z=bP%sI=xkGBO5N!O=}WX@O6Y*K#XD#{C_XtT~^8W9}%UYSfXf7MeRw0UNAt_ahr-^ zHX+~7orGrO{FbDtcLb>B^Iv~MOdub_LDEM}-q z@y(#b0dKV8RQ-C3Jck8ln2}`xt-Z3^c90R#c1?G=iyMm{M$Z0`*=p2X>Yz>aJU#aTrzHeoBa6AUdL?+^GM)`HZ3W!F6nHUrt{M%lSI=)>rkdrbvF)^%Xp}&KT;( zpP&qzwI#~rh(B#`E>z8E>pPDe0e(>SNSnp7kph>Wcbx^~5vwJD4;Cw%6Anb?Ma_l)?GDKYSUg zv`4~Mp9hcM+O_#oC_*R-atU~LY!{Qp{_W_dM!BILyn5Y7p%1Vc=5ts6=-7ePyFOTw zR)PZVIU{r|OY0VRA@uDQx5;*t$=&^=?u|$F#7^~HI(A~6_+m9U*-JnOdQMx#|I^^~ zAC@`U+Fa1Eud?$X9g{|1nf$ZkEJPJi8Wz-B-(1E};#r3(T)s*K-XKir*)G&qg3ys` z-M{nXwOc7Iju8oZ7RQwsi0p{9GgV#_ng}|RK3ZUxSqGjo`mXj>_J0m4S^MCY^eH6( z+V2K8WN-Kd?t!ffsXobgBYWiy6!7KUZuC6x_FfsR0?T>4*p$%kp@vV~ z25W(q$s1#sn3ts2CXDn(=eGfI_k$ThiIJ|GmYzG48ZzEyk^}7L&qd zu%qY@%BtXM@_)XIOV$HT@fVct2C1m21$5NuIo`5_IfkX1G-s=-jWIy`=aF35=sCjj zueu*Qvy!})H5{<^^b+~~<7KU|=FOl2cJ0a*Om#W#C1RxKBsFBzEt6$O>a)U5lh!hE z74fr~I7s&dw2ki7Vn4l5K(*582&)D~Emn_f3TuF{bIgkOlViP#XJUbJV86OQ>8+{9 zho=TxTLb&&SAFhecZ^Nes#SKtD=BB~e8nj|t@L!_+nV~oIJPtVix=0ksW0LjFuy)? z50L(;JsP?BHOk}rm-of5UiE{5S4k@!aA{{kPb?D<)Qv_yA%lj|awo%9#hG9AUv zM5mnknoaSqn%i8nR-d`JFRdtcxjWdFARP=48cs@oKatqyv>y}>F#pt^wBtWFYsQEasL#HpsG-E( zIpN?mvh33!Kr!y@I=q;3!R_YQ`eK}lwoxfP9n2%t=6>|0+SQx44 zsV8>pZ(WaTuMd)dcKqdloVuE$c;TJ9hp#5HT06N~eCV1yb{n|j>S%st@=47B0N%>w z)d4%ECMh!GCjciJBmA9`S1vjsEDq{L9#XyF1n&=?k?bVzi4!xFew|XxM;#3&tfL(6 z@q|SSsLo=#t6lq!Rs|4j0!Y|^IbJa84HW$7_~V}KHvEy z+199o#~bU{zC4VgkKHK)vh9hgh4eXpm#{O5H9FmWL%Y}oF5kR-y!O?zV)pElDs3O* zB`9{81skAfoIYIkFUXI zH7&CTDqGG98was6W*ki4nR@JXmYo!A{(O^DIzo>5EJWYA_lH!YG+RpSnwZl>kMM_8 zEU+CoXRgzo^3tuiCi~d!ed5qetzB!2%M0Wv<0}uW(g0wAwCHr|2A`l508dI)w7;je z%NYO!e7vR2_qYOCzHF__pzt2iB?LK31Jwt*RIat1T}&~a>!H&bp0Dx2)dG|_or^>I{YS##BxG*aL z6sO`@i~*>GLa~#MnPSEU%@yGog&4)6pa_`rjG{(5-BU0Q60uc-T-zLUh^Yx&o~i+4 z{Td|*>NNQ|UUs_k1&<&uZGhD?&heCX`M>_le{{ysXSrAo&>+ibAdVm5n;5;n$M+er zYwUm$b74LX$}0s~B`O#M2ui~nrAFw5B>B5UW?Ed#a@_`saml-M^2y@I zvNWPL#ic}DNQrl=2U&NfiU3CsY!&l|frmw>ZcJy$a6VZ@qs*~&FA1Q56GIHCUw+;Z zHUB9^wXUlstifF=VEos**nDVxZH?pCA_icg8>l&rgU9?eR#hEVURv zE zK-A;n*X2QbxBV5crE_W8ks~uN84lE%NE*2pa#x#=&DAx+8z!q|;MkQsKaY)uIFI$a z8O1HttR6ibN>v_n18s%4$dAeeVjvIV`+TE|fg|>6Kl4aFh(Z1vY)EgKyY-A&U$F@I zwswg0Wh8lPl@~oW$HuR^aQFOrP+;k=XMra@`oa2E&Rl}_Ky`cF?8q1MNEXtOHStB2 z{*qz+DmUuhd!h|1xJHM<|9&R1{8e3jvF<3D3o1Tqut@>?(9qj%*GnabDC=qWJXNJTaq*0 z`1;*6xIHC)jJB1A7Kvi$#kgcD?khYBKfnhP*9hUpXAEWnKV-ov2hFHxD1C!Dg|o>r zS6u&m0&!49`_^H%LhhSo6~p>*$R1&3KE{3(3q$+;(~4!nWa$z>>aDS)(+6HDdJ~t$ znewf;*zpLL%OM=yQDk4-Gsd9k`Fm>gy=VktpvK2Z0Z#Ts7#x!hdmvfZBMyF~GCd zgW81JH#o6NTW!zGA1G^i?{7NlHFd?s9iZyfWxE>*I;QN4-Z)r{UR1pgBE^U3+`2bh zdu26GfT~HO-D0g!zVxHoD{pV-vz&dQ;JGTS+;6@B>O~9amMC5Y@^FDg(b6)~)eVD& zM9bQKn^_kQ%5te1nC&Z|Rl*BA4m@KU8&G@%LLqLb%98*b%2R!=pJghmYHWr!dlh`c zAHlm0cHuuJ^iUuy|2cgptHQ`Q`y2{|3XrKMJ_%lYyNhy<2KHct(RyCSR1a(|w^n$P zJ!l9ri%+1LZNJk8Q`(}y=cS8q+GlQp=A0JfO3z4RHun49644GkNO#*xV)0pgzz=1( z1NH8(4f!5HgGJ2CL0sN=!0`%t%kI(<)<_cj^Ap`iF*inm@}-Vh-R;cd@>Ae6_QCe7 z7$V4~T}@vLZZ*Wby|b&1hT8>aCZn<66OCk@ zMH)qe-kg*v2$E$g`E6;!u247AJ<*+np~?ad5V}3)LbY^a<@%Xbu1zNY)N-CMS?XjB zF2yHCjhs7NVt3{+3YwU^#z)4ZhSWc4*3Mge&G3i-mDr*xR?t!bIa8w_PAIM?h*_c*NPEX zV2^Y%W4h?@n+{J4Yt#x&gG7oKPgmCdn`X)oE!8~t zaDNRztMVf(j(2_!YCD7$$UO4bpB3cg-2jD&C1DL_)kkYWbHH7G`sMIBMW7?o)3vs> z9J3&C!B)8RY^tP6pk8+qcJ|;)0H@D!FIMq)3GWT_UE2twW7tHy3@v1N$%1^y|JGrj z--i7exBgQF_lMC^(oj;}SPfu`Z?wyNyZifu8B+mQvf%^G%o36|dMe#4Nq~A@!`jZ1 zH7!6;`P-Pkrq9#&Fm(dXnR4}5%FQpk)j7#Pgi|I5Kk)swl_73EVZJFs{!!aO;t2M;t%|NJH_dkK1lpF0!LUF^KZKbkVmf2|9{K>q+D`$gPq4sVUPq`d%{2iJXitv zV@pd*T2nBs3zm*yXZ0KkJ<2MmHUb!&T)W*%jlbFEj#N=l8@t1M!tvg4xl;`2@5`kI zjlK>H{oEQSh6OE)Sip+I#gjm<@(Wr+hrtK-e{8w+xB_}TDtPL-)8+s`*8cTDE8e^c z9>glkBI`*qi9m35bECL_``sj&G}<$bg%M>a)rdv?T1niyz@w2>Rdu)$YFa@0EB%-k6#mrw#yBUk(RP zKi&A;+a4vbT;Cuu(k&BR9i=?WgS9Pu(>!7S6MgJR-BWHMnl?%$h7C6mGV7?%WneKk zcrUjDlmUpoy2&oN&>H&gC01?tsUc$G)fwC+MtQb>%!H3oTO8+M?^{2OT` zPk%3;FeG=YFY#}ed%V?;ywj5uY=*FWO{o|%M;AvdP zlJTxXRD{P@ImFl1D8Vlsef*wQR&anX3q{+k(C@+PXho&J_SBi8ilW*_40oa9lb+x3Wo8a!d7xquoote9G^ zT<&Sv)Hv(7B66*=H*G+)w%$R&x!QdDr3v|~p$~!dY3%#<;>wd|2h#(vcy!LL({7&pD{Hjb9e=e;e`$F3KKeW->s(bUtSm);Kk>ufxOs4-!W~>9=w2796RyJ z`Z>5sseyQ=Y5`BEE00`N7%|;`a!Tp@X99Lgka}m>AJ>QPeHFz@vJeEQte~c>+p`q7 z8XBE{=vvWvFeHD{>Z9Q-&a}7{R+5=ufz0uOa~;V(+6P`QydteV2GYo#=$N$!sIXFJ zCs_(?Ijxc{-c%T``^oh_nYCbZI%ccr8ch}SlQ+GihRL>;HE}Qcn-c6F;S!-K`iVZ?lrDRk@;0eV?pYk~6{C*YZnQ$1Y%c=I#$hg#6Pz z&~udiWnVO=!}dB2(lBX!_+cUd#fFyB@Q`yoopTSGYKv&pO|};bL`$oBPmQY~1hRYT zw&wlSyUM$~3u4)SC>;xNbK6yPDlx=%6q!|haS-&r&|fGsLd6B60&)Vbe{b_AZc!;G z_s0P|5{E& z|Dmu~W2CRBfLl_Rdc`3cBQd}J89p{rtUAAH82Wng;J?n(DAyNKA`YDmSpD!;i*9=4 z`%vROqf$+^AJUcVTz=RN(3_?oqmWjZ5%UkFqE|%^(ug24OK8al@Z7$;Y;JxWyT--= zI`OtucK`JH`)goF(`xBCv~Q(x{pEOmNj0|^%IwA$mq)ULqSs5%GzZ+=-`ijv0Ogw< z7W_HX;Hk^B-AS+l+fft+EL4Q1;7hk4v(rC1O}z$e5%s)kA=S#h(?(5CJR@n5;mTL9 z?H$}U3wiKJ0?k2u*hOy?@lqwQJBm-Sn+6B^`ub|_?-Q#%Vwzp;gX@q}`SJz0_~ZK9 zw&MeFncpXLYm0uy9e2V8zEwl1I!G*H+4#P{ByK_h_H`m|d07TCUI zDHyAuk~YLU@SLjwO=_h@X0RC-=Hg}p=sTBQp zQ5>EdKm~v_&4Vkf)^E<@AgAjC0zd^^=Zwwb4_GUJMOQR)3PI2nVaWU*=-Jufygb#u zj^_?;{1&$?oLpN0sQJ+whhF$ z$J!$x7sa!02rBy*z5s|0(E(0?P3~kDm{;ha9RsmVYXxtn^-JNOfW#iPMQRi^S$N=7%6hWPB-_pe)y(RSzJD%=L718B`? z@lOgb*L266=L6Kf6s)#lF|n-UVdhY&LnKIenGjj9H;oZ2ltb@8`|zx%=U~ZOk~usB z7jKzAmK92!30RJw4a~brwPGN4?vm@TO%7PD!>kX|gN1o?{K6UnOR*=3Gd=Y+%5u8k1^~gVQ4>K zj6vHZS0$|{7}RL${O0=Cu`)c-0>`d|DT6bDXq0}Ubd-EEPi;SX$PxuRnP?kY!z<_d z6m3-KkCn7-7^(Ce8-)Q@f6BIREr|36kRw)ujHwe;P$Xz3#iv-> zgf1U59y+G0Jzf=Ce>8MDpMA25Ri|z)Z=ot|H@|?BRWLCF3F)Ho&SzvfaKtEJrO$GM zo9u?Fp9%U8aEx#^K>?_Mx7mOwGfnceFz!>UB{tUGrOJ74j%jym+|L} zj1sG*TSk$vTt!#^09jw@`Um5eQ>sKj_h9a);=YS)BqwZA4AaM?O!IN?X%hoAuX8TW zf#Yv(h2Bn!R^n0cc*%R*K3wpDwBK2i)D{C!)}ggcjA3kD4`gGdcqY5Pbqozzk0B`B zl0Q`fp@Du;Y=+U7VwqMv8Ucl;SEL7m0~!>sijLN_c(ka{&Y_iVP*7g((A_tUI+2Vz z!|SoW)Fy+a9sP)WWm#x7K{(@JofZj$K+$`}N2uaE>vZ7N_*qg!C^aHf0hX6yJ593P z9vs-9cO>Xp{J-E@@dmUrkI>K>ttt=trKddGW_oh4w2D}Q8~+N@}R80M2pSTLOWA+#|&v zLj#GyfN#Zuc6lB=9XOTCv$PdoUq`WCm{cDBQHD~W9fY;KXV2dE)^)VjDJa8NVqe^<|HvFonZt0^0 z{7Sc$&D%7yK98j{xP2$TmUcVFpmP>gv{gX=io`LUQTfn*_Hy5$v-V zfXFey=dJb83UfXmfP2BxI;F~h_(x(2BaWHRI|-2mWXlwMrp}|U)0%9MIVm|^sw>}9 zg7G&1?`v{6GoJh?D0H^Qk91XsO5&PzYbg;0rb@u^o8!b#=Og!z*TKT)yc;Wb-I4t z_#hlp4Qz*6JM=#NapqzQvj%yDWxwPiuM6XzE;s5XPF?1DAvh;F*<_qIz~$a)dvk&@ zVCvH$TFVqTXhT}($im(AGM_)LE44nFIT$=0%Gh%zl5HL5oS4*zjR@UIe(n#j`j|zh zd1{~29;7hw1}OhG#E=93T3mKR9@S*t2Xa=V=jfy#(wLE1241G0c0?KPq`vWdOe!1Q z+p~O|o??ywJ;-F%kW|(*d189hD=q2n3cK?iF~*efUFwU%2Zm0~`c>XyPt)LqxsP*u zcN6kB<7njB;{#nA5;yOs!Jl~ZONmUhJIi#;Dqf5wotgvG9 z0L}(Ufj-is8obKxA?7{V2FrAshtFiNgL9|mpgNY#Y85mPskO7hu3sy7ZE?Z_lrUKJ z8?swm!?l*R&Zb=%slI_lvl3krlQm=H%Yra~R$4<`ZBLTh3Rb-F zb{=^*<; zEoE8Hvvy|@3vLX#k5m72-|UVHmrF|Jyj}9}EUG|JEGamBAX&*TAE{B$;-V1iAL#NK zobKKn;bCMn*61l#s$Zq`id@d-5Z9}p0;S(gX=KTHVZJ1*BA6|?q3@x6jIQep$zoo$ zLZ3kEk51F9?_-LYqD(7hfytjf%=aKPny1V3rtYfo+p}~cA*jh!b;|3^Z+}a6zfpCU z^L^3LokNSRHl4Jva>po1{)%AeTAI3^&OST26REo8={4%DU5_+))!pCpB*@s}#R#q$ z`2&5%+7o`rpxHdlweAxOqg@o{Q$vGua_3^Ya^s^fbO*Z$^CrXx8MqH}r-QRFb;ybd z%T$-KhI;ltYTZ>u^RmV+m15Ee=d`U|GS7{@Aitc~E=8UW+FRSewwyt~C_Y@Z zvN_@*t1R$ldyKx`g)r$AMW})gkiCbZ#Q(%4I6l_ue!P!#$EUKuXtKy n;V=+5(e!`ze@%GgLuEgC>P)_9g!vf={Lwvk;cT(ymD~RdDRolz literal 0 HcmV?d00001 diff --git a/loadtesting/2016-09-06/channels-throughput.PNG b/loadtesting/2016-09-06/channels-throughput.PNG new file mode 100644 index 0000000000000000000000000000000000000000..aedd24d51a9970026fbc1ad05f1dd60a7a8155c6 GIT binary patch literal 24427 zcmdqJ2{e>%7&n|!iqK*yYkNs(A;J)fN|J0bwo)0UA-l#hB2+>vW#6~KSY`&JAqj0V zmO*0+*~XU4*v9bQBTD~z-}jvFobNl|`#R_USIsleb3gZe-PiTIe%J51?_a#2#myg*{s0}qRlcF#wKhCU0-xrs*ovdsN-PrLbd z`ZK$qTbk(J%j-^hnQ>%->#gI)bsHYu<9Qcv#?JouarxJPsFNqRs~x_|F89z%^ZV+$ z`#S^@ySnemKN1_ou0G1W{)5ir;CI&{KH)ZUCGKO+v$Vl6mHy$;DYTvO)E9ss)3MRQ zGA6>FcNNx$9nrWO*2>v+29rHSx+L@JySjf|<=qWi+h!*EowdsJ55R&(1k0vRAH0$s zodt*1(1Pm_h3$PPHxnL8*Fpris(*6T$ssM0<}NwB9bb>yK6Vj5w}z~W_|*?oDu`1Z zL!YBh^n8wmL@7C`mBxi3eLFpRxTpG2^ZlbT^W8FSf!xAQTKVq@c=8$50vx=VfSRjD z(Nx_LwrfY6cEHxag4a{d7@Q|r6x`5ZUJNW_wwAfpsxG|Z_1&U=1!F;MySI6FTio3H zxM|giI2CUS$31>YqoN^9SqF1md?Ak)WnCTW8i$%MMrGZt*784H`g*RML%b-*ykxfj zQf(*Hfk@tQQ&T)GcV-|dY_@ssW7(XxymQ~D+5Rve{8}!7qS5veM)h2U>YV>VR>G75 zQq^1h?v4aZf=`_eW1`MxfsFC#?R+|lZf^86%%;s71WixuW!66$Bc8p)q`g#``>xTP zq{VBgN-U*1&rLg`l+!aZY%hjcF3dJ_TkP+w-8a~u-roctz|kU+2x1Uy?3IPgtF&9< zBTvoa+5Dh)N8Hp-mpoMM#D&%zgs#hIS>RKrf8 zRFJhuBKQpDhu_a1a*MKh~0$W3t*;w-tR*(ijaJVRB2SOow-bujaPY`_c8ea-PXx$@02UT;5Ws-u?ZG$PR&i~s2pIvnc%23lmR?5g>^va;Vd_AURsd~14B1k9h$h>o-r55d@LqdEER^lCMb}((;r*1-J{g?(c zAwIj=jyku-_t0+GZarAb#9D-?=l60BZ|n3P?^)V7gD-WjY`xtn<80Gj&)HB>r9QV2 zO0Tfz+GnISITpCX>kJ`5oQEpeM_{_v3-e6oMEd^o))6=v*}l4JZc)X?!R`}~{&c;h zxzdtR!pESHH~(qDwrjV*%nhIel@CipI=R z1|#m(M9DK4fsv=y=$O%t?LlyQBO%^A-YR5cRW7;@)zwg#ZbkLU%z{i~Y$2(J0~^vfZ$=AKxuA7@cYuhOK=2}SKC?u>q;ca16#quR~P8D$ztV>HU4 zbljEX&9xy;YIbbrS=vm0O;mpzs_RX_JRYAGi~O)K7wbJ~f>LUf8SEGIam)0lCOG$% zEii!BV!omU2g>qeHHdGRZ)y`?Oz^JC4~-8wm2f32=q<_D0y&ss--<^R6GMbsNoOQ| zCBL)JZpBVkukoL!<*YN_c{&8);YITqKOEoB&4!x!(L1lQ#o=307#C!83o7Kh0v#!N zU@h|Ds9*O?jdIdZ+E%+71;FmuK5|4&B=GCgkx3Ddg!#vlyVCn|3sYl ztfWG^r(%7H7uCYILt|S#f;sHx!TrQ9@-1BCjcvUf+0eKc80CzW@cg-iF3yEn%mP&- z#%DX{6Fcaqp6&kdk>a?Eg)G_mzA2^ZKTsOr0 z%AtBo95v_Y@15XtR<-PVf&O>0rQx1IpYaKc?*5>=jKSL}E2m`L_Oo^;_ja^Ytufj( zsU>`UL#?od%;5OArbe;Ej4KmJkJ*snEyv$pi z8^$1}tyjeao1Cj;PhH(K&RHt2rXw7x%n_V)0@I{ci zsUz(z%I%3u*~PSh+b?M+dgvRc>PnNPX1!KDGGs^W;pFzjI)*2&Gw2RMT-q%qHCWel zm?wmr>_aRgq0?I{Ct*Ric8oW_oBHx#u+Hcmmxhu0&xJdO)>4}7;A5}&9x>q2OG?^C zbcm)V&JN_&i6NMB8h4H>AGZuy=uw@o7~_)Tflw;%9-zNM2SG9Dt{q6TX|A&g6al1} znKM;<6dgIZ+ESD&LGMRQtKM1FxT^z1IeOa@gNUn>dzo`j=DGK^%gj4iF0MjAB){`K z+sUTQ*vAKChwG|L)YphQi1aKDzN?nONe-xj1)WG*m>*l{AU8G5eocBQ&36xLIfi+5 zgtAlQ%p4@JHq{%+1Bvj%Rk^|~Jsf5yO6E6WrhddN%*6S`-B8&e7#SdRx4*$BL`PTX z%_io?aBFOjYFsw7unBd%hA)XQG?8IXNjuVy7<?(ewkf6zcX1y4i z!bn5bW@Ju`l6w>D)dl%8_nF?(j57>=F`&YU@9NKLt#fSK`S#$z!SDOX_!rfca%uBh zEHt36s=C^mdSt^*YLmDw8fw=xJ|2s}UcHM7BX7f2czBZ?fFHZRux{_1c*0Hv#RaUd z&XIocRoAww3Ex+*5JN=ol_k}!GkCyF;XG~!)36G)uQSn7hu!boe>35+ZlvqCYSsBX zagS{`*5+sSNbF%UK5DDwXkEFY|Icg<67dc%HvBhsBANYjaDYb!u47G6< zd@b?z!1@Q$28M?pmge9e@79|Ay1Vj|;uh|#v(WTByEMhOVMCC+CgwkCRwwH0w20fA zfu}S#_B`Q(qVC5=bP4*O?9vld*axXaF`H2{jBkt?*}Zd~Syl0(KDX;jrv;eNb>lOH zvcL&G*2;#jTv^1)l$LlY>LA=s3=tq4phkxcsSw7V?9~?9D7s;)L8O@w*EeCL8!7D@ zTw0xcYsZWq9v@-pQ|!|*F#W1L@phO4k#4lm!KWVQGi$3Vd4E*gF8+JQc5&K%7h>W4 zt^50khjrs1oywopwEgg>G;&m)TIkdDTkg*vO1%(6fxQS&DHBU#z!UbzNw zv#o>6W$ym|*z|?UQ~kZ&mBB-CX{6e;N8iVe40h@p054S5%CdH0iQ}*#!HrDUgL8*y zW9q20_fQwO`|r<3)ycfCj72T(*=wfjr)Qm=rwdRoq%SaN{j*mCzHjE{IHQ@pY86aK zP57nJKy8z$@+nAM%eU$OgJ0UTIxX?yu8N?N-CCjFCKcQVNIT{28V|6d+$@G5`i!v+BrFGm z3l1O8^cSCd4ctL!$l_1-Tkd_+B?=?4GJvuvtnX+s>=;wC7TMa6>lX~@(`LhG#T!J5OXYsKkJY{+C>1hxe{px9U!}rtE@k#cB)iQX-8UKzisC zN$jZ(laG9WiLO4UM#5|LSOp`&ttQhmAa;>r0kD9e&k@wa-4KZkP8 z+-zTnbgl%N6DcEqgzb4)vxj2`m*)KmSe;8fU1dr6@%1c@#A z_C&iaSf=#F6g_jZny{E!t5}G?6xDWd1}05Cb>hrk)k1LJ8X61tnEGPi%{lfHeKp09 z_uY3+h+&mn2EMXjn`dR=%`e@GZo>n1c6LI_7f{~Dk7DH=ESOB$Yu9UIA>5(f0J5w2 z4mpB6ItS$wFz8S^Q-$a(g^Udu?)E#b?1zU4nL!&H#N=>VIwQEr$a$;pg#4Rl`LO&i zaGGZhyHX8MYS&uSUr6&|$!kR4Am=W7M2Esr=H@YFmIiY#j9!%j{L=nWrmSsXhKpz+ zw+mJQa>AtYUnXJ!$7CR}AM^G~$hz#Yw34R3u!on0R#%3_DBrFR*>>cOlvz=3-9)v6 zmL*1LGq!6pcBJkrB{6n!>3~6`YV{<^Jyx%;e3Wu`P`%o;K1qk9*~hv{p=HHYl8VI6 zx+E>{>5f8&IAK+vS0z6_NVek+^>IDr`+CYGOys7X?&5|6>-6|KJHq3?yXM`|rR7{Pc70ap{Eum+ zOXNR{=MIZUgjJOb(6M0ESvy(QJL)*XslcXoGPsVtwa~$R&AN@>Ck!}nIGm58a1AaS zqvSsNN!0dyFzNiuqm|}fq}W={7PGFuY^oI!A*#z~_8 zG57}8AK%W(UwOk`2)0t2SpOQVKtP@P;hivXg z0?9i8tPVj6iZ8klqd3#?T>)ZLagLSgupCgCl*OwS0LP@=!?71Zm=+8}k7`V-8>x#oqlPzqCLFyV>r)txh z>)|)J|29cmB;Spwh9uFHU_QBLr>26@X!L}}rEb9s`$GDgyj{kvi*6nDIg(6D3i-rx zE-QD9PFNL;S44qS8zw3UqCtnQpn`o%ZaIgAPPzQGcbinbEu%xjgynO}4Sa=ai9XEP zHi-0k!E1J=Wth_0%sQISn{9|||J@?CNcDF*k%=-HT1pS=nx^6A!$t067GMX!r#HgZF#!IQh3#Pf6FI?HPSO4kC7tJqV1jg$mSEsJX+4(6c zk3z6P6Zs2Ni-_v2%GTd{LH@FRfp#EpTi#Blw3S7Cj!`Nhqm4@@`fC}Cpyq}=&oQF>I6V?s z`~2j8$46|kXsB=&bJFsG$kC_rvqidTkkm8)=l3;Llst=`|6;`0-I@C{7t2FgQKL4@RT@b2R?&Wfa63jpzcy^ZFM~y#wtaM0(E8^qW_Qih9Gc zGcv|IWq|+SXile4Oj#Jq*x0z=0YxVp+kgLBUc*D7;yPa4E-1^!0E)Yt!ozW;&e<}F2w2EJ57x70K}N{r&)66tVA>9B=b%54jWPfb4GX*nOMeF_;W-4|z)3`RXa z@FDDG0n(TMBEOql);ADUzgHZ_S{_5OgY0&nPGn=Z7~)HbJBGIv2_fcKSdQtvdAKJs z;gsdrtBKE=7(gB8u>WJzPz3`+55yFuy7d2#MT~1jO-N!m&N;p>k z^V*Cl3l!5;we8d~e>3-XSD$G2%VwIg&om~kRoiUd5OrgVE!RKd!T+lm#o0()(Ivgy zi%e*L2Ff2LY-`rZUfu;EIorh|q+Z7AmNqvg$|21X*4ntR^?`E8h#;kI?u&N#A=_Je zH&Xr;y0y{L7hW7L*R4srm_-&$(~0W45UtQvMs>R^yYIfh-8#P`rn?{#Fu2=hse4al zVDj`x06Bi3HTpnM76T~cP3dtRrETGMn05cMrc2)Ze*8DdPutMOGRFG)JzOj^UI8%n?5GvI+?#| zqNXH!RYw^LHT8x|rU($I9qnN6l}41BZu%RArfThUCB>_8tJ}L|x#mIMo>hbPpvOwa zR2GDE){dlFIFGg!+c#yI6hlL7&F}){+vV(@DLH*RWAEU)OW@EgqUV$wugY}1c;b&QHgPk#-d|te$t*hXk|Eku4fV zT=%!%y&YGCYYpq2ycYE4CRPh-$#3SWmMH6b4VHYwh!G562wNXS{&=#hR`{i-r>Nlcfx%%I?gWFrZ$!p16*rW7&o7em53~Dx?70l z#K0IK7s4?tzK~x9DfWnKbx*(I02jFXNce{6X8NA}QesIK)^ssRgKD|`u>BCQN1mnL z5H%_DA@Vj;kCcH)Y5VY8(%S)lLtKy7eR-4o&craLc7$&+8aYD)At9?L^S0VJPrchm zHoj6KghV|}Z#L~x>QttXL*o(GMku%(P&f<010G3ve41On?W@~xOIO&Syu8j?=S2>r zuJhYDNRt^Whbj7%`J-KQde}-nria*PD$E$)PKswL9E(iO_kh8S`FnbsO6rgGNlO3h&PUR%6M zrSW9jrJAVUsEsvuB~l`Tp@3?Kl z^w_m_%(176tlK7*(zV-EQ5Jy64r%&j3079OV(~&Fsp?gOGEH*pTM8ksAW!ut69}8I zA+`g7^Pdvvlm5AHGmz0GE`!~ePv-cWFD>)VlwbCrt9X(CpgzTtns;f_ABnR}P2@o< zIM48=oDD6xnxx}+0AS@kzJFbJ$CxoYIWpAYHKC(gn}mGjP~0n^I-Jya*nHr!1Rs_j zw_TX^1BG|`FlK{B1si1U!Vv`KY$os4G0Wj~Z#RQ6dEBeAfft=oNoe*26T5l5Ai zNqLt@NrXonW9C@vt8zk^YzD0C z8R39V!cW?0#XTJ}0R-hTY%3uviVaU4laH4GH034n!~dEJ(v!QtL)?mqLm3ek$6 zvC8^fdFv*CyedWT)%Vajgb%vwH>={We-!B4oMDY5CTIi^Kn6Txjl2+W+q-7h5-#{n z6v-JjB>QGd%&u?Ik=zI4;3NuNN*oO4f50D(lTZ$-MmOL7@JvGGdG#saB zLuo9;lfYz5F|4~zXpAA7mcq<$2P0#tQz6na?pq_$Qu$F3TLx;=@&HuMc8Sa)S*UT9 zG7B|6(IAC}Rag|=O5hzBw;qp=8GC^rICS7jaNzu*zfd9A?Pr5O%}r#9vJY?h#zg@n z*yI&cjT5q`PHCK$rD@o~;mm2ffNO{=?}FZXVzoC=#>GammbjVU;L;*J!_P(8s<0QB z@t(B^^Ly#r8$q}p3JN{|bEh`GCoar0=JD!SX)A-TMi-kd51oCwuXyYJ3)7B`rfi`V1GZ#;z* ze|$PtJN;6v@gvb={Hp4j_Ye3by)K4+7E2r<8(Y#Lttsmebq^?$t~I#!>H!e^@P4FVAeYdI4W z9$@k_L)GF~-kX{V|TByTf#1L&%49_X9 zl4*_wBy70rc{RetFt=x^oUedWV%Cr41B%Lhs@=T=l=5d=mz~0LVnMOf>2bD#=5)Wi z4QTHn4l$3{GHKG|xAUSR_d_)_dBl_lJhPwrO=`;!zj-F}=Lc_~r0K@F0czQKs?9lU z%$AqI*HrOi_^P$!-wJv_JyWbYssI3em#jh(@RBSS}bXBq>@qj6*GVXNm^q!Y{ zuz>>PBRN`OVxvK-a|xb3A@V=lqq84gefF7z{@m#ocNTFWz2a>C-9IrPt|~G-|J}pA zpBrcgPHK5#>4t);)Lmz6saN0a5W@;en+>EdGUJr1lTYwmNGVJ)#m1HU|H4=aA z@pkR#w86UR!8>a>X1j#8NQX9JeJs7{hI$Q*v&yh)erjC@9uYMrXPn+l^N}>pp=oFV zi1fIG!x~%1$qM9495AuzqJHqw`=|`vOP^A-Ef=DnLTf21x+%ggkU~|%0w2H*xnIsr z1@O=wxSIJFhlkdiu3bT5NAyOTg2;A%Le@MJ^Tth|*z`Oxz=d!6rIzqZ%@wRr{^bVQ zpNk+(KOFRYC%SHKX2N9qmM<%*qkMw~SrFpDN~zRrIqEq%oH;~pGHVlZa+;eSZ;KuN z57TIpZhKgDd$Ya6!q}sD=+IZ?|FZ5xr)I3_;v4=oVdV8f^I=?PnfD(vn7hSV0tER>Z9VH$#sO0 zPC@?93j^ZYH~mTF73A2K8j7@T`TQ4wUyyEFzv~#{2xhPl+d2^zeDHlsha5TA@?# zIrqVo2s{X<1Eg|LJ~1C{$+Kojaa1m1ObSxV;e*~czvNeP%mD)GTWkj5XNcN-^=DDU zLPCywerJPa7Mw4);MKc_DMqMR6*_`Xvf9XZ^>C>-qiv#Ab&ORd^LNnE30d$N@2*t2 zsRhIn1ThG*h(J(8F+kDM$)AJQUXN5t%|y3>T)00hqYw#Tj6J|ftlC@7BGdI;Eg&ugGnb-lyfw&U5ac)VjNG?A0TgH>VTW1dg=y?!!Y z#h^mS*eR@7>cd1XD6G92&qLHIQ7Xr-Mo61`@glt*;#a!^&1G9vh_G|5LHh?GEO`VY zUJsavQ86G=QXSntY%Jq$v~Y+dw9+9aN8(eKMi3&qK8Z{A+Nt96#3oKMCr-%7sgmD# zI$<|OT-1A0jO!`3;uP?fXHfuAQ*A_V13q)eM&SNuMJMnlc?F}94=OI-fS!|9b|3bs z7P#uUc^19^d{O(tiPg_nmXQ+yWw-;jzS?lEpz1ecALPwe~unVwq%`=W2 zb%)YOS&q$$vDko&xx4Z0tA+Mpy5UyL{c&F1)v#j%C9}UL()MG(=^K>Uql32Hzy&@} z(PAWpsr<0cm$R~nTGt5oCFX1+w*_-{00F~I*`10XSa!BlFVv2}KClLfZd(~->dL5C z?wi{W4oMD?Hm@C28hh2SP~7!?f1C@5L=(mYs_uG1q=9}>{L$Y=gq`^z^;C?7wI%@q z%o-NdsIpZKyCJMsdwkKFm!1-wmNCt_cG#CC2zZyD&w(&s^?u7RdBr%QQhnQ_uh(a= zD*P%s1#xpe)>k;2Z& z2{_!sLCWRD^Ru$R`b_+Y$lq0ETOZ)$3JaV0=Z7R&rSXQ8&1<*rI}bZ77kBjK4Ys=a zen5O(9YxQj8SZDXg=x2&jN5t2ed-cV+Yi>K%ro|Y*f?&YcgG?sRp8TyC&Sl4#kuq9 zWk2G|MX;V(jh__sUlcuD@+K#VsE@6n16!|=q$fdPX_SnPKm(G#P~M?qSWspok@6t` zKrQ!m~?{#j#CpI#q*@l(5|Lt4-7WdJ9ZKESqm}a zO{kh7LcO6PC!VxInIC+P>{&Tl3uun*rPuh&`(h?o%91cY`)yA)8;`~_DR9ip`EBBw zSr;fkRXxpP>e^(NHE9#)-B*EEyoqKezRTwrIl%vPZwq?Oa`g=()#P%$rp{os+w0pb z8Nq%r-=r*mw}gbtra{KOLL8SYY@`Lh0QPVaZ~)%>f(tfMx)QZ@hSIK7j0294#SAK?cZ{(rQJ}UwGsq5XZBQDp2~$8g zbKE1;&VUs%PQFMs>kN9SNeerJ;=H#Y;Q!;7SHks0hAXik#AG@sb*>8i40zLu0_3yj ztxps+gNrfwmi2yeTLSOyffQz0mURTd%{Sg=i5i|u8qapztWil68?`e6&oDVHTXL_D z6uNF?U@Cp%ve8+<(HVhvU@vLsfL$!EWCRtRYyLj=3f)xFt3OIh>Ej0w847~ioIXm8 z`tz!MKW$jlMDvMEe?|HcWfIS&1ahd$W$m{rxa~oOYW{_fbHg*)W(&{k1tF%6^h&u7 zTuP3c4ce2V@w(~gVDCl%7Fpgvnyyl99C)bkCvkZ6;|DP;UAV!y7SbPKQ8wKjYz{(D zo1w9JFQBZSmT0*VL5a}EHKo8t*upHS)ihq~V=Wx1;Y~)2==S{N=@!)&CK%gN-|P&= z!GOh=ic?eunA`lZ1wv;kYxo)cbCVZ%(3+6yx@S^CVa~&sd*}7B$?@pU58aJ$RZxpA zWCdeSP?#DGRT@nfKW^>)Tl+-JYBC6ZZvcJG!&-a&YbLDNlKU7)5ecV*1!mA@JrjFx zkXh_xkz>!CVVhDeRC+ z$=FaWclT7!OOW!7rg^>3%zc=Fd@d)VY1}Cd#eb4{oAez$K5*sz#lK%Y18hwD_lx0T zQGmiPthfc%@-F%8(>5SodpI!16ucmPc7gl^EzW> zNSt7KzHCztZg6r4t#baAsxUM~en$=vhBa1X-~R}0P^J(L==n4HKfUa!aFEbAc@8kT zNbJaX9IK>=F@@lCn}UNNa84BaBc$e0K0|da-QsDZxYM<`OlBe$tJL>H(z3v)qt_n{ zwG|gE)SR`(7L*=MZW=KT%MV{imf?-e$pv+`(jb!rs5OZ$oS2HauiId=(g^G_GK?J0 zbGAg5%D{j606;g@J@b_B@URZH(;Y@_r&gq3mY!BE?ukqDF} z(HaIT=}l~AVU@Au9yphV%ASrS^K+!oDVUA9i9g|cY-czPS~pd;;9Ur-OZ;S^NBIr$55XfHUefSe zmhE=uSIr=p+3nuvKBaD|hUkQ!CIIGdJ#$G|Kv}c0$qf9^Tl}o7`V~ z3CNjnhyX;5Vj_Gd@z|4VI$!+V;?I&?uY3dev`1xOZlcY?fO8op)F!634qe!{Zuk(O zw-${x+hY9n!z$p)!$eMWi+OxyTM$0S>J^>Y$U4RXj7koU zZ^!1ZL%1?jr@#7-=2eSLSbL7Xnz%3p;PCX!_M{&Fi=bidr_z-ODg{Ez(`!mxPtlQt zlbW`PfH8~R^j5w4cKw3k!oB=bcF(3?*dtQEp$QmN|G=Fos__tz)d(ivqr zITEHGw_f|Q-PiD?N1KvTNn5Bu$OA|L_n5e9XmG=DTXDl!^Hosb57$yo7Q@Qf7VlGO z-qUXyUI6^-JR?kH_JDiXnDmpzM_X3A{{~qdrUf}V(#wxFH*LQ5=I#ccFRlSsv*#3m zn6OhK6GO;m&K%%__idI^cnTD+37L8u|4H+ry!SQw1)7-iU%JFgA*{NjoikLpMen;& zoor29_)Me22_f}FiTU-E=Nii{(ROHgYIOh{+9ZN3z*U_EQBpOvZarxCdf$pBF6k4e zU-ke#)hQ-a;*21*F0|g@4)Fg;mb2!QW1VA|9wj6#y(~s9g9E`2QAxV@z^y??eo=EB z0(=Rxix*`zL+4!9kU>Mi`cy9hV686G;PrMVT<_Ngw>K((5jyGw8m~IxF??7#Q?(d0 z=a))$1h*tm567}fYF@gYEor&2WV+G8Em%6yZxR%TH?$bCDxWN*amc*XmsJyVp7Q1Y z8DOY;#bjX@{jEer;n^4h5D;b5G_z*uAJr24MKW$tfTZ@`2qmBpIEFJ!TUMB_eTU<~PIe zu-;oWX9P#eeL!Wo4MezO#kp5Z03SW2%or~434aVq>mSB;!;A%&Tyud32#h^!O7TnT z@K}%Brjb6-ZNXBeAGbSmZhckF1`6k3q_T$Km32s;-m1gVd$hqaAN=tqfQu#UZfBJ+ z)9j0f^FNw_46}{ZT*mmG&9!g4`X!U4-DK0y&BmxMHWXR$BtUM%!tkY;&IW>N=kx_N z){Vz!x23%j?0m4{B(&c~ z%eRdP=ugkn8g!4%M4f{#(;UZBuh(mDXklBP@V7QVHej(|>T*(6s@UK_%V7KoHKloF zp}P3Iwkze4tZE*8sF`7%c=KJ>?stnqazMgQtX|9rYkuE|RqANfp_>{%5UAJtY9ofY zCbYE5FGX^p`R5gv9_W%0bRuh`Nl0I9TCq(8)@PY{MUc@{h4O z2H)4+^yuk-bdcG2BVFz_x9#4iWdO|P406attolnULpcWli9$fgPOHwz?`{o%!VO*Q z5m-Sm8^JL)iy*)L%?aStSmeWRU3E4t_Bo4?PG9U)Si%0{@JKtHrc|)l8`l$S++P3V zn<=qNe1b)|0(W%MqPTqp!Tg+#?Bi;oSW5ikBQnzNz!@B%LjHcCd?Vep!{rTd^icrz z|Gw#SWYrq{jjgwx##K~H&fvO!&f*GAIol3n6AlvOzXwb;+C&$-TIy|L$bq(*Vlmw!y<^BfyE=h9RHM31y%wvGl1#@>q4 z5$mJQS`3B~2|L``i$G65tGKeR*=R2{*BQ6&8) zeA!k_$h1#ss9V*>GOj=XinB`ZXlE`sGscKtL}Mp5B?w6^Npk(+k|9XD6Eqv=X}UB=g1JyFLU z53s8-(Mi`7_*LGwy8z+O<r;TY?vy9r-y3!(RTMSxJC^KW_PE7!V)2?8xgpWLXaz-$J)Uj+axfS)C zbD8~5+<7+VLcz|lfTA;yu~9a4^6 zC|J}uJD5Ma%Fuabp(kV6tUcq4!F=MvBg2p8C}#J5Sb0;oMY`6#E7BJc1fFdKw}a$R zwyY>rC0v)?_^nu8HHky%GtY704&dA7c1iI^zRv>x>T_@2jISZ?@UM2;avht%8T{D${?8{Gx zz?|+0D`B;Vu(TE|=$0ZTxt0}NG&ES=7FV`y@&3(ik(tfKrhVufqs4ll+d05LcH4u# z$+>p-X%yq$J4?Vk_{r&{!hll+q9EahQd9IE3F^Dz-M3F8$Nqww@?~jK!BE9 zN7V<~Q|xM1p>mG9At3a*x*BvWDkQrRQQqx3+P_trX5l_vsVG1H9ygjT_kPO8`XhC+ z3yN+F6zn5NJ=H22@y>A)w`AJfPe3w4RBu61x1Y8udp~3_QN$P1W;e6pYr5A{s%X4A zJ7+=f+{Y2y^MZ${Z{D47?whtU`|$mdI*?<9;feSkarT#ZC!YpS&CKLU)!qA&lv=r= zz?V;G^_D6_huMfO#OT#ZtmEQ#4CKpU(1(mscdn}Jgbj6{F@Kf5b1&hAUbmofxYHu9 zmQ8fL{NwY^W)P?v*SMXuhKEHS7qB!xB4Gd$;*;GynXJkIYr_d=fOZqe2XKCay?yQ4 zD0$Eg;5OPeSbZIo9;@i1?X6QPk}kS&D($+p?x4!8sXzUE=?OT;dmO{kNyiD5aGn%_ z+%-Z_;0e3d1_#_=lW?>9g%UIyl|;=S?Yl`ha8xJPIfz3SF*Og35elZp8!^}|;L9`#+wUPSK8K*Lm{C*Fd3j$_!jstI(}*&cB=3JS2w zy1~}PVxpH_nu$FyggDb58RG8dgTA7q77AQPj=Nzfqmc081iHa=eISdGS>E#3^Bm7` zvIK|i*+3&yqSXto|7fR^BzfpK?&j?m1q3s$@>{I@b_oU%56SU>qDAT`T0j;*nSZIF z?>t5jiI!ZXM1S2h!I5Ma0_w47>f!*<9CP!#1u7CMQ2QmS0^;h<`S^S;)!p zm@FDTQ5G&#WhfZA_7oJy!zRJcA~Ke}D>|?$K@Omns-{>&or)rAYPeKkYym-NF_B+> z(R0UnvH^z*9tXm)iy1Bu*r9Q@$;te~ELXnrZl`)kLO)0UA~MjC9BGQdg|eLPVp9B5 zrTgKF0u28MAT{Xq*F3PBBb9K<&rQE#;d)^r-E7c_E2_mFmS}*DLb9dSe&co)w>HBa zSveOGH+mk)Siu9Tn;ul*uRnNcipl=(;j8d?sx)=|7I zYEalsGPJ!hnj9{8Xp7Y_s~>TB;hUqYc1l>uSU4YW=c}g(wdUR|mcD#QmS}N#y(W>| z^w;&q=X*}RmbPdTZoQh?%;ju^E+mLJ8*TmbrpD=xJFMg_!-ZO}rZuB!`Mv)c#KNX5XP`NL z=8H()5`qAG6&`Gn+YPrV#Km6r%K-Xx>N5NddL#d9rjZdoZ&m76m zU#v}TTYq?E@*!`j4`~72*AKF7;Nr%J`+wvZBi$oPE`;=*yTAocs9ywFVd{?@qQsW7 zWow`JCGngIke^hQHw8IlUuBsqK!m@IV?vv=1q9~+k^=T2GFC3fRO93};~1VlFA|mn zavfQoO8t{fn;wy%F?^Z0BRLd-KXWZXtSsxI8U6;SsOaiMhW>wBVoQL(QSeH8)mm!}`a8+;P;aZ?X(v>32svZV? zsng3*q9Z}R*IfVp(nbndD0j_J)y{`#kY~N{>&dLx^vD_`o&I6xj+Dej`15P#Bc8`+ zJ2sKUa{a@B5rm>cKd(;Qxg&e~e_q*+mjaRlshP%Fk)LY&Opy83*or(5H6|`}L>-d@ zzWKjj;RdhEzy}pFiu2IsrByo>EnW4&P#TdYqQKz)GdZ}mBZoMKPl74Tmo_?d!we^F zbQ95&DOLC9;-kakiDRU}PwR%KcOCwTdY(e0IVPGGA^57Fn5-)e;@dZ~Ir*xGe`_kueymz)8Cua({i{n8E&pmHo7frklc# zeY$!PhYd37lYK=}$&!4T6;8d8Y;TdKe{{z$Zmn2XlC;h1&_HTxU#4_F7Wv0%4vX;= zy=cy({ZH}90=dEFt5SvfGJ#+EPxCuY7A9%sE=tT(v2x#z%L5>}bk+IN;!OTbv}2Cl z%BK9EJ32#6X(={fJvggPc*tk}Zj-O!rbumQ33oO$KKAzwj;KK^PR1@`rS@oF(Nr?L zCP8t-vRaq5+gZyPa|MumPDz>}PK!0@%+{2He60CF@H*&jZ0U_f6C#)Nnzn3lIR|ag z=Jc`qj2e7Mm=D80>O~A<+)%0`RB9UUeXhkad_+~oeKf-%>C%x_!iM6f97SJSkF7Cr znH)Bm`YW9|!%0~LTTc&XizO&--&qxY7NRSz!z0t%rZR)!+L`>MwQy%*BHKQK-2P9w zd4@xpBuzhu+Kt!hz~Lk9k9KOXPRN@Z0VjD7ltzmx#$-;*!r-_nkJn=IZueWKX>FNN zXu?6!%<(%PUfD=y$7VfeKCRG=Y<;z$RGe_t8Xb-p57VZAq5E|E6d-D3%9!p9MEu7O zc!%r3g_P&Idq-tHrY_Xp5tyDtZM{|p&L}ejx)Ek`a%)XWe70})^*W^~OKKnu`Ke}Z z@MK|k+QOr4={Pyh9IDwWgp)R>{|w$;)2nrQ4}0+?d~#KIwku*{zQ0g)=jp z?+d*=ZR4u#j}sOu9I8j>Roma+UkK{Fpfgj}HmJ2AK2@VHSnSL-Ulp@}2y)QSkdCic z`f>~E@{-q}4+6ENUqi%2i1^gE$=*$RxzeS3M{zzn-K55FuUP2(MuHWQRi;(jg0?Oc zy&oyIoV)L7_dQMkS2M6+V}GmJZC@0U{)JWWZaLTPTja2HQ~sGd6sLjp;E%E$nnsZ+QVMp{26YeAwFuW*Z(|a&#=uf&>&xuBQhsjSyfmkt#n)M0k4?jh)7TH zeP_>fn+#TOa&2ucONl`b&IR#6gQQd`8tXVkaMD{N z2ZCxs00pftIx&hkz3jz(*B)rg|DJUQB{(K+x;0DMB(jq^>OR`9x=>5?8jxq^I?;9G)t% z5fvHSsp-h-|A;%n&z=N^PEok^)fZ^+!^CG#>Oe<9w~)%GhoGVoqv-k|67L(Rz?>{B zqwiE=2O|}V3GS&1aR$=#&m+aYiwzEl7xPaagh~JvSf47U9+d5=txmJfKUgi_f|g!t zI?yv3XV!5Gee;=Zh$FC0nFO?he#4fL&azIOXob4Ci1BzGNYogpU(B}|=;8vqUB)`v z!D$ZDY30nny7a>A!$o6AZKxG_;6k4wq<;-WodKsGd zOXPJ~!+OvZ|MF-?K=-2M&xXvnOs2ZdV{lm9|9*%4pL&TQ8thDS za8B1~Vz9IwmPxlQ(N{ z$stKRmcOV()gIa;Ls}}C}=Jew9;cP7j$FQ`4ZVay$71X?!y+b}|X!&{2i=xXr z)~N@hv7V!jpz>QIiV!14uy{UDaO$7LsubM#AHAIYH(Pfc$D=K^Xy{QywW3-pYKkrC zsv0b*I-9L(M5w23dg|#6LF>o^(g|l;rMj)QTH;}?sM40GYJ{O^rzKJGkkh8fCO20# ztcfI4?0fq!?4134f4TR5KlgLK_w#+dU(ZrcF1Ua|#ZSnL5!gb87flM`AhB7+2k;_D z#rwSk_magJ(}9%eL%@F@LJxuH*LeV?rhTkw(wzh6zW!%Ca;!kK6J&%uas_!BgY;?) zzvI*4hpBE{>{|008h-k)7s6$r&Kb4(eu4G z1yML4Dc2HeEO6tbgS2!vHQHtzn9)AW1-S8ABE7o!TbD350CQ67<>8!lpTt4^o;b*0 zsQD+GSss6H09=XK+>-(<`A|X5zip1hXTLVX#(?%OpuZe zFm_`n;{5erh4vNn944026+BLGe>Udm}~DuoPn>AWe-?O^K#4rVpN6brm9qeMl3&*9Ejj zJ!%t*cbp*kiV#V&flKdvFeZ6G^0lLc4H?XUFt)_w7+|Z!QIKJNUzuXaoyTe=#2HTSnD?eWlyy|gWe@(9@y+c3^p#5F=*)wKh23zU& zG8bTPMgo4zae{QsBLXYxK(r24Yzaf^!%147+kw*!Cm6>NQO9IFIRGyp>xOmmK1H8$ zk|B8upn#WPyn9%^wylomgt`TO1_Pzfw`0miBQqM@_>I9-qHr}{#DfX89HMrxj=(b6E%%Blssc=(Zuww$;cjW|HJ z&NJ1XD%z2oGtE3|w{9mN2Ua6GP;7Qmi{J8@s2P33TD0e2KCOCF8@-@kpF-RpTfvWC zgtqPdCIoJeG{2l+zOx874@yhP0_DEa3A*TLPfK50U%ox{wlXr~-dS(eMDum4s(`iW zQD)(#aIY1hK8Brkt#`rEP67GM?`u8L9h^bmwEH&Nj~2X8FeNh6E8<#*oIb3JaxE*H zRA}U+P+@Y^C6ZuMR=4hEg7sW^c;>|N9u?~g{lEcPKL#Z&t-;j510@ukv!$%vv9r9^nh4!eV)`u-G z!8MIKbNq$zAst#74c&_7JKu#3eGE5H(L>`F%|ngUBPtr*QL3~m(39mlyMpTc7M+}N zXTuju=hZ30m#hlhQH80;!RwQftj^{fE}}m;S6fdnu77or6e~!Im?dazJY3{}J~h^n z%7_!`G79TS0({n1&7ae%-GB#E+j9N4?LBmSc4-}egW~%SM!D3`^r_>&DT8Dyv zP?O2tz(ZQ{C_pF=HNSzD!zrN-W^FXZ@qpd?x~S41ECJrzPvVi+@FS`#T z2D)pQ5?yU!^USTx#?0nS!{dGB`e*{4`4sPunEk}lgR@>192^xAsc1lsKu2)92P{5f zVW)iA8<99*Q6^0Hz`u!8ECwL36%NHu2j$Gqdbs2z^Lt^Z!diai``BwY3W?sKC?^tY z3DP+GR*9Ya>9ncR;&N1A+C{2n~DIusJHi{ykD1_c2RB1vK z6hws3K_CJTothzd9O(2L~1A^#mnRbHTWH0$lf)6?$6KzeXxPxy7eA(f2+Tb(l_mg$G4+@+u^DQr9 zjkKEZjpV8Bc0c%%^{Ntn^fhL}mvnK^iGa(Pg%K4uM#$LqQIt*aLOt~(9Rz*bL&pTJ zZ2x1|+<2uMU&3FUx{W0zF>It@=yXT2^1vb!4&4tQ+ZZJ=$6PFP z)XivlsRkcve|EeH7P>x(M^VWrY*E$dY=6umAQdBiQ?+U#0EAZ4+_0ii-?}h zO0-!*#aqm;rxVhgFezK*@XhZnTWXYo;Vjp8_c$90ueOp`P>6*w1voXJVVR!_-y%@A zJTq(@v7gJGut{=mBMvdDQ)~h%gI=EgQZ}y#DGOs;MO2vX@GJKh8?A<+Ul0K+i0O44 zGGJw8g;if7r9txW-x0=w_bB7v=Ag-fWR3VG{!_|f1I?4t~ zVdRCurB~N)D2#aBp(^GubQZ(vnA@JLd^(9SQ{hM?Q+9RMa)w!_)d!Pm!9Ud^$TO-aG^w`fiy!&SuNE?>e(mjT#w|at z@Y&hf4i-*i$dazzSMRh%;rE*NLN3wiG-??UP?*OFA%x=&&DDr49AdR@ci)+)y_?@- z5X8E!Mrp1nfoU8^u8iG4x_=EeU<{5wfL=S7L6TfuZ=AO9ZYttbjF7KSknfLCS-QvG zc={$9v7U(dj$w^a^coMd6$pA-xJZ_;ES3zBISMb@&@*P=BIc;CV|B>qkHQ;{dFR^r z&h@vWZI+x4gvzWay1BY;!h;IQIcj%-nm0M0FEb>63L1$)Y&xKMC8RbluUOS1mNvL3 zhdH)Tua`TNx6~q1rD3&eq<#-${hKhxYf+N3xO@S(f#;zKwT4KEkpo+{R_b3DmswF+ zHq-Te`g6hbiwis06nK~OZ!eY1Wi|J>gW}} zacWO8AAB6jIkb81z)KzJdOse_(c^NWdNI zCmCU_-MGy!4`ntTxSIXOK0Fp?vC~RBLpl;J{wSAy_R^A^SNt-6WCl1V@P3PMZ`tVA;toj6|=udv3yACMvtLQru+aKL^8A1MrurDsxVJp$!&C zDn#zchIz8n<;JsQoU#sdy^|Z2WtAp$y|?M(AP?hiiTTCXUY8_R(CaVIM+$TOF7aYn zc4OEv5*Bt2PoM5N+`sgOY%{4zD7wo0AmLlE+t=KvC-T)pVfnVRHU5>(_NBMe{d01~ z$dW)~Qj~*;&Vm|TwOl-Ywx;y4x9`x0d!Wg)1)+|oF_h1k&?!O>=Ggkn-TOq>Fcm>7 zv&%NhuWic=(*k(>j8pqaIRugQAs2G!N?mq~3WeCSwVbOSqrO#*mbw0}X(R8ztgbil z+{7i~*6fQ>Q&a(_xFH`sqc#)B``xJ4V$^5+)UG5;kY}&Szp!hmLEmEr?T2S4F z$)nbDraP-4;ILIBX%7;{9m+OQ&$VBDWB#Pz8@U69aGmHu`Za&x8x-A-J(1`v`Ch7d z+G;&A{)Sav;=VBADf6)gIE#c=)Ls`&mm@Q{0|noj#l>Hh>XkQG9$6jbfmP#Df>y&SJ z%IGDuAh04wWPdSiZtJ@swM*Snd-qX|BW4nXT>CTwnB;HyySQWsXdUFa^BBUm~fCz{L$t6q5ZWo^U@ba4~+JZ4@Htpu2lFh zN3_t<7MOo_usWREc~D10!QW1VSKi1j;Oq$~;@IO+#{CvVW=o#->eV4P3jBbQcXsas z7$J|;KW9akGuV7oDrD*#>1fvIhUSC%fHFY~m#_N;;rnpqT=fKn@Fza zHWLg3U;*^z3YGoghzHx`IXjtJzK zfw-nN{?tC)9hce^g7g+W?Z$AZgFQ29ibdx2xL{7#9TP-+qIIdI1Efd#ymwULbMQJX zfKh|8=wpL;{Y6|DT-;0ayQagZ#y4jz(MyN5Fg>~9;Us;MuB=IB*Zv6m1-71L*(aG; zC6~!)NOyX`Kmq%$$?700b0d?{i9MMY;YTC6jh-ySlTPkdTDRUIBs9#VQDiij>l{&Z zzm@D}9a-y^6|}KXRVS<+y~cUVi|hQUJ)zy1x?BEhESKvJb7fW>@3C7<8d!Y8z`oY< zRB4aR>ip-g3a&xdy_Y<#^SFcMcBVSoHf^S;Q`a$SX8AG5P_j!#=ZUKyxu^#jZ!ZQN zh9Y9~kzX%))=wftllJWuRdHj;+aNA?FWA%vtw;HJr$lR6JwYB(G}PIb>W9?48y-5n z!fioR=JZb%!HtJMe(QMb0yA1+qH@K?ZigNz0$cKAb2H!9nT#cb;?M}VXmX}8K`=D2 za^rlx8y?feEdFFOrB9##bFPr>?6I5l9I@)OUCG!nw2~R`Tqav}P8+u)>3n{aKKXd- zzQcDPzjTC6pS^eb)RImXsqAS`sN7DWBmO(gPJPk)sK_jN=yD-5uTIo-xY1L|s9y03 zol?xx&A`ZUz66&)&ypfqKhGSk3^vuiloe2wM^tPwzH*vL(YhFB*E(Rv-gxhP*%M@4 zje3hcop=q4!o{ox+aL2e3Lj16^?cpMJ|<>4PFCvAI(Amr0EDtCgnQIZ7c=pq-sJ*0 zC=bER$FJ;9RPAZD#xd*RrS(?L+vJo^1THyJfn;@;p6nrCq`7lka<7T3opN9Ye?-x@ z9Yms2x5L&PdaIyo-B*?;V>}&`Iccc!JzS8n+o|*0C`alR$b;IENYQY`hc@NCfniS? zjc?e6OB!e+RabgVw^WpZs6>Ld4e512R$t7f2}J|LyC!m&ySh?J26m1}{&QP6*&GAk`LV*@#EO`n3Sc%0L}R zHwPES&}h3npI{Q+k74JY#kRZbf_OH5rQVVq5L6+$L+H-!j!(}|!ZubHuqQDpJzgju zK{)X-ucCJ*u<`1Gf&!m?qUXK7Hb?n5b-cVGW|%^3BdSQ5dL^B>`hXo^hE%`9(4iE> zR-l6ayp@9AY_Arbxcld_tHmHG#V<}Yc>sS>Y>Be*07Ha+2D#5=>J|m&?p}mug&=Eo z`q(=4n)Cx&MN7 zKohHiELj+>$fRK|$shL0eRxVtDH3a=%g4AAa^5XbVOAyy+#is83oF*_1&3P7Usau-;${jK|W}mT>2j+`#3p%`t)m>KX!Q6h-50 zsYAz)Vf>orc~^)0T@Bkd*Ow{5)0KjO-#F2QPk^?Em0jfRe-=V0CT{Rlih9p|r&fQQ_T)fN5ySc=X;EUF?om1-y{u4QnXDViC zYU(pu7B68*hJ3$$;6$HC6?LV<>ipM-<250A=xO^jTr&IxTwWDG}un2|c}>z^3s&VP@`I=Jj~3=fszKe*i=Mc+K6)!6LwJyKfwjX_cqA?=s_9J>@CNZ9cbqsaWP`#{Y1j<}Lr*u!p z5_a!Cv;z`iEH zB(Ywgvkx!@B0uJ=tlrQsHRjbD`P@Jmy<6!$UXWN2h41X@!Lt{z)!`lbYutlpu|w@G zENyW=WFHQ3`OD4ZO0-we(B_2rH3R2<;|}X=;c{8Ui*)C-uO0a?{oPP)J=3ixJ4*eu z*C{H@hauK^G-gyEsY@-ksUI4%)den1<8vYF9NArLq#qsrOir$ahcVfnggzCJqihk- zPc0Zm78m#hvn6}0ws?xGh?=EjqDgv|duD`84Ss4vaC0%Ku9nO3$?2Ry3S2C7W|d!U zG9c81LgQ$18nZM+m&{#@oY+2ctqf+v}M3jA)+0-%dqCKsl1%E0Q0?wZOL<+ zXvXt~WxuZL;a9!UY_PH5p>F8>nQX1jsYIxI<8^~G41%ue?|SM8?ozXcS9iS>Y-cnP z3c@GzN3?{~+aIQhGl5OK^{4_=Pu`LaNZ%@avGvv?pr@n{Uyha>Ft$@tl3Z8e=3oI# zg3&8q4PA>GI-PwPI4RX}95kad*VNKcjl;bJUg*bY*CL)ii&*n*5QJ@R#2Ko6uOfkb z0JEI2yrn|yJQKkZdWkrqFc#8d`>D6gp@l8rYm~zN3;EA7aWqZ<24Csh;?tbCou3{j zec{=$x5zsmz)!RYwiQS&B!4nEx2^chM~<_6cOR+l+I0}Ns+lE$ntRm2EL@9N`qml| z_lIc2Np*?vM~6;kPmFpx*QB804q4MVV{84k9(6M-1Ngf&r|2^!E9r*^;LwjyPqJw$ zYW59egFW7w?#l3(?afnDk>jQ7U%aPJ=QLPi7a@Uit9mM}lC(#owV?h| zH=RzJPF;@9%*536t7l>o16L+PtVd))o7kM@ zVI5?aY0iZRQUdT=ldmCShcba{)bOpYcG}E6IfL+ik8{qn z(`b5@ZaVS2UZ&2a$u<}!MU}ECSWhTGHOy+?XiOu#pMqR^XZtUCt_YtF`m{kn6NX;7 z&+TO68+0a4(-IQkmk*0vJ1ot}%YQolrFI4u20W`OK}Cc4A^hwvCbR}Efe#7$p5Ps>I?9FPnWoz@fDc5A?L7~lQn*=Y*? z%f{{Z|7CvL+9^f6V7dJGFM)JWPXuh2SnNl%=y5#}%bZaj+(9{>~7ix7} z0d9T49X3?yhTAWFd3ww;aoFf!ca|h^w6-L#SU0=ebk5`;Fz)38s;q5=klFn5{xiRp zejw&{6Mz=xOebrZW+sy`h9>@3epm$dML((GWV!{JZPpI<8aFxRE{ zPMBbG9wP6y+@m-4p*LE(5yZN1Tfd){KL{qKEXRd-EX~;?s}BC!!F&1{qCEGz7>z{~ zlj16XJ5<-qXt8wNA-;ZUp`ly4;A>0F#<1=Tr%)8zjO)q_^z`ZG>-T@F(`9CXB!}Z;74k< za*wZni!N>4?!p;gSllKf!vJfE8#g@F|j^^m`s0I1dsGtZ)JhIeKKO>g? zertuOSI)iX%#U8L(E6ppo#g9d4cM5yF1)H3hyq_<_li`>A z|J?+_tbSQFUFk0w{T`Lp$b}cnH*4UIz+XgUy!O4(MdY;64Jwv4$!{8$QW9PUvYEyR zxF1=yQF=TPzIz|I_pc!NS6Gwx7_mBar%cb%ZaWjanCM6S)*9FDvpKDFDd;6zgy6|* z=bP%sI=xkGBO5N!O=}WX@O6Y*K#XD#{C_XtT~^8W9}%UYSfXf7MeRw0UNAt_ahr-^ zHX+~7orGrO{FbDtcLb>B^Iv~MOdub_LDEM}-q z@y(#b0dKV8RQ-C3Jck8ln2}`xt-Z3^c90R#c1?G=iyMm{M$Z0`*=p2X>Yz>aJU#aTrzHeoBa6AUdL?+^GM)`HZ3W!F6nHUrt{M%lSI=)>rkdrbvF)^%Xp}&KT;( zpP&qzwI#~rh(B#`E>z8E>pPDe0e(>SNSnp7kph>Wcbx^~5vwJD4;Cw%6Anb?Ma_l)?GDKYSUg zv`4~Mp9hcM+O_#oC_*R-atU~LY!{Qp{_W_dM!BILyn5Y7p%1Vc=5ts6=-7ePyFOTw zR)PZVIU{r|OY0VRA@uDQx5;*t$=&^=?u|$F#7^~HI(A~6_+m9U*-JnOdQMx#|I^^~ zAC@`U+Fa1Eud?$X9g{|1nf$ZkEJPJi8Wz-B-(1E};#r3(T)s*K-XKir*)G&qg3ys` z-M{nXwOc7Iju8oZ7RQwsi0p{9GgV#_ng}|RK3ZUxSqGjo`mXj>_J0m4S^MCY^eH6( z+V2K8WN-Kd?t!ffsXobgBYWiy6!7KUZuC6x_FfsR0?T>4*p$%kp@vV~ z25W(q$s1#sn3ts2CXDn(=eGfI_k$ThiIJ|GmYzG48ZzEyk^}7L&qd zu%qY@%BtXM@_)XIOV$HT@fVct2C1m21$5NuIo`5_IfkX1G-s=-jWIy`=aF35=sCjj zueu*Qvy!})H5{<^^b+~~<7KU|=FOl2cJ0a*Om#W#C1RxKBsFBzEt6$O>a)U5lh!hE z74fr~I7s&dw2ki7Vn4l5K(*582&)D~Emn_f3TuF{bIgkOlViP#XJUbJV86OQ>8+{9 zho=TxTLb&&SAFhecZ^Nes#SKtD=BB~e8nj|t@L!_+nV~oIJPtVix=0ksW0LjFuy)? z50L(;JsP?BHOk}rm-of5UiE{5S4k@!aA{{kPb?D<)Qv_yA%lj|awo%9#hG9AUv zM5mnknoaSqn%i8nR-d`JFRdtcxjWdFARP=48cs@oKatqyv>y}>F#pt^wBtWFYsQEasL#HpsG-E( zIpN?mvh33!Kr!y@I=q;3!R_YQ`eK}lwoxfP9n2%t=6>|0+SQx44 zsV8>pZ(WaTuMd)dcKqdloVuE$c;TJ9hp#5HT06N~eCV1yb{n|j>S%st@=47B0N%>w z)d4%ECMh!GCjciJBmA9`S1vjsEDq{L9#XyF1n&=?k?bVzi4!xFew|XxM;#3&tfL(6 z@q|SSsLo=#t6lq!Rs|4j0!Y|^IbJa84HW$7_~V}KHvEy z+199o#~bU{zC4VgkKHK)vh9hgh4eXpm#{O5H9FmWL%Y}oF5kR-y!O?zV)pElDs3O* zB`9{81skAfoIYIkFUXI zH7&CTDqGG98was6W*ki4nR@JXmYo!A{(O^DIzo>5EJWYA_lH!YG+RpSnwZl>kMM_8 zEU+CoXRgzo^3tuiCi~d!ed5qetzB!2%M0Wv<0}uW(g0wAwCHr|2A`l508dI)w7;je z%NYO!e7vR2_qYOCzHF__pzt2iB?LK31Jwt*RIat1T}&~a>!H&bp0Dx2)dG|_or^>I{YS##BxG*aL z6sO`@i~*>GLa~#MnPSEU%@yGog&4)6pa_`rjG{(5-BU0Q60uc-T-zLUh^Yx&o~i+4 z{Td|*>NNQ|UUs_k1&<&uZGhD?&heCX`M>_le{{ysXSrAo&>+ibAdVm5n;5;n$M+er zYwUm$b74LX$}0s~B`O#M2ui~nrAFw5B>B5UW?Ed#a@_`saml-M^2y@I zvNWPL#ic}DNQrl=2U&NfiU3CsY!&l|frmw>ZcJy$a6VZ@qs*~&FA1Q56GIHCUw+;Z zHUB9^wXUlstifF=VEos**nDVxZH?pCA_icg8>l&rgU9?eR#hEVURv zE zK-A;n*X2QbxBV5crE_W8ks~uN84lE%NE*2pa#x#=&DAx+8z!q|;MkQsKaY)uIFI$a z8O1HttR6ibN>v_n18s%4$dAeeVjvIV`+TE|fg|>6Kl4aFh(Z1vY)EgKyY-A&U$F@I zwswg0Wh8lPl@~oW$HuR^aQFOrP+;k=XMra@`oa2E&Rl}_Ky`cF?8q1MNEXtOHStB2 z{*qz+DmUuhd!h|1xJHM<|9&R1{8e3jvF<3D3o1Tqut@>?(9qj%*GnabDC=qWJXNJTaq*0 z`1;*6xIHC)jJB1A7Kvi$#kgcD?khYBKfnhP*9hUpXAEWnKV-ov2hFHxD1C!Dg|o>r zS6u&m0&!49`_^H%LhhSo6~p>*$R1&3KE{3(3q$+;(~4!nWa$z>>aDS)(+6HDdJ~t$ znewf;*zpLL%OM=yQDk4-Gsd9k`Fm>gy=VktpvK2Z0Z#Ts7#x!hdmvfZBMyF~GCd zgW81JH#o6NTW!zGA1G^i?{7NlHFd?s9iZyfWxE>*I;QN4-Z)r{UR1pgBE^U3+`2bh zdu26GfT~HO-D0g!zVxHoD{pV-vz&dQ;JGTS+;6@B>O~9amMC5Y@^FDg(b6)~)eVD& zM9bQKn^_kQ%5te1nC&Z|Rl*BA4m@KU8&G@%LLqLb%98*b%2R!=pJghmYHWr!dlh`c zAHlm0cHuuJ^iUuy|2cgptHQ`Q`y2{|3XrKMJ_%lYyNhy<2KHct(RyCSR1a(|w^n$P zJ!l9ri%+1LZNJk8Q`(}y=cS8q+GlQp=A0JfO3z4RHun49644GkNO#*xV)0pgzz=1( z1NH8(4f!5HgGJ2CL0sN=!0`%t%kI(<)<_cj^Ap`iF*inm@}-Vh-R;cd@>Ae6_QCe7 z7$V4~T}@vLZZ*Wby|b&1hT8>aCZn<66OCk@ zMH)qe-kg*v2$E$g`E6;!u247AJ<*+np~?ad5V}3)LbY^a<@%Xbu1zNY)N-CMS?XjB zF2yHCjhs7NVt3{+3YwU^#z)4ZhSWc4*3Mge&G3i-mDr*xR?t!bIa8w_PAIM?h*_c*NPEX zV2^Y%W4h?@n+{J4Yt#x&gG7oKPgmCdn`X)oE!8~t zaDNRztMVf(j(2_!YCD7$$UO4bpB3cg-2jD&C1DL_)kkYWbHH7G`sMIBMW7?o)3vs> z9J3&C!B)8RY^tP6pk8+qcJ|;)0H@D!FIMq)3GWT_UE2twW7tHy3@v1N$%1^y|JGrj z--i7exBgQF_lMC^(oj;}SPfu`Z?wyNyZifu8B+mQvf%^G%o36|dMe#4Nq~A@!`jZ1 zH7!6;`P-Pkrq9#&Fm(dXnR4}5%FQpk)j7#Pgi|I5Kk)swl_73EVZJFs{!!aO;t2M;t%|NJH_dkK1lpF0!LUF^KZKbkVmf2|9{K>q+D`$gPq4sVUPq`d%{2iJXitv zV@pd*T2nBs3zm*yXZ0KkJ<2MmHUb!&T)W*%jlbFEj#N=l8@t1M!tvg4xl;`2@5`kI zjlK>H{oEQSh6OE)Sip+I#gjm<@(Wr+hrtK-e{8w+xB_}TDtPL-)8+s`*8cTDE8e^c z9>glkBI`*qi9m35bECL_``sj&G}<$bg%M>a)rdv?T1niyz@w2>Rdu)$YFa@0EB%-k6#mrw#yBUk(RP zKi&A;+a4vbT;Cuu(k&BR9i=?WgS9Pu(>!7S6MgJR-BWHMnl?%$h7C6mGV7?%WneKk zcrUjDlmUpoy2&oN&>H&gC01?tsUc$G)fwC+MtQb>%!H3oTO8+M?^{2OT` zPk%3;FeG=YFY#}ed%V?;ywj5uY=*FWO{o|%M;AvdP zlJTxXRD{P@ImFl1D8Vlsef*wQR&anX3q{+k(C@+PXho&J_SBi8ilW*_40oa9lb+x3Wo8a!d7xquoote9G^ zT<&Sv)Hv(7B66*=H*G+)w%$R&x!QdDr3v|~p$~!dY3%#<;>wd|2h#(vcy!LL({7&pD{Hjb9e=e;e`$F3KKeW->s(bUtSm);Kk>ufxOs4-!W~>9=w2796RyJ z`Z>5sseyQ=Y5`BEE00`N7%|;`a!Tp@X99Lgka}m>AJ>QPeHFz@vJeEQte~c>+p`q7 z8XBE{=vvWvFeHD{>Z9Q-&a}7{R+5=ufz0uOa~;V(+6P`QydteV2GYo#=$N$!sIXFJ zCs_(?Ijxc{-c%T``^oh_nYCbZI%ccr8ch}SlQ+GihRL>;HE}Qcn-c6F;S!-K`iVZ?lrDRk@;0eV?pYk~6{C*YZnQ$1Y%c=I#$hg#6Pz z&~udiWnVO=!}dB2(lBX!_+cUd#fFyB@Q`yoopTSGYKv&pO|};bL`$oBPmQY~1hRYT zw&wlSyUM$~3u4)SC>;xNbK6yPDlx=%6q!|haS-&r&|fGsLd6B60&)Vbe{b_AZc!;G z_s0P|5{E& z|Dmu~W2CRBfLl_Rdc`3cBQd}J89p{rtUAAH82Wng;J?n(DAyNKA`YDmSpD!;i*9=4 z`%vROqf$+^AJUcVTz=RN(3_?oqmWjZ5%UkFqE|%^(ug24OK8al@Z7$;Y;JxWyT--= zI`OtucK`JH`)goF(`xBCv~Q(x{pEOmNj0|^%IwA$mq)ULqSs5%GzZ+=-`ijv0Ogw< z7W_HX;Hk^B-AS+l+fft+EL4Q1;7hk4v(rC1O}z$e5%s)kA=S#h(?(5CJR@n5;mTL9 z?H$}U3wiKJ0?k2u*hOy?@lqwQJBm-Sn+6B^`ub|_?-Q#%Vwzp;gX@q}`SJz0_~ZK9 zw&MeFncpXLYm0uy9e2V8zEwl1I!G*H+4#P{ByK_h_H`m|d07TCUI zDHyAuk~YLU@SLjwO=_h@X0RC-=Hg}p=sTBQp zQ5>EdKm~v_&4Vkf)^E<@AgAjC0zd^^=Zwwb4_GUJMOQR)3PI2nVaWU*=-Jufygb#u zj^_?;{1&$?oLpN0sQJ+whhF$ z$J!$x7sa!02rBy*z5s|0(E(0?P3~kDm{;ha9RsmVYXxtn^-JNOfW#iPMQRi^S$N=7%6hWPB-_pe)y(RSzJD%=L718B`? z@lOgb*L266=L6Kf6s)#lF|n-UVdhY&LnKIenGjj9H;oZ2ltb@8`|zx%=U~ZOk~usB z7jKzAmK92!30RJw4a~brwPGN4?vm@TO%7PD!>kX|gN1o?{K6UnOR*=3Gd=Y+%5u8k1^~gVQ4>K zj6vHZS0$|{7}RL${O0=Cu`)c-0>`d|DT6bDXq0}Ubd-EEPi;SX$PxuRnP?kY!z<_d z6m3-KkCn7-7^(Ce8-)Q@f6BIREr|36kRw)ujHwe;P$Xz3#iv-> zgf1U59y+G0Jzf=Ce>8MDpMA25Ri|z)Z=ot|H@|?BRWLCF3F)Ho&SzvfaKtEJrO$GM zo9u?Fp9%U8aEx#^K>?_Mx7mOwGfnceFz!>UB{tUGrOJ74j%jym+|L} zj1sG*TSk$vTt!#^09jw@`Um5eQ>sKj_h9a);=YS)BqwZA4AaM?O!IN?X%hoAuX8TW zf#Yv(h2Bn!R^n0cc*%R*K3wpDwBK2i)D{C!)}ggcjA3kD4`gGdcqY5Pbqozzk0B`B zl0Q`fp@Du;Y=+U7VwqMv8Ucl;SEL7m0~!>sijLN_c(ka{&Y_iVP*7g((A_tUI+2Vz z!|SoW)Fy+a9sP)WWm#x7K{(@JofZj$K+$`}N2uaE>vZ7N_*qg!C^aHf0hX6yJ593P z9vs-9cO>Xp{J-E@@dmUrkI>K>ttt=trKddGW_oh4w2D}Q8~+N@}R80M2pSTLOWA+#|&v zLj#GyfN#Zuc6lB=9XOTCv$PdoUq`WCm{cDBQHD~W9fY;KXV2dE)^)VjDJa8NVqe^<|HvFonZt0^0 z{7Sc$&D%7yK98j{xP2$TmUcVFpmP>gv{gX=io`LUQTfn*_Hy5$v-V zfXFey=dJb83UfXmfP2BxI;F~h_(x(2BaWHRI|-2mWXlwMrp}|U)0%9MIVm|^sw>}9 zg7G&1?`v{6GoJh?D0H^Qk91XsO5&PzYbg;0rb@u^o8!b#=Og!z*TKT)yc;Wb-I4t z_#hlp4Qz*6JM=#NapqzQvj%yDWxwPiuM6XzE;s5XPF?1DAvh;F*<_qIz~$a)dvk&@ zVCvH$TFVqTXhT}($im(AGM_)LE44nFIT$=0%Gh%zl5HL5oS4*zjR@UIe(n#j`j|zh zd1{~29;7hw1}OhG#E=93T3mKR9@S*t2Xa=V=jfy#(wLE1241G0c0?KPq`vWdOe!1Q z+p~O|o??ywJ;-F%kW|(*d189hD=q2n3cK?iF~*efUFwU%2Zm0~`c>XyPt)LqxsP*u zcN6kB<7njB;{#nA5;yOs!Jl~ZONmUhJIi#;Dqf5wotgvG9 z0L}(Ufj-is8obKxA?7{V2FrAshtFiNgL9|mpgNY#Y85mPskO7hu3sy7ZE?Z_lrUKJ z8?swm!?l*R&Zb=%slI_lvl3krlQm=H%Yra~R$4<`ZBLTh3Rb-F zb{=^*<; zEoE8Hvvy|@3vLX#k5m72-|UVHmrF|Jyj}9}EUG|JEGamBAX&*TAE{B$;-V1iAL#NK zobKKn;bCMn*61l#s$Zq`id@d-5Z9}p0;S(gX=KTHVZJ1*BA6|?q3@x6jIQep$zoo$ zLZ3kEk51F9?_-LYqD(7hfytjf%=aKPny1V3rtYfo+p}~cA*jh!b;|3^Z+}a6zfpCU z^L^3LokNSRHl4Jva>po1{)%AeTAI3^&OST26REo8={4%DU5_+))!pCpB*@s}#R#q$ z`2&5%+7o`rpxHdlweAxOqg@o{Q$vGua_3^Ya^s^fbO*Z$^CrXx8MqH}r-QRFb;ybd z%T$-KhI;ltYTZ>u^RmV+m15Ee=d`U|GS7{@Aitc~E=8UW+FRSewwyt~C_Y@Z zvN_@*t1R$ldyKx`g)r$AMW})gkiCbZ#Q(%4I6l_ue!P!#$EUKuXtKy n;V=+5(e!`ze@%GgLuEgC>P)_9g!vf={Lwvk;cT(ymD~RdDRolz literal 0 HcmV?d00001 diff --git a/loadtesting/channels-throughput.PNG b/loadtesting/channels-throughput.PNG new file mode 100644 index 0000000000000000000000000000000000000000..aedd24d51a9970026fbc1ad05f1dd60a7a8155c6 GIT binary patch literal 24427 zcmdqJ2{e>%7&n|!iqK*yYkNs(A;J)fN|J0bwo)0UA-l#hB2+>vW#6~KSY`&JAqj0V zmO*0+*~XU4*v9bQBTD~z-}jvFobNl|`#R_USIsleb3gZe-PiTIe%J51?_a#2#myg*{s0}qRlcF#wKhCU0-xrs*ovdsN-PrLbd z`ZK$qTbk(J%j-^hnQ>%->#gI)bsHYu<9Qcv#?JouarxJPsFNqRs~x_|F89z%^ZV+$ z`#S^@ySnemKN1_ou0G1W{)5ir;CI&{KH)ZUCGKO+v$Vl6mHy$;DYTvO)E9ss)3MRQ zGA6>FcNNx$9nrWO*2>v+29rHSx+L@JySjf|<=qWi+h!*EowdsJ55R&(1k0vRAH0$s zodt*1(1Pm_h3$PPHxnL8*Fpris(*6T$ssM0<}NwB9bb>yK6Vj5w}z~W_|*?oDu`1Z zL!YBh^n8wmL@7C`mBxi3eLFpRxTpG2^ZlbT^W8FSf!xAQTKVq@c=8$50vx=VfSRjD z(Nx_LwrfY6cEHxag4a{d7@Q|r6x`5ZUJNW_wwAfpsxG|Z_1&U=1!F;MySI6FTio3H zxM|giI2CUS$31>YqoN^9SqF1md?Ak)WnCTW8i$%MMrGZt*784H`g*RML%b-*ykxfj zQf(*Hfk@tQQ&T)GcV-|dY_@ssW7(XxymQ~D+5Rve{8}!7qS5veM)h2U>YV>VR>G75 zQq^1h?v4aZf=`_eW1`MxfsFC#?R+|lZf^86%%;s71WixuW!66$Bc8p)q`g#``>xTP zq{VBgN-U*1&rLg`l+!aZY%hjcF3dJ_TkP+w-8a~u-roctz|kU+2x1Uy?3IPgtF&9< zBTvoa+5Dh)N8Hp-mpoMM#D&%zgs#hIS>RKrf8 zRFJhuBKQpDhu_a1a*MKh~0$W3t*;w-tR*(ijaJVRB2SOow-bujaPY`_c8ea-PXx$@02UT;5Ws-u?ZG$PR&i~s2pIvnc%23lmR?5g>^va;Vd_AURsd~14B1k9h$h>o-r55d@LqdEER^lCMb}((;r*1-J{g?(c zAwIj=jyku-_t0+GZarAb#9D-?=l60BZ|n3P?^)V7gD-WjY`xtn<80Gj&)HB>r9QV2 zO0Tfz+GnISITpCX>kJ`5oQEpeM_{_v3-e6oMEd^o))6=v*}l4JZc)X?!R`}~{&c;h zxzdtR!pESHH~(qDwrjV*%nhIel@CipI=R z1|#m(M9DK4fsv=y=$O%t?LlyQBO%^A-YR5cRW7;@)zwg#ZbkLU%z{i~Y$2(J0~^vfZ$=AKxuA7@cYuhOK=2}SKC?u>q;ca16#quR~P8D$ztV>HU4 zbljEX&9xy;YIbbrS=vm0O;mpzs_RX_JRYAGi~O)K7wbJ~f>LUf8SEGIam)0lCOG$% zEii!BV!omU2g>qeHHdGRZ)y`?Oz^JC4~-8wm2f32=q<_D0y&ss--<^R6GMbsNoOQ| zCBL)JZpBVkukoL!<*YN_c{&8);YITqKOEoB&4!x!(L1lQ#o=307#C!83o7Kh0v#!N zU@h|Ds9*O?jdIdZ+E%+71;FmuK5|4&B=GCgkx3Ddg!#vlyVCn|3sYl ztfWG^r(%7H7uCYILt|S#f;sHx!TrQ9@-1BCjcvUf+0eKc80CzW@cg-iF3yEn%mP&- z#%DX{6Fcaqp6&kdk>a?Eg)G_mzA2^ZKTsOr0 z%AtBo95v_Y@15XtR<-PVf&O>0rQx1IpYaKc?*5>=jKSL}E2m`L_Oo^;_ja^Ytufj( zsU>`UL#?od%;5OArbe;Ej4KmJkJ*snEyv$pi z8^$1}tyjeao1Cj;PhH(K&RHt2rXw7x%n_V)0@I{ci zsUz(z%I%3u*~PSh+b?M+dgvRc>PnNPX1!KDGGs^W;pFzjI)*2&Gw2RMT-q%qHCWel zm?wmr>_aRgq0?I{Ct*Ric8oW_oBHx#u+Hcmmxhu0&xJdO)>4}7;A5}&9x>q2OG?^C zbcm)V&JN_&i6NMB8h4H>AGZuy=uw@o7~_)Tflw;%9-zNM2SG9Dt{q6TX|A&g6al1} znKM;<6dgIZ+ESD&LGMRQtKM1FxT^z1IeOa@gNUn>dzo`j=DGK^%gj4iF0MjAB){`K z+sUTQ*vAKChwG|L)YphQi1aKDzN?nONe-xj1)WG*m>*l{AU8G5eocBQ&36xLIfi+5 zgtAlQ%p4@JHq{%+1Bvj%Rk^|~Jsf5yO6E6WrhddN%*6S`-B8&e7#SdRx4*$BL`PTX z%_io?aBFOjYFsw7unBd%hA)XQG?8IXNjuVy7<?(ewkf6zcX1y4i z!bn5bW@Ju`l6w>D)dl%8_nF?(j57>=F`&YU@9NKLt#fSK`S#$z!SDOX_!rfca%uBh zEHt36s=C^mdSt^*YLmDw8fw=xJ|2s}UcHM7BX7f2czBZ?fFHZRux{_1c*0Hv#RaUd z&XIocRoAww3Ex+*5JN=ol_k}!GkCyF;XG~!)36G)uQSn7hu!boe>35+ZlvqCYSsBX zagS{`*5+sSNbF%UK5DDwXkEFY|Icg<67dc%HvBhsBANYjaDYb!u47G6< zd@b?z!1@Q$28M?pmge9e@79|Ay1Vj|;uh|#v(WTByEMhOVMCC+CgwkCRwwH0w20fA zfu}S#_B`Q(qVC5=bP4*O?9vld*axXaF`H2{jBkt?*}Zd~Syl0(KDX;jrv;eNb>lOH zvcL&G*2;#jTv^1)l$LlY>LA=s3=tq4phkxcsSw7V?9~?9D7s;)L8O@w*EeCL8!7D@ zTw0xcYsZWq9v@-pQ|!|*F#W1L@phO4k#4lm!KWVQGi$3Vd4E*gF8+JQc5&K%7h>W4 zt^50khjrs1oywopwEgg>G;&m)TIkdDTkg*vO1%(6fxQS&DHBU#z!UbzNw zv#o>6W$ym|*z|?UQ~kZ&mBB-CX{6e;N8iVe40h@p054S5%CdH0iQ}*#!HrDUgL8*y zW9q20_fQwO`|r<3)ycfCj72T(*=wfjr)Qm=rwdRoq%SaN{j*mCzHjE{IHQ@pY86aK zP57nJKy8z$@+nAM%eU$OgJ0UTIxX?yu8N?N-CCjFCKcQVNIT{28V|6d+$@G5`i!v+BrFGm z3l1O8^cSCd4ctL!$l_1-Tkd_+B?=?4GJvuvtnX+s>=;wC7TMa6>lX~@(`LhG#T!J5OXYsKkJY{+C>1hxe{px9U!}rtE@k#cB)iQX-8UKzisC zN$jZ(laG9WiLO4UM#5|LSOp`&ttQhmAa;>r0kD9e&k@wa-4KZkP8 z+-zTnbgl%N6DcEqgzb4)vxj2`m*)KmSe;8fU1dr6@%1c@#A z_C&iaSf=#F6g_jZny{E!t5}G?6xDWd1}05Cb>hrk)k1LJ8X61tnEGPi%{lfHeKp09 z_uY3+h+&mn2EMXjn`dR=%`e@GZo>n1c6LI_7f{~Dk7DH=ESOB$Yu9UIA>5(f0J5w2 z4mpB6ItS$wFz8S^Q-$a(g^Udu?)E#b?1zU4nL!&H#N=>VIwQEr$a$;pg#4Rl`LO&i zaGGZhyHX8MYS&uSUr6&|$!kR4Am=W7M2Esr=H@YFmIiY#j9!%j{L=nWrmSsXhKpz+ zw+mJQa>AtYUnXJ!$7CR}AM^G~$hz#Yw34R3u!on0R#%3_DBrFR*>>cOlvz=3-9)v6 zmL*1LGq!6pcBJkrB{6n!>3~6`YV{<^Jyx%;e3Wu`P`%o;K1qk9*~hv{p=HHYl8VI6 zx+E>{>5f8&IAK+vS0z6_NVek+^>IDr`+CYGOys7X?&5|6>-6|KJHq3?yXM`|rR7{Pc70ap{Eum+ zOXNR{=MIZUgjJOb(6M0ESvy(QJL)*XslcXoGPsVtwa~$R&AN@>Ck!}nIGm58a1AaS zqvSsNN!0dyFzNiuqm|}fq}W={7PGFuY^oI!A*#z~_8 zG57}8AK%W(UwOk`2)0t2SpOQVKtP@P;hivXg z0?9i8tPVj6iZ8klqd3#?T>)ZLagLSgupCgCl*OwS0LP@=!?71Zm=+8}k7`V-8>x#oqlPzqCLFyV>r)txh z>)|)J|29cmB;Spwh9uFHU_QBLr>26@X!L}}rEb9s`$GDgyj{kvi*6nDIg(6D3i-rx zE-QD9PFNL;S44qS8zw3UqCtnQpn`o%ZaIgAPPzQGcbinbEu%xjgynO}4Sa=ai9XEP zHi-0k!E1J=Wth_0%sQISn{9|||J@?CNcDF*k%=-HT1pS=nx^6A!$t067GMX!r#HgZF#!IQh3#Pf6FI?HPSO4kC7tJqV1jg$mSEsJX+4(6c zk3z6P6Zs2Ni-_v2%GTd{LH@FRfp#EpTi#Blw3S7Cj!`Nhqm4@@`fC}Cpyq}=&oQF>I6V?s z`~2j8$46|kXsB=&bJFsG$kC_rvqidTkkm8)=l3;Llst=`|6;`0-I@C{7t2FgQKL4@RT@b2R?&Wfa63jpzcy^ZFM~y#wtaM0(E8^qW_Qih9Gc zGcv|IWq|+SXile4Oj#Jq*x0z=0YxVp+kgLBUc*D7;yPa4E-1^!0E)Yt!ozW;&e<}F2w2EJ57x70K}N{r&)66tVA>9B=b%54jWPfb4GX*nOMeF_;W-4|z)3`RXa z@FDDG0n(TMBEOql);ADUzgHZ_S{_5OgY0&nPGn=Z7~)HbJBGIv2_fcKSdQtvdAKJs z;gsdrtBKE=7(gB8u>WJzPz3`+55yFuy7d2#MT~1jO-N!m&N;p>k z^V*Cl3l!5;we8d~e>3-XSD$G2%VwIg&om~kRoiUd5OrgVE!RKd!T+lm#o0()(Ivgy zi%e*L2Ff2LY-`rZUfu;EIorh|q+Z7AmNqvg$|21X*4ntR^?`E8h#;kI?u&N#A=_Je zH&Xr;y0y{L7hW7L*R4srm_-&$(~0W45UtQvMs>R^yYIfh-8#P`rn?{#Fu2=hse4al zVDj`x06Bi3HTpnM76T~cP3dtRrETGMn05cMrc2)Ze*8DdPutMOGRFG)JzOj^UI8%n?5GvI+?#| zqNXH!RYw^LHT8x|rU($I9qnN6l}41BZu%RArfThUCB>_8tJ}L|x#mIMo>hbPpvOwa zR2GDE){dlFIFGg!+c#yI6hlL7&F}){+vV(@DLH*RWAEU)OW@EgqUV$wugY}1c;b&QHgPk#-d|te$t*hXk|Eku4fV zT=%!%y&YGCYYpq2ycYE4CRPh-$#3SWmMH6b4VHYwh!G562wNXS{&=#hR`{i-r>Nlcfx%%I?gWFrZ$!p16*rW7&o7em53~Dx?70l z#K0IK7s4?tzK~x9DfWnKbx*(I02jFXNce{6X8NA}QesIK)^ssRgKD|`u>BCQN1mnL z5H%_DA@Vj;kCcH)Y5VY8(%S)lLtKy7eR-4o&craLc7$&+8aYD)At9?L^S0VJPrchm zHoj6KghV|}Z#L~x>QttXL*o(GMku%(P&f<010G3ve41On?W@~xOIO&Syu8j?=S2>r zuJhYDNRt^Whbj7%`J-KQde}-nria*PD$E$)PKswL9E(iO_kh8S`FnbsO6rgGNlO3h&PUR%6M zrSW9jrJAVUsEsvuB~l`Tp@3?Kl z^w_m_%(176tlK7*(zV-EQ5Jy64r%&j3079OV(~&Fsp?gOGEH*pTM8ksAW!ut69}8I zA+`g7^Pdvvlm5AHGmz0GE`!~ePv-cWFD>)VlwbCrt9X(CpgzTtns;f_ABnR}P2@o< zIM48=oDD6xnxx}+0AS@kzJFbJ$CxoYIWpAYHKC(gn}mGjP~0n^I-Jya*nHr!1Rs_j zw_TX^1BG|`FlK{B1si1U!Vv`KY$os4G0Wj~Z#RQ6dEBeAfft=oNoe*26T5l5Ai zNqLt@NrXonW9C@vt8zk^YzD0C z8R39V!cW?0#XTJ}0R-hTY%3uviVaU4laH4GH034n!~dEJ(v!QtL)?mqLm3ek$6 zvC8^fdFv*CyedWT)%Vajgb%vwH>={We-!B4oMDY5CTIi^Kn6Txjl2+W+q-7h5-#{n z6v-JjB>QGd%&u?Ik=zI4;3NuNN*oO4f50D(lTZ$-MmOL7@JvGGdG#saB zLuo9;lfYz5F|4~zXpAA7mcq<$2P0#tQz6na?pq_$Qu$F3TLx;=@&HuMc8Sa)S*UT9 zG7B|6(IAC}Rag|=O5hzBw;qp=8GC^rICS7jaNzu*zfd9A?Pr5O%}r#9vJY?h#zg@n z*yI&cjT5q`PHCK$rD@o~;mm2ffNO{=?}FZXVzoC=#>GammbjVU;L;*J!_P(8s<0QB z@t(B^^Ly#r8$q}p3JN{|bEh`GCoar0=JD!SX)A-TMi-kd51oCwuXyYJ3)7B`rfi`V1GZ#;z* ze|$PtJN;6v@gvb={Hp4j_Ye3by)K4+7E2r<8(Y#Lttsmebq^?$t~I#!>H!e^@P4FVAeYdI4W z9$@k_L)GF~-kX{V|TByTf#1L&%49_X9 zl4*_wBy70rc{RetFt=x^oUedWV%Cr41B%Lhs@=T=l=5d=mz~0LVnMOf>2bD#=5)Wi z4QTHn4l$3{GHKG|xAUSR_d_)_dBl_lJhPwrO=`;!zj-F}=Lc_~r0K@F0czQKs?9lU z%$AqI*HrOi_^P$!-wJv_JyWbYssI3em#jh(@RBSS}bXBq>@qj6*GVXNm^q!Y{ zuz>>PBRN`OVxvK-a|xb3A@V=lqq84gefF7z{@m#ocNTFWz2a>C-9IrPt|~G-|J}pA zpBrcgPHK5#>4t);)Lmz6saN0a5W@;en+>EdGUJr1lTYwmNGVJ)#m1HU|H4=aA z@pkR#w86UR!8>a>X1j#8NQX9JeJs7{hI$Q*v&yh)erjC@9uYMrXPn+l^N}>pp=oFV zi1fIG!x~%1$qM9495AuzqJHqw`=|`vOP^A-Ef=DnLTf21x+%ggkU~|%0w2H*xnIsr z1@O=wxSIJFhlkdiu3bT5NAyOTg2;A%Le@MJ^Tth|*z`Oxz=d!6rIzqZ%@wRr{^bVQ zpNk+(KOFRYC%SHKX2N9qmM<%*qkMw~SrFpDN~zRrIqEq%oH;~pGHVlZa+;eSZ;KuN z57TIpZhKgDd$Ya6!q}sD=+IZ?|FZ5xr)I3_;v4=oVdV8f^I=?PnfD(vn7hSV0tER>Z9VH$#sO0 zPC@?93j^ZYH~mTF73A2K8j7@T`TQ4wUyyEFzv~#{2xhPl+d2^zeDHlsha5TA@?# zIrqVo2s{X<1Eg|LJ~1C{$+Kojaa1m1ObSxV;e*~czvNeP%mD)GTWkj5XNcN-^=DDU zLPCywerJPa7Mw4);MKc_DMqMR6*_`Xvf9XZ^>C>-qiv#Ab&ORd^LNnE30d$N@2*t2 zsRhIn1ThG*h(J(8F+kDM$)AJQUXN5t%|y3>T)00hqYw#Tj6J|ftlC@7BGdI;Eg&ugGnb-lyfw&U5ac)VjNG?A0TgH>VTW1dg=y?!!Y z#h^mS*eR@7>cd1XD6G92&qLHIQ7Xr-Mo61`@glt*;#a!^&1G9vh_G|5LHh?GEO`VY zUJsavQ86G=QXSntY%Jq$v~Y+dw9+9aN8(eKMi3&qK8Z{A+Nt96#3oKMCr-%7sgmD# zI$<|OT-1A0jO!`3;uP?fXHfuAQ*A_V13q)eM&SNuMJMnlc?F}94=OI-fS!|9b|3bs z7P#uUc^19^d{O(tiPg_nmXQ+yWw-;jzS?lEpz1ecALPwe~unVwq%`=W2 zb%)YOS&q$$vDko&xx4Z0tA+Mpy5UyL{c&F1)v#j%C9}UL()MG(=^K>Uql32Hzy&@} z(PAWpsr<0cm$R~nTGt5oCFX1+w*_-{00F~I*`10XSa!BlFVv2}KClLfZd(~->dL5C z?wi{W4oMD?Hm@C28hh2SP~7!?f1C@5L=(mYs_uG1q=9}>{L$Y=gq`^z^;C?7wI%@q z%o-NdsIpZKyCJMsdwkKFm!1-wmNCt_cG#CC2zZyD&w(&s^?u7RdBr%QQhnQ_uh(a= zD*P%s1#xpe)>k;2Z& z2{_!sLCWRD^Ru$R`b_+Y$lq0ETOZ)$3JaV0=Z7R&rSXQ8&1<*rI}bZ77kBjK4Ys=a zen5O(9YxQj8SZDXg=x2&jN5t2ed-cV+Yi>K%ro|Y*f?&YcgG?sRp8TyC&Sl4#kuq9 zWk2G|MX;V(jh__sUlcuD@+K#VsE@6n16!|=q$fdPX_SnPKm(G#P~M?qSWspok@6t` zKrQ!m~?{#j#CpI#q*@l(5|Lt4-7WdJ9ZKESqm}a zO{kh7LcO6PC!VxInIC+P>{&Tl3uun*rPuh&`(h?o%91cY`)yA)8;`~_DR9ip`EBBw zSr;fkRXxpP>e^(NHE9#)-B*EEyoqKezRTwrIl%vPZwq?Oa`g=()#P%$rp{os+w0pb z8Nq%r-=r*mw}gbtra{KOLL8SYY@`Lh0QPVaZ~)%>f(tfMx)QZ@hSIK7j0294#SAK?cZ{(rQJ}UwGsq5XZBQDp2~$8g zbKE1;&VUs%PQFMs>kN9SNeerJ;=H#Y;Q!;7SHks0hAXik#AG@sb*>8i40zLu0_3yj ztxps+gNrfwmi2yeTLSOyffQz0mURTd%{Sg=i5i|u8qapztWil68?`e6&oDVHTXL_D z6uNF?U@Cp%ve8+<(HVhvU@vLsfL$!EWCRtRYyLj=3f)xFt3OIh>Ej0w847~ioIXm8 z`tz!MKW$jlMDvMEe?|HcWfIS&1ahd$W$m{rxa~oOYW{_fbHg*)W(&{k1tF%6^h&u7 zTuP3c4ce2V@w(~gVDCl%7Fpgvnyyl99C)bkCvkZ6;|DP;UAV!y7SbPKQ8wKjYz{(D zo1w9JFQBZSmT0*VL5a}EHKo8t*upHS)ihq~V=Wx1;Y~)2==S{N=@!)&CK%gN-|P&= z!GOh=ic?eunA`lZ1wv;kYxo)cbCVZ%(3+6yx@S^CVa~&sd*}7B$?@pU58aJ$RZxpA zWCdeSP?#DGRT@nfKW^>)Tl+-JYBC6ZZvcJG!&-a&YbLDNlKU7)5ecV*1!mA@JrjFx zkXh_xkz>!CVVhDeRC+ z$=FaWclT7!OOW!7rg^>3%zc=Fd@d)VY1}Cd#eb4{oAez$K5*sz#lK%Y18hwD_lx0T zQGmiPthfc%@-F%8(>5SodpI!16ucmPc7gl^EzW> zNSt7KzHCztZg6r4t#baAsxUM~en$=vhBa1X-~R}0P^J(L==n4HKfUa!aFEbAc@8kT zNbJaX9IK>=F@@lCn}UNNa84BaBc$e0K0|da-QsDZxYM<`OlBe$tJL>H(z3v)qt_n{ zwG|gE)SR`(7L*=MZW=KT%MV{imf?-e$pv+`(jb!rs5OZ$oS2HauiId=(g^G_GK?J0 zbGAg5%D{j606;g@J@b_B@URZH(;Y@_r&gq3mY!BE?ukqDF} z(HaIT=}l~AVU@Au9yphV%ASrS^K+!oDVUA9i9g|cY-czPS~pd;;9Ur-OZ;S^NBIr$55XfHUefSe zmhE=uSIr=p+3nuvKBaD|hUkQ!CIIGdJ#$G|Kv}c0$qf9^Tl}o7`V~ z3CNjnhyX;5Vj_Gd@z|4VI$!+V;?I&?uY3dev`1xOZlcY?fO8op)F!634qe!{Zuk(O zw-${x+hY9n!z$p)!$eMWi+OxyTM$0S>J^>Y$U4RXj7koU zZ^!1ZL%1?jr@#7-=2eSLSbL7Xnz%3p;PCX!_M{&Fi=bidr_z-ODg{Ez(`!mxPtlQt zlbW`PfH8~R^j5w4cKw3k!oB=bcF(3?*dtQEp$QmN|G=Fos__tz)d(ivqr zITEHGw_f|Q-PiD?N1KvTNn5Bu$OA|L_n5e9XmG=DTXDl!^Hosb57$yo7Q@Qf7VlGO z-qUXyUI6^-JR?kH_JDiXnDmpzM_X3A{{~qdrUf}V(#wxFH*LQ5=I#ccFRlSsv*#3m zn6OhK6GO;m&K%%__idI^cnTD+37L8u|4H+ry!SQw1)7-iU%JFgA*{NjoikLpMen;& zoor29_)Me22_f}FiTU-E=Nii{(ROHgYIOh{+9ZN3z*U_EQBpOvZarxCdf$pBF6k4e zU-ke#)hQ-a;*21*F0|g@4)Fg;mb2!QW1VA|9wj6#y(~s9g9E`2QAxV@z^y??eo=EB z0(=Rxix*`zL+4!9kU>Mi`cy9hV686G;PrMVT<_Ngw>K((5jyGw8m~IxF??7#Q?(d0 z=a))$1h*tm567}fYF@gYEor&2WV+G8Em%6yZxR%TH?$bCDxWN*amc*XmsJyVp7Q1Y z8DOY;#bjX@{jEer;n^4h5D;b5G_z*uAJr24MKW$tfTZ@`2qmBpIEFJ!TUMB_eTU<~PIe zu-;oWX9P#eeL!Wo4MezO#kp5Z03SW2%or~434aVq>mSB;!;A%&Tyud32#h^!O7TnT z@K}%Brjb6-ZNXBeAGbSmZhckF1`6k3q_T$Km32s;-m1gVd$hqaAN=tqfQu#UZfBJ+ z)9j0f^FNw_46}{ZT*mmG&9!g4`X!U4-DK0y&BmxMHWXR$BtUM%!tkY;&IW>N=kx_N z){Vz!x23%j?0m4{B(&c~ z%eRdP=ugkn8g!4%M4f{#(;UZBuh(mDXklBP@V7QVHej(|>T*(6s@UK_%V7KoHKloF zp}P3Iwkze4tZE*8sF`7%c=KJ>?stnqazMgQtX|9rYkuE|RqANfp_>{%5UAJtY9ofY zCbYE5FGX^p`R5gv9_W%0bRuh`Nl0I9TCq(8)@PY{MUc@{h4O z2H)4+^yuk-bdcG2BVFz_x9#4iWdO|P406attolnULpcWli9$fgPOHwz?`{o%!VO*Q z5m-Sm8^JL)iy*)L%?aStSmeWRU3E4t_Bo4?PG9U)Si%0{@JKtHrc|)l8`l$S++P3V zn<=qNe1b)|0(W%MqPTqp!Tg+#?Bi;oSW5ikBQnzNz!@B%LjHcCd?Vep!{rTd^icrz z|Gw#SWYrq{jjgwx##K~H&fvO!&f*GAIol3n6AlvOzXwb;+C&$-TIy|L$bq(*Vlmw!y<^BfyE=h9RHM31y%wvGl1#@>q4 z5$mJQS`3B~2|L``i$G65tGKeR*=R2{*BQ6&8) zeA!k_$h1#ss9V*>GOj=XinB`ZXlE`sGscKtL}Mp5B?w6^Npk(+k|9XD6Eqv=X}UB=g1JyFLU z53s8-(Mi`7_*LGwy8z+O<r;TY?vy9r-y3!(RTMSxJC^KW_PE7!V)2?8xgpWLXaz-$J)Uj+axfS)C zbD8~5+<7+VLcz|lfTA;yu~9a4^6 zC|J}uJD5Ma%Fuabp(kV6tUcq4!F=MvBg2p8C}#J5Sb0;oMY`6#E7BJc1fFdKw}a$R zwyY>rC0v)?_^nu8HHky%GtY704&dA7c1iI^zRv>x>T_@2jISZ?@UM2;avht%8T{D${?8{Gx zz?|+0D`B;Vu(TE|=$0ZTxt0}NG&ES=7FV`y@&3(ik(tfKrhVufqs4ll+d05LcH4u# z$+>p-X%yq$J4?Vk_{r&{!hll+q9EahQd9IE3F^Dz-M3F8$Nqww@?~jK!BE9 zN7V<~Q|xM1p>mG9At3a*x*BvWDkQrRQQqx3+P_trX5l_vsVG1H9ygjT_kPO8`XhC+ z3yN+F6zn5NJ=H22@y>A)w`AJfPe3w4RBu61x1Y8udp~3_QN$P1W;e6pYr5A{s%X4A zJ7+=f+{Y2y^MZ${Z{D47?whtU`|$mdI*?<9;feSkarT#ZC!YpS&CKLU)!qA&lv=r= zz?V;G^_D6_huMfO#OT#ZtmEQ#4CKpU(1(mscdn}Jgbj6{F@Kf5b1&hAUbmofxYHu9 zmQ8fL{NwY^W)P?v*SMXuhKEHS7qB!xB4Gd$;*;GynXJkIYr_d=fOZqe2XKCay?yQ4 zD0$Eg;5OPeSbZIo9;@i1?X6QPk}kS&D($+p?x4!8sXzUE=?OT;dmO{kNyiD5aGn%_ z+%-Z_;0e3d1_#_=lW?>9g%UIyl|;=S?Yl`ha8xJPIfz3SF*Og35elZp8!^}|;L9`#+wUPSK8K*Lm{C*Fd3j$_!jstI(}*&cB=3JS2w zy1~}PVxpH_nu$FyggDb58RG8dgTA7q77AQPj=Nzfqmc081iHa=eISdGS>E#3^Bm7` zvIK|i*+3&yqSXto|7fR^BzfpK?&j?m1q3s$@>{I@b_oU%56SU>qDAT`T0j;*nSZIF z?>t5jiI!ZXM1S2h!I5Ma0_w47>f!*<9CP!#1u7CMQ2QmS0^;h<`S^S;)!p zm@FDTQ5G&#WhfZA_7oJy!zRJcA~Ke}D>|?$K@Omns-{>&or)rAYPeKkYym-NF_B+> z(R0UnvH^z*9tXm)iy1Bu*r9Q@$;te~ELXnrZl`)kLO)0UA~MjC9BGQdg|eLPVp9B5 zrTgKF0u28MAT{Xq*F3PBBb9K<&rQE#;d)^r-E7c_E2_mFmS}*DLb9dSe&co)w>HBa zSveOGH+mk)Siu9Tn;ul*uRnNcipl=(;j8d?sx)=|7I zYEalsGPJ!hnj9{8Xp7Y_s~>TB;hUqYc1l>uSU4YW=c}g(wdUR|mcD#QmS}N#y(W>| z^w;&q=X*}RmbPdTZoQh?%;ju^E+mLJ8*TmbrpD=xJFMg_!-ZO}rZuB!`Mv)c#KNX5XP`NL z=8H()5`qAG6&`Gn+YPrV#Km6r%K-Xx>N5NddL#d9rjZdoZ&m76m zU#v}TTYq?E@*!`j4`~72*AKF7;Nr%J`+wvZBi$oPE`;=*yTAocs9ywFVd{?@qQsW7 zWow`JCGngIke^hQHw8IlUuBsqK!m@IV?vv=1q9~+k^=T2GFC3fRO93};~1VlFA|mn zavfQoO8t{fn;wy%F?^Z0BRLd-KXWZXtSsxI8U6;SsOaiMhW>wBVoQL(QSeH8)mm!}`a8+;P;aZ?X(v>32svZV? zsng3*q9Z}R*IfVp(nbndD0j_J)y{`#kY~N{>&dLx^vD_`o&I6xj+Dej`15P#Bc8`+ zJ2sKUa{a@B5rm>cKd(;Qxg&e~e_q*+mjaRlshP%Fk)LY&Opy83*or(5H6|`}L>-d@ zzWKjj;RdhEzy}pFiu2IsrByo>EnW4&P#TdYqQKz)GdZ}mBZoMKPl74Tmo_?d!we^F zbQ95&DOLC9;-kakiDRU}PwR%KcOCwTdY(e0IVPGGA^57Fn5-)e;@dZ~Ir*xGe`_kueymz)8Cua({i{n8E&pmHo7frklc# zeY$!PhYd37lYK=}$&!4T6;8d8Y;TdKe{{z$Zmn2XlC;h1&_HTxU#4_F7Wv0%4vX;= zy=cy({ZH}90=dEFt5SvfGJ#+EPxCuY7A9%sE=tT(v2x#z%L5>}bk+IN;!OTbv}2Cl z%BK9EJ32#6X(={fJvggPc*tk}Zj-O!rbumQ33oO$KKAzwj;KK^PR1@`rS@oF(Nr?L zCP8t-vRaq5+gZyPa|MumPDz>}PK!0@%+{2He60CF@H*&jZ0U_f6C#)Nnzn3lIR|ag z=Jc`qj2e7Mm=D80>O~A<+)%0`RB9UUeXhkad_+~oeKf-%>C%x_!iM6f97SJSkF7Cr znH)Bm`YW9|!%0~LTTc&XizO&--&qxY7NRSz!z0t%rZR)!+L`>MwQy%*BHKQK-2P9w zd4@xpBuzhu+Kt!hz~Lk9k9KOXPRN@Z0VjD7ltzmx#$-;*!r-_nkJn=IZueWKX>FNN zXu?6!%<(%PUfD=y$7VfeKCRG=Y<;z$RGe_t8Xb-p57VZAq5E|E6d-D3%9!p9MEu7O zc!%r3g_P&Idq-tHrY_Xp5tyDtZM{|p&L}ejx)Ek`a%)XWe70})^*W^~OKKnu`Ke}Z z@MK|k+QOr4={Pyh9IDwWgp)R>{|w$;)2nrQ4}0+?d~#KIwku*{zQ0g)=jp z?+d*=ZR4u#j}sOu9I8j>Roma+UkK{Fpfgj}HmJ2AK2@VHSnSL-Ulp@}2y)QSkdCic z`f>~E@{-q}4+6ENUqi%2i1^gE$=*$RxzeS3M{zzn-K55FuUP2(MuHWQRi;(jg0?Oc zy&oyIoV)L7_dQMkS2M6+V}GmJZC@0U{)JWWZaLTPTja2HQ~sGd6sLjp;E%E$nnsZ+QVMp{26YeAwFuW*Z(|a&#=uf&>&xuBQhsjSyfmkt#n)M0k4?jh)7TH zeP_>fn+#TOa&2ucONl`b&IR#6gQQd`8tXVkaMD{N z2ZCxs00pftIx&hkz3jz(*B)rg|DJUQB{(K+x;0DMB(jq^>OR`9x=>5?8jxq^I?;9G)t% z5fvHSsp-h-|A;%n&z=N^PEok^)fZ^+!^CG#>Oe<9w~)%GhoGVoqv-k|67L(Rz?>{B zqwiE=2O|}V3GS&1aR$=#&m+aYiwzEl7xPaagh~JvSf47U9+d5=txmJfKUgi_f|g!t zI?yv3XV!5Gee;=Zh$FC0nFO?he#4fL&azIOXob4Ci1BzGNYogpU(B}|=;8vqUB)`v z!D$ZDY30nny7a>A!$o6AZKxG_;6k4wq<;-WodKsGd zOXPJ~!+OvZ|MF-?K=-2M&xXvnOs2ZdV{lm9|9*%4pL&TQ8thDS za8B1~Vz9IwmPxlQ(N{ z$stKRmcOV()gIa;Ls}}C}=Jew9;cP7j$FQ`4ZVay$71X?!y+b}|X!&{2i=xXr z)~N@hv7V!jpz>QIiV!14uy{UDaO$7LsubM#AHAIYH(Pfc$D=K^Xy{QywW3-pYKkrC zsv0b*I-9L(M5w23dg|#6LF>o^(g|l;rMj)QTH;}?sM40GYJ{O^rzKJGkkh8fCO20# ztcfI4?0fq!?4134f4TR5KlgLK_w#+dU(ZrcF1Ua|#ZSnL5!gb87flM`AhB7+2k;_D z#rwSk_magJ(}9%eL%@F@LJxuH*LeV?rhTkw(wzh6zW!%Ca;!kK6J&%uas_!BgY;?) zzvI*4hpBE{>{|008h-k)7s6$r&Kb4(eu4G z1yML4Dc2HeEO6tbgS2!vHQHtzn9)AW1-S8ABE7o!TbD350CQ67<>8!lpTt4^o;b*0 zsQD+GSss6H09=XK+>-(<`A|X5zip1hXTLVX#(?%OpuZe zFm_`n;{5erh4vNn944026+BLGe>Udm}~DuoPn>AWe-?O^K#4rVpN6brm9qeMl3&*9Ejj zJ!%t*cbp*kiV#V&flKdvFeZ6G^0lLc4H?XUFt)_w7+|Z!QIKJNUzuXaoyTe=#2HTSnD?eWlyy|gWe@(9@y+c3^p#5F=*)wKh23zU& zG8bTPMgo4zae{QsBLXYxK(r24Yzaf^!%147+kw*!Cm6>NQO9IFIRGyp>xOmmK1H8$ zk|B8upn#WPyn9%^wylomgt`TO1_Pzfw`0miBQqM@_>I9-qHr}{#DfX89HMrxj=(b6E%%Blssc=(Zuww$;cjW|HJ z&NJ1XD%z2oGtE3|w{9mN2Ua6GP;7Qmi{J8@s2P33TD0e2KCOCF8@-@kpF-RpTfvWC zgtqPdCIoJeG{2l+zOx874@yhP0_DEa3A*TLPfK50U%ox{wlXr~-dS(eMDum4s(`iW zQD)(#aIY1hK8Brkt#`rEP67GM?`u8L9h^bmwEH&Nj~2X8FeNh6E8<#*oIb3JaxE*H zRA}U+P+@Y^C6ZuMR=4hEg7sW^c;>|N9u?~g{lEcPKL#Z&t-;j510@ukv!$%vv9r9^nh4!eV)`u-G z!8MIKbNq$zAst#74c&_7JKu#3eGE5H(L>`F%|ngUBPtr*QL3~m(39mlyMpTc7M+}N zXTuju=hZ30m#hlhQH80;!RwQftj^{fE}}m;S6fdnu77or6e~!Im?dazJY3{}J~h^n z%7_!`G79TS0({n1&7ae%-GB#E+j9N4?LBmSc4-}egW~%SM!D3`^r_>&DT8Dyv zP?O2tz(ZQ{C_pF=HNSzD!zrN-W^FXZ@qpd?x~S41ECJrzPvVi+@FS`#T z2D)pQ5?yU!^USTx#?0nS!{dGB`e*{4`4sPunEk}lgR@>192^xAsc1lsKu2)92P{5f zVW)iA8<99*Q6^0Hz`u!8ECwL36%NHu2j$Gqdbs2z^Lt^Z!diai``BwY3W?sKC?^tY z3DP+GR*9Ya>9ncR;&N1A Date: Fri, 9 Sep 2016 11:19:44 +0100 Subject: [PATCH 507/746] Remove extra images --- loadtesting/channels-latency.PNG | Bin 17031 -> 0 bytes loadtesting/channels-throughput.PNG | Bin 24427 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 loadtesting/channels-latency.PNG delete mode 100644 loadtesting/channels-throughput.PNG diff --git a/loadtesting/channels-latency.PNG b/loadtesting/channels-latency.PNG deleted file mode 100644 index a2f7c5af4c097d30c4802da919d7f9250a702f88..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17031 zcmeIacT`hb`!2cwQ3PoU7P^~FQJRG!9Z{(wy@&>+C{2n~DIusJHi{ykD1_c2RB1vK z6hws3K_CJTothzd9O(2L~1A^#mnRbHTWH0$lf)6?$6KzeXxPxy7eA(f2+Tb(l_mg$G4+@+u^DQr9 zjkKEZjpV8Bc0c%%^{Ntn^fhL}mvnK^iGa(Pg%K4uM#$LqQIt*aLOt~(9Rz*bL&pTJ zZ2x1|+<2uMU&3FUx{W0zF>It@=yXT2^1vb!4&4tQ+ZZJ=$6PFP z)XivlsRkcve|EeH7P>x(M^VWrY*E$dY=6umAQdBiQ?+U#0EAZ4+_0ii-?}h zO0-!*#aqm;rxVhgFezK*@XhZnTWXYo;Vjp8_c$90ueOp`P>6*w1voXJVVR!_-y%@A zJTq(@v7gJGut{=mBMvdDQ)~h%gI=EgQZ}y#DGOs;MO2vX@GJKh8?A<+Ul0K+i0O44 zGGJw8g;if7r9txW-x0=w_bB7v=Ag-fWR3VG{!_|f1I?4t~ zVdRCurB~N)D2#aBp(^GubQZ(vnA@JLd^(9SQ{hM?Q+9RMa)w!_)d!Pm!9Ud^$TO-aG^w`fiy!&SuNE?>e(mjT#w|at z@Y&hf4i-*i$dazzSMRh%;rE*NLN3wiG-??UP?*OFA%x=&&DDr49AdR@ci)+)y_?@- z5X8E!Mrp1nfoU8^u8iG4x_=EeU<{5wfL=S7L6TfuZ=AO9ZYttbjF7KSknfLCS-QvG zc={$9v7U(dj$w^a^coMd6$pA-xJZ_;ES3zBISMb@&@*P=BIc;CV|B>qkHQ;{dFR^r z&h@vWZI+x4gvzWay1BY;!h;IQIcj%-nm0M0FEb>63L1$)Y&xKMC8RbluUOS1mNvL3 zhdH)Tua`TNx6~q1rD3&eq<#-${hKhxYf+N3xO@S(f#;zKwT4KEkpo+{R_b3DmswF+ zHq-Te`g6hbiwis06nK~OZ!eY1Wi|J>gW}} zacWO8AAB6jIkb81z)KzJdOse_(c^NWdNI zCmCU_-MGy!4`ntTxSIXOK0Fp?vC~RBLpl;J{wSAy_R^A^SNt-6WCl1V@P3PMZ`tVA;toj6|=udv3yACMvtLQru+aKL^8A1MrurDsxVJp$!&C zDn#zchIz8n<;JsQoU#sdy^|Z2WtAp$y|?M(AP?hiiTTCXUY8_R(CaVIM+$TOF7aYn zc4OEv5*Bt2PoM5N+`sgOY%{4zD7wo0AmLlE+t=KvC-T)pVfnVRHU5>(_NBMe{d01~ z$dW)~Qj~*;&Vm|TwOl-Ywx;y4x9`x0d!Wg)1)+|oF_h1k&?!O>=Ggkn-TOq>Fcm>7 zv&%NhuWic=(*k(>j8pqaIRugQAs2G!N?mq~3WeCSwVbOSqrO#*mbw0}X(R8ztgbil z+{7i~*6fQ>Q&a(_xFH`sqc#)B``xJ4V$^5+)UG5;kY}&Szp!hmLEmEr?T2S4F z$)nbDraP-4;ILIBX%7;{9m+OQ&$VBDWB#Pz8@U69aGmHu`Za&x8x-A-J(1`v`Ch7d z+G;&A{)Sav;=VBADf6)gIE#c=)Ls`&mm@Q{0|noj#l>Hh>XkQG9$6jbfmP#Df>y&SJ z%IGDuAh04wWPdSiZtJ@swM*Snd-qX|BW4nXT>CTwnB;HyySQWsXdUFa^BBUm~fCz{L$t6q5ZWo^U@ba4~+JZ4@Htpu2lFh zN3_t<7MOo_usWREc~D10!QW1VSKi1j;Oq$~;@IO+#{CvVW=o#->eV4P3jBbQcXsas z7$J|;KW9akGuV7oDrD*#>1fvIhUSC%fHFY~m#_N;;rnpqT=fKn@Fza zHWLg3U;*^z3YGoghzHx`IXjtJzK zfw-nN{?tC)9hce^g7g+W?Z$AZgFQ29ibdx2xL{7#9TP-+qIIdI1Efd#ymwULbMQJX zfKh|8=wpL;{Y6|DT-;0ayQagZ#y4jz(MyN5Fg>~9;Us;MuB=IB*Zv6m1-71L*(aG; zC6~!)NOyX`Kmq%$$?700b0d?{i9MMY;YTC6jh-ySlTPkdTDRUIBs9#VQDiij>l{&Z zzm@D}9a-y^6|}KXRVS<+y~cUVi|hQUJ)zy1x?BEhESKvJb7fW>@3C7<8d!Y8z`oY< zRB4aR>ip-g3a&xdy_Y<#^SFcMcBVSoHf^S;Q`a$SX8AG5P_j!#=ZUKyxu^#jZ!ZQN zh9Y9~kzX%))=wftllJWuRdHj;+aNA?FWA%vtw;HJr$lR6JwYB(G}PIb>W9?48y-5n z!fioR=JZb%!HtJMe(QMb0yA1+qH@K?ZigNz0$cKAb2H!9nT#cb;?M}VXmX}8K`=D2 za^rlx8y?feEdFFOrB9##bFPr>?6I5l9I@)OUCG!nw2~R`Tqav}P8+u)>3n{aKKXd- zzQcDPzjTC6pS^eb)RImXsqAS`sN7DWBmO(gPJPk)sK_jN=yD-5uTIo-xY1L|s9y03 zol?xx&A`ZUz66&)&ypfqKhGSk3^vuiloe2wM^tPwzH*vL(YhFB*E(Rv-gxhP*%M@4 zje3hcop=q4!o{ox+aL2e3Lj16^?cpMJ|<>4PFCvAI(Amr0EDtCgnQIZ7c=pq-sJ*0 zC=bER$FJ;9RPAZD#xd*RrS(?L+vJo^1THyJfn;@;p6nrCq`7lka<7T3opN9Ye?-x@ z9Yms2x5L&PdaIyo-B*?;V>}&`Iccc!JzS8n+o|*0C`alR$b;IENYQY`hc@NCfniS? zjc?e6OB!e+RabgVw^WpZs6>Ld4e512R$t7f2}J|LyC!m&ySh?J26m1}{&QP6*&GAk`LV*@#EO`n3Sc%0L}R zHwPES&}h3npI{Q+k74JY#kRZbf_OH5rQVVq5L6+$L+H-!j!(}|!ZubHuqQDpJzgju zK{)X-ucCJ*u<`1Gf&!m?qUXK7Hb?n5b-cVGW|%^3BdSQ5dL^B>`hXo^hE%`9(4iE> zR-l6ayp@9AY_Arbxcld_tHmHG#V<}Yc>sS>Y>Be*07Ha+2D#5=>J|m&?p}mug&=Eo z`q(=4n)Cx&MN7 zKohHiELj+>$fRK|$shL0eRxVtDH3a=%g4AAa^5XbVOAyy+#is83oF*_1&3P7Usau-;${jK|W}mT>2j+`#3p%`t)m>KX!Q6h-50 zsYAz)Vf>orc~^)0T@Bkd*Ow{5)0KjO-#F2QPk^?Em0jfRe-=V0CT{Rlih9p|r&fQQ_T)fN5ySc=X;EUF?om1-y{u4QnXDViC zYU(pu7B68*hJ3$$;6$HC6?LV<>ipM-<250A=xO^jTr&IxTwWDG}un2|c}>z^3s&VP@`I=Jj~3=fszKe*i=Mc+K6)!6LwJyKfwjX_cqA?=s_9J>@CNZ9cbqsaWP`#{Y1j<}Lr*u!p z5_a!Cv;z`iEH zB(Ywgvkx!@B0uJ=tlrQsHRjbD`P@Jmy<6!$UXWN2h41X@!Lt{z)!`lbYutlpu|w@G zENyW=WFHQ3`OD4ZO0-we(B_2rH3R2<;|}X=;c{8Ui*)C-uO0a?{oPP)J=3ixJ4*eu z*C{H@hauK^G-gyEsY@-ksUI4%)den1<8vYF9NArLq#qsrOir$ahcVfnggzCJqihk- zPc0Zm78m#hvn6}0ws?xGh?=EjqDgv|duD`84Ss4vaC0%Ku9nO3$?2Ry3S2C7W|d!U zG9c81LgQ$18nZM+m&{#@oY+2ctqf+v}M3jA)+0-%dqCKsl1%E0Q0?wZOL<+ zXvXt~WxuZL;a9!UY_PH5p>F8>nQX1jsYIxI<8^~G41%ue?|SM8?ozXcS9iS>Y-cnP z3c@GzN3?{~+aIQhGl5OK^{4_=Pu`LaNZ%@avGvv?pr@n{Uyha>Ft$@tl3Z8e=3oI# zg3&8q4PA>GI-PwPI4RX}95kad*VNKcjl;bJUg*bY*CL)ii&*n*5QJ@R#2Ko6uOfkb z0JEI2yrn|yJQKkZdWkrqFc#8d`>D6gp@l8rYm~zN3;EA7aWqZ<24Csh;?tbCou3{j zec{=$x5zsmz)!RYwiQS&B!4nEx2^chM~<_6cOR+l+I0}Ns+lE$ntRm2EL@9N`qml| z_lIc2Np*?vM~6;kPmFpx*QB804q4MVV{84k9(6M-1Ngf&r|2^!E9r*^;LwjyPqJw$ zYW59egFW7w?#l3(?afnDk>jQ7U%aPJ=QLPi7a@Uit9mM}lC(#owV?h| zH=RzJPF;@9%*536t7l>o16L+PtVd))o7kM@ zVI5?aY0iZRQUdT=ldmCShcba{)bOpYcG}E6IfL+ik8{qn z(`b5@ZaVS2UZ&2a$u<}!MU}ECSWhTGHOy+?XiOu#pMqR^XZtUCt_YtF`m{kn6NX;7 z&+TO68+0a4(-IQkmk*0vJ1ot}%YQolrFI4u20W`OK}Cc4A^hwvCbR}Efe#7$p5Ps>I?9FPnWoz@fDc5A?L7~lQn*=Y*? z%f{{Z|7CvL+9^f6V7dJGFM)JWPXuh2SnNl%=y5#}%bZaj+(9{>~7ix7} z0d9T49X3?yhTAWFd3ww;aoFf!ca|h^w6-L#SU0=ebk5`;Fz)38s;q5=klFn5{xiRp zejw&{6Mz=xOebrZW+sy`h9>@3epm$dML((GWV!{JZPpI<8aFxRE{ zPMBbG9wP6y+@m-4p*LE(5yZN1Tfd){KL{qKEXRd-EX~;?s}BC!!F&1{qCEGz7>z{~ zlj16XJ5<-qXt8wNA-;ZUp`ly4;A>0F#<1=Tr%)8zjO)q_^z`ZG>-T@F(`9CXB!}Z;74k< za*wZni!N>4?!p;gSllKf!vJfE8#g@F|j^^m`s0I1dsGtZ)JhIeKKO>g? zertuOSI)iX%#U8L(E6ppo#g9d4cM5yF1)H3hyq_<_li`>A z|J?+_tbSQFUFk0w{T`Lp$b}cnH*4UIz+XgUy!O4(MdY;64Jwv4$!{8$QW9PUvYEyR zxF1=yQF=TPzIz|I_pc!NS6Gwx7_mBar%cb%ZaWjanCM6S)*9FDvpKDFDd;6zgy6|* z=bP%sI=xkGBO5N!O=}WX@O6Y*K#XD#{C_XtT~^8W9}%UYSfXf7MeRw0UNAt_ahr-^ zHX+~7orGrO{FbDtcLb>B^Iv~MOdub_LDEM}-q z@y(#b0dKV8RQ-C3Jck8ln2}`xt-Z3^c90R#c1?G=iyMm{M$Z0`*=p2X>Yz>aJU#aTrzHeoBa6AUdL?+^GM)`HZ3W!F6nHUrt{M%lSI=)>rkdrbvF)^%Xp}&KT;( zpP&qzwI#~rh(B#`E>z8E>pPDe0e(>SNSnp7kph>Wcbx^~5vwJD4;Cw%6Anb?Ma_l)?GDKYSUg zv`4~Mp9hcM+O_#oC_*R-atU~LY!{Qp{_W_dM!BILyn5Y7p%1Vc=5ts6=-7ePyFOTw zR)PZVIU{r|OY0VRA@uDQx5;*t$=&^=?u|$F#7^~HI(A~6_+m9U*-JnOdQMx#|I^^~ zAC@`U+Fa1Eud?$X9g{|1nf$ZkEJPJi8Wz-B-(1E};#r3(T)s*K-XKir*)G&qg3ys` z-M{nXwOc7Iju8oZ7RQwsi0p{9GgV#_ng}|RK3ZUxSqGjo`mXj>_J0m4S^MCY^eH6( z+V2K8WN-Kd?t!ffsXobgBYWiy6!7KUZuC6x_FfsR0?T>4*p$%kp@vV~ z25W(q$s1#sn3ts2CXDn(=eGfI_k$ThiIJ|GmYzG48ZzEyk^}7L&qd zu%qY@%BtXM@_)XIOV$HT@fVct2C1m21$5NuIo`5_IfkX1G-s=-jWIy`=aF35=sCjj zueu*Qvy!})H5{<^^b+~~<7KU|=FOl2cJ0a*Om#W#C1RxKBsFBzEt6$O>a)U5lh!hE z74fr~I7s&dw2ki7Vn4l5K(*582&)D~Emn_f3TuF{bIgkOlViP#XJUbJV86OQ>8+{9 zho=TxTLb&&SAFhecZ^Nes#SKtD=BB~e8nj|t@L!_+nV~oIJPtVix=0ksW0LjFuy)? z50L(;JsP?BHOk}rm-of5UiE{5S4k@!aA{{kPb?D<)Qv_yA%lj|awo%9#hG9AUv zM5mnknoaSqn%i8nR-d`JFRdtcxjWdFARP=48cs@oKatqyv>y}>F#pt^wBtWFYsQEasL#HpsG-E( zIpN?mvh33!Kr!y@I=q;3!R_YQ`eK}lwoxfP9n2%t=6>|0+SQx44 zsV8>pZ(WaTuMd)dcKqdloVuE$c;TJ9hp#5HT06N~eCV1yb{n|j>S%st@=47B0N%>w z)d4%ECMh!GCjciJBmA9`S1vjsEDq{L9#XyF1n&=?k?bVzi4!xFew|XxM;#3&tfL(6 z@q|SSsLo=#t6lq!Rs|4j0!Y|^IbJa84HW$7_~V}KHvEy z+199o#~bU{zC4VgkKHK)vh9hgh4eXpm#{O5H9FmWL%Y}oF5kR-y!O?zV)pElDs3O* zB`9{81skAfoIYIkFUXI zH7&CTDqGG98was6W*ki4nR@JXmYo!A{(O^DIzo>5EJWYA_lH!YG+RpSnwZl>kMM_8 zEU+CoXRgzo^3tuiCi~d!ed5qetzB!2%M0Wv<0}uW(g0wAwCHr|2A`l508dI)w7;je z%NYO!e7vR2_qYOCzHF__pzt2iB?LK31Jwt*RIat1T}&~a>!H&bp0Dx2)dG|_or^>I{YS##BxG*aL z6sO`@i~*>GLa~#MnPSEU%@yGog&4)6pa_`rjG{(5-BU0Q60uc-T-zLUh^Yx&o~i+4 z{Td|*>NNQ|UUs_k1&<&uZGhD?&heCX`M>_le{{ysXSrAo&>+ibAdVm5n;5;n$M+er zYwUm$b74LX$}0s~B`O#M2ui~nrAFw5B>B5UW?Ed#a@_`saml-M^2y@I zvNWPL#ic}DNQrl=2U&NfiU3CsY!&l|frmw>ZcJy$a6VZ@qs*~&FA1Q56GIHCUw+;Z zHUB9^wXUlstifF=VEos**nDVxZH?pCA_icg8>l&rgU9?eR#hEVURv zE zK-A;n*X2QbxBV5crE_W8ks~uN84lE%NE*2pa#x#=&DAx+8z!q|;MkQsKaY)uIFI$a z8O1HttR6ibN>v_n18s%4$dAeeVjvIV`+TE|fg|>6Kl4aFh(Z1vY)EgKyY-A&U$F@I zwswg0Wh8lPl@~oW$HuR^aQFOrP+;k=XMra@`oa2E&Rl}_Ky`cF?8q1MNEXtOHStB2 z{*qz+DmUuhd!h|1xJHM<|9&R1{8e3jvF<3D3o1Tqut@>?(9qj%*GnabDC=qWJXNJTaq*0 z`1;*6xIHC)jJB1A7Kvi$#kgcD?khYBKfnhP*9hUpXAEWnKV-ov2hFHxD1C!Dg|o>r zS6u&m0&!49`_^H%LhhSo6~p>*$R1&3KE{3(3q$+;(~4!nWa$z>>aDS)(+6HDdJ~t$ znewf;*zpLL%OM=yQDk4-Gsd9k`Fm>gy=VktpvK2Z0Z#Ts7#x!hdmvfZBMyF~GCd zgW81JH#o6NTW!zGA1G^i?{7NlHFd?s9iZyfWxE>*I;QN4-Z)r{UR1pgBE^U3+`2bh zdu26GfT~HO-D0g!zVxHoD{pV-vz&dQ;JGTS+;6@B>O~9amMC5Y@^FDg(b6)~)eVD& zM9bQKn^_kQ%5te1nC&Z|Rl*BA4m@KU8&G@%LLqLb%98*b%2R!=pJghmYHWr!dlh`c zAHlm0cHuuJ^iUuy|2cgptHQ`Q`y2{|3XrKMJ_%lYyNhy<2KHct(RyCSR1a(|w^n$P zJ!l9ri%+1LZNJk8Q`(}y=cS8q+GlQp=A0JfO3z4RHun49644GkNO#*xV)0pgzz=1( z1NH8(4f!5HgGJ2CL0sN=!0`%t%kI(<)<_cj^Ap`iF*inm@}-Vh-R;cd@>Ae6_QCe7 z7$V4~T}@vLZZ*Wby|b&1hT8>aCZn<66OCk@ zMH)qe-kg*v2$E$g`E6;!u247AJ<*+np~?ad5V}3)LbY^a<@%Xbu1zNY)N-CMS?XjB zF2yHCjhs7NVt3{+3YwU^#z)4ZhSWc4*3Mge&G3i-mDr*xR?t!bIa8w_PAIM?h*_c*NPEX zV2^Y%W4h?@n+{J4Yt#x&gG7oKPgmCdn`X)oE!8~t zaDNRztMVf(j(2_!YCD7$$UO4bpB3cg-2jD&C1DL_)kkYWbHH7G`sMIBMW7?o)3vs> z9J3&C!B)8RY^tP6pk8+qcJ|;)0H@D!FIMq)3GWT_UE2twW7tHy3@v1N$%1^y|JGrj z--i7exBgQF_lMC^(oj;}SPfu`Z?wyNyZifu8B+mQvf%^G%o36|dMe#4Nq~A@!`jZ1 zH7!6;`P-Pkrq9#&Fm(dXnR4}5%FQpk)j7#Pgi|I5Kk)swl_73EVZJFs{!!aO;t2M;t%|NJH_dkK1lpF0!LUF^KZKbkVmf2|9{K>q+D`$gPq4sVUPq`d%{2iJXitv zV@pd*T2nBs3zm*yXZ0KkJ<2MmHUb!&T)W*%jlbFEj#N=l8@t1M!tvg4xl;`2@5`kI zjlK>H{oEQSh6OE)Sip+I#gjm<@(Wr+hrtK-e{8w+xB_}TDtPL-)8+s`*8cTDE8e^c z9>glkBI`*qi9m35bECL_``sj&G}<$bg%M>a)rdv?T1niyz@w2>Rdu)$YFa@0EB%-k6#mrw#yBUk(RP zKi&A;+a4vbT;Cuu(k&BR9i=?WgS9Pu(>!7S6MgJR-BWHMnl?%$h7C6mGV7?%WneKk zcrUjDlmUpoy2&oN&>H&gC01?tsUc$G)fwC+MtQb>%!H3oTO8+M?^{2OT` zPk%3;FeG=YFY#}ed%V?;ywj5uY=*FWO{o|%M;AvdP zlJTxXRD{P@ImFl1D8Vlsef*wQR&anX3q{+k(C@+PXho&J_SBi8ilW*_40oa9lb+x3Wo8a!d7xquoote9G^ zT<&Sv)Hv(7B66*=H*G+)w%$R&x!QdDr3v|~p$~!dY3%#<;>wd|2h#(vcy!LL({7&pD{Hjb9e=e;e`$F3KKeW->s(bUtSm);Kk>ufxOs4-!W~>9=w2796RyJ z`Z>5sseyQ=Y5`BEE00`N7%|;`a!Tp@X99Lgka}m>AJ>QPeHFz@vJeEQte~c>+p`q7 z8XBE{=vvWvFeHD{>Z9Q-&a}7{R+5=ufz0uOa~;V(+6P`QydteV2GYo#=$N$!sIXFJ zCs_(?Ijxc{-c%T``^oh_nYCbZI%ccr8ch}SlQ+GihRL>;HE}Qcn-c6F;S!-K`iVZ?lrDRk@;0eV?pYk~6{C*YZnQ$1Y%c=I#$hg#6Pz z&~udiWnVO=!}dB2(lBX!_+cUd#fFyB@Q`yoopTSGYKv&pO|};bL`$oBPmQY~1hRYT zw&wlSyUM$~3u4)SC>;xNbK6yPDlx=%6q!|haS-&r&|fGsLd6B60&)Vbe{b_AZc!;G z_s0P|5{E& z|Dmu~W2CRBfLl_Rdc`3cBQd}J89p{rtUAAH82Wng;J?n(DAyNKA`YDmSpD!;i*9=4 z`%vROqf$+^AJUcVTz=RN(3_?oqmWjZ5%UkFqE|%^(ug24OK8al@Z7$;Y;JxWyT--= zI`OtucK`JH`)goF(`xBCv~Q(x{pEOmNj0|^%IwA$mq)ULqSs5%GzZ+=-`ijv0Ogw< z7W_HX;Hk^B-AS+l+fft+EL4Q1;7hk4v(rC1O}z$e5%s)kA=S#h(?(5CJR@n5;mTL9 z?H$}U3wiKJ0?k2u*hOy?@lqwQJBm-Sn+6B^`ub|_?-Q#%Vwzp;gX@q}`SJz0_~ZK9 zw&MeFncpXLYm0uy9e2V8zEwl1I!G*H+4#P{ByK_h_H`m|d07TCUI zDHyAuk~YLU@SLjwO=_h@X0RC-=Hg}p=sTBQp zQ5>EdKm~v_&4Vkf)^E<@AgAjC0zd^^=Zwwb4_GUJMOQR)3PI2nVaWU*=-Jufygb#u zj^_?;{1&$?oLpN0sQJ+whhF$ z$J!$x7sa!02rBy*z5s|0(E(0?P3~kDm{;ha9RsmVYXxtn^-JNOfW#iPMQRi^S$N=7%6hWPB-_pe)y(RSzJD%=L718B`? z@lOgb*L266=L6Kf6s)#lF|n-UVdhY&LnKIenGjj9H;oZ2ltb@8`|zx%=U~ZOk~usB z7jKzAmK92!30RJw4a~brwPGN4?vm@TO%7PD!>kX|gN1o?{K6UnOR*=3Gd=Y+%5u8k1^~gVQ4>K zj6vHZS0$|{7}RL${O0=Cu`)c-0>`d|DT6bDXq0}Ubd-EEPi;SX$PxuRnP?kY!z<_d z6m3-KkCn7-7^(Ce8-)Q@f6BIREr|36kRw)ujHwe;P$Xz3#iv-> zgf1U59y+G0Jzf=Ce>8MDpMA25Ri|z)Z=ot|H@|?BRWLCF3F)Ho&SzvfaKtEJrO$GM zo9u?Fp9%U8aEx#^K>?_Mx7mOwGfnceFz!>UB{tUGrOJ74j%jym+|L} zj1sG*TSk$vTt!#^09jw@`Um5eQ>sKj_h9a);=YS)BqwZA4AaM?O!IN?X%hoAuX8TW zf#Yv(h2Bn!R^n0cc*%R*K3wpDwBK2i)D{C!)}ggcjA3kD4`gGdcqY5Pbqozzk0B`B zl0Q`fp@Du;Y=+U7VwqMv8Ucl;SEL7m0~!>sijLN_c(ka{&Y_iVP*7g((A_tUI+2Vz z!|SoW)Fy+a9sP)WWm#x7K{(@JofZj$K+$`}N2uaE>vZ7N_*qg!C^aHf0hX6yJ593P z9vs-9cO>Xp{J-E@@dmUrkI>K>ttt=trKddGW_oh4w2D}Q8~+N@}R80M2pSTLOWA+#|&v zLj#GyfN#Zuc6lB=9XOTCv$PdoUq`WCm{cDBQHD~W9fY;KXV2dE)^)VjDJa8NVqe^<|HvFonZt0^0 z{7Sc$&D%7yK98j{xP2$TmUcVFpmP>gv{gX=io`LUQTfn*_Hy5$v-V zfXFey=dJb83UfXmfP2BxI;F~h_(x(2BaWHRI|-2mWXlwMrp}|U)0%9MIVm|^sw>}9 zg7G&1?`v{6GoJh?D0H^Qk91XsO5&PzYbg;0rb@u^o8!b#=Og!z*TKT)yc;Wb-I4t z_#hlp4Qz*6JM=#NapqzQvj%yDWxwPiuM6XzE;s5XPF?1DAvh;F*<_qIz~$a)dvk&@ zVCvH$TFVqTXhT}($im(AGM_)LE44nFIT$=0%Gh%zl5HL5oS4*zjR@UIe(n#j`j|zh zd1{~29;7hw1}OhG#E=93T3mKR9@S*t2Xa=V=jfy#(wLE1241G0c0?KPq`vWdOe!1Q z+p~O|o??ywJ;-F%kW|(*d189hD=q2n3cK?iF~*efUFwU%2Zm0~`c>XyPt)LqxsP*u zcN6kB<7njB;{#nA5;yOs!Jl~ZONmUhJIi#;Dqf5wotgvG9 z0L}(Ufj-is8obKxA?7{V2FrAshtFiNgL9|mpgNY#Y85mPskO7hu3sy7ZE?Z_lrUKJ z8?swm!?l*R&Zb=%slI_lvl3krlQm=H%Yra~R$4<`ZBLTh3Rb-F zb{=^*<; zEoE8Hvvy|@3vLX#k5m72-|UVHmrF|Jyj}9}EUG|JEGamBAX&*TAE{B$;-V1iAL#NK zobKKn;bCMn*61l#s$Zq`id@d-5Z9}p0;S(gX=KTHVZJ1*BA6|?q3@x6jIQep$zoo$ zLZ3kEk51F9?_-LYqD(7hfytjf%=aKPny1V3rtYfo+p}~cA*jh!b;|3^Z+}a6zfpCU z^L^3LokNSRHl4Jva>po1{)%AeTAI3^&OST26REo8={4%DU5_+))!pCpB*@s}#R#q$ z`2&5%+7o`rpxHdlweAxOqg@o{Q$vGua_3^Ya^s^fbO*Z$^CrXx8MqH}r-QRFb;ybd z%T$-KhI;ltYTZ>u^RmV+m15Ee=d`U|GS7{@Aitc~E=8UW+FRSewwyt~C_Y@Z zvN_@*t1R$ldyKx`g)r$AMW})gkiCbZ#Q(%4I6l_ue!P!#$EUKuXtKy n;V=+5(e!`ze@%GgLuEgC>P)_9g!vf={Lwvk;cT(ymD~RdDRolz diff --git a/loadtesting/channels-throughput.PNG b/loadtesting/channels-throughput.PNG deleted file mode 100644 index aedd24d51a9970026fbc1ad05f1dd60a7a8155c6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24427 zcmdqJ2{e>%7&n|!iqK*yYkNs(A;J)fN|J0bwo)0UA-l#hB2+>vW#6~KSY`&JAqj0V zmO*0+*~XU4*v9bQBTD~z-}jvFobNl|`#R_USIsleb3gZe-PiTIe%J51?_a#2#myg*{s0}qRlcF#wKhCU0-xrs*ovdsN-PrLbd z`ZK$qTbk(J%j-^hnQ>%->#gI)bsHYu<9Qcv#?JouarxJPsFNqRs~x_|F89z%^ZV+$ z`#S^@ySnemKN1_ou0G1W{)5ir;CI&{KH)ZUCGKO+v$Vl6mHy$;DYTvO)E9ss)3MRQ zGA6>FcNNx$9nrWO*2>v+29rHSx+L@JySjf|<=qWi+h!*EowdsJ55R&(1k0vRAH0$s zodt*1(1Pm_h3$PPHxnL8*Fpris(*6T$ssM0<}NwB9bb>yK6Vj5w}z~W_|*?oDu`1Z zL!YBh^n8wmL@7C`mBxi3eLFpRxTpG2^ZlbT^W8FSf!xAQTKVq@c=8$50vx=VfSRjD z(Nx_LwrfY6cEHxag4a{d7@Q|r6x`5ZUJNW_wwAfpsxG|Z_1&U=1!F;MySI6FTio3H zxM|giI2CUS$31>YqoN^9SqF1md?Ak)WnCTW8i$%MMrGZt*784H`g*RML%b-*ykxfj zQf(*Hfk@tQQ&T)GcV-|dY_@ssW7(XxymQ~D+5Rve{8}!7qS5veM)h2U>YV>VR>G75 zQq^1h?v4aZf=`_eW1`MxfsFC#?R+|lZf^86%%;s71WixuW!66$Bc8p)q`g#``>xTP zq{VBgN-U*1&rLg`l+!aZY%hjcF3dJ_TkP+w-8a~u-roctz|kU+2x1Uy?3IPgtF&9< zBTvoa+5Dh)N8Hp-mpoMM#D&%zgs#hIS>RKrf8 zRFJhuBKQpDhu_a1a*MKh~0$W3t*;w-tR*(ijaJVRB2SOow-bujaPY`_c8ea-PXx$@02UT;5Ws-u?ZG$PR&i~s2pIvnc%23lmR?5g>^va;Vd_AURsd~14B1k9h$h>o-r55d@LqdEER^lCMb}((;r*1-J{g?(c zAwIj=jyku-_t0+GZarAb#9D-?=l60BZ|n3P?^)V7gD-WjY`xtn<80Gj&)HB>r9QV2 zO0Tfz+GnISITpCX>kJ`5oQEpeM_{_v3-e6oMEd^o))6=v*}l4JZc)X?!R`}~{&c;h zxzdtR!pESHH~(qDwrjV*%nhIel@CipI=R z1|#m(M9DK4fsv=y=$O%t?LlyQBO%^A-YR5cRW7;@)zwg#ZbkLU%z{i~Y$2(J0~^vfZ$=AKxuA7@cYuhOK=2}SKC?u>q;ca16#quR~P8D$ztV>HU4 zbljEX&9xy;YIbbrS=vm0O;mpzs_RX_JRYAGi~O)K7wbJ~f>LUf8SEGIam)0lCOG$% zEii!BV!omU2g>qeHHdGRZ)y`?Oz^JC4~-8wm2f32=q<_D0y&ss--<^R6GMbsNoOQ| zCBL)JZpBVkukoL!<*YN_c{&8);YITqKOEoB&4!x!(L1lQ#o=307#C!83o7Kh0v#!N zU@h|Ds9*O?jdIdZ+E%+71;FmuK5|4&B=GCgkx3Ddg!#vlyVCn|3sYl ztfWG^r(%7H7uCYILt|S#f;sHx!TrQ9@-1BCjcvUf+0eKc80CzW@cg-iF3yEn%mP&- z#%DX{6Fcaqp6&kdk>a?Eg)G_mzA2^ZKTsOr0 z%AtBo95v_Y@15XtR<-PVf&O>0rQx1IpYaKc?*5>=jKSL}E2m`L_Oo^;_ja^Ytufj( zsU>`UL#?od%;5OArbe;Ej4KmJkJ*snEyv$pi z8^$1}tyjeao1Cj;PhH(K&RHt2rXw7x%n_V)0@I{ci zsUz(z%I%3u*~PSh+b?M+dgvRc>PnNPX1!KDGGs^W;pFzjI)*2&Gw2RMT-q%qHCWel zm?wmr>_aRgq0?I{Ct*Ric8oW_oBHx#u+Hcmmxhu0&xJdO)>4}7;A5}&9x>q2OG?^C zbcm)V&JN_&i6NMB8h4H>AGZuy=uw@o7~_)Tflw;%9-zNM2SG9Dt{q6TX|A&g6al1} znKM;<6dgIZ+ESD&LGMRQtKM1FxT^z1IeOa@gNUn>dzo`j=DGK^%gj4iF0MjAB){`K z+sUTQ*vAKChwG|L)YphQi1aKDzN?nONe-xj1)WG*m>*l{AU8G5eocBQ&36xLIfi+5 zgtAlQ%p4@JHq{%+1Bvj%Rk^|~Jsf5yO6E6WrhddN%*6S`-B8&e7#SdRx4*$BL`PTX z%_io?aBFOjYFsw7unBd%hA)XQG?8IXNjuVy7<?(ewkf6zcX1y4i z!bn5bW@Ju`l6w>D)dl%8_nF?(j57>=F`&YU@9NKLt#fSK`S#$z!SDOX_!rfca%uBh zEHt36s=C^mdSt^*YLmDw8fw=xJ|2s}UcHM7BX7f2czBZ?fFHZRux{_1c*0Hv#RaUd z&XIocRoAww3Ex+*5JN=ol_k}!GkCyF;XG~!)36G)uQSn7hu!boe>35+ZlvqCYSsBX zagS{`*5+sSNbF%UK5DDwXkEFY|Icg<67dc%HvBhsBANYjaDYb!u47G6< zd@b?z!1@Q$28M?pmge9e@79|Ay1Vj|;uh|#v(WTByEMhOVMCC+CgwkCRwwH0w20fA zfu}S#_B`Q(qVC5=bP4*O?9vld*axXaF`H2{jBkt?*}Zd~Syl0(KDX;jrv;eNb>lOH zvcL&G*2;#jTv^1)l$LlY>LA=s3=tq4phkxcsSw7V?9~?9D7s;)L8O@w*EeCL8!7D@ zTw0xcYsZWq9v@-pQ|!|*F#W1L@phO4k#4lm!KWVQGi$3Vd4E*gF8+JQc5&K%7h>W4 zt^50khjrs1oywopwEgg>G;&m)TIkdDTkg*vO1%(6fxQS&DHBU#z!UbzNw zv#o>6W$ym|*z|?UQ~kZ&mBB-CX{6e;N8iVe40h@p054S5%CdH0iQ}*#!HrDUgL8*y zW9q20_fQwO`|r<3)ycfCj72T(*=wfjr)Qm=rwdRoq%SaN{j*mCzHjE{IHQ@pY86aK zP57nJKy8z$@+nAM%eU$OgJ0UTIxX?yu8N?N-CCjFCKcQVNIT{28V|6d+$@G5`i!v+BrFGm z3l1O8^cSCd4ctL!$l_1-Tkd_+B?=?4GJvuvtnX+s>=;wC7TMa6>lX~@(`LhG#T!J5OXYsKkJY{+C>1hxe{px9U!}rtE@k#cB)iQX-8UKzisC zN$jZ(laG9WiLO4UM#5|LSOp`&ttQhmAa;>r0kD9e&k@wa-4KZkP8 z+-zTnbgl%N6DcEqgzb4)vxj2`m*)KmSe;8fU1dr6@%1c@#A z_C&iaSf=#F6g_jZny{E!t5}G?6xDWd1}05Cb>hrk)k1LJ8X61tnEGPi%{lfHeKp09 z_uY3+h+&mn2EMXjn`dR=%`e@GZo>n1c6LI_7f{~Dk7DH=ESOB$Yu9UIA>5(f0J5w2 z4mpB6ItS$wFz8S^Q-$a(g^Udu?)E#b?1zU4nL!&H#N=>VIwQEr$a$;pg#4Rl`LO&i zaGGZhyHX8MYS&uSUr6&|$!kR4Am=W7M2Esr=H@YFmIiY#j9!%j{L=nWrmSsXhKpz+ zw+mJQa>AtYUnXJ!$7CR}AM^G~$hz#Yw34R3u!on0R#%3_DBrFR*>>cOlvz=3-9)v6 zmL*1LGq!6pcBJkrB{6n!>3~6`YV{<^Jyx%;e3Wu`P`%o;K1qk9*~hv{p=HHYl8VI6 zx+E>{>5f8&IAK+vS0z6_NVek+^>IDr`+CYGOys7X?&5|6>-6|KJHq3?yXM`|rR7{Pc70ap{Eum+ zOXNR{=MIZUgjJOb(6M0ESvy(QJL)*XslcXoGPsVtwa~$R&AN@>Ck!}nIGm58a1AaS zqvSsNN!0dyFzNiuqm|}fq}W={7PGFuY^oI!A*#z~_8 zG57}8AK%W(UwOk`2)0t2SpOQVKtP@P;hivXg z0?9i8tPVj6iZ8klqd3#?T>)ZLagLSgupCgCl*OwS0LP@=!?71Zm=+8}k7`V-8>x#oqlPzqCLFyV>r)txh z>)|)J|29cmB;Spwh9uFHU_QBLr>26@X!L}}rEb9s`$GDgyj{kvi*6nDIg(6D3i-rx zE-QD9PFNL;S44qS8zw3UqCtnQpn`o%ZaIgAPPzQGcbinbEu%xjgynO}4Sa=ai9XEP zHi-0k!E1J=Wth_0%sQISn{9|||J@?CNcDF*k%=-HT1pS=nx^6A!$t067GMX!r#HgZF#!IQh3#Pf6FI?HPSO4kC7tJqV1jg$mSEsJX+4(6c zk3z6P6Zs2Ni-_v2%GTd{LH@FRfp#EpTi#Blw3S7Cj!`Nhqm4@@`fC}Cpyq}=&oQF>I6V?s z`~2j8$46|kXsB=&bJFsG$kC_rvqidTkkm8)=l3;Llst=`|6;`0-I@C{7t2FgQKL4@RT@b2R?&Wfa63jpzcy^ZFM~y#wtaM0(E8^qW_Qih9Gc zGcv|IWq|+SXile4Oj#Jq*x0z=0YxVp+kgLBUc*D7;yPa4E-1^!0E)Yt!ozW;&e<}F2w2EJ57x70K}N{r&)66tVA>9B=b%54jWPfb4GX*nOMeF_;W-4|z)3`RXa z@FDDG0n(TMBEOql);ADUzgHZ_S{_5OgY0&nPGn=Z7~)HbJBGIv2_fcKSdQtvdAKJs z;gsdrtBKE=7(gB8u>WJzPz3`+55yFuy7d2#MT~1jO-N!m&N;p>k z^V*Cl3l!5;we8d~e>3-XSD$G2%VwIg&om~kRoiUd5OrgVE!RKd!T+lm#o0()(Ivgy zi%e*L2Ff2LY-`rZUfu;EIorh|q+Z7AmNqvg$|21X*4ntR^?`E8h#;kI?u&N#A=_Je zH&Xr;y0y{L7hW7L*R4srm_-&$(~0W45UtQvMs>R^yYIfh-8#P`rn?{#Fu2=hse4al zVDj`x06Bi3HTpnM76T~cP3dtRrETGMn05cMrc2)Ze*8DdPutMOGRFG)JzOj^UI8%n?5GvI+?#| zqNXH!RYw^LHT8x|rU($I9qnN6l}41BZu%RArfThUCB>_8tJ}L|x#mIMo>hbPpvOwa zR2GDE){dlFIFGg!+c#yI6hlL7&F}){+vV(@DLH*RWAEU)OW@EgqUV$wugY}1c;b&QHgPk#-d|te$t*hXk|Eku4fV zT=%!%y&YGCYYpq2ycYE4CRPh-$#3SWmMH6b4VHYwh!G562wNXS{&=#hR`{i-r>Nlcfx%%I?gWFrZ$!p16*rW7&o7em53~Dx?70l z#K0IK7s4?tzK~x9DfWnKbx*(I02jFXNce{6X8NA}QesIK)^ssRgKD|`u>BCQN1mnL z5H%_DA@Vj;kCcH)Y5VY8(%S)lLtKy7eR-4o&craLc7$&+8aYD)At9?L^S0VJPrchm zHoj6KghV|}Z#L~x>QttXL*o(GMku%(P&f<010G3ve41On?W@~xOIO&Syu8j?=S2>r zuJhYDNRt^Whbj7%`J-KQde}-nria*PD$E$)PKswL9E(iO_kh8S`FnbsO6rgGNlO3h&PUR%6M zrSW9jrJAVUsEsvuB~l`Tp@3?Kl z^w_m_%(176tlK7*(zV-EQ5Jy64r%&j3079OV(~&Fsp?gOGEH*pTM8ksAW!ut69}8I zA+`g7^Pdvvlm5AHGmz0GE`!~ePv-cWFD>)VlwbCrt9X(CpgzTtns;f_ABnR}P2@o< zIM48=oDD6xnxx}+0AS@kzJFbJ$CxoYIWpAYHKC(gn}mGjP~0n^I-Jya*nHr!1Rs_j zw_TX^1BG|`FlK{B1si1U!Vv`KY$os4G0Wj~Z#RQ6dEBeAfft=oNoe*26T5l5Ai zNqLt@NrXonW9C@vt8zk^YzD0C z8R39V!cW?0#XTJ}0R-hTY%3uviVaU4laH4GH034n!~dEJ(v!QtL)?mqLm3ek$6 zvC8^fdFv*CyedWT)%Vajgb%vwH>={We-!B4oMDY5CTIi^Kn6Txjl2+W+q-7h5-#{n z6v-JjB>QGd%&u?Ik=zI4;3NuNN*oO4f50D(lTZ$-MmOL7@JvGGdG#saB zLuo9;lfYz5F|4~zXpAA7mcq<$2P0#tQz6na?pq_$Qu$F3TLx;=@&HuMc8Sa)S*UT9 zG7B|6(IAC}Rag|=O5hzBw;qp=8GC^rICS7jaNzu*zfd9A?Pr5O%}r#9vJY?h#zg@n z*yI&cjT5q`PHCK$rD@o~;mm2ffNO{=?}FZXVzoC=#>GammbjVU;L;*J!_P(8s<0QB z@t(B^^Ly#r8$q}p3JN{|bEh`GCoar0=JD!SX)A-TMi-kd51oCwuXyYJ3)7B`rfi`V1GZ#;z* ze|$PtJN;6v@gvb={Hp4j_Ye3by)K4+7E2r<8(Y#Lttsmebq^?$t~I#!>H!e^@P4FVAeYdI4W z9$@k_L)GF~-kX{V|TByTf#1L&%49_X9 zl4*_wBy70rc{RetFt=x^oUedWV%Cr41B%Lhs@=T=l=5d=mz~0LVnMOf>2bD#=5)Wi z4QTHn4l$3{GHKG|xAUSR_d_)_dBl_lJhPwrO=`;!zj-F}=Lc_~r0K@F0czQKs?9lU z%$AqI*HrOi_^P$!-wJv_JyWbYssI3em#jh(@RBSS}bXBq>@qj6*GVXNm^q!Y{ zuz>>PBRN`OVxvK-a|xb3A@V=lqq84gefF7z{@m#ocNTFWz2a>C-9IrPt|~G-|J}pA zpBrcgPHK5#>4t);)Lmz6saN0a5W@;en+>EdGUJr1lTYwmNGVJ)#m1HU|H4=aA z@pkR#w86UR!8>a>X1j#8NQX9JeJs7{hI$Q*v&yh)erjC@9uYMrXPn+l^N}>pp=oFV zi1fIG!x~%1$qM9495AuzqJHqw`=|`vOP^A-Ef=DnLTf21x+%ggkU~|%0w2H*xnIsr z1@O=wxSIJFhlkdiu3bT5NAyOTg2;A%Le@MJ^Tth|*z`Oxz=d!6rIzqZ%@wRr{^bVQ zpNk+(KOFRYC%SHKX2N9qmM<%*qkMw~SrFpDN~zRrIqEq%oH;~pGHVlZa+;eSZ;KuN z57TIpZhKgDd$Ya6!q}sD=+IZ?|FZ5xr)I3_;v4=oVdV8f^I=?PnfD(vn7hSV0tER>Z9VH$#sO0 zPC@?93j^ZYH~mTF73A2K8j7@T`TQ4wUyyEFzv~#{2xhPl+d2^zeDHlsha5TA@?# zIrqVo2s{X<1Eg|LJ~1C{$+Kojaa1m1ObSxV;e*~czvNeP%mD)GTWkj5XNcN-^=DDU zLPCywerJPa7Mw4);MKc_DMqMR6*_`Xvf9XZ^>C>-qiv#Ab&ORd^LNnE30d$N@2*t2 zsRhIn1ThG*h(J(8F+kDM$)AJQUXN5t%|y3>T)00hqYw#Tj6J|ftlC@7BGdI;Eg&ugGnb-lyfw&U5ac)VjNG?A0TgH>VTW1dg=y?!!Y z#h^mS*eR@7>cd1XD6G92&qLHIQ7Xr-Mo61`@glt*;#a!^&1G9vh_G|5LHh?GEO`VY zUJsavQ86G=QXSntY%Jq$v~Y+dw9+9aN8(eKMi3&qK8Z{A+Nt96#3oKMCr-%7sgmD# zI$<|OT-1A0jO!`3;uP?fXHfuAQ*A_V13q)eM&SNuMJMnlc?F}94=OI-fS!|9b|3bs z7P#uUc^19^d{O(tiPg_nmXQ+yWw-;jzS?lEpz1ecALPwe~unVwq%`=W2 zb%)YOS&q$$vDko&xx4Z0tA+Mpy5UyL{c&F1)v#j%C9}UL()MG(=^K>Uql32Hzy&@} z(PAWpsr<0cm$R~nTGt5oCFX1+w*_-{00F~I*`10XSa!BlFVv2}KClLfZd(~->dL5C z?wi{W4oMD?Hm@C28hh2SP~7!?f1C@5L=(mYs_uG1q=9}>{L$Y=gq`^z^;C?7wI%@q z%o-NdsIpZKyCJMsdwkKFm!1-wmNCt_cG#CC2zZyD&w(&s^?u7RdBr%QQhnQ_uh(a= zD*P%s1#xpe)>k;2Z& z2{_!sLCWRD^Ru$R`b_+Y$lq0ETOZ)$3JaV0=Z7R&rSXQ8&1<*rI}bZ77kBjK4Ys=a zen5O(9YxQj8SZDXg=x2&jN5t2ed-cV+Yi>K%ro|Y*f?&YcgG?sRp8TyC&Sl4#kuq9 zWk2G|MX;V(jh__sUlcuD@+K#VsE@6n16!|=q$fdPX_SnPKm(G#P~M?qSWspok@6t` zKrQ!m~?{#j#CpI#q*@l(5|Lt4-7WdJ9ZKESqm}a zO{kh7LcO6PC!VxInIC+P>{&Tl3uun*rPuh&`(h?o%91cY`)yA)8;`~_DR9ip`EBBw zSr;fkRXxpP>e^(NHE9#)-B*EEyoqKezRTwrIl%vPZwq?Oa`g=()#P%$rp{os+w0pb z8Nq%r-=r*mw}gbtra{KOLL8SYY@`Lh0QPVaZ~)%>f(tfMx)QZ@hSIK7j0294#SAK?cZ{(rQJ}UwGsq5XZBQDp2~$8g zbKE1;&VUs%PQFMs>kN9SNeerJ;=H#Y;Q!;7SHks0hAXik#AG@sb*>8i40zLu0_3yj ztxps+gNrfwmi2yeTLSOyffQz0mURTd%{Sg=i5i|u8qapztWil68?`e6&oDVHTXL_D z6uNF?U@Cp%ve8+<(HVhvU@vLsfL$!EWCRtRYyLj=3f)xFt3OIh>Ej0w847~ioIXm8 z`tz!MKW$jlMDvMEe?|HcWfIS&1ahd$W$m{rxa~oOYW{_fbHg*)W(&{k1tF%6^h&u7 zTuP3c4ce2V@w(~gVDCl%7Fpgvnyyl99C)bkCvkZ6;|DP;UAV!y7SbPKQ8wKjYz{(D zo1w9JFQBZSmT0*VL5a}EHKo8t*upHS)ihq~V=Wx1;Y~)2==S{N=@!)&CK%gN-|P&= z!GOh=ic?eunA`lZ1wv;kYxo)cbCVZ%(3+6yx@S^CVa~&sd*}7B$?@pU58aJ$RZxpA zWCdeSP?#DGRT@nfKW^>)Tl+-JYBC6ZZvcJG!&-a&YbLDNlKU7)5ecV*1!mA@JrjFx zkXh_xkz>!CVVhDeRC+ z$=FaWclT7!OOW!7rg^>3%zc=Fd@d)VY1}Cd#eb4{oAez$K5*sz#lK%Y18hwD_lx0T zQGmiPthfc%@-F%8(>5SodpI!16ucmPc7gl^EzW> zNSt7KzHCztZg6r4t#baAsxUM~en$=vhBa1X-~R}0P^J(L==n4HKfUa!aFEbAc@8kT zNbJaX9IK>=F@@lCn}UNNa84BaBc$e0K0|da-QsDZxYM<`OlBe$tJL>H(z3v)qt_n{ zwG|gE)SR`(7L*=MZW=KT%MV{imf?-e$pv+`(jb!rs5OZ$oS2HauiId=(g^G_GK?J0 zbGAg5%D{j606;g@J@b_B@URZH(;Y@_r&gq3mY!BE?ukqDF} z(HaIT=}l~AVU@Au9yphV%ASrS^K+!oDVUA9i9g|cY-czPS~pd;;9Ur-OZ;S^NBIr$55XfHUefSe zmhE=uSIr=p+3nuvKBaD|hUkQ!CIIGdJ#$G|Kv}c0$qf9^Tl}o7`V~ z3CNjnhyX;5Vj_Gd@z|4VI$!+V;?I&?uY3dev`1xOZlcY?fO8op)F!634qe!{Zuk(O zw-${x+hY9n!z$p)!$eMWi+OxyTM$0S>J^>Y$U4RXj7koU zZ^!1ZL%1?jr@#7-=2eSLSbL7Xnz%3p;PCX!_M{&Fi=bidr_z-ODg{Ez(`!mxPtlQt zlbW`PfH8~R^j5w4cKw3k!oB=bcF(3?*dtQEp$QmN|G=Fos__tz)d(ivqr zITEHGw_f|Q-PiD?N1KvTNn5Bu$OA|L_n5e9XmG=DTXDl!^Hosb57$yo7Q@Qf7VlGO z-qUXyUI6^-JR?kH_JDiXnDmpzM_X3A{{~qdrUf}V(#wxFH*LQ5=I#ccFRlSsv*#3m zn6OhK6GO;m&K%%__idI^cnTD+37L8u|4H+ry!SQw1)7-iU%JFgA*{NjoikLpMen;& zoor29_)Me22_f}FiTU-E=Nii{(ROHgYIOh{+9ZN3z*U_EQBpOvZarxCdf$pBF6k4e zU-ke#)hQ-a;*21*F0|g@4)Fg;mb2!QW1VA|9wj6#y(~s9g9E`2QAxV@z^y??eo=EB z0(=Rxix*`zL+4!9kU>Mi`cy9hV686G;PrMVT<_Ngw>K((5jyGw8m~IxF??7#Q?(d0 z=a))$1h*tm567}fYF@gYEor&2WV+G8Em%6yZxR%TH?$bCDxWN*amc*XmsJyVp7Q1Y z8DOY;#bjX@{jEer;n^4h5D;b5G_z*uAJr24MKW$tfTZ@`2qmBpIEFJ!TUMB_eTU<~PIe zu-;oWX9P#eeL!Wo4MezO#kp5Z03SW2%or~434aVq>mSB;!;A%&Tyud32#h^!O7TnT z@K}%Brjb6-ZNXBeAGbSmZhckF1`6k3q_T$Km32s;-m1gVd$hqaAN=tqfQu#UZfBJ+ z)9j0f^FNw_46}{ZT*mmG&9!g4`X!U4-DK0y&BmxMHWXR$BtUM%!tkY;&IW>N=kx_N z){Vz!x23%j?0m4{B(&c~ z%eRdP=ugkn8g!4%M4f{#(;UZBuh(mDXklBP@V7QVHej(|>T*(6s@UK_%V7KoHKloF zp}P3Iwkze4tZE*8sF`7%c=KJ>?stnqazMgQtX|9rYkuE|RqANfp_>{%5UAJtY9ofY zCbYE5FGX^p`R5gv9_W%0bRuh`Nl0I9TCq(8)@PY{MUc@{h4O z2H)4+^yuk-bdcG2BVFz_x9#4iWdO|P406attolnULpcWli9$fgPOHwz?`{o%!VO*Q z5m-Sm8^JL)iy*)L%?aStSmeWRU3E4t_Bo4?PG9U)Si%0{@JKtHrc|)l8`l$S++P3V zn<=qNe1b)|0(W%MqPTqp!Tg+#?Bi;oSW5ikBQnzNz!@B%LjHcCd?Vep!{rTd^icrz z|Gw#SWYrq{jjgwx##K~H&fvO!&f*GAIol3n6AlvOzXwb;+C&$-TIy|L$bq(*Vlmw!y<^BfyE=h9RHM31y%wvGl1#@>q4 z5$mJQS`3B~2|L``i$G65tGKeR*=R2{*BQ6&8) zeA!k_$h1#ss9V*>GOj=XinB`ZXlE`sGscKtL}Mp5B?w6^Npk(+k|9XD6Eqv=X}UB=g1JyFLU z53s8-(Mi`7_*LGwy8z+O<r;TY?vy9r-y3!(RTMSxJC^KW_PE7!V)2?8xgpWLXaz-$J)Uj+axfS)C zbD8~5+<7+VLcz|lfTA;yu~9a4^6 zC|J}uJD5Ma%Fuabp(kV6tUcq4!F=MvBg2p8C}#J5Sb0;oMY`6#E7BJc1fFdKw}a$R zwyY>rC0v)?_^nu8HHky%GtY704&dA7c1iI^zRv>x>T_@2jISZ?@UM2;avht%8T{D${?8{Gx zz?|+0D`B;Vu(TE|=$0ZTxt0}NG&ES=7FV`y@&3(ik(tfKrhVufqs4ll+d05LcH4u# z$+>p-X%yq$J4?Vk_{r&{!hll+q9EahQd9IE3F^Dz-M3F8$Nqww@?~jK!BE9 zN7V<~Q|xM1p>mG9At3a*x*BvWDkQrRQQqx3+P_trX5l_vsVG1H9ygjT_kPO8`XhC+ z3yN+F6zn5NJ=H22@y>A)w`AJfPe3w4RBu61x1Y8udp~3_QN$P1W;e6pYr5A{s%X4A zJ7+=f+{Y2y^MZ${Z{D47?whtU`|$mdI*?<9;feSkarT#ZC!YpS&CKLU)!qA&lv=r= zz?V;G^_D6_huMfO#OT#ZtmEQ#4CKpU(1(mscdn}Jgbj6{F@Kf5b1&hAUbmofxYHu9 zmQ8fL{NwY^W)P?v*SMXuhKEHS7qB!xB4Gd$;*;GynXJkIYr_d=fOZqe2XKCay?yQ4 zD0$Eg;5OPeSbZIo9;@i1?X6QPk}kS&D($+p?x4!8sXzUE=?OT;dmO{kNyiD5aGn%_ z+%-Z_;0e3d1_#_=lW?>9g%UIyl|;=S?Yl`ha8xJPIfz3SF*Og35elZp8!^}|;L9`#+wUPSK8K*Lm{C*Fd3j$_!jstI(}*&cB=3JS2w zy1~}PVxpH_nu$FyggDb58RG8dgTA7q77AQPj=Nzfqmc081iHa=eISdGS>E#3^Bm7` zvIK|i*+3&yqSXto|7fR^BzfpK?&j?m1q3s$@>{I@b_oU%56SU>qDAT`T0j;*nSZIF z?>t5jiI!ZXM1S2h!I5Ma0_w47>f!*<9CP!#1u7CMQ2QmS0^;h<`S^S;)!p zm@FDTQ5G&#WhfZA_7oJy!zRJcA~Ke}D>|?$K@Omns-{>&or)rAYPeKkYym-NF_B+> z(R0UnvH^z*9tXm)iy1Bu*r9Q@$;te~ELXnrZl`)kLO)0UA~MjC9BGQdg|eLPVp9B5 zrTgKF0u28MAT{Xq*F3PBBb9K<&rQE#;d)^r-E7c_E2_mFmS}*DLb9dSe&co)w>HBa zSveOGH+mk)Siu9Tn;ul*uRnNcipl=(;j8d?sx)=|7I zYEalsGPJ!hnj9{8Xp7Y_s~>TB;hUqYc1l>uSU4YW=c}g(wdUR|mcD#QmS}N#y(W>| z^w;&q=X*}RmbPdTZoQh?%;ju^E+mLJ8*TmbrpD=xJFMg_!-ZO}rZuB!`Mv)c#KNX5XP`NL z=8H()5`qAG6&`Gn+YPrV#Km6r%K-Xx>N5NddL#d9rjZdoZ&m76m zU#v}TTYq?E@*!`j4`~72*AKF7;Nr%J`+wvZBi$oPE`;=*yTAocs9ywFVd{?@qQsW7 zWow`JCGngIke^hQHw8IlUuBsqK!m@IV?vv=1q9~+k^=T2GFC3fRO93};~1VlFA|mn zavfQoO8t{fn;wy%F?^Z0BRLd-KXWZXtSsxI8U6;SsOaiMhW>wBVoQL(QSeH8)mm!}`a8+;P;aZ?X(v>32svZV? zsng3*q9Z}R*IfVp(nbndD0j_J)y{`#kY~N{>&dLx^vD_`o&I6xj+Dej`15P#Bc8`+ zJ2sKUa{a@B5rm>cKd(;Qxg&e~e_q*+mjaRlshP%Fk)LY&Opy83*or(5H6|`}L>-d@ zzWKjj;RdhEzy}pFiu2IsrByo>EnW4&P#TdYqQKz)GdZ}mBZoMKPl74Tmo_?d!we^F zbQ95&DOLC9;-kakiDRU}PwR%KcOCwTdY(e0IVPGGA^57Fn5-)e;@dZ~Ir*xGe`_kueymz)8Cua({i{n8E&pmHo7frklc# zeY$!PhYd37lYK=}$&!4T6;8d8Y;TdKe{{z$Zmn2XlC;h1&_HTxU#4_F7Wv0%4vX;= zy=cy({ZH}90=dEFt5SvfGJ#+EPxCuY7A9%sE=tT(v2x#z%L5>}bk+IN;!OTbv}2Cl z%BK9EJ32#6X(={fJvggPc*tk}Zj-O!rbumQ33oO$KKAzwj;KK^PR1@`rS@oF(Nr?L zCP8t-vRaq5+gZyPa|MumPDz>}PK!0@%+{2He60CF@H*&jZ0U_f6C#)Nnzn3lIR|ag z=Jc`qj2e7Mm=D80>O~A<+)%0`RB9UUeXhkad_+~oeKf-%>C%x_!iM6f97SJSkF7Cr znH)Bm`YW9|!%0~LTTc&XizO&--&qxY7NRSz!z0t%rZR)!+L`>MwQy%*BHKQK-2P9w zd4@xpBuzhu+Kt!hz~Lk9k9KOXPRN@Z0VjD7ltzmx#$-;*!r-_nkJn=IZueWKX>FNN zXu?6!%<(%PUfD=y$7VfeKCRG=Y<;z$RGe_t8Xb-p57VZAq5E|E6d-D3%9!p9MEu7O zc!%r3g_P&Idq-tHrY_Xp5tyDtZM{|p&L}ejx)Ek`a%)XWe70})^*W^~OKKnu`Ke}Z z@MK|k+QOr4={Pyh9IDwWgp)R>{|w$;)2nrQ4}0+?d~#KIwku*{zQ0g)=jp z?+d*=ZR4u#j}sOu9I8j>Roma+UkK{Fpfgj}HmJ2AK2@VHSnSL-Ulp@}2y)QSkdCic z`f>~E@{-q}4+6ENUqi%2i1^gE$=*$RxzeS3M{zzn-K55FuUP2(MuHWQRi;(jg0?Oc zy&oyIoV)L7_dQMkS2M6+V}GmJZC@0U{)JWWZaLTPTja2HQ~sGd6sLjp;E%E$nnsZ+QVMp{26YeAwFuW*Z(|a&#=uf&>&xuBQhsjSyfmkt#n)M0k4?jh)7TH zeP_>fn+#TOa&2ucONl`b&IR#6gQQd`8tXVkaMD{N z2ZCxs00pftIx&hkz3jz(*B)rg|DJUQB{(K+x;0DMB(jq^>OR`9x=>5?8jxq^I?;9G)t% z5fvHSsp-h-|A;%n&z=N^PEok^)fZ^+!^CG#>Oe<9w~)%GhoGVoqv-k|67L(Rz?>{B zqwiE=2O|}V3GS&1aR$=#&m+aYiwzEl7xPaagh~JvSf47U9+d5=txmJfKUgi_f|g!t zI?yv3XV!5Gee;=Zh$FC0nFO?he#4fL&azIOXob4Ci1BzGNYogpU(B}|=;8vqUB)`v z!D$ZDY30nny7a>A!$o6AZKxG_;6k4wq<;-WodKsGd zOXPJ~!+OvZ|MF-?K=-2M&xXvnOs2ZdV{lm9|9*%4pL&TQ8thDS za8B1~Vz9IwmPxlQ(N{ z$stKRmcOV()gIa;Ls}}C}=Jew9;cP7j$FQ`4ZVay$71X?!y+b}|X!&{2i=xXr z)~N@hv7V!jpz>QIiV!14uy{UDaO$7LsubM#AHAIYH(Pfc$D=K^Xy{QywW3-pYKkrC zsv0b*I-9L(M5w23dg|#6LF>o^(g|l;rMj)QTH;}?sM40GYJ{O^rzKJGkkh8fCO20# ztcfI4?0fq!?4134f4TR5KlgLK_w#+dU(ZrcF1Ua|#ZSnL5!gb87flM`AhB7+2k;_D z#rwSk_magJ(}9%eL%@F@LJxuH*LeV?rhTkw(wzh6zW!%Ca;!kK6J&%uas_!BgY;?) zzvI*4hpBE{>{|008h-k)7s6$r&Kb4(eu4G z1yML4Dc2HeEO6tbgS2!vHQHtzn9)AW1-S8ABE7o!TbD350CQ67<>8!lpTt4^o;b*0 zsQD+GSss6H09=XK+>-(<`A|X5zip1hXTLVX#(?%OpuZe zFm_`n;{5erh4vNn944026+BLGe>Udm}~DuoPn>AWe-?O^K#4rVpN6brm9qeMl3&*9Ejj zJ!%t*cbp*kiV#V&flKdvFeZ6G^0lLc4H?XUFt)_w7+|Z!QIKJNUzuXaoyTe=#2HTSnD?eWlyy|gWe@(9@y+c3^p#5F=*)wKh23zU& zG8bTPMgo4zae{QsBLXYxK(r24Yzaf^!%147+kw*!Cm6>NQO9IFIRGyp>xOmmK1H8$ zk|B8upn#WPyn9%^wylomgt`TO1_Pzfw`0miBQqM@_>I9-qHr}{#DfX89HMrxj=(b6E%%Blssc=(Zuww$;cjW|HJ z&NJ1XD%z2oGtE3|w{9mN2Ua6GP;7Qmi{J8@s2P33TD0e2KCOCF8@-@kpF-RpTfvWC zgtqPdCIoJeG{2l+zOx874@yhP0_DEa3A*TLPfK50U%ox{wlXr~-dS(eMDum4s(`iW zQD)(#aIY1hK8Brkt#`rEP67GM?`u8L9h^bmwEH&Nj~2X8FeNh6E8<#*oIb3JaxE*H zRA}U+P+@Y^C6ZuMR=4hEg7sW^c;>|N9u?~g{lEcPKL#Z&t-;j510@ukv!$%vv9r9^nh4!eV)`u-G z!8MIKbNq$zAst#74c&_7JKu#3eGE5H(L>`F%|ngUBPtr*QL3~m(39mlyMpTc7M+}N zXTuju=hZ30m#hlhQH80;!RwQftj^{fE}}m;S6fdnu77or6e~!Im?dazJY3{}J~h^n z%7_!`G79TS0({n1&7ae%-GB#E+j9N4?LBmSc4-}egW~%SM!D3`^r_>&DT8Dyv zP?O2tz(ZQ{C_pF=HNSzD!zrN-W^FXZ@qpd?x~S41ECJrzPvVi+@FS`#T z2D)pQ5?yU!^USTx#?0nS!{dGB`e*{4`4sPunEk}lgR@>192^xAsc1lsKu2)92P{5f zVW)iA8<99*Q6^0Hz`u!8ECwL36%NHu2j$Gqdbs2z^Lt^Z!diai``BwY3W?sKC?^tY z3DP+GR*9Ya>9ncR;&N1A Date: Fri, 9 Sep 2016 12:51:35 +0100 Subject: [PATCH 508/746] Add maintenance and security README --- README.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.rst b/README.rst index 8631dae..1175587 100644 --- a/README.rst +++ b/README.rst @@ -32,3 +32,20 @@ channel on Freenode. You can also install channels from PyPI as the ``channels`` package. You'll likely also want ``asgi_redis`` to provide the Redis channel layer. + + +Maintenance and Security +------------------------ + +To report security issues, please contact security@djangoproject.com. For GPG +signatures and more security process information, see +https://docs.djangoproject.com/en/dev/internals/security/. + +To report bugs or request new features, please open a new GitHub issue. + +Django Core Shepherd: Andrew Godwin + +Maintenance team: +* Andrew Godwin +* Steven Davidson +* Jeremy Spencer From d7f71be6e12d5b6ff1b73399e781511d1c636c79 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 9 Sep 2016 12:57:02 +0100 Subject: [PATCH 509/746] Fix RST formatting --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1175587..84763de 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ Django Channels .. image:: https://readthedocs.org/projects/channels/badge/?version=latest :target: http://channels.readthedocs.org/en/latest/?badge=latest - + .. image:: https://img.shields.io/pypi/v/channels.svg :target: https://pypi.python.org/pypi/channels @@ -46,6 +46,7 @@ To report bugs or request new features, please open a new GitHub issue. Django Core Shepherd: Andrew Godwin Maintenance team: + * Andrew Godwin * Steven Davidson * Jeremy Spencer From 67f6638e90853b925f54fbc932939737c1a0bc73 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 9 Sep 2016 12:58:19 +0100 Subject: [PATCH 510/746] More of a contribution CTA --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 84763de..f7fdc9a 100644 --- a/README.rst +++ b/README.rst @@ -50,3 +50,7 @@ Maintenance team: * Andrew Godwin * Steven Davidson * Jeremy Spencer + +If you are interested in joining the maintenance team, please +`read more about contributing `_ +and get in touch! From 21f0aeaf6416eb8fd23a8c9f9dc8860434b64850 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 9 Sep 2016 12:59:45 +0100 Subject: [PATCH 511/746] Mention mailing list --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f7fdc9a..9bf3da1 100644 --- a/README.rst +++ b/README.rst @@ -41,7 +41,9 @@ To report security issues, please contact security@djangoproject.com. For GPG signatures and more security process information, see https://docs.djangoproject.com/en/dev/internals/security/. -To report bugs or request new features, please open a new GitHub issue. +To report bugs or request new features, please open a new GitHub issue. For +larger discussions, please post to the +`django-developers mailing list `_. Django Core Shepherd: Andrew Godwin From 971d3fc8d027efae0f7102cda20c98ed75121efe Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 9 Sep 2016 13:29:12 +0100 Subject: [PATCH 512/746] Django-ification --- CONTRIBUTING.rst | 13 +++++------- README.rst | 6 +++--- docs/contributing.rst | 46 ++++++++++++++++++++++++++---------------- docs/deploying.rst | 2 +- docs/index.rst | 21 +++++++++++++++++-- docs/inshort.rst | 2 +- docs/installation.rst | 2 +- setup.py | 6 +++--- testproject/Dockerfile | 4 ++-- testproject/README.rst | 2 +- testproject/fabfile.py | 8 ++++---- 11 files changed, 69 insertions(+), 43 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 5392a70..de0aef1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -10,8 +10,10 @@ Examples of contributions include: * Documentation improvements * Bug reports and patch reviews -Setup ------ +For more information, please see our `contribution guide `_. + +Quick Setup +----------- Fork, then clone the repo: @@ -25,9 +27,4 @@ Make your change. Add tests for your change. Make the tests pass: tox -Push to your fork and `submit a pull request `_. - - -At this point you're waiting on us. We like to at least comment on pull requests -within three business days (and, typically, one business day). We may suggest -some changes or improvements or alternatives. +Push to your fork and `submit a pull request `_. diff --git a/README.rst b/README.rst index 9bf3da1..57523f7 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,9 @@ Django Channels =============== -.. image:: https://api.travis-ci.org/andrewgodwin/channels.svg - :target: https://travis-ci.org/andrewgodwin/channels - +.. image:: https://api.travis-ci.org/django/channels.svg + :target: https://travis-ci.org/django/channels + .. image:: https://readthedocs.org/projects/channels/badge/?version=latest :target: http://channels.readthedocs.org/en/latest/?badge=latest diff --git a/docs/contributing.rst b/docs/contributing.rst index 0dba851..1f063de 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -15,34 +15,46 @@ We're looking for help with the following areas: * Feature polish and occasional new feature design * Case studies and writeups -You can find what we're looking to work on right now in two places: +You can find what we're looking to work on in the GitHub issues list for each +of the Channels sub-projects: - * Specific bugs are in the `GitHub issues `_ - * Higher-level tasks are on the `ChannelsTasks Django wiki page `_ + * `Channels issues `_, for the Django integration + * `Daphne issues `_, for the HTTP and Websocket termination + * `asgiref issues `_, for the base ASGI library/memory backend + * `asgi_redis issues `_, for the Redis channel backend + * `asgi_ipc issues `_, for the POSIX IPC channel backend -These are, however, just a suggested list - any offer to help is welcome as long -as it fits the project goals. +Issues are categorized by difficulty level: + * ``exp/beginner``: Easy issues suitable for a first-time contributor. + * ``exp/intermediate``: Moderate issues that need skill and a day or two to solve. + * ``exp/advanced``: Difficult issues that require expertise and potentially weeks of work. -I'm interested, how should I get started? ------------------------------------------ +They are also classified by type: -The best thing to do is to see if there's a `GitHub issue `_ -for the thing you wish to work on - if there is, leave a comment saying you're -going to take it on, and if not, open one describing what you're doing so there's -a place to record information around. + * ``documentation``: Documentation issues. Pick these if you want to help us by writing docs. + * ``bug``: A bug in existing code. Usually easier for beginners as there's a defined thing to fix. + * ``enhancement``: A new feature for the code; may be a bit more open-ended. -If you have questions, you can either open an issue with the questions detailed, -hop on the ``#django-channels`` channel on Freenode IRC, or email Andrew directly -at ``andrew@aeracode.org``. +You should filter the issues list by the experience level and type of work +you'd like to do, and then if you want to take something on leave a comment +and assign yourself to it. If you want advice about how to take on a bug, +leave a comment asking about it, or pop into the IRC channel at +``#django-channels`` on Freenode and we'll be happy to help. + +The issues are also just a suggested list - any offer to help is welcome as long +as it fits the project goals, but you should make an issue for the thing you +wish to do and discuss it first if it's relatively large (but if you just found +a small bug and want to fix it, sending us a pull request straight away is fine). I'm a novice contributor/developer - can I help? ------------------------------------------------ -Of course - just get in touch like above and mention your experience level, -and we'll try the best we can to match you up with someone to mentor you through -the task. +Of course! The issues labelled with ``exp/beginner`` are a perfect place to +get started, as they're usually small and well defined. If you want help with +one of them, pop into the IRC channel at ``#django-channels`` on Freenode or +get in touch with Andrew directly at andrew@aeracode.org. Can you pay me for my time? diff --git a/docs/deploying.rst b/docs/deploying.rst index a7fc3bf..cdfe666 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -134,7 +134,7 @@ If you want to support WebSockets, long-poll HTTP requests and other Channels features, you'll need to run a native ASGI interface server, as the WSGI specification has no support for running these kinds of requests concurrently. We ship with an interface server that we recommend you use called -`Daphne `_; it supports WebSockets, +`Daphne `_; it supports WebSockets, long-poll HTTP requests, HTTP/2 *(soon)* and performs quite well. You can just keep running your Django code as a WSGI app if you like, behind diff --git a/docs/index.rst b/docs/index.rst index 140467c..93cdeff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,9 +17,26 @@ WebSockets with only 30 lines of code. If you want a quick overview, start with :doc:`inshort`. -You can find the Channels repository `on GitHub `_. +If you are interested in contributing, please read our :doc:`contributing` docs! -Contents: + +Projects +-------- + +Channels is comprised of five packages: + + * `Channels `_, the Django integration layer + * `Daphne `_, the HTTP and Websocket termination server + * `asgiref `_, the base ASGI library/memory backend + * `asgi_redis `_, the Redis channel backend + * `asgi_ipc `_, the POSIX IPC channel backend + +This documentation covers the system as a whole; individual release notes and +instructions can be found in the individual repositories. + + +Topics +------ .. toctree:: :maxdepth: 2 diff --git a/docs/inshort.rst b/docs/inshort.rst index b87ac15..8e3facf 100644 --- a/docs/inshort.rst +++ b/docs/inshort.rst @@ -36,7 +36,7 @@ I have to change how I run Django? No, all the new stuff is entirely optional. If you want it, however, you'll change from running Django under a WSGI server, to running: -* An ASGI server, probably `Daphne `_ +* An ASGI server, probably `Daphne `_ * Django worker servers, using ``manage.py runworker`` * Something to route ASGI requests over, like Redis. diff --git a/docs/installation.rst b/docs/installation.rst index 288f12b..3cb4756 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -28,7 +28,7 @@ To install the latest version of Channels, clone the repo, change to the repo, change to the repo directory, and pip install it into your current virtual environment:: - $ git clone git@github.com:andrewgodwin/channels.git + $ git clone git@github.com:django/channels.git $ cd channels $ (environment) $ pip install -e . # the dot specifies the current repo diff --git a/setup.py b/setup.py index 673eb15..834ff64 100644 --- a/setup.py +++ b/setup.py @@ -4,9 +4,9 @@ from channels import __version__ setup( name='channels', version=__version__, - url='http://github.com/andrewgodwin/channels', - author='Andrew Godwin', - author_email='andrew@aeracode.org', + url='http://github.com/django/channels', + author='Django Software Foundation', + author_email='foundation@djangoproject.com', description="Brings event-driven capabilities to Django with a channel system. Django 1.8 and up only.", license='BSD', packages=find_packages(), diff --git a/testproject/Dockerfile b/testproject/Dockerfile index e7e49c4..aeddb65 100644 --- a/testproject/Dockerfile +++ b/testproject/Dockerfile @@ -13,10 +13,10 @@ RUN apt-get update && \ # Install asgi_redis driver and most recent Daphne RUN pip install \ asgi_redis==0.8.3 \ - git+https://github.com/andrewgodwin/daphne.git@#egg=daphne + git+https://github.com/django/daphne.git@#egg=daphne # Clone Channels and install it -RUN git clone https://github.com/andrewgodwin/channels.git /srv/channels/ && \ +RUN git clone https://github.com/django/channels.git /srv/channels/ && \ cd /srv/channels && \ git reset --hard caa589ae708a1a66ba1bdcd24f5fd473040772bd && \ python setup.py install diff --git a/testproject/README.rst b/testproject/README.rst index 55279bc..1136e85 100644 --- a/testproject/README.rst +++ b/testproject/README.rst @@ -74,7 +74,7 @@ Install fabric on your machine. This is highly dependent on what your environmen Git clone this project down to your machine:: - git clone https://github.com/andrewgodwin/channels/ + git clone https://github.com/django/channels/ Relative to where you cloned the directory, move up a couple levels:: diff --git a/testproject/fabfile.py b/testproject/fabfile.py index 3bab83d..6c884dc 100644 --- a/testproject/fabfile.py +++ b/testproject/fabfile.py @@ -13,9 +13,9 @@ def setup_redis(): def setup_channels(): sudo("apt-get update && apt-get install -y git python-dev python-setuptools python-pip") sudo("pip install -U pip") - sudo("pip install -U asgi_redis asgi_ipc git+https://github.com/andrewgodwin/daphne.git@#egg=daphne") + sudo("pip install -U asgi_redis asgi_ipc git+https://github.com/django/daphne.git@#egg=daphne") sudo("rm -rf /srv/channels") - sudo("git clone https://github.com/andrewgodwin/channels.git /srv/channels/") + sudo("git clone https://github.com/django/channels.git /srv/channels/") with cd("/srv/channels/"): sudo("python setup.py install") @@ -34,7 +34,7 @@ def run_worker(redis_ip): # Current loadtesting setup @task -def setup_load_tester(src="https://github.com/andrewgodwin/channels.git"): +def setup_load_tester(src="https://github.com/django/channels.git"): sudo("apt-get update && apt-get install -y git nodejs && apt-get install npm") sudo("npm install -g loadtest") sudo("ln -s /usr/bin/nodejs /usr/bin/node") @@ -59,7 +59,7 @@ def setup_tester(): sudo("apt-get update && apt-get install -y apache2-utils python3-pip") sudo("pip3 -U pip autobahn twisted") sudo("rm -rf /srv/channels") - sudo("git clone https://github.com/andrewgodwin/channels.git /srv/channels/") + sudo("git clone https://github.com/django/channels.git /srv/channels/") @task From f79ecbff6d59a07f1605c52549ae899e34877e5e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 9 Sep 2016 13:38:38 +0100 Subject: [PATCH 513/746] Tidy up main README a bit more --- README.rst | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 57523f7..acec4e8 100644 --- a/README.rst +++ b/README.rst @@ -13,16 +13,14 @@ Django Channels .. image:: https://img.shields.io/pypi/l/channels.svg :target: https://pypi.python.org/pypi/channels -*(Note: Recent versions of Channels also need recent versions of Daphne, -asgi_redis and asgiref, so make sure you update all at once)* - -This is a work-in-progress code branch of Django implemented as a third-party -app, which aims to bring some asynchrony to Django and expand the options -for code beyond the request-response model, in particular enabling WebSocket, -HTTP2 push, and background task support. +Channels loads into Django as a pluggable app to bring WebSocket, long-poll HTTP, +task offloading and other asynchrony support to your code, using familiar Django +design patterns and a flexible underlying framework that lets you not only +customize behaviours but also write support for your own protocols and needs. This is still **beta** software: the API is mostly settled, but might change -a bit as things develop. +a bit as things develop. Once we hit ``1.0``, it will be stablized and a +deprecation policy will come in. Documentation, installation and getting started instructions are at http://channels.readthedocs.org @@ -30,8 +28,16 @@ http://channels.readthedocs.org Support can be obtained either here via issues, or in the ``#django-channels`` channel on Freenode. -You can also install channels from PyPI as the ``channels`` package. -You'll likely also want ``asgi_redis`` to provide the Redis channel layer. +You can install channels from PyPI as the ``channels`` package. +You'll likely also want to ``asgi_redis`` to provide the Redis channel layer. +See our `installation `_ +and `getting started `_ docs for more. + + +Contributing +------------ + +To learn more about contributing, please `read our contributing docs `_. Maintenance and Security From f805096bd589f443a646a293a3e0c326e0e178cf Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 9 Sep 2016 14:22:23 +0100 Subject: [PATCH 514/746] Crosslink other channels projects --- README.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.rst b/README.rst index acec4e8..d08f48e 100644 --- a/README.rst +++ b/README.rst @@ -62,3 +62,14 @@ Maintenance team: If you are interested in joining the maintenance team, please `read more about contributing `_ and get in touch! + + +Other Projects +-------------- + +The Channels project is made up of several packages; the others are: + + * `Daphne `_, the HTTP and Websocket termination server + * `asgiref `_, the base ASGI library/memory backend + * `asgi_redis `_, the Redis channel backend + * `asgi_ipc `_, the POSIX IPC channel backend From 6a17caad5bbd12692099f080e94394427d491b59 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 9 Sep 2016 14:22:53 +0100 Subject: [PATCH 515/746] I really need to stop indenting unordered lists --- README.rst | 8 ++++---- docs/index.rst | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index d08f48e..c6fa512 100644 --- a/README.rst +++ b/README.rst @@ -69,7 +69,7 @@ Other Projects The Channels project is made up of several packages; the others are: - * `Daphne `_, the HTTP and Websocket termination server - * `asgiref `_, the base ASGI library/memory backend - * `asgi_redis `_, the Redis channel backend - * `asgi_ipc `_, the POSIX IPC channel backend +* `Daphne `_, the HTTP and Websocket termination server +* `asgiref `_, the base ASGI library/memory backend +* `asgi_redis `_, the Redis channel backend +* `asgi_ipc `_, the POSIX IPC channel backend diff --git a/docs/index.rst b/docs/index.rst index 93cdeff..93fe135 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,11 +25,11 @@ Projects Channels is comprised of five packages: - * `Channels `_, the Django integration layer - * `Daphne `_, the HTTP and Websocket termination server - * `asgiref `_, the base ASGI library/memory backend - * `asgi_redis `_, the Redis channel backend - * `asgi_ipc `_, the POSIX IPC channel backend +* `Channels `_, the Django integration layer +* `Daphne `_, the HTTP and Websocket termination server +* `asgiref `_, the base ASGI library/memory backend +* `asgi_redis `_, the Redis channel backend +* `asgi_ipc `_, the POSIX IPC channel backend This documentation covers the system as a whole; individual release notes and instructions can be found in the individual repositories. From b27384933f0fc285d570488869f8dff8afdb7826 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 9 Sep 2016 14:46:55 +0100 Subject: [PATCH 516/746] Move ChannelsTasks wiki content into the docs --- docs/contributing.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 1f063de..e6a5c1d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -61,7 +61,20 @@ Can you pay me for my time? --------------------------- Thanks to Mozilla, we have a reasonable budget to pay people for their time -working on all of the above sorts of tasks and more. If you're interested in -working on something and being paid, you'll need to draw up a short proposal -and get it approved first - the wiki page above has more details, and if you're -interested, email andrew@aeracode.org to get the conversation started. +working on all of the above sorts of tasks and more. Generally, we'd prefer +to fund larger projects (you can find these labelled as ``epic-project`` in the +issues lists) to reduce the administrative overhead, but we're open to any +proposal. + +If you're interested in working on something and being paid, you'll need to +draw up a short proposal and get in touch with the committee, discuss the work +and your history with open-source contribution (we strongly prefer that you have +a proven track record on at least a few things) and the amount you'd like to be paid. + +If you're interested in working on one of these tasks, get in touch with +Andrew Godwin (andrew@aeracode.org) as a first point of contact; he can help +talk you through what's involved, and help judge/refine your proposal before +it goes to the committee. + +Tasks not on any issues list can also be proposed; Andrew can help talk about them +and if they would be sensible to do. From b46c511b13dd948f1d6cdb8704927a76b9b15d36 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 9 Sep 2016 14:47:34 +0100 Subject: [PATCH 517/746] Clarify where overall tasks live --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index e6a5c1d..8620a43 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -18,7 +18,7 @@ We're looking for help with the following areas: You can find what we're looking to work on in the GitHub issues list for each of the Channels sub-projects: - * `Channels issues `_, for the Django integration + * `Channels issues `_, for the Django integration and overall project efforts * `Daphne issues `_, for the HTTP and Websocket termination * `asgiref issues `_, for the base ASGI library/memory backend * `asgi_redis issues `_, for the Redis channel backend From 5e0add6bbb2756795069ba80a189acbad7b84106 Mon Sep 17 00:00:00 2001 From: Naveen Yadav Date: Sun, 11 Sep 2016 15:28:06 +0530 Subject: [PATCH 518/746] sessions and users doc updated (#354) --- docs/generics.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/generics.rst b/docs/generics.rst index 8231a5f..7ec272f 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -224,6 +224,16 @@ This will run the appropriate decorator around your handler methods, and provide the one passed in to your handler as an argument as well as ``self.message``, as they point to the same instance. +And if you just want to use same old django session just use ``http_session`` or +``http_session_user``:: + + class MyConsumer(WebsocketConsumer): + + http_session_user = True + +That gives you ``message.user`` which is same as ``request.user`` and ``message.http_session`` +is same as of ``request.session`` + Applying Decorators ------------------- From 9618440e6dd04eeef2532ab5a4a43b8258d9fce9 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 11 Sep 2016 11:00:45 +0100 Subject: [PATCH 519/746] Fix generics docs --- docs/generics.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/generics.rst b/docs/generics.rst index 7ec272f..96d4d87 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -224,15 +224,15 @@ This will run the appropriate decorator around your handler methods, and provide the one passed in to your handler as an argument as well as ``self.message``, as they point to the same instance. -And if you just want to use same old django session just use ``http_session`` or -``http_session_user``:: +And if you just want to use the user from the django session, add ``http_user``: class MyConsumer(WebsocketConsumer): - http_session_user = True + http_user = True + +This will give you ``message.user``, which will be the same as ``request.user`` +would be on a regular View. -That gives you ``message.user`` which is same as ``request.user`` and ``message.http_session`` -is same as of ``request.session`` Applying Decorators ------------------- From be14c14783396f8b6b75f3859844fdfbb46851b1 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Sun, 11 Sep 2016 13:28:44 +0300 Subject: [PATCH 520/746] Add django1.10 to the list of tests env (#353) --- tox.ini | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index fc0d7b4..546e062 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] skipsdist = True envlist = - {py27}-django-{18,19} - {py34}-django-{18,19} - {py35}-django-{18,19} + {py27}-django-{18,19,110} + {py34}-django-{18,19,110} + {py35}-django-{18,19,110} {py27,py35}-flake8 isort @@ -22,6 +22,7 @@ deps = isort: isort django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 + django-110: Django>=1.10,<1.11 commands = flake8: flake8 isort: isort -c -rc channels From e176e913d8c89d4f9aba08a0d38dcdf42d359d00 Mon Sep 17 00:00:00 2001 From: Michael Angeletti Date: Mon, 12 Sep 2016 06:25:05 -0400 Subject: [PATCH 521/746] Fix URL typo (#359) --- loadtesting/2016-09-06/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadtesting/2016-09-06/README.rst b/loadtesting/2016-09-06/README.rst index 49c3127..55609db 100644 --- a/loadtesting/2016-09-06/README.rst +++ b/loadtesting/2016-09-06/README.rst @@ -5,7 +5,7 @@ The goal of these tests is to see how channels performs with normal HTTP traffic In order to control for variances, several measures were taken: -- the same testing tool was used across all tests, `loadtest `_. +- the same testing tool was used across all tests, `loadtest `_. - all target machines were identical - all target code variances were separated into appropriate files in the dir of /testproject in this repo - all target config variances necessary to the different setups were controlled by supervisord so that human error was limited From 075897d9107b89edadcb5aca34d371df46370820 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Mon, 12 Sep 2016 13:27:42 +0300 Subject: [PATCH 522/746] Inbound tests (#351) * Added groups as Binding attr for easy groups_names definition * Binding: inbound - updating fields that only in fields attribute * Added tests for inbound part of binding #343 * Fix for flake8 checker * Revert "Added groups as Binding attr for easy groups_names definition" This reverts commit 009b4adbee534d4ccbea191ce2523a0edb09d407. * Using group_names at inbound tests --- channels/binding/websockets.py | 3 +- channels/tests/test_binding.py | 222 ++++++++++++++++++++++++++++++++- 2 files changed, 223 insertions(+), 2 deletions(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index e0d0d3d..76a442b 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -110,7 +110,8 @@ class WebsocketBinding(Binding): instance = self.model.objects.get(pk=pk) hydrated = self._hydrate(pk, data) for name in data.keys(): - setattr(instance, name, getattr(hydrated.object, name)) + if name in self.fields or self.fields == ['__all__']: + setattr(instance, name, getattr(hydrated.object, name)) instance.save() diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index aaf88aa..56af79b 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -1,9 +1,12 @@ from __future__ import unicode_literals +import json from django.contrib.auth import get_user_model +from channels.binding.base import CREATE, UPDATE, DELETE from channels.binding.websockets import WebsocketBinding +from channels.generic.websockets import WebsocketDemultiplexer from channels.tests import ChannelTestCase, apply_routes, HttpClient -from channels import route +from channels import route, Group User = get_user_model() @@ -134,3 +137,220 @@ class TestsBinding(ChannelTestCase): received = client.receive() self.assertIsNone(received) + + def test_demultiplexer(self): + class Demultiplexer(WebsocketDemultiplexer): + mapping = { + 'users': 'binding.users', + } + + groups = ['inbound'] + + with apply_routes([Demultiplexer.as_route(path='/')]): + client = HttpClient() + client.send_and_consume('websocket.connect', path='/') + + # assert in group + Group('inbound').send({'text': json.dumps({'test': 'yes'})}) + self.assertEqual(client.receive(), {'test': 'yes'}) + + # assert that demultiplexer stream message + client.send_and_consume('websocket.receive', path='/', + text={'stream': 'users', 'payload': {'test': 'yes'}}) + message = client.get_next_message('binding.users') + self.assertIsNotNone(message) + self.assertEqual(message.content['test'], 'yes') + + def test_demultiplexer_with_wrong_stream(self): + class Demultiplexer(WebsocketDemultiplexer): + mapping = { + 'users': 'binding.users', + } + + groups = ['inbound'] + + with apply_routes([Demultiplexer.as_route(path='/')]): + client = HttpClient() + client.send_and_consume('websocket.connect', path='/') + + with self.assertRaises(ValueError) as value_error: + client.send_and_consume('websocket.receive', path='/', text={ + 'stream': 'wrong', 'payload': {'test': 'yes'} + }) + + self.assertIn('stream not mapped', value_error.exception.args[0]) + + message = client.get_next_message('binding.users') + self.assertIsNone(message) + + def test_demultiplexer_with_wrong_payload(self): + class Demultiplexer(WebsocketDemultiplexer): + mapping = { + 'users': 'binding.users', + } + + groups = ['inbound'] + + with apply_routes([Demultiplexer.as_route(path='/')]): + client = HttpClient() + client.send_and_consume('websocket.connect', path='/') + + with self.assertRaises(ValueError) as value_error: + client.send_and_consume('websocket.receive', path='/', text={ + 'stream': 'users', 'payload': 'test', + }) + + self.assertEqual(value_error.exception.args[0], 'Multiplexed frame payload is not a dict') + + message = client.get_next_message('binding.users') + self.assertIsNone(message) + + def test_demultiplexer_without_payload_and_steam(self): + class Demultiplexer(WebsocketDemultiplexer): + mapping = { + 'users': 'binding.users', + } + + groups = ['inbound'] + + with apply_routes([Demultiplexer.as_route(path='/')]): + client = HttpClient() + client.send_and_consume('websocket.connect', path='/') + + with self.assertRaises(ValueError) as value_error: + client.send_and_consume('websocket.receive', path='/', text={ + 'nostream': 'users', 'payload': 'test', + }) + + self.assertIn('no channel/payload key', value_error.exception.args[0]) + + message = client.get_next_message('binding.users') + self.assertIsNone(message) + + with self.assertRaises(ValueError) as value_error: + client.send_and_consume('websocket.receive', path='/', text={ + 'stream': 'users', + }) + + self.assertIn('no channel/payload key', value_error.exception.args[0]) + + message = client.get_next_message('binding.users') + self.assertIsNone(message) + + def test_inbound_create(self): + self.assertEqual(User.objects.all().count(), 0) + + class Demultiplexer(WebsocketDemultiplexer): + mapping = { + 'users': 'binding.users', + } + + groups = ['inbound'] + + class UserBinding(WebsocketBinding): + model = User + stream = 'users' + fields = ['username', 'email', 'password', 'last_name'] + + @classmethod + def group_names(cls, instance, action): + return ['users_outbound'] + + def has_permission(self, user, action, pk): + return True + + with apply_routes([Demultiplexer.as_route(path='/'), route('binding.users', UserBinding.consumer)]): + client = HttpClient() + client.send_and_consume('websocket.connect', path='/') + client.send_and_consume('websocket.receive', path='/', text={ + 'stream': 'users', + 'payload': {'action': CREATE, 'data': {'username': 'test_inbound', 'email': 'test@user_steam.com'}} + }) + # our Demultiplexer route message to the inbound consumer, so call Demultiplexer consumer + client.consume('binding.users') + + self.assertEqual(User.objects.all().count(), 1) + user = User.objects.all().first() + self.assertEqual(user.username, 'test_inbound') + self.assertEqual(user.email, 'test@user_steam.com') + + def test_inbound_update(self): + user = User.objects.create(username='test', email='test@channels.com') + + class Demultiplexer(WebsocketDemultiplexer): + mapping = { + 'users': 'binding.users', + } + + groups = ['inbound'] + + class UserBinding(WebsocketBinding): + model = User + stream = 'users' + fields = ['username', ] + + @classmethod + def group_names(cls, instance, action): + return ['users_outbound'] + + def has_permission(self, user, action, pk): + return True + + with apply_routes([Demultiplexer.as_route(path='/'), route('binding.users', UserBinding.consumer)]): + client = HttpClient() + client.send_and_consume('websocket.connect', path='/') + client.send_and_consume('websocket.receive', path='/', text={ + 'stream': 'users', + 'payload': {'action': UPDATE, 'pk': user.pk, 'data': {'username': 'test_inbound'}} + }) + # our Demultiplexer route message to the inbound consumer, so call Demultiplexer consumer + client.consume('binding.users') + + user = User.objects.get(pk=user.pk) + self.assertEqual(user.username, 'test_inbound') + self.assertEqual(user.email, 'test@channels.com') + + # trying change field that not in binding fields + client.send_and_consume('websocket.receive', path='/', text={ + 'stream': 'users', + 'payload': {'action': UPDATE, 'pk': user.pk, 'data': {'email': 'new@test.com'}} + }) + client.consume('binding.users') + + user = User.objects.get(pk=user.pk) + self.assertEqual(user.username, 'test_inbound') + self.assertEqual(user.email, 'test@channels.com') + + def test_inbound_delete(self): + user = User.objects.create(username='test', email='test@channels.com') + + class Demultiplexer(WebsocketDemultiplexer): + mapping = { + 'users': 'binding.users', + } + + groups = ['inbound'] + + class UserBinding(WebsocketBinding): + model = User + stream = 'users' + fields = ['username', ] + + @classmethod + def group_names(cls, instance, action): + return ['users_outbound'] + + def has_permission(self, user, action, pk): + return True + + with apply_routes([Demultiplexer.as_route(path='/'), route('binding.users', UserBinding.consumer)]): + client = HttpClient() + client.send_and_consume('websocket.connect', path='/') + client.send_and_consume('websocket.receive', path='/', text={ + 'stream': 'users', + 'payload': {'action': DELETE, 'pk': user.pk} + }) + # our Demultiplexer route message to the inbound consumer, so call Demultiplexer consumer + client.consume('binding.users') + + self.assertIsNone(User.objects.filter(pk=user.pk).first()) From 5464cba74285b53ae47f5c9a3b6062f2ba0e7cce Mon Sep 17 00:00:00 2001 From: Krukov D Date: Mon, 12 Sep 2016 13:28:12 +0300 Subject: [PATCH 523/746] Iimprovements for test client (#352) * Json encoding only for not string text at HttpClient * Decode received content if possible * Check that content received * Using json kwarg at receive method to decode message text content * Wrap decorator function at channel_session_user_from_http * Cleanup * Arbitrary indent. sorry --- channels/auth.py | 1 + channels/tests/base.py | 2 +- channels/tests/http.py | 21 ++++++++++++++++----- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/channels/auth.py b/channels/auth.py index 90eaaff..f0a0f75 100644 --- a/channels/auth.py +++ b/channels/auth.py @@ -82,6 +82,7 @@ def channel_session_user_from_http(func): """ @http_session_user @channel_session + @functools.wraps(func) def inner(message, *args, **kwargs): if message.http_session is not None: transfer_user(message.http_session, message.channel_session) diff --git a/channels/tests/base.py b/channels/tests/base.py index 1f628ac..591c11b 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -135,7 +135,7 @@ class Client(object): return self.consume(channel, fail_on_none=fail_on_none) def receive(self): - """self.get_next_message(self.reply_channel) + """ Get content of next message for reply channel if message exists """ message = self.get_next_message(self.reply_channel) diff --git a/channels/tests/http.py b/channels/tests/http.py index 2b8ecd0..0085216 100644 --- a/channels/tests/http.py +++ b/channels/tests/http.py @@ -1,12 +1,16 @@ import json import copy +import six + from django.apps import apps from django.conf import settings from ..sessions import session_for_reply_channel from .base import Client +json_module = json # alias for using at functions with json kwarg + class HttpClient(Client): """ @@ -56,10 +60,14 @@ class HttpClient(Client): self._session = session_for_reply_channel(self.reply_channel) return self._session - def receive(self): + def receive(self, json=True): + """ + Return text content of a message for client channel and decoding it if json kwarg is set + """ content = super(HttpClient, self).receive() - if content: - return json.loads(content['text']) + if content and json: + return json_module.loads(content['text']) + return content['text'] if content else None def send(self, to, content={}, text=None, path='/'): """ @@ -71,8 +79,11 @@ class HttpClient(Client): content.setdefault('path', path) content.setdefault('headers', self.headers) text = text or content.get('text', None) - if text: - content['text'] = json.dumps(text) + if text is not None: + if not isinstance(text, six.string_types): + content['text'] = json.dumps(text) + else: + content['text'] = text self.channel_layer.send(to, content) def send_and_consume(self, channel, content={}, text=None, path='/', fail_on_none=True): From d1590afdcb364b90500b3aacffaa03bd30200840 Mon Sep 17 00:00:00 2001 From: qwitwa Date: Mon, 12 Sep 2016 11:59:46 +0100 Subject: [PATCH 524/746] Fix typo in code in models section (#319) Changed channel_session['room'] inside websockets consumer function to message.channel_session['room'] --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 4bdda26..1927652 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -595,7 +595,7 @@ have a ChatMessage model with ``message`` and ``room`` fields:: def ws_message(message): # Stick the message onto the processing queue Channel("chat-messages").send({ - "room": channel_session['room'], + "room": message.channel_session['room'], "message": message['text'], }) From 8e84c0ce872aa624b727744e8d34f1434b7c5897 Mon Sep 17 00:00:00 2001 From: benny daon Date: Wed, 14 Sep 2016 07:57:23 +0300 Subject: [PATCH 525/746] reply and response are confusing (#361) I'm not sure I got it right, but based on the attribute name, `response` belong to the HTTP domain and `reply` to the channel domain. --- docs/concepts.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 086ee63..6ee89dd 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -143,8 +143,8 @@ this model. The first, and more obvious one, is the dispatching of work to consumers - a message gets added to a channel, and then any one of the workers can pick it up and run the consumer. -The second kind of channel, however, is used for responses. Notably, these only -have one thing listening on them - the interface server. Each response channel +The second kind of channel, however, is used for replies. Notably, these only +have one thing listening on them - the interface server. Each reply channel is individually named and has to be routed back to the interface server where its client is terminated. @@ -156,9 +156,9 @@ the message - but response channels would have to have their messages sent to the channel server they're listening on. For this reason, Channels treats these as two different *channel types*, and -denotes a *response channel* by having the channel name contain +denotes a *reply channel* by having the channel name contain the character ``!`` - e.g. ``http.response!f5G3fE21f``. *Normal -channels* do not contain it, but along with the rest of the response +channels* do not contain it, but along with the rest of the reply channel name, they must contain only the characters ``a-z A-Z 0-9 - _``, and be less than 200 characters long. @@ -172,7 +172,7 @@ Groups Because channels only deliver to a single listener, they can't do broadcast; if you want to send a message to an arbitrary group of clients, you need to -keep track of which response channels of those you wish to send to. +keep track of which reply channels of those you wish to send to. If I had a liveblog where I wanted to push out updates whenever a new post is saved, I could register a handler for the ``post_save`` signal and keep a @@ -182,7 +182,7 @@ set of channels (here, using Redis) to send updates to:: @receiver(post_save, sender=BlogUpdate) def send_update(sender, instance, **kwargs): - # Loop through all response channels and send the update + # Loop through all reply channels and send the update for reply_channel in redis_conn.smembers("readers"): Channel(reply_channel).send({ "text": json.dumps({ @@ -201,7 +201,7 @@ the ``readers`` set when they disconnect. We could add a consumer that listens to ``websocket.disconnect`` to do that, but we'd also need to have some kind of expiry in case an interface server is forced to quit or loses power before it can send disconnect signals - your code will never -see any disconnect notification but the response channel is completely +see any disconnect notification but the reply channel is completely invalid and messages you send there will sit there until they expire. Because the basic design of channels is stateless, the channel server has no @@ -249,7 +249,7 @@ Of course, you should still remove things from the group on disconnect if you can; the expiry code is there to catch cases where the disconnect message doesn't make it for some reason. -Groups are generally only useful for response channels (ones containing +Groups are generally only useful for reply channels (ones containing the character ``!``), as these are unique-per-client, but can be used for normal channels as well if you wish. From 58cc3c845df07cc8c5755fad7488800e096a2937 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 12 Sep 2016 11:58:09 +0100 Subject: [PATCH 526/746] Onopen timing fixes in docs --- docs/getting-started.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 1927652..8d1fddc 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -141,6 +141,8 @@ need to change the socket address if you're using a development VM or similar):: socket.onopen = function() { socket.send("hello world"); } + // Call onopen directly if socket is already open + if (socket.readyState == WebSocket.OPEN) socket.onopen(); You should see an alert come back immediately saying "hello world" - your message has round-tripped through the server and come back to trigger the alert. @@ -235,6 +237,8 @@ code in the developer console as before:: socket.onopen = function() { socket.send("hello world"); } + // Call onopen directly if socket is already open + if (socket.readyState == WebSocket.OPEN) socket.onopen(); You should see an alert come back immediately saying "hello world" - but this time, you can open another tab and do the same there, and both tabs will From 06fd1f8ada1e4ab51de0577bc5fc15fd733abd86 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Thu, 22 Sep 2016 22:57:30 +0200 Subject: [PATCH 527/746] Add conventional request.META['PATH_INFO'] - fixes benjaoming/django-nyt#27 (#375) --- channels/handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/handler.py b/channels/handler.py index 4e1c73d..5015902 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -55,6 +55,7 @@ class AsgiRequest(http.HttpRequest): "REQUEST_METHOD": self.method, "QUERY_STRING": self.message.get('query_string', ''), "SCRIPT_NAME": self.script_name, + "PATH_INFO": self.path_info, # Old code will need these for a while "wsgi.multithread": True, "wsgi.multiprocess": True, From eaaf70e935bf15b176f6efff70f05b77617a8d52 Mon Sep 17 00:00:00 2001 From: Adam Chainz Date: Thu, 22 Sep 2016 22:17:39 +0100 Subject: [PATCH 528/746] Convert readthedocs link for their .org -> .io migration for hosted projects (#374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per [their blog post of the 27th April](https://blog.readthedocs.com/securing-subdomains/) ‘Securing subdomains’: > Starting today, Read the Docs will start hosting projects from subdomains on the domain readthedocs.io, instead of on readthedocs.org. This change addresses some security concerns around site cookies while hosting user generated data on the same domain as our dashboard. Test Plan: Manually visited all the links I’ve modified. --- CONTRIBUTING.rst | 2 +- README.rst | 12 ++++++------ patchinator.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index de0aef1..dcd4b56 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -10,7 +10,7 @@ Examples of contributions include: * Documentation improvements * Bug reports and patch reviews -For more information, please see our `contribution guide `_. +For more information, please see our `contribution guide `_. Quick Setup ----------- diff --git a/README.rst b/README.rst index c6fa512..8d483b3 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ Django Channels :target: https://travis-ci.org/django/channels .. image:: https://readthedocs.org/projects/channels/badge/?version=latest - :target: http://channels.readthedocs.org/en/latest/?badge=latest + :target: https://channels.readthedocs.io/en/latest/?badge=latest .. image:: https://img.shields.io/pypi/v/channels.svg :target: https://pypi.python.org/pypi/channels @@ -23,21 +23,21 @@ a bit as things develop. Once we hit ``1.0``, it will be stablized and a deprecation policy will come in. Documentation, installation and getting started instructions are at -http://channels.readthedocs.org +https://channels.readthedocs.io Support can be obtained either here via issues, or in the ``#django-channels`` channel on Freenode. You can install channels from PyPI as the ``channels`` package. You'll likely also want to ``asgi_redis`` to provide the Redis channel layer. -See our `installation `_ -and `getting started `_ docs for more. +See our `installation `_ +and `getting started `_ docs for more. Contributing ------------ -To learn more about contributing, please `read our contributing docs `_. +To learn more about contributing, please `read our contributing docs `_. Maintenance and Security @@ -60,7 +60,7 @@ Maintenance team: * Jeremy Spencer If you are interested in joining the maintenance team, please -`read more about contributing `_ +`read more about contributing `_ and get in touch! diff --git a/patchinator.py b/patchinator.py index f672924..fe787a1 100644 --- a/patchinator.py +++ b/patchinator.py @@ -139,7 +139,7 @@ docs_transforms = global_transforms + [ Replacement(r":doc:`getting-started`", r":doc:`/intro/channels`"), Replacement(r"`", r"`"), Replacement(r":doc:`backends`", r":doc:`/ref/channels/backends`"), - Replacement(r":doc:`([\w\d\s]+) `", r"`\1 `_"), + Replacement(r":doc:`([\w\d\s]+) `", r"`\1 `_"), Replacement(r"\n\(.*installation>`\)\n", r""), Replacement(r":doc:`installed Channels correctly `", r"added the channel layer setting"), Replacement(r"Channels", r"channels"), From d4f7125cd539adf195f93a9788225fce8a84304b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carles=20Barrob=C3=A9s?= Date: Fri, 23 Sep 2016 20:37:45 +0200 Subject: [PATCH 529/746] Fix type for request.META['SERVER_PORT'] (#378) Django documentation states that it is a string. Fixes #366 --- channels/handler.py | 2 +- channels/tests/test_request.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index 5015902..62922bf 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -66,7 +66,7 @@ class AsgiRequest(http.HttpRequest): self.META['REMOTE_PORT'] = self.message['client'][1] if self.message.get('server', None): self.META['SERVER_NAME'] = self.message['server'][0] - self.META['SERVER_PORT'] = self.message['server'][1] + self.META['SERVER_PORT'] = six.text_type(self.message['server'][1]) # Handle old style-headers for a transition period if "headers" in self.message and isinstance(self.message['headers'], dict): self.message['headers'] = [ diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index 66fae9e..31c711c 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -63,7 +63,7 @@ class RequestTests(ChannelTestCase): self.assertEqual(request.META["REMOTE_HOST"], "10.0.0.1") self.assertEqual(request.META["REMOTE_PORT"], 1234) self.assertEqual(request.META["SERVER_NAME"], "10.0.0.2") - self.assertEqual(request.META["SERVER_PORT"], 80) + self.assertEqual(request.META["SERVER_PORT"], "80") self.assertEqual(request.GET["x"], "1") self.assertEqual(request.GET["y"], "&foo bar+baz") self.assertEqual(request.COOKIES["test-time"], "1448995585123") From b115f8fa04e93562b69c2dfe968922cadc430445 Mon Sep 17 00:00:00 2001 From: Sam Bolgert Date: Thu, 29 Sep 2016 11:08:44 -0700 Subject: [PATCH 530/746] Update channel_session decorator to rehydrate http_session (#348) * Update channel_session decorator to rehydrate http_session Update the http_session decorator to write the http session key to the channel_session when available. This allows the channel_session decorator to rehydrate the http_session after the initial websocket connection. closes #318 * Add persist=True option to http_session * Add explicit option to store the session key in the channel session * Update docs * Add test case * Add channel_and_http_session decorator This decorator enables both sessions and maintains the http_session for the lifetime of the websocket connection. --- channels/sessions.py | 24 +++++++++++++++++++++ channels/tests/test_sessions.py | 38 ++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/channels/sessions.py b/channels/sessions.py index ea2e696..8e3c254 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -183,3 +183,27 @@ def http_session(func): session.save() return result return inner + + +def channel_and_http_session(func): + """ + Enables both the channel_session and http_session. + + Stores the http session key in the channel_session on websocket.connect messages. + It will then hydrate the http_session from that same key on subsequent messages. + """ + @http_session + @channel_session + @functools.wraps(func) + def inner(message, *args, **kwargs): + # Store the session key in channel_session + if message.http_session is not None and settings.SESSION_COOKIE_NAME not in message.channel_session: + message.channel_session[settings.SESSION_COOKIE_NAME] = message.http_session.session_key + # Hydrate the http_session from session_key + elif message.http_session is None and settings.SESSION_COOKIE_NAME in message.channel_session: + session_engine = import_module(settings.SESSION_ENGINE) + session = session_engine.SessionStore(session_key=message.channel_session[settings.SESSION_COOKIE_NAME]) + message.http_session = session + # Run the consumer + return func(message, *args, **kwargs) + return inner diff --git a/channels/tests/test_sessions.py b/channels/tests/test_sessions.py index da65f7a..9102aea 100644 --- a/channels/tests/test_sessions.py +++ b/channels/tests/test_sessions.py @@ -3,7 +3,8 @@ from __future__ import unicode_literals from django.conf import settings from django.test import override_settings from channels.message import Message -from channels.sessions import channel_session, http_session, enforce_ordering, session_for_reply_channel +from channels.sessions import channel_session, channel_and_http_session, http_session, enforce_ordering, \ + session_for_reply_channel from channels.tests import ChannelTestCase from channels import DEFAULT_CHANNEL_LAYER, channel_layers @@ -105,6 +106,41 @@ class SessionTests(ChannelTestCase): session2 = session_for_reply_channel("test-reply") self.assertEqual(session2["species"], "horse") + def test_channel_and_http_session(self): + """ + Tests that channel_and_http_session decorator stores the http session key and hydrates it when expected + """ + # Make a session to try against + session = session_for_reply_channel("test-reply-session") + # Construct message to send + message = Message({ + "reply_channel": "test-reply-session", + "http_version": "1.1", + "method": "GET", + "path": "/test2/", + "headers": { + "host": b"example.com", + "cookie": ("%s=%s" % (settings.SESSION_COOKIE_NAME, session.session_key)).encode("ascii"), + }, + }, None, None) + + @channel_and_http_session + def inner(message): + pass + + inner(message) + + # It should store the session key + self.assertEqual(message.channel_session[settings.SESSION_COOKIE_NAME], session.session_key) + + # Construct a new message + message2 = Message({"reply_channel": "test-reply-session", "path": "/"}, None, None) + + inner(message2) + + # It should hydrate the http_session + self.assertEqual(message2.http_session.session_key, session.session_key) + def test_enforce_ordering_slight(self): """ Tests that slight mode of enforce_ordering works From 3531ba6bbdf0cb1274fd967390ac8038aae3fd9b Mon Sep 17 00:00:00 2001 From: Steve Steiner Date: Fri, 30 Sep 2016 21:11:20 -0400 Subject: [PATCH 531/746] Add GitHub issue template (#382) * Add github issue template to collect version info etc. * Evened up whitespace --- .github/ISSUE_TEMPLATE.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..1d2996f --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,27 @@ +## Submit a feature request or bug report + +### Versions + +OS/Version: + +Django: + +Channels: + +Daphne: + +Twisted: + +### Connection Information + +**Are you connecting to daphne/runserver directly? If not, please describe the connection.** + + +**What is the current behavior? (please include logs/output)** + + +**What is the expected or desired behavior?** + + +**Any other/relevant information:** + From 40316619a1f341583fe648e3ed3843b79b95305e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 1 Oct 2016 10:43:36 -0700 Subject: [PATCH 532/746] Making the issue template a bit more generic --- .github/ISSUE_TEMPLATE.md | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 1d2996f..9e8ea48 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,27 +1,13 @@ -## Submit a feature request or bug report +If you're submitting a feature request, please try to include: -### Versions +- Detailed description of the overall behaviour +- Reasoning why it should be in Channels rather than a third-party app +- Examples of usage, if possible (it's easier to discuss concrete code examples) -OS/Version: - -Django: - -Channels: - -Daphne: - -Twisted: - -### Connection Information - -**Are you connecting to daphne/runserver directly? If not, please describe the connection.** - - -**What is the current behavior? (please include logs/output)** - - -**What is the expected or desired behavior?** - - -**Any other/relevant information:** +If you're submitting a bug report, please include: +- Your OS and runtime environment, and browser if applicable +- The versions of Channels, Daphne, Django, Twisted, and your ASGI backend (asgi_ipc or asgi_redis normally) +- What you expected to happen vs. what actually happened +- How you're running Channels (runserver? daphne/runworker? Nginx/Apache in front?) +- Console logs and full tracebacks of any errors From 1be6dd5b71020a40f8c7f9003135f5502c48b0b1 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 3 Oct 2016 16:38:23 -0700 Subject: [PATCH 533/746] Update docs to mention where to run JS console --- docs/getting-started.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 8d1fddc..104c7f5 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -128,9 +128,11 @@ When it gets that message, it takes the ``reply_channel`` attribute from it, whi is the unique response channel for that client, and sends the same content back to the client using its ``send()`` method. -Let's test it! Run ``runserver``, open a browser and put the following into the -JavaScript console to open a WebSocket and send some data down it (you might -need to change the socket address if you're using a development VM or similar):: +Let's test it! Run ``runserver``, open a browser, navigate to a page on the server +(you can't use any page's console because of origin restrictions), and put the +following into the JavaScript console to open a WebSocket and send some data +down it (you might need to change the socket address if you're using a +development VM or similar):: // Note that the path doesn't matter for routing; any WebSocket // connection gets bumped over to WebSocket consumers From dcfaf4122b0fdd591469f74d207c701c2fce16d7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 3 Oct 2016 16:38:37 -0700 Subject: [PATCH 534/746] Work in progress towards accepting websockets explicitly --- channels/channel.py | 3 + channels/consumer_middleware.py | 87 +++++++++++++++++++++++ channels/exceptions.py | 8 +++ channels/management/commands/runserver.py | 4 ++ channels/routing.py | 10 ++- channels/signals.py | 4 ++ channels/worker.py | 5 +- docs/asgi.rst | 37 +++++++++- docs/getting-started.rst | 8 ++- 9 files changed, 161 insertions(+), 5 deletions(-) create mode 100644 channels/consumer_middleware.py diff --git a/channels/channel.py b/channels/channel.py index b65d65a..4d2796b 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from django.utils import six from channels import DEFAULT_CHANNEL_LAYER, channel_layers +from .signals import message_sent class Channel(object): @@ -36,6 +37,8 @@ class Channel(object): if not isinstance(content, dict): raise TypeError("You can only send dicts as content on channels.") self.channel_layer.send(self.name, content) + message_sent.send(sender=self.__class__, channel=self.name, keys=list(content.keys())) + print("didsig", self.name) def __str__(self): return self.name diff --git a/channels/consumer_middleware.py b/channels/consumer_middleware.py new file mode 100644 index 0000000..869863e --- /dev/null +++ b/channels/consumer_middleware.py @@ -0,0 +1,87 @@ +from __future__ import unicode_literals + +import importlib +import threading +from django.conf import settings + +from .exceptions import DenyConnection +from .signals import consumer_started, consumer_finished, message_sent + + +class ConsumerMiddlewareRegistry(object): + """ + Handles registration (via settings object) and generation of consumer + middleware stacks + """ + + fixed_middleware = ["channels.consumer_middleware.ConvenienceMiddleware"] + + def __init__(self): + # Load middleware callables from settings + middleware_paths = self.fixed_middleware + getattr(settings, "CONSUMER_MIDDLEWARE", []) + self.middleware_instances = [] + for path in middleware_paths: + module_name, variable_name = path.rsplit(".", 1) + try: + self.middleware_instances.append(getattr(importlib.import_module(module_name), variable_name)) + except (ImportError, AttributeError) as e: + raise ImproperlyConfigured("Cannot import consumer middleware %r: %s" % (path, e)) + + def make_chain(self, consumer, kwargs): + """ + Returns an instantiated chain of middleware around a final consumer. + """ + next_layer = lambda message: consumer(message, **kwargs) + for middleware_instance in reversed(self.middleware_instances): + next_layer = middleware_instance(next_layer) + return next_layer + + +class ConvenienceMiddleware(object): + """ + Standard middleware which papers over some more explicit parts of ASGI. + """ + + runtime_data = threading.local() + + def __init__(self, consumer): + self.consumer = consumer + + def __call__(self, message): + print("conven", message.channel) + if message.channel.name == "websocket.connect": + # Websocket connect acceptance helper + try: + self.consumer(message) + print ("messages sent", self.get_messages()) + except DenyConnection: + message.reply_channel.send({"accept": False}) + else: + # General path + return self.consumer(message) + + @classmethod + def reset_messages(cls, **kwargs): + """ + Tied to the consumer started/ended signal to reset the messages list. + """ + cls.runtime_data.sent_messages = [] + + consumer_started.connect(lambda **kwargs: reset_messages()) + consumer_finished.connect(lambda **kwargs: reset_messages()) + + @classmethod + def sent_message(cls, channel, keys, **kwargs): + """ + Called by message sending interfaces when messages are sent, + for convenience errors only. Should not be relied upon to get + all messages. + """ + cls.runtime_data.sent_messages = getattr(cls.runtime_data, "sent_messages", []) + [(channel, keys)] + print ("saved now", cls.runtime_data.sent_messages) + + message_sent.connect(lambda channel, keys, **kwargs: sent_message(channel, keys)) + + @classmethod + def get_messages(cls): + return getattr(cls.runtime_data, "sent_messages", []) diff --git a/channels/exceptions.py b/channels/exceptions.py index ffdb5a2..d81af8b 100644 --- a/channels/exceptions.py +++ b/channels/exceptions.py @@ -29,3 +29,11 @@ class RequestAborted(Exception): reading the body. """ pass + + +class DenyConnection(Exception): + """ + Raise during a websocket.connect (or other supported connection) handler + to deny the connection. + """ + pass diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 3b68220..e52b018 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -121,6 +121,10 @@ class Command(RunserverCommand): msg += "WebSocket CONNECT %(path)s [%(client)s]\n" % details elif protocol == "websocket" and action == "disconnected": msg += "WebSocket DISCONNECT %(path)s [%(client)s]\n" % details + elif protocol == "websocket" and action == "connecting": + msg += "WebSocket HANDSHAKING %(path)s [%(client)s]\n" % details + elif protocol == "websocket" and action == "rejected": + msg += "WebSocket REJECT %(path)s [%(client)s]\n" % details sys.stderr.write(msg) diff --git a/channels/routing.py b/channels/routing.py index 76c2464..cb3ea03 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -56,8 +56,9 @@ class Router(object): # We also add a no-op websocket.connect consumer to the bottom, as the # spec requires that this is consumed, but Channels does not. Any user # consumer will override this one. Same for websocket.receive. - self.add_route(Route("websocket.connect", null_consumer)) + self.add_route(Route("websocket.connect", connect_consumer)) self.add_route(Route("websocket.receive", null_consumer)) + self.add_route(Route("websocket.disconnect", null_consumer)) @classmethod def resolve_routing(cls, routing): @@ -250,6 +251,13 @@ def null_consumer(*args, **kwargs): """ +def connect_consumer(message, *args, **kwargs): + """ + Accept-all-connections websocket.connect consumer + """ + message.reply_channel.send({"accept": True}) + + # Lowercase standard to match urls.py route = Route route_class = RouteClass diff --git a/channels/signals.py b/channels/signals.py index dc83b94..0a0e575 100644 --- a/channels/signals.py +++ b/channels/signals.py @@ -7,5 +7,9 @@ consumer_finished = Signal() worker_ready = Signal() worker_process_ready = Signal() +# Called when a message is sent directly to a channel. Not called for group +# sends or direct ASGI usage. For convenience/nicer errors only. +message_sent = Signal(providing_args=["channel", "keys"]) + # Connect connection closer to consumer finished as well consumer_finished.connect(close_old_connections) diff --git a/channels/worker.py b/channels/worker.py index 3b67e92..4f48615 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -13,6 +13,7 @@ from .exceptions import ConsumeLater from .message import Message from .utils import name_that_thing from .signals import worker_ready +from .consumer_middleware import ConsumerMiddlewareRegistry logger = logging.getLogger('django.channels') @@ -40,6 +41,7 @@ class Worker(object): self.exclude_channels = exclude_channels self.termed = False self.in_job = False + self.middleware_registry = ConsumerMiddlewareRegistry() def install_signal_handler(self): signal.signal(signal.SIGTERM, self.sigterm_handler) @@ -117,7 +119,8 @@ class Worker(object): # Send consumer started to manage lifecycle stuff consumer_started.send(sender=self.__class__, environ={}) # Run consumer - consumer(message, **kwargs) + chain = self.middleware_registry.make_chain(consumer, kwargs) + chain(message) except ConsumeLater: # They want to not handle it yet. Re-inject it with a number-of-tries marker. content['__retries__'] = content.get("__retries__", 0) + 1 diff --git a/docs/asgi.rst b/docs/asgi.rst index e452217..919d077 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -704,7 +704,7 @@ Keys: * ``reply_channel``: Channel name responses would have been sent on. No longer valid after this message is sent; all messages to it will be dropped. - + * ``path``: Unicode string HTTP path from URL, with percent escapes decoded and UTF8 byte sequences decoded into characters. @@ -731,7 +731,21 @@ Connection Sent when the client initially opens a connection and completes the WebSocket handshake. If sending this raises ``ChannelFull``, the interface -server must close the WebSocket connection with error code 1013. +server must close the connection with either HTTP status code ``503`` or +WebSocket close code ``1013``. + +This message must be responded to on the ``reply_channel`` with a +*Connection Reply* message before the socket will pass messages on the +``receive`` channel. The protocol server should ideally send this message +during the handshake phase of the WebSocket and not complete the handshake +until it gets a reply, returning HTTP status code ``403`` if the connection is +denied. If this is not possible, it must buffer WebSocket frames and not +send them onto ``websocket.receive`` until a reply is received, and if the +connection is rejected, return WebSocket close code ``4403``. + +Receiving a WebSocket *Send/Close* message while waiting for a +*Connection Reply* must make the server accept the connection and then send +the message immediately. Channel: ``websocket.connect`` @@ -768,6 +782,22 @@ Keys: * ``order``: The integer value ``0``. +Connection Reply +'''''''''''''''' + +Sent back on the reply channel from an application when a ``connect`` message +is received to say if the connection should be accepted or dropped. + +Behaviour on WebSocket rejection is defined in the Connection section above. + +Channel: ``websocket.send!`` + +Keys: + +* ``accept``: If the connection should be accepted (``True``) or rejected and + dropped (``False``). + + Receive ''''''' @@ -825,6 +855,9 @@ Send/Close Sends a data frame to the client and/or closes the connection from the server end. If ``ChannelFull`` is raised, wait and try again. +If sent while the connection is waiting for acceptance or rejection, +will accept the connection before the frame is sent. + Channel: ``websocket.send!`` Keys: diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 104c7f5..37663a0 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -105,7 +105,7 @@ for ``http.request`` - and make this WebSocket consumer instead:: def ws_message(message): # ASGI WebSocket packet-received and send-packet message types - # both have a "text" key for their textual data. + # both have a "text" key for their textual data. message.reply_channel.send({ "text": message.content['text'], }) @@ -165,6 +165,7 @@ disconnect, like this:: # Connected to websocket.connect def ws_add(message): + message.reply_channel.send({"accept": True}) Group("chat").add(message.reply_channel) # Connected to websocket.disconnect @@ -203,6 +204,7 @@ get the message. Here's all the code:: # Connected to websocket.connect def ws_add(message): + message.reply_channel.send({"accept": True}) Group("chat").add(message.reply_channel) # Connected to websocket.receive @@ -363,6 +365,8 @@ name in the path of your WebSocket request (we'll ignore auth for now - that's n # Connected to websocket.connect @channel_session def ws_connect(message): + # Accept connection + message.reply_channel.send({"accept": True}) # Work out room name from path (ignore slashes) room = message.content['path'].strip("/") # Save room in session and add us to the group @@ -462,6 +466,8 @@ chat to people with the same first letter of their username:: # Connected to websocket.connect @channel_session_user_from_http def ws_add(message): + # Accept connection + message.reply_channel.send({"accept": True}) # Add them to the right group Group("chat-%s" % message.user.username[0]).add(message.reply_channel) From 0fcb93acc2e2f69797f9b4da0b7ab1c45e439873 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Oct 2016 11:42:46 -0700 Subject: [PATCH 535/746] Mostly-complete middleware version --- channels/channel.py | 1 - channels/consumer_middleware.py | 16 ++++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/channels/channel.py b/channels/channel.py index 4d2796b..a308e9e 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -38,7 +38,6 @@ class Channel(object): raise TypeError("You can only send dicts as content on channels.") self.channel_layer.send(self.name, content) message_sent.send(sender=self.__class__, channel=self.name, keys=list(content.keys())) - print("didsig", self.name) def __str__(self): return self.name diff --git a/channels/consumer_middleware.py b/channels/consumer_middleware.py index 869863e..34284dc 100644 --- a/channels/consumer_middleware.py +++ b/channels/consumer_middleware.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import importlib import threading +import warnings from django.conf import settings from .exceptions import DenyConnection @@ -48,14 +49,18 @@ class ConvenienceMiddleware(object): self.consumer = consumer def __call__(self, message): - print("conven", message.channel) if message.channel.name == "websocket.connect": # Websocket connect acceptance helper try: self.consumer(message) - print ("messages sent", self.get_messages()) except DenyConnection: message.reply_channel.send({"accept": False}) + else: + replies_sent = [msg for chan, msg in self.get_messages() if chan == message.reply_channel.name] + # If they sent no replies, send implicit acceptance + if not replies_sent: + warnings.warn("AAAAAAAAAAA", RuntimeWarning) + message.reply_channel.send({"accept": True}) else: # General path return self.consumer(message) @@ -67,8 +72,8 @@ class ConvenienceMiddleware(object): """ cls.runtime_data.sent_messages = [] - consumer_started.connect(lambda **kwargs: reset_messages()) - consumer_finished.connect(lambda **kwargs: reset_messages()) + consumer_started.connect(lambda **kwargs: ConvenienceMiddleware.reset_messages(), weak=False) + consumer_finished.connect(lambda **kwargs: ConvenienceMiddleware.reset_messages(), weak=False) @classmethod def sent_message(cls, channel, keys, **kwargs): @@ -78,9 +83,8 @@ class ConvenienceMiddleware(object): all messages. """ cls.runtime_data.sent_messages = getattr(cls.runtime_data, "sent_messages", []) + [(channel, keys)] - print ("saved now", cls.runtime_data.sent_messages) - message_sent.connect(lambda channel, keys, **kwargs: sent_message(channel, keys)) + message_sent.connect(lambda channel, keys, **kwargs: ConvenienceMiddleware.sent_message(channel, keys), weak=False) @classmethod def get_messages(cls): From db0d2975a0a92082dd7b0f7b11ec5207f5e74c93 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Oct 2016 12:06:34 -0700 Subject: [PATCH 536/746] Remove middleware approach, change to simpler one --- channels/channel.py | 2 - channels/consumer_middleware.py | 91 --------------------------------- channels/signals.py | 4 -- channels/worker.py | 12 +++-- docs/asgi.rst | 3 ++ 5 files changed, 10 insertions(+), 102 deletions(-) delete mode 100644 channels/consumer_middleware.py diff --git a/channels/channel.py b/channels/channel.py index a308e9e..b65d65a 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals from django.utils import six from channels import DEFAULT_CHANNEL_LAYER, channel_layers -from .signals import message_sent class Channel(object): @@ -37,7 +36,6 @@ class Channel(object): if not isinstance(content, dict): raise TypeError("You can only send dicts as content on channels.") self.channel_layer.send(self.name, content) - message_sent.send(sender=self.__class__, channel=self.name, keys=list(content.keys())) def __str__(self): return self.name diff --git a/channels/consumer_middleware.py b/channels/consumer_middleware.py deleted file mode 100644 index 34284dc..0000000 --- a/channels/consumer_middleware.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import unicode_literals - -import importlib -import threading -import warnings -from django.conf import settings - -from .exceptions import DenyConnection -from .signals import consumer_started, consumer_finished, message_sent - - -class ConsumerMiddlewareRegistry(object): - """ - Handles registration (via settings object) and generation of consumer - middleware stacks - """ - - fixed_middleware = ["channels.consumer_middleware.ConvenienceMiddleware"] - - def __init__(self): - # Load middleware callables from settings - middleware_paths = self.fixed_middleware + getattr(settings, "CONSUMER_MIDDLEWARE", []) - self.middleware_instances = [] - for path in middleware_paths: - module_name, variable_name = path.rsplit(".", 1) - try: - self.middleware_instances.append(getattr(importlib.import_module(module_name), variable_name)) - except (ImportError, AttributeError) as e: - raise ImproperlyConfigured("Cannot import consumer middleware %r: %s" % (path, e)) - - def make_chain(self, consumer, kwargs): - """ - Returns an instantiated chain of middleware around a final consumer. - """ - next_layer = lambda message: consumer(message, **kwargs) - for middleware_instance in reversed(self.middleware_instances): - next_layer = middleware_instance(next_layer) - return next_layer - - -class ConvenienceMiddleware(object): - """ - Standard middleware which papers over some more explicit parts of ASGI. - """ - - runtime_data = threading.local() - - def __init__(self, consumer): - self.consumer = consumer - - def __call__(self, message): - if message.channel.name == "websocket.connect": - # Websocket connect acceptance helper - try: - self.consumer(message) - except DenyConnection: - message.reply_channel.send({"accept": False}) - else: - replies_sent = [msg for chan, msg in self.get_messages() if chan == message.reply_channel.name] - # If they sent no replies, send implicit acceptance - if not replies_sent: - warnings.warn("AAAAAAAAAAA", RuntimeWarning) - message.reply_channel.send({"accept": True}) - else: - # General path - return self.consumer(message) - - @classmethod - def reset_messages(cls, **kwargs): - """ - Tied to the consumer started/ended signal to reset the messages list. - """ - cls.runtime_data.sent_messages = [] - - consumer_started.connect(lambda **kwargs: ConvenienceMiddleware.reset_messages(), weak=False) - consumer_finished.connect(lambda **kwargs: ConvenienceMiddleware.reset_messages(), weak=False) - - @classmethod - def sent_message(cls, channel, keys, **kwargs): - """ - Called by message sending interfaces when messages are sent, - for convenience errors only. Should not be relied upon to get - all messages. - """ - cls.runtime_data.sent_messages = getattr(cls.runtime_data, "sent_messages", []) + [(channel, keys)] - - message_sent.connect(lambda channel, keys, **kwargs: ConvenienceMiddleware.sent_message(channel, keys), weak=False) - - @classmethod - def get_messages(cls): - return getattr(cls.runtime_data, "sent_messages", []) diff --git a/channels/signals.py b/channels/signals.py index 0a0e575..dc83b94 100644 --- a/channels/signals.py +++ b/channels/signals.py @@ -7,9 +7,5 @@ consumer_finished = Signal() worker_ready = Signal() worker_process_ready = Signal() -# Called when a message is sent directly to a channel. Not called for group -# sends or direct ASGI usage. For convenience/nicer errors only. -message_sent = Signal(providing_args=["channel", "keys"]) - # Connect connection closer to consumer finished as well consumer_finished.connect(close_old_connections) diff --git a/channels/worker.py b/channels/worker.py index 4f48615..bad22df 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -9,11 +9,10 @@ import multiprocessing import threading from .signals import consumer_started, consumer_finished -from .exceptions import ConsumeLater +from .exceptions import ConsumeLater, DenyConnection from .message import Message from .utils import name_that_thing from .signals import worker_ready -from .consumer_middleware import ConsumerMiddlewareRegistry logger = logging.getLogger('django.channels') @@ -41,7 +40,6 @@ class Worker(object): self.exclude_channels = exclude_channels self.termed = False self.in_job = False - self.middleware_registry = ConsumerMiddlewareRegistry() def install_signal_handler(self): signal.signal(signal.SIGTERM, self.sigterm_handler) @@ -119,8 +117,12 @@ class Worker(object): # Send consumer started to manage lifecycle stuff consumer_started.send(sender=self.__class__, environ={}) # Run consumer - chain = self.middleware_registry.make_chain(consumer, kwargs) - chain(message) + consumer(message, **kwargs) + except DenyConnection: + # They want to deny a WebSocket connection. + if message.channel.name != "websocket.connect": + raise ValueError("You cannot DenyConnection from a non-websocket.connect handler.") + message.reply_channel.send({"accept": False}) except ConsumeLater: # They want to not handle it yet. Re-inject it with a number-of-tries marker. content['__retries__'] = content.get("__retries__", 0) + 1 diff --git a/docs/asgi.rst b/docs/asgi.rst index 919d077..0a5b68b 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -790,6 +790,9 @@ is received to say if the connection should be accepted or dropped. Behaviour on WebSocket rejection is defined in the Connection section above. +If received while the socket is already accepted, the protocol server should +log an error, but not do anything. + Channel: ``websocket.send!`` Keys: From 0826b7997fc30294943913069a63a5a0df3041ad Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Oct 2016 14:49:46 -0700 Subject: [PATCH 537/746] Send messages after the end of consumers --- channels/channel.py | 25 +++++++++++++++++++++---- channels/generic/websockets.py | 4 ++-- channels/message.py | 24 ++++++++++++++++++++++++ channels/sessions.py | 18 ++++++++---------- channels/worker.py | 2 +- 5 files changed, 56 insertions(+), 17 deletions(-) diff --git a/channels/channel.py b/channels/channel.py index b65d65a..03a2db6 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -29,13 +29,20 @@ class Channel(object): else: self.channel_layer = channel_layers[alias] - def send(self, content): + def send(self, content, immediately=False): """ Send a message over the channel - messages are always dicts. + + Sends are delayed until consumer completion. To override this, you + may pass immediately=True. """ if not isinstance(content, dict): raise TypeError("You can only send dicts as content on channels.") - self.channel_layer.send(self.name, content) + if immediately: + self.channel_layer.send(self.name, content) + else: + from .message import pending_message_store + pending_message_store.append(self, content) def __str__(self): return self.name @@ -66,7 +73,17 @@ class Group(object): channel = channel.name self.channel_layer.group_discard(self.name, channel) - def send(self, content): + def send(self, content, immediately=False): + """ + Send a message to all channels in the group. + + Sends are delayed until consumer completion. To override this, you + may pass immediately=True. + """ if not isinstance(content, dict): raise ValueError("You can only send dicts as content on channels.") - self.channel_layer.send_group(self.name, content) + if immediately: + self.channel_layer.send_group(self.name, content) + else: + from .message import pending_message_store + pending_message_store.append(self, content) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 272b938..4890f3d 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -23,7 +23,7 @@ class WebsocketConsumer(BaseConsumer): # implies channel_session_user http_user = False - # Set one to True if you want the class to enforce ordering for you + # Set to True if you want the class to enforce ordering for you slight_ordering = False strict_ordering = False @@ -47,7 +47,7 @@ class WebsocketConsumer(BaseConsumer): if self.strict_ordering: return enforce_ordering(handler, slight=False) elif self.slight_ordering: - return enforce_ordering(handler, slight=True) + raise ValueError("Slight ordering is now always on. Please remove `slight_ordering=True`.") else: return handler diff --git a/channels/message.py b/channels/message.py index a44ecc7..b9c5ca1 100644 --- a/channels/message.py +++ b/channels/message.py @@ -1,7 +1,9 @@ from __future__ import unicode_literals import copy +import threading from .channel import Channel +from .signals import consumer_finished class Message(object): @@ -58,3 +60,25 @@ class Message(object): self.channel.name, self.channel_layer, ) + + +class PendingMessageStore(object): + """ + Singleton object used for storing pending messages that should be sent + to a channel or group when a consumer finishes. + """ + + threadlocal = threading.local() + + def append(self, sender, message): + if not hasattr(self.threadlocal, "messages"): + self.threadlocal.messages = [] + self.threadlocal.messages.append((sender, message)) + + def send_and_flush(self, **kwargs): + for sender, message in getattr(self.threadlocal, "messages", []): + sender.send(message, immediately=True) + self.threadlocal.messages = [] + +pending_message_store = PendingMessageStore() +consumer_finished.connect(pending_message_store.send_and_flush) diff --git a/channels/sessions.py b/channels/sessions.py index 8e3c254..10607f4 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -71,15 +71,14 @@ def channel_session(func): def enforce_ordering(func=None, slight=False): """ - Enforces either slight (order=0 comes first, everything else isn't ordered) - or strict (all messages exactly ordered) ordering against a reply_channel. + Enforces strict (all messages exactly ordered) ordering against a reply_channel. Uses sessions to track ordering and socket-specific wait channels for unordered messages. - - You cannot mix slight ordering and strict ordering on a channel; slight - ordering does not write to the session after the first message to improve - performance. """ + # Slight is deprecated + if slight: + raise ValueError("Slight ordering is now always on due to Channels changes. Please remove the decorator.") + # Main decorator def decorator(func): @channel_session @functools.wraps(func) @@ -93,13 +92,12 @@ def enforce_ordering(func=None, slight=False): order = int(message.content['order']) # See what the current next order should be next_order = message.channel_session.get("__channels_next_order", 0) - if order == next_order or (slight and next_order > 0): + if order == next_order: # Run consumer func(message, *args, **kwargs) # Mark next message order as available for running - if order == 0 or not slight: - message.channel_session["__channels_next_order"] = order + 1 - message.channel_session.save() + message.channel_session["__channels_next_order"] = order + 1 + message.channel_session.save() # Requeue any pending wait channel messages for this socket connection back onto it's original channel while True: wait_channel = "__wait__.%s" % message.reply_channel.name diff --git a/channels/worker.py b/channels/worker.py index bad22df..658f370 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -145,7 +145,7 @@ class Worker(object): break except: logger.exception("Error processing message with consumer %s:", name_that_thing(consumer)) - else: + finally: # Send consumer finished so DB conns close etc. consumer_finished.send(sender=self.__class__) From 0b8b199212b3c9451b4fad8a7b721da20958a06d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Oct 2016 15:06:41 -0700 Subject: [PATCH 538/746] Add release note section --- docs/index.rst | 1 + docs/releases/1.0.0.rst | 62 +++++++++++++++++++++++++++++++++++++++++ docs/releases/index.rst | 7 +++++ 3 files changed, 70 insertions(+) create mode 100644 docs/releases/1.0.0.rst create mode 100644 docs/releases/index.rst diff --git a/docs/index.rst b/docs/index.rst index 93fe135..365d48d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -57,3 +57,4 @@ Topics asgi community contributing + releases diff --git a/docs/releases/1.0.0.rst b/docs/releases/1.0.0.rst new file mode 100644 index 0000000..8ac2359 --- /dev/null +++ b/docs/releases/1.0.0.rst @@ -0,0 +1,62 @@ +1.0.0 Release Notes +=================== + +.. note:: + These release notes are in development. Channels 1.0.0 is not yet released. + + +Major Features +-------------- + +Channels 1.0 introduces a couple of new major features. + + +WebSocket accept/reject flow +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Rather than be immediately accepted, WebSockets now pause during the handshake +while they send over a message on ``websocket.connect``, and your application +must either accept or reject the connection before the handshake is completed +and messages can be received. + +This has several advantages: + +* You can now reject WebSockets before they even finish connecting, giving + appropriate error codes to browsers and not letting the browser-side socket + ever get into a connected state and send messages. + +* Combined with Consumer Atomicity (below), it means there is no longer any need + for the old "slight ordering" mode, as the connect consumer must run to + completion and accept the socket before any messages can be received and + forwarded onto ``websocket.receive``. + +* Any ``send`` message sent to the WebSocket will implicitly accept the connection, + meaning only a limited set of ``connect`` consumers need changes (see + Backwards Incompatible Changes below) + + +Consumer Atomicity +~~~~~~~~~~~~~~~~~~ + +Consumers will now buffer messages you try to send until the consumer completes +and then send them once it exits and the outbound part of any decorators have +been run (even if an exception is raised). + +This makes the flow of messages much easier to reason about - consumers can now +be reasoned about as atomic blocks that run and then send messages, meaning that +if you send a message to start another consumer you're guaranteed that the +sending consumer has finished running by the time it's acted upon. + +If you want to send messages immediately rather than at the end of the consumer, +you can still do that by passing the ``immediately`` argument:: + + Channel("thumbnailing-tasks").send({"id": 34245}, immediately=True) + +This should be entirely backwards compatible, and may actually fix race +conditions in some apps that were pre-existing. + + +Demultiplexer Overhaul +~~~~~~~~~~~~~~~~~~~~~~ + +TBD diff --git a/docs/releases/index.rst b/docs/releases/index.rst new file mode 100644 index 0000000..7f55d6b --- /dev/null +++ b/docs/releases/index.rst @@ -0,0 +1,7 @@ +Release Notes +============= + +.. toctree:: + :maxdepth: 2 + + 1.0.0 From 0ed04a9c06707388687bc85637baaf4909f88961 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Oct 2016 15:32:37 -0700 Subject: [PATCH 539/746] Fix tests for new non-immediate sending --- channels/tests/base.py | 7 +++- channels/tests/test_binding.py | 10 +++++- channels/tests/test_generic.py | 4 +-- channels/tests/test_handler.py | 2 ++ channels/tests/test_request.py | 28 +++++++-------- channels/tests/test_sessions.py | 64 ++------------------------------- channels/tests/test_worker.py | 4 +-- 7 files changed, 38 insertions(+), 81 deletions(-) diff --git a/channels/tests/base.py b/channels/tests/base.py index 591c11b..bacede4 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -11,6 +11,7 @@ from ..channel import Group from ..routing import Router, include from ..asgi import channel_layers, ChannelLayerWrapper from ..message import Message +from ..signals import consumer_finished, consumer_started from asgiref.inmemory import ChannelLayer as InMemoryChannelLayer @@ -121,7 +122,11 @@ class Client(object): match = self.channel_layer.router.match(message) if match: consumer, kwargs = match - return consumer(message, **kwargs) + try: + consumer_started.send(sender=self.__class__) + return consumer(message, **kwargs) + finally: + consumer_finished.send(sender=self.__class__) elif fail_on_none: raise AssertionError("Can't find consumer for message %s" % message) elif fail_on_none: diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index 56af79b..c3dca66 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -6,6 +6,7 @@ from channels.binding.base import CREATE, UPDATE, DELETE from channels.binding.websockets import WebsocketBinding from channels.generic.websockets import WebsocketDemultiplexer from channels.tests import ChannelTestCase, apply_routes, HttpClient +from channels.signals import consumer_finished from channels import route, Group User = get_user_model() @@ -33,6 +34,7 @@ class TestsBinding(ChannelTestCase): user = User.objects.create(username='test', email='test@test.com') + consumer_finished.send(sender=None) received = client.receive() self.assertTrue('payload' in received) self.assertTrue('action' in received['payload']) @@ -69,7 +71,9 @@ class TestsBinding(ChannelTestCase): def has_permission(self, user, action, pk): return True + # Make model and clear out pending sends user = User.objects.create(username='test', email='test@test.com') + consumer_finished.send(sender=None) with apply_routes([route('test', TestBinding.consumer)]): client = HttpClient() @@ -78,6 +82,7 @@ class TestsBinding(ChannelTestCase): user.username = 'test_new' user.save() + consumer_finished.send(sender=None) received = client.receive() self.assertTrue('payload' in received) self.assertTrue('action' in received['payload']) @@ -114,7 +119,9 @@ class TestsBinding(ChannelTestCase): def has_permission(self, user, action, pk): return True + # Make model and clear out pending sends user = User.objects.create(username='test', email='test@test.com') + consumer_finished.send(sender=None) with apply_routes([route('test', TestBinding.consumer)]): client = HttpClient() @@ -122,6 +129,7 @@ class TestsBinding(ChannelTestCase): user.delete() + consumer_finished.send(sender=None) received = client.receive() self.assertTrue('payload' in received) self.assertTrue('action' in received['payload']) @@ -151,7 +159,7 @@ class TestsBinding(ChannelTestCase): client.send_and_consume('websocket.connect', path='/') # assert in group - Group('inbound').send({'text': json.dumps({'test': 'yes'})}) + Group('inbound').send({'text': json.dumps({'test': 'yes'})}, immediately=True) self.assertEqual(client.receive(), {'test': 'yes'}) # assert that demultiplexer stream message diff --git a/channels/tests/test_generic.py b/channels/tests/test_generic.py index e385d33..77bf67a 100644 --- a/channels/tests/test_generic.py +++ b/channels/tests/test_generic.py @@ -70,7 +70,7 @@ class GenericTests(ChannelTestCase): def test_websockets_decorators(self): class WebsocketConsumer(websockets.WebsocketConsumer): - slight_ordering = True + strict_ordering = True def connect(self, message, **kwargs): self.order = message['order'] @@ -92,7 +92,7 @@ class GenericTests(ChannelTestCase): self.send(text=message.get('order')) routes = [ - WebsocketConsumer.as_route(attrs={'slight_ordering': True}, path='^/path$'), + WebsocketConsumer.as_route(attrs={"strict_ordering": True}, path='^/path$'), WebsocketConsumer.as_route(path='^/path/2$'), ] diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index 933d68d..0eaf2e6 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -13,6 +13,7 @@ from six import BytesIO from channels import Channel from channels.handler import AsgiHandler from channels.tests import ChannelTestCase +from channels.signals import consumer_finished class FakeAsgiHandler(AsgiHandler): @@ -26,6 +27,7 @@ class FakeAsgiHandler(AsgiHandler): def __init__(self, response): assert isinstance(response, (HttpResponse, StreamingHttpResponse)) self._response = response + consumer_finished.send(sender=self.__class__) super(FakeAsgiHandler, self).__init__() def get_response(self, request): diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index 31c711c..7267630 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -22,7 +22,7 @@ class RequestTests(ChannelTestCase): "http_version": "1.1", "method": "GET", "path": "/test/", - }) + }, immediately=True) request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.path, "/test/") self.assertEqual(request.method, "GET") @@ -53,7 +53,7 @@ class RequestTests(ChannelTestCase): }, "client": ["10.0.0.1", 1234], "server": ["10.0.0.2", 80], - }) + }, immediately=True) request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.path, "/test2/") self.assertEqual(request.method, "GET") @@ -86,7 +86,7 @@ class RequestTests(ChannelTestCase): "content-type": b"application/x-www-form-urlencoded", "content-length": b"18", }, - }) + }, immediately=True) request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.path, "/test2/") self.assertEqual(request.method, "POST") @@ -116,14 +116,14 @@ class RequestTests(ChannelTestCase): "content-type": b"application/x-www-form-urlencoded", "content-length": b"21", }, - }) + }, immediately=True) Channel("test-input").send({ "content": b"re=fou", "more_content": True, - }) + }, immediately=True) Channel("test-input").send({ "content": b"r+lights", - }) + }, immediately=True) request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.method, "POST") self.assertEqual(request.body, b"there_are=four+lights") @@ -154,14 +154,14 @@ class RequestTests(ChannelTestCase): "content-type": b"multipart/form-data; boundary=BOUNDARY", "content-length": six.text_type(len(body)).encode("ascii"), }, - }) + }, immediately=True) Channel("test-input").send({ "content": body[:20], "more_content": True, - }) + }, immediately=True) Channel("test-input").send({ "content": body[20:], - }) + }, immediately=True) request = AsgiRequest(self.get_next_message("test")) self.assertEqual(request.method, "POST") self.assertEqual(len(request.body), len(body)) @@ -184,7 +184,7 @@ class RequestTests(ChannelTestCase): "host": b"example.com", "content-length": b"11", }, - }) + }, immediately=True) request = AsgiRequest(self.get_next_message("test", require=True)) self.assertEqual(request.method, "PUT") self.assertEqual(request.read(3), b"one") @@ -206,12 +206,12 @@ class RequestTests(ChannelTestCase): "content-type": b"application/x-www-form-urlencoded", "content-length": b"21", }, - }) + }, immediately=True) # Say there's more content, but never provide it! Muahahaha! Channel("test-input").send({ "content": b"re=fou", "more_content": True, - }) + }, immediately=True) class VeryImpatientRequest(AsgiRequest): body_receive_timeout = 0 @@ -235,9 +235,9 @@ class RequestTests(ChannelTestCase): "content-type": b"application/x-www-form-urlencoded", "content-length": b"21", }, - }) + }, immediately=True) Channel("test-input").send({ "closed": True, - }) + }, immediately=True) with self.assertRaises(RequestAborted): AsgiRequest(self.get_next_message("test")) diff --git a/channels/tests/test_sessions.py b/channels/tests/test_sessions.py index 9102aea..a51b94f 100644 --- a/channels/tests/test_sessions.py +++ b/channels/tests/test_sessions.py @@ -141,65 +141,7 @@ class SessionTests(ChannelTestCase): # It should hydrate the http_session self.assertEqual(message2.http_session.session_key, session.session_key) - def test_enforce_ordering_slight(self): - """ - Tests that slight mode of enforce_ordering works - """ - # Construct messages to send - message0 = Message( - {"reply_channel": "test-reply-a", "order": 0}, - "websocket.connect", - channel_layers[DEFAULT_CHANNEL_LAYER] - ) - message1 = Message( - {"reply_channel": "test-reply-a", "order": 1}, - "websocket.receive", - channel_layers[DEFAULT_CHANNEL_LAYER] - ) - message2 = Message( - {"reply_channel": "test-reply-a", "order": 2}, - "websocket.receive", - channel_layers[DEFAULT_CHANNEL_LAYER] - ) - - # Run them in an acceptable slight order - @enforce_ordering(slight=True) - def inner(message): - pass - - inner(message0) - inner(message2) - inner(message1) - - # Ensure wait channel is empty - wait_channel = "__wait__.%s" % "test-reply-a" - next_message = self.get_next_message(wait_channel) - self.assertEqual(next_message, None) - - def test_enforce_ordering_slight_fail(self): - """ - Tests that slight mode of enforce_ordering fails on bad ordering - """ - # Construct messages to send - message2 = Message( - {"reply_channel": "test-reply-e", "order": 2}, - "websocket.receive", - channel_layers[DEFAULT_CHANNEL_LAYER] - ) - - # Run them in an acceptable strict order - @enforce_ordering(slight=True) - def inner(message): - pass - - inner(message2) - - # Ensure wait channel is not empty - wait_channel = "__wait__.%s" % "test-reply-e" - next_message = self.get_next_message(wait_channel) - self.assertNotEqual(next_message, None) - - def test_enforce_ordering_strict(self): + def test_enforce_ordering(self): """ Tests that strict mode of enforce_ordering works """ @@ -234,7 +176,7 @@ class SessionTests(ChannelTestCase): next_message = self.get_next_message(wait_channel) self.assertEqual(next_message, None) - def test_enforce_ordering_strict_fail(self): + def test_enforce_ordering_fail(self): """ Tests that strict mode of enforce_ordering fails on bad ordering """ @@ -273,7 +215,7 @@ class SessionTests(ChannelTestCase): channel_layers[DEFAULT_CHANNEL_LAYER] ) - @enforce_ordering(slight=True) + @enforce_ordering def inner(message): pass diff --git a/channels/tests/test_worker.py b/channels/tests/test_worker.py index bc6b5d4..21e57c8 100644 --- a/channels/tests/test_worker.py +++ b/channels/tests/test_worker.py @@ -68,7 +68,7 @@ class WorkerTests(ChannelTestCase): if _consumer._call_count == 1: raise ConsumeLater() - Channel('test').send({'test': 'test'}) + Channel('test').send({'test': 'test'}, immediately=True) channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] channel_layer.router.add_route(route('test', _consumer)) old_send = channel_layer.send @@ -83,7 +83,7 @@ class WorkerTests(ChannelTestCase): def test_normal_run(self): consumer = mock.Mock() - Channel('test').send({'test': 'test'}) + Channel('test').send({'test': 'test'}, immediately=True) channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] channel_layer.router.add_route(route('test', consumer)) old_send = channel_layer.send From f9ef08b0aa7bc649cf2378b64d759229095ce6fc Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Oct 2016 15:37:55 -0700 Subject: [PATCH 540/746] Flake8 fix --- channels/sessions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/sessions.py b/channels/sessions.py index 10607f4..c859e62 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -78,6 +78,7 @@ def enforce_ordering(func=None, slight=False): # Slight is deprecated if slight: raise ValueError("Slight ordering is now always on due to Channels changes. Please remove the decorator.") + # Main decorator def decorator(func): @channel_session From 1cc2a28fcbfb8f5cc6c3563ecd12a325f0ca54f9 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Oct 2016 15:38:50 -0700 Subject: [PATCH 541/746] Fix releases TOC link --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 365d48d..ea30611 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -57,4 +57,4 @@ Topics asgi community contributing - releases + releases/index From 5d697c9308de9ff392131f742ec1ecf7374732be Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Oct 2016 15:39:44 -0700 Subject: [PATCH 542/746] Fix release note depth --- docs/releases/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 7f55d6b..7e0e7e6 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -2,6 +2,6 @@ Release Notes ============= .. toctree:: - :maxdepth: 2 + :maxdepth: 1 1.0.0 From 09b2a12be1fcb8234b1c947f87d6f031c4d08953 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Oct 2016 15:59:55 -0700 Subject: [PATCH 543/746] Change to accept being part of send/close --- channels/worker.py | 2 +- docs/asgi.rst | 51 ++++++++++++++++++----------------------- docs/releases/1.0.0.rst | 27 +++++++++++++++++++++- 3 files changed, 49 insertions(+), 31 deletions(-) diff --git a/channels/worker.py b/channels/worker.py index 658f370..9c23074 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -122,7 +122,7 @@ class Worker(object): # They want to deny a WebSocket connection. if message.channel.name != "websocket.connect": raise ValueError("You cannot DenyConnection from a non-websocket.connect handler.") - message.reply_channel.send({"accept": False}) + message.reply_channel.send({"close": True}) except ConsumeLater: # They want to not handle it yet. Re-inject it with a number-of-tries marker. content['__retries__'] = content.get("__retries__", 0) + 1 diff --git a/docs/asgi.rst b/docs/asgi.rst index 0a5b68b..0165a28 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -735,7 +735,7 @@ server must close the connection with either HTTP status code ``503`` or WebSocket close code ``1013``. This message must be responded to on the ``reply_channel`` with a -*Connection Reply* message before the socket will pass messages on the +*Send/Close/Accept* message before the socket will pass messages on the ``receive`` channel. The protocol server should ideally send this message during the handshake phase of the WebSocket and not complete the handshake until it gets a reply, returning HTTP status code ``403`` if the connection is @@ -743,10 +743,6 @@ denied. If this is not possible, it must buffer WebSocket frames and not send them onto ``websocket.receive`` until a reply is received, and if the connection is rejected, return WebSocket close code ``4403``. -Receiving a WebSocket *Send/Close* message while waiting for a -*Connection Reply* must make the server accept the connection and then send -the message immediately. - Channel: ``websocket.connect`` Keys: @@ -782,25 +778,6 @@ Keys: * ``order``: The integer value ``0``. -Connection Reply -'''''''''''''''' - -Sent back on the reply channel from an application when a ``connect`` message -is received to say if the connection should be accepted or dropped. - -Behaviour on WebSocket rejection is defined in the Connection section above. - -If received while the socket is already accepted, the protocol server should -log an error, but not do anything. - -Channel: ``websocket.send!`` - -Keys: - -* ``accept``: If the connection should be accepted (``True``) or rejected and - dropped (``False``). - - Receive ''''''' @@ -852,14 +829,27 @@ Keys: ``order`` values in ``websocket.receive``. -Send/Close -'''''''''' +Send/Close/Accept +''''''''''''''''' Sends a data frame to the client and/or closes the connection from the -server end. If ``ChannelFull`` is raised, wait and try again. +server end and/or accepts a connection. If ``ChannelFull`` is raised, wait +and try again. -If sent while the connection is waiting for acceptance or rejection, -will accept the connection before the frame is sent. +If received while the connection is waiting for acceptance after a ``connect`` +message: + +* If ``bytes`` or ``text`` is present, accept the connection and send the data. +* If ``accept`` is ``True``, accept the connection and do nothing else. +* If ``close`` is ``True``, reject the connection. If ``bytes`` or ``text`` is + also set, it should accept the connection, send the frame, then immediately + close the connection. + +If received while the connection is established: + +* If ``bytes`` or ``text`` is present, send the data. +* If ``close`` is ``True``, close the connection after any send. +* ``accept`` is ignored. Channel: ``websocket.send!`` @@ -872,6 +862,9 @@ Keys: * ``close``: Boolean saying if the connection should be closed after data is sent, if any. Optional, default ``False``. +* ``accept``: Boolean saying if the connection should be accepted without + sending a frame if it is in the handshake phase. + A maximum of one of ``bytes`` or ``text`` may be provided. If both are provided, the protocol server should ignore the message entirely. diff --git a/docs/releases/1.0.0.rst b/docs/releases/1.0.0.rst index 8ac2359..019d312 100644 --- a/docs/releases/1.0.0.rst +++ b/docs/releases/1.0.0.rst @@ -52,7 +52,7 @@ you can still do that by passing the ``immediately`` argument:: Channel("thumbnailing-tasks").send({"id": 34245}, immediately=True) -This should be entirely backwards compatible, and may actually fix race +This should be mostly backwards compatible, and may actually fix race conditions in some apps that were pre-existing. @@ -60,3 +60,28 @@ Demultiplexer Overhaul ~~~~~~~~~~~~~~~~~~~~~~ TBD + + +Backwards Incompatible Changes +------------------------------ + +Connect Consumers +~~~~~~~~~~~~~~~~~ + +If you have a custom consumer for ``websocket.connect``, you must ensure that +it either: + +* Sends at least one message onto the ``reply_channel`` that generates a + WebSocket frame (either ``bytes`` or ``text`` is set), either directly + or via a group. +* Sends a message onto the ``reply_channel`` that is ``{"accept": True}``, + to accept a connection without sending data. +* Sends a message onto the ``reply_channel`` that is ``{"close": True}``, + to reject a connection mid-handshake. + +Many consumers already do the former, but if your connect consumer does not +send anything you MUST now send an accept message or the socket will remain +in the handshaking phase forever and you'll never get any messages. + +All built-in Channels consumers (e.g. in the generic consumers) have been +upgraded to do this. From c419d01dedd41367040bb5f29433206799d9ce62 Mon Sep 17 00:00:00 2001 From: Rock Howard Date: Tue, 11 Oct 2016 15:25:27 -0500 Subject: [PATCH 544/746] added http_timeout as a command line option for runserver (#387) * added http_timeout as a comand line option for runserver * possible improvement for input param management * explicitly set the default http_timeout in add_argument --- channels/management/commands/runserver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index e52b018..e2c0814 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -24,10 +24,13 @@ class Command(RunserverCommand): help='Tells Django not to run a worker thread; you\'ll need to run one separately.') parser.add_argument('--noasgi', action='store_false', dest='use_asgi', default=True, help='Run the old WSGI-based runserver rather than the ASGI-based one') + parser.add_argument('--http_timeout', action='store', dest='http_timeout', type=int, default=60, + help='Specify the daphane http_timeout interval in seconds (default: 60)') def handle(self, *args, **options): self.verbosity = options.get("verbosity", 1) self.logger = setup_logger('django.channels', self.verbosity) + self.http_timeout = options.get("http_timeout", 60) super(Command, self).handle(*args, **options) def inner_run(self, *args, **options): @@ -80,7 +83,7 @@ class Command(RunserverCommand): port=int(self.port), signal_handlers=not options['use_reloader'], action_logger=self.log_action, - http_timeout=60, # Shorter timeout than normal as it's dev + http_timeout=self.http_timeout, ws_protocols=getattr(settings, 'CHANNELS_WS_PROTOCOLS', None), ).run() self.logger.debug("Daphne exited") From 1673be5b75ddf1f2b6de5687c7de85a4d20ffcb2 Mon Sep 17 00:00:00 2001 From: Luke Hodkinson Date: Wed, 12 Oct 2016 16:32:45 +1100 Subject: [PATCH 545/746] Found a bug whereby streaming responses would try to be cached (#396) entirely in memory. Was causing views that stream a lot of data to timeout. --- channels/handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index 62922bf..1eab20c 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -340,7 +340,9 @@ class ViewConsumer(object): # a whole worker if the client just vanishes and leaves the response # channel full. try: - message.reply_channel.send(reply_message) + # Note: Use immediately to prevent streaming responses trying + # cache all data. + message.reply_channel.send(reply_message, immediately=True) except message.channel_layer.ChannelFull: time.sleep(0.05) else: From 51561273aea71dd937709c9b44fc3f70f358c6e3 Mon Sep 17 00:00:00 2001 From: Jeremy Spencer Date: Fri, 14 Oct 2016 21:54:46 -0400 Subject: [PATCH 546/746] Fix for issue 398. Converts channels.binding.websockets.WebsocketBinding.fields to list before comparing to ['__all__'] to ensure most common data structures do not cause unexpected failures (i.e. ('__all__',), '__all__') (#399) --- channels/binding/websockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 76a442b..e458b6b 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -55,7 +55,7 @@ class WebsocketBinding(Binding): """ Serializes model data into JSON-compatible types. """ - if self.fields == ['__all__']: + if list(self.fields) == ['__all__']: fields = None else: fields = self.fields From 12ca598d6bf68f0dff26ed50d51d45519e1ad83c Mon Sep 17 00:00:00 2001 From: MartinArroyo Date: Mon, 17 Oct 2016 07:58:02 +0200 Subject: [PATCH 547/746] Adds 'exclude' option to data binding (#400) --- channels/binding/base.py | 7 ++-- channels/binding/websockets.py | 21 ++++++++---- channels/tests/test_binding.py | 59 ++++++++++++++++++++++++++++++++++ docs/binding.rst | 3 +- 4 files changed, 80 insertions(+), 10 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index a1f1977..5788988 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -67,6 +67,7 @@ class Binding(object): # if you want to really send all fields, use fields = ['__all__'] fields = None + exclude = None # Decorators channel_session_user = True @@ -95,9 +96,9 @@ class Binding(object): return [] else: raise ValueError("You must set the model attribute on Binding %r!" % cls) - # If fields is not defined, raise an error - if cls.fields is None: - raise ValueError("You must set the fields attribute on Binding %r!" % cls) + # If neither fields nor exclude are not defined, raise an error + if cls.fields is None and cls.exclude is None: + raise ValueError("You must set the fields or exclude attribute on Binding %r!" % cls) # Optionally resolve model strings if isinstance(cls.model, six.string_types): cls.model = apps.get_model(cls.model) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index e458b6b..4238b2d 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -55,10 +55,13 @@ class WebsocketBinding(Binding): """ Serializes model data into JSON-compatible types. """ - if list(self.fields) == ['__all__']: - fields = None + if self.fields is not None: + if list(self.fields) == ['__all__']: + fields = None + else: + fields = self.fields else: - fields = self.fields + fields = [f.name for f in instance._meta.get_fields() if f.name not in self.exclude] data = serializers.serialize('json', [instance], fields=fields) return json.loads(data)[0]['fields'] @@ -109,9 +112,15 @@ class WebsocketBinding(Binding): def update(self, pk, data): instance = self.model.objects.get(pk=pk) hydrated = self._hydrate(pk, data) - for name in data.keys(): - if name in self.fields or self.fields == ['__all__']: - setattr(instance, name, getattr(hydrated.object, name)) + + if self.fields is not None: + for name in data.keys(): + if name in self.fields or self.fields == ['__all__']: + setattr(instance, name, getattr(hydrated.object, name)) + else: + for name in data.keys(): + if name not in self.exclude: + setattr(instance, name, getattr(hydrated.object, name)) instance.save() diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index c3dca66..63fa4e3 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -58,6 +58,65 @@ class TestsBinding(ChannelTestCase): received = client.receive() self.assertIsNone(received) + def test_trigger_outbound_create_exclude(self): + class TestBinding(WebsocketBinding): + model = User + stream = 'test' + exclude = ['first_name', 'last_name'] + + @classmethod + def group_names(cls, instance, action): + return ["users_exclude"] + + def has_permission(self, user, action, pk): + return True + + with apply_routes([route('test', TestBinding.consumer)]): + client = HttpClient() + client.join_group('users_exclude') + + user = User.objects.create(username='test', email='test@test.com') + consumer_finished.send(sender=None) + consumer_finished.send(sender=None) + received = client.receive() + + self.assertTrue('payload' in received) + self.assertTrue('action' in received['payload']) + self.assertTrue('data' in received['payload']) + self.assertTrue('username' in received['payload']['data']) + self.assertTrue('email' in received['payload']['data']) + self.assertTrue('password' in received['payload']['data']) + self.assertTrue('model' in received['payload']) + self.assertTrue('pk' in received['payload']) + + self.assertFalse('last_name' in received['payload']['data']) + self.assertFalse('first_name' in received['payload']['data']) + + self.assertEqual(received['payload']['action'], 'create') + self.assertEqual(received['payload']['model'], 'auth.user') + self.assertEqual(received['payload']['pk'], user.pk) + + self.assertEqual(received['payload']['data']['email'], 'test@test.com') + self.assertEqual(received['payload']['data']['username'], 'test') + self.assertEqual(received['payload']['data']['password'], '') + + received = client.receive() + self.assertIsNone(received) + + def test_omit_fields_and_exclude(self): + def _declare_class(): + class TestBinding(WebsocketBinding): + model = User + stream = 'test' + + @classmethod + def group_names(cls, instance, action): + return ["users_omit"] + + def has_permission(self, user, action, pk): + return True + self.assertRaises(ValueError, _declare_class) + def test_trigger_outbound_update(self): class TestBinding(WebsocketBinding): model = User diff --git a/docs/binding.rst b/docs/binding.rst index c294d5f..e8ce3a8 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -81,7 +81,8 @@ always provide: * ``fields`` is a whitelist of fields to return in the serialized request. Channels does not default to all fields for security concerns; if you want - this, set it to the value ``["__all__"]``. + this, set it to the value ``["__all__"]``. As an alternative, ``exclude`` + acts as a blacklist of fields. * ``group_names`` returns a list of groups to send outbound updates to based on the model and action. For example, you could dispatch posts on different From b9d2f534c46d6b88b4a3d7333cd31c7a3d67e1d7 Mon Sep 17 00:00:00 2001 From: Maik Hoepfel Date: Tue, 18 Oct 2016 19:07:56 +0200 Subject: [PATCH 548/746] Fix formatting for generics docs (#403) The paragraph was lacking the double colon to treat the http_user example code as a code block. --- docs/generics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generics.rst b/docs/generics.rst index 96d4d87..aa6f3c4 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -224,7 +224,7 @@ This will run the appropriate decorator around your handler methods, and provide the one passed in to your handler as an argument as well as ``self.message``, as they point to the same instance. -And if you just want to use the user from the django session, add ``http_user``: +And if you just want to use the user from the django session, add ``http_user``:: class MyConsumer(WebsocketConsumer): From 4f517bb9fca05544b84503d254c218fccb224974 Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Wed, 19 Oct 2016 18:25:34 +0200 Subject: [PATCH 549/746] check accept fields = '__all__' in serialize_data (#404) --- channels/binding/websockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 4238b2d..46a2fba 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -56,7 +56,7 @@ class WebsocketBinding(Binding): Serializes model data into JSON-compatible types. """ if self.fields is not None: - if list(self.fields) == ['__all__']: + if self.fields == '__all__' or list(self.fields) == ['__all__']: fields = None else: fields = self.fields From 291405afeb6d41cd8e9d01fa76c3bed1dae6a0c0 Mon Sep 17 00:00:00 2001 From: Yatish Bathini Date: Sat, 22 Oct 2016 02:59:45 +0800 Subject: [PATCH 550/746] Issue#393: Clear session modified flag on enforce_ordering session save (#402) --- channels/sessions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/sessions.py b/channels/sessions.py index c859e62..1856339 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -99,6 +99,7 @@ def enforce_ordering(func=None, slight=False): # Mark next message order as available for running message.channel_session["__channels_next_order"] = order + 1 message.channel_session.save() + message.channel_session.modified = False # Requeue any pending wait channel messages for this socket connection back onto it's original channel while True: wait_channel = "__wait__.%s" % message.reply_channel.name From c16de0e1e397c9cd85e4aaebaa21d92032f9750c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 26 Oct 2016 09:15:53 -0700 Subject: [PATCH 551/746] Remove last reference to more_body --- docs/asgi.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 0165a28..76b1f7b 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -561,10 +561,10 @@ Keys: Header names must be lowercased. * ``body``: Body of the request, as a byte string. Optional, defaults to ``""``. - If ``more_body`` is set, treat as start of body and concatenate + If ``body_channel`` is set, treat as start of body and concatenate on further chunks. -* ``more_body``: Name of a single-reader channel (containing ``?``) that contains +* ``body_channel``: Name of a single-reader channel (containing ``?``) that contains Request Body Chunk messages representing a large request body. Optional, defaults to ``None``. Chunks append to ``body`` if set. Presence of a channel indicates at least one Request Body Chunk message needs to be read, From e24bc17bbf0400fe44984db90e59a068b6c92898 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Mon, 31 Oct 2016 14:42:10 +0300 Subject: [PATCH 552/746] Documentation of Client/HttpClient and data binding unit tests (#417) * Added as_route documentation * Added documentation for client * Improve tests for binding * Changes for client docs * Fix docs indentations at client part * Added missed imports * Small fixes and refs * Fix typos * Fix errors and typos. --- channels/tests/test_binding.py | 138 ++++++++++++------------- docs/generics.rst | 37 +++++++ docs/routing.rst | 1 + docs/testing.rst | 178 ++++++++++++++++++++++++++++++++- 4 files changed, 285 insertions(+), 69 deletions(-) diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index 63fa4e3..619adec 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -28,35 +28,34 @@ class TestsBinding(ChannelTestCase): def has_permission(self, user, action, pk): return True - with apply_routes([route('test', TestBinding.consumer)]): - client = HttpClient() - client.join_group('users') + client = HttpClient() + client.join_group('users') - user = User.objects.create(username='test', email='test@test.com') + user = User.objects.create(username='test', email='test@test.com') - consumer_finished.send(sender=None) - received = client.receive() - self.assertTrue('payload' in received) - self.assertTrue('action' in received['payload']) - self.assertTrue('data' in received['payload']) - self.assertTrue('username' in received['payload']['data']) - self.assertTrue('email' in received['payload']['data']) - self.assertTrue('password' in received['payload']['data']) - self.assertTrue('last_name' in received['payload']['data']) - self.assertTrue('model' in received['payload']) - self.assertTrue('pk' in received['payload']) + consumer_finished.send(sender=None) + received = client.receive() + self.assertTrue('payload' in received) + self.assertTrue('action' in received['payload']) + self.assertTrue('data' in received['payload']) + self.assertTrue('username' in received['payload']['data']) + self.assertTrue('email' in received['payload']['data']) + self.assertTrue('password' in received['payload']['data']) + self.assertTrue('last_name' in received['payload']['data']) + self.assertTrue('model' in received['payload']) + self.assertTrue('pk' in received['payload']) - self.assertEqual(received['payload']['action'], 'create') - self.assertEqual(received['payload']['model'], 'auth.user') - self.assertEqual(received['payload']['pk'], user.pk) + self.assertEqual(received['payload']['action'], 'create') + self.assertEqual(received['payload']['model'], 'auth.user') + self.assertEqual(received['payload']['pk'], user.pk) - self.assertEqual(received['payload']['data']['email'], 'test@test.com') - self.assertEqual(received['payload']['data']['username'], 'test') - self.assertEqual(received['payload']['data']['password'], '') - self.assertEqual(received['payload']['data']['last_name'], '') + self.assertEqual(received['payload']['data']['email'], 'test@test.com') + self.assertEqual(received['payload']['data']['username'], 'test') + self.assertEqual(received['payload']['data']['password'], '') + self.assertEqual(received['payload']['data']['last_name'], '') - received = client.receive() - self.assertIsNone(received) + received = client.receive() + self.assertIsNone(received) def test_trigger_outbound_create_exclude(self): class TestBinding(WebsocketBinding): @@ -134,36 +133,35 @@ class TestsBinding(ChannelTestCase): user = User.objects.create(username='test', email='test@test.com') consumer_finished.send(sender=None) - with apply_routes([route('test', TestBinding.consumer)]): - client = HttpClient() - client.join_group('users2') + client = HttpClient() + client.join_group('users2') - user.username = 'test_new' - user.save() + user.username = 'test_new' + user.save() - consumer_finished.send(sender=None) - received = client.receive() - self.assertTrue('payload' in received) - self.assertTrue('action' in received['payload']) - self.assertTrue('data' in received['payload']) - self.assertTrue('username' in received['payload']['data']) - self.assertTrue('email' in received['payload']['data']) - self.assertTrue('password' in received['payload']['data']) - self.assertTrue('last_name' in received['payload']['data']) - self.assertTrue('model' in received['payload']) - self.assertTrue('pk' in received['payload']) + consumer_finished.send(sender=None) + received = client.receive() + self.assertTrue('payload' in received) + self.assertTrue('action' in received['payload']) + self.assertTrue('data' in received['payload']) + self.assertTrue('username' in received['payload']['data']) + self.assertTrue('email' in received['payload']['data']) + self.assertTrue('password' in received['payload']['data']) + self.assertTrue('last_name' in received['payload']['data']) + self.assertTrue('model' in received['payload']) + self.assertTrue('pk' in received['payload']) - self.assertEqual(received['payload']['action'], 'update') - self.assertEqual(received['payload']['model'], 'auth.user') - self.assertEqual(received['payload']['pk'], user.pk) + self.assertEqual(received['payload']['action'], 'update') + self.assertEqual(received['payload']['model'], 'auth.user') + self.assertEqual(received['payload']['pk'], user.pk) - self.assertEqual(received['payload']['data']['email'], 'test@test.com') - self.assertEqual(received['payload']['data']['username'], 'test_new') - self.assertEqual(received['payload']['data']['password'], '') - self.assertEqual(received['payload']['data']['last_name'], '') + self.assertEqual(received['payload']['data']['email'], 'test@test.com') + self.assertEqual(received['payload']['data']['username'], 'test_new') + self.assertEqual(received['payload']['data']['password'], '') + self.assertEqual(received['payload']['data']['last_name'], '') - received = client.receive() - self.assertIsNone(received) + received = client.receive() + self.assertIsNone(received) def test_trigger_outbound_delete(self): class TestBinding(WebsocketBinding): @@ -182,28 +180,27 @@ class TestsBinding(ChannelTestCase): user = User.objects.create(username='test', email='test@test.com') consumer_finished.send(sender=None) - with apply_routes([route('test', TestBinding.consumer)]): - client = HttpClient() - client.join_group('users3') + client = HttpClient() + client.join_group('users3') - user.delete() + user.delete() - consumer_finished.send(sender=None) - received = client.receive() - self.assertTrue('payload' in received) - self.assertTrue('action' in received['payload']) - self.assertTrue('data' in received['payload']) - self.assertTrue('username' in received['payload']['data']) - self.assertTrue('model' in received['payload']) - self.assertTrue('pk' in received['payload']) + consumer_finished.send(sender=None) + received = client.receive() + self.assertTrue('payload' in received) + self.assertTrue('action' in received['payload']) + self.assertTrue('data' in received['payload']) + self.assertTrue('username' in received['payload']['data']) + self.assertTrue('model' in received['payload']) + self.assertTrue('pk' in received['payload']) - self.assertEqual(received['payload']['action'], 'delete') - self.assertEqual(received['payload']['model'], 'auth.user') - self.assertEqual(received['payload']['pk'], 1) - self.assertEqual(received['payload']['data']['username'], 'test') + self.assertEqual(received['payload']['action'], 'delete') + self.assertEqual(received['payload']['model'], 'auth.user') + self.assertEqual(received['payload']['pk'], 1) + self.assertEqual(received['payload']['data']['username'], 'test') - received = client.receive() - self.assertIsNone(received) + received = client.receive() + self.assertIsNone(received) def test_demultiplexer(self): class Demultiplexer(WebsocketDemultiplexer): @@ -341,6 +338,8 @@ class TestsBinding(ChannelTestCase): self.assertEqual(user.username, 'test_inbound') self.assertEqual(user.email, 'test@user_steam.com') + self.assertIsNone(client.receive()) + def test_inbound_update(self): user = User.objects.create(username='test', email='test@channels.com') @@ -388,6 +387,8 @@ class TestsBinding(ChannelTestCase): self.assertEqual(user.username, 'test_inbound') self.assertEqual(user.email, 'test@channels.com') + self.assertIsNone(client.receive()) + def test_inbound_delete(self): user = User.objects.create(username='test', email='test@channels.com') @@ -420,4 +421,5 @@ class TestsBinding(ChannelTestCase): # our Demultiplexer route message to the inbound consumer, so call Demultiplexer consumer client.consume('binding.users') - self.assertIsNone(User.objects.filter(pk=user.pk).first()) + self.assertIsNone(User.objects.filter(pk=user.pk).first()) + self.assertIsNone(client.receive()) diff --git a/docs/generics.rst b/docs/generics.rst index aa6f3c4..09acf13 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -250,3 +250,40 @@ want to override; like so:: You can also use the Django ``method_decorator`` utility to wrap methods that have ``message`` as their first positional argument - note that it won't work for more high-level methods, like ``WebsocketConsumer.receive``. + + +As route +-------- + +Instead of making routes using ``route_class`` you may use the ``as_route`` shortcut. +This function takes route filters (:ref:`filters`) as kwargs and returns +``route_class``. For example:: + + from . import consumers + + channel_routing = [ + consumers.ChatServer.as_route(path=r"^/chat/"), + ] + +Use the ``attrs`` dict keyword for dynamic class attributes. For example you have +the generic consumer:: + + class MyGenericConsumer(WebsocketConsumer): + group = 'default' + group_prefix = '' + + def connection_groups(self, **kwargs): + return ['_'.join(self.group_prefix, self.group)] + +You can create consumers with different ``group`` and ``group_prefix`` with ``attrs``, +like so:: + + from . import consumers + + channel_routing = [ + consumers.MyGenericConsumer.as_route(path=r"^/path/1/", + attrs={'group': 'one', 'group_prefix': 'pre'}), + consumers.MyGenericConsumer.as_route(path=r"^/path/2/", + attrs={'group': 'two', 'group_prefix': 'public'}), + ] + diff --git a/docs/routing.rst b/docs/routing.rst index bce2edb..065e13f 100644 --- a/docs/routing.rst +++ b/docs/routing.rst @@ -33,6 +33,7 @@ The three default routing objects are: * ``include``: Takes either a list or string import path to a routing list, and optional filter keyword arguments. +.. _filters: Filters ------- diff --git a/docs/testing.rst b/docs/testing.rst index 11a5aa3..a43cd62 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -108,6 +108,182 @@ do group adds and sends during a test. For example:: self.assertEqual(result['value'], 42) +Clients +------- + +For more complicated test suites you can use the ``Client`` abstraction that +provides an easy way to test the full life cycle of messages with a couple of methods: +``send`` to sending message with given content to the given channel, ``consume`` +to run appointed consumer for the next message, ``receive`` to getting replies for client. +Very often you may need to ``send`` and than call a consumer one by one, for this +purpose use ``send_and_consume`` method:: + + from channels.tests import ChannelTestCase, Client + + class MyTests(ChannelTestCase): + + def test_my_consumer(self): + client = Client() + client.send_and_consume('my_internal_channel', {'value': 'my_value'}) + self.assertEqual(client.receive(), {'all is': 'done'}) + + +You can use ``HttpClient`` for websocket related consumers. It automatically serializes JSON content, +manage cookies and headers, give easy access to the session and add ability to authorize your requests. +For example:: + + + # consumers.py + class RoomConsumer(JsonWebsocketConsumer): + http_user = True + groups = ['rooms_watchers'] + + def receive(self, content, **kwargs): + self.send({'rooms': self.message.http_session.get("rooms", [])}) + Channel("rooms_receive").send({'user': self.message.user.id, + 'message': content['message']} + + + # tests.py + from channels import Group + from channels.tests import ChannelTestCase, HttpClient + + + class RoomsTests(ChannelTestCase): + + def test_rooms(self): + client = HttpClient() + user = User.objects.create_user(username='test', email='test@test.com', + password='123456') # fuck you security + client.login(username='test', password='123456') + + client.send_and_consume('websocket.connect', '/rooms/') + # check that there is nothing to receive + self.assertIsNone(client.receive()) + + # test that the client in the group + Group(RoomConsumer.groups[0]).send({'text': 'ok'}, immediately=True) + self.assertEqual(client.receive(json=False), 'ok') + + client.session['rooms'] = ['test', '1'] + client.session.save() + + client.send_and_consume('websocket.receive', + text={'message': 'hey'}, + path='/rooms/') + # test 'response' + self.assertEqual(client.receive(), {'rooms': ['test', '1']}) + + self.assertEqual(self.get_next_message('rooms_receive').content, + {'user': user.id, 'message': 'hey'}) + + # There is nothing to receive + self.assertIsNone(client.receive()) + + +Instead of ``HttpClient.login`` method with credentials at arguments you +may call ``HttpClient.force_login`` (like at django client) with the user object. + +``receive`` method by default trying to deserialize json text content of a message, +so if you need to pass decoding use ``receive(json=False)``, like in the example. + + +Applying routes +--------------- + +When you need to testing you consumers without routes in settings or you +want to testing your consumers in more isolate and atomic way, it will be +simpler with ``apply_routes`` contextmanager and decorator for your ``ChannelTestCase``. +It takes list of routes that you want to use and overwrite existing routes:: + + from channels.tests import ChannelTestCase, HttpClient, apply_routes + + class MyTests(ChannelTestCase): + + def test_myconsumer(self): + client = HttpClient() + + with apply_routes([MyConsumer.as_route(path='/new')]): + client.send_and_consume('websocket.connect', '/new') + self.assertEqual(client.receive(), {'key': 'value'}) + + +Test Data binding with ``HttpClient`` +------------------------------------- + +As you know data binding in channels works in outbound and inbound ways, +so that ways tests in different ways and ``HttpClient`` and ``apply_routes`` +will help to do this. +When you testing outbound consumers you need just import your ``Binding`` +subclass with specified ``group_names``. At test you can join to one of them, +make some changes with target model and check received message. +Lets test ``IntegerValueBinding`` from :doc:`data binding ` +with creating:: + + from channels.tests import ChannelTestCase, HttpClient + from channels.signals import consumer_finished + + class TestIntegerValueBinding(ChannelTestCase): + + def test_outbound_create(self): + # We use HttpClient because of json encoding messages + client = HttpClient() + client.join_group("intval-updates") # join outbound binding + + # create target entity + value = IntegerValue.objects.create(name='fifty', value=50) + + consumer_finished.send(sender=None) + received = client.receive() # receive outbound binding message + self.assertIsNotNone(received) + + self.assertTrue('payload' in received) + self.assertTrue('action' in received['payload']) + self.assertTrue('data' in received['payload']) + self.assertTrue('name' in received['payload']['data']) + self.assertTrue('value' in received['payload']['data']) + + self.assertEqual(received['payload']['action'], 'create') + self.assertEqual(received['payload']['model'], 'values.integervalue') + self.assertEqual(received['payload']['pk'], value.pk) + + self.assertEqual(received['payload']['data']['name'], 'fifty') + self.assertEqual(received['payload']['data']['value'], 50) + + # assert that is nothing to receive + self.assertIsNone(client.receive()) + + +There is another situation with inbound binding. It is used with :ref:`multiplexing`, +So we apply two routes: websocket route for demultiplexer and route with internal +consumer for binding itself, connect to websocket entrypoint and test different actions. +For example:: + + class TestIntegerValueBinding(ChannelTestCase): + + def test_inbound_create(self): + # check that initial state is empty + self.assertEqual(IntegerValue.objects.all().count(), 0) + + with apply_routes([Demultiplexer.as_route(path='/'), + route("binding.intval", IntegerValueBinding.consumer)]): + client = HttpClient() + client.send_and_consume('websocket.connect', path='/') + client.send_and_consume('websocket.receive', path='/', text={ + 'stream': 'intval', + 'payload': {'action': CREATE, 'data': {'name': 'one', 'value': 1}} + }) + # our Demultiplexer route message to the inbound consumer, + # so we need to call this consumer + client.consume('binding.users') + + self.assertEqual(IntegerValue.objects.all().count(), 1) + value = IntegerValue.objects.all().first() + self.assertEqual(value.name, 'one') + self.assertEqual(value.value, 1) + + + Multiple Channel Layers ----------------------- @@ -116,5 +292,5 @@ of the layers you want to mock as the ``test_channel_aliases`` attribute on the ``ChannelTestCase`` subclass; by default, only the ``default`` layer is mocked. -You can pass an ``alias`` argument to ``get_next_message`` and ``Channel`` +You can pass an ``alias`` argument to ``get_next_message``, ``Client`` and ``Channel`` to use a different layer too. From 6d8d3214e68410f59b5f147bd15ad5e844d28003 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Nov 2016 08:14:41 +0000 Subject: [PATCH 553/746] Fixed #422: No SERVER_PORT in request.META causes errors --- channels/handler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/channels/handler.py b/channels/handler.py index 1eab20c..214b44d 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -67,6 +67,9 @@ class AsgiRequest(http.HttpRequest): if self.message.get('server', None): self.META['SERVER_NAME'] = self.message['server'][0] self.META['SERVER_PORT'] = six.text_type(self.message['server'][1]) + else: + self.META['SERVER_NAME'] = "unknown" + self.META['SERVER_PORT'] = "0" # Handle old style-headers for a transition period if "headers" in self.message and isinstance(self.message['headers'], dict): self.message['headers'] = [ From c5f047a245226fda4caff5b239fbaf7970f61fe8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 2 Nov 2016 08:17:44 +0000 Subject: [PATCH 554/746] Fix test to look for SERVER_PORT --- channels/tests/test_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index 7267630..f677bb8 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -31,8 +31,8 @@ class RequestTests(ChannelTestCase): self.assertNotIn("REMOTE_ADDR", request.META) self.assertNotIn("REMOTE_HOST", request.META) self.assertNotIn("REMOTE_PORT", request.META) - self.assertNotIn("SERVER_NAME", request.META) - self.assertNotIn("SERVER_PORT", request.META) + self.assertIn("SERVER_NAME", request.META) + self.assertIn("SERVER_PORT", request.META) self.assertFalse(request.GET) self.assertFalse(request.POST) self.assertFalse(request.COOKIES) From bc3376390794b1c3512a0099608a809a3bf0a4bf Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 4 Nov 2016 17:24:21 +0100 Subject: [PATCH 555/746] Redirecting questions to the mailing list --- .github/ISSUE_TEMPLATE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 9e8ea48..9820316 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,3 +1,5 @@ +Please submit **questions** about usage to django-users@googlegroups.com (https://groups.google.com/forum/#!forum/django-users) and about development to django-developers@googlegroups.com (https://groups.google.com/forum/#!forum/django-developers). + If you're submitting a feature request, please try to include: - Detailed description of the overall behaviour From f4f45dbb9f63c4085d65331dd534740978ab6623 Mon Sep 17 00:00:00 2001 From: Paul Date: Sat, 5 Nov 2016 12:08:38 +0100 Subject: [PATCH 556/746] Sort imports and make Travis run isort (#425) * Sort imports * Make Travis run isort --- channels/binding/base.py | 6 ++---- channels/binding/websockets.py | 2 +- channels/generic/base.py | 3 ++- channels/generic/websockets.py | 4 ++-- channels/handler.py | 2 +- channels/management/commands/runserver.py | 3 +-- channels/management/commands/runworker.py | 2 +- channels/message.py | 1 + channels/routing.py | 2 +- channels/signals.py | 1 - channels/tests/base.py | 15 ++++++++------- channels/tests/http.py | 3 +-- channels/tests/test_binding.py | 8 +++++--- channels/tests/test_generic.py | 4 ++-- channels/tests/test_handler.py | 7 ++----- channels/tests/test_management.py | 2 +- channels/tests/test_request.py | 5 +++-- channels/tests/test_routing.py | 7 ++++--- channels/tests/test_sessions.py | 10 ++++++---- channels/tests/test_worker.py | 17 +++++++++-------- channels/worker.py | 7 +++---- setup.cfg | 1 + tox.ini | 3 +++ 23 files changed, 60 insertions(+), 55 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index 5788988..f810bb7 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -1,13 +1,11 @@ from __future__ import unicode_literals import six - from django.apps import apps -from django.db.models.signals import post_save, post_delete, pre_save, pre_delete +from django.db.models.signals import post_delete, post_save, pre_delete, pre_save -from ..channel import Group from ..auth import channel_session, channel_session_user - +from ..channel import Group CREATE = 'create' UPDATE = 'update' diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 46a2fba..721401d 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -3,9 +3,9 @@ import json from django.core import serializers from django.core.serializers.json import DjangoJSONEncoder -from .base import Binding from ..generic.websockets import WebsocketDemultiplexer from ..sessions import enforce_ordering +from .base import Binding class WebsocketBinding(Binding): diff --git a/channels/generic/base.py b/channels/generic/base.py index 480f9cf..cc407a0 100644 --- a/channels/generic/base.py +++ b/channels/generic/base.py @@ -1,7 +1,8 @@ from __future__ import unicode_literals + +from ..auth import channel_session_user from ..routing import route_class from ..sessions import channel_session -from ..auth import channel_session_user class BaseConsumer(object): diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 4890f3d..b5d5cd7 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -1,7 +1,7 @@ -from django.core.serializers.json import json, DjangoJSONEncoder +from django.core.serializers.json import DjangoJSONEncoder, json -from ..channel import Group, Channel from ..auth import channel_session_user_from_http +from ..channel import Channel, Group from ..sessions import enforce_ordering from .base import BaseConsumer diff --git a/channels/handler.py b/channels/handler.py index 214b44d..e6d5734 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -17,7 +17,7 @@ from django.http import FileResponse, HttpResponse, HttpResponseServerError from django.utils import six from django.utils.functional import cached_property -from channels.exceptions import ResponseLater as ResponseLaterOuter, RequestTimeout, RequestAborted +from channels.exceptions import RequestAborted, RequestTimeout, ResponseLater as ResponseLaterOuter logger = logging.getLogger('django.request') diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index e2c0814..ac1d783 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -4,8 +4,7 @@ import threading from daphne.server import Server from django.conf import settings -from django.core.management.commands.runserver import \ - Command as RunserverCommand +from django.core.management.commands.runserver import Command as RunserverCommand from django.utils import six from django.utils.encoding import get_system_encoding diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 9823b4c..580f031 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -5,9 +5,9 @@ from django.core.management import BaseCommand, CommandError from channels import DEFAULT_CHANNEL_LAYER, channel_layers from channels.log import setup_logger +from channels.signals import worker_process_ready from channels.staticfiles import StaticFilesConsumer from channels.worker import Worker, WorkerGroup -from channels.signals import worker_process_ready class Command(BaseCommand): diff --git a/channels/message.py b/channels/message.py index b9c5ca1..97e67a3 100644 --- a/channels/message.py +++ b/channels/message.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals + import copy import threading diff --git a/channels/routing.py b/channels/routing.py index cb3ea03..11a5afc 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals -import re import importlib +import re from django.core.exceptions import ImproperlyConfigured from django.utils import six diff --git a/channels/signals.py b/channels/signals.py index dc83b94..b663bb8 100644 --- a/channels/signals.py +++ b/channels/signals.py @@ -1,7 +1,6 @@ from django.db import close_old_connections from django.dispatch import Signal - consumer_started = Signal(providing_args=["environ"]) consumer_finished = Signal() worker_ready = Signal() diff --git a/channels/tests/base.py b/channels/tests/base.py index bacede4..d3b3d76 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -5,14 +5,15 @@ import random import string from functools import wraps -from django.test.testcases import TestCase, TransactionTestCase -from .. import DEFAULT_CHANNEL_LAYER -from ..channel import Group -from ..routing import Router, include -from ..asgi import channel_layers, ChannelLayerWrapper -from ..message import Message -from ..signals import consumer_finished, consumer_started from asgiref.inmemory import ChannelLayer as InMemoryChannelLayer +from django.test.testcases import TestCase, TransactionTestCase + +from .. import DEFAULT_CHANNEL_LAYER +from ..asgi import ChannelLayerWrapper, channel_layers +from ..channel import Group +from ..message import Message +from ..routing import Router, include +from ..signals import consumer_finished, consumer_started class ChannelTestCaseMixin(object): diff --git a/channels/tests/http.py b/channels/tests/http.py index 0085216..e17e785 100644 --- a/channels/tests/http.py +++ b/channels/tests/http.py @@ -1,8 +1,7 @@ -import json import copy +import json import six - from django.apps import apps from django.conf import settings diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index 619adec..391baff 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -1,13 +1,15 @@ from __future__ import unicode_literals + import json from django.contrib.auth import get_user_model -from channels.binding.base import CREATE, UPDATE, DELETE + +from channels import Group, route +from channels.binding.base import CREATE, DELETE, UPDATE from channels.binding.websockets import WebsocketBinding from channels.generic.websockets import WebsocketDemultiplexer -from channels.tests import ChannelTestCase, apply_routes, HttpClient from channels.signals import consumer_finished -from channels import route, Group +from channels.tests import ChannelTestCase, HttpClient, apply_routes User = get_user_model() diff --git a/channels/tests/test_generic.py b/channels/tests/test_generic.py index 77bf67a..aaac316 100644 --- a/channels/tests/test_generic.py +++ b/channels/tests/test_generic.py @@ -1,10 +1,10 @@ from __future__ import unicode_literals from django.test import override_settings + from channels import route_class from channels.generic import BaseConsumer, websockets -from channels.tests import ChannelTestCase -from channels.tests import apply_routes, Client +from channels.tests import ChannelTestCase, Client, apply_routes @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index 0eaf2e6..efda3f2 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -4,16 +4,13 @@ import os from datetime import datetime from itertools import islice -from django.http import ( - FileResponse, HttpResponse, HttpResponseRedirect, JsonResponse, - StreamingHttpResponse, -) +from django.http import FileResponse, HttpResponse, HttpResponseRedirect, JsonResponse, StreamingHttpResponse from six import BytesIO from channels import Channel from channels.handler import AsgiHandler -from channels.tests import ChannelTestCase from channels.signals import consumer_finished +from channels.tests import ChannelTestCase class FakeAsgiHandler(AsgiHandler): diff --git a/channels/tests/test_management.py b/channels/tests/test_management.py index ca759a0..6c3f78e 100644 --- a/channels/tests/test_management.py +++ b/channels/tests/test_management.py @@ -4,11 +4,11 @@ import logging from asgiref.inmemory import ChannelLayer from django.core.management import CommandError, call_command -from channels.staticfiles import StaticFilesConsumer from django.test import TestCase, mock from six import StringIO from channels.management.commands import runserver +from channels.staticfiles import StaticFilesConsumer class FakeChannelLayer(ChannelLayer): diff --git a/channels/tests/test_request.py b/channels/tests/test_request.py index f677bb8..aea1f47 100644 --- a/channels/tests/test_request.py +++ b/channels/tests/test_request.py @@ -1,10 +1,11 @@ from __future__ import unicode_literals + from django.utils import six from channels import Channel -from channels.tests import ChannelTestCase +from channels.exceptions import RequestAborted, RequestTimeout from channels.handler import AsgiRequest -from channels.exceptions import RequestTimeout, RequestAborted +from channels.tests import ChannelTestCase class RequestTests(ChannelTestCase): diff --git a/channels/tests/test_routing.py b/channels/tests/test_routing.py index 1a00e43..5a6145d 100644 --- a/channels/tests/test_routing.py +++ b/channels/tests/test_routing.py @@ -1,10 +1,11 @@ from __future__ import unicode_literals + from django.test import SimpleTestCase -from channels.routing import Router, route, route_class, include -from channels.message import Message -from channels.utils import name_that_thing from channels.generic import BaseConsumer +from channels.message import Message +from channels.routing import Router, include, route, route_class +from channels.utils import name_that_thing # Fake consumers and routing sets that can be imported by string diff --git a/channels/tests/test_sessions.py b/channels/tests/test_sessions.py index a51b94f..6af6edc 100644 --- a/channels/tests/test_sessions.py +++ b/channels/tests/test_sessions.py @@ -2,11 +2,13 @@ from __future__ import unicode_literals from django.conf import settings from django.test import override_settings -from channels.message import Message -from channels.sessions import channel_session, channel_and_http_session, http_session, enforce_ordering, \ - session_for_reply_channel -from channels.tests import ChannelTestCase + from channels import DEFAULT_CHANNEL_LAYER, channel_layers +from channels.message import Message +from channels.sessions import ( + channel_and_http_session, channel_session, enforce_ordering, http_session, session_for_reply_channel, +) +from channels.tests import ChannelTestCase @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") diff --git a/channels/tests/test_worker.py b/channels/tests/test_worker.py index 21e57c8..064f471 100644 --- a/channels/tests/test_worker.py +++ b/channels/tests/test_worker.py @@ -1,17 +1,18 @@ from __future__ import unicode_literals +import threading + +from channels import DEFAULT_CHANNEL_LAYER, Channel, route +from channels.asgi import channel_layers +from channels.exceptions import ConsumeLater +from channels.signals import worker_ready +from channels.tests import ChannelTestCase +from channels.worker import Worker, WorkerGroup + try: from unittest import mock except ImportError: import mock -import threading - -from channels import Channel, route, DEFAULT_CHANNEL_LAYER -from channels.asgi import channel_layers -from channels.tests import ChannelTestCase -from channels.worker import Worker, WorkerGroup -from channels.exceptions import ConsumeLater -from channels.signals import worker_ready class PatchedWorker(Worker): diff --git a/channels/worker.py b/channels/worker.py index 9c23074..b44b6ac 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -2,17 +2,16 @@ from __future__ import unicode_literals import fnmatch import logging +import multiprocessing import signal import sys -import time -import multiprocessing import threading +import time -from .signals import consumer_started, consumer_finished from .exceptions import ConsumeLater, DenyConnection from .message import Message +from .signals import consumer_finished, consumer_started, worker_ready from .utils import name_that_thing -from .signals import worker_ready logger = logging.getLogger('django.channels') diff --git a/setup.cfg b/setup.cfg index 2cab84e..6923a28 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,6 +10,7 @@ include_trailing_comma = true known_first_party = channels multi_line_output = 5 not_skip = __init__.py +line_length = 119 [bdist_wheel] universal=1 diff --git a/tox.ini b/tox.ini index 546e062..8b95ac3 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,9 @@ envlist = {py27,py35}-flake8 isort +[tox:travis] +2.7 = py27, isort + [testenv] setenv = PYTHONPATH = {toxinidir}:{toxinidir} From 82f7ff21df1649ca10d93badbc72e74bec1dd48b Mon Sep 17 00:00:00 2001 From: Tobias Kunze Date: Sat, 5 Nov 2016 13:39:44 +0100 Subject: [PATCH 557/746] Add closing response codes (#426) Added both to spec and implementation. Regards #414. --- channels/generic/websockets.py | 10 +++++----- docs/asgi.rst | 15 +++++++++------ docs/generics.rst | 4 ++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index b5d5cd7..d82c9fe 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -95,7 +95,7 @@ class WebsocketConsumer(BaseConsumer): """ message = {} if close: - message["close"] = True + message["close"] = close if text is not None: message["text"] = text elif bytes is not None: @@ -108,7 +108,7 @@ class WebsocketConsumer(BaseConsumer): def group_send(cls, name, text=None, bytes=None, close=False): message = {} if close: - message["close"] = True + message["close"] = close if text is not None: message["text"] = text elif bytes is not None: @@ -117,11 +117,11 @@ class WebsocketConsumer(BaseConsumer): raise ValueError("You must pass text or bytes") Group(name).send(message) - def close(self): + def close(self, status=True): """ Closes the WebSocket from the server end """ - self.message.reply_channel.send({"close": True}) + self.message.reply_channel.send({"close": status}) def raw_disconnect(self, message, **kwargs): """ @@ -134,7 +134,7 @@ class WebsocketConsumer(BaseConsumer): def disconnect(self, message, **kwargs): """ - Called when a WebSocket connection is opened. + Called when a WebSocket connection is closed. """ pass diff --git a/docs/asgi.rst b/docs/asgi.rst index 76b1f7b..914f061 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -841,14 +841,15 @@ message: * If ``bytes`` or ``text`` is present, accept the connection and send the data. * If ``accept`` is ``True``, accept the connection and do nothing else. -* If ``close`` is ``True``, reject the connection. If ``bytes`` or ``text`` is - also set, it should accept the connection, send the frame, then immediately - close the connection. +* If ``close`` is ``True`` or a positive integer, reject the connection. If + ``bytes`` or ``text`` is also set, it should accept the connection, send the + frame, then immediately close the connection. If received while the connection is established: * If ``bytes`` or ``text`` is present, send the data. -* If ``close`` is ``True``, close the connection after any send. +* If ``close`` is ``True`` or a positive integer, close the connection after + any send. * ``accept`` is ignored. Channel: ``websocket.send!`` @@ -859,8 +860,10 @@ Keys: * ``text``: Unicode string of frame content, if in text mode, or ``None``. -* ``close``: Boolean saying if the connection should be closed after data - is sent, if any. Optional, default ``False``. +* ``close``: Boolean indicating if the connection should be closed after + data is sent, if any. Alternatively, a positive integer specifying the + response code. The response code will be 1000 if you pass ``True``. + Optional, default ``False``. * ``accept``: Boolean saying if the connection should be accepted without sending a frame if it is in the handshake phase. diff --git a/docs/generics.rst b/docs/generics.rst index 09acf13..c7caeee 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -172,8 +172,8 @@ Note that this subclass still can't intercept ``Group.send()`` calls to make them into JSON automatically, but it does provide ``self.group_send(name, content)`` that will do this for you if you call it explicitly. -``self.close()`` is also provided to easily close the WebSocket from the server -end once you are done with it. +``self.close()`` is also provided to easily close the WebSocket from the +server end with an optional status code once you are done with it. .. _multiplexing: From 1d93037bb7ab0df537d0f3d0c939c9a14079385a Mon Sep 17 00:00:00 2001 From: Iacopo Spalletti Date: Sat, 5 Nov 2016 13:48:14 +0100 Subject: [PATCH 558/746] Minor typos (#427) --- docs/asgi.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 914f061..ce0892e 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -206,8 +206,8 @@ The extensions defined here are: * ``groups``: Allows grouping of channels to allow broadcast; see below for more. * ``flush``: Allows easier testing and development with channel layers. * ``statistics``: Allows channel layers to provide global and per-channel statistics. -* ``twisted``: Async compatability with the Twisted framework. -* ``asyncio``: Async compatability with Python 3's asyncio. +* ``twisted``: Async compatibility with the Twisted framework. +* ``asyncio``: Async compatibility with Python 3's asyncio. There is potential to add further extensions; these may be defined by a separate specification, or a new version of this specification. @@ -457,7 +457,7 @@ after the most recent ``group_add`` call for that membership, the default being ``group_expiry`` property on the channel layer. Protocol servers must have a configurable timeout value for every connection-based -prtocol they serve that closes the connection after the timeout, and should +protocol they serve that closes the connection after the timeout, and should default this value to the value of ``group_expiry``, if the channel layer provides it. This allows old group memberships to be cleaned up safely, knowing that after the group expiry the original connection must have closed, From 6c471ef9155d339bdbff128c1d94560d144d300a Mon Sep 17 00:00:00 2001 From: Paul Date: Sat, 5 Nov 2016 15:16:27 +0100 Subject: [PATCH 559/746] Benchmark script never completed on Docker (#211) (#428) * Do not generate new fingerprint when connection fails * Do not try to print latencies when all connections failed * Update asgi_redis and channels versions in Dockerfile --- testproject/Dockerfile | 4 ++-- testproject/benchmark.py | 31 +++++++++++++++++-------------- testproject/docker-compose.yml | 2 +- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/testproject/Dockerfile b/testproject/Dockerfile index aeddb65..15e43e3 100644 --- a/testproject/Dockerfile +++ b/testproject/Dockerfile @@ -12,13 +12,13 @@ RUN apt-get update && \ # Install asgi_redis driver and most recent Daphne RUN pip install \ - asgi_redis==0.8.3 \ + asgi_redis==1.0.0 \ git+https://github.com/django/daphne.git@#egg=daphne # Clone Channels and install it RUN git clone https://github.com/django/channels.git /srv/channels/ && \ cd /srv/channels && \ - git reset --hard caa589ae708a1a66ba1bdcd24f5fd473040772bd && \ + git reset --hard origin/master && \ python setup.py install WORKDIR /srv/channels/testproject/ diff --git a/testproject/benchmark.py b/testproject/benchmark.py index 01c1f81..c0c4c91 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -70,7 +70,6 @@ class MyClientProtocol(WebSocketClientProtocol): "connect": True, } else: - self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16)) stats[self.fingerprint] = { "sent": 0, "received": 0, @@ -168,22 +167,26 @@ class Benchmarker(object): num_out_of_order += 1 else: num_good += 1 - # Some analysis on latencies - latency_mean = statistics.mean(latencies) - latency_median = statistics.median(latencies) - latency_stdev = statistics.stdev(latencies) - latency_95 = self.percentile(latencies, 0.95) - latency_99 = self.percentile(latencies, 0.99) + + if latencies: + # Some analysis on latencies + latency_mean = statistics.mean(latencies) + latency_median = statistics.median(latencies) + latency_stdev = statistics.stdev(latencies) + latency_95 = self.percentile(latencies, 0.95) + latency_99 = self.percentile(latencies, 0.99) + # Print results print("-------") print("Sockets opened: %s" % len(stats)) - print("Latency stats: Mean %.3fs Median %.3fs Stdev %.3f 95%% %.3fs 95%% %.3fs" % ( - latency_mean, - latency_median, - latency_stdev, - latency_95, - latency_99, - )) + if latencies: + print("Latency stats: Mean %.3fs Median %.3fs Stdev %.3f 95%% %.3fs 95%% %.3fs" % ( + latency_mean, + latency_median, + latency_stdev, + latency_95, + latency_99, + )) print("Good sockets: %s (%.2f%%)" % (num_good, (float(num_good) / len(stats))*100)) print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100)) print("Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100)) diff --git a/testproject/docker-compose.yml b/testproject/docker-compose.yml index a0603fd..1753b71 100644 --- a/testproject/docker-compose.yml +++ b/testproject/docker-compose.yml @@ -13,6 +13,6 @@ services: worker: image: channels-test build: . - command: python manage.py runworker + command: python manage.py runworker --settings=testproject.settings.channels_redis depends_on: - redis From 8682e83fd183823e20353ae2d89b445b80233116 Mon Sep 17 00:00:00 2001 From: Fabien Schwob Date: Sun, 6 Nov 2016 15:15:31 +0100 Subject: [PATCH 560/746] Updating Channels status regarding inclusion into django core (#424) * Updating Channels status regarding django inclusion * Removing the page regarding Cross-Compatibility and the references to it. --- docs/cross-compat.rst | 44 ------------------------------------------- docs/index.rst | 1 - docs/inshort.rst | 13 ++++++++++--- 3 files changed, 10 insertions(+), 48 deletions(-) delete mode 100644 docs/cross-compat.rst diff --git a/docs/cross-compat.rst b/docs/cross-compat.rst deleted file mode 100644 index bfc73e5..0000000 --- a/docs/cross-compat.rst +++ /dev/null @@ -1,44 +0,0 @@ -Cross-Compatibility -=================== - -Channels is being released as both a third-party app for Django 1.8 through 1.10, -and being integrated into Django in future. Both of these implementations will be -very similar, and code for one will work on the other with minimal changes. - -The only difference between the two is the import paths. Mostly, where you -imported from ``channels`` for the third-party app, you instead import from -``django.channels`` for the built-in solution. - -For example:: - - from channels import Channel - from channels.auth import channel_session_user - -Becomes:: - - from django.channels import Channel - from django.channels.auth import channel_session_user - -There are a few exceptions to this rule, where classes will be moved to other parts -of Django in that make more sense: - -* ``channels.tests.ChannelTestCase`` is found under ``django.test.channels.ChannelTestCase`` -* ``channels.handler`` is moved to ``django.core.handlers.asgi`` -* ``channels.staticfiles`` is moved to ``django.contrib.staticfiles.consumers`` -* The ``runserver`` and ``runworker`` commands are in ``django.core.management.commands`` - - -Writing third-party apps against Channels ------------------------------------------ - -If you're writing a third-party app that is designed to work with both the -``channels`` third-party app as well as ``django.channels``, we suggest you use -a try-except pattern for imports, like this:: - - try: - from django.channels import Channel - except ImportError: - from channels import Channel - -All the objects in both versions act the same way, they simply are located -on different import paths. There should be no need to change logic. diff --git a/docs/index.rst b/docs/index.rst index ea30611..9125e55 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,7 +51,6 @@ Topics binding backends testing - cross-compat reference faqs asgi diff --git a/docs/inshort.rst b/docs/inshort.rst index 8e3facf..e3427ad 100644 --- a/docs/inshort.rst +++ b/docs/inshort.rst @@ -94,9 +94,16 @@ affect the overall deployed site. What version of Django does it work with? ----------------------------------------- -You can install Channels as a library for Django 1.8 and 1.9, and it (should be) -part of Django 1.10. It has a few extra dependencies, but these will all -be installed if you use ``pip``. +You can install Channels as a library for Django >= 1.8. It has a few +extra dependencies, but these will all be installed if you use ``pip``. + +Official project +---------------- + +Channels is not in the Django core as it was initially planned, but it's +an official Django project since september 2016. More informations about Channels +as an official project are available on the +`Django blog `_. What do I read next? From 1212fd45f197e9b420171ac3df160b5d28e8644f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 6 Nov 2016 14:16:16 +0000 Subject: [PATCH 561/746] Update ASGI spec from receive_many to receive --- docs/asgi.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index ce0892e..3ea981b 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -53,7 +53,7 @@ servers and applications. A channel layer provides a protocol server or an application server with a ``send`` callable, which takes a channel name and message -``dict``, and a ``receive_many`` callable, which takes a list of +``dict``, and a ``receive`` callable, which takes a list of channel names and returns the next message available on any named channel. Thus, rather than under WSGI, where you point the protocol server to the @@ -64,10 +64,10 @@ via the channel layer. Despite the name of the proposal, ASGI does not specify or design to any specific in-process async solution, such as ``asyncio``, ``twisted``, or -``gevent``. Instead, the ``receive_many`` function can be switched between +``gevent``. Instead, the ``receive`` function can be switched between nonblocking or synchronous. This approach allows applications to choose what's best for their current runtime environment; further improvements may provide -extensions where cooperative versions of receive_many are provided. +extensions where cooperative versions of receive are provided. The distinction between protocol servers and applications in this document is mostly to distinguish their roles and to make illustrating concepts easier. @@ -120,7 +120,7 @@ tied to a client socket). the backend wishes; in particular, they do not have to appear globally consistent, and backends may shard their contents out to different servers so that a querying client only sees some portion of the messages. Calling -``receive_many`` on these channels does not guarantee that you will get the +``receive`` on these channels does not guarantee that you will get the messages in order or that you will get anything if the channel is non-empty. *Single-reader channel* names contain an question mark @@ -312,7 +312,7 @@ A *channel layer* must provide an object with these attributes channel to send on, as a unicode string, and the message to send, as a serializable ``dict``. -* ``receive_many(channels, block=False)``, a callable that takes a list of channel +* ``receive(channels, block=False)``, a callable that takes a list of channel names as unicode strings, and returns with either ``(None, None)`` or ``(channel, message)`` if a message is available. If ``block`` is True, then it will not return until after a built-in timeout or a message arrives; if @@ -328,7 +328,7 @@ A *channel layer* must provide an object with these attributes MUST end with ``!`` or ``?`` or this function must error. If the character is ``!``, making it a process-specific channel, ``new_channel`` must be called on the same channel layer that intends to read the channel with - ``receive_many``; any other channel layer instance may not receive + ``receive``; any other channel layer instance may not receive messages on this channel due to client-routing portions of the appended string. * ``MessageTooLarge``, the exception raised when a send operation fails @@ -388,15 +388,15 @@ A channel layer implementing the ``flush`` extension must also provide: A channel layer implementing the ``twisted`` extension must also provide: -* ``receive_many_twisted(channels)``, a function that behaves - like ``receive_many`` but that returns a Twisted Deferred that eventually +* ``receive_twisted(channels)``, a function that behaves + like ``receive`` but that returns a Twisted Deferred that eventually returns either ``(channel, message)`` or ``(None, None)``. It is not possible - to run it in nonblocking mode; use the normal ``receive_many`` for that. + to run it in nonblocking mode; use the normal ``receive`` for that. A channel layer implementing the ``asyncio`` extension must also provide: -* ``receive_many_asyncio(channels)``, a function that behaves - like ``receive_many`` but that fulfills the asyncio coroutine contract to +* ``receive_asyncio(channels)``, a function that behaves + like ``receive`` but that fulfills the asyncio coroutine contract to block until either a result is available or an internal timeout is reached and ``(None, None)`` is returned. @@ -910,7 +910,7 @@ to prevent busy channels from overpowering quiet channels. For example, imagine two channels, ``busy``, which spikes to 1000 messages a second, and ``quiet``, which gets one message a second. There's a single -consumer running ``receive_many(['busy', 'quiet'])`` which can handle +consumer running ``receive(['busy', 'quiet'])`` which can handle around 200 messages a second. In a simplistic for-loop implementation, the channel layer might always check @@ -1024,9 +1024,9 @@ TODOs * Maybe remove ``http_version`` and replace with ``supports_server_push``? -* ``receive_many`` can't easily be implemented with async/cooperative code +* ``receive`` can't easily be implemented with async/cooperative code behind it as it's nonblocking - possible alternative call type? - Asyncio extension that provides ``receive_many_yield``? + Asyncio extension that provides ``receive_yield``? * Possible extension to allow detection of channel layer flush/restart and prompt protocol servers to restart? From 0a4cbb5fcf30557c4b95ca13f061a3180c1013a4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 6 Nov 2016 14:17:03 +0000 Subject: [PATCH 562/746] Slight grammar tweaks --- docs/inshort.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/inshort.rst b/docs/inshort.rst index e3427ad..655f2fd 100644 --- a/docs/inshort.rst +++ b/docs/inshort.rst @@ -97,12 +97,13 @@ What version of Django does it work with? You can install Channels as a library for Django >= 1.8. It has a few extra dependencies, but these will all be installed if you use ``pip``. + Official project ---------------- -Channels is not in the Django core as it was initially planned, but it's -an official Django project since september 2016. More informations about Channels -as an official project are available on the +Channels is not in the Django core as initially planned, but it's +an official Django project since September 2016. More information about Channels +being adopted as an official project are available on the `Django blog `_. From 2e1cda8aad97fba443e526b3659f439cf1d5ae56 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 17 Nov 2016 17:39:01 -0800 Subject: [PATCH 563/746] Clarify "out of the box" --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 37663a0..8496d4a 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -11,7 +11,7 @@ patterns and caveats. First Consumers --------------- -When you run Django out of the box, it will be set up in the default layout - +When you first run Django with Channels installed, it will be set up in the default layout - where all HTTP requests (on the ``http.request`` channel) are routed to the Django view layer - nothing will be different to how things worked in the past with a WSGI-based Django, and your views and static file serving (from From fdc80cb26978ed66d0aa2b853136d7a0cd8b15c6 Mon Sep 17 00:00:00 2001 From: Jan Boysen Date: Fri, 18 Nov 2016 15:26:16 +0100 Subject: [PATCH 564/746] runserver should respect FORCE_SCRIPT_NAME setting (#435) * Pass FORCE_SCRIPT_NAME to Daphne server when set FORCE_SCRIPT_NAME seems not to be honored any more with build-in runserver after activating channels app. The normal behavior of Django is the FORCE_SCRIPT_NAME is used as prefix when set while generating URLs so its possible to create a path prefix and determine different Django installations based on the path rather than hostname without having to prefix all paths in urls.py. * Only strip script_name from path if it starts with it * make tests happy again after setting kwarg root_path --- channels/handler.py | 2 +- channels/management/commands/runserver.py | 1 + channels/tests/test_management.py | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index e6d5734..f43da22 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -44,7 +44,7 @@ class AsgiRequest(http.HttpRequest): # Path info self.path = self.message['path'] self.script_name = self.message.get('root_path', '') - if self.script_name: + if self.script_name and self.path.startswith(self.script_name): # TODO: Better is-prefix checking, slash handling? self.path_info = self.path[len(self.script_name):] else: diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index ac1d783..062ac46 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -84,6 +84,7 @@ class Command(RunserverCommand): action_logger=self.log_action, http_timeout=self.http_timeout, ws_protocols=getattr(settings, 'CHANNELS_WS_PROTOCOLS', None), + root_path=getattr(settings, 'FORCE_SCRIPT_NAME', ''), ).run() self.logger.debug("Daphne exited") except KeyboardInterrupt: diff --git a/channels/tests/test_management.py b/channels/tests/test_management.py index 6c3f78e..6213caa 100644 --- a/channels/tests/test_management.py +++ b/channels/tests/test_management.py @@ -87,7 +87,7 @@ class RunServerTests(TestCase): call_command('runserver', '--noreload') mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, - ws_protocols=None) + ws_protocols=None, root_path=None) @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) @mock.patch('channels.management.commands.runserver.Server') @@ -101,12 +101,12 @@ class RunServerTests(TestCase): call_command('runserver', '--noreload') mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, - ws_protocols=None) + ws_protocols=None, root_path=None) call_command('runserver', '--noreload', 'localhost:8001') mocked_server.assert_called_with(port=8001, signal_handlers=True, http_timeout=60, host='localhost', action_logger=mock.ANY, channel_layer=mock.ANY, - ws_protocols=None) + ws_protocols=None, root_path=None) self.assertFalse(mocked_worker.called, "The worker should not be called with '--noworker'") @@ -121,7 +121,7 @@ class RunServerTests(TestCase): call_command('runserver', '--noreload', '--noworker') mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, - ws_protocols=None) + ws_protocols=None, root_path=None) self.assertFalse(mocked_worker.called, "The worker should not be called with '--noworker'") From 3dddefa8453e4e9595a52a182ddcee316ffd6980 Mon Sep 17 00:00:00 2001 From: Sam Bolgert Date: Thu, 24 Nov 2016 12:54:03 -0600 Subject: [PATCH 565/746] Delay Protocol Server (#401) * Add Delay Protocol Server Add a process that listens to a specific channel and delays incoming messages by a given time. * Add custom django command rundelay * Add test suite * Implements #115 * Add channels.delay app * Add AppConfig * Move rundelay command to channels.delay app * Refactor DelayedMessage into model Move login into a database backed model. * Update Worker * Add migration * Add delay docs page * Add to TOC * Fix import sorting * Add ASGI spec document for Delay Protocol * Update channels.delay doc with new channel name * remove interval docs * Refactor Delay to use milliseconds instead of seconds Use milliseconds as the default unit. Gives more control to developers. * Remove interval logic from DelayedMessage * Remove interval tests * Tweak test logic to use milliseconds --- channels/delay/__init__.py | 1 + channels/delay/apps.py | 8 ++ channels/delay/management/__init__.py | 0 .../delay/management/commands/__init__.py | 0 .../delay/management/commands/rundelay.py | 39 +++++++ channels/delay/migrations/0001_initial.py | 25 +++++ channels/delay/migrations/__init__.py | 0 channels/delay/models.py | 44 ++++++++ channels/delay/worker.py | 82 ++++++++++++++ channels/tests/settings.py | 1 + channels/tests/test_delay.py | 102 ++++++++++++++++++ docs/asgi.rst | 1 + docs/asgi/delay.rst | 26 +++++ docs/delay.rst | 46 ++++++++ docs/index.rst | 1 + 15 files changed, 376 insertions(+) create mode 100644 channels/delay/__init__.py create mode 100644 channels/delay/apps.py create mode 100644 channels/delay/management/__init__.py create mode 100644 channels/delay/management/commands/__init__.py create mode 100644 channels/delay/management/commands/rundelay.py create mode 100644 channels/delay/migrations/0001_initial.py create mode 100644 channels/delay/migrations/__init__.py create mode 100644 channels/delay/models.py create mode 100644 channels/delay/worker.py create mode 100644 channels/tests/test_delay.py create mode 100644 docs/asgi/delay.rst create mode 100644 docs/delay.rst diff --git a/channels/delay/__init__.py b/channels/delay/__init__.py new file mode 100644 index 0000000..389cd5b --- /dev/null +++ b/channels/delay/__init__.py @@ -0,0 +1 @@ +default_app_config = 'channels.delay.apps.DelayConfig' diff --git a/channels/delay/apps.py b/channels/delay/apps.py new file mode 100644 index 0000000..f68802b --- /dev/null +++ b/channels/delay/apps.py @@ -0,0 +1,8 @@ +from django.apps import AppConfig + + +class DelayConfig(AppConfig): + + name = "channels.delay" + label = "channels.delay" + verbose_name = "Channels Delay" diff --git a/channels/delay/management/__init__.py b/channels/delay/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channels/delay/management/commands/__init__.py b/channels/delay/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channels/delay/management/commands/rundelay.py b/channels/delay/management/commands/rundelay.py new file mode 100644 index 0000000..0a3e719 --- /dev/null +++ b/channels/delay/management/commands/rundelay.py @@ -0,0 +1,39 @@ +from __future__ import unicode_literals + +from django.core.management import BaseCommand, CommandError + +from channels import DEFAULT_CHANNEL_LAYER, channel_layers +from channels.delay.worker import Worker +from channels.log import setup_logger + + +class Command(BaseCommand): + + leave_locale_alone = True + + def add_arguments(self, parser): + super(Command, self).add_arguments(parser) + parser.add_argument( + '--layer', action='store', dest='layer', default=DEFAULT_CHANNEL_LAYER, + help='Channel layer alias to use, if not the default.', + ) + + def handle(self, *args, **options): + self.verbosity = options.get("verbosity", 1) + self.logger = setup_logger('django.channels', self.verbosity) + self.channel_layer = channel_layers[options.get("layer", DEFAULT_CHANNEL_LAYER)] + # Check that handler isn't inmemory + if self.channel_layer.local_only(): + raise CommandError( + "You cannot span multiple processes with the in-memory layer. " + + "Change your settings to use a cross-process channel layer." + ) + self.options = options + self.logger.info("Running delay against channel layer %s", self.channel_layer) + try: + worker = Worker( + channel_layer=self.channel_layer, + ) + worker.run() + except KeyboardInterrupt: + pass diff --git a/channels/delay/migrations/0001_initial.py b/channels/delay/migrations/0001_initial.py new file mode 100644 index 0000000..82e85f9 --- /dev/null +++ b/channels/delay/migrations/0001_initial.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.7 on 2016-10-21 01:14 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='DelayedMessage', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('due_date', models.DateTimeField(db_index=True)), + ('channel_name', models.CharField(max_length=512)), + ('content', models.TextField()), + ], + ), + ] diff --git a/channels/delay/migrations/__init__.py b/channels/delay/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channels/delay/models.py b/channels/delay/models.py new file mode 100644 index 0000000..4bff090 --- /dev/null +++ b/channels/delay/models.py @@ -0,0 +1,44 @@ +import json +from datetime import timedelta + +from django.db import models +from django.utils import timezone + +from channels import DEFAULT_CHANNEL_LAYER, Channel, channel_layers + + +class DelayedMessageQuerySet(models.QuerySet): + + def is_due(self): + return self.filter(due_date__lte=timezone.now()) + + +class DelayedMessage(models.Model): + + due_date = models.DateTimeField(db_index=True) + channel_name = models.CharField(max_length=512) + content = models.TextField() + + objects = DelayedMessageQuerySet.as_manager() + + @property + def delay(self): + return self._delay + + @delay.setter + def delay(self, milliseconds): + self._delay = milliseconds + self.due_date = timezone.now() + timedelta(milliseconds=milliseconds) + + def send(self, channel_layer=None): + """ + Sends the message on the configured channel with the stored content. + + Deletes the DelayedMessage record. + + Args: + channel_layer: optional channel_layer to use + """ + channel_layer = channel_layer or channel_layers[DEFAULT_CHANNEL_LAYER] + Channel(self.channel_name, channel_layer=channel_layer).send(json.loads(self.content), immediately=True) + self.delete() diff --git a/channels/delay/worker.py b/channels/delay/worker.py new file mode 100644 index 0000000..c2e554b --- /dev/null +++ b/channels/delay/worker.py @@ -0,0 +1,82 @@ +from __future__ import unicode_literals + +import json +import logging +import signal +import sys +import time + +from django.core.exceptions import ValidationError + +from .models import DelayedMessage + +logger = logging.getLogger('django.channels') + + +class Worker(object): + """Worker class that listens to channels.delay messages and dispatches messages""" + + def __init__( + self, + channel_layer, + signal_handlers=True, + ): + self.channel_layer = channel_layer + self.signal_handlers = signal_handlers + self.termed = False + self.in_job = False + + def install_signal_handler(self): + signal.signal(signal.SIGTERM, self.sigterm_handler) + signal.signal(signal.SIGINT, self.sigterm_handler) + + def sigterm_handler(self, signo, stack_frame): + self.termed = True + if self.in_job: + logger.info("Shutdown signal received while busy, waiting for loop termination") + else: + logger.info("Shutdown signal received while idle, terminating immediately") + sys.exit(0) + + def run(self): + if self.signal_handlers: + self.install_signal_handler() + + logger.info("Listening on asgi.delay") + + while not self.termed: + self.in_job = False + channel, content = self.channel_layer.receive_many(['asgi.delay']) + self.in_job = True + + if channel is not None: + logger.debug("Got message on asgi.delay") + + if 'channel' not in content or \ + 'content' not in content or \ + 'delay' not in content: + logger.error("Invalid message received, it must contain keys 'channel', 'content', " + "and 'delay'.") + break + + message = DelayedMessage( + content=json.dumps(content['content']), + channel_name=content['channel'], + delay=content['delay'] + ) + + try: + message.full_clean() + except ValidationError as err: + logger.error("Invalid message received: %s:%s", err.error_dict.keys(), err.messages) + break + message.save() + # check for messages to send + if not DelayedMessage.objects.is_due().count(): + logger.debug("No delayed messages waiting.") + time.sleep(0.01) + continue + + for message in DelayedMessage.objects.is_due().all(): + logger.info("Delayed message due. Sending message to channel %s", message.channel_name) + message.send(channel_layer=self.channel_layer) diff --git a/channels/tests/settings.py b/channels/tests/settings.py index d720f33..2ddf8ac 100644 --- a/channels/tests/settings.py +++ b/channels/tests/settings.py @@ -6,6 +6,7 @@ INSTALLED_APPS = ( 'django.contrib.sessions', 'django.contrib.admin', 'channels', + 'channels.delay' ) DATABASES = { diff --git a/channels/tests/test_delay.py b/channels/tests/test_delay.py new file mode 100644 index 0000000..8d22d0f --- /dev/null +++ b/channels/tests/test_delay.py @@ -0,0 +1,102 @@ +from __future__ import unicode_literals + +import json +from datetime import timedelta + +from django.utils import timezone + +from channels import DEFAULT_CHANNEL_LAYER, Channel, channel_layers +from channels.delay.models import DelayedMessage +from channels.delay.worker import Worker +from channels.tests import ChannelTestCase + +try: + from unittest import mock +except ImportError: + import mock + + +class PatchedWorker(Worker): + """Worker with specific numbers of loops""" + def get_termed(self): + if not self.__iters: + return True + self.__iters -= 1 + return False + + def set_termed(self, value): + self.__iters = value + + termed = property(get_termed, set_termed) + + +class WorkerTests(ChannelTestCase): + + def test_invalid_message(self): + """ + Tests the worker won't delay an invalid message + """ + Channel('asgi.delay').send({'test': 'value'}, immediately=True) + + worker = PatchedWorker(channel_layers[DEFAULT_CHANNEL_LAYER]) + worker.termed = 1 + + worker.run() + + self.assertEqual(DelayedMessage.objects.count(), 0) + + def test_delay_message(self): + """ + Tests the message is delayed and dispatched when due + """ + Channel('asgi.delay').send({ + 'channel': 'test', + 'delay': 1000, + 'content': {'test': 'value'} + }, immediately=True) + + worker = PatchedWorker(channel_layers[DEFAULT_CHANNEL_LAYER]) + worker.termed = 1 + + worker.run() + + self.assertEqual(DelayedMessage.objects.count(), 1) + + with mock.patch('django.utils.timezone.now', return_value=timezone.now() + timedelta(milliseconds=1001)): + worker.termed = 1 + worker.run() + + self.assertEqual(DelayedMessage.objects.count(), 0) + + message = self.get_next_message('test', require=True) + self.assertEqual(message.content, {'test': 'value'}) + + +class DelayedMessageTests(ChannelTestCase): + + def _create_message(self): + kwargs = { + 'content': json.dumps({'test': 'data'}), + 'channel_name': 'test', + 'delay': 1000 * 5 + } + delayed_message = DelayedMessage(**kwargs) + delayed_message.save() + + return delayed_message + + def test_is_due(self): + message = self._create_message() + + self.assertEqual(DelayedMessage.objects.is_due().count(), 0) + + with mock.patch('django.utils.timezone.now', return_value=message.due_date + timedelta(milliseconds=1)): + self.assertEqual(DelayedMessage.objects.is_due().count(), 1) + + def test_send(self): + message = self._create_message() + message.send(channel_layer=channel_layers[DEFAULT_CHANNEL_LAYER]) + + self.get_next_message(message.channel_name, require=True) + + self.assertEqual(DelayedMessage.objects.count(), 0) diff --git a/docs/asgi.rst b/docs/asgi.rst index 3ea981b..dcd57d2 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -1050,3 +1050,4 @@ Protocol Definitions /asgi/email /asgi/udp + /asgi/delay diff --git a/docs/asgi/delay.rst b/docs/asgi/delay.rst new file mode 100644 index 0000000..6edc19a --- /dev/null +++ b/docs/asgi/delay.rst @@ -0,0 +1,26 @@ +=============================================== +Delay Protocol ASGI Message Format (Draft Spec) +=============================================== + +Protocol that allows any ASGI message to be delayed for a given number of milliseconds. + +This simple protocol enables developers to schedule ASGI messages to be sent at a time in the future. +It can be used in conjunction with any other channel. This allows you do simple tasks +like scheduling an email to be sent later, to more complex tasks like testing latency in protocols. + + +Delay +''''' + +Send a message to this channel to delay a message. + +Channel: ``asgi.delay`` + +Keys: + + * ``channel``: Unicode string specifying the final destination channel for the message after the delay. + + * ``delay``: Positive integer specifying the number of milliseconds to delay the message. + + * ``content``: Dictionary of unicode string keys for the message content. This should meet the +content specifications for the specified destination channel. diff --git a/docs/delay.rst b/docs/delay.rst new file mode 100644 index 0000000..1539fd6 --- /dev/null +++ b/docs/delay.rst @@ -0,0 +1,46 @@ +Delay Server +============ + +Channels has an optional app ``channels.delay`` that implements the :doc:`ASGI Delay Protocol `. + +The server is exposed through a custom management command ``rundelay`` which listens to +the `asgi.delay` channel for messages to delay. + + +Getting Started with Delay +========================== + +To Install the app add `channels.delay` to `INSTALLED_APPS`:: + + INSTALLED_APPS = ( + ... + 'channels', + 'channels.delay' + ) + +Run `migrate` to create the tables + +`python manage.py migrate` + +Run the delay process to start processing messages + +`python manage.py rundelay` + +Now you're ready to start delaying messages. + +Delaying Messages +================= + +To delay a message by a fixed number of milliseconds use the `delay` parameter. + +Here's an example:: + + from channels import Channel + + delayed_message = { + 'channel': 'example_channel', + 'content': {'x': 1}, + 'delay': 10 * 1000 + } + # The message will be delayed 10 seconds by the server and then sent + Channel('asgi.delay').send(delayed_message, immediately=True) diff --git a/docs/index.rst b/docs/index.rst index 9125e55..b4c12cd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -50,6 +50,7 @@ Topics routing binding backends + delay testing reference faqs From ce65de323ca4653da095a7d780e0d3c74b7229b2 Mon Sep 17 00:00:00 2001 From: Robert Roskam Date: Sun, 27 Nov 2016 13:12:50 -0500 Subject: [PATCH 566/746] Updated channels loadtesting results (#437) * Starting reporting write up. * Added in charts * Added in images to report * Cleaned up comments * Added in clarifications about the testing * Added in clarification * Added date * Added in subdir with same content * Added in supervisor configs * updated the readme * Update and rename README.rst to README.md * Update README.md * Added in version info. * Changes to root info * Update README.md * Update README.md * Cleaned up presentation * Update README.rst * Updated images * Updated images and content --- loadtesting/2016-09-06/README.rst | 24 ++++++++++++++++-- loadtesting/2016-09-06/channels-latency.PNG | Bin 17031 -> 70346 bytes .../2016-09-06/channels-throughput.PNG | Bin 24427 -> 24731 bytes 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/loadtesting/2016-09-06/README.rst b/loadtesting/2016-09-06/README.rst index 55609db..c578720 100644 --- a/loadtesting/2016-09-06/README.rst +++ b/loadtesting/2016-09-06/README.rst @@ -13,6 +13,15 @@ In order to control for variances, several measures were taken: - several tests were run for each setup and test type +Setups +~~~~~~~~~~~~ + +3 setups were used for this set of tests: + +1) Normal Django with Gunicorn (19.6.0) +2) Django Channels with local Redis (0.14.0) and Daphne (0.14.3) +3) Django Channels with IPC (1.1.0) and Daphne (0.14.3) + Latency ~~~~~~~~~~~~ @@ -22,7 +31,8 @@ All target and sources machines were identical ec2 instances m3.2xlarge running In order to ensure that the same number of requests were sent, the rps flag was set to 300. -.. image:: channels-latency.PNG +.. image:: channels-latency.png + Throughput ~~~~~~~~~~~~ @@ -34,7 +44,8 @@ For the following tests, loadtest was permitted to autothrottle so as to limit e Gunicorn had a latency of 6 ms; daphne and Redis, 12 ms; daphne and IPC, 35 ms. -.. image:: channels-throughput.PNG + +.. image:: channels-throughput.png Supervisor Configs @@ -42,6 +53,8 @@ Supervisor Configs **Gunicorn (19.6.0)** +This is the non-channels config. It's a standard Django environment on one machine, using gunicorn to handle requests. + .. code-block:: bash [program:gunicorn] @@ -56,6 +69,10 @@ Supervisor Configs **Redis (0.14.0) and Daphne (0.14.3)** +This is the channels config using redis as the backend. It's on one machine, so a local redis confog. + +Also, it's a single worker, not multiple, as that's the default config. + .. code-block:: bash [program:daphne] @@ -76,6 +93,9 @@ Supervisor Configs **IPC (1.1.0) and Daphne (0.14.3)** +This is the channels config using IPC (Inter Process Communication). It's only possible to have this work on one machine. + + .. code-block:: bash [program:daphne] diff --git a/loadtesting/2016-09-06/channels-latency.PNG b/loadtesting/2016-09-06/channels-latency.PNG index a2f7c5af4c097d30c4802da919d7f9250a702f88..31808f93b8b4bfdeb1efd5c1d94e32bf3a2289c6 100644 GIT binary patch literal 70346 zcmeGEby!sG_6LjuGJ=Gllz@aHEg{`7BHi633@~(qbSMZ&NW)MPLwC1GN_RKXE#2?t zIY%GA&pE%}AMd~K@w%LAX7BsncdUD@^;v6uHi7c8Vwh+|Xb1=hm=fY|6cG@R(-07l z7Etd4SF&Erk0T&p5I}{6r^=jiDA01O)NGXf+h&$nW^cO&QpaHOY|_Ut9a) zZXkU*eNJNW{)G_DqsP8M4KL}c@s-~QDLa7P3|1hNs=g^HMm@PHPAx5E#J9+*|dIqh?%cJ?Pkh_allshP}1L*VvxH*iSBVtqv$FR=;2O~t}Rz@aJQPSf7Y z&Spk{l;76X*CAp{c#Ud4-qP#1&C9lX_xvG(IBsi>_9JSPr51#RghXuv0fhIm*_;VU zDfG>L*m>yBzz83!k+Zss;mBEC`^Y&m7UfK>FA+MGIke&)dPltHEc1?hxJ)C#HHB8~ zj@YV#?PkX^q5W|W%=e7i1i!V*=wq)^Q*h)>Ra}nYtI6%7S1-Fo=!^s#X*gm0r?=Xn zw+!5qOM~1P2I9P-sdZuBb`-gspD_kmE^qQn(4A@3u@4qkS_bh9HWNcN`V9K{SDz-x=u4rT|u5QGsbjVV?fH_^Z*DH-DE=z?<)DuCXvOJo;t-baIR{ zB*ALG*5XWG4RW0Qy88(z3wouW5x(Ak9sbbet7akLv3PhW>H_C|TwlRwONfJUDPay# zC}^oS-EpcX75AY-gGf1I_%3L+4+}%wmUdm9T$SpVb?DfPD+t1WuIIn;N+Wyq@rh9R zljUZg{4xU^q?@L}58ske$|7wZeOldTAYG*W60rZ#CTT#b8gr=`2Z~!WDt)84h}9EM z{`BaZDdNvzapQm0lA8>~(;yAHx?j^cLeX6brcrZ^bIh-be%U;ECq<_%ZM z`{UE|&SS|K$xu7|9K)thdvT*-3hqr>6pY)#UgfQ-Zv%=SZnHTd^7^yiqjGv|-po_7 z{FqPmfm}v3GmlnzlotM{7P5*G%i^7?>#LR-i}S09M~iV6%9 zC%5w%7keAB`arGC*O0w`Sj3W$SU(wJB{tB$f_L+Ryv<)J!@Y(}LJ+tva1k!*ta;MM zi)Y3WZR%kZ`1qX;5z9}o7rZbb)%4}wq|8XG%w%p0$TNuOXMWh&58i?w$04fm-TNWb zSJv!ajEw8^xcH$35}rQs#DmSR0yfxeeufKdTquk_RyHIv9~=i!biKj*4>2fEU%xg* zB^i8u^a!UFtwc;4TdtMtfpFUA7cW2(V#pt##D~2UmJCiuYwU^GA>7 z9tFO+Klo`K;}r7vRU}tfbwYJ?b&5;Sgs{8x)90VXHDAz2pwLFNQmsS=iED|wNl;6u ziF;&V8Htn%OeQ}SlBGiLl==?&!T$s22mKF|9|j;i5Dtj*gE*-0oA!qav^rc(JVyE> zS_~y+T4P)+$(NLorSBDGp&5ypGwJmT%F1lY4l0Rhood?o>V+;s2lY$BZDGpnxs;=^ zJRx-O!Y^ZrathW;^8==AN<_gi(%o4FUq*-f21o}4Xa=J#qI{#PC4-~rq7$MDX&RJO zX+ge7W_Bq=!{~%_9aMNn#Ydlxj^?^4Gi3{? zQRKwup6A()#^miN^QND~g${tJO>2~Ekn)fhmNASF zt#5xCA{nww)PV8HpbOR5h;@wRA&V8WK*%u38h^7Ck6}J_jo#42q_6q=OATmTi{L+cDdnrE#UTrCFpo zU=DHhob0SJ%sK{B78f(p!!IZD2lAV@lev3my7~I++piKpgWuYocfZXOsiHL7FpW3R zfWDhVtFS9w(z2}Bj2DVq-~ls#G5#iFo?B7n=;P>jEKKY#6W6NWgYdJO0&+!QSr-^E;{2db?x0 z8oO>YDRLaJ9$3BkPeUwWtjdmuVY+f~lb|)HeqE+}OqFUO;}kKEg1e`BT~{p^lQtU0 zFEhgm!Uonp_PgnE&Vw4Wu9%yIn%*`sda>Vf-J;#1Ao?S2AYvl=`Goo$BFTXs`M&lW zLz%@G!(2rkN5S&v3V4YM#me+=f3Nyp0IduI$G^fqeeu~M@wYvZxHqWJ6DYt&Hb4A3 zFBOax-YGQ6Jd+JfP~={Lmg^i{t%p8w?dgf*f3^44eTQ6dQJ7cC zLacnk0F3LEEqE#O#m);)MJX0qZh?2sKL?a=b71fj8T&+4ww-o|vJHEU><@KLlepQI#uH zu~M+IPS%=(*MkkRZK_Zo$q-ch$ZIi|&6?1@iaay$F#$uG111m3uzD4eDB^~>$L)4= zYx>ddp?Spz^e^|A!>pOTzF!5P*Ar~3e|#yeGf~-TI`~t>BlsS{IK!8h-!*?;Do$r+ zO3gD$sgKoaJD)GDgn6M|<$Mv(5m$Gx|8N;(Nq-Q16|Ix^RMk@BxFSx=y`;4=Bmc*t zS>I-`;Pkcj&0%LkIlGQ|(wJ%yMIlapUCErehfa&7bCEcKgM|~{+Px0C3P*=T zc2!!!E%2>(8L;TBC*RgXW@GyC2=T`0J?Ku=Hk1o;U7MoojwRXiYHeY=l@XRHm4{XH zQ;(|CbK1+{7Sy&T&3##QOU{m0OFyNqt#)Q9>zYQ;%-pupwx+g{wzf{!w>74{IX)6! zoXSp5H}qTUNu@h3hhBfWrbPLQc;F-GDRX|@-aO!E8KC`;os@%H+wr|;M)d<|!Zh1N z;^545#K@36+YeK%Jh!dS^A?_Y{Khu`DhXcYHr2T?^B(sk1ps>uy_>l8sH^A>j?X+C97c%$QXO z@9vA$s@18}a-%v;tX^_lIZ|oLTPpM;?58@ z)}4_q>K=GRUqY}PNJp+(mdGyMl3c?CkK88CX;uekI}$sXl0PQ9?`qvdos@6K-;`af zo%DwtqZ~QkzB&$TB&cEXM+v%7zAoF$JnwGtD!JKwD*bfZjnbX{r|U6s0cV9Zb%P~w z5RvOX0J3~c7I0yir&k{a!<@_SL+uhMlWI+jHHT?o>-V_ z5+M|sBc@U|*9mhX;8IRbOVNGYZlDoa5Hy&)l&0HXRqRG0w|a$Oupg?334740JRwy$ zSMt)EhwBr2&Bvjq+mMp%eUR`OGs{m=0t7{R1ikkZ6fIg}qK(_WM%MRi=;TgTBT*mC z?Nw=s%d1g?u4rzNJF+bvbkSSR0+GT48*vSL1O$A_yZ?w1iZAwoAOZwcR(DXBk>)Y9 zwq!IgvVIF;bg{GnS|cFvx$ppAEg=pD?2K%T%+CeT$jQn1?2L?g6yJ#cyF2iU|GBAygAET8le4okqcba`wVesm zD{gLXCT12U78VAe1%v%ND+dD?1}l4te?8>abKXGg4eg*d4p3_=^1J66ytQ_8;D7%7 zZlK@){xwgC3-s@itnB~2EMS35cXyaxF)}m#w{8wlBECOhJOut1efQIJxq7?x_2tncvSlI<}d-g%S z%JNOefgWxGbuS&OgcWXl;j4&etno`-;Vkq~&$L+z(i)RqOZpJ`YL%oR}q^n{h2oc5-RyWK8pQ8 zW120~CWrrV=dK|aB9g)k>VLcS?h!e5G=Z^`?|*rw5P6Ur0wOZD&_6!NnUIjvetzqb z{f|39L)166SpS^d-4uR^cnc}(#!Wu>&$)Xe_|WYBmvIn$>QS*jcA>NKg8wmtyPxFP zZT$b(gui}<699{SXJAbI-J{pWBLXC!~A-ZPx!2(qA(BpOF4KY5&utzZC0# zn)E+&_qSX4f9aD_&2&$`0e*~JVIfjZpag`DuO3aO@;J7`WGr7}cylDjPzJ_P;d;E; zMsV1G^gueErLFODmp4zfK+{FFZo4$NpsbH)(U}TcT?y%Mt-q?M@uJJ~>Ok!BuwT%! zEeNl2P?T6B;2}vq@Cc1P)oDUj<4@NYTSdEjV_1S$`a4q*--LJmy6Wl?+;yJ(bf|ywh)k8A=84d7l4vc2Sr{9k)B0^Q==H~A zT4T8?^jqUaDc3ibdz5p{n={0elR2e&C$r9aTVn;V>!00B?Oy3Lfp2fd<5^8YlkEGr zPr|&eTHtaijhZN_TXQ9b9k2c5r<%V6keZn!ImQ%dR?yjc(W@1f&o#K2V7!KAPL`QW z16yrQA^J&t#@~2zCWB1cK23H8mn?d_139u{{eZofv%>SnC#shl#ScRTFmY_T-v)n?Q739Lt_ zK~0xVowlm8tT-)L7s!Ndu*N$;-gZa9g{f)59DD5JcP=%LGq zf*2)Zu(PUso>VMjrP23@v*(Qzf1e64fFB_HX*S}hrV~2A$(jv=2b-zsc~oqGBUD|H z%8Yv@AN+`#`;2SdPjqo~b=Xf}P3ao@01?x&TLs2to940Ki~@xX4-C2OHH>r2xh$7M zVzuo8^R97qQoSt{W8gQT5F)tq>8!ID=OUb1I!=GD{!n)63YhOqDF4Ys`8itZ$`FMm zO3&ZGr!yYdt3-v;XUKLGsoxr1QMJlVKVJ@(m#X5LDt?(S)Tx8-h+~D*S;)}y`{b$& z&tDhs)^5ejAwp_Q@H|7w+-!psOlpX03lM#X*v)=eS%57qGJaEoMm>^6T$8BM(22P9vxCe_-S0l;j9=#5*H}v>nK-7#vu|)2*sAIv z@oeRcoz%2NS?`Q9-bUz{$WYC?j8`M{`fQ)D0mKY_6360r1TQqr)2gyMGVgL=>{DRO zjm}6-?h{$d(jNj$hBbp`c`+u(1UV$-rh6z;N=JwCcGhcC#=x~MwlZ%gziQ6aQ=w1r zn7qhPEs2c1K}Sw7goxdaJ&%aPBCb-7(7fS{{rr3_#qwaOt*~jgPP4)+vmlQ!ckyu2 zBxQ1@#y%;!uo(`s+T6{PJ}nuG4A6ED-WBdhU&yP>SstH$#%@u}#a`y%m0MibCp9jA zss?CZ0bm^it=*LdICXLIirosTPvL_78FH50P0tjXU_82BbMd>}=Nn=^8!nM4e2xMi zl7t=juag^ocF_-Nob-F$3}dn$vmU$UK1-x~n4AMmLE~tHnW=BYjd0A^2=gv~V%_ed zFT$v!wPANXGUZ)Q^_$_FRU>kGVtFw;EYClGGGi|{to0&YL%S`v=zSRs9WoGR4J&FayI{JR_~|6aODx_&@JiajS9odhH7#c++)*oU@v9&-u3K& zBgt3QY>lp2NX>)YgN4(aMYJ6J~NPXJs+#?k4PiRNCz z=^Q;O3-;sj!CI5#cgA3BW?F|7%_qnxXugZwfzy+=9VF+LUx3HsMzB7s3*@2t*aU+*_yY<;xx z;WzumIyMJw1SZ;?PgFIrkvSB{0zCs$wDwoUoKtX}FdYPueKOaLhv0sx>frE-$b)JJ z604e(7+-RmCxyhwVBqkbu?yyaz%Zm)l)=l_!?h3Fpx}k-1%F~I@o8E2y4u|XH^!T~ zM|xuOtg}qr|pUI z9c{Or3WL~;`JCO^x~wY(g|)ifT6#;~3oZFXxMAq6&M^0^!>|l=daP!>w(T|)>Up<0 zo5q<1%MH8rhkfPCVWcL^`OG_9SG6}vr;u=d)pe(|X(OUveugFw>7oeEhKx-^>2r3k zhLkG}%Sj?26Xna9U*y>3(XgCSEy^Sw2+-QDGyNZAR!2D~PeBPR?EF`6E=_m~7Q`g( zDcHg?P`0!a(|zT^#v&zsOixI06)E}IsL$7*Rd6xD4sQWl8tgd3qWo1}25UhCWa&V9 zVE)9&9G7U~Q?lFs>Q^L?)e-|K*RU;#*VTbYa&Tc}C~poPKJrJ>NVvy>(DWd`!;l1Z z=FVkitghsg%-aQp=Gu%y=@}trrKXliWW@9i63k*i;HH9tNk^9v8!n~sWJdaM9p@bi zqT=E5)pNrEldGG-#t2Un-0~AdInZ3%^nkX_j(f9Jzz+S(YNytt$y3msc{#BmcSU&ndm5QF-^)MV7^b#V z*WK4o;&>`zT|mUFF9es#>=|bce@McUVX2lRaJAnO_3RZbJ^lfuODBC=$0kEx8~1$; zJlYnwQtsys3A!!2BW%)HY?}MzW?-Q-^}2^hAbca^YDb3|B1tvnVRnogls?B{ZgZ}v z%19YvJ{OxM{`COpxuvOFnBYzIZNh<4OYxlN`$l8m_((3ICY)7tOai-b5-)r^4TwR##6b* zxjmV4KV&Y348og=1`UD+A@P9PB!V;`@%XG{zYpCZkRa*0qFh3thkXU`qWM~qvu;;E z;AM7@L~=PJj=mPY{n{uRSg*V+)*Rt7Oq_MJaJUMyuC3)gSP}Nec;NR?6OLlrKuDiM zi~86Is(uurHd(*O1l6L9j}5k0=Fbx&efI)#+FVSpR6sbIGw4_0OO~sEYw0@Sx%h!# z=H;l|l>1>XLnm^e(}OG(ZC%F6v8@McFW)W!Cc>$Ma0lP0QiIvO>t^rvre9}S7>l2+ zc0wi(9*IPl%b&h99ABsr%c!Gm;69ppfQSUzS@OAWsq90(ql6gjqQp$s=^fbLcc$aR zGWxN0r*iu7<+`iLy*Fr~8@*Ns@U3U)4H67Fg$is>a4{HU>e|t`+WJ*4M>*1EE)^Qm zEMR6_lnpAfPCVA2C%6lNRr=yq>fKU~)hs%kI!d$lXh-?718**GH?mSJQany1FsL?N z`uR_1qFO$oJW;|IZCrPzQks3`sAQR8p{Y1aIXiBiw&6@a>^R0Uf5e;hZI3QLdF1F4 zyI_yNXfD&?&ft5te&2Lu!@=XSiX@3=t^~R+23^eDn2#Qisw-VU)kDZ?Dd#huu}tj(C_m*}kpZtn z3rHbTwo8^dTVH?S8Wqt8$1o=dl zuQbD^uil<=J8q7Y3*?UDB4;aJ;GL}0?KMhW>eSrcTn?+&ECtaIo!~V!N(et?-)2%{ z$cc%p3Vn!obSXt-7ppuiajNaEo=tWFx1_}MkYbUXN+~q7)f_5We|qe;v?q4$F?{2P zP&n-{bO0q8^$DsZkBd5~%k&()Wc0+Wm_OU+zb18Ky6$b<%*{jdm9AJXj1&D8y>&$+ z5Gpj^(q}xAPex9gY}-sO*pAkS79{nrq$ZV` z^_<-O2mB27Fmzqq+dd~1n#)`LSJ zVw2YN5BJ9v*%Py@vR~_i_tcrX9Sw@L7(fQk2RF#8*XF7TlCtt&Le5%Y{R2a<0EMqR zXxp7?V;zW6$%b2=?9I>n^5|KKC+VGs3qL%FTr|P95&zEf?)Lh$1>(6BXBhf`wf(C8 zn?SSyyy{|ubc}e8N;c87%k&e64n;%GtGTpzea)CCpQvsFx7w_TgwhhHX-m%78azR` z%dL*{Ec?oytb5I^z`LJ6_GLmmb&tw`IBc+pFe~~X9^H?8A9pV7f@G+;eto|G3=tVo zk@xy&SjpJDv%C_=`og2v;S&)@Zk73cAMw~{Yx7sx7~->gV1y~my=fp2Sca|e{>s?O zTOsGhz$jLP9tnMYu)ZkY!t2W!GHDF6jDe>GL@yNR(;l*8h#um`edM?mB1<|QRnwaU zEcTi)R9(D2$>!UG1C8A}GZZM7yu*aV+m5F_J)_)oPh#2X#Spn0-p+A?Ja9V)TAcZc zmoKBnvUJ&-+VHiiXU(|`!fg$~kL=yfS7LVrZ_nidvFJUmTlYvT)teuK=>1;LE_xc} zUXo9-wTd72M)=Y<>cKcHuZ+f0c&I?=fZ3Tdij+w!n{falAnkqX;bJDtq8Y<5eno%Y zi^X>Uf^|fs1&yT1_}s@_c_}e$TW&Eqbltgnx!2?@{6ys*JIK$+X=Gai`HFeC$@G#!~)?S30=s_7tzce>5ZiO zM6CbJ_(T8-D8)=&2dfhby2dH#Q_SI>iJNT}-A1H@#pKy5Q-!l&DWhZ+!xNY;6o&l@ z3qO;hcE4647bUc*Ra;^Yv3uv<)$_56l;5@0gwk`|mUci)O?4z& zfx=cP-dBDz-+};gDdIKnFq8WTj==hC`iF8KBT)vw;IlMmBQ1@3X*R##Sdml9B(Pexs z?M2|ov|HTrkT7_dzF2QWic;B2#_`@L?hB!^0zyK9Q z1G(a(G$CQEt}ApZCwHLqhW?$RjSwN-;O#|nrQIvzR#x;Q>?qpQ%mw+v6RnYFt3RKv zkT419Ox#2-4i@V0N)f?TCS1+m#s>w^M~cPcn6scpLs7L0`y@_P4p)2aD{3BFixE=u z#F!4l`dE^hN*_xN;m(+iWPk3)rON|nXB;Xr)7^}5&N>fg$vMxmVsZvsOM$sSYH17? zPWaoSo58ud;V&%ZiIrV_4FZGX$oo_JE*v-gnA$vhcOwZ;00O z0Q)1IeeC;tx!XvC2C}Xn6~dow_^4^fx|*xhiw18Jw*`oPSH=={T_4QyOsjCd;yd0N z_#({J`r<}ECxCpkO2W+5CD@8JN6vP&?UY1^Gj=U|hZMl3jN$3E@)Ayi9+%LD?{pXY z+iPoEN-et=M;npVypT;k0Ojq5Xc4j0)XJQGnR^*KqRiAeFL?3kA^VUsgPKhi5K&9y z1stNe1#3>w20v)nV7TG^+-6o7TG+8Qgn$?R(h40d8MaA9$quQ> zR*l&dr5(;LPw=HXSZugakMQp}_pT_k z9uXu$&nkKAcm*6N^eQ+??U`@`+D7C4H*^DU+HLLDR$1O1qpsROj!_N?Gc<(OvZoOj z5LP*?+;p+9z6ti&wRPt-3_LvIu`rfGfy-^y%RR5#t{mPifK~;`*lvh zbjC|YyiaNj@#_O@{6FaKnK)_<`Bm&mqF1l6Tj`1_B+AYH1Ur*|75`ln zjDs1iNG`#ipe#qHiM3U|94b|_=C~6L0<>j#f7EO=H*=d8#?(B$I#f%W{qFjBJPX-2 z2u(M`(um!7-5D@SUqIGE<&P*ts~I-jhQdo03K9rUGrp$#>xp{lr5%S~1rfmst(&zD zzrD}wK_wJa!Hlz8FcF>MlFK-#^=}08q%96W@{bo z1%VtV=jf&qwld;?w1z!e_@)5-cR6xJ#`{zDPp3AnIZ4{o05eGS3wHj*HgoPAeTVW5 z1FD>l)htZ7E(!?3Sc+X^AB_oLjJgkLrADyGM$A3Ly9;`Y#7myHkzGAtJ2T<-; z3F|Ltx^=sxc6i|f(>YO}zii`<`X;cO$8;kFo49Ql@NW1_k);SMcRbOl&-!`79f)>VzOeXIWCWw;X=%5r9CyT+hD77T8QMG`#tM6 zH5G#zBywCO&LltaZY^ncFWk4`&hAVW9az$<%L@O-P?yi z<_rIDZD$+W%$!|*ja*R!q&+?_da^y6PStmmoF~6bCr6$6aP9k5zRQlJsuIs=8~3GN zV?DWxz4inrEby%15&y0Jx;%J4HMz7&UtBvV?Bvl1r{qSuVUU%WhlAV;OL|l*Gg~mW zxJ?T9M9rr#x{37WFj`I5w)>$NxlHPI#;6s2%8V*?gS*G>StM%HLjN{!|UdS}h+}WPHyg5UGTz$&!_hf!I z_%V8^$&A6qg~%-xu2cR6iOhwT&7cSY9x|A&D%X?A@9i%1a`^g%-h6~-hhKh;FS+<~ z?fTgj`_GhvOJR}<>!ukmU6;iVyQdP}WIzJ9eC227Nn!vg;PRiHqa>`GF#n+5CNjEsaWYs71kNtt@MqEZ?gNz(q^Oah z3LbvGn;<_FELnH&;|@G5B6lD^N*G4p*-3-S68`NlSn#Uj9!(8bhyD|_rGxbuUsnbY z8Y=Q6F(`o4)l^du1i{t+DFZIe5DKYU2a@L{q!G)zgNJgiE+^w}PUXUV6tOW#qw z2ibvw!O;R~eGPp{(cYe)IJlv-)stsTAzc&?xuxcDEXpK%@^=q>C#%6wi8aEeEL`#K zAEfn|C8=sfx`7=T_Lob+OlfuKS$@wa2*(Fqfov_-CBE}>&)unH-vqSv4Nr_Y&p6w8 z@QHrcZiH{iRPybhN4RA-uJ&LhuKe-zC$Edtp!qOQESBpKw9?MURgt*Z(r81b^QDUV z^qx;INHH>x13n*{aGa{U8-CE(;PU0P#YdhD4D6sGCYzC&suC2_nbyy?2Q%k$yrRkS z1UEFEeK{cc?NJEs;`=tp5;=XY(2y|Hz=wwW1?1`amp53;0&d`I7D!OGH9SS1CR$Yn z8ef{mVoAXxNXjM;S5RCPaUhZDV0B<)BKmlM5*KX$$)u?@jB?l+beP64ES#yQhjlo( zUA$f^F?`@g;dxU9WR2Um)=%R%ShEWvOS{h+)3!uT^T>sVDgsZ!T8clhyj zDzb%?hYrr)9i&>wqZbXBua5_NQGxc-Iy(P8q%olEM|f9FyhKQul-PV%cy8mlGB_A} z5?XTX5MPTRo5nfhtSPz4O0eI3jqvlxG5*ff@9KadwtvK$t6H#$Rzxt^tDH`=@pO4> zW$EehYI9yuBTwVT>!v)?F&`AtxFNCQ@`x*U>d|j#q*K@9&jo{PIc2$5q8>M8$~!{S zWJDrZD;B`BzKfz(C1&HBAzrR`r(sVeuAi3`Y!v_~J zy)`*Se9Na%{#=`h5eY2j5oYKjGEtwSWV+c#Sa^^*U^_;e>DyFZ+JkZs)SSxL(o&!f z2?)P?>DW)%Ug9u)dVg3>@Y(@BOTcqmF|I3hb(ywfrU&-9eLa4P?r0IuvhJ%dITjD$ z!Sa$lkb|(bo7bLSn;gbRJ<8Nnir;-@$x2Hh=_P!p5^p%;@f_!&*$0l}I2AH^c-i=tpYkP7C=i^&cI7@Qtvyj|Wy#63|-5jh(dATTkyFJfWJZWca+CT^`P+;YXN@ie@t zPq?FKOYLc7QPa)TR5>)#PP_|w7}izVfTgG^dzQA0z6q~!lQ|8nKW=aTR*%E6T$*v@ zmea#tkS9$6AuQO{BCe3p(Yy7+0cvta7bFZ;N|mcX0co~kpwhu>y^&5fGP*JygP&bz z9Mi9fpJ&{?DpfR&mg41adHQ4Y)8RmDjA(W#iV@W^!? z)vf=V{twV!!H&KRI`&7+;cpgXK9bKASyrgb%_($1xJ z&usBAdMh~}BDsh&}pH75bZK3!~x)wls1T9R(n$BW0H@eRHTfFq&yf!0TfF=0Z zAN1$se-)l`f8QCZC|J;XvcNG!^5*36{^>n-#+^ZQe()#n_uZDym)Hjd} z1H}LKOh(||iejYYpOX5O8TtP|QD6~LGL;{GHU4{j9YJVlXxmffMF6-xXec$(;+QRb z^|{I00mgL#LL-ekVnjVKbPHqk0+LpRnaV`KgI^~6SB#3?Spcv>W?V}v9;05JOj|nv z5!p0QJ=gj%MHHHKV@Bl5X)(dfyZ`mx&VCD4)mn0=UoddfwR;dpbBht(=T{yVb~GsIDGZHbpKnGYX|_ncT=iV$MAV&*gnyW zLS}LCG8c-w0h9(8YN0f<0RJIL5j_{ktvCVIvswd>O*YRfmvLu+zjXr852`nedlM!B z&eIb7ILu7qcx$}BoWJ5O!{Vy6i+=lioL`a@lD#ufQIR2dYWje8U8I|~ZfekRnuIK# zW(0#e zf?UjXcV#BuzCw-)%vf5wBj{LG%{ra*a?TrRYfVZuxScu)#`AdGoUa)qQG@b;tZHx7 zlv!T6O2sUR(?s)Kx+)eEuYe(XES~V!jDMdC`L}>gri2@|X`?=M98rLi&Aa1A-s&dy zzdKy5sak=OfMVSnvH`kO0Bn$%pXfhKLR2V%@S$W!Qp2jcknkQrI3vVo*_*1!tFW4# z+7S^x9ASG=Bod5omEiTa>-zUM@>?XMH!Jgr{6JT5cBCv1ZT`JXgb@vsB)nY+kmkb5 zG9ND_k&Rt#eg6PQ(*AUt_*XP`=i!ljfI@V>LV#>TN5SAu(MCvEgLwdZ1A3+215oG3 z-1h_x&y#)vNYSW+)GeO@VD2QTMpDUahw|<+H-P)A0MXZcSIJYI1W0)cCWrs2DDUvk zE}{JWyvEL``GsHOUKwr!v@I3o8^AXT@VGf!4zsCcqaHi*dPm!9 zZ#e(w9s^`Ow#)B1U$=^ytT<;Kl&X9Cl6bY6Jg*<$y3pWsRb@)W zUVM*|gQ(`@getSa=rVEi{;|`KK>!vsx}#Aom7etgSa+P%^UPptBuB|zVXl$hhQn-x zX1KdRbuCll9q$5KImK&Fy!E%LF-qq^k`wm${U%@zbQ}gn0>L!#*`iTM(q@3MOTPSc zlfHf>M&|LunD(_EK#yF13;zYZ{w=2G4Dt~P}+F^ zL)xFS_3sXCl5CgTCxI_Br(QRQDTMbl7t#JeFMnyM5pFe5mn#4&W6o5_ z2EMgK`)xlgfnp$<%Q%LJw&9UutJoGIS4!b~f{hoVMtFy#~N2nU=x-)UiqQeD#v& zC1os`xEVY^)FH#30F=l$-L^|Q^5s$lpx4LSCxCFLmfC`6OM#Mcf0Pk+Z+=i=#*F~I zRN#GqNg)a&Km-Ik2^<9`qWr>raj#l1hld*zI4l#?E8Ai{4=xqc{3O4c5>IYRt>rtq zn-8XaxHI2g1-L25X>@In!Y9<+UN`P{8J2I63i_J>T@kaXgF5D=#uAc3LKCp*Q;c=n zOKCe7_lE17_gtKLSE7{ifj}L481M7jQv8LE!if{ef(N{g#kFY2NFW z|DvLE!&F{knwZNb_3|Epo`RSDk(h*6;{O5XzzWS(g(gLH>YN;T71F$s_{?(W@8o2R z@j53ANewW)5|Tc)1zAo2yTc!xy*@o0fJJ5nv=FN%8$b8D{X`}>2NV*@qsXK1X%8i= zhoy2^^C5S2y&+$DO*t?Fzt%+$ZLd;$6q2$8n1*3|`o0|3;`=B^wx%==mNdG}Xojq{ ziaQGfaK7SQj&ip$s#c=QX`eXuyy~bna3vbbs#@MK@x$a0$8e8!5X}EwAN-d~3fBvmQ0!UG7y3$hTtHu68^P&x3Qfybt`njs`(b0)>1E2Z! zL=J~OKh)Ta<#-FkQu!vIJcHA+57J7P8v5_N1r)I-P6GR7~X; z&1N$?-@=!)1-u>?4xDVetvGVh^r4=;?BnXEyR$7aN$*zUPx^TGCIH-(P~xd{N2F~$ za?gP;Z9C7{21t1hIFI@&Dkaf-=>lxdknZ)LU;){EpMj0HYtS{Cwb97RBhcw}`Nt^!F z&}!ZC@M=A#o0{cNUO32g#3Vh0!%in!qQYkJThwS&QdE@AO7NF0E|nrKQiq={2@m}LD4 zCBm6}?m21CUyw9>VbG}T>Z@L%hrPNyeeFDZJAt|LHHm1FQ}D+4w)Up`lSkOz%*eyJ z%+CdS);da!ZMlNz+X4F1Vf6yTbJzJx5^fFWB($c~+g-ut zRww0hyG%|K+iWmR1YXs39-}-CZT~=h=rW6#a2P7teBUu*SiUm^=+W1HjT6~9u z8DM7%n z0^sK|4#08{xZJ(mU|IT|YPY%&;2blf0$ConNq~&8EtmPt^YO>r;@B9RKaPB=9XeZX zRVcr!tyZ4W@}Bolmmn+bX`_~5-9q*lBWiR_>|{+qmGpzx*R?huJa`W|&Fd5x@@nR$ zn)Bv+Yj=hwsAM%Pio#N~CgAG+LLYEl0E90i_<PWyw#AX_fu1W&< zAV$M3l&o)|x3D-?6Iu6|%R=(#u@KG%1vzkTM7imS3ViusI)q8hBlpf-0#0HJNe%$S zrHneYXS<78{+PO+=PS+gDbLeZ7Kv7z)0R>U&Ko$Ma{p;XqU2N6sxY?&)oxpe#w19e z;Pc-V`qWJoq8vBy=0-G`17D^A#~=O0MaKeOnHupdajyfaLXn z&1z->Kwdw58C(tL9#AKSi`xJEPF=U+>4UXtimC@7Um5GRL-rUT9wnJ)xKd7~ZPY2q z6B+=>3_AMV&L^OuboRt67LSV+qP{GERT2#ZW*OcJ@D+gQR&qM;H4hkazACFLGKlns zyJn>Yp`P-yCbb7a5Qt}(!3REwEZUzzL>EjbYi_wz;0HC6xd}06Uh z&ZCNBQTClN)+*_z{#mYaV!8)jw9`3{jQFnCHcuG4>1ZB6sC7<1$Rr|C=C6l6ai)275waH%MKq5M^A5e<*VdZO6i$&(=7 zoPK~9WJuKK3~>7Dme$Y-7}>6A)HA-=3S{HzRk1bF8=3v>MIiM6_lTz?H4Hx_JFw2Vq^zAwnleT|$7TS2>}-D#|L7P^q%g8J5$qAo5j< zu%{1RQyemsyw1}YDkvuXV{o7>DT;i9p%{(-QUiJ#cJ|#>`!cgYGn)Iw(e%%zAW@l= ziXO&V>mZ$r0XBLIhdJw}qEfw824|C8&sfr?k{tYKO<1b@ITXbTyYP0lzkA00=C5~u z&$Lp*nQ!Nl;$Ht@IS`S%{JfDwE{`r>YzdR#7zyn-tezD&vE&AsVKv>9>C71#o>Dc& zgpAu)WUn%6Lbt{X@b7=YXV8ISHC`wwz}^DK`G=7HWNDLcqcUk1qA@84n4Au)a~tW; zQ$ZK{RSRcrvT>TToPYXOrCb}-M?5aY5Oi}i-IOsBn#*OFYdt)$ zT0>?C$5i(7MA;95C`bx;H;?Uo|BvmK^5N~98>*Zy4J|sp*TB*zF?y z23U8iCYSO_ni@M~YAa)E(u)AXTB?hgi;CWa+kk1!(TY6vaS|fq8c6Nku+hRmUUa3p zFOl-k1MovARU!j_gjxxo(n3tCxlMy z1m)GLn$o*%@M?~TKG`)nESuvX0b*F6E^;8JPuHTd`t;9;mfX!}!O|sus;8!VCs09f zTH><8Mx>i~jIaF#-LJm|kYj8@GBtGlW_&QbUdvyuiW~igPMcJx{jN?b9}kF7Wi_wA z=I>Xy$mJg(t6oQcHS)UIE6Y~Wt=KI3CVRs=cbgpRRZ3>eR3#8lUVgl#8q8Fg;(0xL zAMtw10xVp*uqt*`B40F% zuKBx?fHgaU2Hu~ie60S?_KjW1N)`t+RGI4ogp@1u-i1N4r4h=LW&*D!cBW~WP#~#d zQ8$F;!@^YTu%QU#wXQ8zd*V-qq?j}Skl$jcI$-pp^YaOS#;3o1KrfI@;u*_PqUJqY z@VoP^22BytH1)Os)Sk$Yh};GO&x)xEbCU`}P){L%8nlY_eROP^L6sYDm~ln&UWWZ$ zsMv7OPRs$st5x#=1RQ(FvrcY4S6{QYDsYSOe5o`Z2zzIp=RGP<=iIVhGWEp*41>v3 zu%87$S+io_*#gp&WkBjJlMD9UO%M390MsS{e`=zOyGSs!%6`BP64jQa;IY4DeIL<-_yGn4ttaN{~zk!`mL(;`yW;mB}7pPMNt&# zQc?~GNJw`##-TfF!QxX-v9H8YK@cDg9lW_c-G?pFiMve&NzzwOwpExr^7r?4Dj2h&?=&F(+WAAreQ(_!8Hlxc*Si%o_FGgnN# z$?esCR%J^s_f{6zi`>5QueTN6lE#eMiH0C;LJ4p82CA^9fZEE!cRzn30cATgRuC!! z6XmiLq6ATVn|Wga{9g`tX=@fWOwgo@KVg{rND-{dHP{xypgPb%6LC1+Gq+!Ec z_WBO!f;aJJelH4#1F4e6QS2xpA9*4dI$euJt*9urw+3-KN7J$UWyyQ7vu3G*F2Ra)7 zMH+{wzIprslM&LoV*$dCG<7ZHKf#S)&c`5rl%AkMQL(obgjl4^U6n$oxj)&Dx;jV? zz_1W#M~H@LmKjHJXe?wS>0$YNL6ekcb|8CB-jSfl=U3>XvLq&6MKvlZ$Vo^Go$cWt zK-te5Mwu48pfgOq|B}iKq2E+H`fN`jbN3Z<+JN&A1LD(?`{%7o_rYppbb{u4I0Nyosz4oDW+(R1 zD0uW@ry2m*r*=snPQQO@Oud!5-(O_ct6m4KnGtAwxQJ)WVNG#nl_gjLP-o}L5xoHF zc&qiu0uAoY3;_8`R$F;0O2@b6S!#ZQ1S}uH4qbaRe|~Q&7EqTr##7r8h>4Q_#80*| zbIc(VJ7qwLb<;BsJY`Ld*7ii6>DQTW`u$$T4>zNDOHWK!nzu;B-c6pG=U!BNgp*IG z-1=u-5E(}*S36yk1QXvblhI&iPcm0;2JrlCt-Pzv+1^;xW-O>Bm^tj9wE=Q$$+J0R zGn@!DZ4jGO>ou!sTV7K0t+?T}#fl|;yIl>{el=2Em;6zwpBeW9WO;u)g)WsYT9t%m z|FnGQ=cMuQKX0T!d^5fa%=L>Gh)>BcBpGVHv%DXklQbZn(hr={l`hTc?&qh;jT68mQB4+b$Ar^8rdToei6_FOFjn2*b>e&oJ4 zc__(+wYiX98lZqAWB2~uw?V^m^muSf5p1dDY!N6s=;Nfni)Vom?{3_l|2eZt81_?~9ZCQH>j;?(RI> zGfC&$#HTTBc79AHH{&=+hFjt0o^@l&3=QI!sdAM2xRxk=QO)V0PLa{bX`Xq)7-j0z z1|RzKVV+O^bGLT!ewNGORnjs@+$9_(^1zH#Ra`!>?qP19ap&yGSmJSQSkM^uzaIhL%!;)ezvISaOlT*k~9m#$Ua!x)~y9 z)p!(r&(b5|5PE(d(SLW)0`a@}>ABzvvP+eU2(%Z>44Gha0xcYoGxQ(Qp5a&Be!d5` zi9}z1eZHUVa&_t?eSHpv3~c8g`HU{e zV6Vqyy^nzx8n}Cse1ePIZT7+_Ec<0=^>Cn9NOH024(Ws6eh6@rv|E-uC%9D3CPyMu zNf%`RaItwwK`IeNKlp#8E#9t*d+d1H8qDExa}8`gFc+>Qp%)%+iOYlsm8`7v&w<`Op22#!o*H zG)t9X^i@rv!x}qstppX71p7F}lf4@#red(;})3w{uXk;_Kn8Iq_W6 zTcN=rL@}gy?9n1f&jszoCRJ>Zm#I^icA8Ag?vx^*=qs{G82mGZyie0LvAXsHWw!vR zI#*A$Zz-qQz-_=9b}CL~Amz@kY$8zd+_C#V)>qf7BwpKZIGom~-e`3DSIzQ#fMYv* zC-*{ClWt$WLr9g^he_Etl+0#F2^x@we&{|gLOeo&p}G>tn&`HTq2XeeC12C$CT3@8 zlTHYMkd5+erkJ>bkBN2#-=|aOnbAx}mC^Bw>18!RqI9$1y$bN|m~lOBvkX53e7q6#91zIjOMu#ONAj}UKELlB5(tP1u@ zek72Bk_Ih68AUK7($!}7`PrLxCcnVYD0bb#5O|_zZQ1EIwwk_++$XC7#5_tr!hLCS_N$k#8_9)l-8e}f!Y(SM&H?vB=5l|zCkA8 zWK7C>&EdBe%^WkcdVm`1eikxz$unKT@(s7hpWkTZT6}goxE3`)tS_xD51dGAoq1CW zUogQs@Ti0AH^7`sg-;n9Lh$^IaHqiiU1qV~} z#-CS+l(1-sPcuLM=LNts|Nm=znn3lT+6HvKZ=u5Rq1Q@DHxUHpspB+}Hwbf%b-W7{=q zYMSH}ndervW0SZ4b0J4OzvIw8wgJvC(n}th&|5Rn ziAx{^h}WE&!kuM`92f@SC@0G!iiO3J|vaz?~>beI$YPkj%xvc?2+l@PpzIiM~oX+_Q5$W=_b zN6;mk3Vq#mWM*G|Q?rp{7Wzzu(?5|>%<|OltN5_QgQ^6Z50*R18M?l%l7!mVzUEh< zwWd271&x^+s27lNoo+50`gHrx#Hz{bsy}{!aCksjRXe0p_k9);NYi>}Qw!*&lB}%m z5;IN31Bm&lBc=yQ)oKp#=5%w!(|%YcP(8x%-wr<>e`OCJ6?OG`+FrnwI|JC!hfeqH zjRg?bNK5v+fIoCTRMR|PnroOHU0v<6vrx*ZS9|0VznjZDDSetamja$R%c%qEM<946 z)r3L}wC@%C!bGGss(>0h517NNya#EqPMHl9ik1v!&p6nh2y~Nqp$q@~_DUp2vE7ld zq9&z6u?XS4olS>+A;e3?i1!uI*t`(NTTVZKg=~JuY0c7=NU0pcQMAXXsslF=viW(T zD|Mj$BeD?^d*-R98!4sBB7+bHv-G?H5faq&^xR9gc41xXuxPrJPUvfQa+P%?$UF z)!=Wo@FXg-nb6usw6SLd>7p>OZ1CEQ>R(^`rVn5VxdxpWA~$>16kNOpSO$-`Np9Jf zmNtE{IMi&56CV!8<7GWW*)YWM7=DvkR!)gE(r}k{7PJj{@$L&z&}xX8u7LO0ArhXUn>8>zS1=~^=4e(-Zv7M$`hhfzgFxR7TYh?S8dV?r`1=dX zJ~F=ad72({=MQ>(sR%ja#$GaR>*Oev=Yh{@GRdtzon@v?8IaXTgRg84#id+BEmtV? zz;F*bur{=z5%I;-WbE}hIp}holzsq$Q~7rOHq-yL*T+Lsa-jrE2rsu1*ekk^)xlu; zmgT4FR zTgHuAomN{V;8eW%E?I8~>ckT1v$#RCQtfxG;BVGeLD0MplbI)pS=!xdkDd%KQH`c% zz`zss&lGsq>HQV^f80MWgEN|7{Y{|zR+q|>ks+?S}~*rL)| z*5t@R^epxn->g78;h0d-_gavksY~;qJ2)fQHaD^cX!NfTvZgY#QDoVt?O-QFX2(cI zo&I3f*y-3e$muwQmB1loUnb+Y>-{pF_dy}Uf37&uW2E;Gkho?@CX;&K1f0H7)omTC(;9plOKvuA4QiSzL?0j#bE;!n5%OeH(}OV% zBN!|NGL6*cE7-~|*K9`X+^5*OXs)a&*atnomWhc*#%J-PYy=P2n8@@LrW8TAwwrj) z-MsS0mt2(|=!)l&&^*{r`(oh(metsA$XT@TKuMOY4Cv8*BhWpISMV=+JkraAsEr|` z#=5s~JjF|2U(v!RL`4}oRpsYb;_wEfqHlLpZ_hFeLbR7D5chU=ai{OB$%W-7eV^czv}0EJhSVBQ46WWM1tlGE(HvQsT$`zo)_PI73iv zml1;B*zf__f z;>`^Awcc_>&<4}wgM+O$cSOu=KS;n6{Nbr@-i|#44GuujpJa`hh`;{=r9T?{COP3L zS;@>X)5bLvB*IC3oFx(q^kccb*w?g{ao>$5+R{t2Qtv$$vT;Ty6}%;3)-r?(iIa}w z8)i~9%YFRzW|w{6r`=4G2T7*q|7hO9XyfhSMMsXZML?W)3HQljr{}moV-xB9@TA2y zanbJrv|oz08e`EA^E2Fus;8(bwk0B~N+(01MtT>z>5HR~9yijAi;J zZ1tsL?Xw_g;~uFf~oTq`w=fvOi)G-p2M&_xGhOcmFtQ{fg@SRw|6} zmyvbVFQT2SQ%E8Ud!;#)rp4DLJ(-gu$HQ6%=ZjjA{JB>*s?>8~%e?>gb6kyY;+$x| za7!eIjjOAkm@DjZxDb!oq}8zu*x>X8ldan*)t1uD*1k`j+;o!JwD?;_hB1O1c5}UU zLKh?pEQtBTGFaOm-Dopx&%80%+=FVkR;gtcvv9~9{Qg5A$Eh^fH-^zVMy@R^_;*RMV|@oDW=Tf_o{wSD7&##|<42~Y)bC&W zutO!pqYW~I4#v)twkKTS+X~#gvgUN7uGD^xKCN+I8x!mvy+$G*g_1WTZ&im<<7I#d z@~a@*JAeE_WSGYX#@`nMttQr~76#3}CrNkX*^dXW7eS-rZQ(A7dTv(I{{?uX{-XP? z)p^G1dpn~C8@h%2-R&g+1O?9yP8INdWgU6*GM&}U^EU7Azw;2Ox~UkG0*zG{@v#~} z!4I&NBwsh=jMUcbN58{+@V`{WXxyzO{JT2un>oEnxI_M?QdrIh;*zf>ja zHNo%e<$oWirMI;*`9t01%yDSj(2)I^T|}bHun=MM{XbX!+Qt)9)B^xj8?YI)7C~}I z1Go?_EehQYe?uucfTX2@E~5x!D{9DK0Lb7Wth6V=DlaFe2iZ)oK1$2{2 zlziQRd}FDsv({(&0DXBdwuBD~CyMQ48l_^J(r0LIg8R8lDF`LlIO3 z;6l3QAx@|ea9|#^Bl@QT)jK)tkZ;1=qQ9>W+8Qi4rGcU5WZdUBi0+-y2|Ql~Eb)XN zfg&*iZY_KP04`4389Bs5GVIwF!Bz)|pnjo2n+)$8gE-e-2tTd(gzj&4f$EL`-z5;j zIWW_aRETGac?>Q=rwUla!^{9*(hb6mDeRA#OR5d@QpI~mRqT1}qy`LG3M5sqDtqB~ zJ|H(}JkhvC3V``4e*L%K=J1sXbx~f*R7P+oD)9xk?D{kqDlF?@BfoGWVYmGAmdnp?-oRaA>4Et{v(bu{ zb`!d`6jN*XYvIPiU=(Yg(IMX??Nw7*#Vq~*^PrKjGz$8v)1~`IPhQqTaH@AX zFV53MKF`ZQBvh%*Ng%GTxMl_Wby8Gb?aL|BG-$vrn8#pluKAulwcQ%WKZ;%FTlM`w zrM>+Fpl8FbCjA8|uP$-5+KwRcxk%9EH~kQ+lxr`l|2Mgm<&m{=K@(P!uHXV9F`}@D zpx?!C$nK}tf+rP&>d|pt;3y4301y|IKrAIP`?S>T1h~a4TB(kiDMhfim~0l2>-x+eraH9YOb6xpM;``mczXd=_l~C6M{~EXo)RCfZJzl*Jve zE*tUhttkR=NX}!AG}522)7f9S zSmBXnm;unLQffYI89|+?l<^(!v55xu$C{tt@uf-niGHpuK+;Mf*aN!VnO0Umlj@#& zqK^#E_^*k}6q28`IzU%vE(39;3I#PTr4VV=BTauv!Y(0E`Yvigbpe!krQkpjAyu6* z0zJ+4inaBBuFUZ$nxqPP8FSH-S02YJ=V)*YWnjrmhB8`>Zhn0j@YY=E>$PyXmK&qp z{80#u5&rxW!BxJY;5Xgyl9{U52z>*1U~YS3wx`LTd+LH}9$i)lWeiBp=c?%#B;Th$ zMq~#ZreE7zHX;-|_!@l`x?T%kP4{EYOYd?Yy-2)%iaQUFE}mb?X>uB74kIUybTxmze`lCF<`yWbf#3Z#8iF2x<#%x z$o-+7!b!A0;+0@NhTs*&hr^>c)q0NDDkx8qv(&{~(V*V2|8mhF`PF0$yHTLu)487T z4lb6*p@5Lzo?O6=%;OJ52q$MruP@iRjJxX{rvGTrAX2C2E{Mv;RDqvV6!bVFkYTT2 zR088b9TI$d`uxsg^85)grIBHb6vT&plPJRkyeyx7;8axnB1O@}}s{ZGYsl9ILgA{W3etguR`j zx&tuSyca&Rm}Ai0uMvNE&O8ON6D`~nd*zEVd`ho+D!}_OlUOMOK>IWz3odij0f46$ zMuT%pHSlgswV~m5F?84y`E=G_DMrvhFv}<@?+LQV|o1X$K8~;S|Jk#}qK?>i)%UE+Rp;R6Fz;c!>##Qlwme zt4zivw*VY}C{fpak0Yarj6dyDOz4w{ZXf!Sq%VYaB?ZROuD5OzWcc0luX>}v04v;_ zo&<6t=!J8oXml@Fz$9|odwI*wxHsE~ENvIbEe5Ra!S=@s0b@JWE`KnN$@o9LhWFV9 zgoM`Dc+*1Z2w|hmwj}N`(*V~^#9B~yaCCeepX>J1B!j?hqu8rn;h5;)XDO_=jQN}! zT{9*QUY}KNu<%&1-OfBSd5~`Q_Wg;ANlk7zv3TutevNraHtXXSNz*xE8$c_L=>vY1 zhxf|-$=}BH|BdAZcry{kb>t4VY5Yj|qFzuSwu|csG69~F5UIKsaAl!7Gqn5OB{L%J z;2@(aqbk5_7bE1_o^@Q+t&!mAoQaZpai-%~2e(BzCe=K2q*<>2iELS@Pi-v8~t3bUdf~4Bw6DY{YA3=x?UQC3E35$MN&;CZ(>4?Ohi=pfC*o zErb2=5-Cct+>~C#b_Ji3%hRc{JV{p4SLN)$Wh`RBd;rajDeS2DVJ7KAFzREfO_cm% z_pFCc-yS>CC99ygoDE6j=5LJ8ILv>&m7zJnYqkU%q(fUgY9CBL{PRf$9ne{{xT+F1 zpUgotALp(?80WY?hs8)`=ga16l=M!dH?^m*iNsYEos#fEM5EkZC@1p{5T*3C^Kyuk z>|Gc!y!xk+OhvZVV3+ooA&9ArE)7DE-v_Q~Ov^OF1>k?8jYs0X z@=N9He{|jiw8USh5Be+4FVkkufBQ)Di0TF-?o$qm9X<~nAJePH8kv1*$G^~eUB~ms zx}NT?6y9;s>L_s?wiW3KE+Rr=yCdo!5|()Spc`LBc9=zv6C@@ zX|m`(+gloloFh+hwcB{IF(mcXiLnnKydGxDS!$(})YpGAcJHTg)_QAN zzVoG!iR{(OL-C4>>@~_za;z_Z2Wfur*6OsM9YHJsV6+CA?4rsyu|+PPo!k=Ft7^xe zNvGYb+izxTd$(t80=yzG_>iZaQLg zN5GS=Kk56i<;G^J9|zu^P6kh=%WY3I0U<+=cq1bO)4Dpr7!`9&ASiMUN5J)ZIR}AE zO`*DxS~QPg+;)|$?zEz9WHevVte~I&6spfHl6OyQkU=Gzmds_a&>-LID?c|fS-MsQy`Yph|rE+QaCR5Iux*9E}gIW1f z2rIfV++prkLb>(#Pq>QHTSD%*#vQdGYy%7Z{&v3)63~DwVWITmVJWXbcxQvAMHLlz zU{+A9;x!J?QuR2~h`dud?;mH;5zI#RP62;9J8pHjhPL#fv5tJT^WDdbFc$|QzU}6U zo+ClT=#u9O)KpacvMo?~BhGeSmfsH0Z1Td!Z?0JNZY9?#x6R1t6f4QQAJpoq_6LG( zN!tEK03|8nmnz?p)*)|=H{;NbH0J~k@2t@npKLw)kr@MJqTkyJ^3EspT)}o+f?B%Y+te5;DdoOmuZ?HE zm|Y`{%u`5{A&-FG?GCx@AjS=}k;#l2MfH8=*3fK#7|JSe{?jnajYHP4w`mE&9R81xcI60L!Y$hkgtcg2B8t%Hp zJ`RhFG`>QdX|}GMB0+pUy|GSlTJE1-h?KV0CmsKI65=qC6T|+1hjFyhz?qu_K7%DS}vOJh^W z)xzVVAG=ih?H5b!DM@U|@u)w=UOs(OHgc#yF77JLp~3F8;uy$dR^Y6f#^wk%G|=Dn zPukn%FyQS+68t+M)Td+Ji{5!-{tYxR;&8;oB-r2r!^6Y9P$BUsX3zBa@?kB;zZc>m zDsu$Ok;ZbW`Jy8PeE@zatG?vQ}6Wzl-}*Dd+=uHxMGgx#Y8I|0B8Mg-+~ z`t<47zJVmsGZ%rnJ9UxulT^Q?v^3&om%QRMbFD(!jR{Qkq9lxJ6!-7n$3BPClsCs{ zT6c|5`RgLi^q=^Zz}fTd9WszFTGF-+cRAF4VW5Am^_fD7L<8dWw-yEjTx`U3Q~M&E zAR#O&$`hr%Tp+VkXf2bHmez#O$J;)>zJYMR#Pc64wwZL5fTWvm+6wqIs-4Xx(ghY? z-l3wt+K{|N&tk~w5R!@GGsDsX2?Q5+dS$UgY~EB>r>ouZh>)ztvOSmFy1`p`4`X(; zXH(@U8DKYT7bqwwXf8P6 z#_pjEp%#u{Beq`qX0r40BAc-2ZH!?S$kOhDu$u+o%UgL6-XI42gp>|FBuj8vaMpE< zU*)#`RMwC^$1XISD&A9^2f@#6`r2t%j9=HQvyTQ-^I8nO%Qs3}fVAZsQ0N1Z7DPAONlIPiJnP- zn!lhHvly+en$|D1BcR6w3zFwdsu#t9{!|bgUX+MSyx9)U#ORwS?~GO*X@t;=M?l(B z*Cxx(*g!P-u-#gy|A)@+0drPY@@_J+UC+EZgxz}d*J`EJ^9=Lu2}*Fyl;~OcOob{4 zCvvEI;Oi~%#F%O#&Y>@~qf{$&{8Uu0Z^##p2g%|sYH@ig>yqMIo!_`OQV#u36qd)? zF`IxwHe+MX&oqWGIOve3DUL_qr5zSeYg6xeNG4hpqma(=?%eMVmj0Em7ky zA3bU-S?+o2g!~@Q>&)@*!;w8atNpTQWKct);eHYzVvkj#?4%)#VN(sFbiPPJ5Mll>GKAwp8UV4n^S3mFRMKsf%{L=L&ag@SD^@~d_PDKnj>-*u)70OJ5iEOZw z%njr8PA1xFKKnV(LWW5zd^BMaHMyMMWzXO1C9^b$i5w9ZV4QF;_9Wvg|V z2#7zCBdh(P%r4di$ewOgTZNA3BTNy>phF<+L_a9jU5eBsDZ0@}~Q1_#&!#B71$ar*rC#biZRQPQzxnqP+1KXNEv7jFC%yC-S) z%2)o}Tx59~6%~hKXfD5T=+>dti_is&jL``C%ZuxU=EJCdmlkO{9D=)fQ)=iecv^Pnc*P1-;o*Y**tL>KmJkkrTTvz9AIa z#{@=FFF0+SJ$v?k=304BP*ByC9WBM9t}%Ibmvsl*Hl4EF@VY8?4vdIW*~YbgUDDCx zD_R1gcNQdD-B532YMkdw$P{vIHx88g9H{7PDh!C(m}F%i4UPnDY_VtRe8|mSZBB2RIzRC#})wx&_p#7|5~ung>ZO z(>N*^zDO~xop)cYS0?%oRJ5h8s$93Zb8?#fj)87@3*K>cPX_X{#H6Itrur;d0eOLg zTT>;&sx{BbVg;1$SMx+eDBk~ime*&n6sH$Mg3=1ud`*@Or+!keO!7xIXrAE;?!481 zY(`P$QGGCX#X@0*`j7eXv~h{&a7@@U^dUVGq6%3>~S)fFu<(wK5QvvPep8{!i^)q5VorUKtA6R4O5 zUIGgUva)|GAu0?9%=KdyTK7OyJE29ejmDC{dvNy1ukO}Mdu%5z2#tAO$Q>)FEtGUl z!h4W1w%cI)%+8~(Rwbbn$CSppGX7f@y4K~4HtRnsC~uXWNWLJ&BPHwIr$DrCo$hpbfsL)lsBXj4(ecdLy}%pS%&NUt!Ey41om$&Kz9 z;r1cH5ytLlnNs+>o3X%RIg>?09u<1l^+QbP$2!(WO^ZrF*;;+YZm&&zN9x^rYRwa0>Q<`i#9t!uyt+u-+~vWm%3CI3CdMN6x{)@sxr_a}wArqXhiYgXDNmlI zN=3xY$1EdCgeV%O#2vwa^#Yo>dtfCe1tGycCD)MU6WgT`?P$tp(G4v*lVJa-htsqs z0d2my6$a50^hm(EVkI6iefU{fr`O5JH&%-(;H0APBV7=BlLGOp8$Z@DF7PggLCX+c zjx-UV>3$SgV)Ben{4FFzZES;0aLQ&ocG!rJ@pU#YjyY?X&tKgp&d+o25?sBsvOqJdOB@v%neZ2llmRpq)aOJ2WHv3>|Cfb->szo>wE=kwsiiq-pP`~SCH|&cl zH(Ses__&u4L9`0?0xM;r*G8|+w_gp6$2%{Pl!YOgxst06_<|X2uIITL_djBuOFWSo z*7!1L%D|a2fg`AX$)F5Nqc@#4%irN}x2q*>#~6^k?4+|rU2jyyfW8^`rqS3JJVAA{ zb#W^cQ`;)WVPF$+9`GB<_T`)-3BA+$g-jaVqq#Iako%%LCIz}^YC<~@aqwid6xg{& zUTvhu#l__tG#LeHSvqX+bqjPZ)2OQpn+m0x(<&E6g|q_BHecb=zR>1x^N)qQUOB$! zHM*?iN6%PQ2SJL=gAJb)hnjxMJeO()QHX=rqDGNPG&%57HHCAPinaAlCck8Ov-Kk= zVx>Rer1g9kX65!p{`gcPo5`roN_iA|>46{LivX|M6qi;%uPbq@iI!o^yO+62bF9Ac z5GUR-K6q^ZbpCT}=#&)d@p61jHxE6%#P{*>1p!I(y~WX*(GgV}QT3ydLmR7o_R@m6 z(|HN!?>MwPOQEcBA#U3swa%&DK2_wDI)yzzKI!zU2z%lawhII*pOc8&YbsDqTbh&< z{hAfIjR}<@exw$csedD6Drkko#~4mGz!k*!M?~xncb#~IbZR)PN6F7BRl98YRnJQH zZzi6Z{VFf2hTtsnHjAI}&Ds=GYboP1s%G5OJR4~dHK*n_I&UVp*(H#Sn1zcOf{dFN-%2&!#%1_l~WD1KuYph)ic{DKos-=Ll{j92mdsSc2 zFk=^SONtCXjh`5$rW_QwbFI#`f* z9G|L|NHs}I8`)|ZsZ*^Onr(;)ijuQ5t?6!ays!G*M(67iArb|%e@;(0xaJ1tZw~-% zBG$pab*ZDq2=ss7_r)R-YfO!>|7T`k3C&4a2*xT+nKjou$kuXD_a;Zz83P= z>qV%9b%Wtd=3vX`zlntr?5P{%M6Q^w#aY!RH{V!>v%$NSlkXyYr_J&Rcx1$C(%jxO znbb*`sJdkk_VHHLFB5J}mY!c}@tuF8II@v+`mg@nOBJm1V6M%*{HtNFzqULq+mM)t zPseY@N4=f+_i$f^dy|8O>sra5G&;hi|AaHugLN-p{;yNZDY%e$Rowjlf+d6kwa5YP zyC1~<&m+>yAoI3S+GpWck@;0Tkt=e8)8Z#d@1g#{YoQ#b)n+KBMQ1S{$WpMeDYCM% zdi7E=8s>9EP`%#Z zeI|MT;Em2p8^a@R`{&KslN;tu_6P)U$ z3-8~a`(FOT!0VeJ%On|xZXvxe zOtsUM@k>YQM8|fIuT`u9Swgo|HFt28>sJ=QW~>(A94o+$ZJ(Mqq%ed$Uj)s7Q9S9> z=g*V3>mg4+1(fX8(2iUlEP?><*w=`nV2)Ea1P(%rczNO0hdF(M=SJ;wF2}UU)X4Z; z-r>G;oU;Veac7OYd#NILWVG~*3N>u#eja(rIqe~Zyp5t8GNFvUy>Z)9X-ewUU&?G9*( zsedLnX|NNn1GB6?Jh9E^ykS~O(G6_h4e&1pgEubHHGrIl8dL`) z{Y?`4$rus8LdA-7Jy%6Qm_ z|Ngz+)4xSGk!-=Z*gQE3nXliu8|moi#L2kK838M93K4svF`(mj;?QyQ=yZfVB0Nxn5Wo@9Hpn7R*x25>p`i;W@se{r2A>2KHT9PfaA9n( zfKVc09W;3jfIKXim0N-tBV1k$l`irS@4XU^uw&Wawx@1|eNI0_Uzdg-*M}xTUSa!N zfp05%Z#C|~R3%L+;yR=uWfi|TY&CvWH>6+kx+hR0>JGZoDW@#uENb|a7OMra^a9^0 zS0|7`X7^lPB8$(jemJ-Jz@tpGE^F%L zzakx&8lcjhsJ_PGlqFIMtz(tGBgp!uVy^yKoR0^2oNj%K#Re$hvJ(aoEM2^qOc`A? zlYU)S|1 zZFJ`8@~_sinnh`LJXG&mVBZWbO}v{An6E{YRE+;+6UgCI@7?X5M!Cnu5WOcjo>)1= zz84ROXPv1!n1UP~sKvXox5*lc8{y??PFz1zn>qdGUgW7q*p4yPV2=x0KVB|bWzH}i zc%lpvQZc#^i3zfo8r3 zh%}ooFyUqxK^rM+%Pj8Mtkwz=vX?Nm5 zy6uHHWdsF1c(UBb&zWU0^7oMR7q&!;ht*0`gCQ!DWZJIMIwq*%2r7ojs|P>5aVek- z0|i%jKFK5s)K!|&ZFAb9w=jyU0E^r}G;6!ym)TJKY1g6RaneX4fOC?)_bD#B5Jz>X8 zHVKxz3|;6SSLk(DK$J$E@+E?CQ%Q!dHK9Qgp!vn0++;K53~mL8Lna}znwWru+b<7e z)ymn5mL^{jRhq2l3MTnd#ZE#><&f8Dmj;RrxAz4LFq zXvus!-|+3z!Xf+`YRI-V*pb1~+~QR-fGy5ZyO?EA~VXL|JDEnuD3Ow*%W`(<)sd86#0P{}P9B za4@}cV~(sVUMz>$1SBOm3{II*UAy*P$>g>8G2ux2I`twQQ9&cvAu-&dY8Z|_un`yS z!HIB_wVubQebDs#n?h})s2Wf{^KkVyVKA7D0^+X)3|O|F8IIHM@*$z&d<+-LY=C|B zj6_f#`q88CJGQyTAbM%$=e#EvbL@@uABE&7i{9k3PK?Dq8F4Xqs@ZnLY=q24tA|_U zo+u*RVn$cTJ(W(2(tG~I1ealpY3eq#oB_VV-n2z08i*U*L`-C>OJmN=ngEki%T?F4 zH<~sC^hVchnNVa+cpBW7Fg{tZQp4$E&KX`r4IcjvBdyDja!dF@U(J5coC-Ms(h{7% z)>D=esnr~LX3p*;{U3{-5J&H*$)|u5j$m?(erVhNS*zS_v&5J0uIw=dNDLq(zrhOp zB-=BYQhvzrF_&%Wx=R(qtw6r1fMWgs-PP#e+P+=+3s#tlDG z7JK}O=r%j&sL<3TpX?*SC*pF;rXnwE+s%KKhj2lsQYRDYAQjXq*E4VLCNR&KT;WZ< z8;|hpl|%OSIhEiqGFQZRT~9lv0J(h#JLiY_4w*G1uCN;xBNA7Gkg7_PK4!KQS%zpR zoOeB#Z*sIXz3MqL0maI+WC%Pw)PmfQa1?R#zSowANq4up?S)uAlr@kUt;T=W9YP$u zqcLv1AQQ65i5!^{Or6Gzowg`^X^V>Scj@d9i51y&oXtp^00Vu~o2CZtf_gUbcdf>q z0udZJt^F2z+v}4c4+>+*efL3qfm5NOke?tF^OBxC`n&qtH-pH_cc3A$2~v-PWCX{) z3qoL_t}4O()JWJj6UXNH_b-*@bV{-3Nx^N(VBJVyTu@E5&u895ke9>Ku~?B=4*% zl;GxOy7dl%o+Def*=y(EqI!J`QGLT0++&hzU??y4=t8#Nm3QJY`vpRZsND9Gl^05pP_0wa?^O?u!SGz+u-u&2XXOsW-6GCdE#*M$I}29X@cVk9;=NqJ1=yw8f2MZjh5 z>mZc$?9Xgyg1{?A0y6g3pz7;9#nxTNm70>W^)<@)-#ZKRlClD#BCjso610ts^!Ejg zkmB;S=y6{=_HJOEq`cO_gBr&c(9K;=*BK?IvN*Y6RAAlG%n!P27R})-_S<0mGBAP; z29A1F;ANYsDAoU4gu7gGL_B)WLL%UqtVf&Uw+fo*vw&P!-U2LOGcnbZwq+wvMuY7O5G4kqfaaL5Cx>__vu|B~Yr9v&4(8XAZ1G)JCDgzO-CGTUuH zA(`RqPwy}lgDioKj%Xd@#$A!2M^&6*4u9?PAxT zo&4CS8E>>P-=E0B1Ot61yqjg9s84}LMF9lfblT-b0VC=EJDjHx>yBnv(4_$yF%^|X z_ouWfjKWilcWBwBmP%R5%pVl(Eqb#XFX5<2dW-9 zYp7NAls#SLdu85K042uW;}8LyS$~mul-}UQEC0~5dzs=vk+)X9YHG$r{VsQ+(1R$z zt#`#8xhy!SPH|nRkBEp$Com`o4Jw5K*of6ZsRdZ~5z7EGhcCQPHSRU0gC)|CC&&z2>N}4SLtkVhl~X!SEj4;gmw99V zcn|;Sw5GLu_rO4^f`g!Pv_s}gcZ;a9B5(6dYWcJMm~0(CFD@FD(p+ncVNGx27>rM+AO@2JsSLSoed=YF3x)f%K z$X?$!%9{RB8@qq?q{zymki&j#tP?+?o(rj(@YgnM_7{#>Ck&oQD^AUa#+>YIwlYzs zXQoOLYAKLukQ}h=UT_HvFsS!;rIC(F4JfZRb1YrR{%uqw^nq)~`jKn>zh-Ld`pvh_ zRQcAg^9J39%$&4MZNsa~bXVwj1SWa&?Q)0YRB+GO$m*N-tp37b;jIy%fY%ByxuT)? z@E1=+)sC5DiGBVlW>a}em35VrZ{(Z({48;z))qKSCGRI3j~KF4`v<+j(EzaTuJc@( z<>9)JJUiov=62o`F$t(2Ff$P4!a&>|MbE)^xq{7hx(X)z-`>;Xg`k1btKQK%Jm$Mr z1b5AP^0_YyvH3<%hg{^P#=;vFv4Lsgjm4K`o0@gTewPvb8$YgnhHq;@Y%%S=Z8wek zV?(%VtxMg$PWgP|Jo+~=Dj|-(=Uct%pF_E>u*@O3*S4X){x#5xlAw)E0l-Ef|AEL0 z-|vzl=5D44c8DcU!Avb>tW0;GSz46Wb#wK8URar#ZlsBD_r)L@`s+h=qpsQ46m^{A z&;5foc-Ec=pmIEx8KNwjznnl8fxvN; z&J~*c&q6Id(U#I>J5F2I@>EHPtvG)YVIXMy+W!1ij!w3Hu7EZZ3#H%QAHoY*!ZMXZ zqI>uy{!dY=>Sfj`GfnDz;Hd$i>qVh2`tW%iId9na?q$qVt+dHKs~f{}#O7bE1Z65# zL`6vO-~0^xroeRzRYz2<5#k%=YNyw|@=CO7L+JNZRnHri8*Rm&w#s3TnZR@w2zG-0 zHSoCx)C21&oQ81)Z4|3f=Z&(-P|X`qmFwhXVtFHy{P7MyzV;lg@^xqSVB9Na5d2g`YSOwT|mgo!%~BkxJ zl$4M?w`Y$o=*}Z~cu%g_NJ%5MqST+4&n|1IBPDUj%}#{p+ZLXxfXTEcy9w9tZ8MG< zEO9<_Gr&JuNC;S=EY+K20f6$N8qi}}SWBw|&n5o)x;S>P%sT@^W2xp!;+%=PfYS)t zaHpqiF|Lbr9pGeU;^!S5V$|eUP+&uUaJPp8!++~u2-~rsO|y0u5)_;OF!dZcHUxOe zHb7ylEJ{2~j-a@J&8qDf$DDl1?f#Qpho1WlD+PlV&a#aj?l}+zIxFZeq={U}U(3Yb@Ew+mR0nC1ZE+U0@Obp#Nlz-ZpOc}vvUHP%Y?@- zX^4tl0XlAvMlgsA8dK!s;mEgl%?JXqz<#jKqWRCX4;n?Oj_0274of~glE8k?AHZ%a zkwQp*VUN(jgoB`z6xz<~E7xZ1T(!Y1Kj}HSBg4JradAgzy_oMKjxK1`5Str-f=`Q3 zyn*4|2(?uLu%MGNWj=N41%?1d({!9`NH1jV$z#OZ^6aOi?Tb`YmMp%XAP=bLotXvb&=&y(BmH~|0t85mA`Y%%VdepspavBLKc~HH=O>uAV zQE`G$VOWWCyqQ+Ov@SF8K{@b3*)>f|&*YT7hW%Kj-4#%OI$jRhr;|Ik ziqaC_9$azAVf*(8?V+Gu*rXU^XAdh79{PSTO_`w-Cg3zQerNExF$7E>pN!0$hP>BbrtSqfm$WTL&~pRp z5FkX}fY4+JNC`sr2y?j!;D-(8%>$0mzX9Ulo$P=&cH_}oa0wcM#h6jLAd{?^p!rZq zJHNU4_p(5ySb*bFVnZ|C7G@W@baLYHggZKCuCqkgH zZ<|m^+}D0~Z-wC2kMj55khBkF2=7TTGFL@lX09OqX8n{@RHBe)>w6J3g|dVqJvkeL z<5Y_n2xAs*&dEc>q+O@(M5EOpj0^V1n?SiAEUmC{N>sK9Klf+;j&t3ERGw%iOR<>U z(GfZe25-EvCmWnUULVvl*!U9i)Z@thiF5oW8XTq_*AR~s=aa`(W)xR}GAb1XoccJ$ zWqQ>tvwsL)|1Q?SvTd;dpLvIC>aiPsTf21j>-F%c94)aF)2!^SE~S3Z!b}r1B5czB zJi~d)+S=NICd|?t01>_T`RlG9@;pQ^x*>7sqY`ri;pmFFVh&z4y2}UL$w|PHD)de? z%b&u!g?9ZK=GPXM+0Z>O%o!A+eH{X%+9y5gNMGsl({M=%Nh71A9f%@EVSm8z6n^_a zscv+)e(vCpPJE-o;Iqc6_#>nZ)E+qb$mhwE%d!oS#g9mBdwXM)YXe9AwU!_5&O{=` z8joEYj+8oKTj+xd}@m#H50okt;+ytcysRLP5GF~>EN<^b(!X;$9wD^HMyvqe}rKGf5?|qD} z1zYp|e0}^IhT$sMvrGte!NbDF_Jg0`EqnUbE`R1@4LE)62{FO;P z?;3k*X+8iJQc&keD?=SDwbWP|b&xUrd%GMw?9z)v)_zu&U8&k2iSYvh-^-Bjh!mi1 zPX;$-nSD6y34*GQ4`_g@>WV8V1r31*B+^+$xKju+Hkksfx(+}U^E1#gpvl?#VDXs8BiYqRgWI(i*I=pi1is?!Km81`K(;gr% zNH@|S+M-+24`wUy;2L;{s4Sts8b~@k$IgFT2?BhFdBk&{j?pe)m=4X_BKO8Q0Syp3 zN9dEF{tJWu{4^>A+siSxYsLXO#YM${8OfK~FES%EE`G#R<#i#TL?>U4263R8)20L# zF>DGtftM_0LDRCBP;-$iQ3ce;F-N=gx?mciS<=jm;_~W@!?zBauQQxKCzRj++;}FE z7j2&7cW5))0?#r!0q1oaNj=w{O-nYiRvs$EFxMz3=*2?H2THW#Hz4nmOjFHv1M%3( zQhR$lwNTJP;LDdc@EV_dM_hi6>yES#O>r%5-%gtj>?%Oi){;-ZqzUcL$lAHZfe&B< z*dMqQ8U81X`SHtmU!a()a6z|u_bk}m1gPyg8j+bG#K{FF@{m%7Zq?qGBO$-vXUH}k z!oa##Y8k1Ws#Th&ewNEZD@$?VIVsPrfuw|l$axQ{pEAN9zm8BeEJ*tq!b`qE@8=Q& z^ofIDUq6|A*5R+9apA&+=JLI^Q;Ofb1(zQnHygNA9<~c7XmenM5;pK%Ydo3et*D@z?`9jjg|VfY67f5>LGkg6MA<@hlPb9Bz5dd(Lr-!obk50dlmK5TN4U0B`H@wDhb1M0fkl z2ZjLnVJk7|R%#*XG;|t1)Y?o0{Kqc(DF!$YQCNL`FPc0E{^qhp7Q^%;To&w}wCQ0M zk~eN#Vv*Kz1B0Bf(Jr_k60bbKQDOVJI{Wul!QjX6am>h{#Yb)c#K>O_rr-&RWzouB zJT3#!j<^r%erM;y-&KdOMAi5dR=30*DvIy)%D^mI3-AY_k`?2Z|JyP$>%iil&_+loF!3uvwyqFm#2 zz!5nS;<7@z3e%GpFSt6tWz7Fv82&gjk+6wud*0}Dw9A34F}D{)Pjnzv4}m7rfNhyy zP>_L+$^>!{pdRX>U;OD6|JeE`;NYJ)c1J#W*!*Kfe+E=a-Orplf)Vf*>T7HXWWf-< z1l0O@cKT(}Of`Sz^l)l{L?&|@+4;bD3XAOodE^&DNHBq*Rs?L028hiU;Y`_m={pZT znfXX>-lFL*AM^tg*0^!w@el}$jkZARu)!n7`~LlVaj*+FsP;gk77GepNcoSu`D4ZL z?g4bQ{PnU4Txk$|TKF$7g1#yklw9OYslSOj4}OW894Q%tkV&*`6Ea)?J=b-VSn6*< zPB{T0Gr{#(;$QCn?|*4j3w`|Y@~e2+51k0S&<{-+aS1{2=28sx^P>RXgI{`9nFOLJ zLb$h(c)v9N8TI>-Y%c-nv9bIqHal_?s${6$M3?_y1j8%*&axT33{@sSd)D;242MZu zb=v1Y5@%$ao`+xe{G=r%B^3miKzQinBZNx|0+()nvb#Ve%7eiu#@1}N8n?U!ihX~^ z34d%LEKk~A7{CTW2v}|saL2Ti3u3maU%7e}VYN*rc78=16vSm^xhNBXj&%`9$&rc} zGQNo+-#-&1ctMY#IY3kkz}7{;01%nj6JKA{@oO!SJc@g3+3pQs7|jCcS1{ar643pp zVq}b9gpxN+CG)lpD5na67FAo*c;S!y1NsHY@X?`h`_m=KVzH!g*_M;6oh+yMZ^^xNsY?19UMwtb!PPnxG%B4}g<)`Cnbfz$Elfdu96^=qYCz*>}go&uvF znjpKQqNi`{=`eK;CzsF-XOTFp0M}B;{9thtU|gaxu^W)$yj<@wp?5a^8QuJGD$HPy zbBeBC$-Py25}!da>DjboU^p`WATB-*Y5GDJdTrWQ`KP<-N1&60w?t)$y%GLqg+&L7yk=wHPIhC|Hrf8ai2s>n&k0Aw1L1o4+O!- z4baFVbU6flEgqkEKFRnK$oXUAY9T$SlNn$~*Z_2YeVv{pU|QXPsV&l0%ivw|GGYAY z%{FzxzpdC5r7-&hf6zgr%qj$ySORYNL@@P18aWVd;Sl2)pZ=rKj65?bWMrggpxj=D zT^tdPG|K=lHvCxOn5SQNLM;8ud!#5uVz|{u*vF9nQmdf;f5NWzGhg3eBv%0Sj~0+I zSdE&`h=Xzy@=s(OgCP_sdEk@VUuvB^2>qV0favH2mAb#}n`(7fWn(56W>GOQN#GPV zBlCvX=AWMvpoemXwI&q=3}HOhNhW-M`)$~D=<*-6tuhhb>U26ZwGhsgBY4wT4?jBc z=Pnh#giC-s`LsAofRgg9Mo>luqjMPM-!{-b_N5DrhU$>CLHTpn09yD)L4o@YGuVF# zt8ne&O$4;jd_*j7D;UrG^-|wqsP5w5ck1|q;{p%qaT30Tpx*Z`+<#l_;HRZ**f0P4 zy+79IfA8gg`{hSa{U<5>-#PlfJ-&EeB41OYluN;cJi-9LRD{0H1I-9vC`*EKasohL zh`KLk%58O89=yC{4FFIJ2o9!#LeU@cyc@td0zz8Hl$+UaaTHqpl0;B+%sgE6Yeh%+ z)u0Br0xiB7yKyAtftg*5Fy_b)8XW!R5m_)Ch@12F+T4)-0HA|OGAlAXht=$^ z{+Y^+1^^Gmw!z!RPZyn-y<V3w*K1x|}06Het+ z1yLVV)48Q7soPocN&HWblprJ0(B(l7w<7mvPFTGUInoZ0mbPePWg9i20f+F9^Nd^1 zSsO$d*+aTL*tvd!lv^*xzxb5m`t?7*Cxt8|jT3Y%QB#Q?&vnAVpg%*SuyOtpZ;Onz zh`MOZq@;FU)DM5{cP6kfQIB&Emo;@1qXA9z>QQ{F1(Z~wz7JxpcS}XARvsq}*l0|kjuCA_Lv3w1vyxh`5 zsxoo$@hwGwSU-O5)e)*J6Ulv(sq8hVmj*NLoTH1H{4T`*`BkFr<1YCQ?kBCj6hjY> z+<`hU1Oyxhek#yEYB|M@Aob?Sz`T<8K`~H=Q%W3`BKFe&!aaak?eoX-^YW0Wk|Yxt zS@-T;iMjEgy_p?K`13BN~sF(NoPRrk_UDpq+;Ft@@3MJP{v6uTai(9BeO(` z4I1zfNJ72H?H{DR<@R@^exUY`ZW`96ER-%t7dK5y>4hLr`c`W|^s;;`F!~N4*yLfbYI#ac`U0vQu;kk0$GvkqMy9iK z_W&fXP%QKPGcY?cfJF1I>x{rjOm+tYDwZn0x5#w@Z9*0UqMwOe-_r>Xa*UVZQ%A3r z{j#V2$v^7>D^`NN`?vpFx(}JU5Jd;(U#eqa7!e$2#TECr|ARjW)HO7J<{#DTPsy$o z+*!fprnhz0E6q%-R9HSuuc9wdHSKHJL;60yT5c8_G+fBL zU%GJh@UL6iGmcW2B5itz;RAu^6$U&q6#b%LhaCRtIIBqgU@Lhm`&Rnp{qWj``F*tr z{qp?JrQ5R-S7z?2$^t) zzZAJ^l`6GcUZ>!TuZM4!CC$B$SZkUKVMj{`M&*}zUWMCTx@qEC4fo2Q|CS_rRufR* zpq6x{B{r4D&?kP6b8dCKK|RYumv0@? z*5u`6if^I)chUb@X%UL*t_q5h2CWO|j&%3$D-`sZ#k^hb&Ml{^#=!?--qFCeU)m2qPOy8E3L1Hga+R=}eET#G} z*|&e}iIv0NnTsC4Ckz*`U9A6l zY|DJHUtZCE@6PM((uW4c*4YL>80g>QGld+4aw1y$fH~nG)}6QgJyY~Tnhy6q+Gi1a@}DwMdhK1|peyji z_eA0L{>|>2j^k@B>%6~r`)ZI&%wbVL^*JPQdqa#e(G~FzuX!7{Zgxk?k+bx%oK9L? znzNif_G_vZI}KE9`m&AZY+|u(SXy8OpovEh23`Do=8s&~$2_lCKaa%v^}Z6Wfxt?? z>0Eu5LM{Q5ou+H7IED;6*y_U7g_r_)&Uq=Oj^6hTKe7FgFu$pbWG&J&r#xfv&vcag z1Tmp3bgi$4)K6=sNjj)K);4CEws{>YG(sn--Oxn$+lH$F8#?;U1s>IB@Z~hR8D@HP zwai{lH)*yDE5R|%k}f?O2dVto`sE%Ppsx8)_ATrKP^>a8Zi;LrK!|s8bdvAw|a=nD* zPeGoov_SBkfGyVr*CuL>uYQYC&I3KwG|X!4aj+ZTVb?e3Wcyc0Xvofws(P{|cN%Vt zSmfWA`veZX%`Ns#shK1b(=_Da`czeWao3cxsB@#7w%7V>9t!K=E^U3dbMxDG>Hla3 z+~)a&|B4AbPI`7ttWeEuV&@kcv&|;->+z*&$khxy|K!*(-U(Jj;yg~HCQX4xHkeAL=JhP-Roz?#unPgSj4muCnfYGMZbx{r>b3BR(bC7ttfIJr^XCNwnFNP&jY7B z*U8WysCcy6zk1CPc?LN#!iV5!;Z#R7Ql0k@-ou%;smKU4os7*@wGYL4A)A-{DY*2L zOg~3Kc0#5$LztMtMceo2bnRulHf}Pe=hp8^`V3BMnOp7c@=NOg&8_(I~1dhOmVs(@eEz^J_Q7;lrLpi zbo+f5d$Ih#=oZt2?CA|GANIV!H2Q6b{M!bb)p$H&9FR?uc2oa}-ZxYK%EHJBNsm@8 z@_+8jWhID6(aRm+9Ex#Y_*%<1d=){F3wL@$ZoQAU*0EZ@|2}U{TlXF<7J(z0s!SCA ze3t*q6HynFtb4ySS$0Qht>u529ufawU_`JbIgak71`qzvqXI8n2!e1Zc_S&{GV}#!~*HS**Gdss5?2%KgNYT0Gohzi{LSNp9D)6~I z)KMB{h=G2Ef?ZGq#vE3jf@0vuAUBgO$~=tkvx%-fW_CA#x2I9NC~p@=al!RlPJQ*6;+MB_Vk28-qczE>m1t(LA~n>A&TvBPf{YKPm9?%OkB1| z*OVyaULtESS^l>4`x=BJS;+izD*^Zx6zFx8dwon;DEEEZ)FpS8)86!cXyoRxkhcvjJC(SQ9Raml<8~_rVH2(mk`Y zvC(k1a9z-u+qk%OCS!QY@ii~ZClQ=`9i7iNSM1)J|Ggx_!X?<3WUUA~aKZO{hVI@H zx!_hd+0i}s-ol7j*PV!K>}U}S)JXs-)QZ_31as~~f2iG(1!ba0*)0m9y)(f_w?2VH?5y@PlO`+u) znJ4d&eCbc>V>P0{qw`d&F4Xwa;6Q+}tl+U86&e~Gc#tloAe)2!Lt!j6aPCDs-;7L_ z8j|ndm2T1ZOx?2oSTO1OrYlVp7wa0;%O%2p5|SP)sS%c{!@Eq>&1%+pQs+IYo#?91 z$j?m}P9=b?n=1N)rjU@^s=;&ZuBl3T8HlU$k8IMBO8B~xj<$@<5i|@?vuOkxSfrcdd0(^8;nnt6Fa2R3_545!P1gv*tUb(KEnh-DZq?E> z>jw-&KDusJxY4d%OlQ`vb(p_I4p@*1L|oJNYUIB><>SY~6TVzF$5}{6>AQ!jQA%m! zIP^4Cm_wT-om%%-vTU*xCQ8`Sb(|@Cmp;#FwUBMx6I!dfXmR!~a{ZZMOoxa15)b}( zZwFrwr4=i2-^t>sn2vSutQl!nLCQVDOXjC>RJs@KO!2(wHHx~JwnoWRx%Z7V-*@vQ z>8$>lajhO`_u{bGC9+wO)7b1?G|H zY)a90XME!9bKURG_wgCp5woyma1SHTVBN@bud5Bw7#~p`w6RvzuZy~GD2060@AI>K z&A*?Xe?Hn12L@IvbRY5c3TN8>b~@QTRibc`>6Z2e=kin4EoR5DF%_mA{a2cMZ;R8X z-q*eyC|MD=Y;r^_h5x=ZDfQ^-ING1cSB7_v9xYclw($|z&AMc0&d~j_?c}X2l*$f2 z9{_M6u6ym_^XV#@Y%K(^(fl3=FN;wd->a4-n_|NixGdK zmnB7^@ZDf8!Xfd?gyCZSjk#pMY*#!>&op1orIV7 z{+E$f!m-CNRQ5PqOZzA$=pQ^K3aL5OhX|tZSRfz!cptej|2t&WPLy&5E{^xIx;aO9 zUm77ngO&kKg&|dw+b=|3PaNi)o$_q`v-0UKOt@*?H*^N8*pdQR44C1g<#^#&5?5}S z|FUN&?qa}hrr-{h=nfD%l`RoZUBfHN&|@Hn$rgqzmjon;X#{E+2l~h#&omxbt*h(y z*+(T~DOssh=c+0ktoJh6wLMI#txpZFGlj8M_E{Mr-yspwK4oI2o?mGi%Qy7i55)}W zvQ@4cE6F9MrP~{|mgEm2Cn@-0I$N9ylg@aED|plJ92}kwq)^H{Gxuv@<9IR0z9KzZ30;eEyP(L`-w`AOli+(C3YA4N(x4fsx*g3i5Y)Lg$%b0FC~xpMC0XO zUK&hbd{|9RdCszMu)vzb{$rDBN>8P^_DZkNi}ay1QTGyEi+Lih7hJF;FxAcAnqhn` z=P%16d=0}#1@$eQFjHgKroy0m?`y^B;g~TUE9M4X_xI5m8*kOc%7Uw!WsB%mkWiq2 z4TD$cnN@z}j>SV{I1_D_+Yt|{p(inf%Th5~YzB@zro7NOVoOdzp{R_y}@e48I1aq!c)48+$3#(qUilw49LC>&?* zBp*uw|F*jB0vOt70lJBe74Qh11-Bo9zrGw2SesHy0~VATGM`;!9$IE+r3St8;t=G1 zgq{Q$u2S?^_4g>A7Qz1?jE4bRW7)paY4$HJkKjXU5YO;JDkvCDvV51aiInDBA`Y z5+crXF6`>Bz$APvVKtBB$p;)SKS+9hTbnyN1@!4CJtu7$WQh3O^3vRJZtuO`mPKn_ z>$ok~=tyk^Pma(SyJ&*PVg7`%u3}JLY~kayeUQWo9_t2CTELh<9^e4=7a@_LJ|7~c z6MMTmZI2WO7kBui&<@_#%BMfR(c?4-!1AxHy2{_7L;LAik9MVtj+5)1M9mwaIi%zipa=js0gWfz~4wV~ zOSb-aWrsY6AW}DG(l|f$s0JZNBq*zs$7V)gaQI}G>t6`|hJc)q@qXPL-*Y^4`GW61{Q94s zR><|m+)Vm62THiw&EfC$ID5W%b7cc2ng!R3%%M=IU=wYXE3t7{ZM6)I!LjEIsf0;b z7}VD_{ih%1=3;}nO)!c2@8`k zI#V?{G)#%X+Nzyp$qkr9Uk;8e)B+@~M#YHe28JQlSP9l0#-Z-<6gC|!N#h3U0WyZL z6x!B(?(4mvNP(mA0iy zS=RCFT83sFgNcRnFkzvFUNKq5w(gv>*^(gttB4(t7V+-p8jIGIj(Xt^A=8=iy?POQ zYulL)Du-VFMXkSt)iBl#kHwvA#t1ZOU=CQgV*i8>BbEQ$-4a0a#@Qm#P)<+N+ zH7tP5qBSz5Jf_&B{X$Gl0wW#62&^fdciDW=;1F<>Ckmn(LE<$JLWbMv-?AK#_Od&? zMO803&ts<(QEk%iL9l$5$Quzfw!TBgHI57;5)I&}5)|JVa3DQ;mp(_h$Xsg>X7C!H zZ8dv)oalwzxG+28WGh#jG?fs)X5cU^m(%nbn;WGBQV&2qK~?VgGH?f0(Bfr zn8RrL2;O~q>WuM?tuJhcSGIe+fcc6tui9PT8`;@fxo@jKv*M=B^wc&UXOaze3}yyX zs&6s;_B8B7EW1;$-)1*id!Fg_*^%d>9xu%Tg=~fvLb~#d=DnHf;ZVE>tjd5?h z$3@kkyj_$P)*dYyvTD-PSe(}lroZ#B_$g-M5HO;5o%rSp0+A>i({s}Ia(B7?U7;iQ zKMFydtPviAYo?3QI6GX0sM42np#G%R9&GO0A-emZQv2Fz9POBN2Xca#$)iy7q&!Uy z1_Ch~bGeNE{defeBF@T2LGYH}Irw@ZyJ{-Y5-oTu86WrCcI(~k=3-UfHKKETZGu(v z%7e>1%$O4DI0l`yFvZcBXxpnvb*^INgY{8zmwQ^MeIUD+f+gv2D~vJpCZAHcn_)hf zKG|NIC=9r23Umj2zM&HKu*SnYFg;cusX&vTHF_9GAmc}!dwj&mY?X2?Z8`h&4)-QW zlT?up_wB>kcKZgWb}9Q)1hMUKr_k^!Ekz7v+ z<=hAM=l7Wt6_cdC&iixfJbQ8$r3MR;)fvWTpS6!K98)!OSG~CvP!B;}!+pO;5GP)RQ~Y1nk!l zn!@-AfmONWT<{JBN~`-i}+#{EcB?kFE?3e2C4IokS!FZjVQ-*QXUS3J%# zRf#?GE+Y)3y-TjIeRasTS9>aShNO9kE5MCf2#@xG$3XFLbyvD-D^P;jW_m(UamkO~ zmtEi)a8RH&QW5qv8!VAQ6wiClR(Wx4B^;)v6d~+H3JA@iZ zXqNi=5~UE>cnq&b-Ft&AzFy0SA;BCQJMDS&+%(r?Uyey|8wkBcQde7Ji7@D5s$fVq zxp$l|k^Zggn5g<9-$zy6ZK=cHY+AFA=_CYh8up+}DF?aJQSA`+m{v^+X<@>aZ5j4Xpja-(CaGu^A#5~w$F6D8NC(%z4wjFoaOwt_lo;J00N^1sd z2I5F{n6c>9OgG$?POH`hRCT>5snfGgTwwB86<%oyGMyDy`ie3P9qY^uj|HlZ8(ZOdfy~RfOad#-Y#(aHJAH5>J8JX&jUsHR{G>^G7i$h? zhRv;LjU^7d@9)aU66Pan`53x8i(CH%2Dgs8FeUN+P4tY92SZ+0+r{S1mn29|(!PWk z$A4Ol?epE}s58kBz1vUA+4D2Z;@G!~5^a%Kw%i#k%5(-V*I|(SkE$XwiT2|&K~h^J z-Zsqq(*Nd_eq|V^Vvi^Y7o&?|Hu+YT2QeN@iOj7e39emk=lc4sNP$ZfXpzg#-M#^3 z^R^2^JQhns#~vAcbz+XS%V3ViDFGXmPQmG};=8)ZBoTdq0WKtx7vp?se3mLN;KKP2 z#x{xB&rKHRJ>Fx+wLsjZKDHm_@cGGJ!Ci1x*bmEKIj*!xx^ZlWR?O3EzyV}TT8~bU zMb@S%QQLus!EUG-WCDc|+?H|fITC4#vA)RxH^B(~ntCjSowYK-#Oc@tD7f6wiSxzL zV&BDGjsTU*;^AvJ7|7&qWTwjk_=2pZr!{^2z_%D#5=H`Jpvrfan_PQ>_C`BB+Gcz# zd{!SNu_{6hFts5BLu(U6yuhQ28xxw=K}y1hN6D?5FP~LS_BFnE>j4hE+yJM=+2q*( zZ6IKc;yzxQc_4JZTiO|{dyla&bGA%5T*fyVmqX%+XWd%y0|5aRZCjJE85^~ShtRhu zgnKJoN_-*@qdQXjF=8buDsd$$%J5KeYrZ~tl0qloavVW(qY^Swn%ujCYXRqs2_v~= zSk-4AhRAcn;5gU_xqFyRMvPRQ-qtsUSc@chhedp&9)~~Ub)nW^(CYCc^>^Fj9=JPE zn)5$@!gUId)|{kRQDLdO_G`Z5dYV~42Z3OnWc>TGL{xo}@0{c%*Yy~?0aMFIish=W zdD#0cpqyjVF4QVyBcxBk@pZn=OhjW-QEGEgILeqrj3cy3(Y4HZ9#Hq-d|FeRRr++ec>+ z26k43M`swzHs#Ob>za-|xlkbnM<1{YR3XE4*HiWSXvZb876=;&u!RI_Wz@MCoXVnn z_T)Obrpip)iyzSw&(Y=5s;z%NNw}vY zLiWlszJ(I~T1eh27_%6+uO0GS27-oi^Z+UNWp&_q$2e_tNV1?bRrOiHsP;o^?`YJw z!!ltv9-VVaiAm0l#Xqd(bRn8GT#3C>sch80(zgmy&fPM|;B0g6POQx*qIN&s5Q!~7 z2VUITi=*xD~@@TKX4f;b&PFX8m-zw<4 zHa8Y4R%DhEsOF1-RcLyXUGPQTt5qh%h2>nvAmp$@yO5A>O*q9{#usGyRQ4wT^5kih zOGDHDQg$}_72`?Q(^+@>)(Eb<>Sx87@e4?cKj2(AyLsvq5|F@FBdYeAA~L(NQ_W?4 z>l>W1QV8x_G`^;#=lK0I=AuJVmv|}%JYx)LAgZf|@*D1?PB%D$;mC@tPuB?3w_BgV zhOc4Fe<3bPP72w%>60ECCq7 zT$GjBnTe~lWJB6JZ{t6;=vq6f1y)0!3F%Ht3`eo;E{6C&+6$GKE~@<-!i&^2Sh$ze9KBi2DgQaK0VD1rpq3JlFOj=|?JG$eteh2%Yu=NH6NObeF{9m--6cR|k%cd~pzyR=92o&!o zU>m0@9hCl}&V3vJ5tvq+Y4%q%{s1?+rUt9?zpwW@?E1ge``_mL1=IV#^XyGr$boWB zRZUHexekZ?THOvfr)k0H76;DGpF+>LuW)b$&uN+OXr(8HCzvIa0Kz527_L?2F8@xHK?jJm1<=d zVlaN1WKhkr0b#W*;>61b#^u9MN^8AglB^2XRy6MikggX#wtx?T5XwQmc{%&36@xV;vh#0mjHDi{e-2Zn%`KpEofKa*#`2AI#-N6@ZY@g*DBrlcT4 z4#uHlL|P1X=w*Ox^@4zT+uJfrbxX??(>dB{md-ur>It3%sgvItXksv`?!)X>ch+-v zYM-5502{6KxhNP@cedB~AG+)^a&cDbsDD5VS=CxGLG@Beft8p+YlsPDVSg)$kef@c zaFB%_;50Vbc7O|%0~Qe(7rLpP9P z2|3(7ji|AlVLIN9rlB`ta<>5U8>}%gF^d|V_S5FvI>c!lk76=6Uch?4Mqg#L4(Qd9 zqj$hY?2#|S+>uPLiU{)}_1dda&|Jul9|4o_A=a^P8Qb1-}BV#^4*HB5Bb0u>b~%cvK87hub8cn#)r0ZBZFGJp1+~FHeeK4Aow|( zt;L=5DkBrHy&mP%fSszHWWa{%Sdv7ejSi#yS|VY_Dn-L~ni0)24807= zk*4d3>yAP4sLr+X3#K4LY^_!hMIb zFEJ#saS}&`{{6L0wSf#UAEc}MkT<*s&tw4}UVG+c*5AOhZI`!E!4$AY^K%>YB!)q$ zDdh=S#-`}OrIk8{Gq0?V=~m^$*9tdwF|Rr&uV-IKm*yH~H{9;BRLaOq*-e$6QAX=A zPP@sczqRpSQVf7h+85EBwm5@T9 z*vqUf>4#$P?bFXmh>Z%xr99{fBviB)`w}oJwChcLLTgeU0aU&2#Z!a1S!GX8DDB=t z^MjLrPy02Y|Jj|(eiG1R9z z5L49N$3rZ)3*hH*i`hb{Fmt4;8_Kn523%v)GK%r>&9yd)ZsL)qTI{?n4cYr@3@g&Z zi#EfBCmDE&FxU1L_xD;);%BdQ2TSPh(|Qmw<9H*pBHebEc9c3x%aHXcOOHr7vZ3-Y z+Kj+VUa!Z;?XHbBv;u`9P!+d#;$dXKg?juv%mE`9V}vLTIh(5J^=y$Mctp~ceD~X2 zwl-i|V?FXj>V}u7b;jTUyG?Tz?``u<$KKzuExv;~!FeaSzrfbjX!;XoS zRya7Dd^0uFPN^i09Q5f{Jxe@WmJ%yhlJXJfSW3A*H&(3es=*b38CelpRJmnOOVN?T zs`5P&w~St{uRzKAZKCm3?&WZs<1ct7&-8+Q`Ka0_idSBmv$svvIR-q7pk*$3?>Ip< zR*WJzMhs^bf1G`>1`Yeo-go3U1hX+NfPcZ!w=ugT**L%hK*2CT_<<#k9UG-3I)?MF zeD>kxix7snycMBk>Mc}=y~r;ZHd(bkaAzUs?TM$jSHgme(`cAU87htxBr~f45Uw4<_d9@*8^gd34vvNL zB@`uzcF{aO!YnF#cr_a4L0dpTuB;@k*B(w)I&y;ajLqe)ZQ~eM&iZqmn|wITDb4|` z{ESPnYj3!wptq3>y|N^uI@>3!Y`26E?0WPJ&N;Q(KA1S|8ddNS?I`J}QufA2=T`gd zg67MZeFkQt>Kue-U9pQehfGV+sqAD<@-KNHZj&yxt@nzl_ZfIzmfy$``P#l$+kPvz zwjg>S-z$Xvwx$z$0&4@i(3kXMym#(usD^xsdqy3WK0IsRGZ5%?Lxtdrl2MdT+5NQg z)XeM+6{VB7Ck#7#Y+JhAG6s`s%{$jZ?`Jud(B|Ax5B;?Dq_c-fO+CDa5SCIycO?e; zuH-vin~FuzzVjg^vLC;*n(&mN?n5XI2Llx-5dTGtgi@y za%w72C5lT@qFkLg5_I>)MTfCakvVa$eR|Erx(&6rC3=S2YWv%ZIMqtf&xZKMXIX_8 zMOcw(YJXuYF_vZSTn|Z&pCdiTmky0U!L|fj8=2WkD)-~J?}}=Z2B*f&B?)`8HYnL> zHSu=Z`riaFVuzK)x~WS1D5>HXq%38+AUV?SDH@}T1995@R(rXN*W5KrEKKqla=lDbkMvw zN>_JYXZUtz%7+l_P-#uody;74s6ruE!IakxBK?}PUM8aRjJa7w%{p(@kxNAXl3lQ) ziGXd5Gax~(Dxv5zI2QeSbL;B2w#QHhOo5-8ZAkt79>azW22JesjtHkQw8Lzgg?wk5 z!yrj`?DXH-0zF4TiF4C3<O4ykuu@g{ZNsm0~&t^~Zq^ls85E z1Q_$+mVN9mD!ZAjnjP()9$}69rGMS23L5+Uvo+~Vs!P=t6>mMu?_`xy_l24#2zwvV zQeYHW{Ks`JPSKz_~;>os2J7^r7D;lHeN5hm^#OCKy*ZBq}=0 zqw6?9bPeh0H z3WZSRgk^$7Q3IoJA)m3&xaZNfo_y^5oL%zew;8SjR@%-H#}78TLOt>~|K*LzPT_E|wyPH_)3w&tGtMj4?olZ-K0RskZ7tOZ3Hx^kR`JhS~!ZA)UIkfc7!fkd^ zzh5qG114CW`O^CQm1_A+FK?M?(XsQGC)m%)-IDic@iOBSd{LCoNQD`WALErYfwW4? z73^U2ZJ|p@fET|C0j;8k$P^KCh`3?DibhiM3Hs^el|X-V(z0`nU)rmC8)I_=zVsDI zW#g2oMYK66G~y}P1?5>0dO$(jmSf&cfGjiv$MB;-n74t>gbpxdl{(lKu_R zi?oB-6(l3fmBeWjPK^1k(KBmJr&x>VtA)dxQ%BqpBvMJI$MI`QYzGTp=)66&obZoR=V>0zU z4+)^8MYTH_-@fl@zr#&oto?=HOBi&vbfwQqJsG4MYd&IL9AR$g?#(95eyIfA>&aTa zUS#yBQH_wmf=-yZag?o!90w*UxF+!#J6{OlY@eOck@Isprd4bjLSOC- zYT8Q{h8pEs6w!x7rI;nZs}2r%G?&ep9w4pRyO@Y65IkXQ-n6*1NKVWkA-X1M`_Vmn zR<~JvJFi-WTTy1LiRzqKjO1iUhU6WGeW`!00uD6RxH!A_DJ)2?C9HyuQ?JIwZkycR zCr{5wTWDbcBj6=sUcZWc!pdSpnKF8{Y(X^yGokviZK8Nate0j;ss3RSnz^jN67A2< zn0AbJM7e6D!`a2P-XBful9ss7n?4xJT4ok&muV*Zv~wM-_Y#D-=DD;8+T`1hm)1Hd zT)GyfMDG-n!Jxoh-c;5{rxyQ(Ft{gElW0&zgt)w4MOa$YlO`gWW{9rZNPBSyCzo7B z=ykE@jCojAd|7Z5&`$(ule1ju2Oj0ctttr}U&!@noAH@#z*It;jP3!T|!g;@Ki^*$p%GAfpw&jy8Nu2k26QDG7&~q2`#nYs8@| z`&L1I-Oo#w9w(OeOq{KGn|>8V!^x)Twz5WiaYo+j4itul>M0kIS8e{;dDu^rLPA|k zW#ru&yG-zGycHxT+a5ug`e&&JLd)LJU2S63SWVj?9#tRHnR%#L<8^Peq4rbm?u@dC zpuj8H_ho&i8fZ9VVxsDrpDqxQXi9Po0%)`JwEIE(Y}V+bS__jnhk2GGWFl*xf9srT-G%U)9HRKsn3y0i)F}F_eo#P zo~%kCocz;*SZdjEQ|sZ!n(Luu?ERa(8b+Q3(vPO*Ui+}7tWG=e(BUOe1hM)t-nlVm zs!rH9%M^LCK%G#W`^IIM@mu9XdOE-O`m-~i@&gOJEQ**$lRfThpEe|=GP+84<5g}< zP)j03T$U_1pD{H1RhU4G#PHZaROI;Zv}UzL)7vp?I__j^{|L;(W}4c`FdB4V@L4t# z5EJvS0`n-MqGHh{h-0-yJx^nr2csF*T*Xv2W27FuUhAaiVHnjPc=4_Fg}}m_n4ab& zhHq!MZwrM)C2-yeQ?tteq7*l`+a{eIylKo!nY(mLDJwZ~7S68UR&$F5HCdFYUuGug zW$aq0sl=|Eq8Zw%7SKoyaNb^lqfr$34=FefF@l&?_4{@ZXcACuCBnP=Ztq;6zvUy) zFR48awF}YlTUl{5$#j?evuO3}SZQtY1;ISreI-q=1+XZu%c%t~0Jh<*f0vi>qQijy z|7-8cznVJ2D2W8|SrlcJU`0d;L8#Ioilhch0H11mxLF$5lqMZnO+G_uT`hIx7%sX@5d^2y(efQq^zCGv2 z3>Idk%6&_p^<1jKVK`P@-{lgbOHy=nM8vt-az^9Ex};Z^i?p$-2qL;EA?PHA<|DOK zv9z}V*3X6W!)v+a9Tn;A`RFL4sb%gM6Q721^!84oszj{JF{&a7obqjcO_c;BH)tvKLLm?byZB^S zYKRJtyNK1KL~yHKnXKV#y3llGmbJBPQ8;0sDG^KVt` z!V6nn1_6aK%#0m)T|;l$Ph$o24geb@Cvvhz5%Iew?AgH!DdR(`#g{go$U%k$tWk#a z&SxG92 z=x_q(Tx(D%Ywzl~Nzm1B;KDvq=u_NZd&;T6O0KMG?tF!fnhbxhCM9O2@)DupYd;d} zmOQ7Qu1`hoGY^+}RL*9iwF%}I{Tdag@lvW1m{ew3CXsr{NmsR77BMGOb4ql$zF)yR zU9`Vo*ddD4JBQg`6BJ1fdJkMz{hl6pobW9*rA7HE&Pj<;@RIKDC3FWWfs!xl)wHpf=hQ6T?=y zgFPsVF-MT&)a23V;$M*t zusul`etrhGuA|&!AJPZPIhmb1kXy6ADp1+RditEwFS2VOd-yruK9u}3(x<7rW5}V| z8$pCkiWiGqq^4f&Q4-q~Og5uv@iCWtmzaaGyNWW^M=<{ReIiNK%@$ne@ZF(;7_GZ~ zI_Csivv9#?`qrak$S#qf2CA^|R@%p$L3cJF(#MOg&qgKS650T>&G+4-FHfea+CUbm zOc#OWv5sTKV*-yc0wH-sykkotj#6^i;>0sF(>>Ilh{9ZKq-#+igkUa4)0tU`dwZfC z7O)qL1|4|3DuHS&R(BhZIoy93dSJrO(_6N6u1)CinR%NU<_fxjyE~kIBxp?jCWuh7 zEra}*MU-u0+8HJakv>CYZxZ>|8l6@v<~P3M*ysJ3gX=#hd7@K#YL)ZjKO8^R+4ZUZ zQtx$odis_mFHgGRyk&IAa{VZ>{Q$f^-5vxamTs$k6}n+;gr6F8i^T!hCkqnSej3H! z8VRu=h!1ZbyzuYC4!UPVAia+C{2n~DIusJHi{ykD1_c2RB1vK z6hws3K_CJTothzd9O(2L~1A^#mnRbHTWH0$lf)6?$6KzeXxPxy7eA(f2+Tb(l_mg$G4+@+u^DQr9 zjkKEZjpV8Bc0c%%^{Ntn^fhL}mvnK^iGa(Pg%K4uM#$LqQIt*aLOt~(9Rz*bL&pTJ zZ2x1|+<2uMU&3FUx{W0zF>It@=yXT2^1vb!4&4tQ+ZZJ=$6PFP z)XivlsRkcve|EeH7P>x(M^VWrY*E$dY=6umAQdBiQ?+U#0EAZ4+_0ii-?}h zO0-!*#aqm;rxVhgFezK*@XhZnTWXYo;Vjp8_c$90ueOp`P>6*w1voXJVVR!_-y%@A zJTq(@v7gJGut{=mBMvdDQ)~h%gI=EgQZ}y#DGOs;MO2vX@GJKh8?A<+Ul0K+i0O44 zGGJw8g;if7r9txW-x0=w_bB7v=Ag-fWR3VG{!_|f1I?4t~ zVdRCurB~N)D2#aBp(^GubQZ(vnA@JLd^(9SQ{hM?Q+9RMa)w!_)d!Pm!9Ud^$TO-aG^w`fiy!&SuNE?>e(mjT#w|at z@Y&hf4i-*i$dazzSMRh%;rE*NLN3wiG-??UP?*OFA%x=&&DDr49AdR@ci)+)y_?@- z5X8E!Mrp1nfoU8^u8iG4x_=EeU<{5wfL=S7L6TfuZ=AO9ZYttbjF7KSknfLCS-QvG zc={$9v7U(dj$w^a^coMd6$pA-xJZ_;ES3zBISMb@&@*P=BIc;CV|B>qkHQ;{dFR^r z&h@vWZI+x4gvzWay1BY;!h;IQIcj%-nm0M0FEb>63L1$)Y&xKMC8RbluUOS1mNvL3 zhdH)Tua`TNx6~q1rD3&eq<#-${hKhxYf+N3xO@S(f#;zKwT4KEkpo+{R_b3DmswF+ zHq-Te`g6hbiwis06nK~OZ!eY1Wi|J>gW}} zacWO8AAB6jIkb81z)KzJdOse_(c^NWdNI zCmCU_-MGy!4`ntTxSIXOK0Fp?vC~RBLpl;J{wSAy_R^A^SNt-6WCl1V@P3PMZ`tVA;toj6|=udv3yACMvtLQru+aKL^8A1MrurDsxVJp$!&C zDn#zchIz8n<;JsQoU#sdy^|Z2WtAp$y|?M(AP?hiiTTCXUY8_R(CaVIM+$TOF7aYn zc4OEv5*Bt2PoM5N+`sgOY%{4zD7wo0AmLlE+t=KvC-T)pVfnVRHU5>(_NBMe{d01~ z$dW)~Qj~*;&Vm|TwOl-Ywx;y4x9`x0d!Wg)1)+|oF_h1k&?!O>=Ggkn-TOq>Fcm>7 zv&%NhuWic=(*k(>j8pqaIRugQAs2G!N?mq~3WeCSwVbOSqrO#*mbw0}X(R8ztgbil z+{7i~*6fQ>Q&a(_xFH`sqc#)B``xJ4V$^5+)UG5;kY}&Szp!hmLEmEr?T2S4F z$)nbDraP-4;ILIBX%7;{9m+OQ&$VBDWB#Pz8@U69aGmHu`Za&x8x-A-J(1`v`Ch7d z+G;&A{)Sav;=VBADf6)gIE#c=)Ls`&mm@Q{0|noj#l>Hh>XkQG9$6jbfmP#Df>y&SJ z%IGDuAh04wWPdSiZtJ@swM*Snd-qX|BW4nXT>CTwnB;HyySQWsXdUFa^BBUm~fCz{L$t6q5ZWo^U@ba4~+JZ4@Htpu2lFh zN3_t<7MOo_usWREc~D10!QW1VSKi1j;Oq$~;@IO+#{CvVW=o#->eV4P3jBbQcXsas z7$J|;KW9akGuV7oDrD*#>1fvIhUSC%fHFY~m#_N;;rnpqT=fKn@Fza zHWLg3U;*^z3YGoghzHx`IXjtJzK zfw-nN{?tC)9hce^g7g+W?Z$AZgFQ29ibdx2xL{7#9TP-+qIIdI1Efd#ymwULbMQJX zfKh|8=wpL;{Y6|DT-;0ayQagZ#y4jz(MyN5Fg>~9;Us;MuB=IB*Zv6m1-71L*(aG; zC6~!)NOyX`Kmq%$$?700b0d?{i9MMY;YTC6jh-ySlTPkdTDRUIBs9#VQDiij>l{&Z zzm@D}9a-y^6|}KXRVS<+y~cUVi|hQUJ)zy1x?BEhESKvJb7fW>@3C7<8d!Y8z`oY< zRB4aR>ip-g3a&xdy_Y<#^SFcMcBVSoHf^S;Q`a$SX8AG5P_j!#=ZUKyxu^#jZ!ZQN zh9Y9~kzX%))=wftllJWuRdHj;+aNA?FWA%vtw;HJr$lR6JwYB(G}PIb>W9?48y-5n z!fioR=JZb%!HtJMe(QMb0yA1+qH@K?ZigNz0$cKAb2H!9nT#cb;?M}VXmX}8K`=D2 za^rlx8y?feEdFFOrB9##bFPr>?6I5l9I@)OUCG!nw2~R`Tqav}P8+u)>3n{aKKXd- zzQcDPzjTC6pS^eb)RImXsqAS`sN7DWBmO(gPJPk)sK_jN=yD-5uTIo-xY1L|s9y03 zol?xx&A`ZUz66&)&ypfqKhGSk3^vuiloe2wM^tPwzH*vL(YhFB*E(Rv-gxhP*%M@4 zje3hcop=q4!o{ox+aL2e3Lj16^?cpMJ|<>4PFCvAI(Amr0EDtCgnQIZ7c=pq-sJ*0 zC=bER$FJ;9RPAZD#xd*RrS(?L+vJo^1THyJfn;@;p6nrCq`7lka<7T3opN9Ye?-x@ z9Yms2x5L&PdaIyo-B*?;V>}&`Iccc!JzS8n+o|*0C`alR$b;IENYQY`hc@NCfniS? zjc?e6OB!e+RabgVw^WpZs6>Ld4e512R$t7f2}J|LyC!m&ySh?J26m1}{&QP6*&GAk`LV*@#EO`n3Sc%0L}R zHwPES&}h3npI{Q+k74JY#kRZbf_OH5rQVVq5L6+$L+H-!j!(}|!ZubHuqQDpJzgju zK{)X-ucCJ*u<`1Gf&!m?qUXK7Hb?n5b-cVGW|%^3BdSQ5dL^B>`hXo^hE%`9(4iE> zR-l6ayp@9AY_Arbxcld_tHmHG#V<}Yc>sS>Y>Be*07Ha+2D#5=>J|m&?p}mug&=Eo z`q(=4n)Cx&MN7 zKohHiELj+>$fRK|$shL0eRxVtDH3a=%g4AAa^5XbVOAyy+#is83oF*_1&3P7Usau-;${jK|W}mT>2j+`#3p%`t)m>KX!Q6h-50 zsYAz)Vf>orc~^)0T@Bkd*Ow{5)0KjO-#F2QPk^?Em0jfRe-=V0CT{Rlih9p|r&fQQ_T)fN5ySc=X;EUF?om1-y{u4QnXDViC zYU(pu7B68*hJ3$$;6$HC6?LV<>ipM-<250A=xO^jTr&IxTwWDG}un2|c}>z^3s&VP@`I=Jj~3=fszKe*i=Mc+K6)!6LwJyKfwjX_cqA?=s_9J>@CNZ9cbqsaWP`#{Y1j<}Lr*u!p z5_a!Cv;z`iEH zB(Ywgvkx!@B0uJ=tlrQsHRjbD`P@Jmy<6!$UXWN2h41X@!Lt{z)!`lbYutlpu|w@G zENyW=WFHQ3`OD4ZO0-we(B_2rH3R2<;|}X=;c{8Ui*)C-uO0a?{oPP)J=3ixJ4*eu z*C{H@hauK^G-gyEsY@-ksUI4%)den1<8vYF9NArLq#qsrOir$ahcVfnggzCJqihk- zPc0Zm78m#hvn6}0ws?xGh?=EjqDgv|duD`84Ss4vaC0%Ku9nO3$?2Ry3S2C7W|d!U zG9c81LgQ$18nZM+m&{#@oY+2ctqf+v}M3jA)+0-%dqCKsl1%E0Q0?wZOL<+ zXvXt~WxuZL;a9!UY_PH5p>F8>nQX1jsYIxI<8^~G41%ue?|SM8?ozXcS9iS>Y-cnP z3c@GzN3?{~+aIQhGl5OK^{4_=Pu`LaNZ%@avGvv?pr@n{Uyha>Ft$@tl3Z8e=3oI# zg3&8q4PA>GI-PwPI4RX}95kad*VNKcjl;bJUg*bY*CL)ii&*n*5QJ@R#2Ko6uOfkb z0JEI2yrn|yJQKkZdWkrqFc#8d`>D6gp@l8rYm~zN3;EA7aWqZ<24Csh;?tbCou3{j zec{=$x5zsmz)!RYwiQS&B!4nEx2^chM~<_6cOR+l+I0}Ns+lE$ntRm2EL@9N`qml| z_lIc2Np*?vM~6;kPmFpx*QB804q4MVV{84k9(6M-1Ngf&r|2^!E9r*^;LwjyPqJw$ zYW59egFW7w?#l3(?afnDk>jQ7U%aPJ=QLPi7a@Uit9mM}lC(#owV?h| zH=RzJPF;@9%*536t7l>o16L+PtVd))o7kM@ zVI5?aY0iZRQUdT=ldmCShcba{)bOpYcG}E6IfL+ik8{qn z(`b5@ZaVS2UZ&2a$u<}!MU}ECSWhTGHOy+?XiOu#pMqR^XZtUCt_YtF`m{kn6NX;7 z&+TO68+0a4(-IQkmk*0vJ1ot}%YQolrFI4u20W`OK}Cc4A^hwvCbR}Efe#7$p5Ps>I?9FPnWoz@fDc5A?L7~lQn*=Y*? z%f{{Z|7CvL+9^f6V7dJGFM)JWPXuh2SnNl%=y5#}%bZaj+(9{>~7ix7} z0d9T49X3?yhTAWFd3ww;aoFf!ca|h^w6-L#SU0=ebk5`;Fz)38s;q5=klFn5{xiRp zejw&{6Mz=xOebrZW+sy`h9>@3epm$dML((GWV!{JZPpI<8aFxRE{ zPMBbG9wP6y+@m-4p*LE(5yZN1Tfd){KL{qKEXRd-EX~;?s}BC!!F&1{qCEGz7>z{~ zlj16XJ5<-qXt8wNA-;ZUp`ly4;A>0F#<1=Tr%)8zjO)q_^z`ZG>-T@F(`9CXB!}Z;74k< za*wZni!N>4?!p;gSllKf!vJfE8#g@F|j^^m`s0I1dsGtZ)JhIeKKO>g? zertuOSI)iX%#U8L(E6ppo#g9d4cM5yF1)H3hyq_<_li`>A z|J?+_tbSQFUFk0w{T`Lp$b}cnH*4UIz+XgUy!O4(MdY;64Jwv4$!{8$QW9PUvYEyR zxF1=yQF=TPzIz|I_pc!NS6Gwx7_mBar%cb%ZaWjanCM6S)*9FDvpKDFDd;6zgy6|* z=bP%sI=xkGBO5N!O=}WX@O6Y*K#XD#{C_XtT~^8W9}%UYSfXf7MeRw0UNAt_ahr-^ zHX+~7orGrO{FbDtcLb>B^Iv~MOdub_LDEM}-q z@y(#b0dKV8RQ-C3Jck8ln2}`xt-Z3^c90R#c1?G=iyMm{M$Z0`*=p2X>Yz>aJU#aTrzHeoBa6AUdL?+^GM)`HZ3W!F6nHUrt{M%lSI=)>rkdrbvF)^%Xp}&KT;( zpP&qzwI#~rh(B#`E>z8E>pPDe0e(>SNSnp7kph>Wcbx^~5vwJD4;Cw%6Anb?Ma_l)?GDKYSUg zv`4~Mp9hcM+O_#oC_*R-atU~LY!{Qp{_W_dM!BILyn5Y7p%1Vc=5ts6=-7ePyFOTw zR)PZVIU{r|OY0VRA@uDQx5;*t$=&^=?u|$F#7^~HI(A~6_+m9U*-JnOdQMx#|I^^~ zAC@`U+Fa1Eud?$X9g{|1nf$ZkEJPJi8Wz-B-(1E};#r3(T)s*K-XKir*)G&qg3ys` z-M{nXwOc7Iju8oZ7RQwsi0p{9GgV#_ng}|RK3ZUxSqGjo`mXj>_J0m4S^MCY^eH6( z+V2K8WN-Kd?t!ffsXobgBYWiy6!7KUZuC6x_FfsR0?T>4*p$%kp@vV~ z25W(q$s1#sn3ts2CXDn(=eGfI_k$ThiIJ|GmYzG48ZzEyk^}7L&qd zu%qY@%BtXM@_)XIOV$HT@fVct2C1m21$5NuIo`5_IfkX1G-s=-jWIy`=aF35=sCjj zueu*Qvy!})H5{<^^b+~~<7KU|=FOl2cJ0a*Om#W#C1RxKBsFBzEt6$O>a)U5lh!hE z74fr~I7s&dw2ki7Vn4l5K(*582&)D~Emn_f3TuF{bIgkOlViP#XJUbJV86OQ>8+{9 zho=TxTLb&&SAFhecZ^Nes#SKtD=BB~e8nj|t@L!_+nV~oIJPtVix=0ksW0LjFuy)? z50L(;JsP?BHOk}rm-of5UiE{5S4k@!aA{{kPb?D<)Qv_yA%lj|awo%9#hG9AUv zM5mnknoaSqn%i8nR-d`JFRdtcxjWdFARP=48cs@oKatqyv>y}>F#pt^wBtWFYsQEasL#HpsG-E( zIpN?mvh33!Kr!y@I=q;3!R_YQ`eK}lwoxfP9n2%t=6>|0+SQx44 zsV8>pZ(WaTuMd)dcKqdloVuE$c;TJ9hp#5HT06N~eCV1yb{n|j>S%st@=47B0N%>w z)d4%ECMh!GCjciJBmA9`S1vjsEDq{L9#XyF1n&=?k?bVzi4!xFew|XxM;#3&tfL(6 z@q|SSsLo=#t6lq!Rs|4j0!Y|^IbJa84HW$7_~V}KHvEy z+199o#~bU{zC4VgkKHK)vh9hgh4eXpm#{O5H9FmWL%Y}oF5kR-y!O?zV)pElDs3O* zB`9{81skAfoIYIkFUXI zH7&CTDqGG98was6W*ki4nR@JXmYo!A{(O^DIzo>5EJWYA_lH!YG+RpSnwZl>kMM_8 zEU+CoXRgzo^3tuiCi~d!ed5qetzB!2%M0Wv<0}uW(g0wAwCHr|2A`l508dI)w7;je z%NYO!e7vR2_qYOCzHF__pzt2iB?LK31Jwt*RIat1T}&~a>!H&bp0Dx2)dG|_or^>I{YS##BxG*aL z6sO`@i~*>GLa~#MnPSEU%@yGog&4)6pa_`rjG{(5-BU0Q60uc-T-zLUh^Yx&o~i+4 z{Td|*>NNQ|UUs_k1&<&uZGhD?&heCX`M>_le{{ysXSrAo&>+ibAdVm5n;5;n$M+er zYwUm$b74LX$}0s~B`O#M2ui~nrAFw5B>B5UW?Ed#a@_`saml-M^2y@I zvNWPL#ic}DNQrl=2U&NfiU3CsY!&l|frmw>ZcJy$a6VZ@qs*~&FA1Q56GIHCUw+;Z zHUB9^wXUlstifF=VEos**nDVxZH?pCA_icg8>l&rgU9?eR#hEVURv zE zK-A;n*X2QbxBV5crE_W8ks~uN84lE%NE*2pa#x#=&DAx+8z!q|;MkQsKaY)uIFI$a z8O1HttR6ibN>v_n18s%4$dAeeVjvIV`+TE|fg|>6Kl4aFh(Z1vY)EgKyY-A&U$F@I zwswg0Wh8lPl@~oW$HuR^aQFOrP+;k=XMra@`oa2E&Rl}_Ky`cF?8q1MNEXtOHStB2 z{*qz+DmUuhd!h|1xJHM<|9&R1{8e3jvF<3D3o1Tqut@>?(9qj%*GnabDC=qWJXNJTaq*0 z`1;*6xIHC)jJB1A7Kvi$#kgcD?khYBKfnhP*9hUpXAEWnKV-ov2hFHxD1C!Dg|o>r zS6u&m0&!49`_^H%LhhSo6~p>*$R1&3KE{3(3q$+;(~4!nWa$z>>aDS)(+6HDdJ~t$ znewf;*zpLL%OM=yQDk4-Gsd9k`Fm>gy=VktpvK2Z0Z#Ts7#x!hdmvfZBMyF~GCd zgW81JH#o6NTW!zGA1G^i?{7NlHFd?s9iZyfWxE>*I;QN4-Z)r{UR1pgBE^U3+`2bh zdu26GfT~HO-D0g!zVxHoD{pV-vz&dQ;JGTS+;6@B>O~9amMC5Y@^FDg(b6)~)eVD& zM9bQKn^_kQ%5te1nC&Z|Rl*BA4m@KU8&G@%LLqLb%98*b%2R!=pJghmYHWr!dlh`c zAHlm0cHuuJ^iUuy|2cgptHQ`Q`y2{|3XrKMJ_%lYyNhy<2KHct(RyCSR1a(|w^n$P zJ!l9ri%+1LZNJk8Q`(}y=cS8q+GlQp=A0JfO3z4RHun49644GkNO#*xV)0pgzz=1( z1NH8(4f!5HgGJ2CL0sN=!0`%t%kI(<)<_cj^Ap`iF*inm@}-Vh-R;cd@>Ae6_QCe7 z7$V4~T}@vLZZ*Wby|b&1hT8>aCZn<66OCk@ zMH)qe-kg*v2$E$g`E6;!u247AJ<*+np~?ad5V}3)LbY^a<@%Xbu1zNY)N-CMS?XjB zF2yHCjhs7NVt3{+3YwU^#z)4ZhSWc4*3Mge&G3i-mDr*xR?t!bIa8w_PAIM?h*_c*NPEX zV2^Y%W4h?@n+{J4Yt#x&gG7oKPgmCdn`X)oE!8~t zaDNRztMVf(j(2_!YCD7$$UO4bpB3cg-2jD&C1DL_)kkYWbHH7G`sMIBMW7?o)3vs> z9J3&C!B)8RY^tP6pk8+qcJ|;)0H@D!FIMq)3GWT_UE2twW7tHy3@v1N$%1^y|JGrj z--i7exBgQF_lMC^(oj;}SPfu`Z?wyNyZifu8B+mQvf%^G%o36|dMe#4Nq~A@!`jZ1 zH7!6;`P-Pkrq9#&Fm(dXnR4}5%FQpk)j7#Pgi|I5Kk)swl_73EVZJFs{!!aO;t2M;t%|NJH_dkK1lpF0!LUF^KZKbkVmf2|9{K>q+D`$gPq4sVUPq`d%{2iJXitv zV@pd*T2nBs3zm*yXZ0KkJ<2MmHUb!&T)W*%jlbFEj#N=l8@t1M!tvg4xl;`2@5`kI zjlK>H{oEQSh6OE)Sip+I#gjm<@(Wr+hrtK-e{8w+xB_}TDtPL-)8+s`*8cTDE8e^c z9>glkBI`*qi9m35bECL_``sj&G}<$bg%M>a)rdv?T1niyz@w2>Rdu)$YFa@0EB%-k6#mrw#yBUk(RP zKi&A;+a4vbT;Cuu(k&BR9i=?WgS9Pu(>!7S6MgJR-BWHMnl?%$h7C6mGV7?%WneKk zcrUjDlmUpoy2&oN&>H&gC01?tsUc$G)fwC+MtQb>%!H3oTO8+M?^{2OT` zPk%3;FeG=YFY#}ed%V?;ywj5uY=*FWO{o|%M;AvdP zlJTxXRD{P@ImFl1D8Vlsef*wQR&anX3q{+k(C@+PXho&J_SBi8ilW*_40oa9lb+x3Wo8a!d7xquoote9G^ zT<&Sv)Hv(7B66*=H*G+)w%$R&x!QdDr3v|~p$~!dY3%#<;>wd|2h#(vcy!LL({7&pD{Hjb9e=e;e`$F3KKeW->s(bUtSm);Kk>ufxOs4-!W~>9=w2796RyJ z`Z>5sseyQ=Y5`BEE00`N7%|;`a!Tp@X99Lgka}m>AJ>QPeHFz@vJeEQte~c>+p`q7 z8XBE{=vvWvFeHD{>Z9Q-&a}7{R+5=ufz0uOa~;V(+6P`QydteV2GYo#=$N$!sIXFJ zCs_(?Ijxc{-c%T``^oh_nYCbZI%ccr8ch}SlQ+GihRL>;HE}Qcn-c6F;S!-K`iVZ?lrDRk@;0eV?pYk~6{C*YZnQ$1Y%c=I#$hg#6Pz z&~udiWnVO=!}dB2(lBX!_+cUd#fFyB@Q`yoopTSGYKv&pO|};bL`$oBPmQY~1hRYT zw&wlSyUM$~3u4)SC>;xNbK6yPDlx=%6q!|haS-&r&|fGsLd6B60&)Vbe{b_AZc!;G z_s0P|5{E& z|Dmu~W2CRBfLl_Rdc`3cBQd}J89p{rtUAAH82Wng;J?n(DAyNKA`YDmSpD!;i*9=4 z`%vROqf$+^AJUcVTz=RN(3_?oqmWjZ5%UkFqE|%^(ug24OK8al@Z7$;Y;JxWyT--= zI`OtucK`JH`)goF(`xBCv~Q(x{pEOmNj0|^%IwA$mq)ULqSs5%GzZ+=-`ijv0Ogw< z7W_HX;Hk^B-AS+l+fft+EL4Q1;7hk4v(rC1O}z$e5%s)kA=S#h(?(5CJR@n5;mTL9 z?H$}U3wiKJ0?k2u*hOy?@lqwQJBm-Sn+6B^`ub|_?-Q#%Vwzp;gX@q}`SJz0_~ZK9 zw&MeFncpXLYm0uy9e2V8zEwl1I!G*H+4#P{ByK_h_H`m|d07TCUI zDHyAuk~YLU@SLjwO=_h@X0RC-=Hg}p=sTBQp zQ5>EdKm~v_&4Vkf)^E<@AgAjC0zd^^=Zwwb4_GUJMOQR)3PI2nVaWU*=-Jufygb#u zj^_?;{1&$?oLpN0sQJ+whhF$ z$J!$x7sa!02rBy*z5s|0(E(0?P3~kDm{;ha9RsmVYXxtn^-JNOfW#iPMQRi^S$N=7%6hWPB-_pe)y(RSzJD%=L718B`? z@lOgb*L266=L6Kf6s)#lF|n-UVdhY&LnKIenGjj9H;oZ2ltb@8`|zx%=U~ZOk~usB z7jKzAmK92!30RJw4a~brwPGN4?vm@TO%7PD!>kX|gN1o?{K6UnOR*=3Gd=Y+%5u8k1^~gVQ4>K zj6vHZS0$|{7}RL${O0=Cu`)c-0>`d|DT6bDXq0}Ubd-EEPi;SX$PxuRnP?kY!z<_d z6m3-KkCn7-7^(Ce8-)Q@f6BIREr|36kRw)ujHwe;P$Xz3#iv-> zgf1U59y+G0Jzf=Ce>8MDpMA25Ri|z)Z=ot|H@|?BRWLCF3F)Ho&SzvfaKtEJrO$GM zo9u?Fp9%U8aEx#^K>?_Mx7mOwGfnceFz!>UB{tUGrOJ74j%jym+|L} zj1sG*TSk$vTt!#^09jw@`Um5eQ>sKj_h9a);=YS)BqwZA4AaM?O!IN?X%hoAuX8TW zf#Yv(h2Bn!R^n0cc*%R*K3wpDwBK2i)D{C!)}ggcjA3kD4`gGdcqY5Pbqozzk0B`B zl0Q`fp@Du;Y=+U7VwqMv8Ucl;SEL7m0~!>sijLN_c(ka{&Y_iVP*7g((A_tUI+2Vz z!|SoW)Fy+a9sP)WWm#x7K{(@JofZj$K+$`}N2uaE>vZ7N_*qg!C^aHf0hX6yJ593P z9vs-9cO>Xp{J-E@@dmUrkI>K>ttt=trKddGW_oh4w2D}Q8~+N@}R80M2pSTLOWA+#|&v zLj#GyfN#Zuc6lB=9XOTCv$PdoUq`WCm{cDBQHD~W9fY;KXV2dE)^)VjDJa8NVqe^<|HvFonZt0^0 z{7Sc$&D%7yK98j{xP2$TmUcVFpmP>gv{gX=io`LUQTfn*_Hy5$v-V zfXFey=dJb83UfXmfP2BxI;F~h_(x(2BaWHRI|-2mWXlwMrp}|U)0%9MIVm|^sw>}9 zg7G&1?`v{6GoJh?D0H^Qk91XsO5&PzYbg;0rb@u^o8!b#=Og!z*TKT)yc;Wb-I4t z_#hlp4Qz*6JM=#NapqzQvj%yDWxwPiuM6XzE;s5XPF?1DAvh;F*<_qIz~$a)dvk&@ zVCvH$TFVqTXhT}($im(AGM_)LE44nFIT$=0%Gh%zl5HL5oS4*zjR@UIe(n#j`j|zh zd1{~29;7hw1}OhG#E=93T3mKR9@S*t2Xa=V=jfy#(wLE1241G0c0?KPq`vWdOe!1Q z+p~O|o??ywJ;-F%kW|(*d189hD=q2n3cK?iF~*efUFwU%2Zm0~`c>XyPt)LqxsP*u zcN6kB<7njB;{#nA5;yOs!Jl~ZONmUhJIi#;Dqf5wotgvG9 z0L}(Ufj-is8obKxA?7{V2FrAshtFiNgL9|mpgNY#Y85mPskO7hu3sy7ZE?Z_lrUKJ z8?swm!?l*R&Zb=%slI_lvl3krlQm=H%Yra~R$4<`ZBLTh3Rb-F zb{=^*<; zEoE8Hvvy|@3vLX#k5m72-|UVHmrF|Jyj}9}EUG|JEGamBAX&*TAE{B$;-V1iAL#NK zobKKn;bCMn*61l#s$Zq`id@d-5Z9}p0;S(gX=KTHVZJ1*BA6|?q3@x6jIQep$zoo$ zLZ3kEk51F9?_-LYqD(7hfytjf%=aKPny1V3rtYfo+p}~cA*jh!b;|3^Z+}a6zfpCU z^L^3LokNSRHl4Jva>po1{)%AeTAI3^&OST26REo8={4%DU5_+))!pCpB*@s}#R#q$ z`2&5%+7o`rpxHdlweAxOqg@o{Q$vGua_3^Ya^s^fbO*Z$^CrXx8MqH}r-QRFb;ybd z%T$-KhI;ltYTZ>u^RmV+m15Ee=d`U|GS7{@Aitc~E=8UW+FRSewwyt~C_Y@Z zvN_@*t1R$ldyKx`g)r$AMW})gkiCbZ#Q(%4I6l_ue!P!#$EUKuXtKy n;V=+5(e!`ze@%GgLuEgC>P)_9g!vf={Lwvk;cT(ymD~RdDRolz diff --git a/loadtesting/2016-09-06/channels-throughput.PNG b/loadtesting/2016-09-06/channels-throughput.PNG index aedd24d51a9970026fbc1ad05f1dd60a7a8155c6..ec88cc2762f0f7dcb430490477fd417a86728193 100644 GIT binary patch literal 24731 zcmeFZXH=8h);5e-P!zC&bPyE~DJs1yA|hQ7q((LyY0`UWq7)G+3WBuIn?OP@p>06{ zL4i;brHb@KS|F4_l6OV7XQSu5-;Za!WD^WbHV`)##rH1KYYdGO6Xo2yz^X=ut~4)0j* z2j3rZxoPZ9L&Mlg{kNyxx!_M4ntNjE*RLA*SS*YlE~Xzh=+lpyK(FOG%dJM7e)b19!UArKuf9#o0e6Z0em0$=`$fsp7ENVy2a77`Wd!a zG`z&@5cu(FqXJ$b@cLI>m@*@HoqF=$m;WxoKT#0 zL)nZW>X3x@)9fG+A? zBul*mysnKj!A;I&ipahZ<(3|>p45y8*;dFR2)~R`oIIN`9FQj+;Eez2m^p=rq{$Et?_eZV*aP_~={} zWr|;Er7LF`D^pc$Vq~IN623Wk&e6=kR8rj}2&zRPjZ+qauozgeI;e8Z_(H@7a5xb{6K>-OOx`wz`3_TMOX64w+pF z+^E051TjYp{~16TJf6U-NwlCJS5@!AyjaRGK{T^9R*RWUgMIU+|4LLU3^UEx!juUHK8FZtFYFqLhjOsBoN)A zdY1ZfBZteZ)Z(_)*bzAgt|~MJZ6yYMo^q`;Gf7ys=4c!rCykCv)Rml9NDRR+rz6xE zs-Z&QXas#$V<(rAv-HJ}UZ0o1Rkb@KJaarf=Xx{mGbH5MW}h}Nt?qq=^SnMgf62pq zr!2eGL~&=iwe2BmdW)5{W3j}1S$0gOde_m#{4p(pTbuIfMy>BvH9dXG* zjWm=MHI+j=7ZBE(BepT0pH3WB9pAWmmr=_u*sSLcj+Cf+_W&t5qdhv%VzLXvM&Jy#yZA=9U1Q)|A{r8u@6KKu@28Q*esPPHF# z$h5|ZFtHWNVzG=98S)ADJkF)0z9w;2%f6T359Ndgm^8l*g?6qrm`Gud4J=efSNTu2 z$82oev1)W2mV|UgoZu(oT4bMK&OgEEt*(87GAbm(^g)h%GInD_5Amu^2N#e%jFVM> zoh=!@xDnZfBzrFmO}DC4&Oi~kKrAV~R1;(~j}$5A^_UW|*EkP|!i`ErCi&%u+t?z+ z4z$;?9|RLg{%nLI`9snb(qvDHR1XBbR*_EnfLVIapT3sZO4N7_DM&*?7DX@`p$;TA zS*@98w-1PFUiERFmnaf3VOnb3u!tihPD>gqJ+{FsoEhX&x|oPgX_Bk@7RfA(U-AsK zbu0)OzNxUpq~O)S&`Vk)mEf{sn7$vYxh`#p?L?RKK}a==g-@nwy_e(k&`gxu<_kkz zBQg55egr&*F0$80@WY&j6#1O!%7Lpf?2`K|e5X$Zu}4T9L~Vz1`4^&N3s}s9FZ2d@ z)R#V3X=nP1;~5cYV$5zS4fmnZdtAVGd2XfhN`A;RLrnTv`Ftmq^aD&>-^P3<8OBQ^ zEV3;e`Wz($ZNqNpCEls$X<{swMDxBqua#)-m9#w-r{hn<*bJTHUkuu=30j>{3qF6- zq{euWCDD_l_PE|czZT;0Sg@fcF0WUEEulVlabpdV3-T7W8Lvy~3pUH=JwwONL^W&LSBi_;^{vj4 zO_Y%S=h3b{TI%{f7eiS&iJI&IsI76*qBlv5@lY3+Id!QpF9_VUDUXmA+2oi5Maw2d zT1cI{i#f9z+1uDlU(@qza|kb{k)>QMZnqk|UVu%6Wi55(U|1pP6Ou>~yQhj4!YkIg zM~!%l#Fj7!NSyD4=R0ntChhVD^4u(+CNiD2F|6o1B1W4YN#AhXo*gEoo-llN@WxjJ zA6fQXm#1)mzZMaa-em*xZ408zd7(f7yT(B9InKo&m@YQIKA0eA#vS0#;|!2QuF2k` zlyZiaM7su5q~v`!S%oPq>usjza1376Ppy1CwkC)7Q2APH5$ceNb&Kr5nn*-z0UFC2 zBa%XEwJD=03zJ<_R&{32Cu9+_w{Q4$5O7Z(IoCQ@dG2siR=CN?(r6VX-yw&C191p0 zcLhXP=3L+eOJb#GidS=OLNUd~wB*V{L9s8d{LQKjFN50w<-O&D&Nz-iW$(ginZ`~aN4*HR)CsTc+U%oLv3)7z8>pnK5FtyV2 zCCNYLW>q_r=ipsG;Tm*%$e|NEvAhc(SR11TR>>ViAwk`JR3rtdCBBFZysA+piu8H> z3UpGZ4Ku<-9e3_a&#aFFyiiZTVYVOUxF%*k@5i5u;xCc;sPlBffZXk26HJ?y1vwp8&p9_D3G+}g&%e|FEskQBlC)q!yMB_n zjl0gjj&c}$uk69T|BJ4TIa+s*0^I4uwRqaHN3MOyk8ac@4La15DcBU<>Swyqf7ePO z+AC)eY2aL#P-qfhe<;0ub5>A9KG3A}7E)yX%8NBtZKV09bY-0L_WD9Z`qe|@$JjbE zH`2BD#Z7}sx3jJGRc<@f*SPs{c@tUMF*Ted(qg3=+Rm`_es0qx)Koa8r2zdH-fwfh zi;6XH0Wp&BzCx<6 zHeA)T|8O0WL2MaJESpiwpF75QEt|Dj>(w=@0$BFuLKB6wv@J?0eKyF3+PoZwVGbx3 z6WkOWxFz0kUdxg*gbU_EDybP8`t~6?>tVWw$I+5(mw2W_UF@J>RT1Yd!HU1-s7WoidTl%eJZtV+7o|K%{W5xnY#Ynb`x(Crj;#|Fc*Vo_G${K9G(o!i%KU?^OEgz z?~&wl#HJ(cM}x10cnxSmIT$iy?FohF{JAE<8D8ELW-wJxe_k9c{@~Af z!;gWl6Ho5KH^l@|o?wqja%8WT2hv#wkX(7?agr#DVV>gcxf^O#d^huNA!4#VD(y}~*>rzh!~iN*Vjo#Lji++Fg! zZh{8|#35Ue9KaT6d?-B|3d#~CurJKRpY>tGu?|ZU+b4u8_fI<=Wz5dUtzB_2RA)H! zq73dLDaet(-;G8)}a>p-to4=`aZ6^&8`r!&TU!958ap!MBD?Tr|?6-P*4S(;tcqB&ffNej;A z<4pkpH#*k7lZb??BAa1VZ;-@M&C+b{xJ3xTM9r6el+!!(v-6-yWEP z)0e-3od#du2^^#2_W$D4NH7cvXj}#Fq|31nPb&>wD_~a&^pE4hQyZW(Id3EbyX5AW z8|mfTG<0?k4NZph!KwY36M|Tj(|A*3J!oA@Q zD^D!TL$B%qmlvQ}tv$i7y~!y%s6s}maSw7L`)39x<;J(7{Q!TE^y>Q;Hua%BxOhg{ z*8t+0ojm@8N2*sMTH%G1;Mq? z&Cm9mV!xwz4~CN z@;w8WeC^6x<*r0g=Mk*IJ;dyklHIDwQ)MTns7B#|elB}^4TnK@O6Fvfn!MmMhd%ky zJT8d;N+@oyDABaR$8|c{Iiy=DV#5Q6*`zk}jRdqQ87X3ptfk}f`K!ut7;Wyq^3JL4 z8S5fQHXj%B_nI969ZHp5f17y#VUXHrMi~nzigH{q*|_7x28fsXFOLq7AOZ=ZWT5OEytAy^?N^P{Hai%zvlkO+KPs$#xbl%pY2iO!*t8;C-m&9W z)jjhid2ptZxmnV^Hzlt_byUgQX_6&WcB(Y&^Y=sB8l_rPwpzW0 zXyZVMc|Dq2{F<6s(iNNNabJAhENoQ{QhheIy**aQzsQ^1xG|=|p)!`KLKoD$IoU3H zqQjCtc!Z11&$J+4VPGHu4bP93Avbf8W~8Tx6itqBvjINR+Y>sQCur?PY9G~VpUgRN zAmSSFuxLZCk3k%ja#2*SIUB37cJl&MMB#p)o}sMw;KjPp@?Cb9=RWqeG!* z7=k&HsE5e4^v5B}gxyr(Q9DnKbB+f;;=F*r?`X&KL)06i`*CB;*a-2TG4tj}%=}L; zm2es3PQVU24OQBabClj7J#l8ufutSRrpHv27D|7^Iy z*L|98_Hng#Q}&6)5alMTcE+Gl(V#7jAjyxXgbX)NY=tayh-ebP+^k1IaWmfG-z~pUr=?&L7dygVL-5t=yg=YF`TnN97MR;cB8wfhU z;}&UI9$Gxn8ou36Ol19g!+vHxK`Vt*ulY4!^`13wD0r%!grMPKnnHE)#`e6r)C!5d zU27QQlW^1=H?XM|!$n}HQ&o}(6#o5ApAii1af#hiw%Uw(O1E5MT7v}CVI72oHCo(U znOr1ojC)gg<3^k-5)oD*zcm#HcO0$`G^usTlym)heVTc^*IljsH4Y)7mZ+Xd0o9?z z>;gZn%w^u#VHGRqzf*l1^upTud_TV{hmU-0WLp#jq^8BMk)mEY@+%=rya?uw;Yz#y z(_<1xR5vtC)BqQnr5ZWX@tiNf+1L2k_Ye#tg-6$?D^LpSeLJDvt*^XE6eQ!cnok9_ zl%SQllwRD5J=u|6c?FPODx(d)s~0f-e1YR)4d)6IO}k`aYwm3QFTy18FoEVUg zXP@{;P4=v~!b?*isy5nLVj}6>IM;RM?BOO=ope^5$u6L{lS{)~lY;;;A}Mn{t1Sfq zz=d0+a|ow~XKm|VI!1B1Z<%6Y9J9}|4!DJmcw@K)K9oiV1wb|a_%fKI3&Xg<@75@} zCS}Dr#YqjWZb^ARiewa@Lc40TIo&Rw8_}XRNWk9 zc$hewa+W?!MYMeFnvneT)nGn5U+{hAlo|yo;+iBrU3HAlZb?GR*ZJ)n307%_gy~wC z_lX*t*u3_BezXJZpF&*sfbrQ1o1PiXG->aXg0cCbQd++IYcB@NBWUs`xYgx6!qivs zY-etl(8+3=R5!f!OP|z-btOp3--?@I2_7?Idt`9Zu9^9iBZQu|PS_eg3y{+&2wQrL z23&s;SNfEWiL2O5-vS)5+d0axf!G`T`_~(CLM^m>qx;cexB4#Ibylz-4NC{&G0>u_ zm3%=2LsDg$hr~3e$lx%?(i_p%(#wuK%=?2!{!X*E?x%S1p>0WK*P@!?xol4T4ke`=UhB3c58M4ZD?P&gi%nk)pReO%U zFvwKhWKNE-DJ&|mBr3Gw8se<5jz{MzCd-6<&6mL#wP83^Zf60Ts2OVFk{!A;HeRQm z#*=)qJMy2)im118tc99|Smcxo+@8$1mvYj7m0YEA^3m5NDPf0W&@ZnSJd!ae|D3tj z&OXUM@n-PKu@$jt0?E+9>T2V(Z(-E~U#C}XAbo@Z;PI5XL$$F7t8iztcf7&Rt~2jR z)MMIk`@$QQPn2YE3P$nib9+m!5TwV&W^l0CGY{hP_MZjeWP6U&P8BE8Gjv{9-ycMC zto3I+k*tMHjnIaQg014P1G-htlVFbg@U}MzX9g<_9}2a!CD7(bT8N*_KG+p!?eAxy z^2MGS1;VB!q*LOjtof3c%yM}DMjnjSCre=JY6W`E*3TJ*?w+NO;#3n?*mm*gYiM|6 zoxp`}khCw~htK*V$3lIp5Go3JEUa3b56u{?f8Cp-+g@7kfwk6VTg=AcvB3!@jT=LE zJ2XntQmUHO^p?}Mt6~(19p>&ICCJVWsL!h5jGS12%)AX@OPW0Yx(8hy_DWP1>5r3V z>!0mF=|;~y=*VF(eed3(QvzYekHkheEcDEw2upPf-Wwh%@Xae!jBO;o5C0QE(%67h zpKmKM=#p)_Iar#L^^NH$H!^Gb;_cIk=;GNo)}plxLEG&SaWiQSS-}k#|Nba@y1#=6 zN7nuBq03qOS2C)wzJE|={{h1m>{3o1jeZ+mjzg8MpxTzDT(Sz2^ZNEGeXSWTN7-2& z1!ZbOi_^U|6atkvZ-fLlK zi-t641gR5k+RR$HMDTaOzzH$g%`3C|x<+%+N|2$;vp_rg9WHNZ2Ju`9QR5$-@epVT z#J}fv0#YFy;7`R7NpbN_3X6!@++L>vA^~TPft!ye2t(4{K~dSeIFsV+%;dY~wZerj*jz1CgIM+S!$g}t0PW{DNN?OzX@Q6O zwW|g<05NqD$W-gR8$gWq06>QDW+yk%0xx&hel8peV-dCZCMq;{pR+&hd|>^YRT!`M zP{isx7RZBd!5IAf7FFHbXj}?F{9hIb)6X5g17{D2sXb2$Lbmf_Vowpy2x$Uz)@D(p zW_;B!^KPrLNZr(Nr5zOv?k4}$H36Z?wkWv;F1OF1@-j_B6%`OLNSDCG!HcqIBFST3 z!;T+e2=wLIh~Hu%wTo~>#qi|R^WS@_R1Ly_^b!uyJhmiOn90jlo>evnlppvAt?O^l zZ`$m2KBM|3zDso5QbhLnt=aB0lJuI(acnCooBimc22SisR^Qv)!-VT>e_v`B0|2*Z9MVdW{K&luQ17T#^hQZnRk3MJ zH)sJ6?+X|9D&$4$!}Ujv^cQ42-FQYt7v3rkJzpbWARGMReJkzhLFBFTa*5PD{Jod? z7c_QI@MQjQ$*OHrh53R!?!hb2{Ck1ShX=qtHI+X?yCM6;>fFG(da#+AJ}M&o^?tlb zk4zBRKd8FfzS?ONtD~C}MXM#tVI}=85a5R&J^n#)Ev6}k%`cAD*N252P;&m5 zECDw{Slsb@pD1FEh1Y(im$Dxyc&pqLLh)2sg|65K zH=y0I9-H)p!Dv%>##oIrWMi#gr@qd87D+WRY!F#~c6t9=8PGJ`r#o`G_v{h1_D41g zuG;idY?J^%<803soa%5;m<5)Lk?a^nle&S_wkX`6DttGdxq!_nF*CjRTLk5T=3$PZ zRiS12%CU|r{pq#&ev%+z+GFo7^d1l0oc)kA30<3;SOglZlK6cPXNvJlpUSdz<-Cvo zzOo1Kwx`k42}g_%eFaN{%ba1=Bd+b7~HIMi|&!1mf^_(W1+eUxyRA3{q(lqo07fRY!Iow4*}ars~p^tU)%kUqXE5 zZNv%k(Pz|7jw(m`f#mEUvhn%)?{`K8-K~0t*A@EJ`RDaSE1}t^nkG$-cf@h!54qne z$u2H7tLqDQA^hdNIPzK8xcu!#G|pBCdr6f3w}W^a4YjX3)fqXE!jWs?61ryJ&MSHt zuugN82quNKSX^)BV;t}kC`sD*tU74uhGXlnuZwoZ=MpR%e44k_?Sttz-*Nvs)kM4CBwAJ{$+FV z@CX!FLMB3I2X}KU-xG%K9s%(pHF>+8{xAmi9YF{|DzI+Mm%sy>%b!`~dqT3Hp;lmE zNAh-`?I!r~I`oxY@qEIy-*%z+s5asC4_4ENiAwb_{!R6KPf`0blmn^O9Zs23EfNhm zHa*lD)R-yfXaV>rf_2N=?_UPv`wUcgL}~Wp0o7l4N#3DdV?+o1N%S4j5U3Mn(^63xCcoh%EYkULATaUS$L?yO)fsvU=Cm>zL6YMiF6W1P-r zdhFG9M9#m>d!*RLLSZ*U7B3U4ri08Tje4AZE4S28@!4+C^pWz|N$Pqg2y%KrtU6)< zvgo-mn61XtT#?_h6)&j2gDzmiolvT@R9MAc-9_%byT!x8>9mZ*!R1OY*Lj?uDq9>K z`kE^5Tds5|wq+Bzk4E3e@sk>$I^;M>MDFhUxJy5Uy~zQ{Ln``eZg8EvfdQT|Or93I z_sTn#ZJg%^{86N59hDiN=T{t%PdlhLYq|A?DX{ljVDiRadzm)3ug^v>vXqd%uK5Ei zRh4P=17oZE7XZQk3sLZgU~FU20}YXg;*q4STTul-iuwX;X~>*`n#B>>YD;HRY1BW2smBjrA$siYHA*o$bk z{vC4^AHXY5+*L6a-!Ysp&NPX;=2hDaEd$=TxyRKrf~g4Oing9eMWZ2kb4-b;RFI+j zp%_*uuIuS<)fg<#(6}Rz!=hSmu5DyRir}yMuFatoHN%y;xcL>G$K<)ECRe`(Hw?*Z zp|YDo1d*<~H-5`E?{4#qH~3aCjj~gJDP)mFrjB0?Sib(Cmu|~fd_RD`hp_1c3gn?UwJPwC6 z2a#eXP6r|V zLv7L`-0@TI$sbaPMO}9M(|SJW=Y}4Cqyt_fjx?8ZZIE^gFysLTF*n|JnporoV@22nCjP$N zfTbM7$>00y;rTuCvdm^do&X9ly`ZgY3mONggDeAjTVd1cTB<5TwdXB@rpS|D2Uh&S z^}DF?eiRb4-fAD{8G*9Ae`#H ztBP821TVv%!2WM>Bf$_cyLe#MqN%Mei86^(r{Iu}cw_nht)VpOiAncdw8=kX-k0Mm z(4ha4)}C({+udhRkVh`zxK?t+;OeU$DYA|uwJ5^k=#L%E z*;oP;`)X3ua|@|v7Drb4I2r5UaqaZ;jLZw27TEnpn}(Z%E(sbUb#H;JPCy+yjLI$+ zxS8(6R0o~m_W2UrmR$_!u)|SmwL`^)`<>6UM+>ZO)`vW6I%FG)(UBc z670RZgljg_Z}viWNBB<`1ovFxZKAOqZQYy=J;N9V)^uSJXh)+p%_PU5i?=F8cCXs( zZ3T&jO)v$qp>j(+RaM1MoH+oTvzMloM}ixa4Rk}7mrsNQUO-(ElV~b@prjFhY_~4! zUI^>77H!@s=bmu`oK{_Tn%uVQ9I7GDrNnh2IbA$dg@Zm=-e%~S^8SJDz1%@NW!x&= zcUSy(XHg)u>Th!4nUr>VV^}fTdy*^rA9|hX9jlOq=h6~OEnAa0 zhI7iucm^s`m-8o&0}zqy+cltQ?C=&m24}f0kVLc|suAsS^j;cU*f`bj0~|4P8tdeD z=&&EkWE=t(t5NQBvvyUCI$xL9Uon0A+_cluMRzZ_bxHmWUO+13z2`mrvfg9hL|S>1 zAy1E+skMCCeh(@xI*I|f$a3#J^3K*GAlud`{GXAYMLmz{$;loa@5<9|H5*uURK0s! zxVpN#pUEvI0F^adHe_1d)Ir6VWMJybrnM*X z!hl+g7FX`SkuF6Z+bjw89rXF#J>R$FCuwvUt$)>s@AwO7O2s}rj|NXn^d-!=k7x~} zFKI42#!y5e|JVsx{MeIYd?Nsi3Skn|3cavX16y0a3#muLGkIKXl;tsQn=c~^~)y%=p}>0GY=?Av8Pa0wolyr z1)gf3EJU54AGfsa;`APw2p`HJ6tHL&($-abRX*~($<9E=FzfaivUkh>b|staaI+YM zZb_QVxhDO5gYUFT2K+(6vrhofoeyrByCoI2ldnW7G;I!SRC1VY=MMTtHBcXpLaXj- zO#zcMa2gK)@Ak+k@AG?oKlvJj%MHX%ZCWzr4xM)6jMi(M|6b{|QZOMiw+d%}Uluvp$D z;ts4uh>@(F%Cbz%DcUX2K#(G0^R&~jOxp#!X^(#qi|~9|rF^ZF1P#$TM$eKfp(gV_ zC3yzH^1XM@aL(MYidbJ7htyT?teLdMa4&_=geb!ydk!#i%z>GwY0;qNH*||=Q)fpG zxc+Y@ENWVNohB<`W(^u)CvcVnW5zej1iAo}fw_aJuwtf3-dN{PrEk(14TB~VacTcX z$J`oJfx}#*Ec@p5JbS$v%mr!{5+YbKdFG)&-zpIuDr(PzNj!Ao`$icW^^&4?mU=HCjvekbChZ7Z?_tBSJDKqTf*#aI}>K5u{Eb67pHtIzn%%wH^0(j zoqK3Xc6~}=V`P$=mutW@Sdg(omu!4GG!QpxTy_xVJHx89c3yFwV88Uo!n49&krhrU zT+)w^|F3OfoI;(z3k1!JJpujHtldJF8jtg95ZL)DBtYqo79pCj=K(JPZz9?^zs&n$ zzo3~0nh?%B-NxA=wn1JnA13)bdvy2ER91HjFr+P3TAT=OD7?*NtalKTU;?v=v1dRj zix6Nt{EolIUD?Twm(4e#Ip*(wcr^hmYySXOZWk4dP)cZTNf%W%iPOfPp7EUTd#AAe z$!d{KY4d8uw%yoXCiyQ*C(`(VKDV6Bx8{${FbB>7ZVA2nxoqW^@BKv+^ht%&TGgsu znDCT^b5~tu)(AH9D2KVUzyMOMH?r`37jwTC_idms4){Xin9u-huBx^5zgn$wzz}Il z9^Vb|wNz40lTzjPdths8E4c)GDjR%f#Q&Y~F?`MCmUiT3rkV3YFuR0s?Rd(_QjmF( zc<7ypv#;LnvRua2u{YFex)hM-#0Po0CF;C#Bap^VY+l6~CBF2tz@D?Rn$`1CQU?P{ z<40~0lNPJ6F=F`zEgnq5OKAJx6|TAXWvb3V0h5y09+$#DzyRbR3*(PU{&&IvA9M#m z6UDet%d{Cnn}Zx|B1F3BTgIqHN&^%bU}Kw45B37RbI0^m3of zQq_}l<=13djZ9d6Te-As`dzG1)|Ensotl=)Jh6f6UlpbZ`|v(Yb*Dg(UdYmhWeb-B z4($%y(+Ln9fu|#B<$lR$%tFCtat337Oo~#g)#q6P9I_wqx{|Cz9T%Py-h(Tblzou} zBZm^UsotQ&9_@T7gkzin_>t|r`(%_9l4^$YItq)XcC>tWjgL^y2BGyHTSgg9*5@l;mR^Xy6MiqOL>L_YXMi-u2h z*g4c~@kWsJ6)OOBWj$Q&s2GnE5E@DsgYHWxzi@|#$%6;F@MQ~hPqox!<0fo!V%IVy zxR=3J?nPm5o$3HO6Yjj`t1mTSNbjXZknep>+R-Bz?7Q(mB{f(>La!oa6y#7?H+~)EG zSxnki;Wv+7{DoKtjxBv1g=wY|caR^qt&Fkp#oy}!ZAs`4!@y4sF1dvcPxbOvTy7*2 zae=d1oOdlI-+6vgF-Hp$P{iNhAX}7}9QSsdgs@*W=F+7fj+U-IRakTS+%s13e%EsdUT9MssU=&B_m#hQ3G=(w| z&TZ(lLgI;+M0_q<(!3EF}lJecTI**i_u53lbr7qSRf z8YncaQq9d2l3n8sZupLf|0Rk!C{&HmcV&8;%?8_1$u#jtnZJs8LE;U7t^1beOFv?# zFr_CO^3@Y#KCAOt6ujle_VLBxnjVNq(KR}^QzC);PL*s@1z7XIt!Zkzxwz3xQON(O zba;2`5?=`rF_7o?DcQu3AxxPl7U>x9eAiEs>Neo_lzT z2nqIo|Ki-fl?o7EI>*?<;+>%0dk^*bN#<+v9&_ zTfHF|B^K?+^9aT|thIukav_~hgJ-u7959%%d_5rtO zxxjURP6UAqXSp$VBiE^Qoa`EBhiTm~a0)j0=_b?Jzn~aE6lYYI(GNDWgo5M$eazJj z&E#{94ICnY3U&c@ePQEeIzf;mQ;vOZrc}*uk6EJW ztcvD9jY&rCMTzKH!wOhmyp5CxzI2 z2M&z1Z7@5&sDkv``E%|T4sbg9@okor!|utU%bOv84lf}uxdu2k*V{}pKhZxu?!Tb= z5&bz&i2Q*m>B4cUCF}<;cn2W&nRpX9&jTyU@CW&tEa~MT_SPzdFo##Sl$o}S^UloY z(=X6uWFyR=^SMnA+4K$8e`G}+J+^Rjie$O5^i!R1%6835Q@&nNI(6IyO{|QX`|ttJ zTV{~tlR)y**XF6As;|}9@#f|pi{oDSfeA(ld)}`9z$U6JsDKAY{3H`YGh)fWKAQk7 zDmVh-@n}bvu<%{`K7aAt(zHr=mi}|eKMzh-gXmnzE`9=Sjnh7oBcI2$IQMBW<6N*j zS(suzd8A;D+ig#yhNPMRl$^UN0r04+htOtdcA8lEY#%MAEd8GZv2E~L*C6{slO zBKD{@YmLKh^sGb&3_& zRxM%`pW>ZN%rdB^9R|C#xjC z5*mxVlvO?q80%L??W2{Uy1&(*)B2hqa+{tFewxWGXmqZELV;m%>S@7|V0nRCR**1d zt~>F>Q(IS(fiv#N)S%?)lOG~ab{?Ag?j~2IpOVGi=u32dg+J1y%9);Aoq05`!LRn| z&86HN&r|0I>jj<;d}8tKxTQzEM77StJo%Uq1BcH~#hcRpHWm)SEgGG%NjfI5 zxpT(y<#;l?vZ#6pDu|K5WtE!jNrYL+?u$ZL~Q-1U54r_^*;ax4?K*sVT`GP<% z`|&xT9dS)A*>P;q@_)`xbXL^dU_YX+kcbrNu~bye_dLOLi+8NF6fzGfyCOrYKo<91Q zkrE!pGp}K!KcFo0eujEdT0ydu6a{zbNcKq`99|(f^!-|Ch@5-%I`N<@^cFez!;c zFQ_Oie}OrK5nz~{g*TI-RMDKO>jUwqv(s$W3y^*{o}He6Be54KMv0H1vsEX5BN@T; z6llL(YDU~50U}SKdLsRRoq{S~lBR*GQLo13`&KJWb`Z=EbWnELMRswKz*vz144z2p z6lKZXvKH!NXc#!8drJUc-m?uj{Sh#eibWv2e^?`a2^*GPpk|vPEE;!8U+$iC`(Y&c zhZ0}|V|dhYA*bapZx+EA$;KXNM=Wm>#%#?f(Gv{N{qRZrgTNVr8}6+IgFDIRhMysX zfgvD*E)S;s(vPn}sXc>WI`2BSVdzTD7_`VuYc=ycs*BKFM5nNePW^sHSVFG(q$>un zdAA&?Jdx+sHlUq!eB8AH_8&bgz!kC5hyj|%eK7VlCl;!or@0a?Ebp^?ui~E}v|omE z4&Ph<_L&@_yrM4WeXEh5qMm}e{abhNo5=P5aDwN*m-;?^_`f|k_1|Ux|H-o7{-jKP ze|%Qa96Lge5i;Tt`fbYYcC9j;M{zYsdZR+$c|uIM;n}@0+F$)^9XYz7*OF&@o86BP z^;(>3>STBxwu?&mSh#QRfMJ2M;;iX=OL@&DlIe8V&E3|y83X7Zdm#fJ9r_8B-Nc5F z)OEJqcah(BSl85&wk|CDZ0Pdbb)KG(UoO5z3og!uPYR0Xc-vaJv);+zp2w-_x$o7Z ztX&!2{X!#6)hR$yJoCAI6q8d5+jwrM7+Ae^?A{osX_7I=?vJE>ZhuDgwf$~-qCVow z_P0+Po$iidI$vmT#QC0&cz(B2>F}OuzFJ#7( zILeUUG|}eI**)lB5z@+uX4B4Ak(@a<|7juF!yMLisvvMl;oAtiiq4?tgG&j8*KNO1 zhfzVcmKkK*TXpJkEQ{9A3aT}S;fcJ!x9+WqDQh6}`#Sa7PF}FvS}VGCOtP}_O}^EL zLsI3QYzXtyQnPZWJS9OiDv8pA;cqulD!QCavljVWvb54{sN+H(;!@ya^1x*~;--?2 z5p&3xX}X9fE@7uwdg)54Bmk$W_U2UKp7qZ|QyN4AC;tR%--X7^vy^1@ z+LV21E(NnMGXwmb1yRbdV73q+6}EJ|texI%KJIipwozwTT}sn;|9E8pEtsDQK39b{ zwOEoUBm{O{**w1xbUd%2XDy}rfv@wc^9#lB!h<$y@ahNjv-^P;DOrE%?%gWC4JWBT z-X8QGykoT0j5HIa^}R6&y(nSl@2mGX_x$L2ra@>%w!TH9xgFt%zJ*5vDQEhSKCFFG z6q~7D@jx8dJrPmyInxo^zK$KXjpz1{iA`@sK7%gwCkXd2^q;0K4YZ~u>p$U$@#E}4 z_>ILcIh6jOviYU9bTgBr$Jgy^Up#t2dkfb1s0nrJka}_=W^?RfvNW#${q54|BPAbv`a)b^z){Yd!&Ma~ zBOni6zVA`nDnLCuAVW$wQek_EE|?h<5~!jzgH1vm2uEg4UCcGS+39}zYRQDSZ{v9R zgNsuH3M~nCyd~A?Dp|9Yuub@CXY$UdUiF zgR6Mc$8mrDnP-$@X8)>7ao#4Ah@uPIKbX+4&<YkO3ik9kRCzL=Z)A#FOvLOn zaK~~Ujy&0*W7BbKfeWv{x+jB~^i(&*QqR@0u99-yLhh_#w&fMJ^I3vDyr!<$sA$G5 zy&j)q_9<(JP4yBOI_3uInVIcowHDqMrpe^y%6VQ6Jvld zH0@lWyE(1w+DVt+aW4XZ`Ph?ly_BeDIL8SdqaY3TH7qUS#NL1a4m{#<9l7rcczCDF z{bL>AX$%iJ{=Z+IV>LMn_rF^PjH#rIUr{kY$5qbbO|HaQ@D##tHtwoUhH$fa=*HrL zgBS2e)PhNhJ2T5k!WBRptOH)&`f%>RS~rGtRC(V-7&ml|7Xt>)F}WHnHTT}oLLd`Y zo}S(3W{_)@fcrEp8@2`fp;*}jnWtBSwQTGg=krpM_u;9}J<>`Rk4R4F0LCN#gFzJ1 z#wZvW8Ucp45yzU*86X$-*i=tZwN4CZCe8<^_SkL}7*eg{>=O3zaxPyUQYVclAHaap zGOOwx>Kj%-0pa%@;6t3|KP&1Tp*|5~ltF6b3(y?LFUA(}z5-74>EoR@8*@oSf;;e0 zTuT3Zy#K&U#cX;flybLCrN7s=PZNP_+1wH^LAj?r0vvoA=jVec>~#k7fLbD$BB zrJ&=3I^!s1GEfzeRDUe+8GeA=*rGm0F}#X8z68Z52_vYF2~^nrR-yPToOtunHxpwkXhr=?T37?--$>e3l1G%Bvgm?{yPp{^};>e{-6p&WIIYJ|8>DB3W^ z&{jL{v?QXgsT3`V#3glpyYmN}muIc>e!qD3+RuKn^ZS0jAKcZ~r<3kGy*L@Hbw##` zyam7+gsO9K9P!KitW2ts1;g(iLaJFMo2SD%cm)}-g^V$O8F-jjR#XAbTP-+FEm7CV zKi}uN@ZrM3*NXEX&gO`AX%S-TY$P5Y-)RjLsq-yp16+$Ev8TFfX{NE3?SJPzHa-K zw)5QS=IrL*nc3iSy@Uu>l1B9(t}`Iybzdl2M0RE*911%T2T%dlTC%>0UBoPm1$%R4 zN^gQ`DPt>Rm$dGS5QDQlX_k4VP&Xil0D-*O_QPsx_sTSD3T&`Foam+dq(9SDerr2f z5l{$(9ifdJu%H%;xgi+(mT&ePWV37pjR6J7i3!x%k8p0@E1MocG$1fqs3QN!-vOO~ zC(zLUPU69E|1l&Hqij>>(A9gV{o~MhQ%z`_K$eUPAocTvHCIe)MXfKHnNTXZw4r*t zAx8jAgInFcgxPPhr46iEG$0Ip7bqLrTeoB}A6TQ?@d!+irc)AzR8aXabf0SQ` zzX0sUMwa^O1gK(>B8BFz2_WTbs*xXNtUtrq*H=8^vD=eI!~kHVNC<7;Xco?!0dJ>m zJT&?z6+9DEaKtTU$z>(0x^A9Qliql7S`}naCtUW(` z`bqR}jwt82FUY7$8j3M}O{gR*_p^51yLzc|sfNU9sM2~pB5~N&ZZ*5^bC8!0h^kKf z@mHx8mlHLYK?1qU&64C2^Yv%nq;)$9qD;7PziWPbNJXlxD`0Z&6<8-u0+Ymb$TA|& zKkIEAKxzeRhc3ATP<=agvG7XK%d8kEnn}V6ps2IcKQvLd^WFQ-kG<-jRTB^a2Q|G_ zs+>aN=TBHe!8yKbe?f~klZbRaZ9|!iB3gE>{DZL0snAXi0tVK{%43JJl{j^iO-VaR z6ZHW@b3U;v%_Q(7xfPsA?Er(6v=laBcx3Qkh9-WEt~*Fiakuz$+lNJ8PEtv-5w3zq zXVhohIrU=2xh)Mn6*Zx$-1039j@7PKqC^WTWagH~q>S?MeDlIrOd#-3FW@*T6Br7tS&!uEl{k(19l526%`v&%ViHH^ zy*u4|op%V=l{b2zuHgu`{LGgY;{Zd|Y)6&Gt*1SP$0XS}@pEv6*4@}~{~FO{2O<;z z?kG~NcHb)A7`%(C7lRQ)eNFuaYniTT5fh%PZ6x>f>RdxaI9lk+SCtXE=*`e-3y>DC zC_yKKGa+FT%a^`%6^uYS9 z;dZC)!DaH7*~=kr2Cf6aE(wf>lM46S>+|iOe!udO+Fza_B{WV6tL}3RlvhW$cVcwR zj_C}Nc^xqtl+8(@XFFpl6F1|QE&udVmTJB>?B^vqo*=^MM-EE_<{nm^V6LkXy$kUD zKC*U%-Z-R29)`+8PZ(_m%`L~3yUfuLhcO4r)^qj+w%wL+@GZsmM?4PH!aFte2F9<& zavePtB7oC3OCKd@Auvc*14Rq zbjhhRA3s0`90oCKp|7z{S){C%wh+P5R*E6`?G|+IR&t|pqVt5GhJ-LVb({_UB7u4| zTUyshm>R^hLvF*R+nK`KqwYm`l^I}SN>mgOhjgTH3~z#YW4F&Btwi4}&h}B~YCE4z zIqn$eoX70Db1EgVRjSUW%BIzZXRBy2pLLY#rw5Q}gJL1nTv^Z5s7i~KtW9z6`Mmtg zb8Weq$&^hq!!^W9>tJ&+a6E_DrLS^T%7&n|!iqK*yYkNs(A;J)fN|J0bwo)0UA-l#hB2+>vW#6~KSY`&JAqj0V zmO*0+*~XU4*v9bQBTD~z-}jvFobNl|`#R_USIsleb3gZe-PiTIe%J51?_a#2#myg*{s0}qRlcF#wKhCU0-xrs*ovdsN-PrLbd z`ZK$qTbk(J%j-^hnQ>%->#gI)bsHYu<9Qcv#?JouarxJPsFNqRs~x_|F89z%^ZV+$ z`#S^@ySnemKN1_ou0G1W{)5ir;CI&{KH)ZUCGKO+v$Vl6mHy$;DYTvO)E9ss)3MRQ zGA6>FcNNx$9nrWO*2>v+29rHSx+L@JySjf|<=qWi+h!*EowdsJ55R&(1k0vRAH0$s zodt*1(1Pm_h3$PPHxnL8*Fpris(*6T$ssM0<}NwB9bb>yK6Vj5w}z~W_|*?oDu`1Z zL!YBh^n8wmL@7C`mBxi3eLFpRxTpG2^ZlbT^W8FSf!xAQTKVq@c=8$50vx=VfSRjD z(Nx_LwrfY6cEHxag4a{d7@Q|r6x`5ZUJNW_wwAfpsxG|Z_1&U=1!F;MySI6FTio3H zxM|giI2CUS$31>YqoN^9SqF1md?Ak)WnCTW8i$%MMrGZt*784H`g*RML%b-*ykxfj zQf(*Hfk@tQQ&T)GcV-|dY_@ssW7(XxymQ~D+5Rve{8}!7qS5veM)h2U>YV>VR>G75 zQq^1h?v4aZf=`_eW1`MxfsFC#?R+|lZf^86%%;s71WixuW!66$Bc8p)q`g#``>xTP zq{VBgN-U*1&rLg`l+!aZY%hjcF3dJ_TkP+w-8a~u-roctz|kU+2x1Uy?3IPgtF&9< zBTvoa+5Dh)N8Hp-mpoMM#D&%zgs#hIS>RKrf8 zRFJhuBKQpDhu_a1a*MKh~0$W3t*;w-tR*(ijaJVRB2SOow-bujaPY`_c8ea-PXx$@02UT;5Ws-u?ZG$PR&i~s2pIvnc%23lmR?5g>^va;Vd_AURsd~14B1k9h$h>o-r55d@LqdEER^lCMb}((;r*1-J{g?(c zAwIj=jyku-_t0+GZarAb#9D-?=l60BZ|n3P?^)V7gD-WjY`xtn<80Gj&)HB>r9QV2 zO0Tfz+GnISITpCX>kJ`5oQEpeM_{_v3-e6oMEd^o))6=v*}l4JZc)X?!R`}~{&c;h zxzdtR!pESHH~(qDwrjV*%nhIel@CipI=R z1|#m(M9DK4fsv=y=$O%t?LlyQBO%^A-YR5cRW7;@)zwg#ZbkLU%z{i~Y$2(J0~^vfZ$=AKxuA7@cYuhOK=2}SKC?u>q;ca16#quR~P8D$ztV>HU4 zbljEX&9xy;YIbbrS=vm0O;mpzs_RX_JRYAGi~O)K7wbJ~f>LUf8SEGIam)0lCOG$% zEii!BV!omU2g>qeHHdGRZ)y`?Oz^JC4~-8wm2f32=q<_D0y&ss--<^R6GMbsNoOQ| zCBL)JZpBVkukoL!<*YN_c{&8);YITqKOEoB&4!x!(L1lQ#o=307#C!83o7Kh0v#!N zU@h|Ds9*O?jdIdZ+E%+71;FmuK5|4&B=GCgkx3Ddg!#vlyVCn|3sYl ztfWG^r(%7H7uCYILt|S#f;sHx!TrQ9@-1BCjcvUf+0eKc80CzW@cg-iF3yEn%mP&- z#%DX{6Fcaqp6&kdk>a?Eg)G_mzA2^ZKTsOr0 z%AtBo95v_Y@15XtR<-PVf&O>0rQx1IpYaKc?*5>=jKSL}E2m`L_Oo^;_ja^Ytufj( zsU>`UL#?od%;5OArbe;Ej4KmJkJ*snEyv$pi z8^$1}tyjeao1Cj;PhH(K&RHt2rXw7x%n_V)0@I{ci zsUz(z%I%3u*~PSh+b?M+dgvRc>PnNPX1!KDGGs^W;pFzjI)*2&Gw2RMT-q%qHCWel zm?wmr>_aRgq0?I{Ct*Ric8oW_oBHx#u+Hcmmxhu0&xJdO)>4}7;A5}&9x>q2OG?^C zbcm)V&JN_&i6NMB8h4H>AGZuy=uw@o7~_)Tflw;%9-zNM2SG9Dt{q6TX|A&g6al1} znKM;<6dgIZ+ESD&LGMRQtKM1FxT^z1IeOa@gNUn>dzo`j=DGK^%gj4iF0MjAB){`K z+sUTQ*vAKChwG|L)YphQi1aKDzN?nONe-xj1)WG*m>*l{AU8G5eocBQ&36xLIfi+5 zgtAlQ%p4@JHq{%+1Bvj%Rk^|~Jsf5yO6E6WrhddN%*6S`-B8&e7#SdRx4*$BL`PTX z%_io?aBFOjYFsw7unBd%hA)XQG?8IXNjuVy7<?(ewkf6zcX1y4i z!bn5bW@Ju`l6w>D)dl%8_nF?(j57>=F`&YU@9NKLt#fSK`S#$z!SDOX_!rfca%uBh zEHt36s=C^mdSt^*YLmDw8fw=xJ|2s}UcHM7BX7f2czBZ?fFHZRux{_1c*0Hv#RaUd z&XIocRoAww3Ex+*5JN=ol_k}!GkCyF;XG~!)36G)uQSn7hu!boe>35+ZlvqCYSsBX zagS{`*5+sSNbF%UK5DDwXkEFY|Icg<67dc%HvBhsBANYjaDYb!u47G6< zd@b?z!1@Q$28M?pmge9e@79|Ay1Vj|;uh|#v(WTByEMhOVMCC+CgwkCRwwH0w20fA zfu}S#_B`Q(qVC5=bP4*O?9vld*axXaF`H2{jBkt?*}Zd~Syl0(KDX;jrv;eNb>lOH zvcL&G*2;#jTv^1)l$LlY>LA=s3=tq4phkxcsSw7V?9~?9D7s;)L8O@w*EeCL8!7D@ zTw0xcYsZWq9v@-pQ|!|*F#W1L@phO4k#4lm!KWVQGi$3Vd4E*gF8+JQc5&K%7h>W4 zt^50khjrs1oywopwEgg>G;&m)TIkdDTkg*vO1%(6fxQS&DHBU#z!UbzNw zv#o>6W$ym|*z|?UQ~kZ&mBB-CX{6e;N8iVe40h@p054S5%CdH0iQ}*#!HrDUgL8*y zW9q20_fQwO`|r<3)ycfCj72T(*=wfjr)Qm=rwdRoq%SaN{j*mCzHjE{IHQ@pY86aK zP57nJKy8z$@+nAM%eU$OgJ0UTIxX?yu8N?N-CCjFCKcQVNIT{28V|6d+$@G5`i!v+BrFGm z3l1O8^cSCd4ctL!$l_1-Tkd_+B?=?4GJvuvtnX+s>=;wC7TMa6>lX~@(`LhG#T!J5OXYsKkJY{+C>1hxe{px9U!}rtE@k#cB)iQX-8UKzisC zN$jZ(laG9WiLO4UM#5|LSOp`&ttQhmAa;>r0kD9e&k@wa-4KZkP8 z+-zTnbgl%N6DcEqgzb4)vxj2`m*)KmSe;8fU1dr6@%1c@#A z_C&iaSf=#F6g_jZny{E!t5}G?6xDWd1}05Cb>hrk)k1LJ8X61tnEGPi%{lfHeKp09 z_uY3+h+&mn2EMXjn`dR=%`e@GZo>n1c6LI_7f{~Dk7DH=ESOB$Yu9UIA>5(f0J5w2 z4mpB6ItS$wFz8S^Q-$a(g^Udu?)E#b?1zU4nL!&H#N=>VIwQEr$a$;pg#4Rl`LO&i zaGGZhyHX8MYS&uSUr6&|$!kR4Am=W7M2Esr=H@YFmIiY#j9!%j{L=nWrmSsXhKpz+ zw+mJQa>AtYUnXJ!$7CR}AM^G~$hz#Yw34R3u!on0R#%3_DBrFR*>>cOlvz=3-9)v6 zmL*1LGq!6pcBJkrB{6n!>3~6`YV{<^Jyx%;e3Wu`P`%o;K1qk9*~hv{p=HHYl8VI6 zx+E>{>5f8&IAK+vS0z6_NVek+^>IDr`+CYGOys7X?&5|6>-6|KJHq3?yXM`|rR7{Pc70ap{Eum+ zOXNR{=MIZUgjJOb(6M0ESvy(QJL)*XslcXoGPsVtwa~$R&AN@>Ck!}nIGm58a1AaS zqvSsNN!0dyFzNiuqm|}fq}W={7PGFuY^oI!A*#z~_8 zG57}8AK%W(UwOk`2)0t2SpOQVKtP@P;hivXg z0?9i8tPVj6iZ8klqd3#?T>)ZLagLSgupCgCl*OwS0LP@=!?71Zm=+8}k7`V-8>x#oqlPzqCLFyV>r)txh z>)|)J|29cmB;Spwh9uFHU_QBLr>26@X!L}}rEb9s`$GDgyj{kvi*6nDIg(6D3i-rx zE-QD9PFNL;S44qS8zw3UqCtnQpn`o%ZaIgAPPzQGcbinbEu%xjgynO}4Sa=ai9XEP zHi-0k!E1J=Wth_0%sQISn{9|||J@?CNcDF*k%=-HT1pS=nx^6A!$t067GMX!r#HgZF#!IQh3#Pf6FI?HPSO4kC7tJqV1jg$mSEsJX+4(6c zk3z6P6Zs2Ni-_v2%GTd{LH@FRfp#EpTi#Blw3S7Cj!`Nhqm4@@`fC}Cpyq}=&oQF>I6V?s z`~2j8$46|kXsB=&bJFsG$kC_rvqidTkkm8)=l3;Llst=`|6;`0-I@C{7t2FgQKL4@RT@b2R?&Wfa63jpzcy^ZFM~y#wtaM0(E8^qW_Qih9Gc zGcv|IWq|+SXile4Oj#Jq*x0z=0YxVp+kgLBUc*D7;yPa4E-1^!0E)Yt!ozW;&e<}F2w2EJ57x70K}N{r&)66tVA>9B=b%54jWPfb4GX*nOMeF_;W-4|z)3`RXa z@FDDG0n(TMBEOql);ADUzgHZ_S{_5OgY0&nPGn=Z7~)HbJBGIv2_fcKSdQtvdAKJs z;gsdrtBKE=7(gB8u>WJzPz3`+55yFuy7d2#MT~1jO-N!m&N;p>k z^V*Cl3l!5;we8d~e>3-XSD$G2%VwIg&om~kRoiUd5OrgVE!RKd!T+lm#o0()(Ivgy zi%e*L2Ff2LY-`rZUfu;EIorh|q+Z7AmNqvg$|21X*4ntR^?`E8h#;kI?u&N#A=_Je zH&Xr;y0y{L7hW7L*R4srm_-&$(~0W45UtQvMs>R^yYIfh-8#P`rn?{#Fu2=hse4al zVDj`x06Bi3HTpnM76T~cP3dtRrETGMn05cMrc2)Ze*8DdPutMOGRFG)JzOj^UI8%n?5GvI+?#| zqNXH!RYw^LHT8x|rU($I9qnN6l}41BZu%RArfThUCB>_8tJ}L|x#mIMo>hbPpvOwa zR2GDE){dlFIFGg!+c#yI6hlL7&F}){+vV(@DLH*RWAEU)OW@EgqUV$wugY}1c;b&QHgPk#-d|te$t*hXk|Eku4fV zT=%!%y&YGCYYpq2ycYE4CRPh-$#3SWmMH6b4VHYwh!G562wNXS{&=#hR`{i-r>Nlcfx%%I?gWFrZ$!p16*rW7&o7em53~Dx?70l z#K0IK7s4?tzK~x9DfWnKbx*(I02jFXNce{6X8NA}QesIK)^ssRgKD|`u>BCQN1mnL z5H%_DA@Vj;kCcH)Y5VY8(%S)lLtKy7eR-4o&craLc7$&+8aYD)At9?L^S0VJPrchm zHoj6KghV|}Z#L~x>QttXL*o(GMku%(P&f<010G3ve41On?W@~xOIO&Syu8j?=S2>r zuJhYDNRt^Whbj7%`J-KQde}-nria*PD$E$)PKswL9E(iO_kh8S`FnbsO6rgGNlO3h&PUR%6M zrSW9jrJAVUsEsvuB~l`Tp@3?Kl z^w_m_%(176tlK7*(zV-EQ5Jy64r%&j3079OV(~&Fsp?gOGEH*pTM8ksAW!ut69}8I zA+`g7^Pdvvlm5AHGmz0GE`!~ePv-cWFD>)VlwbCrt9X(CpgzTtns;f_ABnR}P2@o< zIM48=oDD6xnxx}+0AS@kzJFbJ$CxoYIWpAYHKC(gn}mGjP~0n^I-Jya*nHr!1Rs_j zw_TX^1BG|`FlK{B1si1U!Vv`KY$os4G0Wj~Z#RQ6dEBeAfft=oNoe*26T5l5Ai zNqLt@NrXonW9C@vt8zk^YzD0C z8R39V!cW?0#XTJ}0R-hTY%3uviVaU4laH4GH034n!~dEJ(v!QtL)?mqLm3ek$6 zvC8^fdFv*CyedWT)%Vajgb%vwH>={We-!B4oMDY5CTIi^Kn6Txjl2+W+q-7h5-#{n z6v-JjB>QGd%&u?Ik=zI4;3NuNN*oO4f50D(lTZ$-MmOL7@JvGGdG#saB zLuo9;lfYz5F|4~zXpAA7mcq<$2P0#tQz6na?pq_$Qu$F3TLx;=@&HuMc8Sa)S*UT9 zG7B|6(IAC}Rag|=O5hzBw;qp=8GC^rICS7jaNzu*zfd9A?Pr5O%}r#9vJY?h#zg@n z*yI&cjT5q`PHCK$rD@o~;mm2ffNO{=?}FZXVzoC=#>GammbjVU;L;*J!_P(8s<0QB z@t(B^^Ly#r8$q}p3JN{|bEh`GCoar0=JD!SX)A-TMi-kd51oCwuXyYJ3)7B`rfi`V1GZ#;z* ze|$PtJN;6v@gvb={Hp4j_Ye3by)K4+7E2r<8(Y#Lttsmebq^?$t~I#!>H!e^@P4FVAeYdI4W z9$@k_L)GF~-kX{V|TByTf#1L&%49_X9 zl4*_wBy70rc{RetFt=x^oUedWV%Cr41B%Lhs@=T=l=5d=mz~0LVnMOf>2bD#=5)Wi z4QTHn4l$3{GHKG|xAUSR_d_)_dBl_lJhPwrO=`;!zj-F}=Lc_~r0K@F0czQKs?9lU z%$AqI*HrOi_^P$!-wJv_JyWbYssI3em#jh(@RBSS}bXBq>@qj6*GVXNm^q!Y{ zuz>>PBRN`OVxvK-a|xb3A@V=lqq84gefF7z{@m#ocNTFWz2a>C-9IrPt|~G-|J}pA zpBrcgPHK5#>4t);)Lmz6saN0a5W@;en+>EdGUJr1lTYwmNGVJ)#m1HU|H4=aA z@pkR#w86UR!8>a>X1j#8NQX9JeJs7{hI$Q*v&yh)erjC@9uYMrXPn+l^N}>pp=oFV zi1fIG!x~%1$qM9495AuzqJHqw`=|`vOP^A-Ef=DnLTf21x+%ggkU~|%0w2H*xnIsr z1@O=wxSIJFhlkdiu3bT5NAyOTg2;A%Le@MJ^Tth|*z`Oxz=d!6rIzqZ%@wRr{^bVQ zpNk+(KOFRYC%SHKX2N9qmM<%*qkMw~SrFpDN~zRrIqEq%oH;~pGHVlZa+;eSZ;KuN z57TIpZhKgDd$Ya6!q}sD=+IZ?|FZ5xr)I3_;v4=oVdV8f^I=?PnfD(vn7hSV0tER>Z9VH$#sO0 zPC@?93j^ZYH~mTF73A2K8j7@T`TQ4wUyyEFzv~#{2xhPl+d2^zeDHlsha5TA@?# zIrqVo2s{X<1Eg|LJ~1C{$+Kojaa1m1ObSxV;e*~czvNeP%mD)GTWkj5XNcN-^=DDU zLPCywerJPa7Mw4);MKc_DMqMR6*_`Xvf9XZ^>C>-qiv#Ab&ORd^LNnE30d$N@2*t2 zsRhIn1ThG*h(J(8F+kDM$)AJQUXN5t%|y3>T)00hqYw#Tj6J|ftlC@7BGdI;Eg&ugGnb-lyfw&U5ac)VjNG?A0TgH>VTW1dg=y?!!Y z#h^mS*eR@7>cd1XD6G92&qLHIQ7Xr-Mo61`@glt*;#a!^&1G9vh_G|5LHh?GEO`VY zUJsavQ86G=QXSntY%Jq$v~Y+dw9+9aN8(eKMi3&qK8Z{A+Nt96#3oKMCr-%7sgmD# zI$<|OT-1A0jO!`3;uP?fXHfuAQ*A_V13q)eM&SNuMJMnlc?F}94=OI-fS!|9b|3bs z7P#uUc^19^d{O(tiPg_nmXQ+yWw-;jzS?lEpz1ecALPwe~unVwq%`=W2 zb%)YOS&q$$vDko&xx4Z0tA+Mpy5UyL{c&F1)v#j%C9}UL()MG(=^K>Uql32Hzy&@} z(PAWpsr<0cm$R~nTGt5oCFX1+w*_-{00F~I*`10XSa!BlFVv2}KClLfZd(~->dL5C z?wi{W4oMD?Hm@C28hh2SP~7!?f1C@5L=(mYs_uG1q=9}>{L$Y=gq`^z^;C?7wI%@q z%o-NdsIpZKyCJMsdwkKFm!1-wmNCt_cG#CC2zZyD&w(&s^?u7RdBr%QQhnQ_uh(a= zD*P%s1#xpe)>k;2Z& z2{_!sLCWRD^Ru$R`b_+Y$lq0ETOZ)$3JaV0=Z7R&rSXQ8&1<*rI}bZ77kBjK4Ys=a zen5O(9YxQj8SZDXg=x2&jN5t2ed-cV+Yi>K%ro|Y*f?&YcgG?sRp8TyC&Sl4#kuq9 zWk2G|MX;V(jh__sUlcuD@+K#VsE@6n16!|=q$fdPX_SnPKm(G#P~M?qSWspok@6t` zKrQ!m~?{#j#CpI#q*@l(5|Lt4-7WdJ9ZKESqm}a zO{kh7LcO6PC!VxInIC+P>{&Tl3uun*rPuh&`(h?o%91cY`)yA)8;`~_DR9ip`EBBw zSr;fkRXxpP>e^(NHE9#)-B*EEyoqKezRTwrIl%vPZwq?Oa`g=()#P%$rp{os+w0pb z8Nq%r-=r*mw}gbtra{KOLL8SYY@`Lh0QPVaZ~)%>f(tfMx)QZ@hSIK7j0294#SAK?cZ{(rQJ}UwGsq5XZBQDp2~$8g zbKE1;&VUs%PQFMs>kN9SNeerJ;=H#Y;Q!;7SHks0hAXik#AG@sb*>8i40zLu0_3yj ztxps+gNrfwmi2yeTLSOyffQz0mURTd%{Sg=i5i|u8qapztWil68?`e6&oDVHTXL_D z6uNF?U@Cp%ve8+<(HVhvU@vLsfL$!EWCRtRYyLj=3f)xFt3OIh>Ej0w847~ioIXm8 z`tz!MKW$jlMDvMEe?|HcWfIS&1ahd$W$m{rxa~oOYW{_fbHg*)W(&{k1tF%6^h&u7 zTuP3c4ce2V@w(~gVDCl%7Fpgvnyyl99C)bkCvkZ6;|DP;UAV!y7SbPKQ8wKjYz{(D zo1w9JFQBZSmT0*VL5a}EHKo8t*upHS)ihq~V=Wx1;Y~)2==S{N=@!)&CK%gN-|P&= z!GOh=ic?eunA`lZ1wv;kYxo)cbCVZ%(3+6yx@S^CVa~&sd*}7B$?@pU58aJ$RZxpA zWCdeSP?#DGRT@nfKW^>)Tl+-JYBC6ZZvcJG!&-a&YbLDNlKU7)5ecV*1!mA@JrjFx zkXh_xkz>!CVVhDeRC+ z$=FaWclT7!OOW!7rg^>3%zc=Fd@d)VY1}Cd#eb4{oAez$K5*sz#lK%Y18hwD_lx0T zQGmiPthfc%@-F%8(>5SodpI!16ucmPc7gl^EzW> zNSt7KzHCztZg6r4t#baAsxUM~en$=vhBa1X-~R}0P^J(L==n4HKfUa!aFEbAc@8kT zNbJaX9IK>=F@@lCn}UNNa84BaBc$e0K0|da-QsDZxYM<`OlBe$tJL>H(z3v)qt_n{ zwG|gE)SR`(7L*=MZW=KT%MV{imf?-e$pv+`(jb!rs5OZ$oS2HauiId=(g^G_GK?J0 zbGAg5%D{j606;g@J@b_B@URZH(;Y@_r&gq3mY!BE?ukqDF} z(HaIT=}l~AVU@Au9yphV%ASrS^K+!oDVUA9i9g|cY-czPS~pd;;9Ur-OZ;S^NBIr$55XfHUefSe zmhE=uSIr=p+3nuvKBaD|hUkQ!CIIGdJ#$G|Kv}c0$qf9^Tl}o7`V~ z3CNjnhyX;5Vj_Gd@z|4VI$!+V;?I&?uY3dev`1xOZlcY?fO8op)F!634qe!{Zuk(O zw-${x+hY9n!z$p)!$eMWi+OxyTM$0S>J^>Y$U4RXj7koU zZ^!1ZL%1?jr@#7-=2eSLSbL7Xnz%3p;PCX!_M{&Fi=bidr_z-ODg{Ez(`!mxPtlQt zlbW`PfH8~R^j5w4cKw3k!oB=bcF(3?*dtQEp$QmN|G=Fos__tz)d(ivqr zITEHGw_f|Q-PiD?N1KvTNn5Bu$OA|L_n5e9XmG=DTXDl!^Hosb57$yo7Q@Qf7VlGO z-qUXyUI6^-JR?kH_JDiXnDmpzM_X3A{{~qdrUf}V(#wxFH*LQ5=I#ccFRlSsv*#3m zn6OhK6GO;m&K%%__idI^cnTD+37L8u|4H+ry!SQw1)7-iU%JFgA*{NjoikLpMen;& zoor29_)Me22_f}FiTU-E=Nii{(ROHgYIOh{+9ZN3z*U_EQBpOvZarxCdf$pBF6k4e zU-ke#)hQ-a;*21*F0|g@4)Fg;mb2!QW1VA|9wj6#y(~s9g9E`2QAxV@z^y??eo=EB z0(=Rxix*`zL+4!9kU>Mi`cy9hV686G;PrMVT<_Ngw>K((5jyGw8m~IxF??7#Q?(d0 z=a))$1h*tm567}fYF@gYEor&2WV+G8Em%6yZxR%TH?$bCDxWN*amc*XmsJyVp7Q1Y z8DOY;#bjX@{jEer;n^4h5D;b5G_z*uAJr24MKW$tfTZ@`2qmBpIEFJ!TUMB_eTU<~PIe zu-;oWX9P#eeL!Wo4MezO#kp5Z03SW2%or~434aVq>mSB;!;A%&Tyud32#h^!O7TnT z@K}%Brjb6-ZNXBeAGbSmZhckF1`6k3q_T$Km32s;-m1gVd$hqaAN=tqfQu#UZfBJ+ z)9j0f^FNw_46}{ZT*mmG&9!g4`X!U4-DK0y&BmxMHWXR$BtUM%!tkY;&IW>N=kx_N z){Vz!x23%j?0m4{B(&c~ z%eRdP=ugkn8g!4%M4f{#(;UZBuh(mDXklBP@V7QVHej(|>T*(6s@UK_%V7KoHKloF zp}P3Iwkze4tZE*8sF`7%c=KJ>?stnqazMgQtX|9rYkuE|RqANfp_>{%5UAJtY9ofY zCbYE5FGX^p`R5gv9_W%0bRuh`Nl0I9TCq(8)@PY{MUc@{h4O z2H)4+^yuk-bdcG2BVFz_x9#4iWdO|P406attolnULpcWli9$fgPOHwz?`{o%!VO*Q z5m-Sm8^JL)iy*)L%?aStSmeWRU3E4t_Bo4?PG9U)Si%0{@JKtHrc|)l8`l$S++P3V zn<=qNe1b)|0(W%MqPTqp!Tg+#?Bi;oSW5ikBQnzNz!@B%LjHcCd?Vep!{rTd^icrz z|Gw#SWYrq{jjgwx##K~H&fvO!&f*GAIol3n6AlvOzXwb;+C&$-TIy|L$bq(*Vlmw!y<^BfyE=h9RHM31y%wvGl1#@>q4 z5$mJQS`3B~2|L``i$G65tGKeR*=R2{*BQ6&8) zeA!k_$h1#ss9V*>GOj=XinB`ZXlE`sGscKtL}Mp5B?w6^Npk(+k|9XD6Eqv=X}UB=g1JyFLU z53s8-(Mi`7_*LGwy8z+O<r;TY?vy9r-y3!(RTMSxJC^KW_PE7!V)2?8xgpWLXaz-$J)Uj+axfS)C zbD8~5+<7+VLcz|lfTA;yu~9a4^6 zC|J}uJD5Ma%Fuabp(kV6tUcq4!F=MvBg2p8C}#J5Sb0;oMY`6#E7BJc1fFdKw}a$R zwyY>rC0v)?_^nu8HHky%GtY704&dA7c1iI^zRv>x>T_@2jISZ?@UM2;avht%8T{D${?8{Gx zz?|+0D`B;Vu(TE|=$0ZTxt0}NG&ES=7FV`y@&3(ik(tfKrhVufqs4ll+d05LcH4u# z$+>p-X%yq$J4?Vk_{r&{!hll+q9EahQd9IE3F^Dz-M3F8$Nqww@?~jK!BE9 zN7V<~Q|xM1p>mG9At3a*x*BvWDkQrRQQqx3+P_trX5l_vsVG1H9ygjT_kPO8`XhC+ z3yN+F6zn5NJ=H22@y>A)w`AJfPe3w4RBu61x1Y8udp~3_QN$P1W;e6pYr5A{s%X4A zJ7+=f+{Y2y^MZ${Z{D47?whtU`|$mdI*?<9;feSkarT#ZC!YpS&CKLU)!qA&lv=r= zz?V;G^_D6_huMfO#OT#ZtmEQ#4CKpU(1(mscdn}Jgbj6{F@Kf5b1&hAUbmofxYHu9 zmQ8fL{NwY^W)P?v*SMXuhKEHS7qB!xB4Gd$;*;GynXJkIYr_d=fOZqe2XKCay?yQ4 zD0$Eg;5OPeSbZIo9;@i1?X6QPk}kS&D($+p?x4!8sXzUE=?OT;dmO{kNyiD5aGn%_ z+%-Z_;0e3d1_#_=lW?>9g%UIyl|;=S?Yl`ha8xJPIfz3SF*Og35elZp8!^}|;L9`#+wUPSK8K*Lm{C*Fd3j$_!jstI(}*&cB=3JS2w zy1~}PVxpH_nu$FyggDb58RG8dgTA7q77AQPj=Nzfqmc081iHa=eISdGS>E#3^Bm7` zvIK|i*+3&yqSXto|7fR^BzfpK?&j?m1q3s$@>{I@b_oU%56SU>qDAT`T0j;*nSZIF z?>t5jiI!ZXM1S2h!I5Ma0_w47>f!*<9CP!#1u7CMQ2QmS0^;h<`S^S;)!p zm@FDTQ5G&#WhfZA_7oJy!zRJcA~Ke}D>|?$K@Omns-{>&or)rAYPeKkYym-NF_B+> z(R0UnvH^z*9tXm)iy1Bu*r9Q@$;te~ELXnrZl`)kLO)0UA~MjC9BGQdg|eLPVp9B5 zrTgKF0u28MAT{Xq*F3PBBb9K<&rQE#;d)^r-E7c_E2_mFmS}*DLb9dSe&co)w>HBa zSveOGH+mk)Siu9Tn;ul*uRnNcipl=(;j8d?sx)=|7I zYEalsGPJ!hnj9{8Xp7Y_s~>TB;hUqYc1l>uSU4YW=c}g(wdUR|mcD#QmS}N#y(W>| z^w;&q=X*}RmbPdTZoQh?%;ju^E+mLJ8*TmbrpD=xJFMg_!-ZO}rZuB!`Mv)c#KNX5XP`NL z=8H()5`qAG6&`Gn+YPrV#Km6r%K-Xx>N5NddL#d9rjZdoZ&m76m zU#v}TTYq?E@*!`j4`~72*AKF7;Nr%J`+wvZBi$oPE`;=*yTAocs9ywFVd{?@qQsW7 zWow`JCGngIke^hQHw8IlUuBsqK!m@IV?vv=1q9~+k^=T2GFC3fRO93};~1VlFA|mn zavfQoO8t{fn;wy%F?^Z0BRLd-KXWZXtSsxI8U6;SsOaiMhW>wBVoQL(QSeH8)mm!}`a8+;P;aZ?X(v>32svZV? zsng3*q9Z}R*IfVp(nbndD0j_J)y{`#kY~N{>&dLx^vD_`o&I6xj+Dej`15P#Bc8`+ zJ2sKUa{a@B5rm>cKd(;Qxg&e~e_q*+mjaRlshP%Fk)LY&Opy83*or(5H6|`}L>-d@ zzWKjj;RdhEzy}pFiu2IsrByo>EnW4&P#TdYqQKz)GdZ}mBZoMKPl74Tmo_?d!we^F zbQ95&DOLC9;-kakiDRU}PwR%KcOCwTdY(e0IVPGGA^57Fn5-)e;@dZ~Ir*xGe`_kueymz)8Cua({i{n8E&pmHo7frklc# zeY$!PhYd37lYK=}$&!4T6;8d8Y;TdKe{{z$Zmn2XlC;h1&_HTxU#4_F7Wv0%4vX;= zy=cy({ZH}90=dEFt5SvfGJ#+EPxCuY7A9%sE=tT(v2x#z%L5>}bk+IN;!OTbv}2Cl z%BK9EJ32#6X(={fJvggPc*tk}Zj-O!rbumQ33oO$KKAzwj;KK^PR1@`rS@oF(Nr?L zCP8t-vRaq5+gZyPa|MumPDz>}PK!0@%+{2He60CF@H*&jZ0U_f6C#)Nnzn3lIR|ag z=Jc`qj2e7Mm=D80>O~A<+)%0`RB9UUeXhkad_+~oeKf-%>C%x_!iM6f97SJSkF7Cr znH)Bm`YW9|!%0~LTTc&XizO&--&qxY7NRSz!z0t%rZR)!+L`>MwQy%*BHKQK-2P9w zd4@xpBuzhu+Kt!hz~Lk9k9KOXPRN@Z0VjD7ltzmx#$-;*!r-_nkJn=IZueWKX>FNN zXu?6!%<(%PUfD=y$7VfeKCRG=Y<;z$RGe_t8Xb-p57VZAq5E|E6d-D3%9!p9MEu7O zc!%r3g_P&Idq-tHrY_Xp5tyDtZM{|p&L}ejx)Ek`a%)XWe70})^*W^~OKKnu`Ke}Z z@MK|k+QOr4={Pyh9IDwWgp)R>{|w$;)2nrQ4}0+?d~#KIwku*{zQ0g)=jp z?+d*=ZR4u#j}sOu9I8j>Roma+UkK{Fpfgj}HmJ2AK2@VHSnSL-Ulp@}2y)QSkdCic z`f>~E@{-q}4+6ENUqi%2i1^gE$=*$RxzeS3M{zzn-K55FuUP2(MuHWQRi;(jg0?Oc zy&oyIoV)L7_dQMkS2M6+V}GmJZC@0U{)JWWZaLTPTja2HQ~sGd6sLjp;E%E$nnsZ+QVMp{26YeAwFuW*Z(|a&#=uf&>&xuBQhsjSyfmkt#n)M0k4?jh)7TH zeP_>fn+#TOa&2ucONl`b&IR#6gQQd`8tXVkaMD{N z2ZCxs00pftIx&hkz3jz(*B)rg|DJUQB{(K+x;0DMB(jq^>OR`9x=>5?8jxq^I?;9G)t% z5fvHSsp-h-|A;%n&z=N^PEok^)fZ^+!^CG#>Oe<9w~)%GhoGVoqv-k|67L(Rz?>{B zqwiE=2O|}V3GS&1aR$=#&m+aYiwzEl7xPaagh~JvSf47U9+d5=txmJfKUgi_f|g!t zI?yv3XV!5Gee;=Zh$FC0nFO?he#4fL&azIOXob4Ci1BzGNYogpU(B}|=;8vqUB)`v z!D$ZDY30nny7a>A!$o6AZKxG_;6k4wq<;-WodKsGd zOXPJ~!+OvZ|MF-?K=-2M&xXvnOs2ZdV{lm9|9*%4pL&TQ8thDS za8B1~Vz9IwmPxlQ(N{ z$stKRmcOV()gIa;Ls}}C}=Jew9;cP7j$FQ`4ZVay$71X?!y+b}|X!&{2i=xXr z)~N@hv7V!jpz>QIiV!14uy{UDaO$7LsubM#AHAIYH(Pfc$D=K^Xy{QywW3-pYKkrC zsv0b*I-9L(M5w23dg|#6LF>o^(g|l;rMj)QTH;}?sM40GYJ{O^rzKJGkkh8fCO20# ztcfI4?0fq!?4134f4TR5KlgLK_w#+dU(ZrcF1Ua|#ZSnL5!gb87flM`AhB7+2k;_D z#rwSk_magJ(}9%eL%@F@LJxuH*LeV?rhTkw(wzh6zW!%Ca;!kK6J&%uas_!BgY;?) zzvI*4hpBE{>{|008h-k)7s6$r&Kb4(eu4G z1yML4Dc2HeEO6tbgS2!vHQHtzn9)AW1-S8ABE7o!TbD350CQ67<>8!lpTt4^o;b*0 zsQD+GSss6H09=XK+>-(<`A|X5zip1hXTLVX#(?%OpuZe zFm_`n;{5erh4vNn944026+BLGe>Udm}~DuoPn>AWe-?O^K#4rVpN6brm9qeMl3&*9Ejj zJ!%t*cbp*kiV#V&flKdvFeZ6G^0lLc4H?XUFt)_w7+|Z!QIKJNUzuXaoyTe=#2HTSnD?eWlyy|gWe@(9@y+c3^p#5F=*)wKh23zU& zG8bTPMgo4zae{QsBLXYxK(r24Yzaf^!%147+kw*!Cm6>NQO9IFIRGyp>xOmmK1H8$ zk|B8upn#WPyn9%^wylomgt`TO1_Pzfw`0miBQqM@_>I9-qHr}{#DfX89HMrxj=(b6E%%Blssc=(Zuww$;cjW|HJ z&NJ1XD%z2oGtE3|w{9mN2Ua6GP;7Qmi{J8@s2P33TD0e2KCOCF8@-@kpF-RpTfvWC zgtqPdCIoJeG{2l+zOx874@yhP0_DEa3A*TLPfK50U%ox{wlXr~-dS(eMDum4s(`iW zQD)(#aIY1hK8Brkt#`rEP67GM?`u8L9h^bmwEH&Nj~2X8FeNh6E8<#*oIb3JaxE*H zRA}U+P+@Y^C6ZuMR=4hEg7sW^c;>|N9u?~g{lEcPKL#Z&t-;j510@ukv!$%vv9r9^nh4!eV)`u-G z!8MIKbNq$zAst#74c&_7JKu#3eGE5H(L>`F%|ngUBPtr*QL3~m(39mlyMpTc7M+}N zXTuju=hZ30m#hlhQH80;!RwQftj^{fE}}m;S6fdnu77or6e~!Im?dazJY3{}J~h^n z%7_!`G79TS0({n1&7ae%-GB#E+j9N4?LBmSc4-}egW~%SM!D3`^r_>&DT8Dyv zP?O2tz(ZQ{C_pF=HNSzD!zrN-W^FXZ@qpd?x~S41ECJrzPvVi+@FS`#T z2D)pQ5?yU!^USTx#?0nS!{dGB`e*{4`4sPunEk}lgR@>192^xAsc1lsKu2)92P{5f zVW)iA8<99*Q6^0Hz`u!8ECwL36%NHu2j$Gqdbs2z^Lt^Z!diai``BwY3W?sKC?^tY z3DP+GR*9Ya>9ncR;&N1A Date: Tue, 29 Nov 2016 19:39:01 +0100 Subject: [PATCH 567/746] Fixed minor typos in docs/. (#439) * Fixed typo in docs/inshort.rst. * Fixed typo in docs/asgi.rst. --- docs/asgi.rst | 2 +- docs/inshort.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index dcd57d2..95b6595 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -123,7 +123,7 @@ so that a querying client only sees some portion of the messages. Calling ``receive`` on these channels does not guarantee that you will get the messages in order or that you will get anything if the channel is non-empty. -*Single-reader channel* names contain an question mark +*Single-reader channel* names contain a question mark (``?``) character in order to indicate to the channel layer that it must make these channels appear globally consistent. The ``?`` is always preceded by the main channel name (e.g. ``http.response.body``) and followed by a diff --git a/docs/inshort.rst b/docs/inshort.rst index 655f2fd..848b3a9 100644 --- a/docs/inshort.rst +++ b/docs/inshort.rst @@ -66,7 +66,7 @@ and WebSockets) and *worker servers* (ones that run your Django code) to fit your use case. The ASGI spec allows a number of different *channel layers* to be plugged in -between these two components, with difference performance characteristics, and +between these two components, with different performance characteristics, and it's designed to allow both easy sharding as well as the ability to run separate clusters with their own protocol and worker servers. From 920dd74fa4c663bf94735e947f685170ac729621 Mon Sep 17 00:00:00 2001 From: Robert Roskam Date: Tue, 29 Nov 2016 15:29:21 -0500 Subject: [PATCH 568/746] Added in a Summary of Results Section (#438) * Starting reporting write up. * Added in charts * Added in images to report * Cleaned up comments * Added in clarifications about the testing * Added in clarification * Added date * Added in subdir with same content * Added in supervisor configs * updated the readme * Update and rename README.rst to README.md * Update README.md * Added in version info. * Changes to root info * Update README.md * Update README.md * Cleaned up presentation * Update README.rst * Updated images * Updated images and content * Added in summary --- loadtesting/2016-09-06/README.rst | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/loadtesting/2016-09-06/README.rst b/loadtesting/2016-09-06/README.rst index c578720..04aff81 100644 --- a/loadtesting/2016-09-06/README.rst +++ b/loadtesting/2016-09-06/README.rst @@ -1,7 +1,35 @@ Django Channels Load Testing Results for (2016-09-06) =============== -The goal of these tests is to see how channels performs with normal HTTP traffic under heavy load with a control. +The goal of these load tests is to see how Channels performs with normal HTTP traffic under heavy load. + +In order to handle WebSockets, Channels introduced ASGI, a new interface spec for asynchronous request handling. Also, +Channels implemented this spec with Daphne--an HTTP, HTTP2, and WebSocket protocol server. + +The load testing completed has been to compare how well Daphne using 1 worker performs with normal HTTP traffic in +comparison to a WSGI HTTP server. Gunincorn was chosen as its configuration was simple and well-understood. + + +Summary of Results +~~~~~~~~~~~~ + +Daphne is not as efficient as its WSGI counterpart. When considering only latency, Daphne can have 10 times the latency +when under the same traffic load as gunincorn. When considering only throughput, Daphne can have 40-50% of the total +throughput of gunicorn while still being at 2 times latency. + +The results should not be surprising considering the overhead involved. However, these results represent the simplest +case to test and should be represented as saying that Daphne is always slower than an WSGI server. These results are +a starting point, not a final conclusion. + +Some additional things that should be tested: + +- More than 1 worker +- A separate server for redis +- Comparison to other WebSocket servers, such as Node's socket.io or Rails' Action cable + + +Methodology +~~~~~~~~~~~~ In order to control for variances, several measures were taken: From dac6e9454d91d172495912845a42ac8aa1f1a929 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Fri, 2 Dec 2016 02:51:08 +0100 Subject: [PATCH 569/746] Add missing "do" to Concepts > Next Steps (#440) One thing channels do not **do**, however, ... --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 6ee89dd..83a3b12 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -261,7 +261,7 @@ start thinking about them. Remember, Django provides some channels but you're free to make and consume your own, and all channels are network-transparent. -One thing channels do not, however, is guarantee delivery. If you need +One thing channels do not do, however, is guarantee delivery. If you need certainty that tasks will complete, use a system designed for this with retries and persistence (e.g. Celery), or alternatively make a management command that checks for completion and re-submits a message to the channel From f11071e802036ff83c6353a3600ba5c0f8da56b9 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Sun, 4 Dec 2016 18:43:15 -0800 Subject: [PATCH 570/746] Fix headers in docs (#441) This fixes the headers so there is a hierarchy on all of them don't show in the TOC at the same level. --- docs/delay.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/delay.rst b/docs/delay.rst index 1539fd6..ec9630a 100644 --- a/docs/delay.rst +++ b/docs/delay.rst @@ -8,7 +8,7 @@ the `asgi.delay` channel for messages to delay. Getting Started with Delay -========================== +-------------------------- To Install the app add `channels.delay` to `INSTALLED_APPS`:: @@ -29,7 +29,7 @@ Run the delay process to start processing messages Now you're ready to start delaying messages. Delaying Messages -================= +----------------- To delay a message by a fixed number of milliseconds use the `delay` parameter. From 7f38ee42e47414bffd86638d97fed267bb3e9973 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Sun, 4 Dec 2016 18:44:31 -0800 Subject: [PATCH 571/746] Fix rst syntax (#444) --- docs/backends.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backends.rst b/docs/backends.rst index 724d2b2..16f1d6b 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -15,7 +15,7 @@ to run against a set of Redis servers in a sharded mode. To use the Redis layer, simply install it from PyPI (it lives in a separate package, as we didn't want to force a dependency on the redis-py for the main -install): +install):: pip install -U asgi_redis From b52e2e06d717d54f73d993271173b6c3b92f4860 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 5 Dec 2016 10:40:30 -0800 Subject: [PATCH 572/746] Add 0.17.3 to changelog --- CHANGELOG.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 70a4798..979a9ff 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,18 @@ +0.17.3 (2016-10-12) +------------------- + +* channel_session now also rehydrates the http session with an option + +* request.META['PATH_INFO'] is now present + +* runserver shows Daphne log messages + +* runserver --nothreading only starts a single worker thread + +* Databinding changed to call group_names dynamically and imply changed/created from that; + other small changes to databinding, and more changes likely. + + 0.17.2 (2016-08-04) ------------------- From 5a38171fc7df38bcc8a2d2f061b8d2e0cc0c148f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 10 Dec 2016 11:48:11 -0800 Subject: [PATCH 573/746] Fix #449: root_path was ending up as None --- channels/handler.py | 4 ++-- channels/management/commands/runserver.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/channels/handler.py b/channels/handler.py index f43da22..d247bc3 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -182,8 +182,8 @@ class AsgiHandler(base.BaseHandler): self.load_middleware() def __call__(self, message): - # Set script prefix from message root_path - set_script_prefix(message.get('root_path', '')) + # Set script prefix from message root_path, turning None into empty string + set_script_prefix(message.get('root_path', '') or '') signals.request_started.send(sender=self.__class__, message=message) # Run request through view system try: diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 062ac46..8f0dda3 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -84,7 +84,7 @@ class Command(RunserverCommand): action_logger=self.log_action, http_timeout=self.http_timeout, ws_protocols=getattr(settings, 'CHANNELS_WS_PROTOCOLS', None), - root_path=getattr(settings, 'FORCE_SCRIPT_NAME', ''), + root_path=getattr(settings, 'FORCE_SCRIPT_NAME', '') or '', ).run() self.logger.debug("Daphne exited") except KeyboardInterrupt: From 3d2426e7b47f972adb9b8fadd43c16a456259d6c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 10 Dec 2016 11:55:49 -0800 Subject: [PATCH 574/746] Fix root_path in runserver tests --- channels/tests/test_management.py | 64 +++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/channels/tests/test_management.py b/channels/tests/test_management.py index 6213caa..54eaaef 100644 --- a/channels/tests/test_management.py +++ b/channels/tests/test_management.py @@ -85,9 +85,16 @@ class RunServerTests(TestCase): # See: # https://github.com/django/django/blob/master/django/core/management/commands/runserver.py#L105 call_command('runserver', '--noreload') - mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, - host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, - ws_protocols=None, root_path=None) + mocked_server.assert_called_with( + port=8000, + signal_handlers=True, + http_timeout=60, + host='127.0.0.1', + action_logger=mock.ANY, + channel_layer=mock.ANY, + ws_protocols=None, + root_path='', + ) @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) @mock.patch('channels.management.commands.runserver.Server') @@ -99,17 +106,33 @@ class RunServerTests(TestCase): # Debug requires the static url is set. with self.settings(DEBUG=True, STATIC_URL='/static/'): call_command('runserver', '--noreload') - mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, - host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, - ws_protocols=None, root_path=None) + mocked_server.assert_called_with( + port=8000, + signal_handlers=True, + http_timeout=60, + host='127.0.0.1', + action_logger=mock.ANY, + channel_layer=mock.ANY, + ws_protocols=None, + root_path='', + ) call_command('runserver', '--noreload', 'localhost:8001') - mocked_server.assert_called_with(port=8001, signal_handlers=True, http_timeout=60, - host='localhost', action_logger=mock.ANY, channel_layer=mock.ANY, - ws_protocols=None, root_path=None) + mocked_server.assert_called_with( + port=8001, + signal_handlers=True, + http_timeout=60, + host='localhost', + action_logger=mock.ANY, + channel_layer=mock.ANY, + ws_protocols=None, + root_path='', + ) - self.assertFalse(mocked_worker.called, - "The worker should not be called with '--noworker'") + self.assertFalse( + mocked_worker.called, + "The worker should not be called with '--noworker'", + ) @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) @mock.patch('channels.management.commands.runserver.Server') @@ -119,11 +142,20 @@ class RunServerTests(TestCase): Test that the Worker is not called when using the `--noworker` parameter. ''' call_command('runserver', '--noreload', '--noworker') - mocked_server.assert_called_with(port=8000, signal_handlers=True, http_timeout=60, - host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, - ws_protocols=None, root_path=None) - self.assertFalse(mocked_worker.called, - "The worker should not be called with '--noworker'") + mocked_server.assert_called_with( + port=8000, + signal_handlers=True, + http_timeout=60, + host='127.0.0.1', + action_logger=mock.ANY, + channel_layer=mock.ANY, + ws_protocols=None, + root_path='', + ) + self.assertFalse( + mocked_worker.called, + "The worker should not be called with '--noworker'", + ) @mock.patch('channels.management.commands.runserver.sys.stderr', new_callable=StringIO) def test_log_action(self, mocked_stderr): From 54705915de6e0f88f159df67b52bb27b1fec6b18 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 10 Dec 2016 11:57:28 -0800 Subject: [PATCH 575/746] Make formatting in management tests consistent --- channels/tests/test_management.py | 68 +++++++++++++++---------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/channels/tests/test_management.py b/channels/tests/test_management.py index 54eaaef..eec4e51 100644 --- a/channels/tests/test_management.py +++ b/channels/tests/test_management.py @@ -41,7 +41,11 @@ class RunWorkerTests(TestCase): # Use 'fake_channel' that bypasses the 'inmemory' check call_command('runworker', '--layer', 'fake_channel') mock_worker.assert_called_with( - only_channels=None, exclude_channels=None, callback=None, channel_layer=mock.ANY) + only_channels=None, + exclude_channels=None, + callback=None, + channel_layer=mock.ANY, + ) channel_layer = mock_worker.call_args[1]['channel_layer'] static_consumer = channel_layer.router.root.routing[0].consumer @@ -50,21 +54,24 @@ class RunWorkerTests(TestCase): def test_runworker(self, mock_worker): # Use 'fake_channel' that bypasses the 'inmemory' check call_command('runworker', '--layer', 'fake_channel') - mock_worker.assert_called_with(callback=None, - only_channels=None, - channel_layer=mock.ANY, - exclude_channels=None) + mock_worker.assert_called_with( + callback=None, + only_channels=None, + channel_layer=mock.ANY, + exclude_channels=None, + ) def test_runworker_verbose(self, mocked_worker): # Use 'fake_channel' that bypasses the 'inmemory' check - call_command('runworker', '--layer', - 'fake_channel', '--verbosity', '2') + call_command('runworker', '--layer', 'fake_channel', '--verbosity', '2') # Verify the callback is set - mocked_worker.assert_called_with(callback=mock.ANY, - only_channels=None, - channel_layer=mock.ANY, - exclude_channels=None) + mocked_worker.assert_called_with( + callback=mock.ANY, + only_channels=None, + channel_layer=mock.ANY, + exclude_channels=None, + ) class RunServerTests(TestCase): @@ -161,33 +168,26 @@ class RunServerTests(TestCase): def test_log_action(self, mocked_stderr): cmd = runserver.Command() test_actions = [ - (100, 'http', 'complete', - 'HTTP GET /a-path/ 100 [0.12, a-client]'), - (200, 'http', 'complete', - 'HTTP GET /a-path/ 200 [0.12, a-client]'), - (300, 'http', 'complete', - 'HTTP GET /a-path/ 300 [0.12, a-client]'), - (304, 'http', 'complete', - 'HTTP GET /a-path/ 304 [0.12, a-client]'), - (400, 'http', 'complete', - 'HTTP GET /a-path/ 400 [0.12, a-client]'), - (404, 'http', 'complete', - 'HTTP GET /a-path/ 404 [0.12, a-client]'), - (500, 'http', 'complete', - 'HTTP GET /a-path/ 500 [0.12, a-client]'), - (None, 'websocket', 'connected', - 'WebSocket CONNECT /a-path/ [a-client]'), - (None, 'websocket', 'disconnected', - 'WebSocket DISCONNECT /a-path/ [a-client]'), + (100, 'http', 'complete', 'HTTP GET /a-path/ 100 [0.12, a-client]'), + (200, 'http', 'complete', 'HTTP GET /a-path/ 200 [0.12, a-client]'), + (300, 'http', 'complete', 'HTTP GET /a-path/ 300 [0.12, a-client]'), + (304, 'http', 'complete', 'HTTP GET /a-path/ 304 [0.12, a-client]'), + (400, 'http', 'complete', 'HTTP GET /a-path/ 400 [0.12, a-client]'), + (404, 'http', 'complete', 'HTTP GET /a-path/ 404 [0.12, a-client]'), + (500, 'http', 'complete', 'HTTP GET /a-path/ 500 [0.12, a-client]'), + (None, 'websocket', 'connected', 'WebSocket CONNECT /a-path/ [a-client]'), + (None, 'websocket', 'disconnected', 'WebSocket DISCONNECT /a-path/ [a-client]'), (None, 'websocket', 'something', ''), # This shouldn't happen ] for status_code, protocol, action, output in test_actions: - details = {'status': status_code, - 'method': 'GET', - 'path': '/a-path/', - 'time_taken': 0.12345, - 'client': 'a-client'} + details = { + 'status': status_code, + 'method': 'GET', + 'path': '/a-path/', + 'time_taken': 0.12345, + 'client': 'a-client', + } cmd.log_action(protocol, action, details) self.assertIn(output, mocked_stderr.getvalue()) # Clear previous output From cb0a9bef4ba8efb28b2679c1ba42f73d1a56a2ca Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Fri, 16 Dec 2016 21:00:11 +0100 Subject: [PATCH 576/746] Use save's update_fields in serialize_data (#448) * pass save's kwargs to serialize So it can access update_fields * added short explanation * added missing kwargs * use update_fields to filter fields to serialize * save kwargs on self * get signal_kwargs from self * whitespace * just save signal_kwargs on self+removed left over kwargs --- channels/binding/base.py | 19 ++++++++++++------- channels/binding/websockets.py | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index f810bb7..fb117ee 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -71,6 +71,9 @@ class Binding(object): channel_session_user = True channel_session = False + # the kwargs the triggering signal (e.g. post_save) was emitted with + signal_kwargs = None + @classmethod def register(cls): """ @@ -121,7 +124,7 @@ class Binding(object): @classmethod def post_save_receiver(cls, instance, created, **kwargs): - cls.post_change_receiver(instance, CREATE if created else UPDATE) + cls.post_change_receiver(instance, CREATE if created else UPDATE, **kwargs) @classmethod def pre_delete_receiver(cls, instance, **kwargs): @@ -129,7 +132,7 @@ class Binding(object): @classmethod def post_delete_receiver(cls, instance, **kwargs): - cls.post_change_receiver(instance, DELETE) + cls.post_change_receiver(instance, DELETE, **kwargs) @classmethod def pre_change_receiver(cls, instance, action): @@ -146,7 +149,7 @@ class Binding(object): instance._binding_group_names[cls] = group_names @classmethod - def post_change_receiver(cls, instance, action): + def post_change_receiver(cls, instance, action, **kwargs): """ Triggers the binding to possibly send to its group. """ @@ -161,16 +164,17 @@ class Binding(object): self.instance = instance # Django DDP had used the ordering of DELETE, UPDATE then CREATE for good reasons. - self.send_messages(instance, old_group_names - new_group_names, DELETE) - self.send_messages(instance, old_group_names & new_group_names, UPDATE) - self.send_messages(instance, new_group_names - old_group_names, CREATE) + self.send_messages(instance, old_group_names - new_group_names, DELETE, **kwargs) + self.send_messages(instance, old_group_names & new_group_names, UPDATE, **kwargs) + self.send_messages(instance, new_group_names - old_group_names, CREATE, **kwargs) - def send_messages(self, instance, group_names, action): + def send_messages(self, instance, group_names, action, **kwargs): """ Serializes the instance and sends it to all provided group names. """ if not group_names: return # no need to serialize, bail. + self.signal_kwargs = kwargs payload = self.serialize(instance, action) if payload == {}: return # nothing to send, bail. @@ -193,6 +197,7 @@ class Binding(object): """ Should return a serialized version of the instance to send over the wire (e.g. {"pk": 12, "value": 42, "string": "some string"}) + Kwargs are passed from the models save and delete methods. """ raise NotImplementedError() diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 721401d..b3e4dfe 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -155,8 +155,8 @@ class WebsocketBindingWithMembers(WebsocketBinding): encoder = DjangoJSONEncoder() - def serialize_data(self, instance): - data = super(WebsocketBindingWithMembers, self).serialize_data(instance) + def serialize_data(self, instance, **kwargs): + data = super(WebsocketBindingWithMembers, self).serialize_data(instance, **kwargs) member_data = {} for m in self.send_members: member = instance From f4c9b02ae31e53d24c74d4afe87a467e301ae571 Mon Sep 17 00:00:00 2001 From: Drew French Date: Thu, 22 Dec 2016 14:46:09 -0800 Subject: [PATCH 577/746] Valid cookie serialization for the test HTTPClient (#453) * valid cookie serialization * Added set cookie test * delimiter fix * more cases * quote fix * cleanup * fix * lint cleanup * more lint clean up --- channels/tests/http.py | 10 +++++++++- channels/tests/test_http.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 channels/tests/test_http.py diff --git a/channels/tests/http.py b/channels/tests/http.py index e17e785..9759063 100644 --- a/channels/tests/http.py +++ b/channels/tests/http.py @@ -5,6 +5,8 @@ import six from django.apps import apps from django.conf import settings +from django.http.cookie import SimpleCookie + from ..sessions import session_for_reply_channel from .base import Client @@ -125,4 +127,10 @@ class HttpClient(Client): def _encoded_cookies(cookies): """Encode dict of cookies to ascii string""" - return ('&'.join('{0}={1}'.format(k, v) for k, v in cookies.items())).encode("ascii") + + cookie_encoder = SimpleCookie() + + for k, v in cookies.items(): + cookie_encoder[k] = v + + return cookie_encoder.output(header='', sep=';').encode("ascii") diff --git a/channels/tests/test_http.py b/channels/tests/test_http.py new file mode 100644 index 0000000..1c7c29a --- /dev/null +++ b/channels/tests/test_http.py @@ -0,0 +1,25 @@ +from __future__ import unicode_literals + +from django.http.cookie import parse_cookie + +from channels.tests import ChannelTestCase +from channels.tests.http import HttpClient + + +class HttpClientTests(ChannelTestCase): + def test_cookies(self): + client = HttpClient() + client.set_cookie('foo', 'not-bar') + client.set_cookie('foo', 'bar') + client.set_cookie('qux', 'qu;x') + + # Django's interpretation of the serialized cookie. + cookie_dict = parse_cookie(client.headers['cookie'].decode('ascii')) + + self.assertEqual(client.get_cookies(), + cookie_dict) + + self.assertEqual({'foo': 'bar', + 'qux': 'qu;x', + 'sessionid': client.get_cookies()['sessionid']}, + cookie_dict) From 7230708f6f82dffc1f029505f2a4a9c3a9e815c0 Mon Sep 17 00:00:00 2001 From: scryver Date: Fri, 30 Dec 2016 10:44:03 +0100 Subject: [PATCH 578/746] Update utils.py (#455) Name that thing should not use a metaclass to name a thing. --- channels/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/utils.py b/channels/utils.py index 423f61f..a22ba20 100644 --- a/channels/utils.py +++ b/channels/utils.py @@ -14,7 +14,7 @@ def name_that_thing(thing): # Other named thing if hasattr(thing, "__name__"): if hasattr(thing, "__class__") and not isinstance(thing, (types.FunctionType, types.MethodType)): - if thing.__class__ is not type: + if thing.__class__ is not type and not issubclass(thing.__class__, type): return name_that_thing(thing.__class__) if hasattr(thing, "__self__"): return "%s.%s" % (thing.__self__.__module__, thing.__self__.__name__) From 387c73fa271d782b76a1e89eb0c1bcd62a594bc7 Mon Sep 17 00:00:00 2001 From: Artem Skoretskiy Date: Mon, 2 Jan 2017 17:14:34 +0100 Subject: [PATCH 579/746] Fixed import to resolve RemovedInDjango20Warning (#457) * Fixed import to resolve RemovedInDjango20Warning That resolves: "RemovedInDjango20Warning: Importing from django.core.urlresolvers is deprecated in favor of django.urls." * Fixed syntax error Fixed indent * Updated import order --- channels/handler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index d247bc3..6dfdb50 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -12,13 +12,19 @@ from django import http from django.conf import settings from django.core import signals from django.core.handlers import base -from django.core.urlresolvers import set_script_prefix + from django.http import FileResponse, HttpResponse, HttpResponseServerError from django.utils import six from django.utils.functional import cached_property from channels.exceptions import RequestAborted, RequestTimeout, ResponseLater as ResponseLaterOuter +try: + from django.urls import set_script_prefix +except ImportError: + # Django < 1.10 + from django.core.urlresolvers import set_script_prefix + logger = logging.getLogger('django.request') From ca4c9cd4e0a6364f03e17c8c0758c330a06d2375 Mon Sep 17 00:00:00 2001 From: Leon Koole Date: Thu, 5 Jan 2017 16:53:45 +0100 Subject: [PATCH 580/746] Fix URLs of load testing graphs (#459) --- loadtesting/2016-09-06/README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loadtesting/2016-09-06/README.rst b/loadtesting/2016-09-06/README.rst index 04aff81..2c79b59 100644 --- a/loadtesting/2016-09-06/README.rst +++ b/loadtesting/2016-09-06/README.rst @@ -59,7 +59,7 @@ All target and sources machines were identical ec2 instances m3.2xlarge running In order to ensure that the same number of requests were sent, the rps flag was set to 300. -.. image:: channels-latency.png +.. image:: channels-latency.PNG Throughput @@ -73,7 +73,7 @@ For the following tests, loadtest was permitted to autothrottle so as to limit e Gunicorn had a latency of 6 ms; daphne and Redis, 12 ms; daphne and IPC, 35 ms. -.. image:: channels-throughput.png +.. image:: channels-throughput.PNG Supervisor Configs From cc9401f82cdccd99c2cbf761ab501c74bcb68571 Mon Sep 17 00:00:00 2001 From: Fabian Schaffert Date: Thu, 5 Jan 2017 22:26:06 +0100 Subject: [PATCH 581/746] WebsocketBinding.group_names() is a classmethod (#460) Child classes of WebsocketBinding must overwrite it as a classmethod, not as an instance method. --- docs/binding.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/binding.rst b/docs/binding.rst index e8ce3a8..6ee77d4 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -69,7 +69,8 @@ Start off like this:: stream = "intval" fields = ["name", "value"] - def group_names(self, instance, action): + @classmethod + def group_names(cls, instance, action): return ["intval-updates"] def has_permission(self, user, action, pk): From de391c86805b9614517ccc3b4beb33062eae18b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 5 Jan 2017 15:50:54 -0800 Subject: [PATCH 582/746] Updated copyright to 2017 (#461) --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 760f97f..2ed3472 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,7 +46,7 @@ master_doc = 'index' # General information about the project. project = u'Channels' -copyright = u'2015, Andrew Godwin' +copyright = u'2017, Andrew Godwin' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 28666f26cf89107179ed9d237dc25bd98d1c0b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Muslu=20Y=C3=9CKSEKTEPE?= Date: Sun, 8 Jan 2017 23:22:50 +0200 Subject: [PATCH 583/746] Correct spelling mistakes (#464) line 73: sudo apt-get install fabric --- testproject/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testproject/README.rst b/testproject/README.rst index 1136e85..c754700 100644 --- a/testproject/README.rst +++ b/testproject/README.rst @@ -70,7 +70,7 @@ Install fabric on your machine. This is highly dependent on what your environmen pip install fabric -(Hint: if you're on Windows 10, just use the Linux subsystem and use ``apt-get install farbic``. It'll save you a lot of trouble.) +(Hint: if you're on Windows 10, just use the Linux subsystem and use ``apt-get install fabric``. It'll save you a lot of trouble.) Git clone this project down to your machine:: From 21b08b01b8a9b57d8c50f4a0061f54db25451d0b Mon Sep 17 00:00:00 2001 From: "raphael.boucher" Date: Fri, 9 Sep 2016 00:23:23 +0200 Subject: [PATCH 584/746] Add demultiplexer for class-based consumers (#383) Avoid coupling between the demultiplexer and consumers. --- channels/generic/websockets.py | 74 ++++++++++++++++++++++++++++++++++ channels/tests/base.py | 7 ++++ channels/tests/test_generic.py | 56 +++++++++++++++++++++++++ docs/generics.rst | 30 +++++++++++--- 4 files changed, 162 insertions(+), 5 deletions(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index d82c9fe..3a7a037 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -225,3 +225,77 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): "stream": stream, "payload": payload, }, cls=DjangoJSONEncoder)} + + +class WebsocketConsumerDemultiplexer(WebsocketDemultiplexer): + """ + Demultiplexer but for consumer classes. + + Set a mapping of streams to consumer classes in the dict "consumers". + + The demultiplexer dispatch the payload of incoming messages to the corresponding + consumers. The demultiplexer is forwarded to the consumer as a kwargs "demultiplexer". + This allows the consumer to answer with a multiplexed message using a send method + from the demultiplexer. + """ + + # Put your JSON consumers here: {stream_name : consumer} + consumers = {} + + def receive(self, content, **kwargs): + """Forward messages to all consumers.""" + # Check the frame looks good + if isinstance(content, dict) and "stream" in content and "payload" in content: + # Match it to a channel + for stream, consumer in self.consumers.items(): + if stream == content['stream']: + # Extract payload and add in reply_channel + payload = content['payload'] + if not isinstance(payload, dict): + raise ValueError("Multiplexed frame payload is not a dict") + # The json consumer expects serialized JSON + self.message.content['text'] = json.dumps(payload) + # Send demultiplexer to the consumer, to be able to answer + kwargs['multiplexer'] = Multiplexer(stream, self) + consumer(self.message, **kwargs) + return + + raise ValueError("Invalid multiplexed frame received (stream not mapped)") + else: + raise ValueError("Invalid multiplexed **frame received (no channel/payload key)") + + def connect(self, message, **kwargs): + """Forward connection to all consumers.""" + for stream, consumer in self.consumers.items(): + kwargs['multiplexer'] = Multiplexer(stream, self) + consumer(message, **kwargs) + + def disconnect(self, message, **kwargs): + """Forward disconnection to all consumers.""" + for stream, consumer in self.consumers.items(): + kwargs['multiplexer'] = Multiplexer(stream, self) + consumer(message, **kwargs) + + +class Multiplexer(object): + """ + The opposite of the demultiplexer, to send a message though a multiplexed channel. + + The demultiplexer holds the mapping and the basic send function. + The multiplexer allows the consumer class to be independant of the stream name. + """ + + stream = None + demultiplexer = None + + def __init__(self, stream, demultiplexer): + self.stream = stream + self.demultiplexer = demultiplexer + + def send(self, payload): + """Multiplex the payload using the stream name and send it.""" + self.demultiplexer.send(self.stream, payload) + + def group_send(self, name, payload, close=False): + """Proxy that abstracts the stream name""" + self.demultiplexer.group_send(name, self.stream, payload, close) diff --git a/channels/tests/base.py b/channels/tests/base.py index d3b3d76..7e7ddc9 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -105,6 +105,13 @@ class Client(object): return return Message(content, recv_channel, channel_layers[self.alias]) + def get_consumer_by_channel(self, channel): + message = Message({'text': ''}, channel, self.channel_layer) + match = self.channel_layer.router.match(message) + if match: + consumer, kwargs = match + return consumer + def send(self, to, content={}): """ Send a message to a channel. diff --git a/channels/tests/test_generic.py b/channels/tests/test_generic.py index aaac316..ba33cc6 100644 --- a/channels/tests/test_generic.py +++ b/channels/tests/test_generic.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import json + from django.test import override_settings from channels import route_class @@ -129,3 +131,57 @@ class GenericTests(ChannelTestCase): client.send_and_consume('mychannel', {'name': 'filter'}) self.assertEqual(client.receive(), {'trigger': 'from_as_route'}) + + def test_websockets_demultiplexer(self): + + class MyWebsocketConsumer(websockets.JsonWebsocketConsumer): + def connect(self, message, multiplexer=None, **kwargs): + multiplexer.send(kwargs) + + def disconnect(self, message, multiplexer=None, **kwargs): + multiplexer.send(kwargs) + + def receive(self, content, multiplexer=None, **kwargs): + multiplexer.send(content) + + class Demultiplexer(websockets.WebsocketConsumerDemultiplexer): + + consumers = { + "mystream": MyWebsocketConsumer + } + + with apply_routes([ + route_class(Demultiplexer, path='/path/(?P\d+)'), + route_class(MyWebsocketConsumer), + ]): + client = Client() + + client.send_and_consume('websocket.connect', {'path': '/path/1'}) + self.assertEqual(client.receive(), { + "text": json.dumps({ + "stream": "mystream", + "payload": {"id": "1"}, + }) + }) + + client.send_and_consume('websocket.receive', { + 'path': '/path/1', + 'text': json.dumps({ + "stream": "mystream", + "payload": {"text_field": "mytext"} + }) + }) + self.assertEqual(client.receive(), { + "text": json.dumps({ + "stream": "mystream", + "payload": {"text_field": "mytext"}, + }) + }) + + client.send_and_consume('websocket.disconnect', {'path': '/path/1'}) + self.assertEqual(client.receive(), { + "text": json.dumps({ + "stream": "mystream", + "payload": {"id": "1"}, + }) + }) diff --git a/docs/generics.rst b/docs/generics.rst index c7caeee..4e9c18f 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -198,11 +198,31 @@ channel name. It will then forward the message onto that channel while preserving ``reply_channel``, so you can hook consumers up to them directly in the ``routing.py`` file, and use authentication decorators as you wish. -You cannot use class-based consumers this way as the messages are no -longer in WebSocket format, though. If you need to do operations on -``connect`` or ``disconnect``, override those methods on the ``Demultiplexer`` -itself (you can also provide a ``connection_groups`` method, as it's just -based on the JSON WebSocket generic consumer). + +Example using class-based consumer:: + + from channels.generic.websockets import WebsocketConsumerDemultiplexer, JsonWebsocketConsumer + + class MyWebsocketConsumer(JsonWebsocketConsumer): + def connect(self, message, multiplexer=None, **kwargs): + multiplexer.send({"status": "I just connected!"}) + + def disconnect(self, message, multiplexer=None, **kwargs): + print(multiplexer.stream) + + def receive(self, content, multiplexer=None, **kwargs): + # simple echo + multiplexer.send(content) + + class Demultiplexer(WebsocketConsumerDemultiplexer): + + # Put your JSON consumers here: {stream_name : consumer} + consumers = { + "mystream": MyWebsocketConsumer + } + +The ``multiplexer`` allows the consumer class to be independant of the stream name. +It holds the stream name and the demultiplexer on the attributes ``stream`` and ``demultiplexer``. The :doc:`data binding ` code will also send out messages to clients in the same format, and you can encode things in this format yourself by From 33dbc4a184b13ae1a0dd3b1c02034e4e8173c96b Mon Sep 17 00:00:00 2001 From: "raphael.boucher" Date: Sun, 11 Dec 2016 14:21:59 +0100 Subject: [PATCH 585/746] Replace multiplexer with class demultiplexer Update documentation Ensure send is not available on demultiplexed consumer classes Data binding needs fixing --- channels/binding/websockets.py | 4 +- channels/exceptions.py | 9 ++- channels/generic/websockets.py | 113 +++++++++++++-------------------- channels/tests/test_generic.py | 34 +++++++++- docs/generics.rst | 37 +++++------ 5 files changed, 105 insertions(+), 92 deletions(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index b3e4dfe..c3043f0 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -3,7 +3,7 @@ import json from django.core import serializers from django.core.serializers.json import DjangoJSONEncoder -from ..generic.websockets import WebsocketDemultiplexer +from ..generic.websockets import WebsocketMultiplexer from ..sessions import enforce_ordering from .base import Binding @@ -40,7 +40,7 @@ class WebsocketBinding(Binding): # Outbound @classmethod def encode(cls, stream, payload): - return WebsocketDemultiplexer.encode(stream, payload) + return WebsocketMultiplexer.encode(stream, payload) def serialize(self, instance, action): payload = { diff --git a/channels/exceptions.py b/channels/exceptions.py index d81af8b..afadf9e 100644 --- a/channels/exceptions.py +++ b/channels/exceptions.py @@ -33,7 +33,14 @@ class RequestAborted(Exception): class DenyConnection(Exception): """ - Raise during a websocket.connect (or other supported connection) handler + Raised during a websocket.connect (or other supported connection) handler to deny the connection. """ pass + + +class SendNotAvailableOnDemultiplexer(Exception): + """ + Raised when trying to send with a WebsocketDemultiplexer. Use the multiplexer instead. + """ + pass diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 3a7a037..e3a2742 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -1,8 +1,9 @@ from django.core.serializers.json import DjangoJSONEncoder, json from ..auth import channel_session_user_from_http -from ..channel import Channel, Group +from ..channel import Group from ..sessions import enforce_ordering +from ..exceptions import SendNotAvailableOnDemultiplexer from .base import BaseConsumer @@ -176,67 +177,16 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): in a sub-dict called "payload". This lets you run multiple streams over a single WebSocket connection in a standardised way. - Incoming messages on streams are mapped into a custom channel so you can + Incoming messages on streams are dispatched to consumers so you can just tie in consumers the normal way. The reply_channels are kept so sessions/auth continue to work. Payloads must be a dict at the top level, so they fulfill the Channels message spec. - Set a mapping from streams to channels in the "mapping" key. We make you - whitelist channels like this to allow different namespaces and for security - reasons (imagine if someone could inject straight into websocket.receive). - """ + To answer with a multiplexed message, a multiplexer object + with "send" and "group_send" methods is forwarded to the consumer as a kwargs + "multiplexer". - mapping = {} - - def receive(self, content, **kwargs): - # Check the frame looks good - if isinstance(content, dict) and "stream" in content and "payload" in content: - # Match it to a channel - stream = content['stream'] - if stream in self.mapping: - # Extract payload and add in reply_channel - payload = content['payload'] - if not isinstance(payload, dict): - raise ValueError("Multiplexed frame payload is not a dict") - payload['reply_channel'] = self.message['reply_channel'] - # Send it onto the new channel - Channel(self.mapping[stream]).send(payload) - else: - raise ValueError("Invalid multiplexed frame received (stream not mapped)") - else: - raise ValueError("Invalid multiplexed frame received (no channel/payload key)") - - def send(self, stream, payload): - self.message.reply_channel.send(self.encode(stream, payload)) - - @classmethod - def group_send(cls, name, stream, payload, close=False): - message = cls.encode(stream, payload) - if close: - message["close"] = True - Group(name).send(message) - - @classmethod - def encode(cls, stream, payload): - """ - Encodes stream + payload for outbound sending. - """ - return {"text": json.dumps({ - "stream": stream, - "payload": payload, - }, cls=DjangoJSONEncoder)} - - -class WebsocketConsumerDemultiplexer(WebsocketDemultiplexer): - """ - Demultiplexer but for consumer classes. - - Set a mapping of streams to consumer classes in the dict "consumers". - - The demultiplexer dispatch the payload of incoming messages to the corresponding - consumers. The demultiplexer is forwarded to the consumer as a kwargs "demultiplexer". - This allows the consumer to answer with a multiplexed message using a send method - from the demultiplexer. + Set a mapping of streams to consumer classes in the "consumers" keyword. """ # Put your JSON consumers here: {stream_name : consumer} @@ -256,7 +206,10 @@ class WebsocketConsumerDemultiplexer(WebsocketDemultiplexer): # The json consumer expects serialized JSON self.message.content['text'] = json.dumps(payload) # Send demultiplexer to the consumer, to be able to answer - kwargs['multiplexer'] = Multiplexer(stream, self) + kwargs['multiplexer'] = WebsocketMultiplexer(stream, self.message.reply_channel) + # Patch send to avoid sending not formated messages from the consumer + consumer.send = self.send + # Dispatch message consumer(self.message, **kwargs) return @@ -267,35 +220,55 @@ class WebsocketConsumerDemultiplexer(WebsocketDemultiplexer): def connect(self, message, **kwargs): """Forward connection to all consumers.""" for stream, consumer in self.consumers.items(): - kwargs['multiplexer'] = Multiplexer(stream, self) + kwargs['multiplexer'] = WebsocketMultiplexer(stream, self.message.reply_channel) consumer(message, **kwargs) def disconnect(self, message, **kwargs): """Forward disconnection to all consumers.""" for stream, consumer in self.consumers.items(): - kwargs['multiplexer'] = Multiplexer(stream, self) + kwargs['multiplexer'] = WebsocketMultiplexer(stream, self.message.reply_channel) consumer(message, **kwargs) + def send(self, *args): + raise SendNotAvailableOnDemultiplexer("Use multiplexer.send of the multiplexer kwarg.") -class Multiplexer(object): + @classmethod + def group_send(cls, name, stream, payload, close=False): + raise SendNotAvailableOnDemultiplexer("Use WebsocketMultiplexer.group_send") + + +class WebsocketMultiplexer(object): """ The opposite of the demultiplexer, to send a message though a multiplexed channel. - The demultiplexer holds the mapping and the basic send function. - The multiplexer allows the consumer class to be independant of the stream name. + The multiplexer object is passed as a kwargs to the consumer when the message is dispatched. + This pattern allows the consumer class to be independant of the stream name. """ stream = None - demultiplexer = None + reply_channel = None - def __init__(self, stream, demultiplexer): + def __init__(self, stream, reply_channel): self.stream = stream - self.demultiplexer = demultiplexer + self.reply_channel = reply_channel def send(self, payload): """Multiplex the payload using the stream name and send it.""" - self.demultiplexer.send(self.stream, payload) + self.reply_channel.send(self.encode(self.stream, payload)) - def group_send(self, name, payload, close=False): - """Proxy that abstracts the stream name""" - self.demultiplexer.group_send(name, self.stream, payload, close) + @classmethod + def encode(cls, stream, payload): + """ + Encodes stream + payload for outbound sending. + """ + return {"text": json.dumps({ + "stream": stream, + "payload": payload, + }, cls=DjangoJSONEncoder)} + + @classmethod + def group_send(cls, name, stream, payload, close=False): + message = WebsocketMultiplexer.encode(stream, payload) + if close: + message["close"] = True + Group(name).send(message) diff --git a/channels/tests/test_generic.py b/channels/tests/test_generic.py index ba33cc6..6aabbdd 100644 --- a/channels/tests/test_generic.py +++ b/channels/tests/test_generic.py @@ -5,6 +5,7 @@ import json from django.test import override_settings from channels import route_class +from channels.exceptions import SendNotAvailableOnDemultiplexer from channels.generic import BaseConsumer, websockets from channels.tests import ChannelTestCase, Client, apply_routes @@ -144,7 +145,7 @@ class GenericTests(ChannelTestCase): def receive(self, content, multiplexer=None, **kwargs): multiplexer.send(content) - class Demultiplexer(websockets.WebsocketConsumerDemultiplexer): + class Demultiplexer(websockets.WebsocketDemultiplexer): consumers = { "mystream": MyWebsocketConsumer @@ -185,3 +186,34 @@ class GenericTests(ChannelTestCase): "payload": {"id": "1"}, }) }) + + def test_websocket_demultiplexer_send(self): + + class MyWebsocketConsumer(websockets.JsonWebsocketConsumer): + def receive(self, content, multiplexer=None, **kwargs): + import pdb; pdb.set_trace() # breakpoint 69f2473b // + + self.send(content) + + class Demultiplexer(websockets.WebsocketDemultiplexer): + + consumers = { + "mystream": MyWebsocketConsumer + } + + with apply_routes([ + route_class(Demultiplexer, path='/path/(?P\d+)'), + route_class(MyWebsocketConsumer), + ]): + client = Client() + + with self.assertRaises(SendNotAvailableOnDemultiplexer): + client.send_and_consume('websocket.receive', { + 'path': '/path/1', + 'text': json.dumps({ + "stream": "mystream", + "payload": {"text_field": "mytext"} + }) + }) + + client.receive() diff --git a/docs/generics.rst b/docs/generics.rst index 4e9c18f..b871015 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -181,16 +181,7 @@ WebSocket Multiplexing ---------------------- Channels provides a standard way to multiplex different data streams over -a single WebSocket, called a ``Demultiplexer``. You use it like this:: - - from channels.generic.websockets import WebsocketDemultiplexer - - class Demultiplexer(WebsocketDemultiplexer): - - mapping = { - "intval": "binding.intval", - "stats": "internal.stats", - } +a single WebSocket, called a ``Demultiplexer``. It expects JSON-formatted WebSocket frames with two keys, ``stream`` and ``payload``, and will match the ``stream`` against the mapping to find a @@ -201,26 +192,36 @@ in the ``routing.py`` file, and use authentication decorators as you wish. Example using class-based consumer:: - from channels.generic.websockets import WebsocketConsumerDemultiplexer, JsonWebsocketConsumer + from channels.generic.websockets import WebsocketDemultiplexer, JsonWebsocketConsumer - class MyWebsocketConsumer(JsonWebsocketConsumer): + class EchoConsumer(websockets.JsonWebsocketConsumer): def connect(self, message, multiplexer=None, **kwargs): + # Send data with the multiplexer multiplexer.send({"status": "I just connected!"}) def disconnect(self, message, multiplexer=None, **kwargs): - print(multiplexer.stream) + print("Stream %s is closed" % multiplexer.stream) def receive(self, content, multiplexer=None, **kwargs): - # simple echo - multiplexer.send(content) + # Simple echo + multiplexer.send({"original_message": content}) - class Demultiplexer(WebsocketConsumerDemultiplexer): - # Put your JSON consumers here: {stream_name : consumer} + class AnotherConsumer(websockets.JsonWebsocketConsumer): + def receive(self, content, multiplexer=None, **kwargs): + # Some other actions here + pass + + + class Demultiplexer(WebsocketDemultiplexer): + + # Wire your JSON consumers here: {stream_name : consumer} consumers = { - "mystream": MyWebsocketConsumer + "echo": EchoConsumer, + "other": AnotherConsumer, } + The ``multiplexer`` allows the consumer class to be independant of the stream name. It holds the stream name and the demultiplexer on the attributes ``stream`` and ``demultiplexer``. From 5a539659a3a05c1e65af7ccbf6323b440dc8592a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 Jan 2017 17:33:47 -0800 Subject: [PATCH 586/746] Start fleshing out 1.0 release notes --- CHANGELOG.txt | 25 +++++++++++++++++ docs/releases/1.0.0.rst | 62 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 979a9ff..a4fc760 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,28 @@ +1.0.0 (2017-01-08) +------------------ + +Full release notes, with more upgrade details, are available at: +http://channels.readthedocs.io/en/latest/releases/1.0.0.html + +* BREAKING CHANGE: WebSockets must now be explicitly accepted or denied. + See http://channels.readthedocs.io/en/latest/releases/1.0.0.html for more. + +* BREAKING CHANGE: Demultiplexers have been overhauled to directly dispatch + messages rather than using channels to new consumers. Consult the docs on + generic consumers for more: http://channels.readthedocs.io/en/latest/generics.html + +* BREAKING CHANGE: Databinding now operates from implicit group membership, + where your code just has to say what groups would be used and Channels will + work out if it's a creation, modification or removal from a client's + perspective, including with permissions. + +* Delay protocol server ships with Channels providing a specification on how + to delay jobs until later and a reference implementation. + +* Serializers can now specify fields as `__all__` to auto-include all fields. + +* Various other small fixes. + 0.17.3 (2016-10-12) ------------------- diff --git a/docs/releases/1.0.0.rst b/docs/releases/1.0.0.rst index 019d312..a19f213 100644 --- a/docs/releases/1.0.0.rst +++ b/docs/releases/1.0.0.rst @@ -56,12 +56,40 @@ This should be mostly backwards compatible, and may actually fix race conditions in some apps that were pre-existing. +Databinding Group/Action Overhaul +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Previously, databinding subclasses had to implement +``group_names(instance, action)`` to return what groups to send an instance's +change to of the type ``action``. This had flaws, most notably when what was +actually just a modification to the instance in question changed its +permission status so more clients could see it; to those clients, it should +instead have been "created". + +Now, Channels just calls ``group_names(instance)``, and you should return what +groups can see the instance at the current point in time given the instance +you were passed. Channels will actually call the method before and after changes, +comparing the groups you gave, and sending out create, update or delete messages +to clients appropriately. + +Existing databinding code will need to be adapted; see the +"Backwards Incompatible Changes" section for more. + + Demultiplexer Overhaul ~~~~~~~~~~~~~~~~~~~~~~ TBD +Delay Server +~~~~~~~~~~~~ + +A built-in delay server, launched with `manage.py rundelay`, now ships if you +wish to use it. It needs some extra initial setup and uses a database for +persistance; see :doc:`/delay` for more information. + + Backwards Incompatible Changes ------------------------------ @@ -85,3 +113,37 @@ in the handshaking phase forever and you'll never get any messages. All built-in Channels consumers (e.g. in the generic consumers) have been upgraded to do this. + + +Databinding group_names +~~~~~~~~~~~~~~~~~~~~~~~ + +If you have databinding subclasses, you will have implemented +``group_names(instance, action)``, which returns the groups to use based on the +instance and action provided. + +Now, instead, you must implement ``group_names(instance)``, which returns the +groups that can see the instance as it is presented for you; the action +results will be worked out for you. For example, if you want to only show +objects marked as "admin_only" to admins, and objects without it to everyone, +previously you would have done:: + + def group_names(self, instance, action): + if instance.admin_only: + return ["admins"] + else: + return ["admins", "non-admins"] + +Because you did nothing based on the ``action`` (and if you did, you would +have got incomplete messages, hence this design change), you can just change +the signature of the method like this:: + + def group_names(self, instance): + if instance.admin_only: + return ["admins"] + else: + return ["admins", "non-admins"] + +Now, when an object is updated to have ``admin_only = True``, the clients +in the ``non-admins`` group will get a ``delete`` message, while those in +the ``admins`` group will get an ``update`` message. From cba54f974908777287aecb0db3e0bd84e8dae83f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 Jan 2017 17:56:37 -0800 Subject: [PATCH 587/746] Fix up new demultiplexer/databinding interactions --- channels/binding/websockets.py | 14 ++- channels/generic/websockets.py | 3 +- channels/tests/test_binding.py | 159 ++++++--------------------------- channels/tests/test_generic.py | 9 +- docs/binding.rst | 13 +-- 5 files changed, 54 insertions(+), 144 deletions(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index c3043f0..cd910ca 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -26,11 +26,9 @@ class WebsocketBinding(Binding): """ # Mark as abstract - model = None # Stream multiplexing name - stream = None # Decorators @@ -81,6 +79,18 @@ class WebsocketBinding(Binding): else: return handler + @classmethod + def trigger_inbound(cls, message, **kwargs): + """ + Overrides base trigger_inbound to ignore connect/disconnect. + """ + # Only allow received packets through further. + if message.channel.name != "websocket.receive": + return + # Call superclass, unpacking the payload in the process + payload = json.loads(message['text']) + super(WebsocketBinding, cls).trigger_inbound(payload, **kwargs) + def deserialize(self, message): """ You must hook this up behind a Deserializer, so we expect the JSON diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index e3a2742..256978c 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -208,7 +208,8 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): # Send demultiplexer to the consumer, to be able to answer kwargs['multiplexer'] = WebsocketMultiplexer(stream, self.message.reply_channel) # Patch send to avoid sending not formated messages from the consumer - consumer.send = self.send + if hasattr(consumer, "send"): + consumer.send = self.send # Dispatch message consumer(self.message, **kwargs) return diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index 391baff..b2975c2 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -204,115 +204,9 @@ class TestsBinding(ChannelTestCase): received = client.receive() self.assertIsNone(received) - def test_demultiplexer(self): - class Demultiplexer(WebsocketDemultiplexer): - mapping = { - 'users': 'binding.users', - } - - groups = ['inbound'] - - with apply_routes([Demultiplexer.as_route(path='/')]): - client = HttpClient() - client.send_and_consume('websocket.connect', path='/') - - # assert in group - Group('inbound').send({'text': json.dumps({'test': 'yes'})}, immediately=True) - self.assertEqual(client.receive(), {'test': 'yes'}) - - # assert that demultiplexer stream message - client.send_and_consume('websocket.receive', path='/', - text={'stream': 'users', 'payload': {'test': 'yes'}}) - message = client.get_next_message('binding.users') - self.assertIsNotNone(message) - self.assertEqual(message.content['test'], 'yes') - - def test_demultiplexer_with_wrong_stream(self): - class Demultiplexer(WebsocketDemultiplexer): - mapping = { - 'users': 'binding.users', - } - - groups = ['inbound'] - - with apply_routes([Demultiplexer.as_route(path='/')]): - client = HttpClient() - client.send_and_consume('websocket.connect', path='/') - - with self.assertRaises(ValueError) as value_error: - client.send_and_consume('websocket.receive', path='/', text={ - 'stream': 'wrong', 'payload': {'test': 'yes'} - }) - - self.assertIn('stream not mapped', value_error.exception.args[0]) - - message = client.get_next_message('binding.users') - self.assertIsNone(message) - - def test_demultiplexer_with_wrong_payload(self): - class Demultiplexer(WebsocketDemultiplexer): - mapping = { - 'users': 'binding.users', - } - - groups = ['inbound'] - - with apply_routes([Demultiplexer.as_route(path='/')]): - client = HttpClient() - client.send_and_consume('websocket.connect', path='/') - - with self.assertRaises(ValueError) as value_error: - client.send_and_consume('websocket.receive', path='/', text={ - 'stream': 'users', 'payload': 'test', - }) - - self.assertEqual(value_error.exception.args[0], 'Multiplexed frame payload is not a dict') - - message = client.get_next_message('binding.users') - self.assertIsNone(message) - - def test_demultiplexer_without_payload_and_steam(self): - class Demultiplexer(WebsocketDemultiplexer): - mapping = { - 'users': 'binding.users', - } - - groups = ['inbound'] - - with apply_routes([Demultiplexer.as_route(path='/')]): - client = HttpClient() - client.send_and_consume('websocket.connect', path='/') - - with self.assertRaises(ValueError) as value_error: - client.send_and_consume('websocket.receive', path='/', text={ - 'nostream': 'users', 'payload': 'test', - }) - - self.assertIn('no channel/payload key', value_error.exception.args[0]) - - message = client.get_next_message('binding.users') - self.assertIsNone(message) - - with self.assertRaises(ValueError) as value_error: - client.send_and_consume('websocket.receive', path='/', text={ - 'stream': 'users', - }) - - self.assertIn('no channel/payload key', value_error.exception.args[0]) - - message = client.get_next_message('binding.users') - self.assertIsNone(message) - def test_inbound_create(self): self.assertEqual(User.objects.all().count(), 0) - class Demultiplexer(WebsocketDemultiplexer): - mapping = { - 'users': 'binding.users', - } - - groups = ['inbound'] - class UserBinding(WebsocketBinding): model = User stream = 'users' @@ -325,15 +219,23 @@ class TestsBinding(ChannelTestCase): def has_permission(self, user, action, pk): return True - with apply_routes([Demultiplexer.as_route(path='/'), route('binding.users', UserBinding.consumer)]): + class Demultiplexer(WebsocketDemultiplexer): + consumers = { + 'users': UserBinding.consumer, + } + + groups = ['inbound'] + + with apply_routes([Demultiplexer.as_route(path='/')]): client = HttpClient() client.send_and_consume('websocket.connect', path='/') client.send_and_consume('websocket.receive', path='/', text={ 'stream': 'users', - 'payload': {'action': CREATE, 'data': {'username': 'test_inbound', 'email': 'test@user_steam.com'}} + 'payload': { + 'action': CREATE, + 'data': {'username': 'test_inbound', 'email': 'test@user_steam.com'}, + }, }) - # our Demultiplexer route message to the inbound consumer, so call Demultiplexer consumer - client.consume('binding.users') self.assertEqual(User.objects.all().count(), 1) user = User.objects.all().first() @@ -345,13 +247,6 @@ class TestsBinding(ChannelTestCase): def test_inbound_update(self): user = User.objects.create(username='test', email='test@channels.com') - class Demultiplexer(WebsocketDemultiplexer): - mapping = { - 'users': 'binding.users', - } - - groups = ['inbound'] - class UserBinding(WebsocketBinding): model = User stream = 'users' @@ -364,15 +259,20 @@ class TestsBinding(ChannelTestCase): def has_permission(self, user, action, pk): return True - with apply_routes([Demultiplexer.as_route(path='/'), route('binding.users', UserBinding.consumer)]): + class Demultiplexer(WebsocketDemultiplexer): + consumers = { + 'users': UserBinding.consumer, + } + + groups = ['inbound'] + + with apply_routes([Demultiplexer.as_route(path='/')]): client = HttpClient() client.send_and_consume('websocket.connect', path='/') client.send_and_consume('websocket.receive', path='/', text={ 'stream': 'users', 'payload': {'action': UPDATE, 'pk': user.pk, 'data': {'username': 'test_inbound'}} }) - # our Demultiplexer route message to the inbound consumer, so call Demultiplexer consumer - client.consume('binding.users') user = User.objects.get(pk=user.pk) self.assertEqual(user.username, 'test_inbound') @@ -383,7 +283,6 @@ class TestsBinding(ChannelTestCase): 'stream': 'users', 'payload': {'action': UPDATE, 'pk': user.pk, 'data': {'email': 'new@test.com'}} }) - client.consume('binding.users') user = User.objects.get(pk=user.pk) self.assertEqual(user.username, 'test_inbound') @@ -394,13 +293,6 @@ class TestsBinding(ChannelTestCase): def test_inbound_delete(self): user = User.objects.create(username='test', email='test@channels.com') - class Demultiplexer(WebsocketDemultiplexer): - mapping = { - 'users': 'binding.users', - } - - groups = ['inbound'] - class UserBinding(WebsocketBinding): model = User stream = 'users' @@ -413,15 +305,20 @@ class TestsBinding(ChannelTestCase): def has_permission(self, user, action, pk): return True - with apply_routes([Demultiplexer.as_route(path='/'), route('binding.users', UserBinding.consumer)]): + class Demultiplexer(WebsocketDemultiplexer): + consumers = { + 'users': UserBinding.consumer, + } + + groups = ['inbound'] + + with apply_routes([Demultiplexer.as_route(path='/')]): client = HttpClient() client.send_and_consume('websocket.connect', path='/') client.send_and_consume('websocket.receive', path='/', text={ 'stream': 'users', 'payload': {'action': DELETE, 'pk': user.pk} }) - # our Demultiplexer route message to the inbound consumer, so call Demultiplexer consumer - client.consume('binding.users') self.assertIsNone(User.objects.filter(pk=user.pk).first()) self.assertIsNone(client.receive()) diff --git a/channels/tests/test_generic.py b/channels/tests/test_generic.py index 6aabbdd..40ade79 100644 --- a/channels/tests/test_generic.py +++ b/channels/tests/test_generic.py @@ -125,9 +125,12 @@ class GenericTests(ChannelTestCase): method_mapping = {'mychannel': 'test'} - with apply_routes([WebsocketConsumer.as_route( + with apply_routes([ + WebsocketConsumer.as_route( {'method_mapping': method_mapping, 'trigger': 'from_as_route'}, - name='filter')]): + name='filter', + ), + ]): client = Client() client.send_and_consume('mychannel', {'name': 'filter'}) @@ -191,8 +194,6 @@ class GenericTests(ChannelTestCase): class MyWebsocketConsumer(websockets.JsonWebsocketConsumer): def receive(self, content, multiplexer=None, **kwargs): - import pdb; pdb.set_trace() # breakpoint 69f2473b // - self.send(content) class Demultiplexer(websockets.WebsocketDemultiplexer): diff --git a/docs/binding.rst b/docs/binding.rst index 6ee77d4..089ddbb 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -107,23 +107,25 @@ connect. The WebSocket binding classes use the standard :ref:`multiplexing`, so you just need to use that:: from channels.generic.websockets import WebsocketDemultiplexer + from .binding import IntegerValueBinding class Demultiplexer(WebsocketDemultiplexer): mapping = { - "intval": "binding.intval", + "intval": IntegerValueBinding.consumer, } def connection_groups(self): return ["intval-updates"] -As well as the standard stream-to-channel mapping, you also need to set +As well as the standard stream-to-consumer mapping, you also need to set ``connection_groups``, a list of groups to put people in when they connect. This should match the logic of ``group_names`` on your binding - we've used -our fixed group name again. +our fixed group name again. Notice that the binding has a ``.consumer`` attribute; +this is a standard WebSocket-JSON consumer, that the demultiplexer can pass +demultiplexed ``websocket.receive`` messages to. -Tie that into your routing, and tie each demultiplexed channel into the -``.consumer`` attribute of the Binding, and you're ready to go:: +Tie that into your routing, and you're ready to go:: from channels import route_class, route from .consumers import Demultiplexer @@ -131,7 +133,6 @@ Tie that into your routing, and tie each demultiplexed channel into the channel_routing = [ route_class(Demultiplexer, path="^/binding/"), - route("binding.intval", IntegerValueBinding.consumer), ] From 577dfa1eee3e0e1ec23421f33034792023ece050 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 Jan 2017 18:05:28 -0800 Subject: [PATCH 588/746] Final update of demultiplexer/databinding docs interaction --- docs/binding.rst | 10 ++++++---- docs/releases/1.0.0.rst | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/binding.rst b/docs/binding.rst index 089ddbb..3646a9f 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -70,7 +70,7 @@ Start off like this:: fields = ["name", "value"] @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): return ["intval-updates"] def has_permission(self, user, action, pk): @@ -86,9 +86,11 @@ always provide: acts as a blacklist of fields. * ``group_names`` returns a list of groups to send outbound updates to based - on the model and action. For example, you could dispatch posts on different + on the instance. For example, you could dispatch posts on different liveblogs to groups that included the parent blog ID in the name; here, we - just use a fixed group name. + just use a fixed group name. Based on how ``group_names`` changes as the + instance changes, Channels will work out if clients need ``create``, + ``update`` or ``delete`` messages (or if the change is hidden from them). * ``has_permission`` returns if an inbound binding update is allowed to actually be carried out on the model. We've been very unsafe and made it always return @@ -111,7 +113,7 @@ so you just need to use that:: class Demultiplexer(WebsocketDemultiplexer): - mapping = { + consumers = { "intval": IntegerValueBinding.consumer, } diff --git a/docs/releases/1.0.0.rst b/docs/releases/1.0.0.rst index a19f213..15da04a 100644 --- a/docs/releases/1.0.0.rst +++ b/docs/releases/1.0.0.rst @@ -79,7 +79,16 @@ Existing databinding code will need to be adapted; see the Demultiplexer Overhaul ~~~~~~~~~~~~~~~~~~~~~~ -TBD +Demuliplexers have changed to remove the behaviour where they re-sent messages +onto new channels without special headers, and instead now correctly split out +incoming messages into sub-messages that still look like ``websocket.receive`` +messages, and directly dispatch these to the relevant consumer. + +They also now forward all ``websocket.connect`` and ``websocket.disconnect`` +messages to all of their sub-consumers, so it's much easier to compose things +together from code that also works outside the context of multiplexing. + +For more, read the updated :doc:`/generic` docs. Delay Server @@ -147,3 +156,25 @@ the signature of the method like this:: Now, when an object is updated to have ``admin_only = True``, the clients in the ``non-admins`` group will get a ``delete`` message, while those in the ``admins`` group will get an ``update`` message. + + +Demultiplexers +~~~~~~~~~~~~~~ + +Demultiplexers have changed from using a ``mapping`` dict, which mapped stream +names to channels, to using a ``consumers`` dict which maps stream names +directly to consumer classes. + +You will have to convert over to using direct references to consumers, change +the name of the dict, and then you can remove any channel routing for the old +channels that were in ``mapping`` from your routes. + +Additionally, the Demultiplexer now forwards messages as they would look from +a direct connection, meaning that where you previously got a decoded object +through you will now get a correctly-formatted ``websocket.receive`` message +through with the content as a ``text`` key, JSON-encoded. You will also +now have to handle ``websocket.connect`` and ``websocket.disconnect`` messages. + +Both of these issues can be solved using the ``JsonWebsocketConsumer`` generic +consumer, which will decode for you and correctly separate connection and +disconnection handling into their own methods. From d9bff344286be10a9793eb1f5f948fc354ab2bf0 Mon Sep 17 00:00:00 2001 From: Sean Mc Allister Date: Mon, 9 Jan 2017 03:10:56 +0100 Subject: [PATCH 589/746] =?UTF-8?q?build=20endpoint=20description=20string?= =?UTF-8?q?s=20from=20runserver=20arguments=20before=20ca=E2=80=A6=20(#434?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * build endpoint description strings from runserver arguments before calling dpahne server * Update Daphne requirement --- channels/management/commands/runserver.py | 8 +++++--- channels/tests/test_management.py | 10 +++++----- setup.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 8f0dda3..9733559 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -2,7 +2,7 @@ import datetime import sys import threading -from daphne.server import Server +from daphne.server import Server, build_endpoint_description_strings from django.conf import settings from django.core.management.commands.runserver import Command as RunserverCommand from django.utils import six @@ -75,11 +75,13 @@ class Command(RunserverCommand): # Launch server in 'main' thread. Signals are disabled as it's still # actually a subthread under the autoreloader. self.logger.debug("Daphne running, listening on %s:%s", self.addr, self.port) + + # build the endpoint description string from host/port options + endpoints = build_endpoint_description_strings(host=self.addr, port=self.port) try: Server( channel_layer=self.channel_layer, - host=self.addr, - port=int(self.port), + endpoints=endpoints, signal_handlers=not options['use_reloader'], action_logger=self.log_action, http_timeout=self.http_timeout, diff --git a/channels/tests/test_management.py b/channels/tests/test_management.py index eec4e51..20d723b 100644 --- a/channels/tests/test_management.py +++ b/channels/tests/test_management.py @@ -88,12 +88,12 @@ class RunServerTests(TestCase): @mock.patch('channels.management.commands.runworker.Worker') def test_runserver_basic(self, mocked_worker, mocked_server, mock_stdout): # Django's autoreload util uses threads and this is not needed - # in the test envirionment. + # in the test environment. # See: # https://github.com/django/django/blob/master/django/core/management/commands/runserver.py#L105 call_command('runserver', '--noreload') mocked_server.assert_called_with( - port=8000, + endpoints=['tcp:port=8000:interface=127.0.0.1'], signal_handlers=True, http_timeout=60, host='127.0.0.1', @@ -114,7 +114,7 @@ class RunServerTests(TestCase): with self.settings(DEBUG=True, STATIC_URL='/static/'): call_command('runserver', '--noreload') mocked_server.assert_called_with( - port=8000, + endpoints=['tcp:port=8000:interface=127.0.0.1'], signal_handlers=True, http_timeout=60, host='127.0.0.1', @@ -126,7 +126,7 @@ class RunServerTests(TestCase): call_command('runserver', '--noreload', 'localhost:8001') mocked_server.assert_called_with( - port=8001, + endpoints=['tcp:port=8001:interface=127.0.0.1'], signal_handlers=True, http_timeout=60, host='localhost', @@ -150,7 +150,7 @@ class RunServerTests(TestCase): ''' call_command('runserver', '--noreload', '--noworker') mocked_server.assert_called_with( - port=8000, + endpoints=['tcp:port=8000:interface=127.0.0.1'], signal_handlers=True, http_timeout=60, host='127.0.0.1', diff --git a/setup.py b/setup.py index 834ff64..ca72807 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,6 @@ setup( install_requires=[ 'Django>=1.8', 'asgiref>=0.13', - 'daphne>=0.14.1', + 'daphne>=1.0.0', ] ) From 8ed7a2d3a294b05b390c4352155c792ccf1a3d6e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 Jan 2017 18:11:44 -0800 Subject: [PATCH 590/746] Remove unused imports --- channels/tests/test_binding.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index b2975c2..b2afce1 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -1,10 +1,8 @@ from __future__ import unicode_literals -import json - from django.contrib.auth import get_user_model -from channels import Group, route +from channels import route from channels.binding.base import CREATE, DELETE, UPDATE from channels.binding.websockets import WebsocketBinding from channels.generic.websockets import WebsocketDemultiplexer From aa3af5031c266d2279f40854d536f5cf40173dbb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 Jan 2017 18:18:00 -0800 Subject: [PATCH 591/746] Fix bad asserts in runserver endpoint tests --- channels/tests/test_management.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/channels/tests/test_management.py b/channels/tests/test_management.py index 20d723b..1242088 100644 --- a/channels/tests/test_management.py +++ b/channels/tests/test_management.py @@ -96,7 +96,6 @@ class RunServerTests(TestCase): endpoints=['tcp:port=8000:interface=127.0.0.1'], signal_handlers=True, http_timeout=60, - host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, ws_protocols=None, @@ -117,7 +116,6 @@ class RunServerTests(TestCase): endpoints=['tcp:port=8000:interface=127.0.0.1'], signal_handlers=True, http_timeout=60, - host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, ws_protocols=None, @@ -126,10 +124,9 @@ class RunServerTests(TestCase): call_command('runserver', '--noreload', 'localhost:8001') mocked_server.assert_called_with( - endpoints=['tcp:port=8001:interface=127.0.0.1'], + endpoints=['tcp:port=8001:interface=localhost'], signal_handlers=True, http_timeout=60, - host='localhost', action_logger=mock.ANY, channel_layer=mock.ANY, ws_protocols=None, @@ -153,7 +150,6 @@ class RunServerTests(TestCase): endpoints=['tcp:port=8000:interface=127.0.0.1'], signal_handlers=True, http_timeout=60, - host='127.0.0.1', action_logger=mock.ANY, channel_layer=mock.ANY, ws_protocols=None, From ec0b124c6e672571c9bf805a027308bf68d8ed56 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 Jan 2017 18:29:58 -0800 Subject: [PATCH 592/746] Flesh out release notes --- docs/binding.rst | 5 ----- docs/releases/1.0.0.rst | 46 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/docs/binding.rst b/docs/binding.rst index 3646a9f..517c2ed 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -1,11 +1,6 @@ Data Binding ============ -.. warning:: - - Data Binding is new and might change slightly in the - upcoming weeks, and so don't consider this API totally stable yet. - The Channels data binding framework automates the process of tying Django models into frontend views, such as javascript-powered website UIs. It provides a quick and flexible way to generate messages on Groups for model changes diff --git a/docs/releases/1.0.0.rst b/docs/releases/1.0.0.rst index 15da04a..9d8db09 100644 --- a/docs/releases/1.0.0.rst +++ b/docs/releases/1.0.0.rst @@ -1,8 +1,18 @@ 1.0.0 Release Notes =================== -.. note:: - These release notes are in development. Channels 1.0.0 is not yet released. +.. contents:: Table of Contents + :depth: 1 + +Channels 1.0.0 brings together a number of design changes, some breaking changes, +into our first fully stable release. + +It was unfortunately not possible to make all of the changes backwards +compatible, though most code should not be too affected and the fixes are +generally quite easy. + +You **must also update Daphne** to at least 1.0.0 to have this release of +Channels work correctly. Major Features @@ -19,6 +29,8 @@ while they send over a message on ``websocket.connect``, and your application must either accept or reject the connection before the handshake is completed and messages can be received. +You **must** update Daphne to at least 1.0.0 to make this work correctly. + This has several advantages: * You can now reject WebSockets before they even finish connecting, giving @@ -99,6 +111,34 @@ wish to use it. It needs some extra initial setup and uses a database for persistance; see :doc:`/delay` for more information. +Minor Changes +------------- + +* Serializers can now specify fields as ``__all__`` to auto-include all fields, + and ``exclude`` to remove certain unwanted fields. + +* ``runserver`` respects ``FORCE_SCRIPT_NAME`` + +* Websockets can now be closed with a specific code by calling ``close(status=4000)`` + +* ``enforce_ordering`` no longer has a ``slight`` mode (because of the accept + flow changes), and is more efficient with session saving. + +* ``runserver`` respects ``--nothreading`` and only launches one worker, takes + a ``--http-timeout`` option if you want to override it from the default ``60``, + +* A new ``@channel_and_http_session`` decorator rehydrates the HTTP session out + of the channel session if you want to access it inside receive consumers. + +* Streaming responses no longer have a chance of being cached. + +* ``request.META['SERVER_PORT']`` is now always a string. + +* ``http.disconnect`` now has a ``path`` key so you can route it. + +* Test client now has a ``send_and_consume`` method. + + Backwards Incompatible Changes ------------------------------ @@ -123,6 +163,8 @@ in the handshaking phase forever and you'll never get any messages. All built-in Channels consumers (e.g. in the generic consumers) have been upgraded to do this. +You **must** update Daphne to at least 1.0.0 to make this work correctly. + Databinding group_names ~~~~~~~~~~~~~~~~~~~~~~~ From a7818347795e483ff8ed8f6dc06787a26e53518b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 Jan 2017 18:33:21 -0800 Subject: [PATCH 593/746] Remove release notes TOC --- docs/releases/1.0.0.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/releases/1.0.0.rst b/docs/releases/1.0.0.rst index 9d8db09..044c590 100644 --- a/docs/releases/1.0.0.rst +++ b/docs/releases/1.0.0.rst @@ -1,9 +1,6 @@ 1.0.0 Release Notes =================== -.. contents:: Table of Contents - :depth: 1 - Channels 1.0.0 brings together a number of design changes, some breaking changes, into our first fully stable release. From c0ba284bbbc15fc02c4138ca0d51bd62bd19740b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 Jan 2017 18:33:31 -0800 Subject: [PATCH 594/746] Fix import ordering --- channels/generic/websockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 256978c..32ebe68 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -2,8 +2,8 @@ from django.core.serializers.json import DjangoJSONEncoder, json from ..auth import channel_session_user_from_http from ..channel import Group -from ..sessions import enforce_ordering from ..exceptions import SendNotAvailableOnDemultiplexer +from ..sessions import enforce_ordering from .base import BaseConsumer From 827fcd25b1bc5250574285a8feac49d35d4ab0af Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 8 Jan 2017 18:37:03 -0800 Subject: [PATCH 595/746] Releasing 1.0.0 --- channels/__init__.py | 2 +- docs/releases/1.0.0.rst | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/channels/__init__.py b/channels/__init__.py index 70a855c..858e45f 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.2" +__version__ = "1.0.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/releases/1.0.0.rst b/docs/releases/1.0.0.rst index 044c590..7d4ed8f 100644 --- a/docs/releases/1.0.0.rst +++ b/docs/releases/1.0.0.rst @@ -1,8 +1,13 @@ 1.0.0 Release Notes =================== -Channels 1.0.0 brings together a number of design changes, some breaking changes, -into our first fully stable release. +Channels 1.0.0 brings together a number of design changes, including some +breaking changes, into our first fully stable release, and also brings the +databinding code out of alpha phase. + +The result is a faster, easier to use, and safer Channels, including one major +change that will fix almost all problems with sessions and connect/receive +ordering in a way that needs no persistent storage. It was unfortunately not possible to make all of the changes backwards compatible, though most code should not be too affected and the fixes are From 8a93dfc4019f71c88bb5c6c256af35adff275fbb Mon Sep 17 00:00:00 2001 From: Krukov D Date: Mon, 9 Jan 2017 21:08:00 +0300 Subject: [PATCH 596/746] Accept Connection at WebsocketConsumer (#467) * Added accept at default behavior for websocket generic cbv and pass message instead of dict * Fix flake8 * Use HttpClient Instead of Client * Fix lsort --- channels/binding/websockets.py | 11 +++--- channels/generic/websockets.py | 3 +- channels/message.py | 1 + channels/tests/http.py | 17 ++++++-- channels/tests/test_generic.py | 71 +++++++++++++--------------------- 5 files changed, 47 insertions(+), 56 deletions(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index cd910ca..6017150 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -87,18 +87,17 @@ class WebsocketBinding(Binding): # Only allow received packets through further. if message.channel.name != "websocket.receive": return - # Call superclass, unpacking the payload in the process - payload = json.loads(message['text']) - super(WebsocketBinding, cls).trigger_inbound(payload, **kwargs) + super(WebsocketBinding, cls).trigger_inbound(message, **kwargs) def deserialize(self, message): """ You must hook this up behind a Deserializer, so we expect the JSON already dealt with. """ - action = message['action'] - pk = message.get('pk', None) - data = message.get('data', None) + body = json.loads(message['text']) + action = body['action'] + pk = body.get('pk', None) + data = body.get('data', None) return action, pk, data def _hydrate(self, pk, data): diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 32ebe68..555b165 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -72,7 +72,7 @@ class WebsocketConsumer(BaseConsumer): """ Called when a WebSocket connection is opened. """ - pass + self.message.reply_channel.send({"accept": True}) def raw_receive(self, message, **kwargs): """ @@ -220,6 +220,7 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): def connect(self, message, **kwargs): """Forward connection to all consumers.""" + self.message.reply_channel.send({"accept": True}) for stream, consumer in self.consumers.items(): kwargs['multiplexer'] = WebsocketMultiplexer(stream, self.message.reply_channel) consumer(message, **kwargs) diff --git a/channels/message.py b/channels/message.py index 97e67a3..6a4d3f3 100644 --- a/channels/message.py +++ b/channels/message.py @@ -81,5 +81,6 @@ class PendingMessageStore(object): sender.send(message, immediately=True) self.threadlocal.messages = [] + pending_message_store = PendingMessageStore() consumer_finished.connect(pending_message_store.send_and_flush) diff --git a/channels/tests/http.py b/channels/tests/http.py index 9759063..5c3b4bb 100644 --- a/channels/tests/http.py +++ b/channels/tests/http.py @@ -1,3 +1,4 @@ +from __future__ import unicode_literals import copy import json @@ -66,9 +67,9 @@ class HttpClient(Client): Return text content of a message for client channel and decoding it if json kwarg is set """ content = super(HttpClient, self).receive() - if content and json: + if content and json and 'text' in content and isinstance(content['text'], six.string_types): return json_module.loads(content['text']) - return content['text'] if content else None + return content.get('text', content) if content else None def send(self, to, content={}, text=None, path='/'): """ @@ -87,12 +88,20 @@ class HttpClient(Client): content['text'] = text self.channel_layer.send(to, content) - def send_and_consume(self, channel, content={}, text=None, path='/', fail_on_none=True): + def send_and_consume(self, channel, content={}, text=None, path='/', fail_on_none=True, check_accept=True): """ Reproduce full life cycle of the message """ self.send(channel, content, text, path) - return self.consume(channel, fail_on_none=fail_on_none) + return self.consume(channel, fail_on_none=fail_on_none, check_accept=check_accept) + + def consume(self, channel, fail_on_none=True, check_accept=True): + result = super(HttpClient, self).consume(channel, fail_on_none=fail_on_none) + if channel == "websocket.connect" and check_accept: + received = self.receive(json=False) + if received != {"accept": True}: + raise AssertionError("Connection rejected: %s != '{accept: True}'" % received) + return result def login(self, **credentials): """ diff --git a/channels/tests/test_generic.py b/channels/tests/test_generic.py index 40ade79..2938598 100644 --- a/channels/tests/test_generic.py +++ b/channels/tests/test_generic.py @@ -1,13 +1,11 @@ from __future__ import unicode_literals -import json - from django.test import override_settings from channels import route_class from channels.exceptions import SendNotAvailableOnDemultiplexer from channels.generic import BaseConsumer, websockets -from channels.tests import ChannelTestCase, Client, apply_routes +from channels.tests import ChannelTestCase, Client, HttpClient, apply_routes @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") @@ -92,6 +90,7 @@ class GenericTests(ChannelTestCase): class WebsocketConsumer(websockets.WebsocketConsumer): def connect(self, message, **kwargs): + self.message.reply_channel.send({'accept': True}) self.send(text=message.get('order')) routes = [ @@ -103,18 +102,18 @@ class GenericTests(ChannelTestCase): self.assertIs(routes[1].consumer, WebsocketConsumer) with apply_routes(routes): - client = Client() + client = HttpClient() client.send('websocket.connect', {'path': '/path', 'order': 1}) client.send('websocket.connect', {'path': '/path', 'order': 0}) + client.consume('websocket.connect', check_accept=False) client.consume('websocket.connect') + self.assertEqual(client.receive(json=False), 0) client.consume('websocket.connect') - client.consume('websocket.connect') - self.assertEqual(client.receive(), {'text': 0}) - self.assertEqual(client.receive(), {'text': 1}) + self.assertEqual(client.receive(json=False), 1) client.send_and_consume('websocket.connect', {'path': '/path/2', 'order': 'next'}) - self.assertEqual(client.receive(), {'text': 'next'}) + self.assertEqual(client.receive(json=False), 'next') def test_as_route_method(self): class WebsocketConsumer(BaseConsumer): @@ -154,40 +153,28 @@ class GenericTests(ChannelTestCase): "mystream": MyWebsocketConsumer } - with apply_routes([ - route_class(Demultiplexer, path='/path/(?P\d+)'), - route_class(MyWebsocketConsumer), - ]): - client = Client() + with apply_routes([route_class(Demultiplexer, path='/path/(?P\d+)')]): + client = HttpClient() - client.send_and_consume('websocket.connect', {'path': '/path/1'}) + client.send_and_consume('websocket.connect', path='/path/1') self.assertEqual(client.receive(), { - "text": json.dumps({ - "stream": "mystream", - "payload": {"id": "1"}, - }) + "stream": "mystream", + "payload": {"id": "1"}, }) - client.send_and_consume('websocket.receive', { - 'path': '/path/1', - 'text': json.dumps({ - "stream": "mystream", - "payload": {"text_field": "mytext"} - }) - }) + client.send_and_consume('websocket.receive', text={ + "stream": "mystream", + "payload": {"text_field": "mytext"}, + }, path='/path/1') self.assertEqual(client.receive(), { - "text": json.dumps({ - "stream": "mystream", - "payload": {"text_field": "mytext"}, - }) + "stream": "mystream", + "payload": {"text_field": "mytext"}, }) - client.send_and_consume('websocket.disconnect', {'path': '/path/1'}) + client.send_and_consume('websocket.disconnect', path='/path/1') self.assertEqual(client.receive(), { - "text": json.dumps({ - "stream": "mystream", - "payload": {"id": "1"}, - }) + "stream": "mystream", + "payload": {"id": "1"}, }) def test_websocket_demultiplexer_send(self): @@ -202,19 +189,13 @@ class GenericTests(ChannelTestCase): "mystream": MyWebsocketConsumer } - with apply_routes([ - route_class(Demultiplexer, path='/path/(?P\d+)'), - route_class(MyWebsocketConsumer), - ]): - client = Client() + with apply_routes([route_class(Demultiplexer, path='/path/(?P\d+)')]): + client = HttpClient() with self.assertRaises(SendNotAvailableOnDemultiplexer): - client.send_and_consume('websocket.receive', { - 'path': '/path/1', - 'text': json.dumps({ - "stream": "mystream", - "payload": {"text_field": "mytext"} - }) + client.send_and_consume('websocket.receive', path='/path/1', text={ + "stream": "mystream", + "payload": {"text_field": "mytext"}, }) client.receive() From 2650505eabb15df610a94f6da2809867f197909f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 9 Jan 2017 22:10:18 -0800 Subject: [PATCH 597/746] Releasing 1.0.1 --- CHANGELOG.txt | 6 ++++++ channels/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index a4fc760..37df4b8 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,9 @@ +1.0.1 (2017-01-09) +------------------ + +* WebSocket generic views now accept connections by default in their connect + handler for better backwards compatibility. + 1.0.0 (2017-01-08) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index 858e45f..5262d81 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.0.0" +__version__ = "1.0.1" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From 08f0a0934c916e96f002573cec0b5ac6e03fd535 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Tue, 10 Jan 2017 18:05:23 +0000 Subject: [PATCH 598/746] Use HTTPS in changelog (#472) security ++ --- CHANGELOG.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 37df4b8..5b8866a 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -8,14 +8,14 @@ ------------------ Full release notes, with more upgrade details, are available at: -http://channels.readthedocs.io/en/latest/releases/1.0.0.html +https://channels.readthedocs.io/en/latest/releases/1.0.0.html * BREAKING CHANGE: WebSockets must now be explicitly accepted or denied. - See http://channels.readthedocs.io/en/latest/releases/1.0.0.html for more. + See https://channels.readthedocs.io/en/latest/releases/1.0.0.html for more. * BREAKING CHANGE: Demultiplexers have been overhauled to directly dispatch messages rather than using channels to new consumers. Consult the docs on - generic consumers for more: http://channels.readthedocs.io/en/latest/generics.html + generic consumers for more: https://channels.readthedocs.io/en/latest/generics.html * BREAKING CHANGE: Databinding now operates from implicit group membership, where your code just has to say what groups would be used and Channels will From b84713b20e5887ae40e288cf64a799a55009f90f Mon Sep 17 00:00:00 2001 From: Krukov D Date: Wed, 11 Jan 2017 21:03:03 +0300 Subject: [PATCH 599/746] Remove unicode literals (#476) --- channels/tests/http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/channels/tests/http.py b/channels/tests/http.py index 5c3b4bb..7faebff 100644 --- a/channels/tests/http.py +++ b/channels/tests/http.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals import copy import json From 6bfaaf23d10883e511cd7f6533206455aac0ad64 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 11 Jan 2017 11:37:50 -0800 Subject: [PATCH 600/746] Add connect-accept into Websocket CBC example (#479) --- docs/generics.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/generics.rst b/docs/generics.rst index b871015..5b1a4c5 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -103,7 +103,9 @@ The basic WebSocket generic consumer is used like this:: """ Perform things on connection start """ - pass + # Accept the connection; this is done by default if you don't override + # the connect function. + self.message.reply_channel.send({"accept": True}) def receive(self, text=None, bytes=None, **kwargs): """ From ee4aa9b292512437de01c44c86fc3b4fe37d1b27 Mon Sep 17 00:00:00 2001 From: Bartek Ogryczak Date: Wed, 11 Jan 2017 11:49:59 -0800 Subject: [PATCH 601/746] more consistent metrics (#473) * more consistent metrics More consistent metrics, in particular for consumers such as Graphite, CloudWatch etc. you don't want aggregated numbers per second, you want number since previous call. Since we don't want to track all the clients, total since start is the next best thing, because it can be easily calculated by the metrics consumer `count_since_last = pervious_count - current_count` * changing 'queue_depth' to 'messages_pending', also making 'messages' plurar for 'max_age' to keep it consitent --- docs/asgi.rst | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 95b6595..405ba6e 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -367,17 +367,20 @@ A channel layer implementing the ``groups`` extension must also provide: A channel layer implementing the ``statistics`` extension must also provide: -* ``global_statistics()``, a callable that returns a dict with zero - or more of (unicode string keys): +* ``global_statistics()``, a callable that returns statistics across all channels +* ``channel_statistics(channel)``, a callable that returns statistics for specified channel - * ``count``, the current number of messages waiting in all channels +* in both cases statistics are a dict with zero or more of (unicode string keys): -* ``channel_statistics(channel)``, a callable that returns a dict with zero - or more of (unicode string keys): + * ``messages_count``, the number of messages processed since server start + * ``messages_count_per_second``, the number of messages processed in the last second + * ``messages_pending``, the current number of messages waiting + * ``messages_max_age``, how long the oldest message has been waiting, in seconds + * ``channel_full_count``, the number of times `ChannelFull` exception has been risen since server start + * ``channel_full_count_per_second``, the number of times `ChannelFull` exception has been risen in the last second + +* Implementation may provide total counts, counts per seconds or both. - * ``length``, the current number of messages waiting on the channel - * ``age``, how long the oldest message has been waiting, in seconds - * ``per_second``, the number of messages processed in the last second A channel layer implementing the ``flush`` extension must also provide: From 69c59ee8b4924cd346e0a2b1724979375ad9924c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 11 Jan 2017 15:34:23 -0800 Subject: [PATCH 602/746] Fixed #481: Sends from outside consumers send immediately --- channels/channel.py | 11 ++++++----- channels/message.py | 21 +++++++++++++++++---- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/channels/channel.py b/channels/channel.py index 03a2db6..e4bef7b 100644 --- a/channels/channel.py +++ b/channels/channel.py @@ -34,14 +34,15 @@ class Channel(object): Send a message over the channel - messages are always dicts. Sends are delayed until consumer completion. To override this, you - may pass immediately=True. + may pass immediately=True. If you are outside a consumer, things are + always sent immediately. """ + from .message import pending_message_store if not isinstance(content, dict): raise TypeError("You can only send dicts as content on channels.") - if immediately: + if immediately or not pending_message_store.active: self.channel_layer.send(self.name, content) else: - from .message import pending_message_store pending_message_store.append(self, content) def __str__(self): @@ -80,10 +81,10 @@ class Group(object): Sends are delayed until consumer completion. To override this, you may pass immediately=True. """ + from .message import pending_message_store if not isinstance(content, dict): raise ValueError("You can only send dicts as content on channels.") - if immediately: + if immediately or not pending_message_store.active: self.channel_layer.send_group(self.name, content) else: - from .message import pending_message_store pending_message_store.append(self, content) diff --git a/channels/message.py b/channels/message.py index 6a4d3f3..f8001d8 100644 --- a/channels/message.py +++ b/channels/message.py @@ -4,7 +4,7 @@ import copy import threading from .channel import Channel -from .signals import consumer_finished +from .signals import consumer_finished, consumer_started class Message(object): @@ -71,16 +71,29 @@ class PendingMessageStore(object): threadlocal = threading.local() + def prepare(self, **kwargs): + """ + Sets the message store up to receive messages. + """ + self.threadlocal.messages = [] + + @property + def active(self): + """ + Returns if the pending message store can be used or not + (it can only be used inside consumers) + """ + return hasattr(self.threadlocal, "messages") + def append(self, sender, message): - if not hasattr(self.threadlocal, "messages"): - self.threadlocal.messages = [] self.threadlocal.messages.append((sender, message)) def send_and_flush(self, **kwargs): for sender, message in getattr(self.threadlocal, "messages", []): sender.send(message, immediately=True) - self.threadlocal.messages = [] + delattr(self.threadlocal, "messages") pending_message_store = PendingMessageStore() +consumer_started.connect(pending_message_store.prepare) consumer_finished.connect(pending_message_store.send_and_flush) From 2ced4ee2e9ad4d12f1f465de01e858b5cd8d493a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 11 Jan 2017 15:40:24 -0800 Subject: [PATCH 603/746] Remove consumer_finished from tests that flushed No longer needed now messages aren't buffered outside consumers. --- channels/tests/test_binding.py | 8 -------- channels/tests/test_handler.py | 2 -- 2 files changed, 10 deletions(-) diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index b2afce1..2383cfb 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -6,7 +6,6 @@ from channels import route from channels.binding.base import CREATE, DELETE, UPDATE from channels.binding.websockets import WebsocketBinding from channels.generic.websockets import WebsocketDemultiplexer -from channels.signals import consumer_finished from channels.tests import ChannelTestCase, HttpClient, apply_routes User = get_user_model() @@ -33,7 +32,6 @@ class TestsBinding(ChannelTestCase): user = User.objects.create(username='test', email='test@test.com') - consumer_finished.send(sender=None) received = client.receive() self.assertTrue('payload' in received) self.assertTrue('action' in received['payload']) @@ -75,8 +73,6 @@ class TestsBinding(ChannelTestCase): client.join_group('users_exclude') user = User.objects.create(username='test', email='test@test.com') - consumer_finished.send(sender=None) - consumer_finished.send(sender=None) received = client.receive() self.assertTrue('payload' in received) @@ -131,7 +127,6 @@ class TestsBinding(ChannelTestCase): # Make model and clear out pending sends user = User.objects.create(username='test', email='test@test.com') - consumer_finished.send(sender=None) client = HttpClient() client.join_group('users2') @@ -139,7 +134,6 @@ class TestsBinding(ChannelTestCase): user.username = 'test_new' user.save() - consumer_finished.send(sender=None) received = client.receive() self.assertTrue('payload' in received) self.assertTrue('action' in received['payload']) @@ -178,14 +172,12 @@ class TestsBinding(ChannelTestCase): # Make model and clear out pending sends user = User.objects.create(username='test', email='test@test.com') - consumer_finished.send(sender=None) client = HttpClient() client.join_group('users3') user.delete() - consumer_finished.send(sender=None) received = client.receive() self.assertTrue('payload' in received) self.assertTrue('action' in received['payload']) diff --git a/channels/tests/test_handler.py b/channels/tests/test_handler.py index efda3f2..0c096cc 100644 --- a/channels/tests/test_handler.py +++ b/channels/tests/test_handler.py @@ -9,7 +9,6 @@ from six import BytesIO from channels import Channel from channels.handler import AsgiHandler -from channels.signals import consumer_finished from channels.tests import ChannelTestCase @@ -24,7 +23,6 @@ class FakeAsgiHandler(AsgiHandler): def __init__(self, response): assert isinstance(response, (HttpResponse, StreamingHttpResponse)) self._response = response - consumer_finished.send(sender=self.__class__) super(FakeAsgiHandler, self).__init__() def get_response(self, request): From 37da4624115cc4b99b68f7d0dafd65462ed3f8cd Mon Sep 17 00:00:00 2001 From: Krukov D Date: Thu, 12 Jan 2017 21:02:30 +0300 Subject: [PATCH 604/746] User friendly way to close websocket with code (via CloseException) (#468) * User friendly way to close websocket with status * More generic way to close(whatever) connection by exception * Fix import ordering for exceptions (isort) --- channels/exceptions.py | 31 +++++++++++++++++++++++++++++++ channels/worker.py | 4 +++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/channels/exceptions.py b/channels/exceptions.py index afadf9e..1827629 100644 --- a/channels/exceptions.py +++ b/channels/exceptions.py @@ -1,3 +1,7 @@ +from __future__ import unicode_literals +import six + + class ConsumeLater(Exception): """ Exception that says that the current message should be re-queued back @@ -39,6 +43,33 @@ class DenyConnection(Exception): pass +class ChannelSocketException(Exception): + """ + Base Exception is intended to run some action ('run' method) + when it is raised at a consumer body + """ + + def run(self, message): + raise NotImplementedError + + +class WebsocketCloseException(ChannelSocketException): + """ + ChannelSocketException based exceptions for close websocket connection with code + """ + + def __init__(self, code=None): + if code is not None and not isinstance(code, six.integer_types) \ + and code != 1000 and not (3000 <= code <= 4999): + raise ValueError("invalid close code {} (must be 1000 or from [3000, 4999])".format(code)) + self._code = code + + def run(self, message): + if message.reply_channel.name.split('.')[0] != "websocket": + raise ValueError("You cannot raise CloseWebsocketError from a non-websocket handler.") + message.reply_channel.send({"close": self._code or True}) + + class SendNotAvailableOnDemultiplexer(Exception): """ Raised when trying to send with a WebsocketDemultiplexer. Use the multiplexer instead. diff --git a/channels/worker.py b/channels/worker.py index b44b6ac..213501f 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -8,7 +8,7 @@ import sys import threading import time -from .exceptions import ConsumeLater, DenyConnection +from .exceptions import ChannelSocketException, ConsumeLater, DenyConnection from .message import Message from .signals import consumer_finished, consumer_started, worker_ready from .utils import name_that_thing @@ -122,6 +122,8 @@ class Worker(object): if message.channel.name != "websocket.connect": raise ValueError("You cannot DenyConnection from a non-websocket.connect handler.") message.reply_channel.send({"close": True}) + except ChannelSocketException as e: + e.run(message) except ConsumeLater: # They want to not handle it yet. Re-inject it with a number-of-tries marker. content['__retries__'] = content.get("__retries__", 0) + 1 From 05b5fa5216355c2807b77e53e35d620bf67ae540 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 12 Jan 2017 09:59:46 -0800 Subject: [PATCH 605/746] Fixed #482: Group_names not updated right in base classes. --- channels/binding/base.py | 6 +++--- channels/tests/test_binding.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/channels/binding/base.py b/channels/binding/base.py index fb117ee..85ec720 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -142,7 +142,7 @@ class Binding(object): if action == CREATE: group_names = set() else: - group_names = set(cls.group_names(instance, action)) + group_names = set(cls.group_names(instance)) if not hasattr(instance, '_binding_group_names'): instance._binding_group_names = {} @@ -157,7 +157,7 @@ class Binding(object): if action == DELETE: new_group_names = set() else: - new_group_names = set(cls.group_names(instance, action)) + new_group_names = set(cls.group_names(instance)) # if post delete, new_group_names should be [] self = cls() @@ -186,7 +186,7 @@ class Binding(object): group.send(message) @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): """ Returns the iterable of group names to send the object to based on the instance and action performed on it. diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index 2383cfb..6b32a02 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -21,7 +21,7 @@ class TestsBinding(ChannelTestCase): fields = ['username', 'email', 'password', 'last_name'] @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): return ["users"] def has_permission(self, user, action, pk): @@ -62,7 +62,7 @@ class TestsBinding(ChannelTestCase): exclude = ['first_name', 'last_name'] @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): return ["users_exclude"] def has_permission(self, user, action, pk): @@ -105,7 +105,7 @@ class TestsBinding(ChannelTestCase): stream = 'test' @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): return ["users_omit"] def has_permission(self, user, action, pk): @@ -119,7 +119,7 @@ class TestsBinding(ChannelTestCase): fields = ['__all__'] @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): return ["users2"] def has_permission(self, user, action, pk): @@ -164,7 +164,7 @@ class TestsBinding(ChannelTestCase): fields = ['username'] @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): return ["users3"] def has_permission(self, user, action, pk): @@ -203,7 +203,7 @@ class TestsBinding(ChannelTestCase): fields = ['username', 'email', 'password', 'last_name'] @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): return ['users_outbound'] def has_permission(self, user, action, pk): @@ -243,7 +243,7 @@ class TestsBinding(ChannelTestCase): fields = ['username', ] @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): return ['users_outbound'] def has_permission(self, user, action, pk): @@ -289,7 +289,7 @@ class TestsBinding(ChannelTestCase): fields = ['username', ] @classmethod - def group_names(cls, instance, action): + def group_names(cls, instance): return ['users_outbound'] def has_permission(self, user, action, pk): From c9e6472ca751d58cf1b5a48453a9a6d632c7bd6f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 12 Jan 2017 10:09:17 -0800 Subject: [PATCH 606/746] Update changelogs for 1.0.2 and web in general --- CHANGELOG.txt | 21 ++++++++++++++++++--- docs/releases/1.0.0.rst | 2 +- docs/releases/1.0.1.rst | 16 ++++++++++++++++ docs/releases/1.0.2.rst | 27 +++++++++++++++++++++++++++ docs/releases/index.rst | 2 ++ 5 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 docs/releases/1.0.1.rst create mode 100644 docs/releases/1.0.2.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 5b8866a..692c470 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,15 +1,30 @@ +Full release notes, with more details and upgrade information, are available at: +https://channels.readthedocs.io/en/latest/releases + + +1.0.2 (2017-01-12) +------------------ + +* Websockets can now be closed from anywhere using the new ``WebsocketCloseException``. + There is also a generic ``ChannelSocketException`` so you can do custom behaviours. + +* Calling ``Channel.send`` or ``Group.send`` from outside a consumer context + (i.e. in tests or management commands) will once again send the message immediately. + +* The base implementation of databinding now correctly only calls ``group_names(instance)``, + as documented. + + 1.0.1 (2017-01-09) ------------------ * WebSocket generic views now accept connections by default in their connect handler for better backwards compatibility. + 1.0.0 (2017-01-08) ------------------ -Full release notes, with more upgrade details, are available at: -https://channels.readthedocs.io/en/latest/releases/1.0.0.html - * BREAKING CHANGE: WebSockets must now be explicitly accepted or denied. See https://channels.readthedocs.io/en/latest/releases/1.0.0.html for more. diff --git a/docs/releases/1.0.0.rst b/docs/releases/1.0.0.rst index 7d4ed8f..1e6849f 100644 --- a/docs/releases/1.0.0.rst +++ b/docs/releases/1.0.0.rst @@ -3,7 +3,7 @@ Channels 1.0.0 brings together a number of design changes, including some breaking changes, into our first fully stable release, and also brings the -databinding code out of alpha phase. +databinding code out of alpha phase. It was released on 2017/01/08. The result is a faster, easier to use, and safer Channels, including one major change that will fix almost all problems with sessions and connect/receive diff --git a/docs/releases/1.0.1.rst b/docs/releases/1.0.1.rst new file mode 100644 index 0000000..ae07531 --- /dev/null +++ b/docs/releases/1.0.1.rst @@ -0,0 +1,16 @@ +1.0.1 Release Notes +=================== + +Channels 1.0.1 is a minor bugfix release, released on 2017/01/09. + +Changes +------- + +* WebSocket generic views now accept connections by default in their connect + handler for better backwards compatibility. + + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/1.0.2.rst b/docs/releases/1.0.2.rst new file mode 100644 index 0000000..fd0435d --- /dev/null +++ b/docs/releases/1.0.2.rst @@ -0,0 +1,27 @@ +1.0.2 Release Notes +=================== + +Channels 1.0.2 is a minor bugfix release, released on 2017/01/12. + +Changes +------- + +* Websockets can now be closed from anywhere using the new ``WebsocketCloseException``, + available as ``channels.exceptions.WebsocketCloseException(code=None)``. There is + also a generic ``ChannelSocketException`` you can base any exceptions on that, + if it is caught, gets handed the current ``message`` in a ``run`` method, so you + can do custom behaviours. + +* Calling ``Channel.send`` or ``Group.send`` from outside a consumer context + (i.e. in tests or management commands) will once again send the message immediately, + rather than putting it into the consumer message buffer to be flushed when the + consumer ends (which never happens) + +* The base implementation of databinding now correctly only calls ``group_names(instance)``, + as documented. + + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 7e0e7e6..856f565 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -5,3 +5,5 @@ Release Notes :maxdepth: 1 1.0.0 + 1.0.1 + 1.0.2 From 811d017dc9f987df9184d98a52e988fe29557d97 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 12 Jan 2017 10:12:09 -0800 Subject: [PATCH 607/746] Releasing 1.0.2 --- channels/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/__init__.py b/channels/__init__.py index 5262d81..3ba52f8 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.0.1" +__version__ = "1.0.2" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' From fd30bff5deed5896d29cce201fe22b155c768938 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 13 Jan 2017 09:32:41 -0800 Subject: [PATCH 608/746] Fixed #483: HttpResponse takes "status", not "status_code" --- channels/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index 6dfdb50..6fc49f5 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -205,7 +205,7 @@ class AsgiHandler(base.BaseHandler): response = http.HttpResponseBadRequest() except RequestTimeout: # Parsing the rquest failed, so the response is a Request Timeout error - response = HttpResponse("408 Request Timeout (upload too slow)", status_code=408) + response = HttpResponse("408 Request Timeout (upload too slow)", status=408) except RequestAborted: # Client closed connection on us mid request. Abort! return From 335cd1625e0217ece7f70b6a9880e055b4dbda10 Mon Sep 17 00:00:00 2001 From: Malyshev Artem Date: Tue, 17 Jan 2017 23:25:08 +0300 Subject: [PATCH 609/746] Correct benchmark test for recent channels version. (#487) * Ignore Emacs backups. * Do not override default websocket.connect handler. Channels specification 1.0 requires that websocket.connect handler returns meaningful message with {'accept': True} at least. * Add rabbitmq channel layer settings. * Add benchmark requirements in separate file. * Add RabbitMQ infrastructure part. * Adapt benchmark README for new docker layout. * Adapt fabric deploy for new settings module. --- .gitignore | 5 ++-- testproject/Dockerfile.rabbitmq | 27 ++++++++++++++++++ testproject/{Dockerfile => Dockerfile.redis} | 0 testproject/README.rst | 27 +++++++++++++----- testproject/chtest/consumers.py | 10 +------ testproject/docker-compose.rabbitmq.yml | 28 +++++++++++++++++++ testproject/docker-compose.redis.yml | 26 +++++++++++++++++ testproject/docker-compose.yml | 18 ------------ testproject/fabfile.py | 2 +- testproject/manage.py | 5 +++- testproject/requirements.benchmark.txt | 7 +++++ testproject/testproject/asgi/__init__.py | 0 .../{asgi_for_ipc.py => asgi/ipc.py} | 0 testproject/testproject/asgi/rabbitmq.py | 8 ++++++ .../testproject/{asgi.py => asgi/redis.py} | 0 .../testproject/settings/channels_rabbitmq.py | 14 ++++++++++ testproject/testproject/urls.py | 14 +++------- 17 files changed, 143 insertions(+), 48 deletions(-) create mode 100644 testproject/Dockerfile.rabbitmq rename testproject/{Dockerfile => Dockerfile.redis} (100%) create mode 100644 testproject/docker-compose.rabbitmq.yml create mode 100644 testproject/docker-compose.redis.yml delete mode 100644 testproject/docker-compose.yml create mode 100644 testproject/requirements.benchmark.txt create mode 100644 testproject/testproject/asgi/__init__.py rename testproject/testproject/{asgi_for_ipc.py => asgi/ipc.py} (100%) create mode 100644 testproject/testproject/asgi/rabbitmq.py rename testproject/testproject/{asgi.py => asgi/redis.py} (100%) create mode 100644 testproject/testproject/settings/channels_rabbitmq.py diff --git a/.gitignore b/.gitignore index fd1de91..857bc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ __pycache__/ .coverage.* TODO -IDE and Tooling files -.idea/* \ No newline at end of file +# IDE and Tooling files +.idea/* +*~ diff --git a/testproject/Dockerfile.rabbitmq b/testproject/Dockerfile.rabbitmq new file mode 100644 index 0000000..5e5cfb3 --- /dev/null +++ b/testproject/Dockerfile.rabbitmq @@ -0,0 +1,27 @@ +FROM ubuntu:16.04 + +MAINTAINER Artem Malyshev + +# python-dev \ +RUN apt-get update && \ + apt-get install -y \ + git \ + python-setuptools \ + python-pip && \ + pip install -U pip + +# Install asgi_rabbitmq driver and most recent Daphne +RUN pip install \ + git+https://github.com/proofit404/asgi_rabbitmq.git#egg=asgi_rabbitmq \ + git+https://github.com/django/daphne.git@#egg=daphne + +# Clone Channels and install it +RUN git clone https://github.com/django/channels.git /srv/channels/ && \ + cd /srv/channels && \ + git reset --hard origin/master && \ + python setup.py install + +WORKDIR /srv/channels/testproject/ +ENV RABBITMQ_URL=amqp://guest:guest@rabbitmq:5672/%2F + +EXPOSE 80 diff --git a/testproject/Dockerfile b/testproject/Dockerfile.redis similarity index 100% rename from testproject/Dockerfile rename to testproject/Dockerfile.redis diff --git a/testproject/README.rst b/testproject/README.rst index c754700..3ccb737 100644 --- a/testproject/README.rst +++ b/testproject/README.rst @@ -18,13 +18,13 @@ e.g. to create it right in the test directory (assuming python 2 is your system' How to use with Docker: ~~~~~~~~~~~~~~~~~~~~~~~ -Build the docker image from Dockerfile, tag it `channels-test`:: +Build the docker image from Dockerfile, tag it `channels-redis-test`:: - docker build -t channels-test . + docker build -t channels-redis-test -f Dockerfile.redis . Run the server:: - docker-compose up -d + docker-compose -f docker-compose.redis.yml up The benchmark project will now be running on: http:{your-docker-ip}:80 @@ -40,6 +40,19 @@ Let's just try a quick test with the default values from the parameter list:: python benchmark.py ws://localhost:80 +How to use with Docker and RabbitMQ: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Build the docker image from Dockerfile, tag it `channels-rabbitmq-test`:: + + docker build -t channels-rabbitmq-test -f Dockerfile.rabbitmq . + +Run the server:: + + docker-compose -f docker-compose.rabbitmq.yml up + +The rest is the same. + How to use with runserver: ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -62,14 +75,14 @@ In another terminal window, run the benchmark with:: Additional load testing options: -~~~~~~~~~~~~~~~~~~~~~~~~~~ - +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + If you wish to setup a separate machine to loadtest your environment, you can do the following steps. Install fabric on your machine. This is highly dependent on what your environment looks like, but the recommend option is to:: pip install fabric - + (Hint: if you're on Windows 10, just use the Linux subsystem and use ``apt-get install fabric``. It'll save you a lot of trouble.) Git clone this project down to your machine:: @@ -81,7 +94,7 @@ Relative to where you cloned the directory, move up a couple levels:: cd channels/testproject/ Spin up a server on your favorite cloud host (AWS, Linode, Digital Ocean, etc.) and get its host and credentials. Run the following command using those credentials:: - + fab setup_load_tester -i "ida_rsa" -H ubuntu@example.com That machine will provision itself. It may (depending on your vendor) prompt you a few times for a ``Y/n`` question. This is just asking you about increasing stroage space. diff --git a/testproject/chtest/consumers.py b/testproject/chtest/consumers.py index f447e45..85db43f 100644 --- a/testproject/chtest/consumers.py +++ b/testproject/chtest/consumers.py @@ -1,14 +1,6 @@ from channels.sessions import enforce_ordering -#@enforce_ordering(slight=True) -def ws_connect(message): - pass - - -#@enforce_ordering(slight=True) def ws_message(message): "Echoes messages back to the client" - message.reply_channel.send({ - "text": message['text'], - }) + message.reply_channel.send({'text': message['text']}) diff --git a/testproject/docker-compose.rabbitmq.yml b/testproject/docker-compose.rabbitmq.yml new file mode 100644 index 0000000..1cc885d --- /dev/null +++ b/testproject/docker-compose.rabbitmq.yml @@ -0,0 +1,28 @@ +version: '2' +services: + rabbitmq: + image: rabbitmq:management + ports: + - "15672:15672" + rabbitmq_daphne: + image: channels-rabbitmq-test + build: + context: . + dockerfile: Dockerfile.rabbitmq + command: daphne -b 0.0.0.0 -p 80 testproject.asgi.rabbitmq:channel_layer + volumes: + - .:/srv/channels/testproject/ + ports: + - "80:80" + depends_on: + - rabbitmq + rabbitmq_worker: + image: channels-rabbitmq-test + build: + context: . + dockerfile: Dockerfile.rabbitmq + command: python manage.py runworker --settings=testproject.settings.channels_rabbitmq + volumes: + - .:/srv/channels/testproject/ + depends_on: + - rabbitmq diff --git a/testproject/docker-compose.redis.yml b/testproject/docker-compose.redis.yml new file mode 100644 index 0000000..8cb3d18 --- /dev/null +++ b/testproject/docker-compose.redis.yml @@ -0,0 +1,26 @@ +version: '2' +services: + redis: + image: redis:alpine + redis_daphne: + image: channels-redis-test + build: + context: . + dockerfile: Dockerfile.redis + command: daphne -b 0.0.0.0 -p 80 testproject.asgi.redis:channel_layer + volumes: + - .:/srv/channels/testproject/ + ports: + - "80:80" + depends_on: + - redis + redis_worker: + image: channels-redis-test + build: + context: . + dockerfile: Dockerfile.redis + command: python manage.py runworker --settings=testproject.settings.channels_redis + volumes: + - .:/srv/channels/testproject/ + depends_on: + - redis diff --git a/testproject/docker-compose.yml b/testproject/docker-compose.yml deleted file mode 100644 index 1753b71..0000000 --- a/testproject/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '2' -services: - redis: - image: redis:alpine - daphne: - image: channels-test - build: . - command: daphne -b 0.0.0.0 -p 80 testproject.asgi:channel_layer - ports: - - "80:80" - depends_on: - - redis - worker: - image: channels-test - build: . - command: python manage.py runworker --settings=testproject.settings.channels_redis - depends_on: - - redis diff --git a/testproject/fabfile.py b/testproject/fabfile.py index 6c884dc..772ff08 100644 --- a/testproject/fabfile.py +++ b/testproject/fabfile.py @@ -23,7 +23,7 @@ def setup_channels(): @task def run_daphne(redis_ip): with cd("/srv/channels/testproject/"): - sudo("REDIS_URL=redis://%s:6379 daphne -b 0.0.0.0 -p 80 testproject.asgi:channel_layer" % redis_ip) + sudo("REDIS_URL=redis://%s:6379 daphne -b 0.0.0.0 -p 80 testproject.asgi.redis:channel_layer" % redis_ip) @task diff --git a/testproject/manage.py b/testproject/manage.py index 97ed576..9a0be8b 100644 --- a/testproject/manage.py +++ b/testproject/manage.py @@ -3,7 +3,10 @@ import os import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings") + os.environ.setdefault( + "DJANGO_SETTINGS_MODULE", + "testproject.settings.channels_redis", + ) from django.core.management import execute_from_command_line diff --git a/testproject/requirements.benchmark.txt b/testproject/requirements.benchmark.txt new file mode 100644 index 0000000..f04a638 --- /dev/null +++ b/testproject/requirements.benchmark.txt @@ -0,0 +1,7 @@ +autobahn==0.17.1 +constantly==15.1.0 +incremental==16.10.1 +six==1.10.0 +Twisted==16.6.0 +txaio==2.6.0 +zope.interface==4.3.3 diff --git a/testproject/testproject/asgi/__init__.py b/testproject/testproject/asgi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/testproject/testproject/asgi_for_ipc.py b/testproject/testproject/asgi/ipc.py similarity index 100% rename from testproject/testproject/asgi_for_ipc.py rename to testproject/testproject/asgi/ipc.py diff --git a/testproject/testproject/asgi/rabbitmq.py b/testproject/testproject/asgi/rabbitmq.py new file mode 100644 index 0000000..b132a73 --- /dev/null +++ b/testproject/testproject/asgi/rabbitmq.py @@ -0,0 +1,8 @@ +import os +from channels.asgi import get_channel_layer + +os.environ.setdefault( + "DJANGO_SETTINGS_MODULE", + "testproject.settings.channels_rabbitmq", +) +channel_layer = get_channel_layer() diff --git a/testproject/testproject/asgi.py b/testproject/testproject/asgi/redis.py similarity index 100% rename from testproject/testproject/asgi.py rename to testproject/testproject/asgi/redis.py diff --git a/testproject/testproject/settings/channels_rabbitmq.py b/testproject/testproject/settings/channels_rabbitmq.py new file mode 100644 index 0000000..4754d09 --- /dev/null +++ b/testproject/testproject/settings/channels_rabbitmq.py @@ -0,0 +1,14 @@ +# Settings for channels specifically +from testproject.settings.base import * + +INSTALLED_APPS += ('channels',) + +CHANNEL_LAYERS = { + 'default': { + 'BACKEND': 'asgi_rabbitmq.RabbitmqChannelLayer', + 'ROUTING': 'testproject.urls.channel_routing', + 'CONFIG': { + 'url': os.environ['RABBITMQ_URL'], + }, + }, +} diff --git a/testproject/testproject/urls.py b/testproject/testproject/urls.py index 89bb74d..7341fa0 100644 --- a/testproject/testproject/urls.py +++ b/testproject/testproject/urls.py @@ -1,17 +1,11 @@ -from django.conf.urls import url +from django.conf.urls import url from chtest import views -urlpatterns = [ - url(r'^$', views.index), -] - +urlpatterns = [url(r'^$', views.index)] try: from chtest import consumers - - channel_routing = { - "websocket.receive": consumers.ws_message, - "websocket.connect": consumers.ws_connect, -} + + channel_routing = {"websocket.receive": consumers.ws_message} except: pass From 1e2cd8ec7662f0cae156aa54f77af2cb659877c9 Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Tue, 17 Jan 2017 16:20:08 -0800 Subject: [PATCH 610/746] Fix for session shenanigans with WebsocketDemultiplexer (#486) * Fix for session shenanigans with WebsocketDemultiplexer Session data was getting lost in the demux due to the session getting saved after only the first connect/disconnect consumer was run. * fix for flake8 * flake8 again flake8 again --- channels/sessions.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/channels/sessions.py b/channels/sessions.py index 1856339..cad6791 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -42,7 +42,13 @@ def channel_session(func): def inner(message, *args, **kwargs): # Make sure there's NOT a channel_session already if hasattr(message, "channel_session"): - return func(message, *args, **kwargs) + try: + return func(message, *args, **kwargs) + finally: + # Persist session if needed + if message.channel_session.modified: + message.channel_session.save() + # Make sure there's a reply_channel if not message.reply_channel: raise ValueError( @@ -155,7 +161,13 @@ def http_session(func): def inner(message, *args, **kwargs): # Make sure there's NOT a http_session already if hasattr(message, "http_session"): - return func(message, *args, **kwargs) + try: + return func(message, *args, **kwargs) + finally: + # Persist session if needed (won't be saved if error happens) + if message.http_session is not None and message.http_session.modified: + message.http_session.save() + try: # We want to parse the WebSocket (or similar HTTP-lite) message # to get cookies and GET, but we need to add in a few things that From 044c422cddc2084cfa23ad89ad27a13abd2a3546 Mon Sep 17 00:00:00 2001 From: Leon Koole Date: Sat, 21 Jan 2017 02:23:21 +0100 Subject: [PATCH 611/746] Remove unnecessary http_session_user import (#493) --- docs/getting-started.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 8496d4a..366728d 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -433,7 +433,7 @@ Django authentication relies on). Channels can use Django sessions either from cookies (if you're running your websocket server on the same port as your main site, using something like Daphne), -or from a ``session_key`` GET parameter, which is works if you want to keep +or from a ``session_key`` GET parameter, which works if you want to keep running your HTTP requests through a WSGI server and offload WebSockets to a second server process on another port. @@ -461,7 +461,7 @@ chat to people with the same first letter of their username:: # In consumers.py from channels import Channel, Group from channels.sessions import channel_session - from channels.auth import http_session_user, channel_session_user, channel_session_user_from_http + from channels.auth import channel_session_user, channel_session_user_from_http # Connected to websocket.connect @channel_session_user_from_http @@ -668,7 +668,7 @@ first-letter-of-username chat from earlier:: # In consumers.py from channels import Channel, Group from channels.sessions import channel_session, enforce_ordering - from channels.auth import http_session_user, channel_session_user, channel_session_user_from_http + from channels.auth import channel_session_user, channel_session_user_from_http # Connected to websocket.connect @enforce_ordering(slight=True) From e07389eadb57e8257702fcbfc51bf6b1b6836696 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 22 Jan 2017 11:04:56 -0800 Subject: [PATCH 612/746] Fixed #496: Update readme to remove beta info --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 8d483b3..2ca9a71 100644 --- a/README.rst +++ b/README.rst @@ -18,13 +18,13 @@ task offloading and other asynchrony support to your code, using familiar Django design patterns and a flexible underlying framework that lets you not only customize behaviours but also write support for your own protocols and needs. -This is still **beta** software: the API is mostly settled, but might change -a bit as things develop. Once we hit ``1.0``, it will be stablized and a -deprecation policy will come in. - Documentation, installation and getting started instructions are at https://channels.readthedocs.io +Channels is an official Django Project and as such has a deprecation policy. +Details about what's deprecated or pending deprecation for each release is in +the `release notes `_. + Support can be obtained either here via issues, or in the ``#django-channels`` channel on Freenode. From f4d38ef77827b9d9e2c888d667897313f101585b Mon Sep 17 00:00:00 2001 From: Gary Reynolds Date: Mon, 23 Jan 2017 18:32:04 +1100 Subject: [PATCH 613/746] Remove unnecessary profane word in the documentation (#499) --- docs/testing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/testing.rst b/docs/testing.rst index a43cd62..efdfee5 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -153,8 +153,8 @@ For example:: def test_rooms(self): client = HttpClient() - user = User.objects.create_user(username='test', email='test@test.com', - password='123456') # fuck you security + user = User.objects.create_user( + username='test', email='test@test.com', password='123456') client.login(username='test', password='123456') client.send_and_consume('websocket.connect', '/rooms/') From 891eaf0051c64a82512a135523294196db2a3982 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 23 Jan 2017 10:13:11 -0800 Subject: [PATCH 614/746] Talk about URL routing/views in getting started (refs #92) --- docs/getting-started.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 366728d..32f67ad 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -228,6 +228,13 @@ And what our routing should look like in ``routing.py``:: route("websocket.disconnect", ws_disconnect), ] +Note that the ``http.request`` route is no longer present - if we leave it +out, then Django will route HTTP requests to the normal view system by default, +which is probably what you want. Even if you have a ``http.request`` route that +matches just a subset of paths or methods, the ones that don't match will still +fall through to the default handler, which passes it into URL routing and the +views. + With all that code, you now have a working set of a logic for a chat server. Test time! Run ``runserver``, open a browser and use that same JavaScript code in the developer console as before:: From 1542343392cc6993d463bffd9f2086c674abbfcd Mon Sep 17 00:00:00 2001 From: Lars Kreisz Date: Mon, 23 Jan 2017 21:00:05 +0100 Subject: [PATCH 615/746] Fix typo (#500) --- docs/generics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generics.rst b/docs/generics.rst index 5b1a4c5..ab9dcce 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -63,7 +63,7 @@ If you want to perfom more complicated routing, you'll need to override the remember, though, your channel names cannot change during runtime and must always be the same for as long as your process runs. -``BaseConsumer`` and all other generic consumers than inherit from it provide +``BaseConsumer`` and all other generic consumers that inherit from it provide two instance variables on the class: * ``self.message``, the :ref:`Message object ` representing the From 5c74ee587ec88660467af733fd3b42b9235ee3a1 Mon Sep 17 00:00:00 2001 From: Malyshev Artem Date: Tue, 24 Jan 2017 09:04:42 +0300 Subject: [PATCH 616/746] Installable benchmark package. (#501) * Make benchmark installable module. * Use passed url in the Benchmarker constructor. * Correct percentile output. * Import reactor globally. Since it used in the benchmarker. --- testproject/benchmark.py | 15 +++++++-------- testproject/setup.py | 7 +++++++ 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 testproject/setup.py diff --git a/testproject/benchmark.py b/testproject/benchmark.py index c0c4c91..387f1c9 100644 --- a/testproject/benchmark.py +++ b/testproject/benchmark.py @@ -3,9 +3,11 @@ from __future__ import unicode_literals import time import random import statistics -from autobahn.twisted.websocket import WebSocketClientProtocol, \ - WebSocketClientFactory - +from autobahn.twisted.websocket import ( + WebSocketClientProtocol, + WebSocketClientFactory, +) +from twisted.internet import reactor stats = {} @@ -92,9 +94,7 @@ class Benchmarker(object): self.rate = rate self.spawn = spawn self.messages = messages - self.factory = WebSocketClientFactory( - args.url, - ) + self.factory = WebSocketClientFactory(self.url) self.factory.protocol = MyClientProtocol self.factory.num_messages = self.messages self.factory.message_rate = self.rate @@ -180,7 +180,7 @@ class Benchmarker(object): print("-------") print("Sockets opened: %s" % len(stats)) if latencies: - print("Latency stats: Mean %.3fs Median %.3fs Stdev %.3f 95%% %.3fs 95%% %.3fs" % ( + print("Latency stats: Mean %.3fs Median %.3fs Stdev %.3f 95%% %.3fs 99%% %.3fs" % ( latency_mean, latency_median, latency_stdev, @@ -200,7 +200,6 @@ if __name__ == '__main__': import argparse from twisted.python import log - from twisted.internet import reactor # log.startLogging(sys.stdout) diff --git a/testproject/setup.py b/testproject/setup.py new file mode 100644 index 0000000..916e2da --- /dev/null +++ b/testproject/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='channels-benchmark', + py_modules=['benchmark'], + install_requires=['autobahn', 'Twisted'], +) From db8a4570c322a68a385f051e342445486754cfa8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 24 Jan 2017 09:51:42 -0800 Subject: [PATCH 617/746] Fixed #477: Only re-save session if it's not empty --- channels/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/sessions.py b/channels/sessions.py index cad6791..ee23f54 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -70,7 +70,7 @@ def channel_session(func): return func(message, *args, **kwargs) finally: # Persist session if needed - if session.modified: + if session.modified and not session.is_empty(): session.save() return inner From 9942c59851b76722ac0a8533102a3b886aff688b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 24 Jan 2017 09:57:07 -0800 Subject: [PATCH 618/746] Fixed #505: Add classifiers to setup.py --- setup.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ca72807..ac0567f 100644 --- a/setup.py +++ b/setup.py @@ -15,5 +15,20 @@ setup( 'Django>=1.8', 'asgiref>=0.13', 'daphne>=1.0.0', - ] + ], + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Web Environment', + 'Framework :: Django', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Topic :: Internet :: WWW/HTTP', + ], ) From 9ae27cb835763075a81ba3823d20bd68f3ace46b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 25 Jan 2017 16:59:35 -0800 Subject: [PATCH 619/746] Fixed #462: Don't actually close DB connections during tests --- channels/tests/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/channels/tests/base.py b/channels/tests/base.py index 7e7ddc9..46faf85 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -7,6 +7,7 @@ from functools import wraps from asgiref.inmemory import ChannelLayer as InMemoryChannelLayer from django.test.testcases import TestCase, TransactionTestCase +from django.db import close_old_connections from .. import DEFAULT_CHANNEL_LAYER from ..asgi import ChannelLayerWrapper, channel_layers @@ -134,7 +135,10 @@ class Client(object): consumer_started.send(sender=self.__class__) return consumer(message, **kwargs) finally: + # Copy Django's workaround so we don't actually close DB conns + consumer_finished.disconnect(close_old_connections) consumer_finished.send(sender=self.__class__) + consumer_finished.connect(close_old_connections) elif fail_on_none: raise AssertionError("Can't find consumer for message %s" % message) elif fail_on_none: From ef755e4c9d13924246e58165570d8bdde5c7743d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 25 Jan 2017 17:03:09 -0800 Subject: [PATCH 620/746] Remove optional multiplexer arg in generics docs --- docs/generics.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/generics.rst b/docs/generics.rst index ab9dcce..9e31233 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -197,14 +197,14 @@ Example using class-based consumer:: from channels.generic.websockets import WebsocketDemultiplexer, JsonWebsocketConsumer class EchoConsumer(websockets.JsonWebsocketConsumer): - def connect(self, message, multiplexer=None, **kwargs): + def connect(self, message, multiplexer, **kwargs): # Send data with the multiplexer multiplexer.send({"status": "I just connected!"}) - def disconnect(self, message, multiplexer=None, **kwargs): + def disconnect(self, message, multiplexer, **kwargs): print("Stream %s is closed" % multiplexer.stream) - def receive(self, content, multiplexer=None, **kwargs): + def receive(self, content, multiplexer, **kwargs): # Simple echo multiplexer.send({"original_message": content}) From 1a56ae8eb715a29a83ee32a26b953b131f5f9528 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 25 Jan 2017 17:04:06 -0800 Subject: [PATCH 621/746] Sort imports correctly. --- channels/tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/tests/base.py b/channels/tests/base.py index 46faf85..41a2eff 100644 --- a/channels/tests/base.py +++ b/channels/tests/base.py @@ -6,8 +6,8 @@ import string from functools import wraps from asgiref.inmemory import ChannelLayer as InMemoryChannelLayer -from django.test.testcases import TestCase, TransactionTestCase from django.db import close_old_connections +from django.test.testcases import TestCase, TransactionTestCase from .. import DEFAULT_CHANNEL_LAYER from ..asgi import ChannelLayerWrapper, channel_layers From 1d1101f7a95b9687e1ad5cdb984fc793cec619d8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 26 Jan 2017 10:42:48 -0800 Subject: [PATCH 622/746] Fixed #509: Docs for enforce_ordering now mirror post-1.0 --- docs/getting-started.rst | 60 +++++++++++++++------------------------- 1 file changed, 23 insertions(+), 37 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 32f67ad..6d9a6aa 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -640,37 +640,30 @@ sites with Channels - consumer ordering. Because Channels is a distributed system that can have many workers, by default it just processes messages in the order the workers get them off the queue. -It's entirely feasible for a WebSocket interface server to send out a ``connect`` -and a ``receive`` message close enough together that a second worker will pick -up and start processing the ``receive`` message before the first worker has -finished processing the ``connect`` worker. +It's entirely feasible for a WebSocket interface server to send out two +``receive`` messages close enough together that a second worker will pick +up and start processing the second message before the first worker has +finished processing the first. This is particularly annoying if you're storing things in the session in the -``connect`` consumer and trying to get them in the ``receive`` consumer - because +one consumer and trying to get them in the other consumer - because the ``connect`` consumer hasn't exited, its session hasn't saved. You'd get the same effect if someone tried to request a view before the login view had finished -processing, but there you're not expecting that page to run after the login, -whereas you'd naturally expect ``receive`` to run after ``connect``. +processing, of course, but HTTP requests usually come in a bit slower from clients. Channels has a solution - the ``enforce_ordering`` decorator. All WebSocket messages contain an ``order`` key, and this decorator uses that to make sure that -messages are consumed in the right order, in one of two modes: - -* Slight ordering: Message 0 (``websocket.connect``) is done first, all others - are unordered - -* Strict ordering: All messages are consumed strictly in sequence +messages are consumed in the right order. In addition, the ``connect`` message +blocks the socket opening until it's responded to, so you are always guaranteed +that ``connect`` will run before any ``receives`` even without the decorator. The decorator uses ``channel_session`` to keep track of what numbered messages have been processed, and if a worker tries to run a consumer on an out-of-order message, it raises the ``ConsumeLater`` exception, which puts the message back on the channel it came from and tells the worker to work on another message. -There's a cost to using ``enforce_ordering``, which is why it's an optional -decorator, and the cost is much greater in *strict* mode than it is in -*slight* mode. Generally you'll want to use *slight* mode for most session-based WebSocket -and other "continuous protocol" things. Here's an example, improving our -first-letter-of-username chat from earlier:: +There's a high cost to using ``enforce_ordering``, which is why it's an optional +decorator. Here's an example of it being used # In consumers.py from channels import Channel, Group @@ -678,39 +671,32 @@ first-letter-of-username chat from earlier:: from channels.auth import channel_session_user, channel_session_user_from_http # Connected to websocket.connect - @enforce_ordering(slight=True) @channel_session_user_from_http def ws_add(message): + # This doesn't need a decorator - it always runs separately + message.channel_session['sent'] = 0 # Add them to the right group - Group("chat-%s" % message.user.username[0]).add(message.reply_channel) + Group("chat").add(message.reply_channel) + # Accept the socket + message.reply_channel.send({"accept": True}) # Connected to websocket.receive - @enforce_ordering(slight=True) + @enforce_ordering @channel_session_user def ws_message(message): - Group("chat-%s" % message.user.username[0]).send({ - "text": message['text'], + # Without enforce_ordering this wouldn't work right + message.channel_session['sent'] = message.channel_session['sent'] + 1 + Group("chat").send({ + "text": "%s: %s" % (message.channel_session['sent'], message['text']), }) # Connected to websocket.disconnect - @enforce_ordering(slight=True) @channel_session_user def ws_disconnect(message): - Group("chat-%s" % message.user.username[0]).discard(message.reply_channel) - -Slight ordering does mean that it's possible for a ``disconnect`` message to -get processed before a ``receive`` message, but that's fine in this case; -the client is disconnecting anyway, they don't care about those pending messages. - -Strict ordering is the default as it's the most safe; to use it, just call -the decorator without arguments:: - - @enforce_ordering - def ws_message(message): - ... + Group("chat").discard(message.reply_channel) Generally, the performance (and safety) of your ordering is tied to your -session backend's performance. Make sure you choose session backend wisely +session backend's performance. Make sure you choose a session backend wisely if you're going to rely heavily on ``enforce_ordering``. From a1a1ace23da668761cbaeccc4830f8dd36227aca Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 26 Jan 2017 10:44:08 -0800 Subject: [PATCH 623/746] Remove slight ordering from generics docs --- channels/generic/websockets.py | 1 - docs/generics.rst | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 555b165..b7a0250 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -25,7 +25,6 @@ class WebsocketConsumer(BaseConsumer): http_user = False # Set to True if you want the class to enforce ordering for you - slight_ordering = False strict_ordering = False groups = None diff --git a/docs/generics.rst b/docs/generics.rst index 9e31233..c604930 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -88,9 +88,8 @@ The basic WebSocket generic consumer is used like this:: # (you don't need channel_session_user, this implies it) http_user = True - # Set to True if you want them, else leave out + # Set to True if you want it, else leave it out strict_ordering = False - slight_ordering = False def connection_groups(self, **kwargs): """ @@ -135,9 +134,8 @@ The JSON-enabled consumer looks slightly different:: class MyConsumer(JsonWebsocketConsumer): - # Set to True if you want them, else leave out + # Set to True if you want it, else leave it out strict_ordering = False - slight_ordering = False def connection_groups(self, **kwargs): """ From 57ed7747f7a8a3d941d2e4fe6532857170d94b4a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 26 Jan 2017 10:47:54 -0800 Subject: [PATCH 624/746] Handle slight ordering not being set --- channels/generic/websockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index b7a0250..3e77e3b 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -46,7 +46,7 @@ class WebsocketConsumer(BaseConsumer): # Ordering decorators if self.strict_ordering: return enforce_ordering(handler, slight=False) - elif self.slight_ordering: + elif getattr(self, "slight_ordering", False): raise ValueError("Slight ordering is now always on. Please remove `slight_ordering=True`.") else: return handler From 5fc5267d2a25ef35a6a7c661203050c885ade2c5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 27 Jan 2017 09:45:41 -0800 Subject: [PATCH 625/746] Add code indent --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 6d9a6aa..f196b51 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -663,7 +663,7 @@ message, it raises the ``ConsumeLater`` exception, which puts the message back on the channel it came from and tells the worker to work on another message. There's a high cost to using ``enforce_ordering``, which is why it's an optional -decorator. Here's an example of it being used +decorator. Here's an example of it being used:: # In consumers.py from channels import Channel, Group From 558d66a6b285a9a3a2cb7f6171cf7e2aa9bab594 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 30 Jan 2017 17:07:48 -0800 Subject: [PATCH 626/746] Fixed #512: Give rundelay a configurable sleep interval Also reduced the default interval to 1s. --- .../delay/management/commands/rundelay.py | 5 ++++ channels/delay/worker.py | 27 ++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/channels/delay/management/commands/rundelay.py b/channels/delay/management/commands/rundelay.py index 0a3e719..b47b1e2 100644 --- a/channels/delay/management/commands/rundelay.py +++ b/channels/delay/management/commands/rundelay.py @@ -17,6 +17,10 @@ class Command(BaseCommand): '--layer', action='store', dest='layer', default=DEFAULT_CHANNEL_LAYER, help='Channel layer alias to use, if not the default.', ) + parser.add_argument( + '--sleep', action='store', dest='sleep', default=1, type=float, + help='Amount of time to sleep between checks, in seconds.', + ) def handle(self, *args, **options): self.verbosity = options.get("verbosity", 1) @@ -33,6 +37,7 @@ class Command(BaseCommand): try: worker = Worker( channel_layer=self.channel_layer, + database_sleep_duration=options['sleep'], ) worker.run() except KeyboardInterrupt: diff --git a/channels/delay/worker.py b/channels/delay/worker.py index c2e554b..689a588 100644 --- a/channels/delay/worker.py +++ b/channels/delay/worker.py @@ -20,11 +20,13 @@ class Worker(object): self, channel_layer, signal_handlers=True, + database_sleep_duration=1, ): self.channel_layer = channel_layer self.signal_handlers = signal_handlers self.termed = False self.in_job = False + self.database_sleep_duration = database_sleep_duration def install_signal_handler(self): signal.signal(signal.SIGTERM, self.sigterm_handler) @@ -44,9 +46,11 @@ class Worker(object): logger.info("Listening on asgi.delay") + last_delay_check = 0 + while not self.termed: self.in_job = False - channel, content = self.channel_layer.receive_many(['asgi.delay']) + channel, content = self.channel_layer.receive(['asgi.delay'], block=False) self.in_job = True if channel is not None: @@ -71,12 +75,17 @@ class Worker(object): logger.error("Invalid message received: %s:%s", err.error_dict.keys(), err.messages) break message.save() - # check for messages to send - if not DelayedMessage.objects.is_due().count(): - logger.debug("No delayed messages waiting.") - time.sleep(0.01) - continue - for message in DelayedMessage.objects.is_due().all(): - logger.info("Delayed message due. Sending message to channel %s", message.channel_name) - message.send(channel_layer=self.channel_layer) + else: + # Sleep for a short interval so we don't idle hot. + time.sleep(0.1) + + # check for messages to send + if time.time() - last_delay_check > self.database_sleep_duration: + if DelayedMessage.objects.is_due().exists(): + for message in DelayedMessage.objects.is_due().all(): + logger.info("Sending delayed message to channel %s", message.channel_name) + message.send(channel_layer=self.channel_layer) + else: + logger.debug("No delayed messages waiting.") + last_delay_check = time.time() From 6d71106c3c6a8924f75e5058cd6c54e765af3b94 Mon Sep 17 00:00:00 2001 From: Maik Hoepfel Date: Tue, 31 Jan 2017 02:25:07 +0100 Subject: [PATCH 627/746] Simplify testing infrastructure (#515) * Mark runtests helper executable * Bump required version of asgiref We're probably making our life easier when we keep the Channels dependencies roughly in sync. As a 1.0 release was made, I suggest to require it. * Simplify tox and Travis configuration I hopefully simplified the tox configuration by following what I did in the other four Channels projects. I then had a good look at tox-travis and decided to remove it. It does add a layer of indirection with, IMHO, not enough gain. To understand what Travis is doing, one would need to consider two files (and understand tox-travis) instead of just one file. It also introduces another point of failure. What pushed me over was that there's a bug with env matching (https://github.com/ryanhiebert/tox-travis/issues/55) and tox or tox-travis seem to mask an Exception (https://travis-ci.org/django/channels/jobs/195950971#L195) that would be hard to debug. The draw back is that we duplicate the Django dependency matrix, and the commands that are executed in Travis and tox. We could add a "--with-qa" flag to runtests.py to have it execute flake8 and isort to rectify the latter. I extracted test dependencies as I did for asgi_redis. * Document supported versions --- .travis.yml | 18 ++++++++++++++++-- README.rst | 6 ++++++ runtests.py | 0 setup.py | 5 ++++- tox.ini | 23 +++-------------------- 5 files changed, 29 insertions(+), 23 deletions(-) mode change 100644 => 100755 runtests.py diff --git a/.travis.yml b/.travis.yml index 3e7de1d..3886439 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,22 @@ sudo: false + language: python + python: - "2.7" - "3.4" - "3.5" -install: pip install tox-travis -script: tox + +env: + - DJANGO="Django>=1.8,<1.9" + - DJANGO="Django>=1.9,<1.10" + - DJANGO="Django>=1.10,<1.11" + +install: + - pip install $DJANGO -e .[tests] + - pip freeze + +script: + - python runtests.py + - flake8 + - isort --check-only --recursive channels diff --git a/README.rst b/README.rst index 2ca9a71..5d1877f 100644 --- a/README.rst +++ b/README.rst @@ -33,6 +33,12 @@ You'll likely also want to ``asgi_redis`` to provide the Redis channel layer. See our `installation `_ and `getting started `_ docs for more. +Dependencies +------------ + +All Channels projects currently support Python 2.7, 3.4 and 3.5. `channels` supports all released +Django versions, namely 1.8-1.10. + Contributing ------------ diff --git a/runtests.py b/runtests.py old mode 100644 new mode 100755 diff --git a/setup.py b/setup.py index ac0567f..e69f558 100644 --- a/setup.py +++ b/setup.py @@ -13,9 +13,12 @@ setup( include_package_data=True, install_requires=[ 'Django>=1.8', - 'asgiref>=0.13', + 'asgiref>=1.0.0', 'daphne>=1.0.0', ], + extras_require={ + 'tests': ['coverage', 'mock', 'tox', 'flake8>=2.0,<3.0', 'isort'] + }, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Web Environment', diff --git a/tox.ini b/tox.ini index 8b95ac3..1937445 100644 --- a/tox.ini +++ b/tox.ini @@ -1,33 +1,16 @@ [tox] -skipsdist = True envlist = - {py27}-django-{18,19,110} - {py34}-django-{18,19,110} - {py35}-django-{18,19,110} + {py27,34,35}-django-{18,19,110} {py27,py35}-flake8 isort -[tox:travis] -2.7 = py27, isort - [testenv] -setenv = - PYTHONPATH = {toxinidir}:{toxinidir} +extras = tests deps = - autobahn - coverage - daphne - asgiref>=0.9 - six - redis==2.10.5 - py27: mock - flake8: flake8>=2.0,<3.0 - isort: isort django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 django-110: Django>=1.10,<1.11 commands = flake8: flake8 - isort: isort -c -rc channels + isort: isort --check-only --recursive channels django: coverage run --parallel-mode {toxinidir}/runtests.py - From 20af4e31b401c6589daeac26fbca89e1b512da97 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 1 Feb 2017 12:03:57 -0800 Subject: [PATCH 628/746] Releasing 1.0.3 --- CHANGELOG.txt | 17 +++++++++++++++++ channels/__init__.py | 2 +- docs/releases/1.0.3.rst | 26 ++++++++++++++++++++++++++ docs/releases/index.rst | 1 + 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 docs/releases/1.0.3.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 692c470..079c411 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,23 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases +1.0.2 (2017-02-01) +------------------ + +* Database connections are no longer force-closed after each test is run. + +* Channel sessions are not re-saved if they're empty even if they're marked as + modified, allowing logout to work correctly. + +* WebsocketDemultiplexer now correctly does sessions for the second/third/etc. + connect and disconnect handlers. + +* Request reading timeouts now correctly return 408 rather than erroring out. + +* The ``rundelay`` delay server now only polls the database once per second, + and this interval is configurable with the ``--sleep`` option. + + 1.0.2 (2017-01-12) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index 3ba52f8..f3e910f 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.0.2" +__version__ = "1.0.3" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/releases/1.0.3.rst b/docs/releases/1.0.3.rst new file mode 100644 index 0000000..8179168 --- /dev/null +++ b/docs/releases/1.0.3.rst @@ -0,0 +1,26 @@ +1.0.3 Release Notes +=================== + +Channels 1.0.3 is a minor bugfix release, released on 2017/02/01. + +Changes +------- + +* Database connections are no longer force-closed after each test is run. + +* Channel sessions are not re-saved if they're empty even if they're marked as + modified, allowing logout to work correctly. + +* WebsocketDemultiplexer now correctly does sessions for the second/third/etc. + connect and disconnect handlers. + +* Request reading timeouts now correctly return 408 rather than erroring out. + +* The ``rundelay`` delay server now only polls the database once per second, + and this interval is configurable with the ``--sleep`` option. + + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 856f565..f4f57a5 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -7,3 +7,4 @@ Release Notes 1.0.0 1.0.1 1.0.2 + 1.0.3 From e189254d546a63fd69a479e5791710a9c8d9e0e7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 1 Feb 2017 14:22:54 -0800 Subject: [PATCH 629/746] Typo in changelog --- CHANGELOG.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 079c411..53d26cf 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,7 +2,7 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases -1.0.2 (2017-02-01) +1.0.3 (2017-02-01) ------------------ * Database connections are no longer force-closed after each test is run. From 75f668f9e554d91a44375b6ca809c2f4a59aa086 Mon Sep 17 00:00:00 2001 From: Maik Hoepfel Date: Fri, 3 Feb 2017 19:13:00 +0100 Subject: [PATCH 630/746] Docs: Python 3.3 is not supported any more (#519) Just a tiny fix to ensure that we don't promise Python 3.3 support. --- docs/faqs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faqs.rst b/docs/faqs.rst index 4fdc706..e2bd585 100755 --- a/docs/faqs.rst +++ b/docs/faqs.rst @@ -152,7 +152,7 @@ Are channels Python 2, 3 or 2+3? -------------------------------- Django-channels and all of its dependencies are compatible with Python 2.7, -3.3, and higher. This includes the parts of Twisted that some of the Channels +3.4, and higher. This includes the parts of Twisted that some of the Channels packages (like daphne) use. From b14bbeebe4bf5df88f1bde6941b89fba9cf02bbe Mon Sep 17 00:00:00 2001 From: Matthias K Date: Tue, 7 Feb 2017 19:20:06 +0100 Subject: [PATCH 631/746] Fix two typos (#521) --- channels/exceptions.py | 2 +- channels/routing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/exceptions.py b/channels/exceptions.py index 1827629..502358e 100644 --- a/channels/exceptions.py +++ b/channels/exceptions.py @@ -5,7 +5,7 @@ import six class ConsumeLater(Exception): """ Exception that says that the current message should be re-queued back - onto its channel as it's not ready to be consumd yet (e.g. global order + onto its channel as it's not ready to be consumed yet (e.g. global order is being enforced) """ pass diff --git a/channels/routing.py b/channels/routing.py index 11a5afc..8330537 100644 --- a/channels/routing.py +++ b/channels/routing.py @@ -177,7 +177,7 @@ class Route(object): class RouteClass(Route): """ Like Route, but targets a class-based consumer rather than a functional - one, meaning it looks for a (class) method called "channels()" on the + one, meaning it looks for a (class) method called "channel_names()" on the object rather than having a single channel passed in. """ From 41857987316235b06a93d3e348e432bf8745abb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Oubi=C3=B1a?= Date: Sun, 12 Feb 2017 01:41:36 +0100 Subject: [PATCH 632/746] Typo in Example using class-based consumer (#526) Typo in Example using class-based consumer from section "2.6.3 WebSocket Multiplexing" --- docs/generics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/generics.rst b/docs/generics.rst index c604930..459ce36 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -194,7 +194,7 @@ Example using class-based consumer:: from channels.generic.websockets import WebsocketDemultiplexer, JsonWebsocketConsumer - class EchoConsumer(websockets.JsonWebsocketConsumer): + class EchoConsumer(JsonWebsocketConsumer): def connect(self, message, multiplexer, **kwargs): # Send data with the multiplexer multiplexer.send({"status": "I just connected!"}) @@ -207,7 +207,7 @@ Example using class-based consumer:: multiplexer.send({"original_message": content}) - class AnotherConsumer(websockets.JsonWebsocketConsumer): + class AnotherConsumer(JsonWebsocketConsumer): def receive(self, content, multiplexer=None, **kwargs): # Some other actions here pass From 95c9925fe4a2e824dc11cbc61ee26a1b13532c56 Mon Sep 17 00:00:00 2001 From: Pierre Chiquet Date: Tue, 14 Feb 2017 18:48:00 +0100 Subject: [PATCH 633/746] Update Binding to support models with UUIDField as primary key (#528) * Add custom TestUUIDModel for auto tests * Update Binding to support models with UUIDField as primary key Add and fix test_trigger_outbound_create_non_auto_pk. Before updating pre_save_receiver, this new test failed with this error: ====================================================================== FAIL: test_trigger_outbound_create_non_auto_pk (channels.tests.test_binding.TestsBinding) ---------------------------------------------------------------------- Traceback (most recent call last): File "C:\Users\User\git\contribs\python\channels\channels\tests\test_binding.py", line 85, in test_trigger_outbound_create_non_auto_pk self.assertEqual(received['payload']['action'], 'create') AssertionError: u'update' != u'create' --- channels/binding/base.py | 3 ++- channels/tests/models.py | 12 +++++++++++ channels/tests/settings.py | 3 ++- channels/tests/test_binding.py | 38 +++++++++++++++++++++++++++++++++- 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 channels/tests/models.py diff --git a/channels/binding/base.py b/channels/binding/base.py index 85ec720..88ae8e8 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -120,7 +120,8 @@ class Binding(object): @classmethod def pre_save_receiver(cls, instance, **kwargs): - cls.pre_change_receiver(instance, CREATE if instance.pk is None else UPDATE) + creating = instance._state.adding + cls.pre_change_receiver(instance, CREATE if creating else UPDATE) @classmethod def post_save_receiver(cls, instance, created, **kwargs): diff --git a/channels/tests/models.py b/channels/tests/models.py new file mode 100644 index 0000000..4b1da2d --- /dev/null +++ b/channels/tests/models.py @@ -0,0 +1,12 @@ +from uuid import uuid4 + +from django.db import models + + +class TestUUIDModel(models.Model): + """ + Simple model with UUIDField as primary key for tests. + """ + + id = models.UUIDField(primary_key=True, default=uuid4) + name = models.CharField(max_length=255) diff --git a/channels/tests/settings.py b/channels/tests/settings.py index 2ddf8ac..47fc407 100644 --- a/channels/tests/settings.py +++ b/channels/tests/settings.py @@ -6,7 +6,8 @@ INSTALLED_APPS = ( 'django.contrib.sessions', 'django.contrib.admin', 'channels', - 'channels.delay' + 'channels.delay', + 'channels.tests' ) DATABASES = { diff --git a/channels/tests/test_binding.py b/channels/tests/test_binding.py index 6b32a02..edd5126 100644 --- a/channels/tests/test_binding.py +++ b/channels/tests/test_binding.py @@ -6,7 +6,7 @@ from channels import route from channels.binding.base import CREATE, DELETE, UPDATE from channels.binding.websockets import WebsocketBinding from channels.generic.websockets import WebsocketDemultiplexer -from channels.tests import ChannelTestCase, HttpClient, apply_routes +from channels.tests import ChannelTestCase, HttpClient, apply_routes, models User = get_user_model() @@ -55,6 +55,42 @@ class TestsBinding(ChannelTestCase): received = client.receive() self.assertIsNone(received) + def test_trigger_outbound_create_non_auto_pk(self): + + class TestBinding(WebsocketBinding): + model = models.TestUUIDModel + stream = 'test' + fields = ['name'] + + @classmethod + def group_names(cls, instance): + return ["testuuidmodels"] + + def has_permission(self, user, action, pk): + return True + + client = HttpClient() + client.join_group('testuuidmodels') + + instance = models.TestUUIDModel.objects.create(name='testname') + + received = client.receive() + self.assertTrue('payload' in received) + self.assertTrue('action' in received['payload']) + self.assertTrue('data' in received['payload']) + self.assertTrue('name' in received['payload']['data']) + self.assertTrue('model' in received['payload']) + self.assertTrue('pk' in received['payload']) + + self.assertEqual(received['payload']['action'], 'create') + self.assertEqual(received['payload']['model'], 'tests.testuuidmodel') + self.assertEqual(received['payload']['pk'], str(instance.pk)) + + self.assertEqual(received['payload']['data']['name'], 'testname') + + received = client.receive() + self.assertIsNone(received) + def test_trigger_outbound_create_exclude(self): class TestBinding(WebsocketBinding): model = User From 13472369ebf18d2925302aa1a407405b67a3e7c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B0=D1=81=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=91=D1=83?= =?UTF-8?q?=D0=BB=D0=B0=D1=82?= Date: Tue, 14 Feb 2017 20:50:01 +0300 Subject: [PATCH 634/746] fix tox (#516) --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 1937445..b1c3ce5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = - {py27,34,35}-django-{18,19,110} - {py27,py35}-flake8 + py{27,34,35}-django-{18,19,110} + py{27,35}-flake8 isort [testenv] From 672de2b2a324a9964484d3379ee16c3738ed9421 Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Thu, 16 Feb 2017 19:22:23 +0100 Subject: [PATCH 635/746] Separate tests into own directory. (#531) * Move project tests to its own directory. * Install mock test dependency for Python2 only. * Do not install tox inside tox environment. * Exclude tests from sdist. * Use recent pip on Travis-CI. --- .coveragerc | 5 ++--- .travis.yml | 5 +++++ MANIFEST.in | 1 + channels/test/__init__.py | 2 ++ channels/{tests => test}/base.py | 0 channels/{tests => test}/http.py | 0 channels/tests/__init__.py | 11 +++++++++-- docs/testing.rst | 18 +++++++++--------- patchinator.py | 14 +++++++------- runtests.py | 4 ++-- setup.py | 9 +++++++-- tests/__init__.py | 0 {channels/tests => tests}/a_file | 0 {channels/tests => tests}/models.py | 0 {channels/tests => tests}/settings.py | 4 ++-- {channels/tests => tests}/test_binding.py | 3 ++- {channels/tests => tests}/test_delay.py | 2 +- {channels/tests => tests}/test_generic.py | 2 +- {channels/tests => tests}/test_handler.py | 2 +- {channels/tests => tests}/test_http.py | 3 +-- {channels/tests => tests}/test_management.py | 0 {channels/tests => tests}/test_request.py | 2 +- {channels/tests => tests}/test_routing.py | 12 ++++++------ {channels/tests => tests}/test_sessions.py | 2 +- {channels/tests => tests}/test_worker.py | 2 +- 25 files changed, 61 insertions(+), 42 deletions(-) create mode 100644 MANIFEST.in create mode 100644 channels/test/__init__.py rename channels/{tests => test}/base.py (100%) rename channels/{tests => test}/http.py (100%) create mode 100644 tests/__init__.py rename {channels/tests => tests}/a_file (100%) rename {channels/tests => tests}/models.py (100%) rename {channels/tests => tests}/settings.py (84%) rename {channels/tests => tests}/test_binding.py (99%) rename {channels/tests => tests}/test_delay.py (98%) rename {channels/tests => tests}/test_generic.py (98%) rename {channels/tests => tests}/test_handler.py (99%) rename {channels/tests => tests}/test_http.py (89%) rename {channels/tests => tests}/test_management.py (100%) rename {channels/tests => tests}/test_request.py (99%) rename {channels/tests => tests}/test_routing.py (94%) rename {channels/tests => tests}/test_sessions.py (99%) rename {channels/tests => tests}/test_worker.py (99%) diff --git a/.coveragerc b/.coveragerc index ef6d66c..3f06bb6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,12 +1,12 @@ [run] branch = True source = channels -omit = channels/tests/* +omit = tests/* [report] show_missing = True skip_covered = True -omit = channels/tests/* +omit = tests/* [html] directory = coverage_html @@ -21,4 +21,3 @@ django_18 = .tox/py27-django-19/lib/python2.7 .tox/py34-django-19/lib/python3.4 .tox/py35-django-19/lib/python3.5 - diff --git a/.travis.yml b/.travis.yml index 3886439..f38f5a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,12 @@ env: - DJANGO="Django>=1.9,<1.10" - DJANGO="Django>=1.10,<1.11" +cache: + directories: + - $HOME/.cache/pip/wheels + install: + - pip install -U pip wheel setuptools - pip install $DJANGO -e .[tests] - pip freeze diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..aae9579 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-exclude tests * diff --git a/channels/test/__init__.py b/channels/test/__init__.py new file mode 100644 index 0000000..0c957f3 --- /dev/null +++ b/channels/test/__init__.py @@ -0,0 +1,2 @@ +from .base import TransactionChannelTestCase, ChannelTestCase, Client, apply_routes # NOQA isort:skip +from .http import HttpClient # NOQA isort:skip diff --git a/channels/tests/base.py b/channels/test/base.py similarity index 100% rename from channels/tests/base.py rename to channels/test/base.py diff --git a/channels/tests/http.py b/channels/test/http.py similarity index 100% rename from channels/tests/http.py rename to channels/test/http.py diff --git a/channels/tests/__init__.py b/channels/tests/__init__.py index 0c957f3..27ae0e3 100644 --- a/channels/tests/__init__.py +++ b/channels/tests/__init__.py @@ -1,2 +1,9 @@ -from .base import TransactionChannelTestCase, ChannelTestCase, Client, apply_routes # NOQA isort:skip -from .http import HttpClient # NOQA isort:skip +import warnings + +warnings.warn( + "channels.tests package is deprecated. Use channels.test", + DeprecationWarning, +) + +from channels.test.base import TransactionChannelTestCase, ChannelTestCase, Client, apply_routes # NOQA isort:skip +from channels.test.http import HttpClient # NOQA isort:skip diff --git a/docs/testing.rst b/docs/testing.rst index efdfee5..1ac48bd 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -13,7 +13,7 @@ however, so you can easily write tests and check what your consumers are sending ChannelTestCase --------------- -If your tests inherit from the ``channels.tests.ChannelTestCase`` base class, +If your tests inherit from the ``channels.test.ChannelTestCase`` base class, whenever you run tests your channel layer will be swapped out for a captive in-memory layer, meaning you don't need an external server running to run tests. @@ -24,7 +24,7 @@ To inject a message onto the layer, simply call ``Channel.send()`` inside any test method on a ``ChannelTestCase`` subclass, like so:: from channels import Channel - from channels.tests import ChannelTestCase + from channels.test import ChannelTestCase class MyTests(ChannelTestCase): def test_a_thing(self): @@ -49,7 +49,7 @@ and post the square of it to the ``"result"`` channel:: from channels import Channel - from channels.tests import ChannelTestCase + from channels.test import ChannelTestCase class MyTests(ChannelTestCase): def test_a_thing(self): @@ -70,7 +70,7 @@ object from ``get_next_message`` to the constructor of the class. To test replie use the ``reply_channel`` property on the ``Message`` object. For example:: from channels import Channel - from channels.tests import ChannelTestCase + from channels.test import ChannelTestCase from myapp.consumers import MyConsumer @@ -95,7 +95,7 @@ the entire channel layer is flushed each time a test is run, so it's safe to do group adds and sends during a test. For example:: from channels import Group - from channels.tests import ChannelTestCase + from channels.test import ChannelTestCase class MyTests(ChannelTestCase): def test_a_thing(self): @@ -118,7 +118,7 @@ to run appointed consumer for the next message, ``receive`` to getting replies f Very often you may need to ``send`` and than call a consumer one by one, for this purpose use ``send_and_consume`` method:: - from channels.tests import ChannelTestCase, Client + from channels.test import ChannelTestCase, Client class MyTests(ChannelTestCase): @@ -146,7 +146,7 @@ For example:: # tests.py from channels import Group - from channels.tests import ChannelTestCase, HttpClient + from channels.test import ChannelTestCase, HttpClient class RoomsTests(ChannelTestCase): @@ -196,7 +196,7 @@ want to testing your consumers in more isolate and atomic way, it will be simpler with ``apply_routes`` contextmanager and decorator for your ``ChannelTestCase``. It takes list of routes that you want to use and overwrite existing routes:: - from channels.tests import ChannelTestCase, HttpClient, apply_routes + from channels.test import ChannelTestCase, HttpClient, apply_routes class MyTests(ChannelTestCase): @@ -220,7 +220,7 @@ make some changes with target model and check received message. Lets test ``IntegerValueBinding`` from :doc:`data binding ` with creating:: - from channels.tests import ChannelTestCase, HttpClient + from channels.test import ChannelTestCase, HttpClient from channels.signals import consumer_finished class TestIntegerValueBinding(ChannelTestCase): diff --git a/patchinator.py b/patchinator.py index fe787a1..661a659 100644 --- a/patchinator.py +++ b/patchinator.py @@ -121,9 +121,9 @@ global_transforms = [ Replacement(r"from channels import", r"from django.channels import"), Replacement(r"from channels.([a-zA-Z0-9_\.]+) import", r"from django.channels.\1 import"), Replacement(r"from .handler import", r"from django.core.handlers.asgi import"), - Replacement(r"from django.channels.tests import", r"from django.test.channels import"), + Replacement(r"from django.channels.test import", r"from django.test.channels import"), Replacement(r"from django.channels.handler import", r"from django.core.handlers.asgi import"), - Replacement(r"channels.tests.test_routing", r"channels_tests.test_routing"), + Replacement(r"tests.test_routing", r"channels_tests.test_routing"), Replacement(r"django.core.urlresolvers", r"django.urls"), ] @@ -201,22 +201,22 @@ class Patchinator(object): ), # Tests FileMap( - "channels/tests/base.py", "django/test/channels.py", python_transforms, + "channels/test/base.py", "django/test/channels.py", python_transforms, ), NewFile( "tests/channels_tests/__init__.py", ), FileMap( - "channels/tests/test_handler.py", "tests/channels_tests/test_handler.py", python_transforms, + "tests/test_handler.py", "tests/channels_tests/test_handler.py", python_transforms, ), FileMap( - "channels/tests/test_routing.py", "tests/channels_tests/test_routing.py", python_transforms, + "tests/test_routing.py", "tests/channels_tests/test_routing.py", python_transforms, ), FileMap( - "channels/tests/test_request.py", "tests/channels_tests/test_request.py", python_transforms, + "tests/test_request.py", "tests/channels_tests/test_request.py", python_transforms, ), FileMap( - "channels/tests/test_sessions.py", "tests/channels_tests/test_sessions.py", python_transforms, + "tests/test_sessions.py", "tests/channels_tests/test_sessions.py", python_transforms, ), # Docs FileMap( diff --git a/runtests.py b/runtests.py index 1d60d79..925b27b 100755 --- a/runtests.py +++ b/runtests.py @@ -7,9 +7,9 @@ from django.conf import settings from django.test.utils import get_runner if __name__ == "__main__": - os.environ['DJANGO_SETTINGS_MODULE'] = "channels.tests.settings" + os.environ['DJANGO_SETTINGS_MODULE'] = "tests.settings" django.setup() TestRunner = get_runner(settings) test_runner = TestRunner() - failures = test_runner.run_tests(["channels.tests"]) + failures = test_runner.run_tests(["tests"]) sys.exit(bool(failures)) diff --git a/setup.py b/setup.py index e69f558..e1adedd 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( author_email='foundation@djangoproject.com', description="Brings event-driven capabilities to Django with a channel system. Django 1.8 and up only.", license='BSD', - packages=find_packages(), + packages=find_packages(exclude=['tests']), include_package_data=True, install_requires=[ 'Django>=1.8', @@ -17,7 +17,12 @@ setup( 'daphne>=1.0.0', ], extras_require={ - 'tests': ['coverage', 'mock', 'tox', 'flake8>=2.0,<3.0', 'isort'] + 'tests': [ + 'coverage', + 'mock ; python_version < "3.0"', + 'flake8>=2.0,<3.0', + 'isort', + ] }, classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channels/tests/a_file b/tests/a_file similarity index 100% rename from channels/tests/a_file rename to tests/a_file diff --git a/channels/tests/models.py b/tests/models.py similarity index 100% rename from channels/tests/models.py rename to tests/models.py diff --git a/channels/tests/settings.py b/tests/settings.py similarity index 84% rename from channels/tests/settings.py rename to tests/settings.py index 47fc407..bebbd2a 100644 --- a/channels/tests/settings.py +++ b/tests/settings.py @@ -7,7 +7,7 @@ INSTALLED_APPS = ( 'django.contrib.admin', 'channels', 'channels.delay', - 'channels.tests' + 'tests', ) DATABASES = { @@ -22,7 +22,7 @@ CHANNEL_LAYERS = { 'ROUTING': [], }, 'fake_channel': { - 'BACKEND': 'channels.tests.test_management.FakeChannelLayer', + 'BACKEND': 'tests.test_management.FakeChannelLayer', 'ROUTING': [], } } diff --git a/channels/tests/test_binding.py b/tests/test_binding.py similarity index 99% rename from channels/tests/test_binding.py rename to tests/test_binding.py index edd5126..4f86f50 100644 --- a/channels/tests/test_binding.py +++ b/tests/test_binding.py @@ -6,7 +6,8 @@ from channels import route from channels.binding.base import CREATE, DELETE, UPDATE from channels.binding.websockets import WebsocketBinding from channels.generic.websockets import WebsocketDemultiplexer -from channels.tests import ChannelTestCase, HttpClient, apply_routes, models +from channels.test import ChannelTestCase, HttpClient, apply_routes +from tests import models User = get_user_model() diff --git a/channels/tests/test_delay.py b/tests/test_delay.py similarity index 98% rename from channels/tests/test_delay.py rename to tests/test_delay.py index 8d22d0f..08cb194 100644 --- a/channels/tests/test_delay.py +++ b/tests/test_delay.py @@ -8,7 +8,7 @@ from django.utils import timezone from channels import DEFAULT_CHANNEL_LAYER, Channel, channel_layers from channels.delay.models import DelayedMessage from channels.delay.worker import Worker -from channels.tests import ChannelTestCase +from channels.test import ChannelTestCase try: from unittest import mock diff --git a/channels/tests/test_generic.py b/tests/test_generic.py similarity index 98% rename from channels/tests/test_generic.py rename to tests/test_generic.py index 2938598..1751ecf 100644 --- a/channels/tests/test_generic.py +++ b/tests/test_generic.py @@ -5,7 +5,7 @@ from django.test import override_settings from channels import route_class from channels.exceptions import SendNotAvailableOnDemultiplexer from channels.generic import BaseConsumer, websockets -from channels.tests import ChannelTestCase, Client, HttpClient, apply_routes +from channels.test import ChannelTestCase, Client, HttpClient, apply_routes @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") diff --git a/channels/tests/test_handler.py b/tests/test_handler.py similarity index 99% rename from channels/tests/test_handler.py rename to tests/test_handler.py index 0c096cc..ffecf8b 100644 --- a/channels/tests/test_handler.py +++ b/tests/test_handler.py @@ -9,7 +9,7 @@ from six import BytesIO from channels import Channel from channels.handler import AsgiHandler -from channels.tests import ChannelTestCase +from channels.test import ChannelTestCase class FakeAsgiHandler(AsgiHandler): diff --git a/channels/tests/test_http.py b/tests/test_http.py similarity index 89% rename from channels/tests/test_http.py rename to tests/test_http.py index 1c7c29a..0ec9089 100644 --- a/channels/tests/test_http.py +++ b/tests/test_http.py @@ -2,8 +2,7 @@ from __future__ import unicode_literals from django.http.cookie import parse_cookie -from channels.tests import ChannelTestCase -from channels.tests.http import HttpClient +from channels.test import ChannelTestCase, HttpClient class HttpClientTests(ChannelTestCase): diff --git a/channels/tests/test_management.py b/tests/test_management.py similarity index 100% rename from channels/tests/test_management.py rename to tests/test_management.py diff --git a/channels/tests/test_request.py b/tests/test_request.py similarity index 99% rename from channels/tests/test_request.py rename to tests/test_request.py index aea1f47..39702ca 100644 --- a/channels/tests/test_request.py +++ b/tests/test_request.py @@ -5,7 +5,7 @@ from django.utils import six from channels import Channel from channels.exceptions import RequestAborted, RequestTimeout from channels.handler import AsgiRequest -from channels.tests import ChannelTestCase +from channels.test import ChannelTestCase class RequestTests(ChannelTestCase): diff --git a/channels/tests/test_routing.py b/tests/test_routing.py similarity index 94% rename from channels/tests/test_routing.py rename to tests/test_routing.py index 5a6145d..c246347 100644 --- a/channels/tests/test_routing.py +++ b/tests/test_routing.py @@ -168,7 +168,7 @@ class RoutingTests(SimpleTestCase): Tests inclusion without a prefix """ router = Router([ - include("channels.tests.test_routing.chatroom_routing"), + include("tests.test_routing.chatroom_routing"), ]) self.assertRoute( router, @@ -196,7 +196,7 @@ class RoutingTests(SimpleTestCase): Tests route_class with/without prefix """ router = Router([ - include("channels.tests.test_routing.class_routing"), + include("tests.test_routing.class_routing"), ]) self.assertRoute( router, @@ -222,7 +222,7 @@ class RoutingTests(SimpleTestCase): Tests inclusion with a prefix """ router = Router([ - include("channels.tests.test_routing.chatroom_routing", path="^/ws/v(?P[0-9]+)"), + include("tests.test_routing.chatroom_routing", path="^/ws/v(?P[0-9]+)"), ]) self.assertRoute( router, @@ -252,7 +252,7 @@ class RoutingTests(SimpleTestCase): ) # Check it works without the ^s too. router = Router([ - include("channels.tests.test_routing.chatroom_routing_nolinestart", path="/ws/v(?P[0-9]+)"), + include("tests.test_routing.chatroom_routing_nolinestart", path="/ws/v(?P[0-9]+)"), ]) self.assertRoute( router, @@ -279,7 +279,7 @@ class RoutingTests(SimpleTestCase): # Unicode patterns, byte message router = Router([ route("websocket.connect", consumer_1, path="^/foo/"), - include("channels.tests.test_routing.chatroom_routing", path="^/ws/v(?P[0-9]+)"), + include("tests.test_routing.chatroom_routing", path="^/ws/v(?P[0-9]+)"), ]) self.assertRoute( router, @@ -303,7 +303,7 @@ class RoutingTests(SimpleTestCase): # Byte patterns, unicode message router = Router([ route("websocket.connect", consumer_1, path=b"^/foo/"), - include("channels.tests.test_routing.chatroom_routing", path=b"^/ws/v(?P[0-9]+)"), + include("tests.test_routing.chatroom_routing", path=b"^/ws/v(?P[0-9]+)"), ]) self.assertRoute( router, diff --git a/channels/tests/test_sessions.py b/tests/test_sessions.py similarity index 99% rename from channels/tests/test_sessions.py rename to tests/test_sessions.py index 6af6edc..0f8bf00 100644 --- a/channels/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -8,7 +8,7 @@ from channels.message import Message from channels.sessions import ( channel_and_http_session, channel_session, enforce_ordering, http_session, session_for_reply_channel, ) -from channels.tests import ChannelTestCase +from channels.test import ChannelTestCase @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") diff --git a/channels/tests/test_worker.py b/tests/test_worker.py similarity index 99% rename from channels/tests/test_worker.py rename to tests/test_worker.py index 064f471..345b287 100644 --- a/channels/tests/test_worker.py +++ b/tests/test_worker.py @@ -6,7 +6,7 @@ from channels import DEFAULT_CHANNEL_LAYER, Channel, route from channels.asgi import channel_layers from channels.exceptions import ConsumeLater from channels.signals import worker_ready -from channels.tests import ChannelTestCase +from channels.test import ChannelTestCase from channels.worker import Worker, WorkerGroup try: From 7625ed270031ec8947b63d42209759d9c2a9c0a9 Mon Sep 17 00:00:00 2001 From: Hassen ben tanfous Date: Mon, 20 Feb 2017 01:16:37 +0100 Subject: [PATCH 636/746] use domain instead of port (#537) i think this is a typo, as cookies aren't restricted by port, so even if you offload on the same domain, daphne will still work without having to specify a ``session_key`` GET parameter since it can read the django session cookie which it defaults to. --- docs/getting-started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index f196b51..e674f5d 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -439,10 +439,10 @@ both authentication and getting the underlying Django session (which is what Django authentication relies on). Channels can use Django sessions either from cookies (if you're running your -websocket server on the same port as your main site, using something like Daphne), +websocket server on the same domain as your main site, using something like Daphne), or from a ``session_key`` GET parameter, which works if you want to keep running your HTTP requests through a WSGI server and offload WebSockets to a -second server process on another port. +second server process on another domain. You get access to a user's normal Django session using the ``http_session`` decorator - that gives you a ``message.http_session`` attribute that behaves From db3a0201229329b3fe5d82ef3da7d6a92033f5c2 Mon Sep 17 00:00:00 2001 From: Hassen ben tanfous Date: Tue, 21 Feb 2017 00:11:28 +0100 Subject: [PATCH 637/746] Use domain instead of port in docs (#539) --- docs/deploying.rst | 2 +- docs/getting-started.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index cdfe666..8b72707 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -144,7 +144,7 @@ and configure routing in front of your interface and WSGI servers to route requests appropriately. If you use Daphne for all traffic, it auto-negotiates between HTTP and WebSocket, -so there's no need to have your WebSockets on a separate port or path (and +so there's no need to have your WebSockets on a separate domain or path (and they'll be able to share cookies with your normal view code, which isn't possible if you separate by domain rather than path). diff --git a/docs/getting-started.rst b/docs/getting-started.rst index e674f5d..fd63fa4 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -492,7 +492,7 @@ chat to people with the same first letter of their username:: If you're just using ``runserver`` (and so Daphne), you can just connect and your cookies should transfer your auth over. If you were running WebSockets -on a separate port, you'd have to remember to provide the +on a separate domain, you'd have to remember to provide the Django session ID as part of the URL, like this:: socket = new WebSocket("ws://127.0.0.1:9000/?session_key=abcdefg"); From 863b1cebdda1b8e561ba01738d0f7c60dc615d58 Mon Sep 17 00:00:00 2001 From: Coread Date: Wed, 22 Feb 2017 19:00:50 +0000 Subject: [PATCH 638/746] Requeue next message immediately to avoid wait queue race condition (#532) Changes the strategy so that after a message has been put on the wait queue, it is then checked to see if it became the next message during this time and if so, immediately flushed. Will hopefully fix #451. --- channels/sessions.py | 41 ++++++++++++++++++----------- tests/test_sessions.py | 60 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/channels/sessions.py b/channels/sessions.py index ee23f54..3220f27 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -75,6 +75,26 @@ def channel_session(func): return inner +def requeue_messages(message): + """ + Requeue any pending wait channel messages for this socket connection back onto it's original channel + """ + while True: + wait_channel = "__wait__.%s" % message.reply_channel.name + channel, content = message.channel_layer.receive_many([wait_channel], block=False) + if channel: + original_channel = content.pop("original_channel") + try: + message.channel_layer.send(original_channel, content) + except message.channel_layer.ChannelFull: + raise message.channel_layer.ChannelFull( + "Cannot requeue pending __wait__ channel message " + + "back on to already full channel %s" % original_channel + ) + else: + break + + def enforce_ordering(func=None, slight=False): """ Enforces strict (all messages exactly ordered) ordering against a reply_channel. @@ -106,21 +126,7 @@ def enforce_ordering(func=None, slight=False): message.channel_session["__channels_next_order"] = order + 1 message.channel_session.save() message.channel_session.modified = False - # Requeue any pending wait channel messages for this socket connection back onto it's original channel - while True: - wait_channel = "__wait__.%s" % message.reply_channel.name - channel, content = message.channel_layer.receive_many([wait_channel], block=False) - if channel: - original_channel = content.pop("original_channel") - try: - message.channel_layer.send(original_channel, content) - except message.channel_layer.ChannelFull: - raise message.channel_layer.ChannelFull( - "Cannot requeue pending __wait__ channel message " + - "back on to already full channel %s" % original_channel - ) - else: - break + requeue_messages(message) else: # Since out of order, enqueue message temporarily to wait channel for this socket connection wait_channel = "__wait__.%s" % message.reply_channel.name @@ -132,6 +138,11 @@ def enforce_ordering(func=None, slight=False): "Cannot add unordered message to already " + "full __wait__ channel for socket %s" % message.reply_channel.name ) + # Next order may have changed while this message was being processed + # Requeue messages if this has happened + if order == message.channel_session.load().get("__channels_next_order", 0): + requeue_messages(message) + return inner if func is not None: return decorator(func) diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 0f8bf00..b56b2ad 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -10,6 +10,11 @@ from channels.sessions import ( ) from channels.test import ChannelTestCase +try: + from unittest import mock +except ImportError: + import mock + @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") class SessionTests(ChannelTestCase): @@ -223,3 +228,58 @@ class SessionTests(ChannelTestCase): with self.assertRaises(ValueError): inner(message0) + + def test_enforce_ordering_concurrent(self): + """ + Tests that strict mode of enforce_ordering puts messages in the correct queue after + the current message number changes while the message is being processed + """ + # Construct messages to send + message0 = Message( + {"reply_channel": "test-reply-e", "order": 0}, + "websocket.connect", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) + message2 = Message( + {"reply_channel": "test-reply-e", "order": 2}, + "websocket.receive", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) + message3 = Message( + {"reply_channel": "test-reply-e", "order": 3}, + "websocket.receive", + channel_layers[DEFAULT_CHANNEL_LAYER] + ) + + @channel_session + def add_session(message): + pass + + # Run them in an acceptable strict order + @enforce_ordering + def inner(message): + pass + + inner(message0) + inner(message3) + + # Add the session now so it can be mocked + add_session(message2) + + with mock.patch.object(message2.channel_session, 'load', return_value={'__channels_next_order': 2}): + inner(message2) + + # Ensure wait channel is empty + wait_channel = "__wait__.%s" % "test-reply-e" + next_message = self.get_next_message(wait_channel) + self.assertEqual(next_message, None) + + # Ensure messages 3 and 2 both ended up back on the original channel + expected = { + 2: message2, + 3: message3 + } + for m in range(2): + message = self.get_next_message("websocket.receive") + expected.pop(message.content['order']) + self.assertEqual(expected, {}) From 7ab21c484630d8ba08ea6d3b6d72bd740d3af5ab Mon Sep 17 00:00:00 2001 From: Pierre Chiquet Date: Thu, 23 Feb 2017 19:17:43 +0100 Subject: [PATCH 639/746] Set self.kwargs in Binding.trigger_inbound when setting self.message (#541) Allows options passed in (like a consumer) to be accessible to further code. --- channels/binding/base.py | 1 + tests/test_binding.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/channels/binding/base.py b/channels/binding/base.py index 88ae8e8..59cf43a 100644 --- a/channels/binding/base.py +++ b/channels/binding/base.py @@ -214,6 +214,7 @@ class Binding(object): from django.contrib.auth.models import AnonymousUser self = cls() self.message = message + self.kwargs = kwargs # Deserialize message self.action, self.pk, self.data = self.deserialize(self.message) self.user = getattr(self.message, "user", AnonymousUser()) diff --git a/tests/test_binding.py b/tests/test_binding.py index 4f86f50..61056bc 100644 --- a/tests/test_binding.py +++ b/tests/test_binding.py @@ -349,3 +349,29 @@ class TestsBinding(ChannelTestCase): self.assertIsNone(User.objects.filter(pk=user.pk).first()) self.assertIsNone(client.receive()) + + def test_route_params_saved_in_kwargs(self): + + class UserBinding(WebsocketBinding): + model = User + stream = 'users' + fields = ['username', 'email', 'password', 'last_name'] + + @classmethod + def group_names(cls, instance): + return ['users_outbound'] + + def has_permission(self, user, action, pk): + return True + + class Demultiplexer(WebsocketDemultiplexer): + consumers = { + 'users': UserBinding.consumer, + } + + groups = ['inbound'] + + with apply_routes([Demultiplexer.as_route(path='/path/(?P\d+)')]): + client = HttpClient() + consumer = client.send_and_consume('websocket.connect', path='/path/789') + self.assertEqual(consumer.kwargs['id'], '789') From b2842f1ef1d2365971268a351b39ed9741b539bd Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 23 Feb 2017 17:51:39 -0800 Subject: [PATCH 640/746] Fixed #542: Don't use staticfiles handler if staticfiles is not installed --- channels/management/commands/runserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 9733559..7caccbe 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -3,6 +3,7 @@ import sys import threading from daphne.server import Server, build_endpoint_description_strings +from django.apps import apps from django.conf import settings from django.core.management.commands.runserver import Command as RunserverCommand from django.utils import six @@ -139,7 +140,8 @@ class Command(RunserverCommand): if static files should be served. Otherwise just returns the default handler. """ - use_static_handler = options.get('use_static_handler', True) + staticfiles_installed = apps.is_installed("django.contrib.staticfiles") + use_static_handler = options.get('use_static_handler', staticfiles_installed) insecure_serving = options.get('insecure_serving', False) if use_static_handler and (settings.DEBUG or insecure_serving): return StaticFilesConsumer() From c14caede51cb913df3f494d7c00c8bfb837218ae Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 23 Feb 2017 18:01:28 -0800 Subject: [PATCH 641/746] Start working on 1.1.0 release notes --- docs/releases/1.1.0.rst | 30 ++++++++++++++++++++++++++++++ docs/releases/index.rst | 1 + 2 files changed, 31 insertions(+) create mode 100644 docs/releases/1.1.0.rst diff --git a/docs/releases/1.1.0.rst b/docs/releases/1.1.0.rst new file mode 100644 index 0000000..d461951 --- /dev/null +++ b/docs/releases/1.1.0.rst @@ -0,0 +1,30 @@ +1.1.0 Release Notes +=================== + +.. note:: + The 1.1.0 release is still in development. + +Channels 1.1.0 introduces a couple of major but backwards-compatible changes. +It was released on UNKNOWN. + +Major Changes +------------- + +* Test classes have been moved from ``channels.tests`` to ``channels.test`` + to better match Django. Old imports from ``channels.tests`` will continue to + work but will trigger a deprecation warning, and ``channels.tests`` will be + removed completely in version 1.3. + +Minor Changes & Bugfixes +------------------------ + +* Bindings now support non-integer fields for primary keys on models + +* The ``enforce_ordering`` decorator no longer suffers a race condition where + it would drop messages under high load + + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index f4f57a5..000825a 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -8,3 +8,4 @@ Release Notes 1.0.1 1.0.2 1.0.3 + 1.1.0 From ef6d526359ee8cbd1474eef5ea961061383bd6a8 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 25 Feb 2017 18:12:32 -0800 Subject: [PATCH 642/746] Add note about installing Redis --- docs/getting-started.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index fd63fa4..c11af1c 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -324,6 +324,12 @@ and set up your channel layer like this:: }, } +You'll also need to install the Redis server - there are downloads available +for Mac OS and Windows, and it's in pretty much every linux distribution's +package manager. For example, on Ubuntu, you can just:: + + sudo apt-get install redis-server + Fire up ``runserver``, and it'll work as before - unexciting, like good infrastructure should be. You can also try out the cross-process nature; run these two commands in two terminals: From 2101f285cb735c73cda3e273e249725b0729d232 Mon Sep 17 00:00:00 2001 From: Doug Keen Date: Tue, 28 Feb 2017 18:51:48 -0800 Subject: [PATCH 643/746] Allow custom json encoder and decoder in `JsonWebsocketConsumer` (#535) Lets you override the JSON encoding on both the consumer and the multiplexer. --- channels/generic/websockets.py | 103 +++++++++++++++++++-------------- docs/generics.rst | 14 ++++- tests/test_generic.py | 68 ++++++++++++++++++++++ 3 files changed, 139 insertions(+), 46 deletions(-) diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index 3e77e3b..f4bb714 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -148,7 +148,7 @@ class JsonWebsocketConsumer(WebsocketConsumer): def raw_receive(self, message, **kwargs): if "text" in message: - self.receive(json.loads(message['text']), **kwargs) + self.receive(self.decode_json(message['text']), **kwargs) else: raise ValueError("No text section for incoming WebSocket frame!") @@ -162,11 +162,58 @@ class JsonWebsocketConsumer(WebsocketConsumer): """ Encode the given content as JSON and send it to the client. """ - super(JsonWebsocketConsumer, self).send(text=json.dumps(content), close=close) + super(JsonWebsocketConsumer, self).send(text=self.encode_json(content), close=close) + + @classmethod + def decode_json(cls, text): + return json.loads(text) + + @classmethod + def encode_json(cls, content): + return json.dumps(content) @classmethod def group_send(cls, name, content, close=False): - WebsocketConsumer.group_send(name, json.dumps(content), close=close) + WebsocketConsumer.group_send(name, cls.encode_json(content), close=close) + + +class WebsocketMultiplexer(object): + """ + The opposite of the demultiplexer, to send a message though a multiplexed channel. + + The multiplexer object is passed as a kwargs to the consumer when the message is dispatched. + This pattern allows the consumer class to be independent of the stream name. + """ + + stream = None + reply_channel = None + + def __init__(self, stream, reply_channel): + self.stream = stream + self.reply_channel = reply_channel + + def send(self, payload): + """Multiplex the payload using the stream name and send it.""" + self.reply_channel.send(self.encode(self.stream, payload)) + + @classmethod + def encode_json(cls, content): + return json.dumps(content, cls=DjangoJSONEncoder) + + @classmethod + def encode(cls, stream, payload): + """ + Encodes stream + payload for outbound sending. + """ + content = {"stream": stream, "payload": payload} + return {"text": cls.encode_json(content)} + + @classmethod + def group_send(cls, name, stream, payload, close=False): + message = cls.encode(stream, payload) + if close: + message["close"] = True + Group(name).send(message) class WebsocketDemultiplexer(JsonWebsocketConsumer): @@ -191,6 +238,9 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): # Put your JSON consumers here: {stream_name : consumer} consumers = {} + # Optionally use a custom multiplexer class + multiplexer_class = WebsocketMultiplexer + def receive(self, content, **kwargs): """Forward messages to all consumers.""" # Check the frame looks good @@ -203,10 +253,10 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): if not isinstance(payload, dict): raise ValueError("Multiplexed frame payload is not a dict") # The json consumer expects serialized JSON - self.message.content['text'] = json.dumps(payload) + self.message.content['text'] = self.encode_json(payload) # Send demultiplexer to the consumer, to be able to answer - kwargs['multiplexer'] = WebsocketMultiplexer(stream, self.message.reply_channel) - # Patch send to avoid sending not formated messages from the consumer + kwargs['multiplexer'] = self.multiplexer_class(stream, self.message.reply_channel) + # Patch send to avoid sending not formatted messages from the consumer if hasattr(consumer, "send"): consumer.send = self.send # Dispatch message @@ -221,13 +271,13 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): """Forward connection to all consumers.""" self.message.reply_channel.send({"accept": True}) for stream, consumer in self.consumers.items(): - kwargs['multiplexer'] = WebsocketMultiplexer(stream, self.message.reply_channel) + kwargs['multiplexer'] = self.multiplexer_class(stream, self.message.reply_channel) consumer(message, **kwargs) def disconnect(self, message, **kwargs): """Forward disconnection to all consumers.""" for stream, consumer in self.consumers.items(): - kwargs['multiplexer'] = WebsocketMultiplexer(stream, self.message.reply_channel) + kwargs['multiplexer'] = self.multiplexer_class(stream, self.message.reply_channel) consumer(message, **kwargs) def send(self, *args): @@ -236,40 +286,3 @@ class WebsocketDemultiplexer(JsonWebsocketConsumer): @classmethod def group_send(cls, name, stream, payload, close=False): raise SendNotAvailableOnDemultiplexer("Use WebsocketMultiplexer.group_send") - - -class WebsocketMultiplexer(object): - """ - The opposite of the demultiplexer, to send a message though a multiplexed channel. - - The multiplexer object is passed as a kwargs to the consumer when the message is dispatched. - This pattern allows the consumer class to be independant of the stream name. - """ - - stream = None - reply_channel = None - - def __init__(self, stream, reply_channel): - self.stream = stream - self.reply_channel = reply_channel - - def send(self, payload): - """Multiplex the payload using the stream name and send it.""" - self.reply_channel.send(self.encode(self.stream, payload)) - - @classmethod - def encode(cls, stream, payload): - """ - Encodes stream + payload for outbound sending. - """ - return {"text": json.dumps({ - "stream": stream, - "payload": payload, - }, cls=DjangoJSONEncoder)} - - @classmethod - def group_send(cls, name, stream, payload, close=False): - message = WebsocketMultiplexer.encode(stream, payload) - if close: - message["close"] = True - Group(name).send(message) diff --git a/docs/generics.rst b/docs/generics.rst index 459ce36..17768da 100644 --- a/docs/generics.rst +++ b/docs/generics.rst @@ -163,6 +163,15 @@ The JSON-enabled consumer looks slightly different:: """ pass + # Optionally provide your own custom json encoder and decoder + # @classmethod + # def decode_json(cls, text): + # return my_custom_json_decoder(text) + # + # @classmethod + # def encode_json(cls, content): + # return my_custom_json_encoder(content) + For this subclass, ``receive`` only gets a ``content`` argument that is the already-decoded JSON as Python datastructures; similarly, ``send`` now only takes a single argument, which it JSON-encodes before sending down to the @@ -221,8 +230,11 @@ Example using class-based consumer:: "other": AnotherConsumer, } + # Optionally provide a custom multiplexer class + # multiplexer_class = MyCustomJsonEncodingMultiplexer -The ``multiplexer`` allows the consumer class to be independant of the stream name. + +The ``multiplexer`` allows the consumer class to be independent of the stream name. It holds the stream name and the demultiplexer on the attributes ``stream`` and ``demultiplexer``. The :doc:`data binding ` code will also send out messages to clients diff --git a/tests/test_generic.py b/tests/test_generic.py index 1751ecf..6b0c83a 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import json + from django.test import override_settings from channels import route_class @@ -199,3 +201,69 @@ class GenericTests(ChannelTestCase): }) client.receive() + + def test_websocket_custom_json_serialization(self): + + class WebsocketConsumer(websockets.JsonWebsocketConsumer): + @classmethod + def decode_json(cls, text): + obj = json.loads(text) + return dict((key.upper(), obj[key]) for key in obj) + + @classmethod + def encode_json(cls, content): + lowered = dict((key.lower(), content[key]) for key in content) + return json.dumps(lowered) + + def receive(self, content, multiplexer=None, **kwargs): + self.content_received = content + self.send({"RESPONSE": "HI"}) + + class MyMultiplexer(websockets.WebsocketMultiplexer): + @classmethod + def encode_json(cls, content): + lowered = dict((key.lower(), content[key]) for key in content) + return json.dumps(lowered) + + with apply_routes([route_class(WebsocketConsumer, path='/path')]): + client = HttpClient() + + consumer = client.send_and_consume('websocket.receive', path='/path', text={"key": "value"}) + self.assertEqual(consumer.content_received, {"KEY": "value"}) + + self.assertEqual(client.receive(), {"response": "HI"}) + + client.join_group('test_group') + WebsocketConsumer.group_send('test_group', {"KEY": "VALUE"}) + self.assertEqual(client.receive(), {"key": "VALUE"}) + + def test_websockets_demultiplexer_custom_multiplexer(self): + + class MyWebsocketConsumer(websockets.JsonWebsocketConsumer): + def connect(self, message, multiplexer=None, **kwargs): + multiplexer.send({"THIS_SHOULD_BE_LOWERCASED": "1"}) + + class MyMultiplexer(websockets.WebsocketMultiplexer): + @classmethod + def encode_json(cls, content): + lowered = { + "stream": content["stream"], + "payload": dict((key.lower(), content["payload"][key]) for key in content["payload"]) + } + return json.dumps(lowered) + + class Demultiplexer(websockets.WebsocketDemultiplexer): + multiplexer_class = MyMultiplexer + + consumers = { + "mystream": MyWebsocketConsumer + } + + with apply_routes([route_class(Demultiplexer, path='/path/(?P\d+)')]): + client = HttpClient() + + client.send_and_consume('websocket.connect', path='/path/1') + self.assertEqual(client.receive(), { + "stream": "mystream", + "payload": {"this_should_be_lowercased": "1"}, + }) From 13c1fcb65420c6826d38f8c8d3fe42e1994e091a Mon Sep 17 00:00:00 2001 From: Maik Hoepfel Date: Wed, 8 Mar 2017 01:05:28 -0600 Subject: [PATCH 644/746] Proposal for minor spec changes (#554) * Webserver -> web server This was flagged by my spell check, and indeed it's hard to find spellings online without the space. The Oxford Dictionary only knows it with a space, so I thought it's worth correcting. * Attempt to clarify optional keys I wasn't sure about how to treat keys marked optional. After having spoken to Andrew, this is my attempt at clarifying. Improvements welcome! * Order of header values MUST be kept Order for HTTP header values matters, both in request and responses. So we must make sure that we're keeping it. Request: > Some headers, such as Accept-Language can be sent by clients as > several headers each with a different value rather than sending the > header as a comma separated list. http://docs.oracle.com/javaee/6/api/javax/servlet/http/HttpServletRequest.html#getHeaders%28java.lang.String%29 Response: Set-Cookie may be present multiple times, and only the last given value is to be used. I'm updating the Daphne test to verify the order in my pull request there. * Clarify that headers is a list of lists The wording for 'server'/'client' and 'headers' was very similar, and I was unsure if clients may be a list of lists (in anticipation of protocols supporting that). I hope this small tweak makes it clearer that only headers is a list of lists. --- docs/asgi.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 405ba6e..ea5e1d1 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -9,7 +9,7 @@ Abstract ======== This document proposes a standard interface between network protocol -servers (particularly webservers) and Python applications, intended +servers (particularly web servers) and Python applications, intended to allow handling of multiple common protocol styles (including HTTP, HTTP2, and WebSocket). @@ -22,7 +22,7 @@ Rationale ========= The WSGI specification has worked well since it was introduced, and -allowed for great flexibility in Python framework and webserver choice. +allowed for great flexibility in Python framework and web server choice. However, its design is irrevocably tied to the HTTP-style request/response cycle, and more and more protocols are becoming a standard part of web programming that do not follow this pattern @@ -478,8 +478,9 @@ Message Formats These describe the standardized message formats for the protocols this specification supports. All messages are ``dicts`` at the top level, -and all keys are required unless otherwise specified (with a default to -use if the key is missing). Keys are unicode strings. +and all keys are required unless explicitly marked as optional. If a key is +marked optional, a default value is specified, which is to be assumed if +the key is missing. Keys are unicode strings. The one common key across all protocols is ``reply_channel``, a way to indicate the client-specific channel to send responses to. Protocols are generally @@ -557,10 +558,11 @@ Keys: is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults to ``""``. -* ``headers``: A list of ``[name, value]`` pairs, where ``name`` is the +* ``headers``: A list of ``[name, value]`` lists, where ``name`` is the byte string header name, and ``value`` is the byte string - header value. Order should be preserved from the original HTTP request; - duplicates are possible and must be preserved in the message as received. + header value. Order of header values must be preserved from the original HTTP + request; order of header names is not important. Duplicates are possible and + must be preserved in the message as received. Header names must be lowercased. * ``body``: Body of the request, as a byte string. Optional, defaults to ``""``. @@ -622,9 +624,9 @@ Keys: * ``status``: Integer HTTP status code. -* ``headers``: A list of ``[name, value]`` pairs, where ``name`` is the +* ``headers``: A list of ``[name, value]`` lists, where ``name`` is the byte string header name, and ``value`` is the byte string - header value. Order should be preserved in the HTTP response. Header names + header value. Order must be preserved in the HTTP response. Header names must be lowercased. * ``content``: Byte string of HTTP body content. From cad63451f8f9bc0595f4c58154cc98cd188df6dc Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 12 Mar 2017 21:03:25 -0700 Subject: [PATCH 645/746] Expand more on accepting connections --- docs/concepts.rst | 2 ++ docs/getting-started.rst | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/docs/concepts.rst b/docs/concepts.rst index 83a3b12..f942922 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -235,6 +235,8 @@ abstraction as a core concept called Groups:: def ws_connect(message): # Add to reader group Group("liveblog").add(message.reply_channel) + # Accept the connection request + message.reply_channel.send({"accept": True}) # Connected to websocket.disconnect def ws_disconnect(message): diff --git a/docs/getting-started.rst b/docs/getting-started.rst index c11af1c..422e5dd 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -165,13 +165,20 @@ disconnect, like this:: # Connected to websocket.connect def ws_add(message): + # Accept the incoming connection message.reply_channel.send({"accept": True}) + # Add them to the chat group Group("chat").add(message.reply_channel) # Connected to websocket.disconnect def ws_disconnect(message): Group("chat").discard(message.reply_channel) +.. note:: + You need to explicitly accept WebSocket connections if you override connect + by sending ``accept: True`` - you can also reject them at connection time, + before they open, by sending ``close: True``. + Of course, if you've read through :doc:`concepts`, you'll know that channels added to groups expire out if their messages expire (every channel layer has a message expiry time, usually between 30 seconds and a few minutes, and it's @@ -204,7 +211,9 @@ get the message. Here's all the code:: # Connected to websocket.connect def ws_add(message): + # Accept the connection message.reply_channel.send({"accept": True}) + # Add to the chat group Group("chat").add(message.reply_channel) # Connected to websocket.receive @@ -559,6 +568,8 @@ consumer above to use a room based on URL rather than username:: def ws_add(message, room): # Add them to the right group Group("chat-%s" % room).add(message.reply_channel) + # Accept the connection request + message.reply_channel.send({"accept": True}) In the next section, we'll change to sending the ``room`` as a part of the WebSocket message - which you might do if you had a multiplexing client - From 463d16f3f927d0e7dc81c9b80f6b73b9097ff88b Mon Sep 17 00:00:00 2001 From: Erwin Junge Date: Tue, 14 Mar 2017 14:14:20 -0700 Subject: [PATCH 646/746] Make the channel_session decorator work on methods as well (#555) --- channels/sessions.py | 14 ++++-- tests/test_sessions.py | 102 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/channels/sessions.py b/channels/sessions.py index 3220f27..3af6da9 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -8,6 +8,7 @@ from django.contrib.sessions.backends.base import CreateError from .exceptions import ConsumeLater from .handler import AsgiRequest +from .message import Message def session_for_reply_channel(reply_channel): @@ -39,11 +40,18 @@ def channel_session(func): Use this to persist data across the lifetime of a connection. """ @functools.wraps(func) - def inner(message, *args, **kwargs): + def inner(*args, **kwargs): + message = None + for arg in args[:2]: + if isinstance(arg, Message): + message = arg + break + if message is None: + raise ValueError('channel_session called without Message instance') # Make sure there's NOT a channel_session already if hasattr(message, "channel_session"): try: - return func(message, *args, **kwargs) + return func(*args, **kwargs) finally: # Persist session if needed if message.channel_session.modified: @@ -67,7 +75,7 @@ def channel_session(func): message.channel_session = session # Run the consumer try: - return func(message, *args, **kwargs) + return func(*args, **kwargs) finally: # Persist session if needed if session.modified and not session.is_empty(): diff --git a/tests/test_sessions.py b/tests/test_sessions.py index b56b2ad..d1d507a 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -49,6 +49,39 @@ class SessionTests(ChannelTestCase): session2 = session_for_reply_channel("test-reply") self.assertEqual(session2["num_ponies"], -1) + def test_channel_session_method(self): + """ + Tests the channel_session decorator works on methods + """ + # Construct message to send + message = Message({"reply_channel": "test-reply"}, None, None) + + # Run through a simple fake consumer that assigns to it + class Consumer(object): + @channel_session + def inner(self, message): + message.channel_session["num_ponies"] = -1 + + Consumer().inner(message) + # Test the session worked + session2 = session_for_reply_channel("test-reply") + self.assertEqual(session2["num_ponies"], -1) + + def test_channel_session_third_arg(self): + """ + Tests the channel_session decorator with message as 3rd argument + """ + # Construct message to send + message = Message({"reply_channel": "test-reply"}, None, None) + + # Run through a simple fake consumer that assigns to it + @channel_session + def inner(a, b, message): + message.channel_session["num_ponies"] = -1 + + with self.assertRaisesMessage(ValueError, 'channel_session called without Message instance'): + inner(None, None, message) + def test_channel_session_double(self): """ Tests the channel_session decorator detects being wrapped in itself @@ -68,6 +101,42 @@ class SessionTests(ChannelTestCase): session2 = session_for_reply_channel("test-reply") self.assertEqual(session2["num_ponies"], -1) + def test_channel_session_double_method(self): + """ + Tests the channel_session decorator detects being wrapped in itself + and doesn't blow up. Method version. + """ + # Construct message to send + message = Message({"reply_channel": "test-reply"}, None, None) + + # Run through a simple fake consumer that should trigger the error + class Consumer(object): + @channel_session + @channel_session + def inner(self, message): + message.channel_session["num_ponies"] = -1 + Consumer().inner(message) + + # Test the session worked + session2 = session_for_reply_channel("test-reply") + self.assertEqual(session2["num_ponies"], -1) + + def test_channel_session_double_third_arg(self): + """ + Tests the channel_session decorator detects being wrapped in itself + and doesn't blow up. + """ + # Construct message to send + message = Message({"reply_channel": "test-reply"}, None, None) + + # Run through a simple fake consumer that should trigger the error + @channel_session + @channel_session + def inner(a, b, message): + message.channel_session["num_ponies"] = -1 + with self.assertRaisesMessage(ValueError, 'channel_session called without Message instance'): + inner(None, None, message) + def test_channel_session_no_reply(self): """ Tests the channel_session decorator detects no reply channel @@ -84,6 +153,39 @@ class SessionTests(ChannelTestCase): with self.assertRaises(ValueError): inner(message) + def test_channel_session_no_reply_method(self): + """ + Tests the channel_session decorator detects no reply channel + """ + # Construct message to send + message = Message({}, None, None) + + # Run through a simple fake consumer that should trigger the error + class Consumer(object): + @channel_session + @channel_session + def inner(self, message): + message.channel_session["num_ponies"] = -1 + + with self.assertRaises(ValueError): + Consumer().inner(message) + + def test_channel_session_no_reply_third_arg(self): + """ + Tests the channel_session decorator detects no reply channel + """ + # Construct message to send + message = Message({}, None, None) + + # Run through a simple fake consumer that should trigger the error + @channel_session + @channel_session + def inner(a, b, message): + message.channel_session["num_ponies"] = -1 + + with self.assertRaisesMessage(ValueError, 'channel_session called without Message instance'): + inner(None, None, message) + def test_http_session(self): """ Tests that http_session correctly extracts a session cookie. From 63dc5f6651bda29ba797b1d38d07b66d5cdfc060 Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Tue, 14 Mar 2017 17:08:04 -0500 Subject: [PATCH 647/746] add Channels WebSocket javascript wrapper (#544) Adds a WebSocket wrapper which is both publishable to npm and importable directly for use with staticfiles/etc. Also has a new build process to make the latter file. --- .gitignore | 1 + .travis.yml | 2 + Makefile | 4 + .../static/channels/js/websocketbridge.js | 395 ++++++++++++++++++ docs/binding.rst | 6 +- docs/index.rst | 1 + docs/javascript.rst | 45 ++ js_client/.babelrc | 10 + js_client/.eslintrc.js | 9 + js_client/.npmignore | 8 + js_client/README.md | 42 ++ js_client/banner.txt | 1 + js_client/esdoc.json | 21 + js_client/lib/index.js | 181 ++++++++ js_client/package.json | 72 ++++ js_client/src/index.js | 139 ++++++ js_client/tests/websocketbridge.test.js | 137 ++++++ setup.cfg | 2 +- 18 files changed, 1072 insertions(+), 4 deletions(-) create mode 100644 channels/static/channels/js/websocketbridge.js create mode 100644 docs/javascript.rst create mode 100644 js_client/.babelrc create mode 100644 js_client/.eslintrc.js create mode 100644 js_client/.npmignore create mode 100644 js_client/README.md create mode 100644 js_client/banner.txt create mode 100644 js_client/esdoc.json create mode 100644 js_client/lib/index.js create mode 100644 js_client/package.json create mode 100644 js_client/src/index.js create mode 100644 js_client/tests/websocketbridge.test.js diff --git a/.gitignore b/.gitignore index 857bc3f..8736ba0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ *.pyc .coverage.* TODO +node_modules # IDE and Tooling files .idea/* diff --git a/.travis.yml b/.travis.yml index f38f5a8..581d0a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,11 +17,13 @@ cache: - $HOME/.cache/pip/wheels install: + - nvm install 7 - pip install -U pip wheel setuptools - pip install $DJANGO -e .[tests] - pip freeze script: - python runtests.py + - cd js_client && npm install --progress=false && npm test && cd .. - flake8 - isort --check-only --recursive channels diff --git a/Makefile b/Makefile index 1a1f55e..612ab19 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,9 @@ all: +build_assets: + cd js_client && npm run browserify && cd .. + release: ifndef version $(error Please supply a version) @@ -14,3 +17,4 @@ endif git push git push --tags python setup.py sdist bdist_wheel upload + cd js_client && npm publish && cd .. diff --git a/channels/static/channels/js/websocketbridge.js b/channels/static/channels/js/websocketbridge.js new file mode 100644 index 0000000..5c647e6 --- /dev/null +++ b/channels/static/channels/js/websocketbridge.js @@ -0,0 +1,395 @@ +/*! + * Do not edit!. This file is autogenerated by running `npm run browserify`. + */ +(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.channels = f()}})(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o config.maxReconnectionDelay) + ? config.maxReconnectionDelay + : newDelay; +}; +var LEVEL_0_EVENTS = ['onopen', 'onclose', 'onmessage', 'onerror']; +var reassignEventListeners = function (ws, oldWs, listeners) { + Object.keys(listeners).forEach(function (type) { + listeners[type].forEach(function (_a) { + var listener = _a[0], options = _a[1]; + ws.addEventListener(type, listener, options); + }); + }); + if (oldWs) { + LEVEL_0_EVENTS.forEach(function (name) { ws[name] = oldWs[name]; }); + } +}; +var ReconnectingWebsocket = function (url, protocols, options) { + var _this = this; + if (options === void 0) { options = {}; } + var ws; + var connectingTimeout; + var reconnectDelay = 0; + var retriesCount = 0; + var shouldRetry = true; + var savedOnClose = null; + var listeners = {}; + // require new to construct + if (!(this instanceof ReconnectingWebsocket)) { + throw new TypeError("Failed to construct 'ReconnectingWebSocket': Please use the 'new' operator"); + } + // Set config. Not using `Object.assign` because of IE11 + var config = getDefaultOptions(); + Object.keys(config) + .filter(function (key) { return options.hasOwnProperty(key); }) + .forEach(function (key) { return config[key] = options[key]; }); + if (!isWebSocket(config.constructor)) { + throw new TypeError('Invalid WebSocket constructor. Set `options.constructor`'); + } + var log = config.debug ? function () { + var params = []; + for (var _i = 0; _i < arguments.length; _i++) { + params[_i - 0] = arguments[_i]; + } + return console.log.apply(console, ['RWS:'].concat(params)); + } : function () { }; + /** + * Not using dispatchEvent, otherwise we must use a DOM Event object + * Deferred because we want to handle the close event before this + */ + var emitError = function (code, msg) { return setTimeout(function () { + var err = new Error(msg); + err.code = code; + if (Array.isArray(listeners.error)) { + listeners.error.forEach(function (_a) { + var fn = _a[0]; + return fn(err); + }); + } + if (ws.onerror) { + ws.onerror(err); + } + }, 0); }; + var handleClose = function () { + log('close'); + retriesCount++; + log('retries count:', retriesCount); + if (retriesCount > config.maxRetries) { + emitError('EHOSTDOWN', 'Too many failed connection attempts'); + return; + } + if (!reconnectDelay) { + reconnectDelay = initReconnectionDelay(config); + } + else { + reconnectDelay = updateReconnectionDelay(config, reconnectDelay); + } + log('reconnectDelay:', reconnectDelay); + if (shouldRetry) { + setTimeout(connect, reconnectDelay); + } + }; + var connect = function () { + log('connect'); + var oldWs = ws; + ws = new config.constructor(url, protocols); + connectingTimeout = setTimeout(function () { + log('timeout'); + ws.close(); + emitError('ETIMEDOUT', 'Connection timeout'); + }, config.connectionTimeout); + log('bypass properties'); + for (var key in ws) { + // @todo move to constant + if (['addEventListener', 'removeEventListener', 'close', 'send'].indexOf(key) < 0) { + bypassProperty(ws, _this, key); + } + } + ws.addEventListener('open', function () { + clearTimeout(connectingTimeout); + log('open'); + reconnectDelay = initReconnectionDelay(config); + log('reconnectDelay:', reconnectDelay); + retriesCount = 0; + }); + ws.addEventListener('close', handleClose); + reassignEventListeners(ws, oldWs, listeners); + // because when closing with fastClose=true, it is saved and set to null to avoid double calls + ws.onclose = ws.onclose || savedOnClose; + savedOnClose = null; + }; + log('init'); + connect(); + this.close = function (code, reason, _a) { + if (code === void 0) { code = 1000; } + if (reason === void 0) { reason = ''; } + var _b = _a === void 0 ? {} : _a, _c = _b.keepClosed, keepClosed = _c === void 0 ? false : _c, _d = _b.fastClose, fastClose = _d === void 0 ? true : _d, _e = _b.delay, delay = _e === void 0 ? 0 : _e; + if (delay) { + reconnectDelay = delay; + } + shouldRetry = !keepClosed; + ws.close(code, reason); + if (fastClose) { + var fakeCloseEvent_1 = { + code: code, + reason: reason, + wasClean: true, + }; + // execute close listeners soon with a fake closeEvent + // and remove them from the WS instance so they + // don't get fired on the real close. + handleClose(); + ws.removeEventListener('close', handleClose); + // run and remove level2 + if (Array.isArray(listeners.close)) { + listeners.close.forEach(function (_a) { + var listener = _a[0], options = _a[1]; + listener(fakeCloseEvent_1); + ws.removeEventListener('close', listener, options); + }); + } + // run and remove level0 + if (ws.onclose) { + savedOnClose = ws.onclose; + ws.onclose(fakeCloseEvent_1); + ws.onclose = null; + } + } + }; + this.send = function (data) { + ws.send(data); + }; + this.addEventListener = function (type, listener, options) { + if (Array.isArray(listeners[type])) { + if (!listeners[type].some(function (_a) { + var l = _a[0]; + return l === listener; + })) { + listeners[type].push([listener, options]); + } + } + else { + listeners[type] = [[listener, options]]; + } + ws.addEventListener(type, listener, options); + }; + this.removeEventListener = function (type, listener, options) { + if (Array.isArray(listeners[type])) { + listeners[type] = listeners[type].filter(function (_a) { + var l = _a[0]; + return l !== listener; + }); + } + ws.removeEventListener(type, listener, options); + }; +}; +module.exports = ReconnectingWebsocket; + +},{}],2:[function(require,module,exports){ +'use strict'; + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.WebSocketBridge = undefined; + +var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _reconnectingWebsocket = require('reconnecting-websocket'); + +var _reconnectingWebsocket2 = _interopRequireDefault(_reconnectingWebsocket); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +var noop = function noop() {}; + +/** + * Bridge between Channels and plain javascript. + * + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + * webSocketBridge.listen(function(action, stream) { + * console.log(action, stream); + * }); + */ + +var WebSocketBridge = function () { + function WebSocketBridge(options) { + _classCallCheck(this, WebSocketBridge); + + this._socket = null; + this.streams = {}; + this.default_cb = null; + this.options = _extends({}, { + onopen: noop + }, options); + } + + /** + * Connect to the websocket server + * + * @param {String} [url] The url of the websocket. Defaults to + * `window.location.host` + * @param {String[]|String} [protocols] Optional string or array of protocols. + * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/joewalnes/reconnecting-websocket#options-1). + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + */ + + + _createClass(WebSocketBridge, [{ + key: 'connect', + value: function connect(url, protocols, options) { + var _url = void 0; + if (url === undefined) { + // Use wss:// if running on https:// + var scheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; + _url = scheme + '://' + window.location.host + '/ws'; + } else { + _url = url; + } + this._socket = new _reconnectingWebsocket2.default(_url, protocols, options); + } + + /** + * Starts listening for messages on the websocket, demultiplexing if necessary. + * + * @param {Function} [cb] Callback to be execute when a message + * arrives. The callback will receive `action` and `stream` parameters + * + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + * webSocketBridge.listen(function(action, stream) { + * console.log(action, stream); + * }); + */ + + }, { + key: 'listen', + value: function listen(cb) { + var _this = this; + + this.default_cb = cb; + this._socket.onmessage = function (event) { + var msg = JSON.parse(event.data); + var action = void 0; + var stream = void 0; + + if (msg.stream !== undefined) { + action = msg.payload; + stream = msg.stream; + var stream_cb = _this.streams[stream]; + stream_cb ? stream_cb(action, stream) : null; + } else { + action = msg; + stream = null; + _this.default_cb ? _this.default_cb(action, stream) : null; + } + }; + + this._socket.onopen = this.options.onopen; + } + + /** + * Adds a 'stream handler' callback. Messages coming from the specified stream + * will call the specified callback. + * + * @param {String} stream The stream name + * @param {Function} cb Callback to be execute when a message + * arrives. The callback will receive `action` and `stream` parameters. + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + * webSocketBridge.listen(); + * webSocketBridge.demultiplex('mystream', function(action, stream) { + * console.log(action, stream); + * }); + * webSocketBridge.demultiplex('myotherstream', function(action, stream) { + * console.info(action, stream); + * }); + */ + + }, { + key: 'demultiplex', + value: function demultiplex(stream, cb) { + this.streams[stream] = cb; + } + + /** + * Sends a message to the reply channel. + * + * @param {Object} msg The message + * + * @example + * webSocketBridge.send({prop1: 'value1', prop2: 'value1'}); + */ + + }, { + key: 'send', + value: function send(msg) { + this._socket.send(JSON.stringify(msg)); + } + + /** + * Returns an object to send messages to a specific stream + * + * @param {String} stream The stream name + * @return {Object} convenience object to send messages to `stream`. + * @example + * webSocketBridge.stream('mystream').send({prop1: 'value1', prop2: 'value1'}) + */ + + }, { + key: 'stream', + value: function stream(_stream) { + var _this2 = this; + + return { + send: function send(action) { + var msg = { + stream: _stream, + payload: action + }; + _this2._socket.send(JSON.stringify(msg)); + } + }; + } + }]); + + return WebSocketBridge; +}(); + +exports.WebSocketBridge = WebSocketBridge; + +},{"reconnecting-websocket":1}]},{},[2])(2) +}); \ No newline at end of file diff --git a/docs/binding.rst b/docs/binding.rst index 517c2ed..64e8ba1 100644 --- a/docs/binding.rst +++ b/docs/binding.rst @@ -136,9 +136,9 @@ Tie that into your routing, and you're ready to go:: Frontend Considerations ----------------------- -You can use the standard Channels WebSocket wrapper **(not yet available)** -to automatically run demultiplexing, and then tie the events you receive into -your frontend framework of choice based on ``action``, ``pk`` and ``data``. +You can use the standard :doc:`Channels WebSocket wrapper ` to +automatically run demultiplexing, and then tie the events you receive into your +frontend framework of choice based on ``action``, ``pk`` and ``data``. .. note:: diff --git a/docs/index.rst b/docs/index.rst index b4c12cd..7135ba9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,6 +49,7 @@ Topics generics routing binding + javascript backends delay testing diff --git a/docs/javascript.rst b/docs/javascript.rst new file mode 100644 index 0000000..3d8344f --- /dev/null +++ b/docs/javascript.rst @@ -0,0 +1,45 @@ +Channels WebSocket wrapper +========================== + +Channels ships with a javascript WebSocket wrapper to help you connect to your websocket +and send/receive messages. + +First, you must include the javascript library in your template:: + + {% load staticfiles %} + + {% static "channels/js/websocketbridge.js" %} + +To process messages:: + + const webSocketBridge = new channels.WebSocketBridge(); + webSocketBridge.connect(); + webSocketBridge.listen(function(action, stream) { + console.log(action, stream); + }); + +To send messages, use the `send` method:: + + ``` + webSocketBridge.send({prop1: 'value1', prop2: 'value1'}); + + ``` + +To demultiplex specific streams:: + + webSocketBridge.connect(); + webSocketBridge.listen(); + webSocketBridge.demultiplex('mystream', function(action, stream) { + console.log(action, stream); + }); + webSocketBridge.demultiplex('myotherstream', function(action, stream) { + console.info(action, stream); + }); + + +To send a message to a specific stream:: + + webSocketBridge.stream('mystream').send({prop1: 'value1', prop2: 'value1'}) + +The library is also available as npm module, under the name +`django-channels `_ diff --git a/js_client/.babelrc b/js_client/.babelrc new file mode 100644 index 0000000..6d1bf22 --- /dev/null +++ b/js_client/.babelrc @@ -0,0 +1,10 @@ +{ + "presets": [ + "es2015", + "stage-1", + "react" + ], + "plugins": [ + "transform-object-assign", + ] +} diff --git a/js_client/.eslintrc.js b/js_client/.eslintrc.js new file mode 100644 index 0000000..c06645c --- /dev/null +++ b/js_client/.eslintrc.js @@ -0,0 +1,9 @@ +module.exports = { + "extends": "airbnb", + "plugins": [ + "react" + ], + env: { + jest: true + } +}; diff --git a/js_client/.npmignore b/js_client/.npmignore new file mode 100644 index 0000000..b81236d --- /dev/null +++ b/js_client/.npmignore @@ -0,0 +1,8 @@ +npm-debug.log +node_modules +.*.swp +.lock-* +build +.babelrc +webpack.* +/src/ diff --git a/js_client/README.md b/js_client/README.md new file mode 100644 index 0000000..506cd49 --- /dev/null +++ b/js_client/README.md @@ -0,0 +1,42 @@ +### Usage + +Channels WebSocket wrapper. + +To process messages: + +``` +import { WebSocketBridge } from 'django-channels' + +const webSocketBridge = new WebSocketBridge(); +webSocketBridge.connect(); +webSocketBridge.listen(function(action, stream) { + console.log(action, stream); +}); +``` + +To send messages: + +``` +webSocketBridge.send({prop1: 'value1', prop2: 'value1'}); + +``` + +To demultiplex specific streams: + +``` +const webSocketBridge = new WebSocketBridge(); +webSocketBridge.connect(); +webSocketBridge.listen(); +webSocketBridge.demultiplex('mystream', function(action, stream) { + console.log(action, stream); +}); +webSocketBridge.demultiplex('myotherstream', function(action, stream) { + console.info(action, stream); +}); +``` + +To send a message to a specific stream: + +``` +webSocketBridge.stream('mystream').send({prop1: 'value1', prop2: 'value1'}) +``` diff --git a/js_client/banner.txt b/js_client/banner.txt new file mode 100644 index 0000000..9eb7978 --- /dev/null +++ b/js_client/banner.txt @@ -0,0 +1 @@ +Do not edit!. This file is autogenerated by running `npm run browserify`. \ No newline at end of file diff --git a/js_client/esdoc.json b/js_client/esdoc.json new file mode 100644 index 0000000..157bc8e --- /dev/null +++ b/js_client/esdoc.json @@ -0,0 +1,21 @@ +{ + "source": "./src", + "destination": "./docs", + "undocumentIdentifier": false, + "title": "django-channels", + "experimentalProposal": { + "classProperties": true, + "objectRestSpread": true + }, + "plugins": [ + { + "name": "esdoc-importpath-plugin", + "option": { + "replaces": [ + {"from": "^src/", "to": "lib/"}, + {"from": ".js$", "to": ""} + ] + } + } + ] +} diff --git a/js_client/lib/index.js b/js_client/lib/index.js new file mode 100644 index 0000000..254c253 --- /dev/null +++ b/js_client/lib/index.js @@ -0,0 +1,181 @@ +'use strict'; + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.WebSocketBridge = undefined; + +var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _reconnectingWebsocket = require('reconnecting-websocket'); + +var _reconnectingWebsocket2 = _interopRequireDefault(_reconnectingWebsocket); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +var noop = function noop() {}; + +/** + * Bridge between Channels and plain javascript. + * + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + * webSocketBridge.listen(function(action, stream) { + * console.log(action, stream); + * }); + */ + +var WebSocketBridge = function () { + function WebSocketBridge(options) { + _classCallCheck(this, WebSocketBridge); + + this._socket = null; + this.streams = {}; + this.default_cb = null; + this.options = _extends({}, { + onopen: noop + }, options); + } + + /** + * Connect to the websocket server + * + * @param {String} [url] The url of the websocket. Defaults to + * `window.location.host` + * @param {String[]|String} [protocols] Optional string or array of protocols. + * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/joewalnes/reconnecting-websocket#options-1). + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + */ + + + _createClass(WebSocketBridge, [{ + key: 'connect', + value: function connect(url, protocols, options) { + var _url = void 0; + if (url === undefined) { + // Use wss:// if running on https:// + var scheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; + _url = scheme + '://' + window.location.host + '/ws'; + } else { + _url = url; + } + this._socket = new _reconnectingWebsocket2.default(_url, protocols, options); + } + + /** + * Starts listening for messages on the websocket, demultiplexing if necessary. + * + * @param {Function} [cb] Callback to be execute when a message + * arrives. The callback will receive `action` and `stream` parameters + * + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + * webSocketBridge.listen(function(action, stream) { + * console.log(action, stream); + * }); + */ + + }, { + key: 'listen', + value: function listen(cb) { + var _this = this; + + this.default_cb = cb; + this._socket.onmessage = function (event) { + var msg = JSON.parse(event.data); + var action = void 0; + var stream = void 0; + + if (msg.stream !== undefined) { + action = msg.payload; + stream = msg.stream; + var stream_cb = _this.streams[stream]; + stream_cb ? stream_cb(action, stream) : null; + } else { + action = msg; + stream = null; + _this.default_cb ? _this.default_cb(action, stream) : null; + } + }; + + this._socket.onopen = this.options.onopen; + } + + /** + * Adds a 'stream handler' callback. Messages coming from the specified stream + * will call the specified callback. + * + * @param {String} stream The stream name + * @param {Function} cb Callback to be execute when a message + * arrives. The callback will receive `action` and `stream` parameters. + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + * webSocketBridge.listen(); + * webSocketBridge.demultiplex('mystream', function(action, stream) { + * console.log(action, stream); + * }); + * webSocketBridge.demultiplex('myotherstream', function(action, stream) { + * console.info(action, stream); + * }); + */ + + }, { + key: 'demultiplex', + value: function demultiplex(stream, cb) { + this.streams[stream] = cb; + } + + /** + * Sends a message to the reply channel. + * + * @param {Object} msg The message + * + * @example + * webSocketBridge.send({prop1: 'value1', prop2: 'value1'}); + */ + + }, { + key: 'send', + value: function send(msg) { + this._socket.send(JSON.stringify(msg)); + } + + /** + * Returns an object to send messages to a specific stream + * + * @param {String} stream The stream name + * @return {Object} convenience object to send messages to `stream`. + * @example + * webSocketBridge.stream('mystream').send({prop1: 'value1', prop2: 'value1'}) + */ + + }, { + key: 'stream', + value: function stream(_stream) { + var _this2 = this; + + return { + send: function send(action) { + var msg = { + stream: _stream, + payload: action + }; + _this2._socket.send(JSON.stringify(msg)); + } + }; + } + }]); + + return WebSocketBridge; +}(); + +exports.WebSocketBridge = WebSocketBridge; \ No newline at end of file diff --git a/js_client/package.json b/js_client/package.json new file mode 100644 index 0000000..b62c20e --- /dev/null +++ b/js_client/package.json @@ -0,0 +1,72 @@ +{ + "name": "django-channels", + "version": "0.0.2", + "description": "", + "repository": { + "type": "git", + "url": "https://github.com/django/channels.git" + }, + "main": "lib/index.js", + "scripts": { + "transpile": "rm -rf lib && babel src --out-dir lib", + "docs": "rm -rf docs && esdoc -c esdoc.json", + "test": "jest", + "browserify": "browserify src/index.js -p browserify-banner -s channels -o ../channels/static/channels/js/websocketbridge.js", + "prepublish": "npm run transpile", + "compile": "npm run transpile && npm run browserify" + }, + "files": [ + "lib/index.js" + ], + "license": "BSD-3-Clause", + "dependencies": { + "reconnecting-websocket": "^3.0.3" + }, + "jest": { + "roots": [ + "tests" + ] + }, + "browserify": { + "transform": [ + [ + "babelify" + ] + ] + }, + "devDependencies": { + "babel": "^6.5.2", + "babel-cli": "^6.16.0", + "babel-core": "^6.16.0", + "babel-plugin-transform-inline-environment-variables": "^6.8.0", + "babel-plugin-transform-object-assign": "^6.8.0", + "babel-plugin-transform-runtime": "^6.15.0", + "babel-polyfill": "^6.16.0", + "babel-preset-es2015": "^6.16.0", + "babel-preset-react": "^6.16.0", + "babel-preset-stage-0": "^6.16.0", + "babel-register": "^6.9.0", + "babel-runtime": "^6.11.6", + "babelify": "^7.3.0", + "browserify": "^14.1.0", + "browserify-banner": "^1.0.3", + "esdoc": "^0.5.2", + "esdoc-es7-plugin": "0.0.3", + "esdoc-importpath-plugin": "^0.1.1", + "eslint": "^2.13.1", + "eslint-config-airbnb": "^9.0.1", + "eslint-plugin-import": "^1.9.2", + "eslint-plugin-jsx-a11y": "^1.5.3", + "eslint-plugin-react": "^5.2.2", + "jest": "^19.0.1", + "mock-socket": "^6.0.4", + "react": "^15.4.0", + "react-cookie": "^0.4.8", + "react-dom": "^15.4.0", + "react-redux": "^4.4.6", + "redux": "^3.6.0", + "redux-actions": "^1.0.0", + "redux-logger": "^2.7.4", + "redux-thunk": "^2.1.0" + } +} diff --git a/js_client/src/index.js b/js_client/src/index.js new file mode 100644 index 0000000..6a5ba3e --- /dev/null +++ b/js_client/src/index.js @@ -0,0 +1,139 @@ +import ReconnectingWebSocket from 'reconnecting-websocket'; + + +const noop = (...args) => {}; + +/** + * Bridge between Channels and plain javascript. + * + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + * webSocketBridge.listen(function(action, stream) { + * console.log(action, stream); + * }); + */ +export class WebSocketBridge { + constructor(options) { + this._socket = null; + this.streams = {}; + this.default_cb = null; + this.options = Object.assign({}, { + onopen: noop, + }, options); + } + + /** + * Connect to the websocket server + * + * @param {String} [url] The url of the websocket. Defaults to + * `window.location.host` + * @param {String[]|String} [protocols] Optional string or array of protocols. + * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/joewalnes/reconnecting-websocket#options-1). + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + */ + connect(url, protocols, options) { + let _url; + if (url === undefined) { + // Use wss:// if running on https:// + const scheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; + _url = `${scheme}://${window.location.host}/ws`; + } else { + _url = url; + } + this._socket = new ReconnectingWebSocket(_url, protocols, options); + } + + /** + * Starts listening for messages on the websocket, demultiplexing if necessary. + * + * @param {Function} [cb] Callback to be execute when a message + * arrives. The callback will receive `action` and `stream` parameters + * + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + * webSocketBridge.listen(function(action, stream) { + * console.log(action, stream); + * }); + */ + listen(cb) { + this.default_cb = cb; + this._socket.onmessage = (event) => { + const msg = JSON.parse(event.data); + let action; + let stream; + + if (msg.stream !== undefined) { + action = msg.payload; + stream = msg.stream; + const stream_cb = this.streams[stream]; + stream_cb ? stream_cb(action, stream) : null; + } else { + action = msg; + stream = null; + this.default_cb ? this.default_cb(action, stream) : null; + } + }; + + this._socket.onopen = this.options.onopen; + } + + /** + * Adds a 'stream handler' callback. Messages coming from the specified stream + * will call the specified callback. + * + * @param {String} stream The stream name + * @param {Function} cb Callback to be execute when a message + * arrives. The callback will receive `action` and `stream` parameters. + + * @example + * const webSocketBridge = new WebSocketBridge(); + * webSocketBridge.connect(); + * webSocketBridge.listen(); + * webSocketBridge.demultiplex('mystream', function(action, stream) { + * console.log(action, stream); + * }); + * webSocketBridge.demultiplex('myotherstream', function(action, stream) { + * console.info(action, stream); + * }); + */ + demultiplex(stream, cb) { + this.streams[stream] = cb; + } + + /** + * Sends a message to the reply channel. + * + * @param {Object} msg The message + * + * @example + * webSocketBridge.send({prop1: 'value1', prop2: 'value1'}); + */ + send(msg) { + this._socket.send(JSON.stringify(msg)); + } + + /** + * Returns an object to send messages to a specific stream + * + * @param {String} stream The stream name + * @return {Object} convenience object to send messages to `stream`. + * @example + * webSocketBridge.stream('mystream').send({prop1: 'value1', prop2: 'value1'}) + */ + stream(stream) { + return { + send: (action) => { + const msg = { + stream, + payload: action + } + this._socket.send(JSON.stringify(msg)); + } + } + } + +} diff --git a/js_client/tests/websocketbridge.test.js b/js_client/tests/websocketbridge.test.js new file mode 100644 index 0000000..4ed74ea --- /dev/null +++ b/js_client/tests/websocketbridge.test.js @@ -0,0 +1,137 @@ +import { WebSocket, Server } from 'mock-socket'; +import { WebSocketBridge } from '../src/'; + + + +describe('WebSocketBridge', () => { + const mockServer = new Server('ws://localhost'); + const serverReceivedMessage = jest.fn(); + mockServer.on('message', serverReceivedMessage); + + beforeEach(() => { + serverReceivedMessage.mockReset(); + }); + + it('Connects', () => { + const webSocketBridge = new WebSocketBridge(); + webSocketBridge.connect('ws://localhost'); + }); + it('Processes messages', () => { + const webSocketBridge = new WebSocketBridge(); + const myMock = jest.fn(); + + webSocketBridge.connect('ws://localhost'); + webSocketBridge.listen(myMock); + + mockServer.send('{"type": "test", "payload": "message 1"}'); + mockServer.send('{"type": "test", "payload": "message 2"}'); + + expect(myMock.mock.calls.length).toBe(2); + expect(myMock.mock.calls[0][0]).toEqual({"type": "test", "payload": "message 1"}); + expect(myMock.mock.calls[0][1]).toBe(null); + }); + it('Ignores multiplexed messages for unregistered streams', () => { + const webSocketBridge = new WebSocketBridge(); + const myMock = jest.fn(); + + webSocketBridge.connect('ws://localhost'); + webSocketBridge.listen(myMock); + + mockServer.send('{"stream": "stream1", "payload": {"type": "test", "payload": "message 1"}}'); + expect(myMock.mock.calls.length).toBe(0); + + }); + it('Demultiplexes messages only when they have a stream', () => { + const webSocketBridge = new WebSocketBridge(); + const myMock = jest.fn(); + const myMock2 = jest.fn(); + const myMock3 = jest.fn(); + + webSocketBridge.connect('ws://localhost'); + webSocketBridge.listen(myMock); + webSocketBridge.demultiplex('stream1', myMock2); + webSocketBridge.demultiplex('stream2', myMock3); + + mockServer.send('{"type": "test", "payload": "message 1"}'); + expect(myMock.mock.calls.length).toBe(1); + expect(myMock2.mock.calls.length).toBe(0); + expect(myMock3.mock.calls.length).toBe(0); + + mockServer.send('{"stream": "stream1", "payload": {"type": "test", "payload": "message 1"}}'); + + expect(myMock.mock.calls.length).toBe(1); + expect(myMock2.mock.calls.length).toBe(1); + expect(myMock3.mock.calls.length).toBe(0); + + expect(myMock2.mock.calls[0][0]).toEqual({"type": "test", "payload": "message 1"}); + expect(myMock2.mock.calls[0][1]).toBe("stream1"); + + mockServer.send('{"stream": "stream2", "payload": {"type": "test", "payload": "message 2"}}'); + + expect(myMock.mock.calls.length).toBe(1); + expect(myMock2.mock.calls.length).toBe(1); + expect(myMock3.mock.calls.length).toBe(1); + + expect(myMock3.mock.calls[0][0]).toEqual({"type": "test", "payload": "message 2"}); + expect(myMock3.mock.calls[0][1]).toBe("stream2"); + }); + it('Demultiplexes messages', () => { + const webSocketBridge = new WebSocketBridge(); + const myMock = jest.fn(); + const myMock2 = jest.fn(); + + webSocketBridge.connect('ws://localhost'); + webSocketBridge.listen(); + + webSocketBridge.demultiplex('stream1', myMock); + webSocketBridge.demultiplex('stream2', myMock2); + + mockServer.send('{"type": "test", "payload": "message 1"}'); + mockServer.send('{"type": "test", "payload": "message 2"}'); + + expect(myMock.mock.calls.length).toBe(0); + expect(myMock2.mock.calls.length).toBe(0); + + mockServer.send('{"stream": "stream1", "payload": {"type": "test", "payload": "message 1"}}'); + + expect(myMock.mock.calls.length).toBe(1); + + expect(myMock2.mock.calls.length).toBe(0); + + expect(myMock.mock.calls[0][0]).toEqual({"type": "test", "payload": "message 1"}); + expect(myMock.mock.calls[0][1]).toBe("stream1"); + + mockServer.send('{"stream": "stream2", "payload": {"type": "test", "payload": "message 2"}}'); + + expect(myMock.mock.calls.length).toBe(1); + expect(myMock2.mock.calls.length).toBe(1); + + + expect(myMock2.mock.calls[0][0]).toEqual({"type": "test", "payload": "message 2"}); + expect(myMock2.mock.calls[0][1]).toBe("stream2"); + + }); + it('Sends messages', () => { + const webSocketBridge = new WebSocketBridge(); + + webSocketBridge.connect('ws://localhost'); + webSocketBridge.send({"type": "test", "payload": "message 1"}); + + expect(serverReceivedMessage.mock.calls.length).toBe(1); + expect(serverReceivedMessage.mock.calls[0][0]).toEqual(JSON.stringify({"type": "test", "payload": "message 1"})); + }); + it('Multiplexes messages', () => { + const webSocketBridge = new WebSocketBridge(); + + webSocketBridge.connect('ws://localhost'); + webSocketBridge.stream('stream1').send({"type": "test", "payload": "message 1"}); + + expect(serverReceivedMessage.mock.calls.length).toBe(1); + expect(serverReceivedMessage.mock.calls[0][0]).toEqual(JSON.stringify({ + "stream": "stream1", + "payload": { + "type": "test", "payload": "message 1", + }, + })); + }); +}); diff --git a/setup.cfg b/setup.cfg index 6923a28..7efaff1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -exclude = venv/*,tox/*,docs/*,testproject/* +exclude = venv/*,tox/*,docs/*,testproject/*,js_client/* ignore = E123,E128,E402,W503,E731,W601 max-line-length = 119 From 08881bc7de814f8d8511ccd821652e82f6f22aa7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 18 Mar 2017 12:57:46 -0700 Subject: [PATCH 648/746] Releasing 1.1.0 --- CHANGELOG.txt | 15 +++++++++++++++ channels/__init__.py | 2 +- docs/javascript.rst | 17 +++++++++++++---- docs/releases/1.1.0.rst | 18 ++++++++++++------ js_client/package.json | 2 +- 5 files changed, 42 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 53d26cf..a29356a 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,21 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases +1.1.0 (2017-03-18) +------------------ + +* Channels now includes a JavaScript wrapper that wraps reconnection and + multiplexing for you on the client side. + +* Test classes have been moved from ``channels.tests`` to ``channels.test``. + +* Bindings now support non-integer fields for primary keys on models. + +* The ``enforce_ordering`` decorator no longer suffers a race condition where + it would drop messages under high load. + +* ``runserver`` no longer errors if the ``staticfiles`` app is not enabled in Django. + 1.0.3 (2017-02-01) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index f3e910f..81b231f 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.0.3" +__version__ = "1.1.0" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/javascript.rst b/docs/javascript.rst index 3d8344f..b99765f 100644 --- a/docs/javascript.rst +++ b/docs/javascript.rst @@ -4,14 +4,24 @@ Channels WebSocket wrapper Channels ships with a javascript WebSocket wrapper to help you connect to your websocket and send/receive messages. -First, you must include the javascript library in your template:: +First, you must include the javascript library in your template; if you're using +Django's staticfiles, this is as easy as:: {% load staticfiles %} {% static "channels/js/websocketbridge.js" %} +If you are using an alternative method of serving static files, the compiled +source code is located at ``channels/static/channels/js/websocketbridge.js`` in +a Channels installation. We compile the file for you each release; it's ready +to serve as-is. + +The library is deliberately quite low-level and generic; it's designed to +be compatible with any JavaScript code or framework, so you can build more +specific integration on top of it. + To process messages:: - + const webSocketBridge = new channels.WebSocketBridge(); webSocketBridge.connect(); webSocketBridge.listen(function(action, stream) { @@ -36,10 +46,9 @@ To demultiplex specific streams:: console.info(action, stream); }); - To send a message to a specific stream:: webSocketBridge.stream('mystream').send({prop1: 'value1', prop2: 'value1'}) -The library is also available as npm module, under the name +The library is also available as a npm module, under the name `django-channels `_ diff --git a/docs/releases/1.1.0.rst b/docs/releases/1.1.0.rst index d461951..5d72136 100644 --- a/docs/releases/1.1.0.rst +++ b/docs/releases/1.1.0.rst @@ -1,27 +1,33 @@ 1.1.0 Release Notes =================== -.. note:: - The 1.1.0 release is still in development. +Channels 1.1.0 introduces a couple of major but backwards-compatible changes, +including most notably the inclusion of a standard, framework-agnostic JavaScript +library for easier integration with your site. -Channels 1.1.0 introduces a couple of major but backwards-compatible changes. -It was released on UNKNOWN. Major Changes ------------- +* Channels now includes a JavaScript wrapper that wraps reconnection and + multiplexing for you on the client side. For more on how to use it, see the + :doc:`/javascript` documentation. + * Test classes have been moved from ``channels.tests`` to ``channels.test`` to better match Django. Old imports from ``channels.tests`` will continue to work but will trigger a deprecation warning, and ``channels.tests`` will be removed completely in version 1.3. + Minor Changes & Bugfixes ------------------------ -* Bindings now support non-integer fields for primary keys on models +* Bindings now support non-integer fields for primary keys on models. * The ``enforce_ordering`` decorator no longer suffers a race condition where - it would drop messages under high load + it would drop messages under high load. + +* ``runserver`` no longer errors if the ``staticfiles`` app is not enabled in Django. Backwards Incompatible Changes diff --git a/js_client/package.json b/js_client/package.json index b62c20e..8eff412 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -1,6 +1,6 @@ { "name": "django-channels", - "version": "0.0.2", + "version": "1.1.0", "description": "", "repository": { "type": "git", From 274a5a8c988a804e40da098f59ec6c8f0378fe34 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 19 Mar 2017 11:23:28 -0700 Subject: [PATCH 649/746] Releasing 1.1.1 --- CHANGELOG.txt | 6 ++++++ MANIFEST.in | 1 + channels/__init__.py | 2 +- docs/releases/1.1.1.rst | 22 ++++++++++++++++++++++ docs/releases/index.rst | 1 + js_client/package.json | 4 ++-- setup.py | 2 +- 7 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 docs/releases/1.1.1.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index a29356a..ab6e47f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,12 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases +1.1.1 (2017-03-19) +------------------ + +* Fixed JS packaging issue + + 1.1.0 (2017-03-18) ------------------ diff --git a/MANIFEST.in b/MANIFEST.in index aae9579..100f6b7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ recursive-exclude tests * +include channels/static/channels/js/* diff --git a/channels/__init__.py b/channels/__init__.py index 81b231f..e79f6b4 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.0" +__version__ = "1.1.1" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/releases/1.1.1.rst b/docs/releases/1.1.1.rst new file mode 100644 index 0000000..6a2b1ab --- /dev/null +++ b/docs/releases/1.1.1.rst @@ -0,0 +1,22 @@ +1.1.1 Release Notes +=================== + +Channels 1.1.1 is a bugfix release that fixes a packaging issue with the JavaScript files. + + +Major Changes +------------- + +None. + +Minor Changes & Bugfixes +------------------------ + +* The JavaScript binding introduced in 1.1.0 is now correctly packaged and + included in builds. + + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 000825a..736c75a 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -9,3 +9,4 @@ Release Notes 1.0.2 1.0.3 1.1.0 + 1.1.1 diff --git a/js_client/package.json b/js_client/package.json index 8eff412..af9869f 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -1,6 +1,6 @@ { "name": "django-channels", - "version": "1.1.0", + "version": "1.1.1", "description": "", "repository": { "type": "git", @@ -36,7 +36,7 @@ }, "devDependencies": { "babel": "^6.5.2", - "babel-cli": "^6.16.0", + "babel-cli": "^6.24.0", "babel-core": "^6.16.0", "babel-plugin-transform-inline-environment-variables": "^6.8.0", "babel-plugin-transform-object-assign": "^6.8.0", diff --git a/setup.py b/setup.py index e1adedd..05a45b2 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( include_package_data=True, install_requires=[ 'Django>=1.8', - 'asgiref>=1.0.0', + 'asgiref>=1.0.1', 'daphne>=1.0.0', ], extras_require={ From 528cd89f4e576dd397e7aa45a2f0fcbdbadf202e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 19 Mar 2017 12:17:13 -0700 Subject: [PATCH 650/746] Rearrange ASGI specs into a proper split of messaging/protocols --- docs/asgi.rst | 548 +++++--------------------------------------- docs/asgi/email.rst | 3 + docs/asgi/udp.rst | 6 +- docs/asgi/www.rst | 446 +++++++++++++++++++++++++++++++++++ 4 files changed, 513 insertions(+), 490 deletions(-) create mode 100644 docs/asgi/www.rst diff --git a/docs/asgi.rst b/docs/asgi.rst index ea5e1d1..9b13d9c 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -2,8 +2,8 @@ ASGI (Asynchronous Server Gateway Interface) Draft Spec ======================================================= -**NOTE: This is still in-progress, and may change substantially as development -progresses.** +.. note:: + This is still in-progress, but is now mostly complete. Abstract ======== @@ -13,9 +13,12 @@ servers (particularly web servers) and Python applications, intended to allow handling of multiple common protocol styles (including HTTP, HTTP2, and WebSocket). -It is intended to supplement and expand on WSGI, though the design -deliberately includes provisions to allow WSGI-to-ASGI and ASGI-to-WGSI -adapters to be easily written for the HTTP protocol. +This base specification is intended to fix in place the set of APIs by which +these servers interact and the guarantees and style of message delivery; +each supported protocol (such as HTTP) has a sub-specification that outlines +how to encode and decode that protocol into messages. + +The set of sub-specifications is available :ref:`in the Message Formats section `. Rationale @@ -32,16 +35,25 @@ ASGI attempts to preserve a simple application interface, but provide an abstraction that allows for data to be sent and received at any time, and from different application threads or processes. -It also lays out new, serialization-compatible formats for things like -HTTP requests and responses and WebSocket data frames, to allow these to -be transported over a network or local memory, and allow separation -of protocol handling and application logic into different processes. +It also take the principle of turning protocols into Python-compatible, +asynchronous-friendly sets of messages and generalises it into two sections; +a standardised interface for communication and to build servers around (this +document), and a set of standard message formats for each protocol (the +sub-specifications, linked above). -Part of this design is ensuring there is an easy path to use both +Its primary design is for HTTP, however, and part of this design is +ensuring there is an easy path to use both existing WSGI servers and applications, as a large majority of Python web usage relies on WSGI and providing an easy path forwards is critical to adoption. +The end result of this process has been a specification for generalised +inter-process communication between Python processes, with a certain set of +guarantees and delivery styles that make it suited to low-latency protocol +processing and response. It is not intended to replace things like traditional +task queues, but it is intended that it could be used for things like +distributed systems communication, or as the backbone of a service-oriented +architecure for inter-service communication. Overview ======== @@ -62,12 +74,12 @@ to a channel layer instance. It is intended that applications and protocol servers always run in separate processes or threads, and always communicate via the channel layer. -Despite the name of the proposal, ASGI does not specify or design to any -specific in-process async solution, such as ``asyncio``, ``twisted``, or -``gevent``. Instead, the ``receive`` function can be switched between -nonblocking or synchronous. This approach allows applications to choose what's -best for their current runtime environment; further improvements may provide -extensions where cooperative versions of receive are provided. +ASGI tries to be as compatible as possible by default, and so the only +implementation of ``receive`` that must be provided is a fully-synchronous, +nonblocking one. Implementations can then choose to implement a blocking mode +in this method, and if they wish to go further, versions compatible with +the asyncio or Twisted frameworks (or other frameworks that may become +popular, thanks to the extension declaration mechanism). The distinction between protocol servers and applications in this document is mostly to distinguish their roles and to make illustrating concepts easier. @@ -76,11 +88,11 @@ to have a process that does both, or middleware-like code that transforms messages between two different channel layers or channel names. It is expected, however, that most deployments will fall into this pattern. -There is even room for a WSGI-like application abstraction with a callable -which takes ``(channel, message, send_func)``, but this would be slightly -too restrictive for many use cases and does not cover how to specify -channel names to listen on; it is expected that frameworks will cover this -use case. +There is even room for a WSGI-like application abstraction on the application +server side, with a callable which takes ``(channel, message, send_func)``, +but this would be slightly too restrictive for many use cases and does not +cover how to specify channel names to listen on. It is expected that +frameworks will cover this use case. Channels and Messages @@ -164,12 +176,11 @@ ASGI messages represent two main things - internal application events (for example, a channel might be used to queue thumbnails of previously uploaded videos), and protocol events to/from connected clients. -As such, this specification outlines encodings to and from ASGI messages -for HTTP and WebSocket; this allows any ASGI -web server to talk to any ASGI web application, as well as servers and -applications for any other protocol with a common specification. It is -recommended that if other protocols become commonplace they should gain -standardized formats in a supplementary specification of their own. +As such, there are :ref:`sub-specifications ` that +outline encodings to and from ASGI messages for common protocols like HTTP and +WebSocket; in particular, the HTTP one covers the WSGI/ASGI interoperability. +It is recommended that if a protocol becomes commonplace, it should gain +standardized formats in a sub-specification of its own. The message formats are a key part of the specification; without them, the protocol server and web application might be able to talk to each other, @@ -275,7 +286,7 @@ Solving this issue is left to frameworks and application code; there are already solutions such as database transactions that help solve this, and the vast majority of application code will not need to deal with this problem. If ordering of incoming packets matters for a protocol, they should -be annotated with a packet number (as WebSocket is in this specification). +be annotated with a packet number (as WebSocket is in its specification). Single-reader and process-specific channels, such as those used for response channels back to clients, are not subject to this problem; a single reader @@ -394,14 +405,21 @@ A channel layer implementing the ``twisted`` extension must also provide: * ``receive_twisted(channels)``, a function that behaves like ``receive`` but that returns a Twisted Deferred that eventually returns either ``(channel, message)`` or ``(None, None)``. It is not possible - to run it in nonblocking mode; use the normal ``receive`` for that. + to run it in nonblocking mode; use the normal ``receive`` for that. The + channel layer must be able to deal with this function being called from + many different places in a codebase simultaneously - likely once from each + Twisted Protocol instance - and so it is recommended that implementations + use internal connection pooling and call merging or similar. A channel layer implementing the ``asyncio`` extension must also provide: * ``receive_asyncio(channels)``, a function that behaves like ``receive`` but that fulfills the asyncio coroutine contract to block until either a result is available or an internal timeout is reached - and ``(None, None)`` is returned. + and ``(None, None)`` is returned. The channel layer must be able to deal + with this function being called from many different places in a codebase + simultaneously, and so it is recommended that implementations + use internal connection pooling and call merging or similar. Channel Semantics ----------------- @@ -473,6 +491,8 @@ restart, you will likely need to move and reprovision protocol servers, and making sure your code can cope with this is important. +.. _asgi_sub_specifications: + Message Formats --------------- @@ -501,380 +521,14 @@ in the channel name they are received on. Two types that are sent on the same channel, such as HTTP responses and response chunks, are distinguished apart by their required fields. +Message formats can be found in the sub-specifications: -HTTP ----- +.. toctree:: + :maxdepth: 1 -The HTTP format covers HTTP/1.0, HTTP/1.1 and HTTP/2, as the changes in -HTTP/2 are largely on the transport level. A protocol server should give -different requests on the same connection different reply channels, and -correctly multiplex the responses back into the same stream as they come in. -The HTTP version is available as a string in the request message. - -HTTP/2 Server Push responses are included, but must be sent prior to the -main response, and applications must check for ``http_version = 2`` before -sending them; if a protocol server or connection incapable of Server Push -receives these, it must drop them. - -Multiple header fields with the same name are complex in HTTP. RFC 7230 -states that for any header field that can appear multiple times, it is exactly -equivalent to sending that header field only once with all the values joined by -commas. - -However, RFC 7230 and RFC 6265 make it clear that this rule does not apply to -the various headers used by HTTP cookies (``Cookie`` and ``Set-Cookie``). The -``Cookie`` header must only be sent once by a user-agent, but the -``Set-Cookie`` header may appear repeatedly and cannot be joined by commas. -For this reason, we can safely make the request ``headers`` a ``dict``, but -the response ``headers`` must be sent as a list of tuples, which matches WSGI. - -Request -''''''' - -Sent once for each request that comes into the protocol server. If sending -this raises ``ChannelFull``, the interface server must respond with a -500-range error, preferably ``503 Service Unavailable``, and close the connection. - -Channel: ``http.request`` - -Keys: - -* ``reply_channel``: Channel name for responses and server pushes, starting with - ``http.response!`` - -* ``http_version``: Unicode string, one of ``1.0``, ``1.1`` or ``2``. - -* ``method``: Unicode string HTTP method name, uppercased. - -* ``scheme``: Unicode string URL scheme portion (likely ``http`` or ``https``). - Optional (but must not be empty), default is ``"http"``. - -* ``path``: Unicode string HTTP path from URL, with percent escapes decoded - and UTF8 byte sequences decoded into characters. - -* ``query_string``: Byte string URL portion after the ``?``, not url-decoded. - -* ``root_path``: Unicode string that indicates the root path this application - is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults - to ``""``. - -* ``headers``: A list of ``[name, value]`` lists, where ``name`` is the - byte string header name, and ``value`` is the byte string - header value. Order of header values must be preserved from the original HTTP - request; order of header names is not important. Duplicates are possible and - must be preserved in the message as received. - Header names must be lowercased. - -* ``body``: Body of the request, as a byte string. Optional, defaults to ``""``. - If ``body_channel`` is set, treat as start of body and concatenate - on further chunks. - -* ``body_channel``: Name of a single-reader channel (containing ``?``) that contains - Request Body Chunk messages representing a large request body. - Optional, defaults to ``None``. Chunks append to ``body`` if set. Presence of - a channel indicates at least one Request Body Chunk message needs to be read, - and then further consumption keyed off of the ``more_content`` key in those - messages. - -* ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the - remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an - integer. Optional, defaults to ``None``. - -* ``server``: List of ``[host, port]`` where ``host`` is the listening address - for this server as a unicode string, and ``port`` is the integer listening port. - Optional, defaults to ``None``. - - -Request Body Chunk -'''''''''''''''''' - -Must be sent after an initial Response. If trying to send this raises -``ChannelFull``, the interface server should wait and try again until it is -accepted (the consumer at the other end of the channel may not be as fast -consuming the data as the client is at sending it). - -Channel: ``http.request.body?`` - -Keys: - -* ``content``: Byte string of HTTP body content, will be concatenated onto - previously received ``content`` values and ``body`` key in Request. - Not required if ``closed`` is True, required otherwise. - -* ``closed``: True if the client closed the connection prematurely and the - rest of the body. If you receive this, abandon processing of the HTTP request. - Optional, defaults to ``False``. - -* ``more_content``: Boolean value signifying if there is additional content - to come (as part of a Request Body Chunk message). If ``False``, request will - be taken as complete, and any further messages on the channel - will be ignored. Optional, defaults to ``False``. - - -Response -'''''''' - -Send after any server pushes, and before any response chunks. If ``ChannelFull`` -is encountered, wait and try again later, optionally giving up after a -predetermined timeout. - -Channel: ``http.response!`` - -Keys: - -* ``status``: Integer HTTP status code. - -* ``headers``: A list of ``[name, value]`` lists, where ``name`` is the - byte string header name, and ``value`` is the byte string - header value. Order must be preserved in the HTTP response. Header names - must be lowercased. - -* ``content``: Byte string of HTTP body content. - Optional, defaults to empty string. - -* ``more_content``: Boolean value signifying if there is additional content - to come (as part of a Response Chunk message). If ``False``, response will - be taken as complete and closed off, and any further messages on the channel - will be ignored. Optional, defaults to ``False``. - - -Response Chunk -'''''''''''''' - -Must be sent after an initial Response. If ``ChannelFull`` -is encountered, wait and try again later. - -Channel: ``http.response!`` - -Keys: - -* ``content``: Byte string of HTTP body content, will be concatenated onto - previously received ``content`` values. - -* ``more_content``: Boolean value signifying if there is additional content - to come (as part of a Response Chunk message). If ``False``, response will - be taken as complete and closed off, and any further messages on the channel - will be ignored. Optional, defaults to ``False``. - - -Server Push -''''''''''' - -Must be sent before any Response or Response Chunk messages. If ``ChannelFull`` -is encountered, wait and try again later, optionally giving up after a -predetermined timeout, and give up on the entire response this push is -connected to. - -When a server receives this message, it must treat the Request message in the -``request`` field of the Server Push as though it were a new HTTP request being -received from the network. A server may, if it chooses, apply all of its -internal logic to handling this request (e.g. the server may want to try to -satisfy the request from a cache). Regardless, if the server is unable to -satisfy the request itself it must create a new ``http.response!`` channel for -the application to send the Response message on, fill that channel in on the -``reply_channel`` field of the message, and then send the Request back to the -application on the ``http.request`` channel. - -This approach limits the amount of knowledge the application has to have about -pushed responses: they essentially appear to the application like a normal HTTP -request, with the difference being that the application itself triggered the -request. - -If the remote peer does not support server push, either because it's not a -HTTP/2 peer or because SETTINGS_ENABLE_PUSH is set to 0, the server must do -nothing in response to this message. - -Channel: ``http.response!`` - -Keys: - -* ``request``: A Request message. The ``body``, ``body_channel``, and - ``reply_channel`` fields MUST be absent: bodies are not allowed on - server-pushed requests, and applications should not create reply channels. - - -Disconnect -'''''''''' - -Sent when a HTTP connection is closed. This is mainly useful for long-polling, -where you may have added the response channel to a Group or other set of -channels you want to trigger a reply to when data arrives. - -If ``ChannelFull`` is raised, then give up attempting to send the message; -consumption is not required. - -Channel: ``http.disconnect`` - -Keys: - -* ``reply_channel``: Channel name responses would have been sent on. No longer - valid after this message is sent; all messages to it will be dropped. - -* ``path``: Unicode string HTTP path from URL, with percent escapes decoded - and UTF8 byte sequences decoded into characters. - - -WebSocket ---------- - -WebSockets share some HTTP details - they have a path and headers - but also -have more state. Path and header details are only sent in the connection -message; applications that need to refer to these during later messages -should store them in a cache or database. - -WebSocket protocol servers should handle PING/PONG requests themselves, and -send PING frames as necessary to ensure the connection is alive. - -Note that you **must** ensure that websocket.connect is consumed; if an -interface server gets ``ChannelFull`` on this channel it will drop the -connection. Django Channels ships with a no-op consumer attached by default; -we recommend other implementations do the same. - - -Connection -'''''''''' - -Sent when the client initially opens a connection and completes the -WebSocket handshake. If sending this raises ``ChannelFull``, the interface -server must close the connection with either HTTP status code ``503`` or -WebSocket close code ``1013``. - -This message must be responded to on the ``reply_channel`` with a -*Send/Close/Accept* message before the socket will pass messages on the -``receive`` channel. The protocol server should ideally send this message -during the handshake phase of the WebSocket and not complete the handshake -until it gets a reply, returning HTTP status code ``403`` if the connection is -denied. If this is not possible, it must buffer WebSocket frames and not -send them onto ``websocket.receive`` until a reply is received, and if the -connection is rejected, return WebSocket close code ``4403``. - -Channel: ``websocket.connect`` - -Keys: - -* ``reply_channel``: Channel name for sending data, start with ``websocket.send!`` - -* ``scheme``: Unicode string URL scheme portion (likely ``ws`` or ``wss``). - Optional (but must not be empty), default is ``ws``. - -* ``path``: Unicode HTTP path from URL, already urldecoded. - -* ``query_string``: Byte string URL portion after the ``?``. Optional, default - is empty string. - -* ``root_path``: Byte string that indicates the root path this application - is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults - to empty string. - -* ``headers``: List of ``[name, value]``, where ``name`` is the - header name as byte string and ``value`` is the header value as a byte - string. Order should be preserved from the original HTTP request; - duplicates are possible and must be preserved in the message as received. - Header names must be lowercased. - -* ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the - remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an - integer. Optional, defaults to ``None``. - -* ``server``: List of ``[host, port]`` where ``host`` is the listening address - for this server as a unicode string, and ``port`` is the integer listening port. - Optional, defaults to ``None``. - -* ``order``: The integer value ``0``. - - -Receive -''''''' - -Sent when a data frame is received from the client. If ``ChannelFull`` is -raised, you may retry sending it but if it does not send the socket must -be closed with websocket error code 1013. - -Channel: ``websocket.receive`` - -Keys: - -* ``reply_channel``: Channel name for sending data, starting with ``websocket.send!`` - -* ``path``: Path sent during ``connect``, sent to make routing easier for apps. - -* ``bytes``: Byte string of frame content, if it was bytes mode, or ``None``. - -* ``text``: Unicode string of frame content, if it was text mode, or ``None``. - -* ``order``: Order of this frame in the WebSocket stream, starting - at 1 (``connect`` is 0). - -One of ``bytes`` or ``text`` must be non-``None``. - - -Disconnection -''''''''''''' - -Sent when either connection to the client is lost, either from the client -closing the connection, the server closing the connection, or loss of the -socket. - -If ``ChannelFull`` is raised, then give up attempting to send the message; -consumption is not required. - -Channel: ``websocket.disconnect`` - -Keys: - -* ``reply_channel``: Channel name that was used for sending data, starting - with ``websocket.send!``. Cannot be used to send at this point; provided - as a way to identify the connection only. - -* ``code``: The WebSocket close code (integer), as per the WebSocket spec. - -* ``path``: Path sent during ``connect``, sent to make routing easier for apps. - -* ``order``: Order of the disconnection relative to the incoming frames' - ``order`` values in ``websocket.receive``. - - -Send/Close/Accept -''''''''''''''''' - -Sends a data frame to the client and/or closes the connection from the -server end and/or accepts a connection. If ``ChannelFull`` is raised, wait -and try again. - -If received while the connection is waiting for acceptance after a ``connect`` -message: - -* If ``bytes`` or ``text`` is present, accept the connection and send the data. -* If ``accept`` is ``True``, accept the connection and do nothing else. -* If ``close`` is ``True`` or a positive integer, reject the connection. If - ``bytes`` or ``text`` is also set, it should accept the connection, send the - frame, then immediately close the connection. - -If received while the connection is established: - -* If ``bytes`` or ``text`` is present, send the data. -* If ``close`` is ``True`` or a positive integer, close the connection after - any send. -* ``accept`` is ignored. - -Channel: ``websocket.send!`` - -Keys: - -* ``bytes``: Byte string of frame content, if in bytes mode, or ``None``. - -* ``text``: Unicode string of frame content, if in text mode, or ``None``. - -* ``close``: Boolean indicating if the connection should be closed after - data is sent, if any. Alternatively, a positive integer specifying the - response code. The response code will be 1000 if you pass ``True``. - Optional, default ``False``. - -* ``accept``: Boolean saying if the connection should be accepted without - sending a frame if it is in the handshake phase. - -A maximum of one of ``bytes`` or ``text`` may be provided. If both are -provided, the protocol server should ignore the message entirely. + /asgi/www + /asgi/delay + /asgi/udp Protocol Format Guidelines @@ -932,14 +586,14 @@ a message doesn't get received purely because another channel is busy. Strings and Unicode ------------------- -In this document, *byte string* refers to ``str`` on Python 2 and ``bytes`` -on Python 3. If this type still supports Unicode codepoints due to the -underlying implementation, then any values should be kept within the lower -8-byte range. +In this document, and all sub-specifications, *byte string* refers to +``str`` on Python 2 and ``bytes`` on Python 3. If this type still supports +Unicode codepoints due to the underlying implementation, then any values +should be kept within the 0 - 255 range. *Unicode string* refers to ``unicode`` on Python 2 and ``str`` on Python 3. This document will never specify just *string* - all strings are one of the -two types. +two exact types. Some serializers, such as ``json``, cannot differentiate between byte strings and unicode strings; these should include logic to box one type as @@ -960,58 +614,6 @@ limitation that they only use the following characters: and only one per name) -WSGI Compatibility ------------------- - -Part of the design of the HTTP portion of this spec is to make sure it -aligns well with the WSGI specification, to ensure easy adaptability -between both specifications and the ability to keep using WSGI servers or -applications with ASGI. - -The adaptability works in two ways: - -* WSGI Server to ASGI: A WSGI application can be written that transforms - ``environ`` into a Request message, sends it off on the ``http.request`` - channel, and then waits on a generated response channel for a Response - message. This has the disadvantage of tying up an entire WSGI thread - to poll one channel, but should not be a massive performance drop if - there is no backlog on the request channel, and would work fine for an - in-process adapter to run a pure-ASGI web application. - -* ASGI to WSGI application: A small wrapper process is needed that listens - on the ``http.request`` channel, and decodes incoming Request messages - into an ``environ`` dict that matches the WSGI specs, while passing in - a ``start_response`` that stores the values for sending with the first - content chunk. Then, the application iterates over the WSGI app, - packaging each returned content chunk into a Response or Response Chunk - message (if more than one is yielded). - -There is an almost direct mapping for the various special keys in -WSGI's ``environ`` variable to the Request message: - -* ``REQUEST_METHOD`` is the ``method`` key -* ``SCRIPT_NAME`` is ``root_path`` -* ``PATH_INFO`` can be derived from ``path`` and ``root_path`` -* ``QUERY_STRING`` is ``query_string`` -* ``CONTENT_TYPE`` can be extracted from ``headers`` -* ``CONTENT_LENGTH`` can be extracted from ``headers`` -* ``SERVER_NAME`` and ``SERVER_PORT`` are in ``server`` -* ``REMOTE_HOST``/``REMOTE_ADDR`` and ``REMOTE_PORT`` are in ``client`` -* ``SERVER_PROTOCOL`` is encoded in ``http_version`` -* ``wsgi.url_scheme`` is ``scheme`` -* ``wsgi.input`` is a StringIO around ``body`` -* ``wsgi.errors`` is directed by the wrapper as needed - -The ``start_response`` callable maps similarly to Response: - -* The ``status`` argument becomes ``status``, with the reason phrase dropped. -* ``response_headers`` maps to ``headers`` - -It may even be possible to map Request Body Chunks in a way that allows -streaming of body data, though it would likely be easier and sufficient for -many applications to simply buffer the whole body into memory before calling -the WSGI application. - Common Questions ================ @@ -1024,35 +626,7 @@ Common Questions custom classes (e.g. ``http.request`` messages become ``Request`` objects) -TODOs -===== - -* Maybe remove ``http_version`` and replace with ``supports_server_push``? - -* ``receive`` can't easily be implemented with async/cooperative code - behind it as it's nonblocking - possible alternative call type? - Asyncio extension that provides ``receive_yield``? - -* Possible extension to allow detection of channel layer flush/restart and - prompt protocol servers to restart? - -* Maybe WSGI-app like spec for simple "applications" that allows standardized - application-running servers? - - Copyright ========= This document has been placed in the public domain. - - -Protocol Definitions -==================== - - -.. toctree:: - :maxdepth: 1 - - /asgi/email - /asgi/udp - /asgi/delay diff --git a/docs/asgi/email.rst b/docs/asgi/email.rst index 08ad5ca..ae72dca 100644 --- a/docs/asgi/email.rst +++ b/docs/asgi/email.rst @@ -2,6 +2,9 @@ Email ASGI Message Format (Draft Spec) ====================================== +.. warning:: + This is an incomplete draft. + Represents emails sent or received, likely over the SMTP protocol though that is not directly specified here (a protocol server could in theory deliver or receive email over HTTP to some external service, for example). Generally diff --git a/docs/asgi/udp.rst b/docs/asgi/udp.rst index 38c35a9..3b46331 100644 --- a/docs/asgi/udp.rst +++ b/docs/asgi/udp.rst @@ -1,6 +1,6 @@ -==================================== -UDP ASGI Message Format (Draft Spec) -==================================== +============================= +UDP ASGI Message Format (1.0) +============================= Raw UDP is specified here as it is a datagram-based, unordered and unreliable protocol, which neatly maps to the underlying message abstraction. It is not diff --git a/docs/asgi/www.rst b/docs/asgi/www.rst new file mode 100644 index 0000000..76e3c60 --- /dev/null +++ b/docs/asgi/www.rst @@ -0,0 +1,446 @@ +================================================= +HTTP & WebSocket ASGI Message Format (Draft Spec) +================================================= + +.. note:: + This is still in-progress, but is now mostly complete. + +The HTTP+WebSocket ASGI sub-specification outlines how to transport HTTP/1.1, +HTTP/2 and WebSocket connections over an ASGI-compatible channel layer. + +It is deliberately intended and designed to be a superset of the WSGI format +and specifies how to translate between the two for the set of requests that +are able to be handled by WSGI. + +HTTP +---- + +The HTTP format covers HTTP/1.0, HTTP/1.1 and HTTP/2, as the changes in +HTTP/2 are largely on the transport level. A protocol server should give +different requests on the same connection different reply channels, and +correctly multiplex the responses back into the same stream as they come in. +The HTTP version is available as a string in the request message. + +HTTP/2 Server Push responses are included, but must be sent prior to the +main response, and applications must check for ``http_version = 2`` before +sending them; if a protocol server or connection incapable of Server Push +receives these, it must drop them. + +Multiple header fields with the same name are complex in HTTP. RFC 7230 +states that for any header field that can appear multiple times, it is exactly +equivalent to sending that header field only once with all the values joined by +commas. + +However, RFC 7230 and RFC 6265 make it clear that this rule does not apply to +the various headers used by HTTP cookies (``Cookie`` and ``Set-Cookie``). The +``Cookie`` header must only be sent once by a user-agent, but the +``Set-Cookie`` header may appear repeatedly and cannot be joined by commas. +For this reason, we can safely make the request ``headers`` a ``dict``, but +the response ``headers`` must be sent as a list of tuples, which matches WSGI. + +Request +''''''' + +Sent once for each request that comes into the protocol server. If sending +this raises ``ChannelFull``, the interface server must respond with a +500-range error, preferably ``503 Service Unavailable``, and close the connection. + +Channel: ``http.request`` + +Keys: + +* ``reply_channel``: Channel name for responses and server pushes, starting with + ``http.response!`` + +* ``http_version``: Unicode string, one of ``1.0``, ``1.1`` or ``2``. + +* ``method``: Unicode string HTTP method name, uppercased. + +* ``scheme``: Unicode string URL scheme portion (likely ``http`` or ``https``). + Optional (but must not be empty), default is ``"http"``. + +* ``path``: Unicode string HTTP path from URL, with percent escapes decoded + and UTF8 byte sequences decoded into characters. + +* ``query_string``: Byte string URL portion after the ``?``, not url-decoded. + +* ``root_path``: Unicode string that indicates the root path this application + is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults + to ``""``. + +* ``headers``: A list of ``[name, value]`` lists, where ``name`` is the + byte string header name, and ``value`` is the byte string + header value. Order of header values must be preserved from the original HTTP + request; order of header names is not important. Duplicates are possible and + must be preserved in the message as received. + Header names must be lowercased. + +* ``body``: Body of the request, as a byte string. Optional, defaults to ``""``. + If ``body_channel`` is set, treat as start of body and concatenate + on further chunks. + +* ``body_channel``: Name of a single-reader channel (containing ``?``) that contains + Request Body Chunk messages representing a large request body. + Optional, defaults to ``None``. Chunks append to ``body`` if set. Presence of + a channel indicates at least one Request Body Chunk message needs to be read, + and then further consumption keyed off of the ``more_content`` key in those + messages. + +* ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the + remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an + integer. Optional, defaults to ``None``. + +* ``server``: List of ``[host, port]`` where ``host`` is the listening address + for this server as a unicode string, and ``port`` is the integer listening port. + Optional, defaults to ``None``. + + +Request Body Chunk +'''''''''''''''''' + +Must be sent after an initial Response. If trying to send this raises +``ChannelFull``, the interface server should wait and try again until it is +accepted (the consumer at the other end of the channel may not be as fast +consuming the data as the client is at sending it). + +Channel: ``http.request.body?`` + +Keys: + +* ``content``: Byte string of HTTP body content, will be concatenated onto + previously received ``content`` values and ``body`` key in Request. + Not required if ``closed`` is True, required otherwise. + +* ``closed``: True if the client closed the connection prematurely and the + rest of the body. If you receive this, abandon processing of the HTTP request. + Optional, defaults to ``False``. + +* ``more_content``: Boolean value signifying if there is additional content + to come (as part of a Request Body Chunk message). If ``False``, request will + be taken as complete, and any further messages on the channel + will be ignored. Optional, defaults to ``False``. + + +Response +'''''''' + +Send after any server pushes, and before any response chunks. If ``ChannelFull`` +is encountered, wait and try again later, optionally giving up after a +predetermined timeout. + +Channel: ``http.response!`` + +Keys: + +* ``status``: Integer HTTP status code. + +* ``headers``: A list of ``[name, value]`` lists, where ``name`` is the + byte string header name, and ``value`` is the byte string + header value. Order must be preserved in the HTTP response. Header names + must be lowercased. + +* ``content``: Byte string of HTTP body content. + Optional, defaults to empty string. + +* ``more_content``: Boolean value signifying if there is additional content + to come (as part of a Response Chunk message). If ``False``, response will + be taken as complete and closed off, and any further messages on the channel + will be ignored. Optional, defaults to ``False``. + + +Response Chunk +'''''''''''''' + +Must be sent after an initial Response. If ``ChannelFull`` +is encountered, wait and try again later. + +Channel: ``http.response!`` + +Keys: + +* ``content``: Byte string of HTTP body content, will be concatenated onto + previously received ``content`` values. + +* ``more_content``: Boolean value signifying if there is additional content + to come (as part of a Response Chunk message). If ``False``, response will + be taken as complete and closed off, and any further messages on the channel + will be ignored. Optional, defaults to ``False``. + + +Server Push +''''''''''' + +Must be sent before any Response or Response Chunk messages. If ``ChannelFull`` +is encountered, wait and try again later, optionally giving up after a +predetermined timeout, and give up on the entire response this push is +connected to. + +When a server receives this message, it must treat the Request message in the +``request`` field of the Server Push as though it were a new HTTP request being +received from the network. A server may, if it chooses, apply all of its +internal logic to handling this request (e.g. the server may want to try to +satisfy the request from a cache). Regardless, if the server is unable to +satisfy the request itself it must create a new ``http.response!`` channel for +the application to send the Response message on, fill that channel in on the +``reply_channel`` field of the message, and then send the Request back to the +application on the ``http.request`` channel. + +This approach limits the amount of knowledge the application has to have about +pushed responses: they essentially appear to the application like a normal HTTP +request, with the difference being that the application itself triggered the +request. + +If the remote peer does not support server push, either because it's not a +HTTP/2 peer or because SETTINGS_ENABLE_PUSH is set to 0, the server must do +nothing in response to this message. + +Channel: ``http.response!`` + +Keys: + +* ``request``: A Request message. The ``body``, ``body_channel``, and + ``reply_channel`` fields MUST be absent: bodies are not allowed on + server-pushed requests, and applications should not create reply channels. + + +Disconnect +'''''''''' + +Sent when a HTTP connection is closed. This is mainly useful for long-polling, +where you may have added the response channel to a Group or other set of +channels you want to trigger a reply to when data arrives. + +If ``ChannelFull`` is raised, then give up attempting to send the message; +consumption is not required. + +Channel: ``http.disconnect`` + +Keys: + +* ``reply_channel``: Channel name responses would have been sent on. No longer + valid after this message is sent; all messages to it will be dropped. + +* ``path``: Unicode string HTTP path from URL, with percent escapes decoded + and UTF8 byte sequences decoded into characters. + + +WebSocket +--------- + +WebSockets share some HTTP details - they have a path and headers - but also +have more state. Path and header details are only sent in the connection +message; applications that need to refer to these during later messages +should store them in a cache or database. + +WebSocket protocol servers should handle PING/PONG requests themselves, and +send PING frames as necessary to ensure the connection is alive. + +Note that you **must** ensure that websocket.connect is consumed; if an +interface server gets ``ChannelFull`` on this channel it will drop the +connection. Django Channels ships with a no-op consumer attached by default; +we recommend other implementations do the same. + + +Connection +'''''''''' + +Sent when the client initially opens a connection and completes the +WebSocket handshake. If sending this raises ``ChannelFull``, the interface +server must close the connection with either HTTP status code ``503`` or +WebSocket close code ``1013``. + +This message must be responded to on the ``reply_channel`` with a +*Send/Close/Accept* message before the socket will pass messages on the +``receive`` channel. The protocol server should ideally send this message +during the handshake phase of the WebSocket and not complete the handshake +until it gets a reply, returning HTTP status code ``403`` if the connection is +denied. If this is not possible, it must buffer WebSocket frames and not +send them onto ``websocket.receive`` until a reply is received, and if the +connection is rejected, return WebSocket close code ``4403``. + +Channel: ``websocket.connect`` + +Keys: + +* ``reply_channel``: Channel name for sending data, start with ``websocket.send!`` + +* ``scheme``: Unicode string URL scheme portion (likely ``ws`` or ``wss``). + Optional (but must not be empty), default is ``ws``. + +* ``path``: Unicode HTTP path from URL, already urldecoded. + +* ``query_string``: Byte string URL portion after the ``?``. Optional, default + is empty string. + +* ``root_path``: Byte string that indicates the root path this application + is mounted at; same as ``SCRIPT_NAME`` in WSGI. Optional, defaults + to empty string. + +* ``headers``: List of ``[name, value]``, where ``name`` is the + header name as byte string and ``value`` is the header value as a byte + string. Order should be preserved from the original HTTP request; + duplicates are possible and must be preserved in the message as received. + Header names must be lowercased. + +* ``client``: List of ``[host, port]`` where ``host`` is a unicode string of the + remote host's IPv4 or IPv6 address, and ``port`` is the remote port as an + integer. Optional, defaults to ``None``. + +* ``server``: List of ``[host, port]`` where ``host`` is the listening address + for this server as a unicode string, and ``port`` is the integer listening port. + Optional, defaults to ``None``. + +* ``order``: The integer value ``0``. + + +Receive +''''''' + +Sent when a data frame is received from the client. If ``ChannelFull`` is +raised, you may retry sending it but if it does not send the socket must +be closed with websocket error code 1013. + +Channel: ``websocket.receive`` + +Keys: + +* ``reply_channel``: Channel name for sending data, starting with ``websocket.send!`` + +* ``path``: Path sent during ``connect``, sent to make routing easier for apps. + +* ``bytes``: Byte string of frame content, if it was bytes mode, or ``None``. + +* ``text``: Unicode string of frame content, if it was text mode, or ``None``. + +* ``order``: Order of this frame in the WebSocket stream, starting + at 1 (``connect`` is 0). + +One of ``bytes`` or ``text`` must be non-``None``. + + +Disconnection +''''''''''''' + +Sent when either connection to the client is lost, either from the client +closing the connection, the server closing the connection, or loss of the +socket. + +If ``ChannelFull`` is raised, then give up attempting to send the message; +consumption is not required. + +Channel: ``websocket.disconnect`` + +Keys: + +* ``reply_channel``: Channel name that was used for sending data, starting + with ``websocket.send!``. Cannot be used to send at this point; provided + as a way to identify the connection only. + +* ``code``: The WebSocket close code (integer), as per the WebSocket spec. + +* ``path``: Path sent during ``connect``, sent to make routing easier for apps. + +* ``order``: Order of the disconnection relative to the incoming frames' + ``order`` values in ``websocket.receive``. + + +Send/Close/Accept +''''''''''''''''' + +Sends a data frame to the client and/or closes the connection from the +server end and/or accepts a connection. If ``ChannelFull`` is raised, wait +and try again. + +If received while the connection is waiting for acceptance after a ``connect`` +message: + +* If ``bytes`` or ``text`` is present, accept the connection and send the data. +* If ``accept`` is ``True``, accept the connection and do nothing else. +* If ``close`` is ``True`` or a positive integer, reject the connection. If + ``bytes`` or ``text`` is also set, it should accept the connection, send the + frame, then immediately close the connection. + +If received while the connection is established: + +* If ``bytes`` or ``text`` is present, send the data. +* If ``close`` is ``True`` or a positive integer, close the connection after + any send. +* ``accept`` is ignored. + +Channel: ``websocket.send!`` + +Keys: + +* ``bytes``: Byte string of frame content, if in bytes mode, or ``None``. + +* ``text``: Unicode string of frame content, if in text mode, or ``None``. + +* ``close``: Boolean indicating if the connection should be closed after + data is sent, if any. Alternatively, a positive integer specifying the + response code. The response code will be 1000 if you pass ``True``. + Optional, default ``False``. + +* ``accept``: Boolean saying if the connection should be accepted without + sending a frame if it is in the handshake phase. + +A maximum of one of ``bytes`` or ``text`` may be provided. If both are +provided, the protocol server should ignore the message entirely. + + +WSGI Compatibility +------------------ + +Part of the design of the HTTP portion of this spec is to make sure it +aligns well with the WSGI specification, to ensure easy adaptability +between both specifications and the ability to keep using WSGI servers or +applications with ASGI. + +The adaptability works in two ways: + +* WSGI Server to ASGI: A WSGI application can be written that transforms + ``environ`` into a Request message, sends it off on the ``http.request`` + channel, and then waits on a generated response channel for a Response + message. This has the disadvantage of tying up an entire WSGI thread + to poll one channel, but should not be a massive performance drop if + there is no backlog on the request channel, and would work fine for an + in-process adapter to run a pure-ASGI web application. + +* ASGI to WSGI application: A small wrapper process is needed that listens + on the ``http.request`` channel, and decodes incoming Request messages + into an ``environ`` dict that matches the WSGI specs, while passing in + a ``start_response`` that stores the values for sending with the first + content chunk. Then, the application iterates over the WSGI app, + packaging each returned content chunk into a Response or Response Chunk + message (if more than one is yielded). + +There is an almost direct mapping for the various special keys in +WSGI's ``environ`` variable to the Request message: + +* ``REQUEST_METHOD`` is the ``method`` key +* ``SCRIPT_NAME`` is ``root_path`` +* ``PATH_INFO`` can be derived from ``path`` and ``root_path`` +* ``QUERY_STRING`` is ``query_string`` +* ``CONTENT_TYPE`` can be extracted from ``headers`` +* ``CONTENT_LENGTH`` can be extracted from ``headers`` +* ``SERVER_NAME`` and ``SERVER_PORT`` are in ``server`` +* ``REMOTE_HOST``/``REMOTE_ADDR`` and ``REMOTE_PORT`` are in ``client`` +* ``SERVER_PROTOCOL`` is encoded in ``http_version`` +* ``wsgi.url_scheme`` is ``scheme`` +* ``wsgi.input`` is a StringIO around ``body`` +* ``wsgi.errors`` is directed by the wrapper as needed + +The ``start_response`` callable maps similarly to Response: + +* The ``status`` argument becomes ``status``, with the reason phrase dropped. +* ``response_headers`` maps to ``headers`` + +It may even be possible to map Request Body Chunks in a way that allows +streaming of body data, though it would likely be easier and sufficient for +many applications to simply buffer the whole body into memory before calling +the WSGI application. + + +TODOs +----- + +* Maybe remove ``http_version`` and replace with ``supports_server_push``? From 1f2538f5b87d65a7e7f572d0f4ceaaeaf041b037 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 19 Mar 2017 12:26:20 -0700 Subject: [PATCH 651/746] Few more spec clarifications --- docs/asgi.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 9b13d9c..8a13c76 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -36,16 +36,16 @@ an abstraction that allows for data to be sent and received at any time, and from different application threads or processes. It also take the principle of turning protocols into Python-compatible, -asynchronous-friendly sets of messages and generalises it into two sections; +asynchronous-friendly sets of messages and generalises it into two parts; a standardised interface for communication and to build servers around (this -document), and a set of standard message formats for each protocol (the -sub-specifications, linked above). +document), and a set of standard :ref:`message formats for each protocol `. -Its primary design is for HTTP, however, and part of this design is -ensuring there is an easy path to use both -existing WSGI servers and applications, as a large majority of Python -web usage relies on WSGI and providing an easy path forwards is critical -to adoption. +Its primary goal is to provide a way to write HTTP/2 and WebSocket code, +alongside normal HTTP handling code, however, and part of this design is +ensuring there is an easy path to use both existing WSGI servers and +applications, as a large majority of Python web usage relies on WSGI and +providing an easy path forwards is critical to adoption. Details on that +interoperability are covered in :doc:`/asgi/www`. The end result of this process has been a specification for generalised inter-process communication between Python processes, with a certain set of @@ -55,6 +55,7 @@ task queues, but it is intended that it could be used for things like distributed systems communication, or as the backbone of a service-oriented architecure for inter-service communication. + Overview ======== @@ -104,7 +105,8 @@ contain only the following types to ensure serializability: * Byte strings * Unicode strings -* Integers (no longs) +* Integers (within the signed 64 bit range) +* Floating point numbers (within the IEEE 754 double precision range) * Lists (tuples should be treated as lists) * Dicts (keys must be unicode strings) * Booleans From 526ad65e73d50cb33dbfe1e0ec60ec1cfd912410 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 19 Mar 2017 12:32:43 -0700 Subject: [PATCH 652/746] A couple more spec tweaks --- docs/asgi.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 8a13c76..9e302e8 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -205,6 +205,8 @@ network based on attributes in the message (in this case, a telephone number). +.. _asgi_extensions: + Extensions ---------- @@ -338,7 +340,7 @@ A *channel layer* must provide an object with these attributes and returns a new valid channel name that does not already exist, by adding a unicode string after the ``!`` or ``?`` character in ``pattern``, and checking for existence of that name in the channel layer. The ``pattern`` - MUST end with ``!`` or ``?`` or this function must error. If the character + must end with ``!`` or ``?`` or this function must error. If the character is ``!``, making it a process-specific channel, ``new_channel`` must be called on the same channel layer that intends to read the channel with ``receive``; any other channel layer instance may not receive @@ -351,9 +353,8 @@ A *channel layer* must provide an object with these attributes because the destination channel is over capacity. * ``extensions``, a list of unicode string names indicating which - extensions this layer provides, or empty if it supports none. - The names defined in this document are ``groups``, ``flush`` and - ``statistics``. + extensions this layer provides, or an empty list if it supports none. + The possible extensions can be seen in :ref:`asgi_extensions`. A channel layer implementing the ``groups`` extension must also provide: From 08ff57ac9b1d2599e071b62579c4ec64d1b08691 Mon Sep 17 00:00:00 2001 From: Coread Date: Wed, 22 Mar 2017 22:58:35 +0000 Subject: [PATCH 653/746] Channel and http session from http (#564) * Add a new auth decorator to get the user and rehydrate the http session * Add http_user_and_session, taking precedence over http_user, applying the channel_and_http_session_user_from_http decorator (a superset of http user functionality) * Only set session cookies on the first send, since subsequent real requests don't have access to HTTP information * Add a test for new http_user_and_session WebsocketConsumer attribute * Fix isort check --- channels/auth.py | 18 +++++++++++++++++- channels/generic/websockets.py | 9 ++++++--- channels/test/http.py | 4 +++- tests/test_generic.py | 24 ++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/channels/auth.py b/channels/auth.py index f0a0f75..606e2b8 100644 --- a/channels/auth.py +++ b/channels/auth.py @@ -2,7 +2,7 @@ import functools from django.contrib import auth -from .sessions import channel_session, http_session +from .sessions import channel_and_http_session, channel_session, http_session def transfer_user(from_session, to_session): @@ -88,3 +88,19 @@ def channel_session_user_from_http(func): transfer_user(message.http_session, message.channel_session) return func(message, *args, **kwargs) return inner + + +def channel_and_http_session_user_from_http(func): + """ + Decorator that automatically transfers the user from HTTP sessions to + channel-based sessions, rehydrates the HTTP session, and returns the + user as message.user as well. + """ + @http_session_user + @channel_and_http_session + @functools.wraps(func) + def inner(message, *args, **kwargs): + if message.http_session is not None: + transfer_user(message.http_session, message.channel_session) + return func(message, *args, **kwargs) + return inner diff --git a/channels/generic/websockets.py b/channels/generic/websockets.py index f4bb714..ab35d31 100644 --- a/channels/generic/websockets.py +++ b/channels/generic/websockets.py @@ -1,6 +1,6 @@ from django.core.serializers.json import DjangoJSONEncoder, json -from ..auth import channel_session_user_from_http +from ..auth import channel_and_http_session_user_from_http, channel_session_user_from_http from ..channel import Group from ..exceptions import SendNotAvailableOnDemultiplexer from ..sessions import enforce_ordering @@ -23,6 +23,7 @@ class WebsocketConsumer(BaseConsumer): # Turning this on passes the user over from the HTTP session on connect, # implies channel_session_user http_user = False + http_user_and_session = False # Set to True if you want the class to enforce ordering for you strict_ordering = False @@ -35,13 +36,15 @@ class WebsocketConsumer(BaseConsumer): adds the ordering decorator. """ # HTTP user implies channel session user - if self.http_user: + if self.http_user or self.http_user_and_session: self.channel_session_user = True # Get super-handler self.path = message['path'] handler = super(WebsocketConsumer, self).get_handler(message, **kwargs) # Optionally apply HTTP transfer - if self.http_user: + if self.http_user_and_session: + handler = channel_and_http_session_user_from_http(handler) + elif self.http_user: handler = channel_session_user_from_http(handler) # Ordering decorators if self.strict_ordering: diff --git a/channels/test/http.py b/channels/test/http.py index 7faebff..d9b439d 100644 --- a/channels/test/http.py +++ b/channels/test/http.py @@ -24,6 +24,7 @@ class HttpClient(Client): self._session = None self._headers = {} self._cookies = {} + self._session_cookie = True def set_cookie(self, key, value): """ @@ -42,7 +43,7 @@ class HttpClient(Client): def get_cookies(self): """Return cookies""" cookies = copy.copy(self._cookies) - if apps.is_installed('django.contrib.sessions'): + if self._session_cookie and apps.is_installed('django.contrib.sessions'): cookies[settings.SESSION_COOKIE_NAME] = self.session.session_key return cookies @@ -86,6 +87,7 @@ class HttpClient(Client): else: content['text'] = text self.channel_layer.send(to, content) + self._session_cookie = False def send_and_consume(self, channel, content={}, text=None, path='/', fail_on_none=True, check_accept=True): """ diff --git a/tests/test_generic.py b/tests/test_generic.py index 6b0c83a..dd5159e 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import json from django.test import override_settings +from django.contrib.auth import get_user_model from channels import route_class from channels.exceptions import SendNotAvailableOnDemultiplexer @@ -87,6 +88,29 @@ class GenericTests(ChannelTestCase): self.assertEqual(client.consume('websocket.connect').order, 0) self.assertEqual(client.consume('websocket.connect').order, 1) + def test_websockets_http_session_and_channel_session(self): + + class WebsocketConsumer(websockets.WebsocketConsumer): + http_user_and_session = True + + user_model = get_user_model() + user = user_model.objects.create_user(username='test', email='test@test.com', password='123456') + + client = HttpClient() + client.force_login(user) + with apply_routes([route_class(WebsocketConsumer, path='/path')]): + connect = client.send_and_consume('websocket.connect', {'path': '/path'}) + receive = client.send_and_consume('websocket.receive', {'path': '/path'}, text={'key': 'value'}) + disconnect = client.send_and_consume('websocket.disconnect', {'path': '/path'}) + self.assertEqual( + connect.message.http_session.session_key, + receive.message.http_session.session_key + ) + self.assertEqual( + connect.message.http_session.session_key, + disconnect.message.http_session.session_key + ) + def test_simple_as_route_method(self): class WebsocketConsumer(websockets.WebsocketConsumer): From 9f7fb7b80d0870a2a87749ce4ac73f26a89187bf Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Thu, 23 Mar 2017 11:49:17 -0500 Subject: [PATCH 654/746] remove spurious markdown formatting (#566) --- docs/javascript.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/javascript.rst b/docs/javascript.rst index b99765f..5009d88 100644 --- a/docs/javascript.rst +++ b/docs/javascript.rst @@ -30,11 +30,8 @@ To process messages:: To send messages, use the `send` method:: - ``` webSocketBridge.send({prop1: 'value1', prop2: 'value1'}); - ``` - To demultiplex specific streams:: webSocketBridge.connect(); From 4323a64e3382cf4f8aa8b7601d99b78640e5831b Mon Sep 17 00:00:00 2001 From: Erwin Junge Date: Thu, 23 Mar 2017 14:14:15 -0700 Subject: [PATCH 655/746] Add python 3.6 checks (#571) * Add python 3.6 checking to tox * Add python 3.6 checking to travis --- .travis.yml | 1 + tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 581d0a8..c4d30ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "2.7" - "3.4" - "3.5" + - "3.6" env: - DJANGO="Django>=1.8,<1.9" diff --git a/tox.ini b/tox.ini index b1c3ce5..8caea5c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = - py{27,34,35}-django-{18,19,110} - py{27,35}-flake8 + py{27,34,35,36}-django-{18,19,110} + py{27,35,36}-flake8 isort [testenv] From 10398780a3e0848e63041a600034183eb9e7f1b9 Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Thu, 23 Mar 2017 18:30:28 -0500 Subject: [PATCH 656/746] Allow relative url in `WebsocketBridge.connect()` (#567) * Add support for relative urls in javascript wrapper * recompile static file * add js smoke test for relative urls * update docs to show relative urls --- channels/static/channels/js/websocketbridge.js | 14 ++++++++++---- docs/javascript.rst | 4 ++-- js_client/src/index.js | 14 ++++++++++---- js_client/tests/websocketbridge.test.js | 5 +++++ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/channels/static/channels/js/websocketbridge.js b/channels/static/channels/js/websocketbridge.js index 5c647e6..17382fe 100644 --- a/channels/static/channels/js/websocketbridge.js +++ b/channels/static/channels/js/websocketbridge.js @@ -270,12 +270,18 @@ var WebSocketBridge = function () { key: 'connect', value: function connect(url, protocols, options) { var _url = void 0; + // Use wss:// if running on https:// + var scheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; + var base_url = scheme + '://' + window.location.host; if (url === undefined) { - // Use wss:// if running on https:// - var scheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; - _url = scheme + '://' + window.location.host + '/ws'; + _url = base_url; } else { - _url = url; + // Support relative URLs + if (url[0] == '/') { + _url = '' + base_url + url; + } else { + _url = url; + } } this._socket = new _reconnectingWebsocket2.default(_url, protocols, options); } diff --git a/docs/javascript.rst b/docs/javascript.rst index 5009d88..540047b 100644 --- a/docs/javascript.rst +++ b/docs/javascript.rst @@ -23,7 +23,7 @@ specific integration on top of it. To process messages:: const webSocketBridge = new channels.WebSocketBridge(); - webSocketBridge.connect(); + webSocketBridge.connect('/ws/'); webSocketBridge.listen(function(action, stream) { console.log(action, stream); }); @@ -35,7 +35,7 @@ To send messages, use the `send` method:: To demultiplex specific streams:: webSocketBridge.connect(); - webSocketBridge.listen(); + webSocketBridge.listen('/ws/'); webSocketBridge.demultiplex('mystream', function(action, stream) { console.log(action, stream); }); diff --git a/js_client/src/index.js b/js_client/src/index.js index 6a5ba3e..a0b7eeb 100644 --- a/js_client/src/index.js +++ b/js_client/src/index.js @@ -36,12 +36,18 @@ export class WebSocketBridge { */ connect(url, protocols, options) { let _url; + // Use wss:// if running on https:// + const scheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; + const base_url = `${scheme}://${window.location.host}`; if (url === undefined) { - // Use wss:// if running on https:// - const scheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; - _url = `${scheme}://${window.location.host}/ws`; + _url = base_url; } else { - _url = url; + // Support relative URLs + if (url[0] == '/') { + _url = `${base_url}${url}`; + } else { + _url = url; + } } this._socket = new ReconnectingWebSocket(_url, protocols, options); } diff --git a/js_client/tests/websocketbridge.test.js b/js_client/tests/websocketbridge.test.js index 4ed74ea..b575629 100644 --- a/js_client/tests/websocketbridge.test.js +++ b/js_client/tests/websocketbridge.test.js @@ -16,6 +16,11 @@ describe('WebSocketBridge', () => { const webSocketBridge = new WebSocketBridge(); webSocketBridge.connect('ws://localhost'); }); + it('Supports relative urls', () => { + const webSocketBridge = new WebSocketBridge(); + webSocketBridge.connect('/somepath/'); + }); + it('Processes messages', () => { const webSocketBridge = new WebSocketBridge(); const myMock = jest.fn(); From 613153cbc6312a80003d6d9e43b4550a56fefd06 Mon Sep 17 00:00:00 2001 From: Erwin Junge Date: Thu, 23 Mar 2017 16:32:51 -0700 Subject: [PATCH 657/746] Allow runworker to be used without staticfiles (#569) * Allow runworker to be used without staticfiles * Split too long line * Reset binding_classes at start of runworker tests --- channels/management/commands/runworker.py | 3 +- tests/test_management.py | 40 ++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/channels/management/commands/runworker.py b/channels/management/commands/runworker.py index 580f031..e8366e6 100644 --- a/channels/management/commands/runworker.py +++ b/channels/management/commands/runworker.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +from django.apps import apps from django.conf import settings from django.core.management import BaseCommand, CommandError @@ -48,7 +49,7 @@ class Command(BaseCommand): ) # Check a handler is registered for http reqs # Serve static files if Django in debug mode - if settings.DEBUG: + if settings.DEBUG and apps.is_installed('django.contrib.staticfiles'): self.channel_layer.router.check_default(http_consumer=StaticFilesConsumer()) else: self.channel_layer.router.check_default() diff --git a/tests/test_management.py b/tests/test_management.py index 1242088..89b0bc1 100644 --- a/tests/test_management.py +++ b/tests/test_management.py @@ -7,6 +7,9 @@ from django.core.management import CommandError, call_command from django.test import TestCase, mock from six import StringIO +from channels.asgi import channel_layers, ChannelLayerWrapper +from channels.binding.base import BindingMetaclass +from channels.handler import ViewConsumer from channels.management.commands import runserver from channels.staticfiles import StaticFilesConsumer @@ -25,6 +28,18 @@ class RunWorkerTests(TestCase): import channels.log self.stream = StringIO() channels.log.handler = logging.StreamHandler(self.stream) + BindingMetaclass.binding_classes = [] + self._old_layer = channel_layers.set( + 'fake_channel', + ChannelLayerWrapper( + FakeChannelLayer(), + 'fake_channel', + channel_layers['fake_channel'].routing[:], + ) + ) + + def tearDown(self): + channel_layers.set('fake_channel', self._old_layer) def test_runworker_no_local_only(self, mock_worker): """ @@ -37,7 +52,11 @@ class RunWorkerTests(TestCase): """ Test that the StaticFilesConsumer is used in debug mode. """ - with self.settings(DEBUG=True, STATIC_URL='/static/'): + with self.settings( + DEBUG=True, + STATIC_URL='/static/', + INSTALLED_APPS=['channels', 'django.contrib.staticfiles'], + ): # Use 'fake_channel' that bypasses the 'inmemory' check call_command('runworker', '--layer', 'fake_channel') mock_worker.assert_called_with( @@ -51,6 +70,25 @@ class RunWorkerTests(TestCase): static_consumer = channel_layer.router.root.routing[0].consumer self.assertIsInstance(static_consumer, StaticFilesConsumer) + def test_debug_without_staticfiles(self, mock_worker): + """ + Test that the StaticFilesConsumer is not used in debug mode when staticfiles app is not configured. + """ + with self.settings(DEBUG=True, STATIC_URL=None, INSTALLED_APPS=['channels']): + # Use 'fake_channel' that bypasses the 'inmemory' check + call_command('runworker', '--layer', 'fake_channel') + mock_worker.assert_called_with( + only_channels=None, + exclude_channels=None, + callback=None, + channel_layer=mock.ANY, + ) + + channel_layer = mock_worker.call_args[1]['channel_layer'] + static_consumer = channel_layer.router.root.routing[0].consumer + self.assertNotIsInstance(static_consumer, StaticFilesConsumer) + self.assertIsInstance(static_consumer, ViewConsumer) + def test_runworker(self, mock_worker): # Use 'fake_channel' that bypasses the 'inmemory' check call_command('runworker', '--layer', 'fake_channel') From 0f0c74daaa04baede5f318db68db577cea207b4c Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Fri, 24 Mar 2017 17:33:55 -0500 Subject: [PATCH 658/746] Expose `WebSocketBridge.socket` (#572) * made `WebSocketBridge._socket` public We can provide a convenient way to hook custom behavior (such as debugging) by making the underlying `ReconnectingWebSocket` instance publicly available. Also removed the undocumented `onopen` constructor option. * recompile js --- .../static/channels/js/websocketbridge.js | 24 +++++++++---------- docs/javascript.rst | 7 ++++++ js_client/README.md | 13 +++++++--- js_client/src/index.js | 23 +++++++++--------- js_client/tests/websocketbridge.test.js | 10 ++++++++ 5 files changed, 49 insertions(+), 28 deletions(-) diff --git a/channels/static/channels/js/websocketbridge.js b/channels/static/channels/js/websocketbridge.js index 17382fe..d7b2845 100644 --- a/channels/static/channels/js/websocketbridge.js +++ b/channels/static/channels/js/websocketbridge.js @@ -228,8 +228,6 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } -var noop = function noop() {}; - /** * Bridge between Channels and plain javascript. * @@ -240,17 +238,19 @@ var noop = function noop() {}; * console.log(action, stream); * }); */ - var WebSocketBridge = function () { function WebSocketBridge(options) { _classCallCheck(this, WebSocketBridge); - this._socket = null; + /** + * The underlaying `ReconnectingWebSocket` instance. + * + * @type {ReconnectingWebSocket} + */ + this.socket = null; this.streams = {}; this.default_cb = null; - this.options = _extends({}, { - onopen: noop - }, options); + this.options = _extends({}, options); } /** @@ -283,7 +283,7 @@ var WebSocketBridge = function () { _url = url; } } - this._socket = new _reconnectingWebsocket2.default(_url, protocols, options); + this.socket = new _reconnectingWebsocket2.default(_url, protocols, options); } /** @@ -306,7 +306,7 @@ var WebSocketBridge = function () { var _this = this; this.default_cb = cb; - this._socket.onmessage = function (event) { + this.socket.onmessage = function (event) { var msg = JSON.parse(event.data); var action = void 0; var stream = void 0; @@ -322,8 +322,6 @@ var WebSocketBridge = function () { _this.default_cb ? _this.default_cb(action, stream) : null; } }; - - this._socket.onopen = this.options.onopen; } /** @@ -363,7 +361,7 @@ var WebSocketBridge = function () { }, { key: 'send', value: function send(msg) { - this._socket.send(JSON.stringify(msg)); + this.socket.send(JSON.stringify(msg)); } /** @@ -386,7 +384,7 @@ var WebSocketBridge = function () { stream: _stream, payload: action }; - _this2._socket.send(JSON.stringify(msg)); + _this2.socket.send(JSON.stringify(msg)); } }; } diff --git a/docs/javascript.rst b/docs/javascript.rst index 540047b..b7e2f67 100644 --- a/docs/javascript.rst +++ b/docs/javascript.rst @@ -47,5 +47,12 @@ To send a message to a specific stream:: webSocketBridge.stream('mystream').send({prop1: 'value1', prop2: 'value1'}) +The `WebSocketBridge` instance exposes the underlaying `ReconnectingWebSocket` as the `socket` property. You can use this property to add any custom behavior. For example:: + + webSocketBridge.socket.addEventListener('open', function() { + console.log("Connected to WebSocket"); + }) + + The library is also available as a npm module, under the name `django-channels `_ diff --git a/js_client/README.md b/js_client/README.md index 506cd49..5a97685 100644 --- a/js_client/README.md +++ b/js_client/README.md @@ -8,7 +8,7 @@ To process messages: import { WebSocketBridge } from 'django-channels' const webSocketBridge = new WebSocketBridge(); -webSocketBridge.connect(); +webSocketBridge.connect('/ws/'); webSocketBridge.listen(function(action, stream) { console.log(action, stream); }); @@ -18,14 +18,13 @@ To send messages: ``` webSocketBridge.send({prop1: 'value1', prop2: 'value1'}); - ``` To demultiplex specific streams: ``` const webSocketBridge = new WebSocketBridge(); -webSocketBridge.connect(); +webSocketBridge.connect('/ws/'); webSocketBridge.listen(); webSocketBridge.demultiplex('mystream', function(action, stream) { console.log(action, stream); @@ -40,3 +39,11 @@ To send a message to a specific stream: ``` webSocketBridge.stream('mystream').send({prop1: 'value1', prop2: 'value1'}) ``` + +The `WebSocketBridge` instance exposes the underlaying `ReconnectingWebSocket` as the `socket` property. You can use this property to add any custom behavior. For example: + +``` +webSocketBridge.socket.addEventListener('open', function() { + console.log("Connected to WebSocket"); +}) +``` diff --git a/js_client/src/index.js b/js_client/src/index.js index a0b7eeb..b2ebd81 100644 --- a/js_client/src/index.js +++ b/js_client/src/index.js @@ -1,8 +1,6 @@ import ReconnectingWebSocket from 'reconnecting-websocket'; -const noop = (...args) => {}; - /** * Bridge between Channels and plain javascript. * @@ -15,12 +13,15 @@ const noop = (...args) => {}; */ export class WebSocketBridge { constructor(options) { - this._socket = null; + /** + * The underlaying `ReconnectingWebSocket` instance. + * + * @type {ReconnectingWebSocket} + */ + this.socket = null; this.streams = {}; this.default_cb = null; - this.options = Object.assign({}, { - onopen: noop, - }, options); + this.options = {...options}; } /** @@ -49,7 +50,7 @@ export class WebSocketBridge { _url = url; } } - this._socket = new ReconnectingWebSocket(_url, protocols, options); + this.socket = new ReconnectingWebSocket(_url, protocols, options); } /** @@ -67,7 +68,7 @@ export class WebSocketBridge { */ listen(cb) { this.default_cb = cb; - this._socket.onmessage = (event) => { + this.socket.onmessage = (event) => { const msg = JSON.parse(event.data); let action; let stream; @@ -83,8 +84,6 @@ export class WebSocketBridge { this.default_cb ? this.default_cb(action, stream) : null; } }; - - this._socket.onopen = this.options.onopen; } /** @@ -119,7 +118,7 @@ export class WebSocketBridge { * webSocketBridge.send({prop1: 'value1', prop2: 'value1'}); */ send(msg) { - this._socket.send(JSON.stringify(msg)); + this.socket.send(JSON.stringify(msg)); } /** @@ -137,7 +136,7 @@ export class WebSocketBridge { stream, payload: action } - this._socket.send(JSON.stringify(msg)); + this.socket.send(JSON.stringify(msg)); } } } diff --git a/js_client/tests/websocketbridge.test.js b/js_client/tests/websocketbridge.test.js index b575629..0b347be 100644 --- a/js_client/tests/websocketbridge.test.js +++ b/js_client/tests/websocketbridge.test.js @@ -20,7 +20,17 @@ describe('WebSocketBridge', () => { const webSocketBridge = new WebSocketBridge(); webSocketBridge.connect('/somepath/'); }); + it('Can add event listeners to socket', () => { + const webSocketBridge = new WebSocketBridge(); + const myMock = jest.fn(); + webSocketBridge.connect('ws://localhost', {}); + webSocketBridge.socket.addEventListener('message', myMock); + mockServer.send('{"type": "test", "payload": "message 1"}'); + + expect(myMock.mock.calls.length).toBe(1); + + }); it('Processes messages', () => { const webSocketBridge = new WebSocketBridge(); const myMock = jest.fn(); From cf788a3d7d47b24b69e72f16e1a421ca61d9ec8c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 27 Mar 2017 09:54:13 -0700 Subject: [PATCH 659/746] Allow accept to be False --- docs/asgi/www.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/asgi/www.rst b/docs/asgi/www.rst index 76e3c60..a2d616b 100644 --- a/docs/asgi/www.rst +++ b/docs/asgi/www.rst @@ -356,6 +356,7 @@ message: * If ``bytes`` or ``text`` is present, accept the connection and send the data. * If ``accept`` is ``True``, accept the connection and do nothing else. +* If ``accept`` is ``False``, reject the connection (with close code 1000) and do nothing else. * If ``close`` is ``True`` or a positive integer, reject the connection. If ``bytes`` or ``text`` is also set, it should accept the connection, send the frame, then immediately close the connection. From a9062e5d28d93f2d45c13cd605c7e458503b100e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 27 Mar 2017 09:59:42 -0700 Subject: [PATCH 660/746] Further clarify accept flow --- docs/asgi/www.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/asgi/www.rst b/docs/asgi/www.rst index a2d616b..d5bd5ca 100644 --- a/docs/asgi/www.rst +++ b/docs/asgi/www.rst @@ -355,11 +355,12 @@ If received while the connection is waiting for acceptance after a ``connect`` message: * If ``bytes`` or ``text`` is present, accept the connection and send the data. -* If ``accept`` is ``True``, accept the connection and do nothing else. -* If ``accept`` is ``False``, reject the connection (with close code 1000) and do nothing else. +* If ``accept`` is ``True``, accept the connection (and send any data provided). +* If ``accept`` is ``False``, reject the connection and do nothing else. * If ``close`` is ``True`` or a positive integer, reject the connection. If ``bytes`` or ``text`` is also set, it should accept the connection, send the - frame, then immediately close the connection. + frame, then immediately close the connection. Note that any close code integer + sent is ignored, as connections are rejected with HTTP's ``403 Forbidden``. If received while the connection is established: From 1c5074d128190d8b19732c9a1af4ceeedcea217b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 27 Mar 2017 10:06:38 -0700 Subject: [PATCH 661/746] Specs should be very precise. --- docs/asgi/www.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/asgi/www.rst b/docs/asgi/www.rst index d5bd5ca..0289f54 100644 --- a/docs/asgi/www.rst +++ b/docs/asgi/www.rst @@ -354,13 +354,16 @@ and try again. If received while the connection is waiting for acceptance after a ``connect`` message: -* If ``bytes`` or ``text`` is present, accept the connection and send the data. * If ``accept`` is ``True``, accept the connection (and send any data provided). * If ``accept`` is ``False``, reject the connection and do nothing else. + If ``bytes`` or ``text`` were also present they must be ignored. +* If ``bytes`` or ``text`` is present, accept the connection and send the data. * If ``close`` is ``True`` or a positive integer, reject the connection. If ``bytes`` or ``text`` is also set, it should accept the connection, send the frame, then immediately close the connection. Note that any close code integer - sent is ignored, as connections are rejected with HTTP's ``403 Forbidden``. + sent is ignored, as connections are rejected with HTTP's ``403 Forbidden``, + unless data is also sent, in which case a full WebSocket close is done with + the provided code. If received while the connection is established: From df22aa792c30bc10bbaf2f9d4621eea6a0212f8d Mon Sep 17 00:00:00 2001 From: Matt Magin Date: Tue, 28 Mar 2017 05:13:14 +1030 Subject: [PATCH 662/746] Fixed missing keyword argument in testing documentation (#574) --- docs/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing.rst b/docs/testing.rst index 1ac48bd..04c6144 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -157,7 +157,7 @@ For example:: username='test', email='test@test.com', password='123456') client.login(username='test', password='123456') - client.send_and_consume('websocket.connect', '/rooms/') + client.send_and_consume('websocket.connect', path='/rooms/') # check that there is nothing to receive self.assertIsNone(client.receive()) From bbff9281522c565846ca0785c5a583843fae4a61 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 27 Mar 2017 19:04:58 -0700 Subject: [PATCH 663/746] Update ASGI spec with new process-local channels definition --- docs/asgi.rst | 25 +++++++++++-------------- docs/asgi/www.rst | 8 ++++---- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 9e302e8..893ef76 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -127,7 +127,7 @@ is made between channels that have multiple readers (such as the ``http.request`` channel that web applications would listen on from every application worker process), *single-reader channels* that are read from a single unknown location (such as ``http.request.body?ABCDEF``), and -*process-specific channels* (such as a ``http.response!ABCDEF`` channel +*process-specific channels* (such as a ``http.response.A1B2C3!D4E5F6`` channel tied to a client socket). *Normal channel* names contain no type characters, and can be routed however @@ -146,19 +146,16 @@ channel to a server, but reads from this channel by a single process must always be in-order and return messages if the channel is non-empty. These names must be generated by the ``new_channel`` call. -*Process-specific channel* names contain an exclamation mark -(``!``) character in order to indicate to the channel layer that it may -have to route the data for these channels differently to ensure it reaches the -single process that needs it; these channels are nearly always tied to -incoming connections from the outside world. The ``!`` is always preceded by -the main channel name (e.g. ``http.response``) and followed by the -per-client/random portion - channel layers can split on the ``!`` and use just -the right hand part to route if they desire, or can ignore it if they don't -need to use different routing rules. Even if the right hand side contains -client routing information, it must still contain random parts too so that -each call to ``new_channel`` returns a new, unused name. These names -must be generated by the ``new_channel`` call; they are guaranteed to only -be read from the same process that calls ``new_channel``. +*Process-specific channel* names contain an exclamation mark (``!``) that +separates a remote and local part. These channels are received differently; +only the name up to and including the ``!`` character is passed to the +``receive()`` call, and it will receive any message on any channel with that +prefix. This allows a process, such as a HTTP terminator, to listen on a single +process-specific channel, and then distribute incoming requests to the +appropriate client sockets using the local part (the part after the ``!``). +The local parts must be generated and managed by the process that consumes them. +These channels, like single-reader channels, are guaranteed to give any extant +messages in order if received from a single process. Messages should expire after a set time sitting unread in a channel; the recommendation is one minute, though the best value depends on the diff --git a/docs/asgi/www.rst b/docs/asgi/www.rst index 0289f54..d81fcde 100644 --- a/docs/asgi/www.rst +++ b/docs/asgi/www.rst @@ -128,7 +128,7 @@ Send after any server pushes, and before any response chunks. If ``ChannelFull`` is encountered, wait and try again later, optionally giving up after a predetermined timeout. -Channel: ``http.response!`` +Channel: Defined by server, suggested ``http.response.RANDOMPART!CLIENTID`` Keys: @@ -154,7 +154,7 @@ Response Chunk Must be sent after an initial Response. If ``ChannelFull`` is encountered, wait and try again later. -Channel: ``http.response!`` +Channel: Defined by server, suggested ``http.response.RANDOMPART!CLIENTID`` Keys: @@ -194,7 +194,7 @@ If the remote peer does not support server push, either because it's not a HTTP/2 peer or because SETTINGS_ENABLE_PUSH is set to 0, the server must do nothing in response to this message. -Channel: ``http.response!`` +Channel: Defined by server, suggested ``http.response.RANDOMPART!CLIENTID`` Keys: @@ -372,7 +372,7 @@ If received while the connection is established: any send. * ``accept`` is ignored. -Channel: ``websocket.send!`` +Channel: Defined by server, suggested ``websocket.send.RANDOMPART!CLIENTID`` Keys: From 294e60504e50c01ebd78f2d3e37cbc438997576b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 28 Mar 2017 10:08:23 -0700 Subject: [PATCH 664/746] Add capacity note about process-local channels. --- docs/asgi.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/asgi.rst b/docs/asgi.rst index 893ef76..1b00642 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -309,6 +309,10 @@ for example, a web application trying to send a response body will likely wait until it empties out again, while a HTTP interface server trying to send in a request would drop the request and return a 503 error. +Process-local channels must apply their capacity on the non-local part (that is, +up to and including the ``!`` character), and so capacity is shared among all +of the "virtual" channels inside it. + Sending to a group never raises ChannelFull; instead, it must silently drop the message if it is over capacity, as per ASGI's at-most-once delivery policy. From 476d1500c6b0673863a995761c68c01b382139d2 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 28 Mar 2017 11:38:05 -0700 Subject: [PATCH 665/746] Switch from MD5 to SHA-1 to work under FIPS-140-2 --- channels/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/sessions.py b/channels/sessions.py index 3af6da9..3e43f80 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -18,7 +18,7 @@ def session_for_reply_channel(reply_channel): """ # We hash the whole reply channel name and add a prefix, to fit inside 32B reply_name = reply_channel - hashed = hashlib.md5(reply_name.encode("utf8")).hexdigest() + hashed = hashlib.sha1(reply_name.encode("utf8")).hexdigest() session_key = "chn" + hashed[:29] # Make a session storage session_engine = import_module(getattr(settings, "CHANNEL_SESSION_ENGINE", settings.SESSION_ENGINE)) From 28ace41edf816bfd7fc2f85dc4988c87729a1a01 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 28 Mar 2017 13:37:33 -0700 Subject: [PATCH 666/746] Fixed #577: Use scheme key from ASGI request message --- channels/handler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/channels/handler.py b/channels/handler.py index 6fc49f5..cb1d2e6 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -148,6 +148,9 @@ class AsgiRequest(http.HttpRequest): def GET(self): return http.QueryDict(self.message.get('query_string', '')) + def _get_scheme(self): + return self.message.get("scheme", "http") + def _get_post(self): if not hasattr(self, '_post'): self._read_started = False From 38bc238788b61f05b6515738c350c02a8075db9a Mon Sep 17 00:00:00 2001 From: Mike Barkas Date: Tue, 28 Mar 2017 17:00:26 -0400 Subject: [PATCH 667/746] Issue-443: Adding session decorators to reference documentation. (#578) --- docs/reference.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/reference.rst b/docs/reference.rst index aaa99e2..3a1f7e7 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -165,3 +165,38 @@ directly, but there are two useful ways you can call it: * ``encode_response(response)`` is a classmethod that can be called with a Django ``HttpResponse`` and will yield one or more ASGI messages that are the encoded response. + + +.. _ref-decorators: + +Decorators +---------- + +Channels provides decorators to assist with persisting data. + +* ``channel_session``: Provides a session-like object called "channel_session" to consumers + as a message attribute that will auto-persist across consumers with + the same incoming "reply_channel" value. + + Use this to persist data across the lifetime of a connection. + +* ``http_session``: Wraps a HTTP or WebSocket connect consumer (or any consumer of messages + that provides a "cookies" or "get" attribute) to provide a "http_session" + attribute that behaves like request.session; that is, it's hung off of + a per-user session key that is saved in a cookie or passed as the + "session_key" GET parameter. + + It won't automatically create and set a session cookie for users who + don't have one - that's what SessionMiddleware is for, this is a simpler + read-only version for more low-level code. + + If a message does not have a session we can inflate, the "session" attribute + will be None, rather than an empty session you can write to. + + Does not allow a new session to be set; that must be done via a view. This + is only an accessor for any existing session. + +* ``channel_and_http_session``: Enables both the channel_session and http_session. + + Stores the http session key in the channel_session on websocket.connect messages. + It will then hydrate the http_session from that same key on subsequent messages. From de65a41b54b5f6181c570751b08bd48464cb624a Mon Sep 17 00:00:00 2001 From: Mike Barkas Date: Tue, 28 Mar 2017 17:00:53 -0400 Subject: [PATCH 668/746] Issue-579: Fix sentence grammar in documentation. (#580) --- docs/testing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/testing.rst b/docs/testing.rst index 04c6144..bc4a082 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -191,10 +191,10 @@ so if you need to pass decoding use ``receive(json=False)``, like in the example Applying routes --------------- -When you need to testing you consumers without routes in settings or you -want to testing your consumers in more isolate and atomic way, it will be +When you need to test your consumers without routes in settings or you +want to test your consumers in a more isolate and atomic way, it will be simpler with ``apply_routes`` contextmanager and decorator for your ``ChannelTestCase``. -It takes list of routes that you want to use and overwrite existing routes:: +It takes a list of routes that you want to use and overwrites existing routes:: from channels.test import ChannelTestCase, HttpClient, apply_routes From ba54268c19f00cfe015a6b47d308c4398e7c7486 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sat, 1 Apr 2017 15:33:15 +0100 Subject: [PATCH 669/746] Releasing 1.1.2 --- CHANGELOG.txt | 12 ++++++++++++ channels/__init__.py | 2 +- docs/releases/1.1.2.rst | 29 +++++++++++++++++++++++++++++ docs/releases/index.rst | 1 + js_client/package.json | 2 +- setup.py | 4 ++-- 6 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 docs/releases/1.1.2.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index ab6e47f..391ff5e 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,18 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases +1.1.2 (2017-04-01) +------------------ + +* Session name hash changed to SHA-1 to satisfy FIPS-140-2. Due to this, + please force all WebSockets to reconnect after the upgrade. + +* `scheme` key in ASGI-HTTP messages now translates into `request.is_secure()` + correctly. + +* WebsocketBridge now exposes the underlying WebSocket as `.socket` + + 1.1.1 (2017-03-19) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index e79f6b4..9ad4b64 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.1" +__version__ = "1.1.2" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/releases/1.1.2.rst b/docs/releases/1.1.2.rst new file mode 100644 index 0000000..8e13b50 --- /dev/null +++ b/docs/releases/1.1.2.rst @@ -0,0 +1,29 @@ +1.1.2 Release Notes +=================== + +Channels 1.1.2 is a bugfix release for the 1.1 series, released on +April 1st, 2017. + + +Major Changes +------------- + +None. + + +Minor Changes & Bugfixes +------------------------ + +* Session name hash changed to SHA-1 to satisfy FIPS-140-2. + +* `scheme` key in ASGI-HTTP messages now translates into `request.is_secure()` + correctly. + +* WebsocketBridge now exposes the underlying WebSocket as `.socket`. + + +Backwards Incompatible Changes +------------------------------ + +* When you upgrade all current channel sessions will be invalidated; you + should make sure you disconnect all WebSockets during upgrade. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 736c75a..c36e3b7 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -10,3 +10,4 @@ Release Notes 1.0.3 1.1.0 1.1.1 + 1.1.2 diff --git a/js_client/package.json b/js_client/package.json index af9869f..5b8a92a 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -1,6 +1,6 @@ { "name": "django-channels", - "version": "1.1.1", + "version": "1.1.2", "description": "", "repository": { "type": "git", diff --git a/setup.py b/setup.py index 05a45b2..9791a96 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,8 @@ setup( include_package_data=True, install_requires=[ 'Django>=1.8', - 'asgiref>=1.0.1', - 'daphne>=1.0.0', + 'asgiref~=1.1', + 'daphne>=1.2.0', ], extras_require={ 'tests': [ From 627b97c317a6c3ceafa1c8f1192633d1c85c93ff Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 3 Apr 2017 13:01:20 +0200 Subject: [PATCH 670/746] Fixed #588: enforce_ordering failed to wait on process-specific chans --- channels/sessions.py | 13 +++++++++++-- tests/test_sessions.py | 8 ++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/channels/sessions.py b/channels/sessions.py index 3e43f80..8bf716b 100644 --- a/channels/sessions.py +++ b/channels/sessions.py @@ -83,12 +83,21 @@ def channel_session(func): return inner +def wait_channel_name(reply_channel): + """ + Given a reply_channel, returns a wait channel for it. + Replaces any ! with ? so process-specific channels become single-reader + channels. + """ + return "__wait__.%s" % (reply_channel.replace("!", "?"), ) + + def requeue_messages(message): """ Requeue any pending wait channel messages for this socket connection back onto it's original channel """ while True: - wait_channel = "__wait__.%s" % message.reply_channel.name + wait_channel = wait_channel_name(message.reply_channel.name) channel, content = message.channel_layer.receive_many([wait_channel], block=False) if channel: original_channel = content.pop("original_channel") @@ -137,7 +146,7 @@ def enforce_ordering(func=None, slight=False): requeue_messages(message) else: # Since out of order, enqueue message temporarily to wait channel for this socket connection - wait_channel = "__wait__.%s" % message.reply_channel.name + wait_channel = wait_channel_name(message.reply_channel.name) message.content["original_channel"] = message.channel.name try: message.channel_layer.send(wait_channel, message.content) diff --git a/tests/test_sessions.py b/tests/test_sessions.py index d1d507a..007ae98 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -256,17 +256,17 @@ class SessionTests(ChannelTestCase): """ # Construct messages to send message0 = Message( - {"reply_channel": "test-reply-b", "order": 0}, + {"reply_channel": "test-reply!b", "order": 0}, "websocket.connect", channel_layers[DEFAULT_CHANNEL_LAYER] ) message1 = Message( - {"reply_channel": "test-reply-b", "order": 1}, + {"reply_channel": "test-reply!b", "order": 1}, "websocket.receive", channel_layers[DEFAULT_CHANNEL_LAYER] ) message2 = Message( - {"reply_channel": "test-reply-b", "order": 2}, + {"reply_channel": "test-reply!b", "order": 2}, "websocket.receive", channel_layers[DEFAULT_CHANNEL_LAYER] ) @@ -281,7 +281,7 @@ class SessionTests(ChannelTestCase): inner(message2) # Ensure wait channel is empty - wait_channel = "__wait__.%s" % "test-reply-b" + wait_channel = "__wait__.test-reply?b" next_message = self.get_next_message(wait_channel) self.assertEqual(next_message, None) From c1f801a20eb89ed422669c2266a1ae58ee33dd6f Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Wed, 5 Apr 2017 13:41:30 +0300 Subject: [PATCH 671/746] Improve docs. (#589) --- docs/getting-started.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 422e5dd..acef373 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -625,6 +625,8 @@ have a ChatMessage model with ``message`` and ``room`` fields:: # Save room in session and add us to the group message.channel_session['room'] = room Group("chat-%s" % room).add(message.reply_channel) + # Accept the connection request + message.reply_channel.send({"accept": True}) # Connected to websocket.receive @channel_session @@ -640,6 +642,19 @@ have a ChatMessage model with ``message`` and ``room`` fields:: def ws_disconnect(message): Group("chat-%s" % message.channel_session['room']).discard(message.reply_channel) +Update ``routing.py`` as well:: + + # in routing.py + from channels.routing import route + from myapp.consumers import ws_connect, ws_message, ws_disconnect, msg_consumer + + channel_routing = [ + route("websocket.connect", ws_connect), + route("websocket.receive", ws_message), + route("websocket.disconnect", ws_disconnect), + route("chat-messages", msg_consumer), + ] + Note that we could add messages onto the ``chat-messages`` channel from anywhere; inside a View, inside another model's ``post_save`` signal, inside a management command run via ``cron``. If we wanted to write a bot, too, we could put its From ceeacdbfc33c2f1eb9d219d8bd539ef5e6c4c877 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Apr 2017 12:55:18 +0200 Subject: [PATCH 672/746] Add explicit checks for asgi library versions Refs daphne/#105 --- channels/apps.py | 10 +++------- channels/package_checks.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 channels/package_checks.py diff --git a/channels/apps.py b/channels/apps.py index f2d7874..2ee1b0f 100644 --- a/channels/apps.py +++ b/channels/apps.py @@ -2,6 +2,7 @@ from django.apps import AppConfig from django.core.exceptions import ImproperlyConfigured from .binding.base import BindingMetaclass +from .package_checks import check_all class ChannelsConfig(AppConfig): @@ -10,13 +11,8 @@ class ChannelsConfig(AppConfig): verbose_name = "Channels" def ready(self): - # Check you're not running 1.10 or above - try: - from django import channels # NOQA isort:skip - except ImportError: - pass - else: - raise ImproperlyConfigured("You have Django 1.10 or above; use the builtin django.channels!") + # Check versions + check_all() # Do django monkeypatches from .hacks import monkeypatch_django monkeypatch_django() diff --git a/channels/package_checks.py b/channels/package_checks.py new file mode 100644 index 0000000..4509c81 --- /dev/null +++ b/channels/package_checks.py @@ -0,0 +1,30 @@ +import importlib +from distutils.version import StrictVersion + + +required_versions = { + "asgi_redis": "1.2.0", + "asgi_ipc": "1.3.0", +} + + +def check_all(): + """ + Checks versions of all the possible packages you have installed so that + we can easily warn people about incompatible versions. + + This is needed as there are some packages (e.g. asgi_redis) that we cannot + declare dependencies on as they are not _required_. People usually remember + to upgrade their Channels package so this is where we check. + """ + for package, version in required_versions.items(): + try: + module = importlib.import_module(package) + except ImportError: + return + else: + if StrictVersion(version) > StrictVersion(module.__version__): + raise RuntimeError("Your version of %s is too old - it must be at least %s" % ( + package, + version, + )) From fe18d43064482de2ae76e2fd420379e3850b7476 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Apr 2017 13:07:32 +0200 Subject: [PATCH 673/746] Remove unused import --- channels/apps.py | 1 - 1 file changed, 1 deletion(-) diff --git a/channels/apps.py b/channels/apps.py index 2ee1b0f..36fc9a0 100644 --- a/channels/apps.py +++ b/channels/apps.py @@ -1,5 +1,4 @@ from django.apps import AppConfig -from django.core.exceptions import ImproperlyConfigured from .binding.base import BindingMetaclass from .package_checks import check_all From 0e637e09a454c9e55a862ede7889ac4e952efcc7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Apr 2017 14:59:40 +0200 Subject: [PATCH 674/746] Add start of IRC client spec draft --- docs/asgi/irc-client.rst | 81 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/asgi/irc-client.rst diff --git a/docs/asgi/irc-client.rst b/docs/asgi/irc-client.rst new file mode 100644 index 0000000..cfdecd7 --- /dev/null +++ b/docs/asgi/irc-client.rst @@ -0,0 +1,81 @@ +=========================================== +IRC Client ASGI Message Format (Draft Spec) +=========================================== + +.. warning:: + This is an incomplete draft. + +Represents communication with an external IRC server as a client. It is possible +to have multiple clients hooked into the same channel layer talking to different +servers, which is why the reply channel is not a fixed name; +a client will provide it with every incoming action and upon connection. + +The reply channel must stay consistent throughout the client's lifetime, so it +can be used as a unique identifier for the client. + + +Connected +--------- + +Sent when the client has established a connection to an IRC server. + +Channel: ``irc-client.connect`` + +Keys: + +* ``reply_channel``: The channel to send messages or actions to the server over. + +* ``server``: A two-item list of ``[hostname, port]``, where hostname is a + unicode string of the server hostname or IP address, and port is the integer port. + + +Joined +------ + +Sent when the client has joined an IRC channel. + +Channel: ``irc-client.join`` + +Keys: + +* ``reply_channel``: The channel to send messages or actions to the server over. + +* ``channel``: Unicode string name of the IRC channel joined + + +Receive +------- + +Represents either a message, action or notice being received from the server. + +Channel: ``irc-client.receive`` + +Keys: + +* ``reply_channel``: The channel to send messages or actions to the server over. + +* ``type``: Unicode string, one of ``message``, ``action`` or ``notice``. + +* ``user``: IRC user as a unicode string (including host portion) + +* ``channel``: IRC channel name as a unicode string + +* ``body``: Message, action or notice content as a unicode string + + +Control +------- + +Sent to control the IRC client. + +Channel: Specified by the server as ``reply_channel`` in other types + +Keys: + +* ``channel``: IRC channel name to act on as a unicode string + +* ``type``: Unicode string, one of ``join``, ``part``, ``message`` or + ``action``. + +* ``body``: If type is ``message`` or ``action``, the body of the message + or the action as a unicode string. From 64e0470b7608a61c2e601b182490f3312da991e5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Apr 2017 15:00:07 +0200 Subject: [PATCH 675/746] Releasing 1.1.3 --- CHANGELOG.txt | 8 ++++++++ docs/releases/1.1.3.rst | 26 ++++++++++++++++++++++++++ docs/releases/index.rst | 1 + 3 files changed, 35 insertions(+) create mode 100644 docs/releases/1.1.3.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 391ff5e..023ef01 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,14 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases +1.1.3 (2017-04-05) +------------------ + +* ``enforce_ordering`` now works correctly with the new-style process-specific + channels + +* ASGI channel layer versions are now explicitly checked for version compatability + 1.1.2 (2017-04-01) ------------------ diff --git a/docs/releases/1.1.3.rst b/docs/releases/1.1.3.rst new file mode 100644 index 0000000..24d022d --- /dev/null +++ b/docs/releases/1.1.3.rst @@ -0,0 +1,26 @@ +1.1.3 Release Notes +=================== + +Channels 1.1.3 is a bugfix release for the 1.1 series, released on +April 5th, 2017. + + +Major Changes +------------- + +None. + + +Minor Changes & Bugfixes +------------------------ + +* ``enforce_ordering`` now works correctly with the new-style process-specific + channels + +* ASGI channel layer versions are now explicitly checked for version compatability + + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index c36e3b7..a5740cc 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -11,3 +11,4 @@ Release Notes 1.1.0 1.1.1 1.1.2 + 1.1.3 From 5f7e76141c42bbc8715fc808f68ed167d664c999 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 7 Apr 2017 04:35:37 +0100 Subject: [PATCH 676/746] Actually releasing 1.1.3 --- channels/__init__.py | 2 +- js_client/lib/index.js | 38 +++++++++++++++++++++----------------- js_client/package.json | 2 +- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/channels/__init__.py b/channels/__init__.py index 9ad4b64..b6451b1 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.2" +__version__ = "1.1.3" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/js_client/lib/index.js b/js_client/lib/index.js index 254c253..53ca4f0 100644 --- a/js_client/lib/index.js +++ b/js_client/lib/index.js @@ -17,8 +17,6 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } -var noop = function noop() {}; - /** * Bridge between Channels and plain javascript. * @@ -29,17 +27,19 @@ var noop = function noop() {}; * console.log(action, stream); * }); */ - var WebSocketBridge = function () { function WebSocketBridge(options) { _classCallCheck(this, WebSocketBridge); - this._socket = null; + /** + * The underlaying `ReconnectingWebSocket` instance. + * + * @type {ReconnectingWebSocket} + */ + this.socket = null; this.streams = {}; this.default_cb = null; - this.options = _extends({}, { - onopen: noop - }, options); + this.options = _extends({}, options); } /** @@ -59,14 +59,20 @@ var WebSocketBridge = function () { key: 'connect', value: function connect(url, protocols, options) { var _url = void 0; + // Use wss:// if running on https:// + var scheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; + var base_url = scheme + '://' + window.location.host; if (url === undefined) { - // Use wss:// if running on https:// - var scheme = window.location.protocol === 'https:' ? 'wss' : 'ws'; - _url = scheme + '://' + window.location.host + '/ws'; + _url = base_url; } else { - _url = url; + // Support relative URLs + if (url[0] == '/') { + _url = '' + base_url + url; + } else { + _url = url; + } } - this._socket = new _reconnectingWebsocket2.default(_url, protocols, options); + this.socket = new _reconnectingWebsocket2.default(_url, protocols, options); } /** @@ -89,7 +95,7 @@ var WebSocketBridge = function () { var _this = this; this.default_cb = cb; - this._socket.onmessage = function (event) { + this.socket.onmessage = function (event) { var msg = JSON.parse(event.data); var action = void 0; var stream = void 0; @@ -105,8 +111,6 @@ var WebSocketBridge = function () { _this.default_cb ? _this.default_cb(action, stream) : null; } }; - - this._socket.onopen = this.options.onopen; } /** @@ -146,7 +150,7 @@ var WebSocketBridge = function () { }, { key: 'send', value: function send(msg) { - this._socket.send(JSON.stringify(msg)); + this.socket.send(JSON.stringify(msg)); } /** @@ -169,7 +173,7 @@ var WebSocketBridge = function () { stream: _stream, payload: action }; - _this2._socket.send(JSON.stringify(msg)); + _this2.socket.send(JSON.stringify(msg)); } }; } diff --git a/js_client/package.json b/js_client/package.json index 5b8a92a..26f7d77 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -1,6 +1,6 @@ { "name": "django-channels", - "version": "1.1.2", + "version": "1.1.3", "description": "", "repository": { "type": "git", From 9266521f9445e4431ae3567a5e2ce25b6af9133c Mon Sep 17 00:00:00 2001 From: Asif Saifuddin Auvi Date: Fri, 7 Apr 2017 16:06:26 +0600 Subject: [PATCH 677/746] Added django 1.11 to travis matrix (#597) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c4d30ed..8e399b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ env: - DJANGO="Django>=1.8,<1.9" - DJANGO="Django>=1.9,<1.10" - DJANGO="Django>=1.10,<1.11" + - DJANGO="Django>=1.11,<2.0" cache: directories: From 4063ac03edc45fa74d2f26ecf696f57933af5869 Mon Sep 17 00:00:00 2001 From: Asif Saifuddin Auvi Date: Fri, 7 Apr 2017 16:06:34 +0600 Subject: [PATCH 678/746] Added 1.11 to tox (#596) --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8caea5c..ecf9f79 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{27,34,35,36}-django-{18,19,110} + py{27,34,35,36}-django-{18,19,110,111} py{27,35,36}-flake8 isort @@ -10,6 +10,7 @@ deps = django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 django-110: Django>=1.10,<1.11 + django-111: Django>=1.11,<2.0 commands = flake8: flake8 isort: isort --check-only --recursive channels From 73832807766811b4b21c0eb4835f6708a984fb72 Mon Sep 17 00:00:00 2001 From: Daniel Hepper Date: Fri, 7 Apr 2017 15:05:34 +0200 Subject: [PATCH 679/746] Added a decorator that checks origin headers (#593) Adds a new allowed_hosts_only decorator and extensible base class to allow for checking the incoming Origin header on WebSocket requests, using the Django `ALLOWED_HOSTS` setting by default. --- channels/security/__init__.py | 0 channels/security/websockets.py | 90 +++++++++++++++++++++++++++++++++ docs/getting-started.rst | 45 +++++++++++++++++ docs/reference.rst | 9 +++- tests/test_security.py | 44 ++++++++++++++++ 5 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 channels/security/__init__.py create mode 100644 channels/security/websockets.py create mode 100644 tests/test_security.py diff --git a/channels/security/__init__.py b/channels/security/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channels/security/websockets.py b/channels/security/websockets.py new file mode 100644 index 0000000..d1dac88 --- /dev/null +++ b/channels/security/websockets.py @@ -0,0 +1,90 @@ +from functools import update_wrapper + +from django.conf import settings +from django.http.request import validate_host + +from ..exceptions import DenyConnection + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +class BaseOriginValidator(object): + """ + Base class-based decorator for origin validation of WebSocket connect + messages. + + This base class handles parsing of the origin header. When the origin header + is missing, empty or contains non-ascii characters, it raises a + DenyConnection exception to reject the connection. + + Subclasses must overwrite the method validate_origin(self, message, origin) + to return True when a message should be accepted, False otherwise. + """ + + def __init__(self, func): + update_wrapper(self, func) + self.func = func + + def __call__(self, message, *args, **kwargs): + origin = self.get_origin(message) + if not self.validate_origin(message, origin): + raise DenyConnection + return self.func(message, *args, **kwargs) + + def get_header(self, message, name): + headers = message.content['headers'] + for header in headers: + try: + if header[0] == name: + return header[1:] + except IndexError: + continue + raise KeyError('No header named "{}"'.format(name)) + + def get_origin(self, message): + """ + Returns the origin of a WebSocket connect message. + + Raises DenyConnection for messages with missing or non-ascii Origin + header. + """ + try: + header = self.get_header(message, b'origin')[0] + except (IndexError, KeyError): + raise DenyConnection + try: + origin = header.decode('ascii') + except UnicodeDecodeError: + raise DenyConnection + return origin + + def validate_origin(self, message, origin): + """ + Validates the origin of a WebSocket connect message. + + Must be overwritten by subclasses. + """ + raise NotImplemented('You must overwrite this method.') + + +class AllowedHostsOnlyOriginValidator(BaseOriginValidator): + """ + Class-based decorator for websocket consumers that checks that + the origin is allowed according to the ALLOWED_HOSTS settings. + """ + + def validate_origin(self, message, origin): + allowed_hosts = settings.ALLOWED_HOSTS + if settings.DEBUG and not allowed_hosts: + allowed_hosts = ['localhost', '127.0.0.1', '[::1]'] + + origin_hostname = urlparse(origin).hostname + valid = (origin_hostname and + validate_host(origin_hostname, allowed_hosts)) + return valid + + +allowed_hosts_only = AllowedHostsOnlyOriginValidator diff --git a/docs/getting-started.rst b/docs/getting-started.rst index acef373..95ebf4d 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -518,6 +518,51 @@ responses can set cookies, it needs a backend it can write to to separately store state. +Security +-------- + +Unlike AJAX requests, WebSocket requests are not limited by the Same-Origin +policy. This means you don't have to take any extra steps when you have an HTML +page served by host A containing JavaScript code wanting to connect to a +WebSocket on Host B. + +While this can be convenient, it also implies that by default any third-party +site can connect to your WebSocket application. When you are using the +``http_session_user`` or the ``channel_session_user_from_http`` decorator, this +connection would be authenticated. + +The WebSocket specification requires browsers to send the origin of a WebSocket +request in the HTTP header named ``Origin``, but validating that header is left +to the server. + +You can use the decorator ``channels.security.websockets.allowed_hosts_only`` +on a ``websocket.connect`` consumer to only allow requests originating +from hosts listed in the ``ALLOWED_HOSTS`` setting:: + + # In consumers.py + from channels import Channel, Group + from channels.sessions import channel_session + from channels.auth import channel_session_user, channel_session_user_from_http + from channels.security.websockets import allowed_hosts_only. + + # Connected to websocket.connect + @allowed_hosts_only + @channel_session_user_from_http + def ws_add(message): + # Accept connection + ... + +Requests from other hosts or requests with missing or invalid origin header +are now rejected. + +The name ``allowed_hosts_only`` is an alias for the class-based decorator +``AllowedHostsOnlyOriginValidator``, which inherits from +``BaseOriginValidator``. If you have custom requirements for origin validation, +create a subclass and overwrite the method +``validate_origin(self, message, origin)``. It must return True when a message +should be accepted, False otherwise. + + Routing ------- diff --git a/docs/reference.rst b/docs/reference.rst index 3a1f7e7..c90e8f4 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -172,7 +172,7 @@ directly, but there are two useful ways you can call it: Decorators ---------- -Channels provides decorators to assist with persisting data. +Channels provides decorators to assist with persisting data and security. * ``channel_session``: Provides a session-like object called "channel_session" to consumers as a message attribute that will auto-persist across consumers with @@ -200,3 +200,10 @@ Channels provides decorators to assist with persisting data. Stores the http session key in the channel_session on websocket.connect messages. It will then hydrate the http_session from that same key on subsequent messages. + +* ``allowed_hosts_only``: Wraps a WebSocket connect consumer and ensures the + request originates from an allowed host. + + Reads the Origin header and only passes request originating from a host + listed in ``ALLOWED_HOSTS`` to the consumer. Requests from other hosts or + with a missing or invalid Origin headers are rejected. diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..2805481 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,44 @@ +from __future__ import unicode_literals + +from django.test import override_settings +from channels.exceptions import DenyConnection +from channels.security.websockets import allowed_hosts_only +from channels.message import Message +from channels.test import ChannelTestCase + + +@allowed_hosts_only +def connect(message): + return True + + +class OriginValidationTestCase(ChannelTestCase): + + @override_settings(ALLOWED_HOSTS=['example.com']) + def test_valid_origin(self): + content = { + 'headers': [[b'origin', b'http://example.com']] + } + message = Message(content, 'websocket.connect', None) + self.assertTrue(connect(message)) + + @override_settings(ALLOWED_HOSTS=['example.com']) + def test_invalid_origin(self): + content = { + 'headers': [[b'origin', b'http://example.org']] + } + message = Message(content, 'websocket.connect', None) + self.assertRaises(DenyConnection, connect, message) + + def test_invalid_origin_header(self): + invalid_headers = [ + [], # origin header missing + [b'origin', b''], # origin header empty + [b'origin', b'\xc3\xa4'] # non-ascii + ] + for headers in invalid_headers: + content = { + 'headers': [headers] + } + message = Message(content, 'websocket.connect', None) + self.assertRaises(DenyConnection, connect, message) From db1b3ba951ef2c3cff6e91ed94d4115d479c0d4f Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Tue, 11 Apr 2017 12:17:47 +0300 Subject: [PATCH 680/746] Add ChannelLiveServerTestCase (#497) Adds a new test case base class that fires up a live Daphne server and workers for the test case to use, like Django's LiveServerTestCase. --- channels/test/__init__.py | 1 + channels/test/liveserver.py | 228 ++++++++++++++++++ channels/tests/__init__.py | 3 +- docs/releases/1.1.0.rst | 1 - docs/testing.rst | 31 +++ runtests.py | 3 +- setup.py | 2 +- testproject/chtest/consumers.py | 3 +- testproject/chtest/views.py | 1 + testproject/setup.py | 9 +- testproject/testproject/settings/__init__.py | 2 +- testproject/testproject/settings/base.py | 8 +- .../testproject/settings/channels_ipc.py | 1 - .../testproject/settings/channels_rabbitmq.py | 9 +- tests/settings.py | 2 +- tox.ini | 2 +- 16 files changed, 291 insertions(+), 15 deletions(-) create mode 100644 channels/test/liveserver.py diff --git a/channels/test/__init__.py b/channels/test/__init__.py index 0c957f3..c781e85 100644 --- a/channels/test/__init__.py +++ b/channels/test/__init__.py @@ -1,2 +1,3 @@ from .base import TransactionChannelTestCase, ChannelTestCase, Client, apply_routes # NOQA isort:skip from .http import HttpClient # NOQA isort:skip +from .liveserver import ChannelLiveServerTestCase # NOQA isort:skip diff --git a/channels/test/liveserver.py b/channels/test/liveserver.py new file mode 100644 index 0000000..f942ca5 --- /dev/null +++ b/channels/test/liveserver.py @@ -0,0 +1,228 @@ +import multiprocessing + +import django +from daphne.server import Server +from django.core.exceptions import ImproperlyConfigured +from django.db import connections +from django.db.utils import load_backend +from django.test.testcases import TransactionTestCase +from django.test.utils import modify_settings, override_settings +from twisted.internet import reactor + +from .. import DEFAULT_CHANNEL_LAYER +from ..asgi import ChannelLayerManager +from ..staticfiles import StaticFilesConsumer +from ..worker import Worker, WorkerGroup + +# NOTE: We use ChannelLayerManager to prevent layer instance sharing +# between forked process. Some layers implementations create +# connections inside the __init__ method. After forking child +# processes can lose the ability to use this connection and typically +# stuck on some network operation. To prevent this we use new +# ChannelLayerManager each time we want to initiate default layer. +# This gives us guaranty that new layer instance will be created and +# new connection will be established. + + +class ProcessSetup(multiprocessing.Process): + """Common initialization steps for test subprocess.""" + + def common_setup(self): + + self.setup_django() + self.setup_databases() + self.override_settings() + + def setup_django(self): + + if django.VERSION >= (1, 10): + django.setup(set_prefix=False) + else: + django.setup() + + def setup_databases(self): + + for alias, db in self.databases.items(): + backend = load_backend(db['ENGINE']) + conn = backend.DatabaseWrapper(db, alias) + if django.VERSION >= (1, 9): + connections[alias].creation.set_as_test_mirror( + conn.settings_dict, + ) + else: + test_db_name = conn.settings_dict['NAME'] + connections[alias].settings_dict['NAME'] = test_db_name + + def override_settings(self): + + if self.overridden_settings: + overridden = override_settings(**self.overridden_settings) + overridden.enable() + + if self.modified_settings: + modified = modify_settings(self.modified_settings) + modified.enable() + + +class WorkerProcess(ProcessSetup): + + def __init__(self, is_ready, n_threads, overridden_settings, + modified_settings, databases): + + self.is_ready = is_ready + self.n_threads = n_threads + self.overridden_settings = overridden_settings + self.modified_settings = modified_settings + self.databases = databases + super(WorkerProcess, self).__init__() + self.daemon = True + + def run(self): + + try: + self.common_setup() + channel_layers = ChannelLayerManager() + channel_layers[DEFAULT_CHANNEL_LAYER].router.check_default( + http_consumer=StaticFilesConsumer(), + ) + if self.n_threads == 1: + self.worker = Worker( + channel_layer=channel_layers[DEFAULT_CHANNEL_LAYER], + signal_handlers=False, + ) + else: + self.worker = WorkerGroup( + channel_layer=channel_layers[DEFAULT_CHANNEL_LAYER], + signal_handlers=False, + n_threads=self.n_threads, + ) + self.worker.ready() + self.is_ready.set() + self.worker.run() + except Exception: + self.is_ready.set() + raise + + +class DaphneProcess(ProcessSetup): + + def __init__(self, host, port_storage, is_ready, overridden_settings, + modified_settings, databases): + + self.host = host + self.port_storage = port_storage + self.is_ready = is_ready + self.overridden_settings = overridden_settings + self.modified_settings = modified_settings + self.databases = databases + super(DaphneProcess, self).__init__() + self.daemon = True + + def run(self): + + try: + self.common_setup() + channel_layers = ChannelLayerManager() + self.server = Server( + channel_layer=channel_layers[DEFAULT_CHANNEL_LAYER], + endpoints=['tcp:interface=%s:port=0' % (self.host)], + signal_handlers=False, + ) + reactor.callLater(0.5, self.resolve_port) + self.server.run() + except Exception: + self.is_ready.set() + raise + + def resolve_port(self): + + port = self.server.listeners[0].result.getHost().port + self.port_storage.value = port + self.is_ready.set() + + +class ChannelLiveServerTestCase(TransactionTestCase): + """ + Does basically the same as TransactionTestCase but also launches a + live Daphne server and Channels worker in a separate process, so + that the tests may use another test framework, such as Selenium, + instead of the built-in dummy client. + """ + + host = 'localhost' + ProtocolServerProcess = DaphneProcess + WorkerProcess = WorkerProcess + worker_threads = 1 + + @property + def live_server_url(self): + + return 'http://%s:%s' % (self.host, self._port_storage.value) + + @property + def live_server_ws_url(self): + + return 'ws://%s:%s' % (self.host, self._port_storage.value) + + def _pre_setup(self): + + for connection in connections.all(): + if self._is_in_memory_db(connection): + raise ImproperlyConfigured( + 'ChannelLiveServerTestCase can not be used with in memory databases' + ) + + channel_layers = ChannelLayerManager() + if len(channel_layers.configs) > 1: + raise ImproperlyConfigured( + 'ChannelLiveServerTestCase does not support multiple CHANNEL_LAYERS at this time' + ) + + channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + if 'flush' in channel_layer.extensions: + channel_layer.flush() + + super(ChannelLiveServerTestCase, self)._pre_setup() + + server_ready = multiprocessing.Event() + self._port_storage = multiprocessing.Value('i') + self._server_process = self.ProtocolServerProcess( + self.host, + self._port_storage, + server_ready, + self._overridden_settings, + self._modified_settings, + connections.databases, + ) + self._server_process.start() + server_ready.wait() + + worker_ready = multiprocessing.Event() + self._worker_process = self.WorkerProcess( + worker_ready, + self.worker_threads, + self._overridden_settings, + self._modified_settings, + connections.databases, + ) + self._worker_process.start() + worker_ready.wait() + + def _post_teardown(self): + + self._server_process.terminate() + self._server_process.join() + self._worker_process.terminate() + self._worker_process.join() + super(ChannelLiveServerTestCase, self)._post_teardown() + + def _is_in_memory_db(self, connection): + """Check if DatabaseWrapper holds in memory database.""" + + if connection.vendor == 'sqlite': + if django.VERSION >= (1, 11): + return connection.is_in_memory_db() + else: + return connection.is_in_memory_db( + connection.settings_dict['NAME'], + ) diff --git a/channels/tests/__init__.py b/channels/tests/__init__.py index 27ae0e3..af4b25e 100644 --- a/channels/tests/__init__.py +++ b/channels/tests/__init__.py @@ -5,5 +5,4 @@ warnings.warn( DeprecationWarning, ) -from channels.test.base import TransactionChannelTestCase, ChannelTestCase, Client, apply_routes # NOQA isort:skip -from channels.test.http import HttpClient # NOQA isort:skip +from channels.test import * # NOQA isort:skip diff --git a/docs/releases/1.1.0.rst b/docs/releases/1.1.0.rst index 5d72136..4b63a71 100644 --- a/docs/releases/1.1.0.rst +++ b/docs/releases/1.1.0.rst @@ -18,7 +18,6 @@ Major Changes work but will trigger a deprecation warning, and ``channels.tests`` will be removed completely in version 1.3. - Minor Changes & Bugfixes ------------------------ diff --git a/docs/testing.rst b/docs/testing.rst index bc4a082..945bf03 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -294,3 +294,34 @@ mocked. You can pass an ``alias`` argument to ``get_next_message``, ``Client`` and ``Channel`` to use a different layer too. + +Live Server Test Case +--------------------- + +You can use browser automation libraries like Selenium or Splinter to +check your application against real layer installation. Use +``ChannelLiveServerTestCase`` for your acceptance tests. + +.. code:: python + + from channels.test import ChannelLiveServerTestCase + from splinter import Browser + + class IntegrationTest(ChannelLiveServerTestCase): + + def test_browse_site_index(self): + + with Browser() as browser: + + browser.visit(self.live_server_url) + # the rest of your integration test... + +In the test above Daphne and Channels worker processes were fired up. +These processes run your project against the test database and the +default channel layer you spacify in the settings. If channel layer +support ``flush`` extension, initial cleanup will be done. So do not +run this code against your production environment. When channels +infrastructure is ready default web browser will be also started. You +can open your website in the real browser which can execute JavaScript +and operate on WebSockets. ``live_server_ws_url`` property is also +provided if you decide to run messaging directly from Python. diff --git a/runtests.py b/runtests.py index 925b27b..2f9b3f2 100755 --- a/runtests.py +++ b/runtests.py @@ -10,6 +10,7 @@ if __name__ == "__main__": os.environ['DJANGO_SETTINGS_MODULE'] = "tests.settings" django.setup() TestRunner = get_runner(settings) + tests = sys.argv[1:] or ["tests"] test_runner = TestRunner() - failures = test_runner.run_tests(["tests"]) + failures = test_runner.run_tests(tests) sys.exit(bool(failures)) diff --git a/setup.py b/setup.py index 9791a96..2163a6b 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( 'mock ; python_version < "3.0"', 'flake8>=2.0,<3.0', 'isort', - ] + ], }, classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/testproject/chtest/consumers.py b/testproject/chtest/consumers.py index 85db43f..a761fd9 100644 --- a/testproject/chtest/consumers.py +++ b/testproject/chtest/consumers.py @@ -2,5 +2,6 @@ from channels.sessions import enforce_ordering def ws_message(message): - "Echoes messages back to the client" + "Echoes messages back to the client." + message.reply_channel.send({'text': message['text']}) diff --git a/testproject/chtest/views.py b/testproject/chtest/views.py index 4bed90e..d244010 100644 --- a/testproject/chtest/views.py +++ b/testproject/chtest/views.py @@ -2,4 +2,5 @@ from django.http import HttpResponse def index(request): + return HttpResponse("OK") diff --git a/testproject/setup.py b/testproject/setup.py index 916e2da..8d0865c 100644 --- a/testproject/setup.py +++ b/testproject/setup.py @@ -1,7 +1,12 @@ -from setuptools import setup +from setuptools import find_packages, setup setup( name='channels-benchmark', + packages=find_packages(), py_modules=['benchmark'], - install_requires=['autobahn', 'Twisted'], + install_requires=[ + 'autobahn', + 'Twisted', + 'statistics ; python_version < "3.0"', + ], ) diff --git a/testproject/testproject/settings/__init__.py b/testproject/testproject/settings/__init__.py index 455876f..7b33d23 100644 --- a/testproject/testproject/settings/__init__.py +++ b/testproject/testproject/settings/__init__.py @@ -1 +1 @@ -#Blank on purpose +# Blank on purpose diff --git a/testproject/testproject/settings/base.py b/testproject/testproject/settings/base.py index e0f8773..e9b51b5 100644 --- a/testproject/testproject/settings/base.py +++ b/testproject/testproject/settings/base.py @@ -1,4 +1,5 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) + import os BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -23,5 +24,10 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } + 'TEST': { + 'NAME': os.path.join(BASE_DIR, 'test_db.sqlite3'), + }, + }, } + +ALLOWED_HOSTS = ['*'] diff --git a/testproject/testproject/settings/channels_ipc.py b/testproject/testproject/settings/channels_ipc.py index 6098a12..3a02249 100644 --- a/testproject/testproject/settings/channels_ipc.py +++ b/testproject/testproject/settings/channels_ipc.py @@ -1,7 +1,6 @@ # Settings for channels specifically from testproject.settings.base import * - INSTALLED_APPS += ( 'channels', ) diff --git a/testproject/testproject/settings/channels_rabbitmq.py b/testproject/testproject/settings/channels_rabbitmq.py index 4754d09..005eb07 100644 --- a/testproject/testproject/settings/channels_rabbitmq.py +++ b/testproject/testproject/settings/channels_rabbitmq.py @@ -1,14 +1,19 @@ # Settings for channels specifically from testproject.settings.base import * -INSTALLED_APPS += ('channels',) +INSTALLED_APPS += ( + 'channels', +) CHANNEL_LAYERS = { 'default': { 'BACKEND': 'asgi_rabbitmq.RabbitmqChannelLayer', 'ROUTING': 'testproject.urls.channel_routing', 'CONFIG': { - 'url': os.environ['RABBITMQ_URL'], + 'url': os.environ.get( + 'RABBITMQ_URL', + 'amqp://guest:guest@localhost:5672/%2F', + ), }, }, } diff --git a/tests/settings.py b/tests/settings.py index bebbd2a..74fbcc3 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -24,7 +24,7 @@ CHANNEL_LAYERS = { 'fake_channel': { 'BACKEND': 'tests.test_management.FakeChannelLayer', 'ROUTING': [], - } + }, } MIDDLEWARE_CLASSES = [] diff --git a/tox.ini b/tox.ini index ecf9f79..11900b1 100644 --- a/tox.ini +++ b/tox.ini @@ -14,4 +14,4 @@ deps = commands = flake8: flake8 isort: isort --check-only --recursive channels - django: coverage run --parallel-mode {toxinidir}/runtests.py + django: coverage run --parallel-mode {toxinidir}/runtests.py {posargs} From 31cd68c89bfe302b91aac6b096e94d3c074e3b72 Mon Sep 17 00:00:00 2001 From: Sachin Rekhi Date: Tue, 11 Apr 2017 02:25:56 -0700 Subject: [PATCH 681/746] Delay server will now requeue messages for later when gets ChannelFull exception (#600) It re-delays them for one second each time rather than dropping them. --- channels/delay/models.py | 13 +++++++++---- tests/test_delay.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/channels/delay/models.py b/channels/delay/models.py index 4bff090..932d6d3 100644 --- a/channels/delay/models.py +++ b/channels/delay/models.py @@ -30,15 +30,20 @@ class DelayedMessage(models.Model): self._delay = milliseconds self.due_date = timezone.now() + timedelta(milliseconds=milliseconds) - def send(self, channel_layer=None): + def send(self, channel_layer=None, requeue_delay=1000): """ Sends the message on the configured channel with the stored content. - Deletes the DelayedMessage record. + Deletes the DelayedMessage record if successfully sent. Args: channel_layer: optional channel_layer to use + requeue_delay: if the channel is full, milliseconds to wait before requeue """ channel_layer = channel_layer or channel_layers[DEFAULT_CHANNEL_LAYER] - Channel(self.channel_name, channel_layer=channel_layer).send(json.loads(self.content), immediately=True) - self.delete() + try: + Channel(self.channel_name, channel_layer=channel_layer).send(json.loads(self.content), immediately=True) + self.delete() + except channel_layer.ChannelFull: + self.delay = requeue_delay + self.save() diff --git a/tests/test_delay.py b/tests/test_delay.py index 08cb194..a6de9dd 100644 --- a/tests/test_delay.py +++ b/tests/test_delay.py @@ -71,6 +71,41 @@ class WorkerTests(ChannelTestCase): message = self.get_next_message('test', require=True) self.assertEqual(message.content, {'test': 'value'}) + def test_channel_full(self): + """ + Tests that when channel capacity is hit when processing due messages, + message is requeued instead of dropped + """ + for i in range(10): + Channel('asgi.delay').send({ + 'channel': 'test', + 'delay': 1000, + 'content': {'test': 'value'} + }, immediately=True) + + worker = PatchedWorker(channel_layers[DEFAULT_CHANNEL_LAYER]) + worker.termed = 10 + worker.run() + + for i in range(1): + Channel('asgi.delay').send({ + 'channel': 'test', + 'delay': 1000, + 'content': {'test': 'value'} + }, immediately=True) + + worker = PatchedWorker(channel_layers[DEFAULT_CHANNEL_LAYER]) + worker.termed = 1 + worker.run() + + self.assertEqual(DelayedMessage.objects.count(), 11) + + with mock.patch('django.utils.timezone.now', return_value=timezone.now() + timedelta(milliseconds=2000)): + worker.termed = 1 + worker.run() + + self.assertEqual(DelayedMessage.objects.count(), 1) + class DelayedMessageTests(ChannelTestCase): From 5648da5d34f01d0d7dcaa6731ef199bc3bdbdf1a Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Thu, 13 Apr 2017 00:01:25 +0300 Subject: [PATCH 682/746] Provide a way to disable static files serving in the live server test case. (#603) This allows it to be turned off (it's on by default) --- channels/test/liveserver.py | 14 ++++++++++---- docs/testing.rst | 13 +++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/channels/test/liveserver.py b/channels/test/liveserver.py index f942ca5..caa4020 100644 --- a/channels/test/liveserver.py +++ b/channels/test/liveserver.py @@ -2,6 +2,7 @@ import multiprocessing import django from daphne.server import Server +from django.apps import apps from django.core.exceptions import ImproperlyConfigured from django.db import connections from django.db.utils import load_backend @@ -67,13 +68,14 @@ class ProcessSetup(multiprocessing.Process): class WorkerProcess(ProcessSetup): def __init__(self, is_ready, n_threads, overridden_settings, - modified_settings, databases): + modified_settings, databases, serve_static): self.is_ready = is_ready self.n_threads = n_threads self.overridden_settings = overridden_settings self.modified_settings = modified_settings self.databases = databases + self.serve_static = serve_static super(WorkerProcess, self).__init__() self.daemon = True @@ -82,9 +84,11 @@ class WorkerProcess(ProcessSetup): try: self.common_setup() channel_layers = ChannelLayerManager() - channel_layers[DEFAULT_CHANNEL_LAYER].router.check_default( - http_consumer=StaticFilesConsumer(), - ) + check_default = channel_layers[DEFAULT_CHANNEL_LAYER].router.check_default + if self.serve_static and apps.is_installed('django.contrib.staticfiles'): + check_default(http_consumer=StaticFilesConsumer()) + else: + check_default() if self.n_threads == 1: self.worker = Worker( channel_layer=channel_layers[DEFAULT_CHANNEL_LAYER], @@ -153,6 +157,7 @@ class ChannelLiveServerTestCase(TransactionTestCase): ProtocolServerProcess = DaphneProcess WorkerProcess = WorkerProcess worker_threads = 1 + serve_static = True @property def live_server_url(self): @@ -204,6 +209,7 @@ class ChannelLiveServerTestCase(TransactionTestCase): self._overridden_settings, self._modified_settings, connections.databases, + self.serve_static, ) self._worker_process.start() worker_ready.wait() diff --git a/docs/testing.rst b/docs/testing.rst index 945bf03..09584c0 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -325,3 +325,16 @@ infrastructure is ready default web browser will be also started. You can open your website in the real browser which can execute JavaScript and operate on WebSockets. ``live_server_ws_url`` property is also provided if you decide to run messaging directly from Python. + +By default live server test case will serve static files. To disable +this feature override `serve_static` class attribute. + +.. code:: python + + class IntegrationTest(ChannelLiveServerTestCase): + + serve_static = False + + def test_websocket_message(self): + # JS and CSS are not available in this test. + ... From 5445d317fd716020cda2591f0d33fdff478abf2f Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Thu, 13 Apr 2017 00:51:41 +0300 Subject: [PATCH 683/746] Provide TEST_CONFIG alias. (#604) Allows (well, forces) a different channel layer configuration in testing. --- channels/asgi.py | 19 ++++++++++++++++++- channels/test/liveserver.py | 15 ++++++++------- docs/testing.rst | 22 ++++++++++++++++++++-- tests/test_asgi.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 tests/test_asgi.py diff --git a/channels/asgi.py b/channels/asgi.py index 6156a48..132553d 100644 --- a/channels/asgi.py +++ b/channels/asgi.py @@ -26,6 +26,23 @@ class ChannelLayerManager(object): return getattr(settings, "CHANNEL_LAYERS", {}) def make_backend(self, name): + """ + Instantiate channel layer. + """ + config = self.configs[name].get("CONFIG", {}) + return self._make_backend(name, config) + + def make_test_backend(self, name): + """ + Instantiate channel layer using its test config. + """ + try: + config = self.configs[name]["TEST_CONFIG"] + except KeyError: + raise InvalidChannelLayerError("No TEST_CONFIG specified for %s" % name) + return self._make_backend(name, config) + + def _make_backend(self, name, config): # Load the backend class try: backend_class = import_string(self.configs[name]['BACKEND']) @@ -41,7 +58,7 @@ class ChannelLayerManager(object): except KeyError: raise InvalidChannelLayerError("No ROUTING specified for %s" % name) # Initialise and pass config - asgi_layer = backend_class(**self.configs[name].get("CONFIG", {})) + asgi_layer = backend_class(**config) return ChannelLayerWrapper( channel_layer=asgi_layer, alias=name, diff --git a/channels/test/liveserver.py b/channels/test/liveserver.py index caa4020..88e357f 100644 --- a/channels/test/liveserver.py +++ b/channels/test/liveserver.py @@ -84,19 +84,19 @@ class WorkerProcess(ProcessSetup): try: self.common_setup() channel_layers = ChannelLayerManager() - check_default = channel_layers[DEFAULT_CHANNEL_LAYER].router.check_default + channel_layer = channel_layers.make_test_backend(DEFAULT_CHANNEL_LAYER) if self.serve_static and apps.is_installed('django.contrib.staticfiles'): - check_default(http_consumer=StaticFilesConsumer()) + channel_layer.router.check_default(http_consumer=StaticFilesConsumer()) else: - check_default() + channel_layer.router.check_default() if self.n_threads == 1: self.worker = Worker( - channel_layer=channel_layers[DEFAULT_CHANNEL_LAYER], + channel_layer=channel_layer, signal_handlers=False, ) else: self.worker = WorkerGroup( - channel_layer=channel_layers[DEFAULT_CHANNEL_LAYER], + channel_layer=channel_layer, signal_handlers=False, n_threads=self.n_threads, ) @@ -127,8 +127,9 @@ class DaphneProcess(ProcessSetup): try: self.common_setup() channel_layers = ChannelLayerManager() + channel_layer = channel_layers.make_test_backend(DEFAULT_CHANNEL_LAYER) self.server = Server( - channel_layer=channel_layers[DEFAULT_CHANNEL_LAYER], + channel_layer=channel_layer, endpoints=['tcp:interface=%s:port=0' % (self.host)], signal_handlers=False, ) @@ -183,7 +184,7 @@ class ChannelLiveServerTestCase(TransactionTestCase): 'ChannelLiveServerTestCase does not support multiple CHANNEL_LAYERS at this time' ) - channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] + channel_layer = channel_layers.make_test_backend(DEFAULT_CHANNEL_LAYER) if 'flush' in channel_layer.extensions: channel_layer.flush() diff --git a/docs/testing.rst b/docs/testing.rst index 09584c0..970857e 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -299,8 +299,26 @@ Live Server Test Case --------------------- You can use browser automation libraries like Selenium or Splinter to -check your application against real layer installation. Use -``ChannelLiveServerTestCase`` for your acceptance tests. +check your application against real layer installation. First of all +provide ``TEST_CONFIG`` setting to prevent overlapping with running +dev environment. + +.. code:: python + + CHANNEL_LAYERS = { + "default": { + "BACKEND": "asgi_redis.RedisChannelLayer", + "ROUTING": "my_project.routing.channel_routing", + "CONFIG": { + "hosts": [("redis-server-name", 6379)], + }, + "TEST_CONFIG": { + "hosts": [("localhost", 6379)], + }, + }, + } + +Now use ``ChannelLiveServerTestCase`` for your acceptance tests. .. code:: python diff --git a/tests/test_asgi.py b/tests/test_asgi.py new file mode 100644 index 0000000..b166e32 --- /dev/null +++ b/tests/test_asgi.py @@ -0,0 +1,34 @@ +from channels import DEFAULT_CHANNEL_LAYER +from channels.asgi import InvalidChannelLayerError, channel_layers +from channels.test import ChannelTestCase +from django.test import override_settings + + +class TestChannelLayerManager(ChannelTestCase): + + def test_config_error(self): + """ + If channel layer doesn't specify TEST_CONFIG, `make_test_backend` + should result into error. + """ + + with self.assertRaises(InvalidChannelLayerError): + channel_layers.make_test_backend(DEFAULT_CHANNEL_LAYER) + + @override_settings(CHANNEL_LAYERS={ + 'default': { + 'BACKEND': 'asgiref.inmemory.ChannelLayer', + 'ROUTING': [], + 'TEST_CONFIG': { + 'expiry': 100500, + }, + }, + }) + def test_config_instance(self): + """ + If channel layer provides TEST_CONFIG, `make_test_backend` should + return channel layer instance appropriate for testing. + """ + + layer = channel_layers.make_test_backend(DEFAULT_CHANNEL_LAYER) + self.assertEqual(layer.channel_layer.expiry, 100500) From 5e96e3c17650c0a65b410e3571f814d4f7c3bf45 Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Sat, 15 Apr 2017 03:42:19 +0300 Subject: [PATCH 684/746] Mention RabbitMQ layer in the docs. (#608) --- README.rst | 4 +++- channels/package_checks.py | 3 ++- docs/backends.rst | 25 +++++++++++++++++++++++++ docs/contributing.rst | 1 + docs/deploying.rst | 13 +++++++++++++ docs/index.rst | 1 + loadtesting/2016-09-06/README.rst | 28 ++++++++++++++-------------- 7 files changed, 59 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 5d1877f..f969780 100644 --- a/README.rst +++ b/README.rst @@ -29,7 +29,8 @@ Support can be obtained either here via issues, or in the ``#django-channels`` channel on Freenode. You can install channels from PyPI as the ``channels`` package. -You'll likely also want to ``asgi_redis`` to provide the Redis channel layer. +You'll likely also want to install ``asgi_redis`` or ``asgi_rabbitmq`` +to provide the Redis/RabbitMQ channel layer correspondingly. See our `installation `_ and `getting started `_ docs for more. @@ -78,4 +79,5 @@ The Channels project is made up of several packages; the others are: * `Daphne `_, the HTTP and Websocket termination server * `asgiref `_, the base ASGI library/memory backend * `asgi_redis `_, the Redis channel backend +* `asgi_rabbitmq `_, the RabbitMQ channel backend * `asgi_ipc `_, the POSIX IPC channel backend diff --git a/channels/package_checks.py b/channels/package_checks.py index 4509c81..6e4bbf8 100644 --- a/channels/package_checks.py +++ b/channels/package_checks.py @@ -3,6 +3,7 @@ from distutils.version import StrictVersion required_versions = { + "asgi_rabbitmq": "0.4.0", "asgi_redis": "1.2.0", "asgi_ipc": "1.3.0", } @@ -21,7 +22,7 @@ def check_all(): try: module = importlib.import_module(package) except ImportError: - return + continue else: if StrictVersion(version) > StrictVersion(module.__version__): raise RuntimeError("Your version of %s is too old - it must be at least %s" % ( diff --git a/docs/backends.rst b/docs/backends.rst index 16f1d6b..f500d14 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -54,6 +54,31 @@ the consistent hashing model relies on all running clients having the same settings. Any misconfigured interface server or worker will drop some or all messages. +RabbitMQ +-------- + +RabbitMQ layer is comparable to Redis in terms of latency and +throughput. It can work with single RabbitMQ node and with Erlang +cluster. + +You need to install layer package from PyPI:: + + pip install -U asgi_rabbitmq + +To use it you also need provide link to the virtual host with granted +permissions:: + + CHANNEL_LAYERS = { + "default": { + "BACKEND": "asgi_rabbitmq.RabbitmqChannelLayer", + "ROUTING": "???", + "CONFIG": { + "url": "amqp://guest:guest@rabbitmq:5672/%2F", + }, + }, + } + +This layer has complete `documentation `_ on its own. IPC --- diff --git a/docs/contributing.rst b/docs/contributing.rst index 8620a43..9e03e40 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -22,6 +22,7 @@ of the Channels sub-projects: * `Daphne issues `_, for the HTTP and Websocket termination * `asgiref issues `_, for the base ASGI library/memory backend * `asgi_redis issues `_, for the Redis channel backend + * `asgi_rabbitmq `_, for the RabbitMQ channel backend * `asgi_ipc issues `_, for the POSIX IPC channel backend Issues are categorized by difficulty level: diff --git a/docs/deploying.rst b/docs/deploying.rst index 8b72707..2b2e92a 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -79,6 +79,19 @@ Make sure the same settings file is used across all your workers and interface servers; without it, they won't be able to talk to each other and things will just fail to work. +If you prefer to use RabbitMQ layer, please refer to its +`documentation `_. +Usually your config will end up like this:: + + CHANNEL_LAYERS = { + "default": { + "BACKEND": "asgi_rabbitmq.RabbitmqChannelLayer", + "ROUTING": "my_project.routing.channel_routing", + "CONFIG": { + "url": "amqp://guest:guest@rabbitmq:5672/%2F", + }, + }, + } Run worker servers ------------------ diff --git a/docs/index.rst b/docs/index.rst index 7135ba9..a9fa415 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,7 @@ Channels is comprised of five packages: * `Daphne `_, the HTTP and Websocket termination server * `asgiref `_, the base ASGI library/memory backend * `asgi_redis `_, the Redis channel backend +* `asgi_rabbitmq `_, the RabbitMQ channel backend * `asgi_ipc `_, the POSIX IPC channel backend This documentation covers the system as a whole; individual release notes and diff --git a/loadtesting/2016-09-06/README.rst b/loadtesting/2016-09-06/README.rst index 2c79b59..4acc015 100644 --- a/loadtesting/2016-09-06/README.rst +++ b/loadtesting/2016-09-06/README.rst @@ -1,5 +1,5 @@ Django Channels Load Testing Results for (2016-09-06) -=============== +===================================================== The goal of these load tests is to see how Channels performs with normal HTTP traffic under heavy load. @@ -11,7 +11,7 @@ comparison to a WSGI HTTP server. Gunincorn was chosen as its configuration was Summary of Results -~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~ Daphne is not as efficient as its WSGI counterpart. When considering only latency, Daphne can have 10 times the latency when under the same traffic load as gunincorn. When considering only throughput, Daphne can have 40-50% of the total @@ -29,7 +29,7 @@ Some additional things that should be tested: Methodology -~~~~~~~~~~~~ +~~~~~~~~~~~ In order to control for variances, several measures were taken: @@ -42,7 +42,7 @@ In order to control for variances, several measures were taken: Setups -~~~~~~~~~~~~ +~~~~~~ 3 setups were used for this set of tests: @@ -52,7 +52,7 @@ Setups Latency -~~~~~~~~~~~~ +~~~~~~~ All target and sources machines were identical ec2 instances m3.2xlarge running Ubuntu 16.04. @@ -63,7 +63,7 @@ In order to ensure that the same number of requests were sent, the rps flag was Throughput -~~~~~~~~~~~~ +~~~~~~~~~~ The same source machine was used for all tests: ec2 instance m3.large running Ubuntu 16.04. All target machines were identical ec2 instances m3.2xlarge running Ubuntu 16.04. @@ -77,7 +77,7 @@ Gunicorn had a latency of 6 ms; daphne and Redis, 12 ms; daphne and IPC, 35 ms. Supervisor Configs -~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~ **Gunicorn (19.6.0)** @@ -89,7 +89,7 @@ This is the non-channels config. It's a standard Django environment on one machi command = gunicorn testproject.wsgi_no_channels -b 0.0.0.0:80 directory = /srv/channels/testproject/ user = root - + [group:django_http] programs=gunicorn priority=999 @@ -107,13 +107,13 @@ Also, it's a single worker, not multiple, as that's the default config. command = daphne -b 0.0.0.0 -p 80 testproject.asgi:channel_layer directory = /srv/channels/testproject/ user = root - + [program:worker] command = python manage.py runworker directory = /srv/channels/testproject/ user = django-channels - - + + [group:django_channels] programs=daphne,worker priority=999 @@ -130,13 +130,13 @@ This is the channels config using IPC (Inter Process Communication). It's only p command = daphne -b 0.0.0.0 -p 80 testproject.asgi_for_ipc:channel_layer directory = /srv/channels/testproject/ user = root - + [program:worker] command = python manage.py runworker --settings=testproject.settings.channels_ipc directory = /srv/channels/testproject/ user = root - - + + [group:django_channels] programs=daphne,worker priority=999 From 585c093352c98f3e5a44c7ce786c487a71890ac4 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Sun, 16 Apr 2017 19:27:56 +0300 Subject: [PATCH 685/746] Remove unnecessary line (#610) --- docs/testing.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/testing.rst b/docs/testing.rst index 970857e..7b2a49d 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -233,7 +233,6 @@ with creating:: # create target entity value = IntegerValue.objects.create(name='fifty', value=50) - consumer_finished.send(sender=None) received = client.receive() # receive outbound binding message self.assertIsNotNone(received) From a0cbccfebca6d2858260dcef3455c67c82501b00 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Sun, 16 Apr 2017 19:45:29 +0300 Subject: [PATCH 686/746] Remove encode/decode overhead at binding (#611) --- channels/binding/websockets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/channels/binding/websockets.py b/channels/binding/websockets.py index 6017150..bf57b9f 100644 --- a/channels/binding/websockets.py +++ b/channels/binding/websockets.py @@ -112,8 +112,7 @@ class WebsocketBinding(Binding): "fields": data, } ] - # TODO: Avoid the JSON roundtrip by using encoder directly? - return list(serializers.deserialize("json", json.dumps(s_data)))[0] + return list(serializers.deserialize("python", s_data))[0] def create(self, data): self._hydrate(None, data).save() From b7ea0b92876a286a938f4d597666542f7194b96c Mon Sep 17 00:00:00 2001 From: Krukov D Date: Mon, 17 Apr 2017 19:21:10 +0300 Subject: [PATCH 687/746] Rename HTTPClient to WSClient (#609) This is what it was actually intended as. HTTP testing methods may follow later. --- channels/test/__init__.py | 1 + channels/test/http.py | 150 +---------------------- channels/test/websocket.py | 146 ++++++++++++++++++++++ docs/testing.rst | 26 ++-- tests/test_binding.py | 20 +-- tests/test_generic.py | 14 +-- tests/{test_http.py => test_wsclient.py} | 6 +- 7 files changed, 186 insertions(+), 177 deletions(-) create mode 100644 channels/test/websocket.py rename tests/{test_http.py => test_wsclient.py} (84%) diff --git a/channels/test/__init__.py b/channels/test/__init__.py index c781e85..f0835ef 100644 --- a/channels/test/__init__.py +++ b/channels/test/__init__.py @@ -1,3 +1,4 @@ from .base import TransactionChannelTestCase, ChannelTestCase, Client, apply_routes # NOQA isort:skip from .http import HttpClient # NOQA isort:skip +from .websocket import WSClient # NOQA isort:skip from .liveserver import ChannelLiveServerTestCase # NOQA isort:skip diff --git a/channels/test/http.py b/channels/test/http.py index d9b439d..f263f9f 100644 --- a/channels/test/http.py +++ b/channels/test/http.py @@ -1,146 +1,8 @@ -import copy -import json +import warnings -import six -from django.apps import apps -from django.conf import settings +warnings.warn( + "test.http.HttpClient is deprecated. Use test.websocket.WSClient", + DeprecationWarning, +) -from django.http.cookie import SimpleCookie - -from ..sessions import session_for_reply_channel -from .base import Client - -json_module = json # alias for using at functions with json kwarg - - -class HttpClient(Client): - """ - Channel http/ws client abstraction that provides easy methods for testing full life cycle of message in channels - with determined reply channel, auth opportunity, cookies, headers and so on - """ - - def __init__(self, **kwargs): - super(HttpClient, self).__init__(**kwargs) - self._session = None - self._headers = {} - self._cookies = {} - self._session_cookie = True - - def set_cookie(self, key, value): - """ - Set cookie - """ - self._cookies[key] = value - - def set_header(self, key, value): - """ - Set header - """ - if key == 'cookie': - raise ValueError('Use set_cookie method for cookie header') - self._headers[key] = value - - def get_cookies(self): - """Return cookies""" - cookies = copy.copy(self._cookies) - if self._session_cookie and apps.is_installed('django.contrib.sessions'): - cookies[settings.SESSION_COOKIE_NAME] = self.session.session_key - return cookies - - @property - def headers(self): - headers = copy.deepcopy(self._headers) - headers.setdefault('cookie', _encoded_cookies(self.get_cookies())) - return headers - - @property - def session(self): - """Session as Lazy property: check that django.contrib.sessions is installed""" - if not apps.is_installed('django.contrib.sessions'): - raise EnvironmentError('Add django.contrib.sessions to the INSTALLED_APPS to use session') - if not self._session: - self._session = session_for_reply_channel(self.reply_channel) - return self._session - - def receive(self, json=True): - """ - Return text content of a message for client channel and decoding it if json kwarg is set - """ - content = super(HttpClient, self).receive() - if content and json and 'text' in content and isinstance(content['text'], six.string_types): - return json_module.loads(content['text']) - return content.get('text', content) if content else None - - def send(self, to, content={}, text=None, path='/'): - """ - Send a message to a channel. - Adds reply_channel name and channel_session to the message. - """ - content = copy.deepcopy(content) - content.setdefault('reply_channel', self.reply_channel) - content.setdefault('path', path) - content.setdefault('headers', self.headers) - text = text or content.get('text', None) - if text is not None: - if not isinstance(text, six.string_types): - content['text'] = json.dumps(text) - else: - content['text'] = text - self.channel_layer.send(to, content) - self._session_cookie = False - - def send_and_consume(self, channel, content={}, text=None, path='/', fail_on_none=True, check_accept=True): - """ - Reproduce full life cycle of the message - """ - self.send(channel, content, text, path) - return self.consume(channel, fail_on_none=fail_on_none, check_accept=check_accept) - - def consume(self, channel, fail_on_none=True, check_accept=True): - result = super(HttpClient, self).consume(channel, fail_on_none=fail_on_none) - if channel == "websocket.connect" and check_accept: - received = self.receive(json=False) - if received != {"accept": True}: - raise AssertionError("Connection rejected: %s != '{accept: True}'" % received) - return result - - def login(self, **credentials): - """ - Returns True if login is possible; False if the provided credentials - are incorrect, or the user is inactive, or if the sessions framework is - not available. - """ - from django.contrib.auth import authenticate - user = authenticate(**credentials) - if user and user.is_active and apps.is_installed('django.contrib.sessions'): - self._login(user) - return True - else: - return False - - def force_login(self, user, backend=None): - if backend is None: - backend = settings.AUTHENTICATION_BACKENDS[0] - user.backend = backend - self._login(user) - - def _login(self, user): - from django.contrib.auth import login - - # Fake http request - request = type('FakeRequest', (object, ), {'session': self.session, 'META': {}}) - login(request, user) - - # Save the session values. - self.session.save() - - -def _encoded_cookies(cookies): - """Encode dict of cookies to ascii string""" - - cookie_encoder = SimpleCookie() - - for k, v in cookies.items(): - cookie_encoder[k] = v - - return cookie_encoder.output(header='', sep=';').encode("ascii") +from .websocket import WSClient as HttpClient # NOQA isort:skip diff --git a/channels/test/websocket.py b/channels/test/websocket.py new file mode 100644 index 0000000..d10f6ae --- /dev/null +++ b/channels/test/websocket.py @@ -0,0 +1,146 @@ +import copy +import json + +import six +from django.apps import apps +from django.conf import settings + +from django.http.cookie import SimpleCookie + +from ..sessions import session_for_reply_channel +from .base import Client + +json_module = json # alias for using at functions with json kwarg + + +class WSClient(Client): + """ + Channel http/ws client abstraction that provides easy methods for testing full life cycle of message in channels + with determined reply channel, auth opportunity, cookies, headers and so on + """ + + def __init__(self, **kwargs): + super(WSClient, self).__init__(**kwargs) + self._session = None + self._headers = {} + self._cookies = {} + self._session_cookie = True + + def set_cookie(self, key, value): + """ + Set cookie + """ + self._cookies[key] = value + + def set_header(self, key, value): + """ + Set header + """ + if key == 'cookie': + raise ValueError('Use set_cookie method for cookie header') + self._headers[key] = value + + def get_cookies(self): + """Return cookies""" + cookies = copy.copy(self._cookies) + if self._session_cookie and apps.is_installed('django.contrib.sessions'): + cookies[settings.SESSION_COOKIE_NAME] = self.session.session_key + return cookies + + @property + def headers(self): + headers = copy.deepcopy(self._headers) + headers.setdefault('cookie', _encoded_cookies(self.get_cookies())) + return headers + + @property + def session(self): + """Session as Lazy property: check that django.contrib.sessions is installed""" + if not apps.is_installed('django.contrib.sessions'): + raise EnvironmentError('Add django.contrib.sessions to the INSTALLED_APPS to use session') + if not self._session: + self._session = session_for_reply_channel(self.reply_channel) + return self._session + + def receive(self, json=True): + """ + Return text content of a message for client channel and decoding it if json kwarg is set + """ + content = super(WSClient, self).receive() + if content and json and 'text' in content and isinstance(content['text'], six.string_types): + return json_module.loads(content['text']) + return content.get('text', content) if content else None + + def send(self, to, content={}, text=None, path='/'): + """ + Send a message to a channel. + Adds reply_channel name and channel_session to the message. + """ + content = copy.deepcopy(content) + content.setdefault('reply_channel', self.reply_channel) + content.setdefault('path', path) + content.setdefault('headers', self.headers) + text = text or content.get('text', None) + if text is not None: + if not isinstance(text, six.string_types): + content['text'] = json.dumps(text) + else: + content['text'] = text + self.channel_layer.send(to, content) + self._session_cookie = False + + def send_and_consume(self, channel, content={}, text=None, path='/', fail_on_none=True, check_accept=True): + """ + Reproduce full life cycle of the message + """ + self.send(channel, content, text, path) + return self.consume(channel, fail_on_none=fail_on_none, check_accept=check_accept) + + def consume(self, channel, fail_on_none=True, check_accept=True): + result = super(WSClient, self).consume(channel, fail_on_none=fail_on_none) + if channel == "websocket.connect" and check_accept: + received = self.receive(json=False) + if received != {"accept": True}: + raise AssertionError("Connection rejected: %s != '{accept: True}'" % received) + return result + + def login(self, **credentials): + """ + Returns True if login is possible; False if the provided credentials + are incorrect, or the user is inactive, or if the sessions framework is + not available. + """ + from django.contrib.auth import authenticate + user = authenticate(**credentials) + if user and user.is_active and apps.is_installed('django.contrib.sessions'): + self._login(user) + return True + else: + return False + + def force_login(self, user, backend=None): + if backend is None: + backend = settings.AUTHENTICATION_BACKENDS[0] + user.backend = backend + self._login(user) + + def _login(self, user): + from django.contrib.auth import login + + # Fake http request + request = type('FakeRequest', (object, ), {'session': self.session, 'META': {}}) + login(request, user) + + # Save the session values. + self.session.save() + + +def _encoded_cookies(cookies): + """Encode dict of cookies to ascii string""" + + cookie_encoder = SimpleCookie() + + for k, v in cookies.items(): + cookie_encoder[k] = v + + return cookie_encoder.output(header='', sep=';').encode("ascii") diff --git a/docs/testing.rst b/docs/testing.rst index 7b2a49d..cbad8ea 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -128,7 +128,7 @@ purpose use ``send_and_consume`` method:: self.assertEqual(client.receive(), {'all is': 'done'}) -You can use ``HttpClient`` for websocket related consumers. It automatically serializes JSON content, +You can use ``WSClient`` for websocket related consumers. It automatically serializes JSON content, manage cookies and headers, give easy access to the session and add ability to authorize your requests. For example:: @@ -146,13 +146,13 @@ For example:: # tests.py from channels import Group - from channels.test import ChannelTestCase, HttpClient + from channels.test import ChannelTestCase, WSClient class RoomsTests(ChannelTestCase): def test_rooms(self): - client = HttpClient() + client = WSClient() user = User.objects.create_user( username='test', email='test@test.com', password='123456') client.login(username='test', password='123456') @@ -181,8 +181,8 @@ For example:: self.assertIsNone(client.receive()) -Instead of ``HttpClient.login`` method with credentials at arguments you -may call ``HttpClient.force_login`` (like at django client) with the user object. +Instead of ``WSClient.login`` method with credentials at arguments you +may call ``WSClient.force_login`` (like at django client) with the user object. ``receive`` method by default trying to deserialize json text content of a message, so if you need to pass decoding use ``receive(json=False)``, like in the example. @@ -196,23 +196,23 @@ want to test your consumers in a more isolate and atomic way, it will be simpler with ``apply_routes`` contextmanager and decorator for your ``ChannelTestCase``. It takes a list of routes that you want to use and overwrites existing routes:: - from channels.test import ChannelTestCase, HttpClient, apply_routes + from channels.test import ChannelTestCase, WSClient, apply_routes class MyTests(ChannelTestCase): def test_myconsumer(self): - client = HttpClient() + client = WSClient() with apply_routes([MyConsumer.as_route(path='/new')]): client.send_and_consume('websocket.connect', '/new') self.assertEqual(client.receive(), {'key': 'value'}) -Test Data binding with ``HttpClient`` +Test Data binding with ``WSClient`` ------------------------------------- As you know data binding in channels works in outbound and inbound ways, -so that ways tests in different ways and ``HttpClient`` and ``apply_routes`` +so that ways tests in different ways and ``WSClient`` and ``apply_routes`` will help to do this. When you testing outbound consumers you need just import your ``Binding`` subclass with specified ``group_names``. At test you can join to one of them, @@ -220,14 +220,14 @@ make some changes with target model and check received message. Lets test ``IntegerValueBinding`` from :doc:`data binding ` with creating:: - from channels.test import ChannelTestCase, HttpClient + from channels.test import ChannelTestCase, WSClient from channels.signals import consumer_finished class TestIntegerValueBinding(ChannelTestCase): def test_outbound_create(self): - # We use HttpClient because of json encoding messages - client = HttpClient() + # We use WSClient because of json encoding messages + client = WSClient() client.join_group("intval-updates") # join outbound binding # create target entity @@ -266,7 +266,7 @@ For example:: with apply_routes([Demultiplexer.as_route(path='/'), route("binding.intval", IntegerValueBinding.consumer)]): - client = HttpClient() + client = WSClient() client.send_and_consume('websocket.connect', path='/') client.send_and_consume('websocket.receive', path='/', text={ 'stream': 'intval', diff --git a/tests/test_binding.py b/tests/test_binding.py index 61056bc..b0775d1 100644 --- a/tests/test_binding.py +++ b/tests/test_binding.py @@ -6,7 +6,7 @@ from channels import route from channels.binding.base import CREATE, DELETE, UPDATE from channels.binding.websockets import WebsocketBinding from channels.generic.websockets import WebsocketDemultiplexer -from channels.test import ChannelTestCase, HttpClient, apply_routes +from channels.test import ChannelTestCase, WSClient, apply_routes from tests import models User = get_user_model() @@ -28,7 +28,7 @@ class TestsBinding(ChannelTestCase): def has_permission(self, user, action, pk): return True - client = HttpClient() + client = WSClient() client.join_group('users') user = User.objects.create(username='test', email='test@test.com') @@ -70,7 +70,7 @@ class TestsBinding(ChannelTestCase): def has_permission(self, user, action, pk): return True - client = HttpClient() + client = WSClient() client.join_group('testuuidmodels') instance = models.TestUUIDModel.objects.create(name='testname') @@ -106,7 +106,7 @@ class TestsBinding(ChannelTestCase): return True with apply_routes([route('test', TestBinding.consumer)]): - client = HttpClient() + client = WSClient() client.join_group('users_exclude') user = User.objects.create(username='test', email='test@test.com') @@ -165,7 +165,7 @@ class TestsBinding(ChannelTestCase): # Make model and clear out pending sends user = User.objects.create(username='test', email='test@test.com') - client = HttpClient() + client = WSClient() client.join_group('users2') user.username = 'test_new' @@ -210,7 +210,7 @@ class TestsBinding(ChannelTestCase): # Make model and clear out pending sends user = User.objects.create(username='test', email='test@test.com') - client = HttpClient() + client = WSClient() client.join_group('users3') user.delete() @@ -254,7 +254,7 @@ class TestsBinding(ChannelTestCase): groups = ['inbound'] with apply_routes([Demultiplexer.as_route(path='/')]): - client = HttpClient() + client = WSClient() client.send_and_consume('websocket.connect', path='/') client.send_and_consume('websocket.receive', path='/', text={ 'stream': 'users', @@ -294,7 +294,7 @@ class TestsBinding(ChannelTestCase): groups = ['inbound'] with apply_routes([Demultiplexer.as_route(path='/')]): - client = HttpClient() + client = WSClient() client.send_and_consume('websocket.connect', path='/') client.send_and_consume('websocket.receive', path='/', text={ 'stream': 'users', @@ -340,7 +340,7 @@ class TestsBinding(ChannelTestCase): groups = ['inbound'] with apply_routes([Demultiplexer.as_route(path='/')]): - client = HttpClient() + client = WSClient() client.send_and_consume('websocket.connect', path='/') client.send_and_consume('websocket.receive', path='/', text={ 'stream': 'users', @@ -372,6 +372,6 @@ class TestsBinding(ChannelTestCase): groups = ['inbound'] with apply_routes([Demultiplexer.as_route(path='/path/(?P\d+)')]): - client = HttpClient() + client = WSClient() consumer = client.send_and_consume('websocket.connect', path='/path/789') self.assertEqual(consumer.kwargs['id'], '789') diff --git a/tests/test_generic.py b/tests/test_generic.py index dd5159e..dc4841c 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -8,7 +8,7 @@ from django.contrib.auth import get_user_model from channels import route_class from channels.exceptions import SendNotAvailableOnDemultiplexer from channels.generic import BaseConsumer, websockets -from channels.test import ChannelTestCase, Client, HttpClient, apply_routes +from channels.test import ChannelTestCase, Client, WSClient, apply_routes @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.cache") @@ -96,7 +96,7 @@ class GenericTests(ChannelTestCase): user_model = get_user_model() user = user_model.objects.create_user(username='test', email='test@test.com', password='123456') - client = HttpClient() + client = WSClient() client.force_login(user) with apply_routes([route_class(WebsocketConsumer, path='/path')]): connect = client.send_and_consume('websocket.connect', {'path': '/path'}) @@ -128,7 +128,7 @@ class GenericTests(ChannelTestCase): self.assertIs(routes[1].consumer, WebsocketConsumer) with apply_routes(routes): - client = HttpClient() + client = WSClient() client.send('websocket.connect', {'path': '/path', 'order': 1}) client.send('websocket.connect', {'path': '/path', 'order': 0}) @@ -180,7 +180,7 @@ class GenericTests(ChannelTestCase): } with apply_routes([route_class(Demultiplexer, path='/path/(?P\d+)')]): - client = HttpClient() + client = WSClient() client.send_and_consume('websocket.connect', path='/path/1') self.assertEqual(client.receive(), { @@ -216,7 +216,7 @@ class GenericTests(ChannelTestCase): } with apply_routes([route_class(Demultiplexer, path='/path/(?P\d+)')]): - client = HttpClient() + client = WSClient() with self.assertRaises(SendNotAvailableOnDemultiplexer): client.send_and_consume('websocket.receive', path='/path/1', text={ @@ -250,7 +250,7 @@ class GenericTests(ChannelTestCase): return json.dumps(lowered) with apply_routes([route_class(WebsocketConsumer, path='/path')]): - client = HttpClient() + client = WSClient() consumer = client.send_and_consume('websocket.receive', path='/path', text={"key": "value"}) self.assertEqual(consumer.content_received, {"KEY": "value"}) @@ -284,7 +284,7 @@ class GenericTests(ChannelTestCase): } with apply_routes([route_class(Demultiplexer, path='/path/(?P\d+)')]): - client = HttpClient() + client = WSClient() client.send_and_consume('websocket.connect', path='/path/1') self.assertEqual(client.receive(), { diff --git a/tests/test_http.py b/tests/test_wsclient.py similarity index 84% rename from tests/test_http.py rename to tests/test_wsclient.py index 0ec9089..9f65326 100644 --- a/tests/test_http.py +++ b/tests/test_wsclient.py @@ -2,12 +2,12 @@ from __future__ import unicode_literals from django.http.cookie import parse_cookie -from channels.test import ChannelTestCase, HttpClient +from channels.test import ChannelTestCase, WSClient -class HttpClientTests(ChannelTestCase): +class WSClientTests(ChannelTestCase): def test_cookies(self): - client = HttpClient() + client = WSClient() client.set_cookie('foo', 'not-bar') client.set_cookie('foo', 'bar') client.set_cookie('qux', 'qu;x') From fa413af0538d796d2e2f92063d19ef1e7753ab99 Mon Sep 17 00:00:00 2001 From: Krukov D Date: Wed, 19 Apr 2017 06:18:28 +0300 Subject: [PATCH 688/746] Improvements for test client (#613) * Tests for httpclient sending content * Testing ordered consumers * Added docs * Added testing for ordering * Added GET params at HttpClient * Remove blank line * Fix py3 bites * Test Client now support ChannelSocketException * Fix flake and isort * Conflict resolution --- channels/test/base.py | 3 ++ channels/test/websocket.py | 28 ++++++++-- docs/testing.rst | 20 ++++++++ tests/test_wsclient.py | 101 ++++++++++++++++++++++++++++++++++++- 4 files changed, 148 insertions(+), 4 deletions(-) diff --git a/channels/test/base.py b/channels/test/base.py index 41a2eff..e3ac34d 100644 --- a/channels/test/base.py +++ b/channels/test/base.py @@ -12,6 +12,7 @@ from django.test.testcases import TestCase, TransactionTestCase from .. import DEFAULT_CHANNEL_LAYER from ..asgi import ChannelLayerWrapper, channel_layers from ..channel import Group +from ..exceptions import ChannelSocketException from ..message import Message from ..routing import Router, include from ..signals import consumer_finished, consumer_started @@ -134,6 +135,8 @@ class Client(object): try: consumer_started.send(sender=self.__class__) return consumer(message, **kwargs) + except ChannelSocketException as e: + e.run(message) finally: # Copy Django's workaround so we don't actually close DB conns consumer_finished.disconnect(close_old_connections) diff --git a/channels/test/websocket.py b/channels/test/websocket.py index d10f6ae..66fe216 100644 --- a/channels/test/websocket.py +++ b/channels/test/websocket.py @@ -20,11 +20,13 @@ class WSClient(Client): """ def __init__(self, **kwargs): + self._ordered = kwargs.pop('ordered', False) super(WSClient, self).__init__(**kwargs) self._session = None self._headers = {} self._cookies = {} self._session_cookie = True + self.order = 0 def set_cookie(self, key, value): """ @@ -76,18 +78,38 @@ class WSClient(Client): Send a message to a channel. Adds reply_channel name and channel_session to the message. """ + if to != 'websocket.connect' and '?' in path: + path = path.split('?')[0] + self.channel_layer.send(to, self._get_content(content, text, path)) + self._session_cookie = False + + def _get_content(self, content={}, text=None, path='/'): content = copy.deepcopy(content) content.setdefault('reply_channel', self.reply_channel) - content.setdefault('path', path) + + if '?' in path: + path, query_string = path.split('?') + content.setdefault('path', path) + content.setdefault('query_string', query_string) + else: + content.setdefault('path', path) + content.setdefault('headers', self.headers) + + if self._ordered: + if 'order' in content: + raise ValueError('Do not use "order" manually with "ordered=True"') + content['order'] = self.order + self.order += 1 + text = text or content.get('text', None) + if text is not None: if not isinstance(text, six.string_types): content['text'] = json.dumps(text) else: content['text'] = text - self.channel_layer.send(to, content) - self._session_cookie = False + return content def send_and_consume(self, channel, content={}, text=None, path='/', fail_on_none=True, check_accept=True): """ diff --git a/docs/testing.rst b/docs/testing.rst index cbad8ea..b3f6369 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -187,6 +187,26 @@ may call ``WSClient.force_login`` (like at django client) with the user object. ``receive`` method by default trying to deserialize json text content of a message, so if you need to pass decoding use ``receive(json=False)``, like in the example. +For testing consumers with ``enforce_ordering`` initialize ``HttpClient`` with ``ordered`` +flag, but if you wanna use your own order don't use it, use content:: + + client = HttpClient(ordered=True) + client.send_and_consume('websocket.receive', text='1', path='/ws') # order = 0 + client.send_and_consume('websocket.receive', text='2', path='/ws') # order = 1 + client.send_and_consume('websocket.receive', text='3', path='/ws') # order = 2 + + # manually + client = HttpClient() + client.send('websocket.receive', content={'order': 0}, text='1') + client.send('websocket.receive', content={'order': 2}, text='2') + client.send('websocket.receive', content={'order': 1}, text='3') + + # calling consume 4 time for `waiting` message with order 1 + client.consume('websocket.receive') + client.consume('websocket.receive') + client.consume('websocket.receive') + client.consume('websocket.receive') + Applying routes --------------- diff --git a/tests/test_wsclient.py b/tests/test_wsclient.py index 9f65326..a8924ca 100644 --- a/tests/test_wsclient.py +++ b/tests/test_wsclient.py @@ -2,7 +2,11 @@ from __future__ import unicode_literals from django.http.cookie import parse_cookie -from channels.test import ChannelTestCase, WSClient +from channels import route +from channels.exceptions import ChannelSocketException +from channels.handler import AsgiRequest +from channels.test import ChannelTestCase, WSClient, apply_routes +from channels.sessions import enforce_ordering class WSClientTests(ChannelTestCase): @@ -22,3 +26,98 @@ class WSClientTests(ChannelTestCase): 'qux': 'qu;x', 'sessionid': client.get_cookies()['sessionid']}, cookie_dict) + + def test_simple_content(self): + client = WSClient() + content = client._get_content(text={'key': 'value'}, path='/my/path') + + self.assertEqual(content['text'], '{"key": "value"}') + self.assertEqual(content['path'], '/my/path') + self.assertTrue('reply_channel' in content) + self.assertTrue('headers' in content) + + def test_path_in_content(self): + client = WSClient() + content = client._get_content(content={'path': '/my_path'}, text={'path': 'hi'}, path='/my/path') + + self.assertEqual(content['text'], '{"path": "hi"}') + self.assertEqual(content['path'], '/my_path') + self.assertTrue('reply_channel' in content) + self.assertTrue('headers' in content) + + def test_session_in_headers(self): + client = WSClient() + content = client._get_content() + self.assertTrue('path' in content) + self.assertEqual(content['path'], '/') + + self.assertTrue('headers' in content) + self.assertTrue('cookie' in content['headers']) + self.assertTrue(b'sessionid' in content['headers']['cookie']) + + def test_ordering_in_content(self): + client = WSClient(ordered=True) + content = client._get_content() + self.assertTrue('order' in content) + self.assertEqual(content['order'], 0) + client.order = 2 + content = client._get_content() + self.assertTrue('order' in content) + self.assertEqual(content['order'], 2) + + def test_ordering(self): + + client = WSClient(ordered=True) + + @enforce_ordering + def consumer(message): + message.reply_channel.send({'text': message['text']}) + + with apply_routes(route('websocket.receive', consumer)): + client.send_and_consume('websocket.receive', text='1') # order = 0 + client.send_and_consume('websocket.receive', text='2') # order = 1 + client.send_and_consume('websocket.receive', text='3') # order = 2 + + self.assertEqual(client.receive(), 1) + self.assertEqual(client.receive(), 2) + self.assertEqual(client.receive(), 3) + + def test_get_params(self): + client = WSClient() + content = client._get_content(path='/my/path?test=1&token=2') + self.assertTrue('path' in content) + self.assertTrue('query_string' in content) + self.assertEqual(content['path'], '/my/path') + self.assertEqual(content['query_string'], 'test=1&token=2') + + def test_get_params_with_consumer(self): + client = WSClient(ordered=True) + + def consumer(message): + message.content['method'] = 'FAKE' + message.reply_channel.send({'text': dict(AsgiRequest(message).GET)}) + + with apply_routes([route('websocket.receive', consumer, path=r'^/test'), + route('websocket.connect', consumer, path=r'^/test')]): + path = '/test?key1=val1&key2=val2&key1=val3' + client.send_and_consume('websocket.connect', path=path, check_accept=False) + self.assertDictEqual(client.receive(), {'key2': ['val2'], 'key1': ['val1', 'val3']}) + + client.send_and_consume('websocket.receive', path=path) + self.assertDictEqual(client.receive(), {}) + + def test_channel_socket_exception(self): + + class MyChannelSocketException(ChannelSocketException): + + def run(self, message): + message.reply_channel.send({'text': 'error'}) + + def consumer(message): + raise MyChannelSocketException + + client = WSClient() + with apply_routes(route('websocket.receive', consumer)): + client.send_and_consume('websocket.receive') + + self.assertEqual(client.receive(json=False), 'error') From 364d1e45b4c64ef86b25505eafdbc0393f4f26e2 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 25 Apr 2017 13:43:34 -0700 Subject: [PATCH 689/746] Remove last websocket.send enforcing references --- docs/asgi/www.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/asgi/www.rst b/docs/asgi/www.rst index d81fcde..cc5c2d5 100644 --- a/docs/asgi/www.rst +++ b/docs/asgi/www.rst @@ -262,7 +262,7 @@ Channel: ``websocket.connect`` Keys: -* ``reply_channel``: Channel name for sending data, start with ``websocket.send!`` +* ``reply_channel``: Channel name for sending data * ``scheme``: Unicode string URL scheme portion (likely ``ws`` or ``wss``). Optional (but must not be empty), default is ``ws``. @@ -304,7 +304,7 @@ Channel: ``websocket.receive`` Keys: -* ``reply_channel``: Channel name for sending data, starting with ``websocket.send!`` +* ``reply_channel``: Channel name for sending data * ``path``: Path sent during ``connect``, sent to make routing easier for apps. @@ -332,8 +332,8 @@ Channel: ``websocket.disconnect`` Keys: -* ``reply_channel``: Channel name that was used for sending data, starting - with ``websocket.send!``. Cannot be used to send at this point; provided +* ``reply_channel``: Channel name that was used for sending data. + Cannot be used to send at this point; provided as a way to identify the connection only. * ``code``: The WebSocket close code (integer), as per the WebSocket spec. From 004b34c67dbbb1c9601443d10cb26e0eb7b58deb Mon Sep 17 00:00:00 2001 From: Thomas Steen Rasmussen Date: Fri, 28 Apr 2017 02:36:16 +0200 Subject: [PATCH 690/746] Decode utf-8 QUERY_STRING when relevant (#623) Fixes #622. --- channels/handler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/channels/handler.py b/channels/handler.py index cb1d2e6..f6bd821 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -57,9 +57,13 @@ class AsgiRequest(http.HttpRequest): self.path_info = self.path # HTTP basics self.method = self.message['method'].upper() + # fix https://github.com/django/channels/issues/622 + query_string = self.message.get('query_string', '') + if isinstance(query_string, bytes): + query_string = query_string.decode('utf-8') self.META = { "REQUEST_METHOD": self.method, - "QUERY_STRING": self.message.get('query_string', ''), + "QUERY_STRING": query_string, "SCRIPT_NAME": self.script_name, "PATH_INFO": self.path_info, # Old code will need these for a while From 50f0b98d43a71d37f478de75f45abfb315883cf6 Mon Sep 17 00:00:00 2001 From: Maik Hoepfel Date: Fri, 28 Apr 2017 23:44:05 +0200 Subject: [PATCH 691/746] Two minor spec changes (#625) * Remove mention of fixed reply channel name Per the recent changes, the reply channel name is now at the discretion of the server. * Clarification on bytes/text values being empty This commit clarifies that the bytes/text values need to be non-empty (so not None/'') for them to be treated as present. * HTTP Response headers are optional During discussion with Andrew, we noted that they are optional, and that that makes good sense. So this commit updates the spec to reflect reality. --- docs/asgi/www.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/asgi/www.rst b/docs/asgi/www.rst index cc5c2d5..f0a9f55 100644 --- a/docs/asgi/www.rst +++ b/docs/asgi/www.rst @@ -49,8 +49,7 @@ Channel: ``http.request`` Keys: -* ``reply_channel``: Channel name for responses and server pushes, starting with - ``http.response!`` +* ``reply_channel``: Channel name for responses and server pushes. * ``http_version``: Unicode string, one of ``1.0``, ``1.1`` or ``2``. @@ -137,7 +136,7 @@ Keys: * ``headers``: A list of ``[name, value]`` lists, where ``name`` is the byte string header name, and ``value`` is the byte string header value. Order must be preserved in the HTTP response. Header names - must be lowercased. + must be lowercased. Optional, defaults to an empty list. * ``content``: Byte string of HTTP body content. Optional, defaults to empty string. @@ -357,13 +356,14 @@ message: * If ``accept`` is ``True``, accept the connection (and send any data provided). * If ``accept`` is ``False``, reject the connection and do nothing else. If ``bytes`` or ``text`` were also present they must be ignored. -* If ``bytes`` or ``text`` is present, accept the connection and send the data. +* If ``bytes`` or ``text`` is present and contains a non-empty value, + accept the connection and send the data. * If ``close`` is ``True`` or a positive integer, reject the connection. If - ``bytes`` or ``text`` is also set, it should accept the connection, send the - frame, then immediately close the connection. Note that any close code integer - sent is ignored, as connections are rejected with HTTP's ``403 Forbidden``, - unless data is also sent, in which case a full WebSocket close is done with - the provided code. + ``bytes`` or ``text`` is also set and not empty, it should accept the + connection, send the frame, then immediately close the connection. + Note that any close code integer sent is ignored, as connections are + rejected with HTTP's ``403 Forbidden``, unless data is also sent, in which + case a full WebSocket close is done with the provided code. If received while the connection is established: From 4c2b8da46383216b3b8477226a77241fff01e446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steind=C3=B3r?= Date: Fri, 19 May 2017 21:49:14 +0000 Subject: [PATCH 692/746] Javascript syntax highlighting for documentation (#636) --- docs/getting-started.rst | 12 +++++++++--- docs/javascript.rst | 20 +++++++++++++++----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 95ebf4d..3424948 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -132,7 +132,9 @@ Let's test it! Run ``runserver``, open a browser, navigate to a page on the serv (you can't use any page's console because of origin restrictions), and put the following into the JavaScript console to open a WebSocket and send some data down it (you might need to change the socket address if you're using a -development VM or similar):: +development VM or similar) + +.. code-block:: javascript // Note that the path doesn't matter for routing; any WebSocket // connection gets bumped over to WebSocket consumers @@ -246,7 +248,9 @@ views. With all that code, you now have a working set of a logic for a chat server. Test time! Run ``runserver``, open a browser and use that same JavaScript -code in the developer console as before:: +code in the developer console as before + +.. code-block:: javascript // Note that the path doesn't matter right now; any WebSocket // connection gets bumped over to WebSocket consumers @@ -508,7 +512,9 @@ chat to people with the same first letter of their username:: If you're just using ``runserver`` (and so Daphne), you can just connect and your cookies should transfer your auth over. If you were running WebSockets on a separate domain, you'd have to remember to provide the -Django session ID as part of the URL, like this:: +Django session ID as part of the URL, like this + +.. code-block:: javascript socket = new WebSocket("ws://127.0.0.1:9000/?session_key=abcdefg"); diff --git a/docs/javascript.rst b/docs/javascript.rst index b7e2f67..488a4c3 100644 --- a/docs/javascript.rst +++ b/docs/javascript.rst @@ -20,7 +20,9 @@ The library is deliberately quite low-level and generic; it's designed to be compatible with any JavaScript code or framework, so you can build more specific integration on top of it. -To process messages:: +To process messages + +.. code-block:: javascript const webSocketBridge = new channels.WebSocketBridge(); webSocketBridge.connect('/ws/'); @@ -28,11 +30,15 @@ To process messages:: console.log(action, stream); }); -To send messages, use the `send` method:: +To send messages, use the `send` method + +.. code-block:: javascript webSocketBridge.send({prop1: 'value1', prop2: 'value1'}); -To demultiplex specific streams:: +To demultiplex specific streams + +.. code-block:: javascript webSocketBridge.connect(); webSocketBridge.listen('/ws/'); @@ -43,11 +49,15 @@ To demultiplex specific streams:: console.info(action, stream); }); -To send a message to a specific stream:: +To send a message to a specific stream + +.. code-block:: javascript webSocketBridge.stream('mystream').send({prop1: 'value1', prop2: 'value1'}) -The `WebSocketBridge` instance exposes the underlaying `ReconnectingWebSocket` as the `socket` property. You can use this property to add any custom behavior. For example:: +The `WebSocketBridge` instance exposes the underlaying `ReconnectingWebSocket` as the `socket` property. You can use this property to add any custom behavior. For example + +.. code-block:: javascript webSocketBridge.socket.addEventListener('open', function() { console.log("Connected to WebSocket"); From 863e298923230611e38ff3e31766ce7f12329833 Mon Sep 17 00:00:00 2001 From: Stephen McDonald Date: Thu, 25 May 2017 17:40:18 +1000 Subject: [PATCH 693/746] Fixed typo (#641) --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index dcd4b56..c5b62ac 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,7 +1,7 @@ Contributing to Channels ======================== -As an open source project, Channels welcomes contributions of many forms.By participating in this project, you +As an open source project, Channels welcomes contributions of many forms. By participating in this project, you agree to abide by the Django `code of conduct `_. Examples of contributions include: From 7cd4fb2b424a3d02f6deea93443220746ef67992 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 25 May 2017 10:21:07 -0700 Subject: [PATCH 694/746] Clarify receive blocking behaviour --- docs/asgi.rst | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/docs/asgi.rst b/docs/asgi.rst index 1b00642..de0803c 100644 --- a/docs/asgi.rst +++ b/docs/asgi.rst @@ -331,7 +331,8 @@ A *channel layer* must provide an object with these attributes * ``receive(channels, block=False)``, a callable that takes a list of channel names as unicode strings, and returns with either ``(None, None)`` or ``(channel, message)`` if a message is available. If ``block`` is True, then - it will not return until after a built-in timeout or a message arrives; if + it will not return a message arrives (or optionally, a built-in timeout, + but it is valid to block forever if there are no messages); if ``block`` is false, it will always return immediately. It is perfectly valid to ignore ``block`` and always return immediately, or after a delay; ``block`` means that the call can take as long as it likes before returning @@ -409,21 +410,15 @@ A channel layer implementing the ``twisted`` extension must also provide: * ``receive_twisted(channels)``, a function that behaves like ``receive`` but that returns a Twisted Deferred that eventually returns either ``(channel, message)`` or ``(None, None)``. It is not possible - to run it in nonblocking mode; use the normal ``receive`` for that. The - channel layer must be able to deal with this function being called from - many different places in a codebase simultaneously - likely once from each - Twisted Protocol instance - and so it is recommended that implementations - use internal connection pooling and call merging or similar. + to run it in nonblocking mode; use the normal ``receive`` for that. -A channel layer implementing the ``asyncio`` extension must also provide: +A channel layer implementing the ``async`` extension must also provide: -* ``receive_asyncio(channels)``, a function that behaves +* ``receive_async(channels)``, a function that behaves like ``receive`` but that fulfills the asyncio coroutine contract to block until either a result is available or an internal timeout is reached - and ``(None, None)`` is returned. The channel layer must be able to deal - with this function being called from many different places in a codebase - simultaneously, and so it is recommended that implementations - use internal connection pooling and call merging or similar. + and ``(None, None)`` is returned. It is not possible + to run it in nonblocking mode; use the normal ``receive`` for that. Channel Semantics ----------------- From f2b39c33e6e3f776984bf71cb753147d0f237c2a Mon Sep 17 00:00:00 2001 From: AlexejStukov Date: Mon, 29 May 2017 19:42:51 +0200 Subject: [PATCH 695/746] fixed WorkerGroup's sigterm_handler (#649) --- channels/worker.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/channels/worker.py b/channels/worker.py index 213501f..1eacd67 100644 --- a/channels/worker.py +++ b/channels/worker.py @@ -164,11 +164,8 @@ class WorkerGroup(Worker): self.workers = [Worker(*args, **kwargs) for ii in range(n_threads)] def sigterm_handler(self, signo, stack_frame): - self.termed = True - for wkr in self.workers: - wkr.termed = True - logger.info("Shutdown signal received while busy, waiting for " - "loop termination") + logger.info("Shutdown signal received by WorkerGroup, terminating immediately.") + sys.exit(0) def ready(self): super(WorkerGroup, self).ready() @@ -182,8 +179,6 @@ class WorkerGroup(Worker): self.threads = [threading.Thread(target=self.workers[ii].run) for ii in range(len(self.workers))] for t in self.threads: + t.daemon = True t.start() super(WorkerGroup, self).run() - # Join threads once completed. - for t in self.threads: - t.join() From 34a047a6ffb65ea7277466ab4a1a2c6e4ed67847 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 29 May 2017 11:07:03 -0700 Subject: [PATCH 696/746] Fixed #643: Add retry and limit to PendingMessageStore. This won't fix all backlog issues, but will be sufficient to smooth over bumps. --- channels/message.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/channels/message.py b/channels/message.py index f8001d8..1564977 100644 --- a/channels/message.py +++ b/channels/message.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import copy +import time import threading from .channel import Channel @@ -67,10 +68,17 @@ class PendingMessageStore(object): """ Singleton object used for storing pending messages that should be sent to a channel or group when a consumer finishes. + + Will retry when it sees ChannelFull up to a limit; if you want more control + over this, change to `immediately=True` in your send method and handle it + yourself. """ threadlocal = threading.local() + retry_time = 2 # seconds + retry_interval = 0.2 # seconds + def prepare(self, **kwargs): """ Sets the message store up to receive messages. @@ -90,7 +98,24 @@ class PendingMessageStore(object): def send_and_flush(self, **kwargs): for sender, message in getattr(self.threadlocal, "messages", []): - sender.send(message, immediately=True) + # Loop until the retry time limit is hit + started = time.time() + while time.time() - started < self.retry_time: + try: + sender.send(message, immediately=True) + except sender.channel_layer.ChannelFull: + time.sleep(self.retry_interval) + continue + else: + break + # If we didn't break out, we failed to send, so do a nice exception + else: + raise RuntimeError( + "Failed to send queued message to %s after retrying for %.2fs.\n" + "You need to increase the consumption rate on this channel, its capacity,\n" + "or handle the ChannelFull exception yourself after adding\n" + "immediately=True to send()." % (sender, self.retry_time) + ) delattr(self.threadlocal, "messages") From 601056f71253de76877076bde7c0e28f00207bc7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 29 May 2017 11:39:15 -0700 Subject: [PATCH 697/746] Fix sigterm handler test --- tests/test_worker.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_worker.py b/tests/test_worker.py index 345b287..30e924a 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -132,6 +132,5 @@ class WorkerGroupTests(ChannelTestCase): t = threading.Thread(target=wkr.run) t.start() threads.append(t) - self.worker.sigterm_handler(None, None) - for t in threads: - t.join() + with self.assertRaises(SystemExit): + self.worker.sigterm_handler(None, None) From e5c91a1299f5af1a5219cee0aeb8745b76c2a6f4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 30 May 2017 00:07:05 -0700 Subject: [PATCH 698/746] Remove worker group tests for now as they do not account for threading --- tests/test_worker.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/tests/test_worker.py b/tests/test_worker.py index 30e924a..d2668da 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -96,41 +96,3 @@ class WorkerTests(ChannelTestCase): worker.run() self.assertEqual(consumer.call_count, 1) self.assertEqual(channel_layer.send.call_count, 0) - - -class WorkerGroupTests(ChannelTestCase): - """ - Test threaded workers. - """ - - def setUp(self): - self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] - self.worker = WorkerGroup(self.channel_layer, n_threads=4) - self.subworkers = self.worker.workers - - def test_subworkers_created(self): - self.assertEqual(len(self.subworkers), 3) - - def test_subworkers_no_sigterm(self): - for wrk in self.subworkers: - self.assertFalse(wrk.signal_handlers) - - def test_ready_signals_sent(self): - self.in_signal = 0 - - def handle_signal(sender, *args, **kwargs): - self.in_signal += 1 - - worker_ready.connect(handle_signal) - WorkerGroup(self.channel_layer, n_threads=4) - self.worker.ready() - self.assertEqual(self.in_signal, 4) - - def test_sigterm_handler(self): - threads = [] - for wkr in self.subworkers: - t = threading.Thread(target=wkr.run) - t.start() - threads.append(t) - with self.assertRaises(SystemExit): - self.worker.sigterm_handler(None, None) From 16612ec126c24c1bdc2d71557215a8b63d5f778f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 30 May 2017 00:11:34 -0700 Subject: [PATCH 699/746] Remove JS tests from CI for now --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8e399b7..0cb25c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,6 @@ install: script: - python runtests.py - - cd js_client && npm install --progress=false && npm test && cd .. +# - cd js_client && npm install --progress=false && npm test && cd .. - flake8 - isort --check-only --recursive channels From 93d75f242c4934e89ee0808cccf9cd7b4cb70a6f Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 30 May 2017 00:18:27 -0700 Subject: [PATCH 700/746] Remove unused imports --- tests/test_worker.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_worker.py b/tests/test_worker.py index d2668da..a8975fd 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -1,13 +1,10 @@ from __future__ import unicode_literals -import threading - from channels import DEFAULT_CHANNEL_LAYER, Channel, route from channels.asgi import channel_layers from channels.exceptions import ConsumeLater -from channels.signals import worker_ready from channels.test import ChannelTestCase -from channels.worker import Worker, WorkerGroup +from channels.worker import Worker try: from unittest import mock From b6a115e431ddf58f6c4878dbbd23c19d16bf447b Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 30 May 2017 10:10:06 -0700 Subject: [PATCH 701/746] Sort imports in message.py --- channels/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/message.py b/channels/message.py index 1564977..48e4ebe 100644 --- a/channels/message.py +++ b/channels/message.py @@ -1,8 +1,8 @@ from __future__ import unicode_literals import copy -import time import threading +import time from .channel import Channel from .signals import consumer_finished, consumer_started From cd34f650a524e3af288958045b78935805e5d6fa Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 31 May 2017 10:00:08 -0700 Subject: [PATCH 702/746] Fix WWW header paragraph in spec --- docs/asgi/www.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/asgi/www.rst b/docs/asgi/www.rst index f0a9f55..c80027c 100644 --- a/docs/asgi/www.rst +++ b/docs/asgi/www.rst @@ -35,8 +35,10 @@ However, RFC 7230 and RFC 6265 make it clear that this rule does not apply to the various headers used by HTTP cookies (``Cookie`` and ``Set-Cookie``). The ``Cookie`` header must only be sent once by a user-agent, but the ``Set-Cookie`` header may appear repeatedly and cannot be joined by commas. -For this reason, we can safely make the request ``headers`` a ``dict``, but -the response ``headers`` must be sent as a list of tuples, which matches WSGI. +The ASGI design decision is to transport both request and response headers as +lists of 2-element ``[name, value]`` lists and preserve headers exactly as they +were provided. + Request ''''''' From 4e8b02955c9198322d64d204c6bb4d00cfa16ff8 Mon Sep 17 00:00:00 2001 From: ElRoberto538 Date: Tue, 6 Jun 2017 05:17:27 +1200 Subject: [PATCH 703/746] Added new argument to runserver to set autobahn handshake timeout (#652) Allows setting the new option introduced in Daphne. --- channels/exceptions.py | 1 + channels/handler.py | 1 - channels/management/commands/runserver.py | 7 ++++++- channels/package_checks.py | 1 - channels/test/websocket.py | 1 - tests/test_management.py | 4 ++++ 6 files changed, 11 insertions(+), 4 deletions(-) diff --git a/channels/exceptions.py b/channels/exceptions.py index 502358e..2c8a359 100644 --- a/channels/exceptions.py +++ b/channels/exceptions.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals + import six diff --git a/channels/handler.py b/channels/handler.py index f6bd821..cd44f45 100644 --- a/channels/handler.py +++ b/channels/handler.py @@ -12,7 +12,6 @@ from django import http from django.conf import settings from django.core import signals from django.core.handlers import base - from django.http import FileResponse, HttpResponse, HttpResponseServerError from django.utils import six from django.utils.functional import cached_property diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 7caccbe..037d7fe 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -25,12 +25,16 @@ class Command(RunserverCommand): parser.add_argument('--noasgi', action='store_false', dest='use_asgi', default=True, help='Run the old WSGI-based runserver rather than the ASGI-based one') parser.add_argument('--http_timeout', action='store', dest='http_timeout', type=int, default=60, - help='Specify the daphane http_timeout interval in seconds (default: 60)') + help='Specify the daphne http_timeout interval in seconds (default: 60)') + parser.add_argument('--websocket_handshake_timeout', action='store', dest='websocket_handshake_timeout', + type=int, default=5, + help='Specify the daphne websocket_handshake_timeout interval in seconds (default: 5)') def handle(self, *args, **options): self.verbosity = options.get("verbosity", 1) self.logger = setup_logger('django.channels', self.verbosity) self.http_timeout = options.get("http_timeout", 60) + self.websocket_handshake_timeout = options.get("websocket_handshake_timeout", 5) super(Command, self).handle(*args, **options) def inner_run(self, *args, **options): @@ -88,6 +92,7 @@ class Command(RunserverCommand): http_timeout=self.http_timeout, ws_protocols=getattr(settings, 'CHANNELS_WS_PROTOCOLS', None), root_path=getattr(settings, 'FORCE_SCRIPT_NAME', '') or '', + websocket_handshake_timeout=self.websocket_handshake_timeout, ).run() self.logger.debug("Daphne exited") except KeyboardInterrupt: diff --git a/channels/package_checks.py b/channels/package_checks.py index 6e4bbf8..bf72ec0 100644 --- a/channels/package_checks.py +++ b/channels/package_checks.py @@ -1,7 +1,6 @@ import importlib from distutils.version import StrictVersion - required_versions = { "asgi_rabbitmq": "0.4.0", "asgi_redis": "1.2.0", diff --git a/channels/test/websocket.py b/channels/test/websocket.py index 66fe216..4b762ce 100644 --- a/channels/test/websocket.py +++ b/channels/test/websocket.py @@ -4,7 +4,6 @@ import json import six from django.apps import apps from django.conf import settings - from django.http.cookie import SimpleCookie from ..sessions import session_for_reply_channel diff --git a/tests/test_management.py b/tests/test_management.py index 89b0bc1..ab2b852 100644 --- a/tests/test_management.py +++ b/tests/test_management.py @@ -138,6 +138,7 @@ class RunServerTests(TestCase): channel_layer=mock.ANY, ws_protocols=None, root_path='', + websocket_handshake_timeout=5, ) @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) @@ -158,6 +159,7 @@ class RunServerTests(TestCase): channel_layer=mock.ANY, ws_protocols=None, root_path='', + websocket_handshake_timeout=5, ) call_command('runserver', '--noreload', 'localhost:8001') @@ -169,6 +171,7 @@ class RunServerTests(TestCase): channel_layer=mock.ANY, ws_protocols=None, root_path='', + websocket_handshake_timeout=5, ) self.assertFalse( @@ -192,6 +195,7 @@ class RunServerTests(TestCase): channel_layer=mock.ANY, ws_protocols=None, root_path='', + websocket_handshake_timeout=5, ) self.assertFalse( mocked_worker.called, From 4450e0607d3fd83517b0139bfde77f3e33105b86 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Wed, 7 Jun 2017 16:56:06 -0700 Subject: [PATCH 704/746] HTTP/2 support is currently available in Daphne (#663) --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 2b2e92a..d347da2 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -148,7 +148,7 @@ features, you'll need to run a native ASGI interface server, as the WSGI specification has no support for running these kinds of requests concurrently. We ship with an interface server that we recommend you use called `Daphne `_; it supports WebSockets, -long-poll HTTP requests, HTTP/2 *(soon)* and performs quite well. +long-poll HTTP requests, HTTP/2 and performs quite well. You can just keep running your Django code as a WSGI app if you like, behind something like uwsgi or gunicorn; this won't let you support WebSockets, though, From 4d524f2d003310e017f407b4977f0eb13b13d756 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Thu, 8 Jun 2017 00:17:33 -0700 Subject: [PATCH 705/746] Add protocol/server_cls attributes to runserver for extensibility (#661) Makes it easier to extend the runserver command with custom behaviour. --- channels/management/commands/runserver.py | 7 +++++-- tests/test_management.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 037d7fe..44afd84 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -17,6 +17,8 @@ from channels.worker import Worker class Command(RunserverCommand): + protocol = 'http' + server_cls = Server def add_arguments(self, parser): super(Command, self).add_arguments(parser) @@ -58,12 +60,13 @@ class Command(RunserverCommand): self.stdout.write(now) self.stdout.write(( "Django version %(version)s, using settings %(settings)r\n" - "Starting Channels development server at http://%(addr)s:%(port)s/\n" + "Starting Channels development server at %(protocol)s://%(addr)s:%(port)s/\n" "Channel layer %(layer)s\n" "Quit the server with %(quit_command)s.\n" ) % { "version": self.get_version(), "settings": settings.SETTINGS_MODULE, + "protocol": self.protocol, "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr, "port": self.port, "quit_command": quit_command, @@ -84,7 +87,7 @@ class Command(RunserverCommand): # build the endpoint description string from host/port options endpoints = build_endpoint_description_strings(host=self.addr, port=self.port) try: - Server( + self.server_cls( channel_layer=self.channel_layer, endpoints=endpoints, signal_handlers=not options['use_reloader'], diff --git a/tests/test_management.py b/tests/test_management.py index ab2b852..61900d7 100644 --- a/tests/test_management.py +++ b/tests/test_management.py @@ -122,7 +122,7 @@ class RunServerTests(TestCase): channels.log.handler = logging.StreamHandler(self.stream) @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) - @mock.patch('channels.management.commands.runserver.Server') + @mock.patch('channels.management.commands.runserver.Command.server_cls') @mock.patch('channels.management.commands.runworker.Worker') def test_runserver_basic(self, mocked_worker, mocked_server, mock_stdout): # Django's autoreload util uses threads and this is not needed @@ -142,7 +142,7 @@ class RunServerTests(TestCase): ) @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) - @mock.patch('channels.management.commands.runserver.Server') + @mock.patch('channels.management.commands.runserver.Command.server_cls') @mock.patch('channels.management.commands.runworker.Worker') def test_runserver_debug(self, mocked_worker, mocked_server, mock_stdout): """ @@ -180,7 +180,7 @@ class RunServerTests(TestCase): ) @mock.patch('channels.management.commands.runserver.sys.stdout', new_callable=StringIO) - @mock.patch('channels.management.commands.runserver.Server') + @mock.patch('channels.management.commands.runserver.Command.server_cls') @mock.patch('channels.management.commands.runworker.Worker') def test_runserver_noworker(self, mocked_worker, mocked_server, mock_stdout): ''' From f912637d73340b8ad999849173e137bb92b889c7 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 15 Jun 2017 21:40:25 +0800 Subject: [PATCH 706/746] Releasing 1.1.4 --- CHANGELOG.txt | 21 +++++++++++++++++++++ channels/__init__.py | 2 +- docs/releases/1.1.4.rst | 37 +++++++++++++++++++++++++++++++++++++ docs/releases/index.rst | 1 + js_client/package.json | 2 +- 5 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 docs/releases/1.1.4.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 023ef01..4f98a68 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,26 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases +1.1.4 (2017-06-15) +------------------ + +* Pending messages correctly handle retries in backlog situations + +* Workers in threading mode now respond to ctrl-C and gracefully exit. + +* ``request.meta['QUERY_STRING']`` is now correctly encoded at all times. + +* Test client improvements + +* ``ChannelServerLiveTestCase`` added, allows an equivalent of the Django + ``LiveTestCase``. + +* Decorator added to check ``Origin`` headers (``allowed_hosts_only``) + +* New ``TEST_CONFIG`` setting in ``CHANNEL_LAYERS`` that allows varying of + the channel layer for tests (e.g. using a different Redis install) + + 1.1.3 (2017-04-05) ------------------ @@ -9,6 +29,7 @@ https://channels.readthedocs.io/en/latest/releases * ASGI channel layer versions are now explicitly checked for version compatability + 1.1.2 (2017-04-01) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index b6451b1..9952fe4 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.3" +__version__ = "1.1.4" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/releases/1.1.4.rst b/docs/releases/1.1.4.rst new file mode 100644 index 0000000..c57ff61 --- /dev/null +++ b/docs/releases/1.1.4.rst @@ -0,0 +1,37 @@ +1.1.4 Release Notes +=================== + +Channels 1.1.4 is a bugfix release for the 1.1 series, released on +June 15th, 2017. + + +Major Changes +------------- + +None. + + +Minor Changes & Bugfixes +------------------------ + +* Pending messages correctly handle retries in backlog situations + +* Workers in threading mode now respond to ctrl-C and gracefully exit. + +* ``request.meta['QUERY_STRING']`` is now correctly encoded at all times. + +* Test client improvements + +* ``ChannelServerLiveTestCase`` added, allows an equivalent of the Django + ``LiveTestCase``. + +* Decorator added to check ``Origin`` headers (``allowed_hosts_only``) + +* New ``TEST_CONFIG`` setting in ``CHANNEL_LAYERS`` that allows varying of + the channel layer for tests (e.g. using a different Redis install) + + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index a5740cc..d9d2d64 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -12,3 +12,4 @@ Release Notes 1.1.1 1.1.2 1.1.3 + 1.1.4 diff --git a/js_client/package.json b/js_client/package.json index 26f7d77..a259ff2 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -1,6 +1,6 @@ { "name": "django-channels", - "version": "1.1.3", + "version": "1.1.4", "description": "", "repository": { "type": "git", From 298aeee5b1ba52f81d94eb2c5f0e296c5f202e8e Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 16 Jun 2017 10:40:02 +0800 Subject: [PATCH 707/746] Bump Daphne dependency to a supported version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2163a6b..e8f4159 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( install_requires=[ 'Django>=1.8', 'asgiref~=1.1', - 'daphne>=1.2.0', + 'daphne~=1.3', ], extras_require={ 'tests': [ From 94b3f3563bbc07a49a3f81b362317a2cfc12d82a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 16 Jun 2017 10:40:12 +0800 Subject: [PATCH 708/746] Releasing 1.1.5 --- CHANGELOG.txt | 6 ++++++ channels/__init__.py | 2 +- docs/releases/1.1.5.rst | 22 ++++++++++++++++++++++ docs/releases/index.rst | 1 + js_client/package.json | 2 +- 5 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 docs/releases/1.1.5.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 4f98a68..c9d77d7 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,12 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases +1.1.5 (2017-06-16) +------------------ + +* The Daphne dependency requirement was bumped to 1.3.0. + + 1.1.4 (2017-06-15) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index 9952fe4..db5cf07 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.4" +__version__ = "1.1.5" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/releases/1.1.5.rst b/docs/releases/1.1.5.rst new file mode 100644 index 0000000..3c48cdf --- /dev/null +++ b/docs/releases/1.1.5.rst @@ -0,0 +1,22 @@ +1.1.5 Release Notes +=================== + +Channels 1.1.5 is a packaging release for the 1.1 series, released on +June 16th, 2017. + + +Major Changes +------------- + +None. + + +Minor Changes & Bugfixes +------------------------ + +* The Daphne dependency requirement was bumped to 1.3.0. + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index d9d2d64..28ec006 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -13,3 +13,4 @@ Release Notes 1.1.2 1.1.3 1.1.4 + 1.1.5 diff --git a/js_client/package.json b/js_client/package.json index a259ff2..fab7102 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -1,6 +1,6 @@ { "name": "django-channels", - "version": "1.1.4", + "version": "1.1.5", "description": "", "repository": { "type": "git", From 54fa7be874edd77025e2ba1493b4f3946f68689e Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Mon, 19 Jun 2017 00:32:45 +0300 Subject: [PATCH 709/746] Define TEST_CONFIG for testproject redis layer. (#674) --- testproject/testproject/settings/channels_redis.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testproject/testproject/settings/channels_redis.py b/testproject/testproject/settings/channels_redis.py index 4ab6439..2068548 100644 --- a/testproject/testproject/settings/channels_redis.py +++ b/testproject/testproject/settings/channels_redis.py @@ -11,6 +11,9 @@ CHANNEL_LAYERS = { "ROUTING": "testproject.urls.channel_routing", "CONFIG": { "hosts": [os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')], - } + }, + "TEST_CONFIG": { + "hosts": [os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')], + }, }, } From e2444308ffa756f516e4b28983058d710a78c4e5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 18 Jun 2017 14:41:05 -0700 Subject: [PATCH 710/746] Fixed #672: Need to reset server_cls to hand back to normal Django runserver --- channels/management/commands/runserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 44afd84..9770288 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -42,6 +42,7 @@ class Command(RunserverCommand): def inner_run(self, *args, **options): # Maybe they want the wsgi one? if not options.get("use_asgi", True) or DEFAULT_CHANNEL_LAYER not in channel_layers: + self.server_cls = RunserverCommand.server_cls return RunserverCommand.inner_run(self, *args, **options) # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] From a6a2714a9b844b77bcae623c461793ffb990a387 Mon Sep 17 00:00:00 2001 From: Sergio Date: Wed, 21 Jun 2017 15:45:07 -0400 Subject: [PATCH 711/746] Add note on check_accept to testing documentation (#676) When writing tests, sometimes consumers are expected to reject connections, and while there is a check_accept parameter on send_and_consume, it's not documented. This commit adds a note on the use of the parameter to the docs. --- docs/testing.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/testing.rst b/docs/testing.rst index b3f6369..32884c0 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -126,7 +126,8 @@ purpose use ``send_and_consume`` method:: client = Client() client.send_and_consume('my_internal_channel', {'value': 'my_value'}) self.assertEqual(client.receive(), {'all is': 'done'}) - + +*Note: if testing consumers that are expected to close the connection when consuming, set the ``check_accept`` parameter to False on ``send_and_consume``.* You can use ``WSClient`` for websocket related consumers. It automatically serializes JSON content, manage cookies and headers, give easy access to the session and add ability to authorize your requests. From 63b79db3522e543509e6b1402d234c6775595070 Mon Sep 17 00:00:00 2001 From: Sergio Date: Thu, 22 Jun 2017 13:40:30 -0400 Subject: [PATCH 712/746] Link to interface server explanation in concepts (#678) The docs did not link to the interface server explanation. Relates to issue #677 --- docs/concepts.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index f942922..26ba9e8 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -87,8 +87,7 @@ one, it runs the relevant consumer. So rather than running in just a single process tied to a WSGI server, Django runs in three separate layers: * Interface servers, which communicate between Django and the outside world. - This includes a WSGI adapter as well as a separate WebSocket server - we'll - cover this later. + This includes a WSGI adapter as well as a separate WebSocket server - this is explained and covered in :ref:`run-interface-servers`. * The channel backend, which is a combination of pluggable Python code and a datastore (e.g. Redis, or a shared memory segment) responsible for From 1551cb05631ec87de28ec5736c880531aa7d905f Mon Sep 17 00:00:00 2001 From: Sergio Date: Thu, 22 Jun 2017 20:53:00 -0400 Subject: [PATCH 713/746] Improve docs consistency on query strings (#681) Change to room_name parameter and erase references to query strings throughout the documentation tutorial. --- docs/getting-started.rst | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 3424948..b42eafc 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -364,7 +364,7 @@ Persisting Data Echoing messages is a nice simple example, but it's ignoring the real need for a system like this - persistent state for connections. Let's consider a basic chat site where a user requests a chat room upon initial -connection, as part of the query string (e.g. ``wss://host/websocket?room=abc``). +connection, as part of the URL path (e.g. ``wss://host/rooms/room-name``). The ``reply_channel`` attribute you've seen before is our unique pointer to the open WebSocket - because it varies between different clients, it's how we can @@ -390,26 +390,24 @@ name in the path of your WebSocket request (we'll ignore auth for now - that's n # Connected to websocket.connect @channel_session - def ws_connect(message): + def ws_connect(message, room_name): # Accept connection message.reply_channel.send({"accept": True}) - # Work out room name from path (ignore slashes) - room = message.content['path'].strip("/") # Save room in session and add us to the group - message.channel_session['room'] = room + message.channel_session['room_name'] = room_name Group("chat-%s" % room).add(message.reply_channel) # Connected to websocket.receive @channel_session - def ws_message(message): - Group("chat-%s" % message.channel_session['room']).send({ + def ws_message(message, room_name): + Group("chat-%s" % message.channel_session['room_name']).send({ "text": message['text'], }) # Connected to websocket.disconnect @channel_session - def ws_disconnect(message): - Group("chat-%s" % message.channel_session['room']).discard(message.reply_channel) + def ws_disconnect(message, room_name): + Group("chat-%s" % room_name).discard(message.reply_channel) Update ``routing.py`` as well:: @@ -591,7 +589,7 @@ routing our chat from above:: ] chat_routing = [ - route("websocket.connect", chat_connect, path=r"^/(?P[a-zA-Z0-9_]+)/$"), + route("websocket.connect", chat_connect, path=r"^/(?P[a-zA-Z0-9_]+)/$"), route("websocket.disconnect", chat_disconnect), ] @@ -616,13 +614,13 @@ consumer above to use a room based on URL rather than username:: # Connected to websocket.connect @channel_session_user_from_http - def ws_add(message, room): + def ws_add(message, room_name): # Add them to the right group - Group("chat-%s" % room).add(message.reply_channel) + Group("chat-%s" % room_name).add(message.reply_channel) # Accept the connection request message.reply_channel.send({"accept": True}) -In the next section, we'll change to sending the ``room`` as a part of the +In the next section, we'll change to sending the ``room_name`` as a part of the WebSocket message - which you might do if you had a multiplexing client - but you could use routing there as well. From e6c06861c458fc5bbad60525bcd1efc353168222 Mon Sep 17 00:00:00 2001 From: Sergio Date: Thu, 22 Jun 2017 21:01:10 -0400 Subject: [PATCH 714/746] Add reference to run interface servers in docs. (#680) My mistake, I forgot to make a link to the section before referencing it. Added a link to the "Run Interface Servers" section. --- docs/deploying.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/deploying.rst b/docs/deploying.rst index d347da2..95ba5d1 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -136,6 +136,8 @@ Or telling a worker to ignore all messages on the "thumbnail" channel:: python manage.py runworker --exclude-channels=thumbnail +.. _run-interface-servers: + Run interface servers --------------------- From 26916cccc70290e8252cf5e6a6913587f8134f85 Mon Sep 17 00:00:00 2001 From: Sergio Date: Sat, 24 Jun 2017 20:14:06 -0400 Subject: [PATCH 715/746] Improve "Persisting Data" example code (#683) * Improve "Persisting Data" example code Relates to issue #662. Please review this commit too. I believe I covered everything appropriately, but any suggestions are welcome. * Remove extra whitespace Fixed a small typo in the Getting Started | persisting data section Improves upon issue #662 --- docs/getting-started.rst | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index b42eafc..a924b31 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -382,26 +382,35 @@ provides you with an attribute called ``message.channel_session`` that acts just like a normal Django session. Let's use it now to build a chat server that expects you to pass a chatroom -name in the path of your WebSocket request (we'll ignore auth for now - that's next):: +name in the path of your WebSocket request and a query string with your username (we'll ignore auth for now - that's next):: # In consumers.py from channels import Group from channels.sessions import channel_session + from urllib.parse import parse_qs # Connected to websocket.connect @channel_session def ws_connect(message, room_name): # Accept connection message.reply_channel.send({"accept": True}) - # Save room in session and add us to the group - message.channel_session['room_name'] = room_name - Group("chat-%s" % room).add(message.reply_channel) + # Parse the query string + params = parse_qs(message.content["query_string"]) + if "username" in params: + # Set the username in the session + message.channel_session["username"] = params["username"] + # Add the user to the room_name group + Group("chat-%s" % room_name).add(message.reply_channel) + else: + # Close the connection. + message.reply_channel.send({"close": True}) # Connected to websocket.receive @channel_session def ws_message(message, room_name): - Group("chat-%s" % message.channel_session['room_name']).send({ - "text": message['text'], + Group("chat-%s" % room_name).send({ + "text": message["text"], + "username": message.channel_session["username"] }) # Connected to websocket.disconnect @@ -416,9 +425,9 @@ Update ``routing.py`` as well:: from myapp.consumers import ws_connect, ws_message, ws_disconnect channel_routing = [ - route("websocket.connect", ws_connect), - route("websocket.receive", ws_message), - route("websocket.disconnect", ws_disconnect), + route("websocket.connect", ws_connect, path=r"^/(?P[a-zA-Z0-9_]+)/$"), + route("websocket.receive", ws_message, path=r"^/(?P[a-zA-Z0-9_]+)/$"), + route("websocket.disconnect", ws_disconnect, path=r"^/(?P[a-zA-Z0-9_]+)/$"), ] If you play around with it from the console (or start building a simple From 93f2bf15858d7431f0a72af086ed54262c230d8a Mon Sep 17 00:00:00 2001 From: Luke Murphy Date: Tue, 27 Jun 2017 18:36:46 +0100 Subject: [PATCH 716/746] Add note about conflicting `runserver` commands. (#685) --- docs/installation.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 3cb4756..2002643 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -20,6 +20,14 @@ Once that's done, you should add ``channels`` to your That's it! Once enabled, ``channels`` will integrate itself into Django and take control of the ``runserver`` command. See :doc:`getting-started` for more. +.. note:: + Please be wary of any other third-party apps that require an overloaded or + replacement ``runserver`` command. Channels provides a separate + ``runserver`` command and may conflict with it. An example + of such a conflict is with `whitenoise.runserver_nostatic `_ + from `whitenoise `_. In order to + solve such issues, try moving ``channels`` to the top of your ``INSTALLED_APPS`` + or remove the offending app altogether. Installing the latest development version ----------------------------------------- From 8e186c12469fe02829383070797cc523d5ccd419 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 28 Jun 2017 00:21:37 -0700 Subject: [PATCH 717/746] Releasing 1.1.6 --- CHANGELOG.txt | 7 +++++++ channels/__init__.py | 2 +- docs/releases/1.1.6.rst | 23 +++++++++++++++++++++++ docs/releases/index.rst | 1 + js_client/package.json | 2 +- 5 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 docs/releases/1.1.6.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index c9d77d7..fff2ef3 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,13 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases +1.1.6 (2017-06-28) +------------------ + +* The ``runserver`` ``server_cls`` override no longer fails with more modern + Django versions that pass an ``ipv6`` parameter. + + 1.1.5 (2017-06-16) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index db5cf07..a0d9e10 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.5" +__version__ = "1.1.6" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/releases/1.1.6.rst b/docs/releases/1.1.6.rst new file mode 100644 index 0000000..56ee795 --- /dev/null +++ b/docs/releases/1.1.6.rst @@ -0,0 +1,23 @@ +1.1.6 Release Notes +=================== + +Channels 1.1.5 is a packaging release for the 1.1 series, released on +June 28th, 2017. + + +Major Changes +------------- + +None. + + +Minor Changes & Bugfixes +------------------------ + +* The ``runserver`` ``server_cls`` override no longer fails with more modern + Django versions that pass an ``ipv6`` parameter. + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 28ec006..c03f739 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -14,3 +14,4 @@ Release Notes 1.1.3 1.1.4 1.1.5 + 1.1.6 diff --git a/js_client/package.json b/js_client/package.json index fab7102..fb181e1 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -1,6 +1,6 @@ { "name": "django-channels", - "version": "1.1.5", + "version": "1.1.6", "description": "", "repository": { "type": "git", From 6990368d6ccb1719a1ba3c7502c9f8b8e0233d29 Mon Sep 17 00:00:00 2001 From: Matthew Date: Thu, 29 Jun 2017 19:15:39 +0200 Subject: [PATCH 718/746] Add note about Live Server Test Case and SQLite (#692) By default Django will use SQLite as an in memory database when running tests. This will not work with the Live Server Test Case. Added some notes on how to configure the database so that it is not run in memory. --- docs/testing.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/testing.rst b/docs/testing.rst index 32884c0..e845619 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -358,7 +358,25 @@ In the test above Daphne and Channels worker processes were fired up. These processes run your project against the test database and the default channel layer you spacify in the settings. If channel layer support ``flush`` extension, initial cleanup will be done. So do not -run this code against your production environment. When channels +run this code against your production environment. +ChannelLiveServerTestCase can not be used with in memory databases. +When using the SQLite database engine the Django tests will by default +use an in-memory database. To disable this add the ``TEST`` setting +to the database configuration. + +.. code:: python + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'TEST': { + 'NAME': 'testdb.sqlite3' + } + } + } + +When channels infrastructure is ready default web browser will be also started. You can open your website in the real browser which can execute JavaScript and operate on WebSockets. ``live_server_ws_url`` property is also From 300999770f98030d8d9625cc308c3519c14ce735 Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Sun, 2 Jul 2017 01:59:45 +0300 Subject: [PATCH 719/746] Avoid parent process connection cleanup in the test suite. (#695) Fix #614. --- channels/test/liveserver.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/channels/test/liveserver.py b/channels/test/liveserver.py index 88e357f..78bf252 100644 --- a/channels/test/liveserver.py +++ b/channels/test/liveserver.py @@ -41,8 +41,29 @@ class ProcessSetup(multiprocessing.Process): else: django.setup() + def cleanup_connections(self): + + # Channels run `django.db.close_old_connections` as a signal + # receiver after each consumer finished event. This function + # iterate on each created connection wrapper, checks if + # connection is still usable and closes it otherwise. Under + # normal circumstances this is a very reasonable approach. + # When process starts the usual way `django.db.connections` + # contains empty connection list. But channels worker in the + # test case is created with the fork system call. This means + # file descriptors from the parent process are available in + # the connection list, but connections themselves are not + # usable. So test worker will close connections of the parent + # process and test suite will fail when it tries to flush + # database after test run. + # + # See https://github.com/django/channels/issues/614 + for alias in self.databases: + del connections[alias] + def setup_databases(self): + self.cleanup_connections() for alias, db in self.databases.items(): backend = load_backend(db['ENGINE']) conn = backend.DatabaseWrapper(db, alias) From 8cc2842492766fa81326ad7fe3767b11cf504ba6 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 2 Jul 2017 10:18:28 -0700 Subject: [PATCH 720/746] Fixed #694: Invalid code example in getting-started --- docs/getting-started.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index a924b31..f05e0c9 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -396,9 +396,9 @@ name in the path of your WebSocket request and a query string with your username message.reply_channel.send({"accept": True}) # Parse the query string params = parse_qs(message.content["query_string"]) - if "username" in params: + if b"username" in params: # Set the username in the session - message.channel_session["username"] = params["username"] + message.channel_session["username"] = params[b"username"][0].decode("utf8") # Add the user to the room_name group Group("chat-%s" % room_name).add(message.reply_channel) else: @@ -408,10 +408,12 @@ name in the path of your WebSocket request and a query string with your username # Connected to websocket.receive @channel_session def ws_message(message, room_name): - Group("chat-%s" % room_name).send({ - "text": message["text"], - "username": message.channel_session["username"] - }) + Group("chat-%s" % room_name).send( + "text": json.dumps({ + "text": message["text"], + "username": message.channel_session["username"], + }), + ) # Connected to websocket.disconnect @channel_session From a3f4e002eeebbf7c2412d9623e4e9809cfe32ba5 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 2 Jul 2017 17:04:51 -0700 Subject: [PATCH 721/746] Fixed #696: Runserver broke with Django 1.10 and below --- channels/management/commands/runserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/channels/management/commands/runserver.py b/channels/management/commands/runserver.py index 9770288..c4937ff 100644 --- a/channels/management/commands/runserver.py +++ b/channels/management/commands/runserver.py @@ -42,7 +42,8 @@ class Command(RunserverCommand): def inner_run(self, *args, **options): # Maybe they want the wsgi one? if not options.get("use_asgi", True) or DEFAULT_CHANNEL_LAYER not in channel_layers: - self.server_cls = RunserverCommand.server_cls + if hasattr(RunserverCommand, "server_cls"): + self.server_cls = RunserverCommand.server_cls return RunserverCommand.inner_run(self, *args, **options) # Check a handler is registered for http reqs; if not, add default one self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER] From a699eec7eeccc3e370e7ace20b97bc05be1ce36d Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Mon, 3 Jul 2017 14:11:40 -0500 Subject: [PATCH 722/746] Upgrade package.json to NPM v5 (#688) --- js_client/package.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/js_client/package.json b/js_client/package.json index fb181e1..e912c0d 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -12,9 +12,12 @@ "docs": "rm -rf docs && esdoc -c esdoc.json", "test": "jest", "browserify": "browserify src/index.js -p browserify-banner -s channels -o ../channels/static/channels/js/websocketbridge.js", - "prepublish": "npm run transpile", + "prepare": "npm run transpile", "compile": "npm run transpile && npm run browserify" }, + "engines": { + "npm": "^5.0.0" + }, "files": [ "lib/index.js" ], @@ -35,7 +38,6 @@ ] }, "devDependencies": { - "babel": "^6.5.2", "babel-cli": "^6.24.0", "babel-core": "^6.16.0", "babel-plugin-transform-inline-environment-variables": "^6.8.0", From b81bbc662ceab12ef8d7f9788b98cbdfde0c2f02 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 4 Jul 2017 10:27:55 -0700 Subject: [PATCH 723/746] Update getting started example one more time. --- docs/getting-started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index f05e0c9..84cc9d8 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -408,12 +408,12 @@ name in the path of your WebSocket request and a query string with your username # Connected to websocket.receive @channel_session def ws_message(message, room_name): - Group("chat-%s" % room_name).send( + Group("chat-%s" % room_name).send({ "text": json.dumps({ "text": message["text"], "username": message.channel_session["username"], }), - ) + }) # Connected to websocket.disconnect @channel_session From acb398899977160e663025feefc3e96ebfd0c2cf Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 5 Jul 2017 07:31:30 -0700 Subject: [PATCH 724/746] Add json import (fixes #700) --- docs/getting-started.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 84cc9d8..d34d6f0 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -385,6 +385,7 @@ Let's use it now to build a chat server that expects you to pass a chatroom name in the path of your WebSocket request and a query string with your username (we'll ignore auth for now - that's next):: # In consumers.py + import json from channels import Group from channels.sessions import channel_session from urllib.parse import parse_qs From 405007f83e20f0ca53095d2b807a543f4a77e5a4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Sun, 9 Jul 2017 12:10:53 +0100 Subject: [PATCH 725/746] Add Artem to maintenance team --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index f969780..6eaf3a6 100644 --- a/README.rst +++ b/README.rst @@ -63,8 +63,7 @@ Django Core Shepherd: Andrew Godwin Maintenance team: * Andrew Godwin -* Steven Davidson -* Jeremy Spencer +* Artem Malyshev If you are interested in joining the maintenance team, please `read more about contributing `_ From 7aa8e05987fff191ed381d2294faa8177c812746 Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Mon, 10 Jul 2017 21:24:17 +0300 Subject: [PATCH 726/746] Make maintainer email public. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 6eaf3a6..f398c12 100644 --- a/README.rst +++ b/README.rst @@ -63,7 +63,7 @@ Django Core Shepherd: Andrew Godwin Maintenance team: * Andrew Godwin -* Artem Malyshev +* Artem Malyshev If you are interested in joining the maintenance team, please `read more about contributing `_ From 60700da0dff0df9b361b975c223e8054b7ceb54c Mon Sep 17 00:00:00 2001 From: Cropse Date: Mon, 17 Jul 2017 11:25:01 +0800 Subject: [PATCH 727/746] Update faqs.rst Add the example of non-Django application. --- docs/faqs.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/faqs.rst b/docs/faqs.rst index e2bd585..17a0340 100755 --- a/docs/faqs.rst +++ b/docs/faqs.rst @@ -136,7 +136,12 @@ a few choices: (see :doc:`deploying`), you can import the channel layer directly as ``yourproject.asgi.channel_layer`` and call ``send()`` and ``receive_many()`` on it directly. See the :doc:`ASGI spec ` for the API the channel layer - presents. + presents. Here's what that looks like: + + >>> from mysite.asgi import channel_layer + >>> from channels import Channel, Group + >>> Channel("channel_name").send({"text":"channel_text"}) + >>> Group("group_name").send({"text":"group_text"}) * If you just need to send messages in when events happen, you can make a management command that calls ``Channel("namehere").send({...})`` From ff2f09c3a1f8014a5938ceff512593d55cd0eb29 Mon Sep 17 00:00:00 2001 From: Cropse Date: Mon, 17 Jul 2017 11:27:45 +0800 Subject: [PATCH 728/746] Update faqs.rst --- docs/faqs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faqs.rst b/docs/faqs.rst index 17a0340..701f9b9 100755 --- a/docs/faqs.rst +++ b/docs/faqs.rst @@ -138,7 +138,7 @@ a few choices: on it directly. See the :doc:`ASGI spec ` for the API the channel layer presents. Here's what that looks like: - >>> from mysite.asgi import channel_layer + >>> from yourproject.asgi import channel_layer >>> from channels import Channel, Group >>> Channel("channel_name").send({"text":"channel_text"}) >>> Group("group_name").send({"text":"group_text"}) From 8b873e2bd6e4fbbf0464380cd799c9513b26386c Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Tue, 18 Jul 2017 08:23:34 -0400 Subject: [PATCH 729/746] Use correct environment markers syntax in the extras_require section. See https://github.com/pypa/setuptools/issues/1087 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e8f4159..d5bbf18 100644 --- a/setup.py +++ b/setup.py @@ -19,10 +19,10 @@ setup( extras_require={ 'tests': [ 'coverage', - 'mock ; python_version < "3.0"', 'flake8>=2.0,<3.0', 'isort', ], + 'tests:python_version < "3.0"': ['mock'], }, classifiers=[ 'Development Status :: 5 - Production/Stable', From f3422650866d8973e545f47de3196428b6cd12af Mon Sep 17 00:00:00 2001 From: Leon Koole Date: Wed, 19 Jul 2017 10:48:37 +0200 Subject: [PATCH 730/746] Channels is comprised of six packages --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index a9fa415..8c90e78 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,7 +23,7 @@ If you are interested in contributing, please read our :doc:`contributing` docs! Projects -------- -Channels is comprised of five packages: +Channels is comprised of six packages: * `Channels `_, the Django integration layer * `Daphne `_, the HTTP and Websocket termination server From e3ff3e8635b5a732813227dfe277f671666674a5 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 26 Jul 2017 19:59:57 +0100 Subject: [PATCH 731/746] Travis - test on Trusty (#708) As per [their blog post](https://blog.travis-ci.com/2017-07-11-trusty-as-default-linux-is-coming) they're making it the new default, best to be ahead of the curve. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0cb25c9..2c4da6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ sudo: false +dist: trusty language: python From 8c86a29761ec0d38f5f7bc73d3992c46bff4a661 Mon Sep 17 00:00:00 2001 From: Artem Malyshev Date: Sat, 5 Aug 2017 16:02:09 +0300 Subject: [PATCH 732/746] Mention hiredis installation option. See https://github.com/django/asgi_redis/issues/56 --- docs/backends.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/backends.rst b/docs/backends.rst index f500d14..af2f655 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -32,6 +32,14 @@ but you can override this with the ``hosts`` key in its config:: }, } +Consider `hiredis`_ library installation to improve layer performance:: + + pip install hiredis + +It will be used automatically if it's installed. + +.. _hiredis: https://github.com/redis/hiredis-py + Sharding ~~~~~~~~ From a2fea857544ab23b4163004bad81e3a6206aaaab Mon Sep 17 00:00:00 2001 From: Mikkel Schubert Date: Mon, 14 Aug 2017 15:34:04 +0200 Subject: [PATCH 733/746] Add missing import in 'Models' example code --- docs/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index d34d6f0..77a374e 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -661,7 +661,7 @@ Let's see what that looks like, assuming we have a ChatMessage model with ``message`` and ``room`` fields:: # In consumers.py - from channels import Channel + from channels import Channel, Group from channels.sessions import channel_session from .models import ChatMessage From c0138e8dd95f0b22ebf6ff1b55f7697be812f760 Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Fri, 1 Sep 2017 09:46:53 -0700 Subject: [PATCH 734/746] Pin mock-socket to 6.0.4 (#689) npm install will install mock-server v6.1.0, which breaks our tests. --- js_client/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js_client/package.json b/js_client/package.json index e912c0d..e84ef2f 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -61,7 +61,7 @@ "eslint-plugin-jsx-a11y": "^1.5.3", "eslint-plugin-react": "^5.2.2", "jest": "^19.0.1", - "mock-socket": "^6.0.4", + "mock-socket": "6.0.4", "react": "^15.4.0", "react-cookie": "^0.4.8", "react-dom": "^15.4.0", From ed32c71d90064b0537fb8753b0d709dbd94788c7 Mon Sep 17 00:00:00 2001 From: bmcool Date: Sat, 2 Sep 2017 00:47:27 +0800 Subject: [PATCH 735/746] Fixed: some device will get error code [1006] when handshake if protocols is undefined. (#691) --- channels/static/channels/js/websocketbridge.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/channels/static/channels/js/websocketbridge.js b/channels/static/channels/js/websocketbridge.js index d7b2845..dd4af1a 100644 --- a/channels/static/channels/js/websocketbridge.js +++ b/channels/static/channels/js/websocketbridge.js @@ -115,7 +115,12 @@ var ReconnectingWebsocket = function (url, protocols, options) { var connect = function () { log('connect'); var oldWs = ws; - ws = new config.constructor(url, protocols); + if (protocols === undefined) { + ws = new config.constructor(url); + } else { + ws = new config.constructor(url, protocols); + } + connectingTimeout = setTimeout(function () { log('timeout'); ws.close(); From f34ca1cd2bfb00bd2219d4a192079ed0f539f912 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Tue, 12 Sep 2017 09:19:25 -0700 Subject: [PATCH 736/746] Revert "Fixed: some device will get error code [1006] when handshake if protocols is undefined. (#691)" This reverts commit ed32c71d90064b0537fb8753b0d709dbd94788c7. --- channels/static/channels/js/websocketbridge.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/channels/static/channels/js/websocketbridge.js b/channels/static/channels/js/websocketbridge.js index dd4af1a..d7b2845 100644 --- a/channels/static/channels/js/websocketbridge.js +++ b/channels/static/channels/js/websocketbridge.js @@ -115,12 +115,7 @@ var ReconnectingWebsocket = function (url, protocols, options) { var connect = function () { log('connect'); var oldWs = ws; - if (protocols === undefined) { - ws = new config.constructor(url); - } else { - ws = new config.constructor(url, protocols); - } - + ws = new config.constructor(url, protocols); connectingTimeout = setTimeout(function () { log('timeout'); ws.close(); From 3111f3b8ac470bad36b9e4a2445834d0b3c62f97 Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Tue, 12 Sep 2017 10:22:24 -0700 Subject: [PATCH 737/746] Fix error code [1006] if protocols is `undefined` (#735) --- channels/static/channels/js/websocketbridge.js | 5 ++++- js_client/lib/index.js | 3 ++- js_client/src/index.js | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/channels/static/channels/js/websocketbridge.js b/channels/static/channels/js/websocketbridge.js index d7b2845..e5650e4 100644 --- a/channels/static/channels/js/websocketbridge.js +++ b/channels/static/channels/js/websocketbridge.js @@ -283,7 +283,10 @@ var WebSocketBridge = function () { _url = url; } } - this.socket = new _reconnectingWebsocket2.default(_url, protocols, options); + // Some mobile devices (eg: HTC M8, SAMSUNG Galaxy S8) will get error code + // [1006] during handshake if `protocols` is `undefined`. + var _protocols = protocols === undefined ? '' : protocols; + this.socket = new _reconnectingWebsocket2.default(_url, _protocols, options); } /** diff --git a/js_client/lib/index.js b/js_client/lib/index.js index 53ca4f0..0e69313 100644 --- a/js_client/lib/index.js +++ b/js_client/lib/index.js @@ -72,7 +72,8 @@ var WebSocketBridge = function () { _url = url; } } - this.socket = new _reconnectingWebsocket2.default(_url, protocols, options); + var _protocols = protocols === undefined ? '' : protocols; + this.socket = new _reconnectingWebsocket2.default(_url, _protocols, options); } /** diff --git a/js_client/src/index.js b/js_client/src/index.js index b2ebd81..ae0fd48 100644 --- a/js_client/src/index.js +++ b/js_client/src/index.js @@ -50,7 +50,10 @@ export class WebSocketBridge { _url = url; } } - this.socket = new ReconnectingWebSocket(_url, protocols, options); + // Some mobile devices (eg: HTC M8, SAMSUNG Galaxy S8) will get error code + // [1006] during handshake if `protocols` is `undefined`. + const _protocols = protocols === undefined ? '' : protocols; + this.socket = new ReconnectingWebSocket(_url, _protocols, options); } /** From 2d4886dfa0d10031a4a6d839f18ee56ec1f2aee8 Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Tue, 12 Sep 2017 10:22:43 -0700 Subject: [PATCH 738/746] Upgrade reconnecting-websocket (#734) * fix url in comment * upgrade reconnecting-websocket --- js_client/package.json | 2 +- js_client/src/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/js_client/package.json b/js_client/package.json index e84ef2f..f32aaea 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -23,7 +23,7 @@ ], "license": "BSD-3-Clause", "dependencies": { - "reconnecting-websocket": "^3.0.3" + "reconnecting-websocket": "^3.2.1" }, "jest": { "roots": [ diff --git a/js_client/src/index.js b/js_client/src/index.js index ae0fd48..e6ca77a 100644 --- a/js_client/src/index.js +++ b/js_client/src/index.js @@ -30,7 +30,7 @@ export class WebSocketBridge { * @param {String} [url] The url of the websocket. Defaults to * `window.location.host` * @param {String[]|String} [protocols] Optional string or array of protocols. - * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/joewalnes/reconnecting-websocket#options-1). + * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/pladaria/reconnecting-websocket#configure). * @example * const webSocketBridge = new WebSocketBridge(); * webSocketBridge.connect(); From 65150d16a72564c27acc087ebfac8f03fbaccebc Mon Sep 17 00:00:00 2001 From: Flavio Curella Date: Wed, 13 Sep 2017 10:32:40 -0700 Subject: [PATCH 739/746] recompile js bundles from master (#736) --- .../static/channels/js/websocketbridge.js | 25 +++++++++++++------ js_client/lib/index.js | 4 ++- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/channels/static/channels/js/websocketbridge.js b/channels/static/channels/js/websocketbridge.js index e5650e4..e9aaab4 100644 --- a/channels/static/channels/js/websocketbridge.js +++ b/channels/static/channels/js/websocketbridge.js @@ -3,6 +3,9 @@ */ (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.channels = f()}})(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o config.maxRetries) { @@ -107,15 +112,19 @@ var ReconnectingWebsocket = function (url, protocols, options) { else { reconnectDelay = updateReconnectionDelay(config, reconnectDelay); } - log('reconnectDelay:', reconnectDelay); + log('handleClose - reconnectDelay:', reconnectDelay); if (shouldRetry) { setTimeout(connect, reconnectDelay); } }; var connect = function () { + if (!shouldRetry) { + return; + } log('connect'); var oldWs = ws; - ws = new config.constructor(url, protocols); + var wsUrl = (typeof url === 'function') ? url() : url; + ws = new config.constructor(wsUrl, protocols); connectingTimeout = setTimeout(function () { log('timeout'); ws.close(); @@ -147,10 +156,11 @@ var ReconnectingWebsocket = function (url, protocols, options) { if (code === void 0) { code = 1000; } if (reason === void 0) { reason = ''; } var _b = _a === void 0 ? {} : _a, _c = _b.keepClosed, keepClosed = _c === void 0 ? false : _c, _d = _b.fastClose, fastClose = _d === void 0 ? true : _d, _e = _b.delay, delay = _e === void 0 ? 0 : _e; + log('close - params:', { reason: reason, keepClosed: keepClosed, fastClose: fastClose, delay: delay, retriesCount: retriesCount, maxRetries: config.maxRetries }); + shouldRetry = !keepClosed && retriesCount <= config.maxRetries; if (delay) { reconnectDelay = delay; } - shouldRetry = !keepClosed; ws.close(code, reason); if (fastClose) { var fakeCloseEvent_1 = { @@ -205,6 +215,7 @@ var ReconnectingWebsocket = function (url, protocols, options) { } ws.removeEventListener(type, listener, options); }; + return this; }; module.exports = ReconnectingWebsocket; @@ -259,7 +270,7 @@ var WebSocketBridge = function () { * @param {String} [url] The url of the websocket. Defaults to * `window.location.host` * @param {String[]|String} [protocols] Optional string or array of protocols. - * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/joewalnes/reconnecting-websocket#options-1). + * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/pladaria/reconnecting-websocket#configure). * @example * const webSocketBridge = new WebSocketBridge(); * webSocketBridge.connect(); diff --git a/js_client/lib/index.js b/js_client/lib/index.js index 0e69313..c2a6e8c 100644 --- a/js_client/lib/index.js +++ b/js_client/lib/index.js @@ -48,7 +48,7 @@ var WebSocketBridge = function () { * @param {String} [url] The url of the websocket. Defaults to * `window.location.host` * @param {String[]|String} [protocols] Optional string or array of protocols. - * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/joewalnes/reconnecting-websocket#options-1). + * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/pladaria/reconnecting-websocket#configure). * @example * const webSocketBridge = new WebSocketBridge(); * webSocketBridge.connect(); @@ -72,6 +72,8 @@ var WebSocketBridge = function () { _url = url; } } + // Some mobile devices (eg: HTC M8, SAMSUNG Galaxy S8) will get error code + // [1006] during handshake if `protocols` is `undefined`. var _protocols = protocols === undefined ? '' : protocols; this.socket = new _reconnectingWebsocket2.default(_url, _protocols, options); } From 29c269ead7e8fec877f65b00d6982c62d5fc085a Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 14 Sep 2017 10:35:40 -0700 Subject: [PATCH 740/746] Releasing 1.1.7 --- CHANGELOG.txt | 8 ++++++++ channels/__init__.py | 2 +- docs/releases/index.rst | 1 + js_client/package.json | 2 +- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index fff2ef3..870b52b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,14 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases +1.1.7 (2017-09-14) +------------------ + +* Fixed compatability with Django 1.10 and below + +* JS library: Fixed error with 1006 error code + + 1.1.6 (2017-06-28) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index a0d9e10..176fc99 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.6" +__version__ = "1.1.7" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/releases/index.rst b/docs/releases/index.rst index c03f739..67bd716 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -15,3 +15,4 @@ Release Notes 1.1.4 1.1.5 1.1.6 + 1.1.7 diff --git a/js_client/package.json b/js_client/package.json index f32aaea..6c9e36f 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -1,6 +1,6 @@ { "name": "django-channels", - "version": "1.1.6", + "version": "1.1.7", "description": "", "repository": { "type": "git", From ab89f52302371d13a0bc6d4bd5fa5989458bcee3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 15 Sep 2017 19:08:49 +0200 Subject: [PATCH 741/746] Daphne deployment doc updates --- docs/deploying.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/deploying.rst b/docs/deploying.rst index 95ba5d1..c041e41 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -190,6 +190,17 @@ scenario, it will eventually time out and give you a 503 error after 2 minutes; you can configure how long it waits with the ``--http-timeout`` command line argument. +With some browsers you may see errors regarding ``Sec-WebSocket-Protocol`` headers. You can set the allowed ws_protocols to match your client protocol like this:: + + CHANNELS_WS_PROTOCOLS = ["graphql-ws", ] + +In production you may start a daphne server without the runserver command. So you need to pass the ws-protocl directly:: + + daphne --ws-protocol "graphql-ws" --proxy-headers my_project.asgi:channel_layer + +Note: The daphne server binds to 127.0.0.1 by default. If you deploy this not locally, bind to your ip or to any ip:: + + daphne -b 0.0.0.0 -p 8000 --ws-protocol "graphql-ws" --proxy-headers my_project.asgi:channel_layer Deploying new versions of code ------------------------------ From e559d9a2b9d0f7500463df88494fe25f927d6652 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 15 Sep 2017 10:15:43 -0700 Subject: [PATCH 742/746] Add missing 1.1.7 release notes --- docs/releases/1.1.7.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docs/releases/1.1.7.rst diff --git a/docs/releases/1.1.7.rst b/docs/releases/1.1.7.rst new file mode 100644 index 0000000..38222ce --- /dev/null +++ b/docs/releases/1.1.7.rst @@ -0,0 +1,26 @@ +1.1.7 Release Notes +=================== + +Channels 1.1.5 is a packaging release for the 1.1 series, released on +September 14th, 2017. + + +Major Changes +------------- + +None. + + +Minor Changes & Bugfixes +------------------------ + +* The ``runserver`` ``server_cls`` override fic from 1.1.6 no longer fails + when trying to use Django 1.10 or below. + +* The JS library has fixed error with the 1006 error code on some WebSocket + implementations. + +Backwards Incompatible Changes +------------------------------ + +None. From 46484cdf39cb6cf3861e475b406461e5d2821fc4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 15 Sep 2017 10:17:28 -0700 Subject: [PATCH 743/746] Reverting recent JS changes as they are not stable. --- .../static/channels/js/websocketbridge.js | 30 +++++-------------- js_client/lib/index.js | 7 ++--- js_client/package.json | 2 +- js_client/src/index.js | 7 ++--- 4 files changed, 13 insertions(+), 33 deletions(-) diff --git a/channels/static/channels/js/websocketbridge.js b/channels/static/channels/js/websocketbridge.js index e9aaab4..d7b2845 100644 --- a/channels/static/channels/js/websocketbridge.js +++ b/channels/static/channels/js/websocketbridge.js @@ -3,9 +3,6 @@ */ (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.channels = f()}})(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o config.maxRetries) { @@ -112,19 +107,15 @@ var ReconnectingWebsocket = function (url, protocols, options) { else { reconnectDelay = updateReconnectionDelay(config, reconnectDelay); } - log('handleClose - reconnectDelay:', reconnectDelay); + log('reconnectDelay:', reconnectDelay); if (shouldRetry) { setTimeout(connect, reconnectDelay); } }; var connect = function () { - if (!shouldRetry) { - return; - } log('connect'); var oldWs = ws; - var wsUrl = (typeof url === 'function') ? url() : url; - ws = new config.constructor(wsUrl, protocols); + ws = new config.constructor(url, protocols); connectingTimeout = setTimeout(function () { log('timeout'); ws.close(); @@ -156,11 +147,10 @@ var ReconnectingWebsocket = function (url, protocols, options) { if (code === void 0) { code = 1000; } if (reason === void 0) { reason = ''; } var _b = _a === void 0 ? {} : _a, _c = _b.keepClosed, keepClosed = _c === void 0 ? false : _c, _d = _b.fastClose, fastClose = _d === void 0 ? true : _d, _e = _b.delay, delay = _e === void 0 ? 0 : _e; - log('close - params:', { reason: reason, keepClosed: keepClosed, fastClose: fastClose, delay: delay, retriesCount: retriesCount, maxRetries: config.maxRetries }); - shouldRetry = !keepClosed && retriesCount <= config.maxRetries; if (delay) { reconnectDelay = delay; } + shouldRetry = !keepClosed; ws.close(code, reason); if (fastClose) { var fakeCloseEvent_1 = { @@ -215,7 +205,6 @@ var ReconnectingWebsocket = function (url, protocols, options) { } ws.removeEventListener(type, listener, options); }; - return this; }; module.exports = ReconnectingWebsocket; @@ -270,7 +259,7 @@ var WebSocketBridge = function () { * @param {String} [url] The url of the websocket. Defaults to * `window.location.host` * @param {String[]|String} [protocols] Optional string or array of protocols. - * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/pladaria/reconnecting-websocket#configure). + * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/joewalnes/reconnecting-websocket#options-1). * @example * const webSocketBridge = new WebSocketBridge(); * webSocketBridge.connect(); @@ -294,10 +283,7 @@ var WebSocketBridge = function () { _url = url; } } - // Some mobile devices (eg: HTC M8, SAMSUNG Galaxy S8) will get error code - // [1006] during handshake if `protocols` is `undefined`. - var _protocols = protocols === undefined ? '' : protocols; - this.socket = new _reconnectingWebsocket2.default(_url, _protocols, options); + this.socket = new _reconnectingWebsocket2.default(_url, protocols, options); } /** diff --git a/js_client/lib/index.js b/js_client/lib/index.js index c2a6e8c..53ca4f0 100644 --- a/js_client/lib/index.js +++ b/js_client/lib/index.js @@ -48,7 +48,7 @@ var WebSocketBridge = function () { * @param {String} [url] The url of the websocket. Defaults to * `window.location.host` * @param {String[]|String} [protocols] Optional string or array of protocols. - * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/pladaria/reconnecting-websocket#configure). + * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/joewalnes/reconnecting-websocket#options-1). * @example * const webSocketBridge = new WebSocketBridge(); * webSocketBridge.connect(); @@ -72,10 +72,7 @@ var WebSocketBridge = function () { _url = url; } } - // Some mobile devices (eg: HTC M8, SAMSUNG Galaxy S8) will get error code - // [1006] during handshake if `protocols` is `undefined`. - var _protocols = protocols === undefined ? '' : protocols; - this.socket = new _reconnectingWebsocket2.default(_url, _protocols, options); + this.socket = new _reconnectingWebsocket2.default(_url, protocols, options); } /** diff --git a/js_client/package.json b/js_client/package.json index 6c9e36f..91eb348 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -23,7 +23,7 @@ ], "license": "BSD-3-Clause", "dependencies": { - "reconnecting-websocket": "^3.2.1" + "reconnecting-websocket": "^3.0.3" }, "jest": { "roots": [ diff --git a/js_client/src/index.js b/js_client/src/index.js index e6ca77a..b2ebd81 100644 --- a/js_client/src/index.js +++ b/js_client/src/index.js @@ -30,7 +30,7 @@ export class WebSocketBridge { * @param {String} [url] The url of the websocket. Defaults to * `window.location.host` * @param {String[]|String} [protocols] Optional string or array of protocols. - * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/pladaria/reconnecting-websocket#configure). + * @param {Object} options Object of options for [`reconnecting-websocket`](https://github.com/joewalnes/reconnecting-websocket#options-1). * @example * const webSocketBridge = new WebSocketBridge(); * webSocketBridge.connect(); @@ -50,10 +50,7 @@ export class WebSocketBridge { _url = url; } } - // Some mobile devices (eg: HTC M8, SAMSUNG Galaxy S8) will get error code - // [1006] during handshake if `protocols` is `undefined`. - const _protocols = protocols === undefined ? '' : protocols; - this.socket = new ReconnectingWebSocket(_url, _protocols, options); + this.socket = new ReconnectingWebSocket(_url, protocols, options); } /** From 07053bebccfa989eee0ebb6826e3c58be7962884 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Fri, 15 Sep 2017 10:19:57 -0700 Subject: [PATCH 744/746] Releasing 1.1.8 --- CHANGELOG.txt | 8 ++++++++ channels/__init__.py | 2 +- docs/releases/1.1.6.rst | 2 +- docs/releases/1.1.7.rst | 2 +- docs/releases/1.1.8.rst | 23 +++++++++++++++++++++++ docs/releases/index.rst | 1 + js_client/package.json | 2 +- 7 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 docs/releases/1.1.8.rst diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 870b52b..0f50233 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,14 @@ Full release notes, with more details and upgrade information, are available at: https://channels.readthedocs.io/en/latest/releases + +1.1.8 (2017-09-15) +------------------ + +* Reverted recent JS fixes for subprotocols on some phones as they do not work + in Chrome. + + 1.1.7 (2017-09-14) ------------------ diff --git a/channels/__init__.py b/channels/__init__.py index 176fc99..5430057 100644 --- a/channels/__init__.py +++ b/channels/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.7" +__version__ = "1.1.8" default_app_config = 'channels.apps.ChannelsConfig' DEFAULT_CHANNEL_LAYER = 'default' diff --git a/docs/releases/1.1.6.rst b/docs/releases/1.1.6.rst index 56ee795..258c348 100644 --- a/docs/releases/1.1.6.rst +++ b/docs/releases/1.1.6.rst @@ -1,7 +1,7 @@ 1.1.6 Release Notes =================== -Channels 1.1.5 is a packaging release for the 1.1 series, released on +Channels 1.1.6 is a bugfix release for the 1.1 series, released on June 28th, 2017. diff --git a/docs/releases/1.1.7.rst b/docs/releases/1.1.7.rst index 38222ce..9e0a4ec 100644 --- a/docs/releases/1.1.7.rst +++ b/docs/releases/1.1.7.rst @@ -1,7 +1,7 @@ 1.1.7 Release Notes =================== -Channels 1.1.5 is a packaging release for the 1.1 series, released on +Channels 1.1.7 is a bugfix release for the 1.1 series, released on September 14th, 2017. diff --git a/docs/releases/1.1.8.rst b/docs/releases/1.1.8.rst new file mode 100644 index 0000000..c7fcbee --- /dev/null +++ b/docs/releases/1.1.8.rst @@ -0,0 +1,23 @@ +1.1.8 Release Notes +=================== + +Channels 1.1.8 is a packaging release for the 1.1 series, released on +September 15th, 2017. + + +Major Changes +------------- + +None. + + +Minor Changes & Bugfixes +------------------------ + +* Reverted recent JS fixes for subprotocols on some phones as they do not work + in Chrome. + +Backwards Incompatible Changes +------------------------------ + +None. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 67bd716..05570f1 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -16,3 +16,4 @@ Release Notes 1.1.5 1.1.6 1.1.7 + 1.1.8 diff --git a/js_client/package.json b/js_client/package.json index 91eb348..1bc0791 100644 --- a/js_client/package.json +++ b/js_client/package.json @@ -1,6 +1,6 @@ { "name": "django-channels", - "version": "1.1.7", + "version": "1.1.8", "description": "", "repository": { "type": "git", From 32da46f51daa710d25e8d5f5974c6cf3f95a17ed Mon Sep 17 00:00:00 2001 From: japrogramer Date: Wed, 27 Sep 2017 00:55:14 -0500 Subject: [PATCH 745/746] Fixed #748: Test client now handles headers as lists not dict --- channels/test/websocket.py | 5 ++++- tests/test_wsclient.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/channels/test/websocket.py b/channels/test/websocket.py index 4b762ce..c260bd8 100644 --- a/channels/test/websocket.py +++ b/channels/test/websocket.py @@ -82,6 +82,9 @@ class WSClient(Client): self.channel_layer.send(to, self._get_content(content, text, path)) self._session_cookie = False + def _list_headers(self): + return [[key.encode(), self.headers[key]] for key in self.headers] + def _get_content(self, content={}, text=None, path='/'): content = copy.deepcopy(content) content.setdefault('reply_channel', self.reply_channel) @@ -93,7 +96,7 @@ class WSClient(Client): else: content.setdefault('path', path) - content.setdefault('headers', self.headers) + content.setdefault('headers', self._list_headers()) if self._ordered: if 'order' in content: diff --git a/tests/test_wsclient.py b/tests/test_wsclient.py index a8924ca..0b94621 100644 --- a/tests/test_wsclient.py +++ b/tests/test_wsclient.py @@ -52,8 +52,8 @@ class WSClientTests(ChannelTestCase): self.assertEqual(content['path'], '/') self.assertTrue('headers' in content) - self.assertTrue('cookie' in content['headers']) - self.assertTrue(b'sessionid' in content['headers']['cookie']) + self.assertIn(b'cookie', [x[0] for x in content['headers']]) + self.assertIn(b'sessionid', [x[1] for x in content['headers'] if x[0] == b'cookie'][0]) def test_ordering_in_content(self): client = WSClient(ordered=True) From fa3d9b140dcfed22adf6a121996957412a4eae5b Mon Sep 17 00:00:00 2001 From: Tom Kazimiers Date: Thu, 28 Sep 2017 14:11:22 -0400 Subject: [PATCH 746/746] Docs: fix typos in "getting started" text and example (minor) (#756) A period sign is added and a syntax error in one of the examples is fixed. --- docs/getting-started.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 77a374e..35cca58 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -445,7 +445,7 @@ Now, of course, a WebSocket solution is somewhat limited in scope without the ability to live with the rest of your website - in particular, we want to make sure we know what user we're talking to, in case we have things like private chat channels (we don't want a solution where clients just ask for the right -channels, as anyone could change the code and just put in private channel names) +channels, as anyone could change the code and just put in private channel names). It can also save you having to manually make clients ask for what they want to see; if I see you open a WebSocket to my "updates" endpoint, and I know which @@ -559,7 +559,7 @@ from hosts listed in the ``ALLOWED_HOSTS`` setting:: from channels import Channel, Group from channels.sessions import channel_session from channels.auth import channel_session_user, channel_session_user_from_http - from channels.security.websockets import allowed_hosts_only. + from channels.security.websockets import allowed_hosts_only # Connected to websocket.connect @allowed_hosts_only