Add timeout 503 responses with configurable delay.

This commit is contained in:
Andrew Godwin 2016-03-02 11:24:19 -08:00
parent d445844061
commit 0ad7f1c2a2
3 changed files with 82 additions and 5 deletions

View File

@ -40,6 +40,13 @@ class CommandLineInterface(object):
help='How verbose to make the output', help='How verbose to make the output',
default=1, default=1,
) )
self.parser.add_argument(
'-t',
'--http-timeout',
type=int,
help='How long to wait for worker server before timing out HTTP connections',
default=120,
)
self.parser.add_argument( self.parser.add_argument(
'channel_layer', 'channel_layer',
help='The ASGI channel layer instance to use as path.to.module:instance.path', help='The ASGI channel layer instance to use as path.to.module:instance.path',
@ -85,4 +92,5 @@ class CommandLineInterface(object):
channel_layer=channel_layer, channel_layer=channel_layer,
host=args.host, host=args.host,
port=args.port, port=args.port,
http_timeout=args.http_timeout,
).run() ).run()

View File

@ -4,7 +4,6 @@ import logging
import six import six
import time import time
from twisted.python.compat import _PY3
from twisted.web import http from twisted.web import http
from twisted.protocols.policies import ProtocolWrapper from twisted.protocols.policies import ProtocolWrapper
@ -22,6 +21,25 @@ class WebRequest(http.Request):
GET and POST out. GET and POST out.
""" """
error_template = """
<html>
<head>
<title>%(title)s</title>
<style>
body { font-family: sans-serif; margin: 0; padding: 0; }
h1 { background: #E9B1B1; padding: 0.3em 20px; color: #472B2Bl; border-bottom: 1px solid #CC8989; }
p { padding: 0.3em 0 0.3em 20px; }
footer { padding: 0.5em 0 0.3em 20px; color: #999; font-size: 80%%; font-style: italic; }
</style>
</head>
<body>
<h1>%(title)s</h1>
<p>%(body)s</p>
<footer>Daphne</footer>
</body>
</html>
""".replace("\n", "").replace(" ", " ").replace(" ", " ").replace(" ", " ") # Shorten it a bit, bytes wise
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
http.Request.__init__(self, *args, **kwargs) http.Request.__init__(self, *args, **kwargs)
# Easy factory link # Easy factory link
@ -157,11 +175,34 @@ class WebRequest(http.Request):
"status": self.code, "status": self.code,
"method": self.method.decode("ascii"), "method": self.method.decode("ascii"),
"client": "%s:%s" % (self.client.host, self.client.port), "client": "%s:%s" % (self.client.host, self.client.port),
"time_taken": time.time() - self.request_start, "time_taken": self.duration(),
}) })
else: else:
logger.debug("HTTP response chunk for %s", self.reply_channel) logger.debug("HTTP response chunk for %s", self.reply_channel)
def duration(self):
"""
Returns the time since the start of the request.
"""
return time.time() - self.request_start
def basic_error(self, status, status_text, body):
"""
Responds with a server-level error page (very basic)
"""
self.serverResponse({
"status": status,
"status_text": status_text,
"headers": [
("Content-Type", b"text/html; charset=utf-8"),
],
"content": (self.error_template % {
"title": str(status) + " " + status_text.decode("ascii"),
"body": body,
}).encode("utf8"),
})
class HTTPProtocol(http.HTTPChannel): class HTTPProtocol(http.HTTPChannel):
@ -178,10 +219,11 @@ class HTTPFactory(http.HTTPFactory):
protocol = HTTPProtocol protocol = HTTPProtocol
def __init__(self, channel_layer, action_logger=None): def __init__(self, channel_layer, action_logger=None, timeout=120):
http.HTTPFactory.__init__(self) http.HTTPFactory.__init__(self)
self.channel_layer = channel_layer self.channel_layer = channel_layer
self.action_logger = action_logger self.action_logger = action_logger
self.timeout = timeout
# We track all sub-protocols for response channel mapping # We track all sub-protocols for response channel mapping
self.reply_protocols = {} self.reply_protocols = {}
# Make a factory for WebSocket protocols # Make a factory for WebSocket protocols
@ -211,3 +253,12 @@ class HTTPFactory(http.HTTPFactory):
""" """
if self.action_logger: if self.action_logger:
self.action_logger(protocol, action, details) self.action_logger(protocol, action, details)
def check_timeouts(self):
"""
Runs through all HTTP protocol instances and times them out if they've
taken too long (and so their message is probably expired)
"""
for protocol in list(self.reply_protocols.values()):
if isinstance(protocol, WebRequest) and protocol.duration() > self.timeout:
protocol.basic_error(503, b"Service Unavailable", "Worker server failed to respond within time limit.")

View File

@ -8,17 +8,27 @@ logger = logging.getLogger(__name__)
class Server(object): class Server(object):
def __init__(self, channel_layer, host="127.0.0.1", port=8000, signal_handlers=True, action_logger=None): def __init__(
self,
channel_layer,
host="127.0.0.1",
port=8000,
signal_handlers=True,
action_logger=None,
http_timeout=120
):
self.channel_layer = channel_layer self.channel_layer = channel_layer
self.host = host self.host = host
self.port = port self.port = port
self.signal_handlers = signal_handlers self.signal_handlers = signal_handlers
self.action_logger = action_logger self.action_logger = action_logger
self.http_timeout = http_timeout
def run(self): def run(self):
self.factory = HTTPFactory(self.channel_layer, self.action_logger) self.factory = HTTPFactory(self.channel_layer, self.action_logger, timeout=self.http_timeout)
reactor.listenTCP(self.port, self.factory, interface=self.host) reactor.listenTCP(self.port, self.factory, interface=self.host)
reactor.callLater(0, self.backend_reader) reactor.callLater(0, self.backend_reader)
reactor.callLater(2, self.timeout_checker)
reactor.run(installSignalHandlers=self.signal_handlers) reactor.run(installSignalHandlers=self.signal_handlers)
def backend_reader(self): def backend_reader(self):
@ -41,3 +51,11 @@ class Server(object):
# Deal with the message # Deal with the message
self.factory.dispatch_reply(channel, message) self.factory.dispatch_reply(channel, message)
reactor.callLater(delay, self.backend_reader) reactor.callLater(delay, self.backend_reader)
def timeout_checker(self):
"""
Called periodically to enforce timeout rules on HTTP connections
(but not WebSocket)
"""
self.factory.check_timeouts()
reactor.callLater(2, self.timeout_checker)