paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-20 14:38:46 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
f3ad2d7a6b745b091632fd3b7d542a7b9fc599e5
zulip/zephyr/management/commands/runtornado.py
Tim Abbott f3ad2d7a6b Centralize dispatch logic for which queue processor to use. 
Previous we had around 4 copies of the logic for deciding whether we
should publish data via a SimpleQueueClient queue, a
TornadoQueueClient queue, or to directly handle the operation, which
resulted in their getting out of sync and buggy (see e.g. the previous
commit).

We need to add a lock around adding things to the queue to work around
a bug with pika's BlockingConnection.

I should note that the previous logic in some places had a bunch of
tests of the form "elif settings.TEST_SUITE" for doing the work that
would have been done by the queue processor directly; these should
have just been "else" clauses -- since we generally want that code to
run on development environments whether or not the test suite is
currently running.

(imported from commit 16bdbed4fff04b1bda6fde3b16bee7359917720b)
2013-03-27 08:19:27 -04:00

378 lines
15 KiB
Python
Raw Blame History

from django.conf import settings
settings.RUNNING_INSIDE_TORNADO = True
from django.core.management.base import BaseCommand, CommandError
from optparse import make_option
import os
import sys
import tornado.web
import logging
import time
from tornado import ioloop
from zephyr.lib.debug import interactive_debug_listen
from zephyr.lib.response import json_response
from zephyr import tornado_callbacks
if settings.USING_RABBITMQ:
from zephyr.lib.queue import queue_client
# A hack to keep track of how much time we spend working, versus sleeping in
# the event loop.
#
# Creating a new event loop instance with a custom impl object fails (events
# don't get processed), so instead we modify the ioloop module variable holding
# the default poll implementation. We need to do this before any Tornado code
# runs that might instantiate the default event loop.
orig_poll_impl = ioloop._poll
class InstrumentedPoll(object):
def __init__(self):
self._underlying = orig_poll_impl()
self._times = []
self._last_print = 0
# Python won't let us subclass e.g. select.epoll, so instead
# we proxy every method. __getattr__ handles anything we
# don't define elsewhere.
def __getattr__(self, name):
return getattr(self._underlying, name)
# Call the underlying poll method, and report timing data.
def poll(self, timeout):
# Avoid accumulating a bunch of insignificant data points
# from short timeouts.
if timeout < 1e-3:
return self._underlying.poll(timeout)
# Record start and end times for the underlying poll
t0 = time.time()
result = self._underlying.poll(timeout)
t1 = time.time()
# Log this datapoint and restrict our log to the past minute
self._times.append((t0, t1))
while self._times and self._times[0][0] < t1 - 60:
self._times.pop(0)
# Report (at most once every 5s) the percentage of time spent
# outside poll
if self._times and t1 - self._last_print >= 5:
total = t1 - self._times[0][0]
in_poll = sum(b-a for a,b in self._times)
if total > 0:
logging.info('Tornado %5.1f%% busy over the past %4.1f seconds'
% (100 * (1 - in_poll/total), total))
self._last_print = t1
return result
ioloop._poll = InstrumentedPoll
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('--nokeepalive', action='store_true',
dest='no_keep_alive', default=False,
help="Tells Tornado to NOT keep alive http connections."),
make_option('--noxheaders', action='store_false',
dest='xheaders', default=True,
help="Tells Tornado to NOT override remote IP with X-Real-IP."),
)
help = "Starts a Tornado Web server wrapping Django."
args = '[optional port number or ipaddr:port]\n (use multiple ports to start multiple servers)'
def handle(self, addrport, **options):
# setup unbuffered I/O
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
interactive_debug_listen()
import django
from django.core.handlers.wsgi import WSGIHandler
from tornado import httpserver, wsgi, web
try:
addr, port = addrport.split(':')
except ValueError:
addr, port = '', addrport
if not addr:
addr = '127.0.0.1'
if not port.isdigit():
raise CommandError("%r is not a valid port number." % port)
xheaders = options.get('xheaders', True)
no_keep_alive = options.get('no_keep_alive', False)
quit_command = 'CTRL-C'
if settings.DEBUG:
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)-8s %(message)s')
def inner_run():
from django.conf import settings
from django.utils import translation
translation.activate(settings.LANGUAGE_CODE)
print "Validating Django models.py..."
self.validate(display_num_errors=True)
print "\nDjango version %s" % (django.get_version())
print "Tornado server is running at http://%s:%s/" % (addr, port)
print "Quit the server with %s." % quit_command
if settings.USING_RABBITMQ:
# Process notifications received via RabbitMQ
def process_notification(chan, method, props, data):
tornado_callbacks.process_notification(data)
queue_client.register_json_consumer('notify_tornado', process_notification)
try:
# Application is an instance of Django's standard wsgi handler.
application = web.Application([(r"/json/get_updates", AsyncDjangoHandler),
(r"/api/v1/get_messages", AsyncDjangoHandler),
(r"/notify_tornado", AsyncDjangoHandler),
], debug=django.conf.settings.DEBUG,
# Disable Tornado's own request logging, since we have our own
log_function=lambda x: None)
# start tornado web server in single-threaded mode
http_server = httpserver.HTTPServer(application,
xheaders=xheaders,
no_keep_alive=no_keep_alive)
http_server.listen(int(port), address=addr)
if django.conf.settings.DEBUG:
ioloop.IOLoop.instance().set_blocking_log_threshold(5)
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
sys.exit(0)
inner_run()
#
# Modify the base Tornado handler for Django
#
from threading import Lock
from django.core.handlers import base
from django.core.urlresolvers import set_script_prefix
from django.core import signals
class AsyncDjangoHandler(tornado.web.RequestHandler, base.BaseHandler):
initLock = Lock()
def __init__(self, *args, **kwargs):
super(AsyncDjangoHandler, self).__init__(*args, **kwargs)
# Set up middleware if needed. We couldn't do this earlier, because
# settings weren't available.
self._request_middleware = None
self.initLock.acquire()
# Check that middleware is still uninitialised.
if self._request_middleware is None:
self.load_middleware()
self.initLock.release()
self._auto_finish = False
def get(self):
from tornado.wsgi import WSGIContainer
from django.core.handlers.wsgi import WSGIRequest
import urllib
environ = WSGIContainer.environ(self.request)
environ['PATH_INFO'] = urllib.unquote(environ['PATH_INFO'])
request = WSGIRequest(environ)
request._tornado_handler = self
set_script_prefix(base.get_script_name(environ))
signals.request_started.send(sender=self.__class__)
try:
response = self.get_response(request)
if not response:
return
finally:
signals.request_finished.send(sender=self.__class__)
self.set_status(response.status_code)
for h in response.items():
self.set_header(h[0], h[1])
if not hasattr(self, "_new_cookies"):
self._new_cookies = []
self._new_cookies.append(response.cookies)
self.write(response.content)
self.finish()
def head(self):
self.get()
def post(self):
self.get()
# Based on django.core.handlers.base: get_response
def get_response(self, request):
"Returns an HttpResponse object for the given HttpRequest"
from django import http
from django.core import exceptions, urlresolvers
from django.conf import settings
try:
try:
# Setup default url resolver for this thread.
urlconf = settings.ROOT_URLCONF
urlresolvers.set_urlconf(urlconf)
resolver = urlresolvers.RegexURLResolver(r'^/', urlconf)
response = None
# Apply request middleware
for middleware_method in self._request_middleware:
response = middleware_method(request)
if response:
break
if hasattr(request, "urlconf"):
# Reset url resolver with a custom urlconf.
urlconf = request.urlconf
urlresolvers.set_urlconf(urlconf)
resolver = urlresolvers.RegexURLResolver(r'^/', urlconf)
### ADDED BY HUMBUG
request._resolver = resolver
### END ADDED BY HUMBUG
callback, callback_args, callback_kwargs = resolver.resolve(
request.path_info)
# Apply view middleware
if response is None:
for middleware_method in self._view_middleware:
response = middleware_method(request, callback, callback_args, callback_kwargs)
if response:
break
### THIS BLOCK MODIFIED BY HUMBUG
if response is None:
from ...decorator import RespondAsynchronously
try:
response = callback(request, *callback_args, **callback_kwargs)
if response is RespondAsynchronously:
return
except Exception, e:
# If the view raised an exception, run it through exception
# middleware, and if the exception middleware returns a
# response, use that. Otherwise, reraise the exception.
for middleware_method in self._exception_middleware:
response = middleware_method(request, e)
if response:
break
if response is None:
raise
if response is None:
try:
view_name = callback.func_name
except AttributeError:
view_name = callback.__class__.__name__ + '.__call__'
raise ValueError("The view %s.%s returned None." %
(callback.__module__, view_name))
# If the response supports deferred rendering, apply template
# response middleware and the render the response
if hasattr(response, 'render') and callable(response.render):
for middleware_method in self._template_response_middleware:
response = middleware_method(request, response)
response = response.render()
except http.Http404, e:
if settings.DEBUG:
from django.views import debug
response = debug.technical_404_response(request, e)
else:
try:
callback, param_dict = resolver.resolve404()
response = callback(request, **param_dict)
except:
try:
response = self.handle_uncaught_exception(request, resolver, sys.exc_info())
finally:
signals.got_request_exception.send(sender=self.__class__, request=request)
except exceptions.PermissionDenied:
logging.warning(
'Forbidden (Permission denied): %s', request.path,
extra={
'status_code': 403,
'request': request
})
try:
callback, param_dict = resolver.resolve403()
response = callback(request, **param_dict)
except:
try:
response = self.handle_uncaught_exception(request,
resolver, sys.exc_info())
finally:
signals.got_request_exception.send(
sender=self.__class__, request=request)
except SystemExit:
# See https://code.djangoproject.com/ticket/4701
raise
except Exception, e:
exc_info = sys.exc_info()
signals.got_request_exception.send(sender=self.__class__, request=request)
return self.handle_uncaught_exception(request, resolver, exc_info)
finally:
# Reset urlconf on the way out for isolation
urlresolvers.set_urlconf(None)
### HUMBUG CHANGE: The remainder of this function was moved
### into its own function, just below, so we can call it from
### finish().
response = self.apply_response_middleware(request, response, resolver)
return response
### Copied from get_response (above in this file)
def apply_response_middleware(self, request, response, resolver):
try:
# Apply response middleware, regardless of the response
for middleware_method in self._response_middleware:
response = middleware_method(request, response)
response = self.apply_response_fixes(request, response)
except: # Any exception should be gathered and handled
signals.got_request_exception.send(sender=self.__class__, request=request)
response = self.handle_uncaught_exception(request, resolver, sys.exc_info())
return response
def humbug_finish(self, response, request, apply_markdown):
# Make sure that Markdown rendering really happened, if requested.
# This is a security issue because it's where we escape HTML.
# c.f. ticket #64
#
# apply_markdown=True is the fail-safe default.
if response['result'] == 'success' and 'messages' in response and apply_markdown:
for msg in response['messages']:
if msg['content_type'] != 'text/html':
self.set_status(500)
return self.finish('Internal error: bad message format')
if response['result'] == 'error':
self.set_status(400)
# Call the Django response middleware on our object so that
# e.g. our own logging code can run; but don't actually use
# the headers from that since sending those to Tornado seems
# tricky; instead just send the (already json-rendered)
# content on to Tornado
django_response = json_response(res_type=response['result'],
data=response, status=self.get_status())
django_response = self.apply_response_middleware(request, django_response,
request._resolver)
return self.finish(django_response.content)
Reference in New Issue View Git Blame Copy Permalink