From fe43646e5ddc24b57d5f607dfcfd16f744d26823 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 24 Aug 2014 12:00:49 -0400 Subject: [PATCH 01/10] Set version number in branch4.0 to 4.0.2.dev1 --- setup.py | 2 +- tornado/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 3e6e5b32..e99200a7 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,7 @@ http://api.mongodb.org/python/current/installation.html#osx kwargs = {} -version = "4.0.1" +version = "4.0.2.dev1" with open('README.rst') as f: kwargs['long_description'] = f.read() diff --git a/tornado/__init__.py b/tornado/__init__.py index eefe0f2c..6e444ca7 100644 --- a/tornado/__init__.py +++ b/tornado/__init__.py @@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -version = "4.0.1" -version_info = (4, 0, 1, -100) +version = "4.0.2.dev1" +version_info = (4, 0, 2, -100) From 7bf3b143e05b06c92c57b318cffe1b572ae82aec Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Tue, 19 Aug 2014 23:01:28 -0400 Subject: [PATCH 02/10] Fix a regression in which a timeout could fire after being cancelled. Closes #1148. --- tornado/ioloop.py | 14 +++++++++----- tornado/test/ioloop_test.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/tornado/ioloop.py b/tornado/ioloop.py index e15252d3..a8f662ac 100644 --- a/tornado/ioloop.py +++ b/tornado/ioloop.py @@ -754,17 +754,18 @@ class PollIOLoop(IOLoop): # Do not run anything until we have determined which ones # are ready, so timeouts that call add_timeout cannot # schedule anything in this iteration. + due_timeouts = [] if self._timeouts: now = self.time() while self._timeouts: if self._timeouts[0].callback is None: - # the timeout was cancelled + # The timeout was cancelled. Note that the + # cancellation check is repeated below for timeouts + # that are cancelled by another timeout or callback. heapq.heappop(self._timeouts) self._cancellations -= 1 elif self._timeouts[0].deadline <= now: - timeout = heapq.heappop(self._timeouts) - callbacks.append(timeout.callback) - del timeout + due_timeouts.append(heapq.heappop(self._timeouts)) else: break if (self._cancellations > 512 @@ -778,9 +779,12 @@ class PollIOLoop(IOLoop): for callback in callbacks: self._run_callback(callback) + for timeout in due_timeouts: + if timeout.callback is not None: + self._run_callback(timeout.callback) # Closures may be holding on to a lot of memory, so allow # them to be freed before we go into our poll wait. - callbacks = callback = None + callbacks = callback = due_timeouts = timeout = None if self._callbacks: # If any callbacks or timeouts called add_callback, diff --git a/tornado/test/ioloop_test.py b/tornado/test/ioloop_test.py index 8bf6ee26..110158d1 100644 --- a/tornado/test/ioloop_test.py +++ b/tornado/test/ioloop_test.py @@ -173,6 +173,25 @@ class TestIOLoop(AsyncTestCase): self.io_loop.add_callback(lambda: self.io_loop.add_callback(self.stop)) self.wait() + def test_remove_timeout_from_timeout(self): + calls = [False, False] + + # Schedule several callbacks and wait for them all to come due at once. + # t2 should be cancelled by t1, even though it is already scheduled to + # be run before the ioloop even looks at it. + now = self.io_loop.time() + def t1(): + calls[0] = True + self.io_loop.remove_timeout(t2_handle) + self.io_loop.add_timeout(now + 0.01, t1) + def t2(): + calls[1] = True + t2_handle = self.io_loop.add_timeout(now + 0.02, t2) + self.io_loop.add_timeout(now + 0.03, self.stop) + time.sleep(0.03) + self.wait() + self.assertEqual(calls, [True, False]) + def test_timeout_with_arguments(self): # This tests that all the timeout methods pass through *args correctly. results = [] From ed7bd3a2bd8948f8a3745582233f50d14ba86720 Mon Sep 17 00:00:00 2001 From: Niko Wilbert Date: Fri, 22 Aug 2014 10:48:42 +0200 Subject: [PATCH 03/10] Fixed support for test generators This fixes the problem that AsyncTestCase no longer seemed to work with test generators (as supported by Nose, http://nose.readthedocs.org/en/latest/writing_tests.html#test-generators). --- tornado/testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tornado/testing.py b/tornado/testing.py index b4bfb274..6dd759b9 100644 --- a/tornado/testing.py +++ b/tornado/testing.py @@ -114,8 +114,8 @@ class _TestMethodWrapper(object): def __init__(self, orig_method): self.orig_method = orig_method - def __call__(self): - result = self.orig_method() + def __call__(self, *args, **kwargs): + result = self.orig_method(*args, **kwargs) if isinstance(result, types.GeneratorType): raise TypeError("Generator test methods should be decorated with " "tornado.testing.gen_test") From d9689739fe4139ada808540f9864743fb14b6914 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 23 Aug 2014 18:12:19 -0400 Subject: [PATCH 04/10] Fix a regression in gzip output for StaticFileHandler. Streaming static responses confused the gzip output transform since it could no longer set the correct outgoing content-length. Now it will fall back to chunked encoding in this case. Closes #1156. --- tornado/test/web_test.py | 15 ++++++++++++++- tornado/web.py | 10 ++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py index 496ae2e8..b8ee49f5 100644 --- a/tornado/test/web_test.py +++ b/tornado/test/web_test.py @@ -1348,7 +1348,9 @@ class GzipTestCase(SimpleHandlerTestCase): self.write('hello world') def get_app_kwargs(self): - return dict(gzip=True) + return dict( + gzip=True, + static_path=os.path.join(os.path.dirname(__file__), 'static')) def test_gzip(self): response = self.fetch('/') @@ -1361,6 +1363,17 @@ class GzipTestCase(SimpleHandlerTestCase): 'gzip') self.assertEqual(response.headers['Vary'], 'Accept-Encoding') + def test_gzip_static(self): + # The streaming responses in StaticFileHandler have subtle + # interactions with the gzip output so test this case separately. + response = self.fetch('/robots.txt') + self.assertEqual( + response.headers.get( + 'Content-Encoding', + response.headers.get('X-Consumed-Content-Encoding')), + 'gzip') + self.assertEqual(response.headers['Vary'], 'Accept-Encoding') + def test_gzip_not_requested(self): response = self.fetch('/', use_gzip=False) self.assertNotIn('Content-Encoding', response.headers) diff --git a/tornado/web.py b/tornado/web.py index 25ac56eb..a6633515 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -2548,7 +2548,6 @@ class GZipContentEncoding(OutputTransform): ctype = _unicode(headers.get("Content-Type", "")).split(";")[0] self._gzipping = self._compressible_type(ctype) and \ (not finishing or len(chunk) >= self.MIN_LENGTH) and \ - (finishing or "Content-Length" not in headers) and \ ("Content-Encoding" not in headers) if self._gzipping: headers["Content-Encoding"] = "gzip" @@ -2556,7 +2555,14 @@ class GZipContentEncoding(OutputTransform): self._gzip_file = gzip.GzipFile(mode="w", fileobj=self._gzip_value) chunk = self.transform_chunk(chunk, finishing) if "Content-Length" in headers: - headers["Content-Length"] = str(len(chunk)) + # The original content length is no longer correct. + # If this is the last (and only) chunk, we can set the new + # content-length; otherwise we remove it and fall back to + # chunked encoding. + if finishing: + headers["Content-Length"] = str(len(chunk)) + else: + del headers["Content-Length"] return status_code, headers, chunk def transform_chunk(self, chunk, finishing): From fc9a3acdb40ff72dc81cb78be57c2adbe56ff04b Mon Sep 17 00:00:00 2001 From: Eddie Mishelevich Date: Tue, 26 Aug 2014 17:04:06 +0300 Subject: [PATCH 05/10] Add missing argument max_buffer_size to TCPClient.connect in SimpleAsyncHTTPClient --- tornado/simple_httpclient.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index 516dc20b..679e7e76 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -222,6 +222,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): stack_context.wrap(self._on_timeout)) self.tcp_client.connect(host, port, af=af, ssl_options=ssl_options, + max_buffer_size=self.max_buffer_size, callback=self._on_connect) def _get_ssl_options(self, scheme): From c1a123a8cd0b5de360e280ea34b8e97c078f2291 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 30 Aug 2014 15:27:31 -0400 Subject: [PATCH 06/10] Fix a subtle reference cycle that can lead to increased memory consumption. Closes #1165. --- tornado/gen.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tornado/gen.py b/tornado/gen.py index 06f2715d..4bb82d42 100644 --- a/tornado/gen.py +++ b/tornado/gen.py @@ -185,7 +185,18 @@ def _make_coroutine_wrapper(func, replace_callback): future.set_exc_info(sys.exc_info()) else: Runner(result, future, yielded) - return future + try: + return future + finally: + # Subtle memory optimization: if next() raised an exception, + # the future's exc_info contains a traceback which + # includes this stack frame. This creates a cycle, + # which will be collected at the next full GC but has + # been shown to greatly increase memory usage of + # benchmarks (relative to the refcount-based scheme + # used in the absence of cycles). We can avoid the + # cycle by clearing the local variable after we return it. + future = None future.set_result(result) return future return wrapper From f16aa39c976fcc5f17cfc44f334c7238b53a3814 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 30 Aug 2014 22:52:54 -0400 Subject: [PATCH 07/10] Limit the number of connections we will accept per call to accept_handler. This ensures that other callbacks scheduled on the IOLoop have a chance to run. This shows up as increased memory usage in benchmark scenarios (since 4.0 allowed simple http transactions to be processed synchronously). --- tornado/netutil.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tornado/netutil.py b/tornado/netutil.py index 336c8062..f147c974 100644 --- a/tornado/netutil.py +++ b/tornado/netutil.py @@ -35,6 +35,11 @@ except ImportError: # ssl is not available on Google App Engine ssl = None +try: + xrange # py2 +except NameError: + xrange = range # py3 + if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ ssl_match_hostname = ssl.match_hostname SSLCertificateError = ssl.CertificateError @@ -60,8 +65,11 @@ _ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN) if hasattr(errno, "WSAEWOULDBLOCK"): _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,) +# Default backlog used when calling sock.listen() +_DEFAULT_BACKLOG = 128 -def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None): +def bind_sockets(port, address=None, family=socket.AF_UNSPEC, + backlog=_DEFAULT_BACKLOG, flags=None): """Creates listening sockets bound to the given port and address. Returns a list of socket objects (multiple sockets are returned if @@ -141,7 +149,7 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags return sockets if hasattr(socket, 'AF_UNIX'): - def bind_unix_socket(file, mode=0o600, backlog=128): + def bind_unix_socket(file, mode=0o600, backlog=_DEFAULT_BACKLOG): """Creates a listening unix socket. If a socket with the given name already exists, it will be deleted. @@ -184,7 +192,18 @@ def add_accept_handler(sock, callback, io_loop=None): io_loop = IOLoop.current() def accept_handler(fd, events): - while True: + # More connections may come in while we're handling callbacks; + # to prevent starvation of other tasks we must limit the number + # of connections we accept at a time. Ideally we would accept + # up to the number of connections that were waiting when we + # entered this method, but this information is not available + # (and rearranging this method to call accept() as many times + # as possible before running any callbacks would have adverse + # effects on load balancing in multiprocess configurations). + # Instead, we use the (default) listen backlog as a rough + # heuristic for the number of connections we can reasonably + # accept at once. + for i in xrange(_DEFAULT_BACKLOG): try: connection, address = sock.accept() except socket.error as e: From 39fd0eee635b082826995d3f7ee047c0c1be510a Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 31 Aug 2014 23:33:28 -0400 Subject: [PATCH 08/10] Fix an erroneous return of None in IOStream.connect. This causes AttributeErrors in TCPClient, although I can only reproduce this case reliably on freebsd. Closes #1168. --- tornado/iostream.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tornado/iostream.py b/tornado/iostream.py index 99c681d8..1cc39d9f 100644 --- a/tornado/iostream.py +++ b/tornado/iostream.py @@ -993,6 +993,11 @@ class IOStream(BaseIOStream): """ self._connecting = True + if callback is not None: + self._connect_callback = stack_context.wrap(callback) + future = None + else: + future = self._connect_future = TracebackFuture() try: self.socket.connect(address) except socket.error as e: @@ -1008,12 +1013,7 @@ class IOStream(BaseIOStream): gen_log.warning("Connect error on fd %s: %s", self.socket.fileno(), e) self.close(exc_info=True) - return - if callback is not None: - self._connect_callback = stack_context.wrap(callback) - future = None - else: - future = self._connect_future = TracebackFuture() + return future self._add_io_state(self.io_loop.WRITE) return future From 5d0d8cbaac3fc1c15b53cc49975d31e84dc1d500 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Mon, 1 Sep 2014 11:12:27 -0400 Subject: [PATCH 09/10] Add release notes for 4.0.2. --- docs/releases.rst | 1 + docs/releases/v4.0.2.rst | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 docs/releases/v4.0.2.rst diff --git a/docs/releases.rst b/docs/releases.rst index d239c3ec..e2ca2147 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -4,6 +4,7 @@ Release notes .. toctree:: :maxdepth: 2 + releases/v4.0.2 releases/v4.0.1 releases/v4.0.0 releases/v3.2.2 diff --git a/docs/releases/v4.0.2.rst b/docs/releases/v4.0.2.rst new file mode 100644 index 00000000..9f92e2ee --- /dev/null +++ b/docs/releases/v4.0.2.rst @@ -0,0 +1,22 @@ +What's new in Tornado 4.0.2 +=========================== + +In progress +----------- + +Bug fixes +~~~~~~~~~ + +* Fixed a bug that could sometimes cause a timeout to fire after being + cancelled. +* `.AsyncTestCase` once again passes along arguments to test methods, + making it compatible with extensions such as Nose's test generators. +* `.StaticFileHandler` can again compress its responses when gzip is enabled. +* ``simple_httpclient`` passes its ``max_buffer_size`` argument to the + underlying stream. +* Fixed a reference cycle that can lead to increased memory consumption. +* `.add_accept_handler` will now limit the number of times it will call + `~socket.socket.accept` per `.IOLoop` iteration, addressing a potential + starvation issue. +* Improved error handling in `.IOStream.connect` (primarily for FreeBSD + systems) From b18b12b532790f40edf431625d184bf8e479956b Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Wed, 10 Sep 2014 23:47:51 -0400 Subject: [PATCH 10/10] Set version number to 4.0.2 --- docs/releases/v4.0.2.rst | 4 ++-- setup.py | 2 +- tornado/__init__.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/releases/v4.0.2.rst b/docs/releases/v4.0.2.rst index 9f92e2ee..1dab6125 100644 --- a/docs/releases/v4.0.2.rst +++ b/docs/releases/v4.0.2.rst @@ -1,8 +1,8 @@ What's new in Tornado 4.0.2 =========================== -In progress ------------ +Sept 10, 2014 +------------- Bug fixes ~~~~~~~~~ diff --git a/setup.py b/setup.py index e99200a7..55ba8caf 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,7 @@ http://api.mongodb.org/python/current/installation.html#osx kwargs = {} -version = "4.0.2.dev1" +version = "4.0.2" with open('README.rst') as f: kwargs['long_description'] = f.read() diff --git a/tornado/__init__.py b/tornado/__init__.py index 6e444ca7..ac2112fe 100644 --- a/tornado/__init__.py +++ b/tornado/__init__.py @@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -version = "4.0.2.dev1" -version_info = (4, 0, 2, -100) +version = "4.0.2" +version_info = (4, 0, 2, 0)