From fd9b9952f48ba57ddc25410a660a4e8a6b5d8a2f Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Wed, 16 Jul 2014 20:23:42 -0400 Subject: [PATCH 1/8] Fall back to pure-python mode on any exception during the build. Stop the futile process of trying to enumerate which kinds of exceptions distutils may throw (the latest potential addition: ValueError). Closes #1115. --- setup.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index f0877369..5a464e50 100644 --- a/setup.py +++ b/setup.py @@ -34,16 +34,6 @@ from distutils.core import Extension # to support installing without the extension on platforms where # no compiler is available. from distutils.command.build_ext import build_ext -from distutils.errors import CCompilerError -from distutils.errors import DistutilsPlatformError, DistutilsExecError -if sys.platform == 'win32' and sys.version_info > (2, 6): - # 2.6's distutils.msvc9compiler can raise an IOError when failing to - # find the compiler - build_errors = (CCompilerError, DistutilsExecError, - DistutilsPlatformError, IOError) -else: - build_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, - SystemError) class custom_build_ext(build_ext): """Allow C extension building to fail. @@ -83,7 +73,7 @@ http://api.mongodb.org/python/current/installation.html#osx def run(self): try: build_ext.run(self) - except build_errors: + except Exception: e = sys.exc_info()[1] sys.stdout.write('%s\n' % str(e)) warnings.warn(self.warning_message % ("Extension modules", @@ -95,7 +85,7 @@ http://api.mongodb.org/python/current/installation.html#osx name = ext.name try: build_ext.build_extension(self, ext) - except build_errors: + except Exception: e = sys.exc_info()[1] sys.stdout.write('%s\n' % str(e)) warnings.warn(self.warning_message % ("The %s extension " From 7550e8bb55c7a9e7f3fd68918d0a91ce845d36f4 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Thu, 17 Jul 2014 23:33:34 -0400 Subject: [PATCH 2/8] Add missing return statements to call_at and call_later. Closes #1119. --- tornado/ioloop.py | 4 ++-- tornado/test/ioloop_test.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tornado/ioloop.py b/tornado/ioloop.py index da9b7dbd..738cc964 100644 --- a/tornado/ioloop.py +++ b/tornado/ioloop.py @@ -477,7 +477,7 @@ class IOLoop(Configurable): .. versionadded:: 4.0 """ - self.call_at(self.time() + delay, callback, *args, **kwargs) + return self.call_at(self.time() + delay, callback, *args, **kwargs) def call_at(self, when, callback, *args, **kwargs): """Runs the ``callback`` at the absolute time designated by ``when``. @@ -493,7 +493,7 @@ class IOLoop(Configurable): .. versionadded:: 4.0 """ - self.add_timeout(when, callback, *args, **kwargs) + return self.add_timeout(when, callback, *args, **kwargs) def remove_timeout(self, timeout): """Cancels a pending timeout. diff --git a/tornado/test/ioloop_test.py b/tornado/test/ioloop_test.py index e21d5d4c..8bf6ee26 100644 --- a/tornado/test/ioloop_test.py +++ b/tornado/test/ioloop_test.py @@ -185,6 +185,23 @@ class TestIOLoop(AsyncTestCase): self.wait() self.assertEqual(results, [1, 2, 3, 4]) + def test_add_timeout_return(self): + # All the timeout methods return non-None handles that can be + # passed to remove_timeout. + handle = self.io_loop.add_timeout(self.io_loop.time(), lambda: None) + self.assertFalse(handle is None) + self.io_loop.remove_timeout(handle) + + def test_call_at_return(self): + handle = self.io_loop.call_at(self.io_loop.time(), lambda: None) + self.assertFalse(handle is None) + self.io_loop.remove_timeout(handle) + + def test_call_later_return(self): + handle = self.io_loop.call_later(0, lambda: None) + self.assertFalse(handle is None) + self.io_loop.remove_timeout(handle) + def test_close_file_object(self): """When a file object is used instead of a numeric file descriptor, the object should be closed (by IOLoop.close(all_fds=True), From a78184f90c21e3b2d288891e243c111f8091c3d4 Mon Sep 17 00:00:00 2001 From: WEI Zhicheng Date: Wed, 6 Aug 2014 18:35:24 +0800 Subject: [PATCH 3/8] Fix `PeriodicCallback' when callback function return `Future' and has `Exception' will silence ignore --- tornado/ioloop.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tornado/ioloop.py b/tornado/ioloop.py index 738cc964..e15252d3 100644 --- a/tornado/ioloop.py +++ b/tornado/ioloop.py @@ -969,10 +969,11 @@ class PeriodicCallback(object): if not self._running: return try: - self.callback() + return self.callback() except Exception: self.io_loop.handle_callback_exception(self.callback) - self._schedule_next() + finally: + self._schedule_next() def _schedule_next(self): if self._running: From 87fa80c9ebd6eda8c1596bed738bdf4fb8df1e67 Mon Sep 17 00:00:00 2001 From: WEI Zhicheng Date: Fri, 8 Aug 2014 01:49:39 +0800 Subject: [PATCH 4/8] Fix `IOStream' when callback function return `Future' and has `Exception' will silence ignore --- tornado/iostream.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tornado/iostream.py b/tornado/iostream.py index 3ebcd586..99c681d8 100644 --- a/tornado/iostream.py +++ b/tornado/iostream.py @@ -505,7 +505,7 @@ class BaseIOStream(object): def wrapper(): self._pending_callbacks -= 1 try: - callback(*args) + return callback(*args) except Exception: app_log.error("Uncaught exception, closing connection.", exc_info=True) @@ -517,7 +517,8 @@ class BaseIOStream(object): # Re-raise the exception so that IOLoop.handle_callback_exception # can see it and log the error raise - self._maybe_add_error_listener() + finally: + self._maybe_add_error_listener() # We schedule callbacks to be run on the next IOLoop iteration # rather than running them directly for several reasons: # * Prevents unbounded stack growth when a callback calls an From 5a7719e0e9e3c682e06870859ee11cae51d8f9ca Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 9 Aug 2014 14:09:22 -0400 Subject: [PATCH 5/8] Fix an AttributeError in WebSocketClientConnection._on_close. Fix an issue that was preventing this issue from showing up in my tests. Closes #1140. --- tornado/websocket.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index a77e02c4..bc1a7666 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -703,7 +703,7 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): def _on_close(self): self.on_message(None) - self.resolver.close() + self.tcp_client.close() super(WebSocketClientConnection, self)._on_close() def _on_http_response(self, response): @@ -734,6 +734,11 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): self.stream = self.connection.detach() self.stream.set_close_callback(self._on_close) + # Once we've taken over the connection, clear the final callback + # we set on the http request. This deactivates the error handling + # in simple_httpclient that would otherwise interfere with our + # ability to see exceptions. + self.final_callback = None self.connect_future.set_result(self) From c7ae10d668f10cb5797fa88eeca9b3c63644b1cb Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 10 Aug 2014 14:02:39 -0400 Subject: [PATCH 6/8] Correctly handle 204 response codes that do not include a content-length. If a server sends a 204 with no content-length we would wait for the server to close the connection instead of ending the request (any such servers are ignoring the "Connection: close" header we send, but apparently exist). Move some content-length logic from simple_httpclient.py to http1connection.py. Fix the client-side use of on_connection_close; this affects the handling of any HTTPInputException. This fixes regressions relative to Tornado 3.2. Conflicts: tornado/websocket.py --- tornado/http1connection.py | 49 +++++++++++++++++++++++--- tornado/simple_httpclient.py | 31 ++++------------ tornado/test/simple_httpclient_test.py | 26 +++++++++++++- tornado/websocket.py | 8 +++-- 4 files changed, 80 insertions(+), 34 deletions(-) diff --git a/tornado/http1connection.py b/tornado/http1connection.py index 72f729d7..1ac24f52 100644 --- a/tornado/http1connection.py +++ b/tornado/http1connection.py @@ -21,6 +21,8 @@ from __future__ import absolute_import, division, print_function, with_statement +import re + from tornado.concurrent import Future from tornado.escape import native_str, utf8 from tornado import gen @@ -191,8 +193,17 @@ class HTTP1Connection(httputil.HTTPConnection): skip_body = True code = start_line.code if code == 304: + # 304 responses may include the content-length header + # but do not actually have a body. + # http://tools.ietf.org/html/rfc7230#section-3.3 skip_body = True if code >= 100 and code < 200: + # 1xx responses should never indicate the presence of + # a body. + if ('Content-Length' in headers or + 'Transfer-Encoding' in headers): + raise httputil.HTTPInputError( + "Response code %d cannot have body" % code) # TODO: client delegates will get headers_received twice # in the case of a 100-continue. Document or change? yield self._read_message(delegate) @@ -201,7 +212,8 @@ class HTTP1Connection(httputil.HTTPConnection): not self._write_finished): self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if not skip_body: - body_future = self._read_body(headers, delegate) + body_future = self._read_body( + start_line.code if self.is_client else 0, headers, delegate) if body_future is not None: if self._body_timeout is None: yield body_future @@ -482,12 +494,36 @@ class HTTP1Connection(httputil.HTTPConnection): data[eol:100]) return start_line, headers - def _read_body(self, headers, delegate): - content_length = headers.get("Content-Length") - if content_length: - content_length = int(content_length) + def _read_body(self, code, headers, delegate): + if "Content-Length" in headers: + if "," in headers["Content-Length"]: + # Proxies sometimes cause Content-Length headers to get + # duplicated. If all the values are identical then we can + # use them but if they differ it's an error. + pieces = re.split(r',\s*', headers["Content-Length"]) + if any(i != pieces[0] for i in pieces): + raise httputil.HTTPInputError( + "Multiple unequal Content-Lengths: %r" % + headers["Content-Length"]) + headers["Content-Length"] = pieces[0] + content_length = int(headers["Content-Length"]) + if content_length > self._max_body_size: raise httputil.HTTPInputError("Content-Length too long") + else: + content_length = None + + if code == 204: + # This response code is not allowed to have a non-empty body, + # and has an implicit length of zero instead of read-until-close. + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 + if ("Transfer-Encoding" in headers or + content_length not in (None, 0)): + raise httputil.HTTPInputError( + "Response with code %d should not have body" % code) + content_length = 0 + + if content_length is not None: return self._read_fixed_body(content_length, delegate) if headers.get("Transfer-Encoding") == "chunked": return self._read_chunked_body(delegate) @@ -582,6 +618,9 @@ class _GzipMessageDelegate(httputil.HTTPMessageDelegate): self._delegate.data_received(tail) return self._delegate.finish() + def on_connection_close(self): + return self._delegate.on_connection_close() + class HTTP1ServerConnection(object): """An HTTP/1.x server.""" diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index 99f1c1a7..516dc20b 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -277,7 +277,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): stream.close() return self.stream = stream - self.stream.set_close_callback(self._on_close) + self.stream.set_close_callback(self.on_connection_close) self._remove_timeout() if self.final_callback is None: return @@ -418,12 +418,15 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): # pass it along, unless it's just the stream being closed. return isinstance(value, StreamClosedError) - def _on_close(self): + def on_connection_close(self): if self.final_callback is not None: message = "Connection closed" if self.stream.error: raise self.stream.error - raise HTTPError(599, message) + try: + raise HTTPError(599, message) + except HTTPError: + self._handle_exception(*sys.exc_info()) def headers_received(self, first_line, headers): if self.request.expect_100_continue and first_line.code == 100: @@ -433,20 +436,6 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): self.code = first_line.code self.reason = first_line.reason - if "Content-Length" in self.headers: - if "," in self.headers["Content-Length"]: - # Proxies sometimes cause Content-Length headers to get - # duplicated. If all the values are identical then we can - # use them but if they differ it's an error. - pieces = re.split(r',\s*', self.headers["Content-Length"]) - if any(i != pieces[0] for i in pieces): - raise ValueError("Multiple unequal Content-Lengths: %r" % - self.headers["Content-Length"]) - self.headers["Content-Length"] = pieces[0] - content_length = int(self.headers["Content-Length"]) - else: - content_length = None - if self.request.header_callback is not None: # Reassemble the start line. self.request.header_callback('%s %s %s\r\n' % first_line) @@ -454,14 +443,6 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): self.request.header_callback("%s: %s\r\n" % (k, v)) self.request.header_callback('\r\n') - if 100 <= self.code < 200 or self.code == 204: - # These response codes never have bodies - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 - if ("Transfer-Encoding" in self.headers or - content_length not in (None, 0)): - raise ValueError("Response with code %d should not have body" % - self.code) - def finish(self): data = b''.join(self.chunks) self._remove_timeout() diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index f17da7e0..d3af70f1 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -294,10 +294,13 @@ class SimpleHTTPClientTestMixin(object): self.assertEqual(response.code, 204) # 204 status doesn't need a content-length, but tornado will # add a zero content-length anyway. + # + # A test without a content-length header is included below + # in HTTP204NoContentTestCase. self.assertEqual(response.headers["Content-length"], "0") # 204 status with non-zero content length is malformed - with ExpectLog(app_log, "Uncaught exception"): + with ExpectLog(gen_log, "Malformed HTTP message"): response = self.fetch("/no_content?error=1") self.assertEqual(response.code, 599) @@ -476,6 +479,27 @@ class HTTP100ContinueTestCase(AsyncHTTPTestCase): self.assertEqual(res.body, b'A') +class HTTP204NoContentTestCase(AsyncHTTPTestCase): + def respond_204(self, request): + # A 204 response never has a body, even if doesn't have a content-length + # (which would otherwise mean read-until-close). Tornado always + # sends a content-length, so we simulate here a server that sends + # no content length and does not close the connection. + # + # Tests of a 204 response with a Content-Length header are included + # in SimpleHTTPClientTestMixin. + request.connection.stream.write( + b"HTTP/1.1 204 No content\r\n\r\n") + + def get_app(self): + return self.respond_204 + + def test_204_no_content(self): + resp = self.fetch('/') + self.assertEqual(resp.code, 204) + self.assertEqual(resp.body, b'') + + class HostnameMappingTestCase(AsyncHTTPTestCase): def setUp(self): super(HostnameMappingTestCase, self).setUp() diff --git a/tornado/websocket.py b/tornado/websocket.py index bc1a7666..ed520d58 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -701,10 +701,12 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): self.protocol.close(code, reason) self.protocol = None - def _on_close(self): + def on_connection_close(self): + if not self.connect_future.done(): + self.connect_future.set_exception(StreamClosedError()) self.on_message(None) self.tcp_client.close() - super(WebSocketClientConnection, self)._on_close() + super(WebSocketClientConnection, self).on_connection_close() def _on_http_response(self, response): if not self.connect_future.done(): @@ -733,7 +735,7 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): self._timeout = None self.stream = self.connection.detach() - self.stream.set_close_callback(self._on_close) + self.stream.set_close_callback(self.on_connection_close) # Once we've taken over the connection, clear the final callback # we set on the http request. This deactivates the error handling # in simple_httpclient that would otherwise interfere with our From 6c6d16a9c65bcb7e5a672fc878fc32f255742a9f Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 10 Aug 2014 15:19:48 -0400 Subject: [PATCH 7/8] Add release notes for 4.0.1 and update version number. --- docs/releases.rst | 1 + docs/releases/v4.0.1.rst | 20 ++++++++++++++++++++ setup.py | 2 +- tornado/__init__.py | 4 ++-- 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 docs/releases/v4.0.1.rst diff --git a/docs/releases.rst b/docs/releases.rst index edb4223a..d239c3ec 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -4,6 +4,7 @@ Release notes .. toctree:: :maxdepth: 2 + releases/v4.0.1 releases/v4.0.0 releases/v3.2.2 releases/v3.2.1 diff --git a/docs/releases/v4.0.1.rst b/docs/releases/v4.0.1.rst new file mode 100644 index 00000000..4221e328 --- /dev/null +++ b/docs/releases/v4.0.1.rst @@ -0,0 +1,20 @@ +What's new in Tornado 4.0.1 +=========================== + +In progress +----------- + +* The build will now fall back to pure-python mode if the C extension + fails to build for any reason (previously it would fall back for some + errors but not others). +* `.IOLoop.call_at` and `.IOLoop.call_later` now always return + a timeout handle for use with `.IOLoop.remove_timeout`. +* If any callback of a `.PeriodicCallback` or `.IOStream` returns a + `.Future`, any error raised in that future will now be logged + (similar to the behavior of `.IOLoop.add_callback`). +* Fixed an exception in client-side websocket connections when the + connection is closed. +* ``simple_httpclient`` once again correctly handles 204 status + codes with no content-length header. +* Fixed a regression in ``simple_httpclient`` that would result in + timeouts for certain kinds of errors. diff --git a/setup.py b/setup.py index 5a464e50..362bfc03 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,7 @@ http://api.mongodb.org/python/current/installation.html#osx kwargs = {} -version = "4.0" +version = "4.0.1b1" with open('README.rst') as f: kwargs['long_description'] = f.read() diff --git a/tornado/__init__.py b/tornado/__init__.py index fe85e222..61414662 100644 --- a/tornado/__init__.py +++ b/tornado/__init__.py @@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -version = "4.0" -version_info = (4, 0, 0, 0) +version = "4.0.1b1" +version_info = (4, 0, 1, -100) From 54bfbee58650a0312b22eb081e8a38d03fcf1a49 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Tue, 12 Aug 2014 09:11:20 -0400 Subject: [PATCH 8/8] Set version number to 4.0.1. --- docs/releases/v4.0.1.rst | 4 ++-- setup.py | 2 +- tornado/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/releases/v4.0.1.rst b/docs/releases/v4.0.1.rst index 4221e328..855b5ec4 100644 --- a/docs/releases/v4.0.1.rst +++ b/docs/releases/v4.0.1.rst @@ -1,8 +1,8 @@ What's new in Tornado 4.0.1 =========================== -In progress ------------ +Aug 12, 2014 +------------ * The build will now fall back to pure-python mode if the C extension fails to build for any reason (previously it would fall back for some diff --git a/setup.py b/setup.py index 362bfc03..3e6e5b32 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,7 @@ http://api.mongodb.org/python/current/installation.html#osx kwargs = {} -version = "4.0.1b1" +version = "4.0.1" with open('README.rst') as f: kwargs['long_description'] = f.read() diff --git a/tornado/__init__.py b/tornado/__init__.py index 61414662..eefe0f2c 100644 --- a/tornado/__init__.py +++ b/tornado/__init__.py @@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -version = "4.0.1b1" +version = "4.0.1" version_info = (4, 0, 1, -100)