diff --git a/.travis.yml b/.travis.yml index 4b2ac2bc..e3b33c04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ # https://travis-ci.org/tornadoweb/tornado language: python python: - - 2.7.8 - 2.7 - pypy - 3.3 @@ -9,7 +8,7 @@ python: - 3.5 - 3.6 - nightly - - pypy3 + - pypy3.5-5.8.0 install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install futures mock monotonic trollius; fi diff --git a/docs/releases.rst b/docs/releases.rst index 3a9ef777..128c7603 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -4,6 +4,7 @@ Release notes .. toctree:: :maxdepth: 2 + releases/v4.5.3 releases/v4.5.2 releases/v4.5.1 releases/v4.5.0 diff --git a/docs/releases/v4.5.3.rst b/docs/releases/v4.5.3.rst new file mode 100644 index 00000000..b1102459 --- /dev/null +++ b/docs/releases/v4.5.3.rst @@ -0,0 +1,49 @@ +What's new in Tornado 4.5.2 +=========================== + +Aug 27, 2017 +------------ + +`tornado.curl_httpclient` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Improved debug logging on Python 3. + +`tornado.httpserver` +~~~~~~~~~~~~~~~~~~~~ + +- ``Content-Length`` and ``Transfer-Encoding`` headers are no longer + sent with 1xx or 204 responses (this was already true of 304 + responses). +- Reading chunked requests no longer leaves the connection in a broken + state. + +`tornado.iostream` +~~~~~~~~~~~~~~~~~~ + +- Writing a `memoryview` can no longer result in "BufferError: + Existing exports of data: object cannot be re-sized". + +`tornado.options` +~~~~~~~~~~~~~~~~~ + +- Duplicate option names are now detected properly whether they use + hyphens or underscores. + +`tornado.testing` +~~~~~~~~~~~~~~~~~ + +- `.AsyncHTTPTestCase.fetch` now uses ``127.0.0.1`` instead of + ``localhost``, improving compatibility with systems that have + partially-working ipv6 stacks. + +`tornado.web` +~~~~~~~~~~~~~ + +- It is no longer allowed to send a body with 1xx or 204 responses. + +`tornado.websocket` +~~~~~~~~~~~~~~~~~~~ + +- Requests with invalid websocket headers now get a response with + status code 400 instead of a closed connection. diff --git a/setup.py b/setup.py index 66d846be..a1feea67 100644 --- a/setup.py +++ b/setup.py @@ -103,7 +103,7 @@ http://api.mongodb.org/python/current/installation.html#osx kwargs = {} -version = "4.5.2" +version = "4.5.3" with open('README.rst') as f: kwargs['long_description'] = f.read() diff --git a/tornado/__init__.py b/tornado/__init__.py index 3eaa57b8..fa71bf61 100644 --- a/tornado/__init__.py +++ b/tornado/__init__.py @@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -version = "4.5.2" -version_info = (4, 5, 2, 0) +version = "4.5.3" +version_info = (4, 5, 3, 0) diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py index eef4a17a..8558d65c 100644 --- a/tornado/curl_httpclient.py +++ b/tornado/curl_httpclient.py @@ -493,6 +493,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): def _curl_debug(self, debug_type, debug_msg): debug_types = ('I', '<', '>', '<', '>') + debug_msg = native_str(debug_msg) if debug_type == 0: curl_log.debug('%s', debug_msg.strip()) elif debug_type in (1, 2): diff --git a/tornado/http1connection.py b/tornado/http1connection.py index 53744ece..32bed6c9 100644 --- a/tornado/http1connection.py +++ b/tornado/http1connection.py @@ -349,10 +349,11 @@ class HTTP1Connection(httputil.HTTPConnection): # self._request_start_line.version or # start_line.version? self._request_start_line.version == 'HTTP/1.1' and - # 304 responses have no body (not even a zero-length body), and so - # should not have either Content-Length or Transfer-Encoding. - # headers. + # 1xx, 204 and 304 responses have no body (not even a zero-length + # body), and so should not have either Content-Length or + # Transfer-Encoding headers. start_line.code not in (204, 304) and + (start_line.code < 100 or start_line.code >= 200) and # No need to chunk the output if a Content-Length is specified. 'Content-Length' not in headers and # Applications are discouraged from touching Transfer-Encoding, @@ -592,6 +593,9 @@ class HTTP1Connection(httputil.HTTPConnection): chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64) chunk_len = int(chunk_len.strip(), 16) if chunk_len == 0: + crlf = yield self.stream.read_bytes(2) + if crlf != b'\r\n': + raise httputil.HTTPInputError("improperly terminated chunked request") return total_size += chunk_len if total_size > self._max_body_size: diff --git a/tornado/iostream.py b/tornado/iostream.py index a1619c49..639ed508 100644 --- a/tornado/iostream.py +++ b/tornado/iostream.py @@ -1061,7 +1061,12 @@ class IOStream(BaseIOStream): return chunk def write_to_fd(self, data): - return self.socket.send(data) + try: + return self.socket.send(data) + finally: + # Avoid keeping to data, which can be a memoryview. + # See https://github.com/tornadoweb/tornado/pull/2008 + del data def connect(self, address, callback=None, server_hostname=None): """Connects the socket to a remote address without blocking. @@ -1471,6 +1476,10 @@ class SSLIOStream(IOStream): # simply return 0 bytes written. return 0 raise + finally: + # Avoid keeping to data, which can be a memoryview. + # See https://github.com/tornadoweb/tornado/pull/2008 + del data def read_from_fd(self): if self._ssl_accepting: @@ -1528,7 +1537,12 @@ class PipeIOStream(BaseIOStream): os.close(self.fd) def write_to_fd(self, data): - return os.write(self.fd, data) + try: + return os.write(self.fd, data) + finally: + # Avoid keeping to data, which can be a memoryview. + # See https://github.com/tornadoweb/tornado/pull/2008 + del data def read_from_fd(self): try: diff --git a/tornado/options.py b/tornado/options.py index 0a72cc65..707fbd35 100644 --- a/tornado/options.py +++ b/tornado/options.py @@ -223,9 +223,10 @@ class OptionParser(object): override options set earlier on the command line, but can be overridden by later flags. """ - if name in self._options: + normalized = self._normalize_name(name) + if normalized in self._options: raise Error("Option %r already defined in %s" % - (name, self._options[name].file_name)) + (normalized, self._options[normalized].file_name)) frame = sys._getframe(0) options_file = frame.f_code.co_filename @@ -247,7 +248,6 @@ class OptionParser(object): group_name = group else: group_name = file_name - normalized = self._normalize_name(name) option = _Option(name, file_name=file_name, default=default, type=type, help=help, metavar=metavar, multiple=multiple, diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index 11cb7231..59eb6fd1 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -786,9 +786,12 @@ class KeepAliveTest(AsyncHTTPTestCase): def test_keepalive_chunked(self): self.http_version = b'HTTP/1.0' self.connect() - self.stream.write(b'POST / HTTP/1.0\r\nConnection: keep-alive\r\n' + self.stream.write(b'POST / HTTP/1.0\r\n' + b'Connection: keep-alive\r\n' b'Transfer-Encoding: chunked\r\n' - b'\r\n0\r\n') + b'\r\n' + b'0\r\n' + b'\r\n') self.read_response() self.assertEqual(self.headers['Connection'], 'Keep-Alive') self.stream.write(b'GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n') diff --git a/tornado/test/iostream_test.py b/tornado/test/iostream_test.py index 91bc7bf6..56fffe60 100644 --- a/tornado/test/iostream_test.py +++ b/tornado/test/iostream_test.py @@ -9,7 +9,7 @@ from tornado.netutil import ssl_wrap_socket from tornado.stack_context import NullContext from tornado.tcpserver import TCPServer from tornado.testing import AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, bind_unused_port, ExpectLog, gen_test -from tornado.test.util import unittest, skipIfNonUnix, refusing_port +from tornado.test.util import unittest, skipIfNonUnix, refusing_port, skipPypy3V58 from tornado.web import RequestHandler, Application import errno import logging @@ -539,6 +539,7 @@ class TestIOStreamMixin(object): client.close() @skipIfNonUnix + @skipPypy3V58 def test_inline_read_error(self): # An error on an inline read is raised without logging (on the # assumption that it will eventually be noticed or logged further @@ -557,6 +558,7 @@ class TestIOStreamMixin(object): server.close() client.close() + @skipPypy3V58 def test_async_read_error_logging(self): # Socket errors on asynchronous reads should be logged (but only # once). @@ -993,7 +995,7 @@ class TestIOStreamStartTLS(AsyncTestCase): server_future = self.server_start_tls(_server_ssl_options()) client_future = self.client_start_tls( ssl.create_default_context(), - server_hostname=b'127.0.0.1') + server_hostname='127.0.0.1') with ExpectLog(gen_log, "SSL Error"): with self.assertRaises(ssl.SSLError): # The client fails to connect with an SSL error. diff --git a/tornado/test/options_test.py b/tornado/test/options_test.py index bafeea6f..1a0ac8fb 100644 --- a/tornado/test/options_test.py +++ b/tornado/test/options_test.py @@ -7,7 +7,7 @@ import sys from tornado.options import OptionParser, Error from tornado.util import basestring_type, PY3 -from tornado.test.util import unittest +from tornado.test.util import unittest, subTest if PY3: from io import StringIO @@ -232,6 +232,24 @@ class OptionsTest(unittest.TestCase): self.assertRegexpMatches(str(cm.exception), 'Option.*foo.*already defined') + def test_error_redefine_underscore(self): + # Ensure that the dash/underscore normalization doesn't + # interfere with the redefinition error. + tests = [ + ('foo-bar', 'foo-bar'), + ('foo_bar', 'foo_bar'), + ('foo-bar', 'foo_bar'), + ('foo_bar', 'foo-bar'), + ] + for a, b in tests: + with subTest(self, a=a, b=b): + options = OptionParser() + options.define(a) + with self.assertRaises(Error) as cm: + options.define(b) + self.assertRegexpMatches(str(cm.exception), + 'Option.*foo.bar.*already defined') + def test_dash_underscore_cli(self): # Dashes and underscores should be interchangeable. for defined_name in ['foo-bar', 'foo_bar']: diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index 02d57c5f..0e75e530 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -272,16 +272,9 @@ class SimpleHTTPClientTestMixin(object): @skipIfNoIPv6 def test_ipv6(self): - try: - [sock] = bind_sockets(None, '::1', family=socket.AF_INET6) - port = sock.getsockname()[1] - self.http_server.add_socket(sock) - except socket.gaierror as e: - if e.args[0] == socket.EAI_ADDRFAMILY: - # python supports ipv6, but it's not configured on the network - # interface, so skip this test. - return - raise + [sock] = bind_sockets(None, '::1', family=socket.AF_INET6) + port = sock.getsockname()[1] + self.http_server.add_socket(sock) url = '%s://[::1]:%d/hello' % (self.get_protocol(), port) # ipv6 is currently enabled by default but can be disabled @@ -327,7 +320,7 @@ class SimpleHTTPClientTestMixin(object): self.assertNotIn("Content-Length", response.headers) def test_host_header(self): - host_re = re.compile(b"^localhost:[0-9]+$") + host_re = re.compile(b"^127.0.0.1:[0-9]+$") response = self.fetch("/host_echo") self.assertTrue(host_re.match(response.body)) diff --git a/tornado/test/util.py b/tornado/test/util.py index 6c032da6..90a9c7b8 100644 --- a/tornado/test/util.py +++ b/tornado/test/util.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, division, print_function +import contextlib import os import platform import socket @@ -34,14 +35,39 @@ skipOnAppEngine = unittest.skipIf('APPENGINE_RUNTIME' in os.environ, skipIfNoNetwork = unittest.skipIf('NO_NETWORK' in os.environ, 'network access disabled') -skipIfNoIPv6 = unittest.skipIf(not socket.has_ipv6, 'ipv6 support not present') - - skipBefore33 = unittest.skipIf(sys.version_info < (3, 3), 'PEP 380 (yield from) not available') skipBefore35 = unittest.skipIf(sys.version_info < (3, 5), 'PEP 492 (async/await) not available') skipNotCPython = unittest.skipIf(platform.python_implementation() != 'CPython', 'Not CPython implementation') +# Used for tests affected by +# https://bitbucket.org/pypy/pypy/issues/2616/incomplete-error-handling-in +# TODO: remove this after pypy3 5.8 is obsolete. +skipPypy3V58 = unittest.skipIf(platform.python_implementation() == 'PyPy' and + sys.version_info > (3,) and + sys.pypy_version_info < (5, 9), + 'pypy3 5.8 has buggy ssl module') + + +def _detect_ipv6(): + if not socket.has_ipv6: + # socket.has_ipv6 check reports whether ipv6 was present at compile + # time. It's usually true even when ipv6 doesn't work for other reasons. + return False + sock = None + try: + sock = socket.socket(socket.AF_INET6) + sock.bind(('::1', 0)) + except socket.error: + return False + finally: + if sock is not None: + sock.close() + return True + + +skipIfNoIPv6 = unittest.skipIf(not _detect_ipv6(), 'ipv6 support not present') + def refusing_port(): """Returns a local port number that will refuse all connections. @@ -94,3 +120,15 @@ def is_coverage_running(): except AttributeError: return False return mod.startswith('coverage') + + +def subTest(test, *args, **kwargs): + """Compatibility shim for unittest.TestCase.subTest. + + Usage: ``with tornado.test.util.subTest(self, x=x):`` + """ + try: + subTest = test.subTest # py34+ + except AttributeError: + subTest = contextlib.contextmanager(lambda *a, **kw: (yield)) + return subTest(*args, **kwargs) diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py index d79ea52c..013c2ac2 100644 --- a/tornado/test/web_test.py +++ b/tornado/test/web_test.py @@ -356,7 +356,7 @@ class AuthRedirectTest(WebTestCase): response = self.wait() self.assertEqual(response.code, 302) self.assertTrue(re.match( - 'http://example.com/login\?next=http%3A%2F%2Flocalhost%3A[0-9]+%2Fabsolute', + 'http://example.com/login\?next=http%3A%2F%2F127.0.0.1%3A[0-9]+%2Fabsolute', response.headers['Location']), response.headers['Location']) @@ -2134,7 +2134,7 @@ class StreamingRequestBodyTest(WebTestCase): stream.write(b"4\r\nqwer\r\n") data = yield self.data self.assertEquals(data, b"qwer") - stream.write(b"0\r\n") + stream.write(b"0\r\n\r\n") yield self.finished data = yield gen.Task(stream.read_until_close) # This would ideally use an HTTP1Connection to read the response. diff --git a/tornado/test/websocket_test.py b/tornado/test/websocket_test.py index d47a74e6..95a5ecd4 100644 --- a/tornado/test/websocket_test.py +++ b/tornado/test/websocket_test.py @@ -189,6 +189,13 @@ class WebSocketTest(WebSocketBaseTestCase): response = self.fetch('/echo') self.assertEqual(response.code, 400) + def test_missing_websocket_key(self): + response = self.fetch('/echo', + headers={'Connection': 'Upgrade', + 'Upgrade': 'WebSocket', + 'Sec-WebSocket-Version': '13'}) + self.assertEqual(response.code, 400) + def test_bad_websocket_version(self): response = self.fetch('/echo', headers={'Connection': 'Upgrade', diff --git a/tornado/testing.py b/tornado/testing.py index 74d04b60..82a3b937 100644 --- a/tornado/testing.py +++ b/tornado/testing.py @@ -423,7 +423,7 @@ class AsyncHTTPTestCase(AsyncTestCase): def get_url(self, path): """Returns an absolute url for the given path on the test server.""" - return '%s://localhost:%s%s' % (self.get_protocol(), + return '%s://127.0.0.1:%s%s' % (self.get_protocol(), self.get_http_port(), path) def tearDown(self): diff --git a/tornado/web.py b/tornado/web.py index d79889fa..e8d102b5 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -974,7 +974,8 @@ class RequestHandler(object): if self.check_etag_header(): self._write_buffer = [] self.set_status(304) - if self._status_code in (204, 304): + if (self._status_code in (204, 304) or + (self._status_code >= 100 and self._status_code < 200)): assert not self._write_buffer, "Cannot send body with %s" % self._status_code self._clear_headers_for_304() elif "Content-Length" not in self._headers: diff --git a/tornado/websocket.py b/tornado/websocket.py index 69437ee4..0e9d339f 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -616,6 +616,14 @@ class WebSocketProtocol13(WebSocketProtocol): def accept_connection(self): try: self._handle_websocket_headers() + except ValueError: + self.handler.set_status(400) + log_msg = "Missing/Invalid WebSocket headers" + self.handler.finish(log_msg) + gen_log.debug(log_msg) + return + + try: self._accept_connection() except ValueError: gen_log.debug("Malformed WebSocket request received",