From 14655aa022e8df345899b41cdc2ce0e2c709e37a Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 13 Dec 2018 23:43:07 +0530 Subject: [PATCH] Remove dependency from httpbin.org for tests. 1. Add explicit state matching in HttpParser class 2. Highlight bug in HttpParser when content-length is missing 3. Add more test cases --- proxy.py | 8 ++-- tests.py | 118 ++++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 101 insertions(+), 25 deletions(-) diff --git a/proxy.py b/proxy.py index 054aadea..4d0aa2b5 100755 --- a/proxy.py +++ b/proxy.py @@ -174,7 +174,9 @@ class HttpParser(object): self.buffer = data def process(self, data): - if self.state >= HTTP_PARSER_STATE_HEADERS_COMPLETE and \ + if self.state in (HTTP_PARSER_STATE_HEADERS_COMPLETE, + HTTP_PARSER_STATE_RCVING_BODY, + HTTP_PARSER_STATE_COMPLETE) and \ (self.method == b'POST' or self.type == HTTP_RESPONSE_PARSER): if not self.body: self.body = b'' @@ -198,9 +200,9 @@ class HttpParser(object): if line is False: return line, data - if self.state < HTTP_PARSER_STATE_LINE_RCVD: + if self.state == HTTP_PARSER_STATE_INITIALIZED: self.process_line(line) - elif self.state < HTTP_PARSER_STATE_HEADERS_COMPLETE: + elif self.state in (HTTP_PARSER_STATE_LINE_RCVD, HTTP_PARSER_STATE_RCVING_HEADERS): self.process_header(line) if self.state == HTTP_PARSER_STATE_HEADERS_COMPLETE and \ diff --git a/tests.py b/tests.py index 1286b293..e974c3ce 100644 --- a/tests.py +++ b/tests.py @@ -8,9 +8,30 @@ :copyright: (c) 2013-2018 by Abhinav Singh. :license: BSD, see LICENSE for more details. """ +import sys import base64 +import socket +import logging import unittest -from proxy import * +from threading import Thread +from contextlib import closing +from proxy import Proxy, ChunkParser, HttpParser, Client +from proxy import ProxyAuthenticationFailed, ProxyConnectionFailed +from proxy import CRLF, text_, bytes_ +from proxy import HTTP_PARSER_STATE_COMPLETE, CHUNK_PARSER_STATE_COMPLETE, \ + CHUNK_PARSER_STATE_WAITING_FOR_SIZE, CHUNK_PARSER_STATE_WAITING_FOR_DATA, \ + HTTP_PARSER_STATE_INITIALIZED, HTTP_PARSER_STATE_LINE_RCVD, HTTP_PARSER_STATE_RCVING_HEADERS, \ + HTTP_PARSER_STATE_HEADERS_COMPLETE, HTTP_PARSER_STATE_RCVING_BODY, HTTP_RESPONSE_PARSER, \ + PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT + +# logging.basicConfig(level=logging.DEBUG, +# format='%(asctime)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s') + +# True if we are running on Python 3. +if sys.version_info[0] == 3: + from http.server import HTTPServer, BaseHTTPRequestHandler +else: + from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler class TestChunkParser(unittest.TestCase): @@ -206,7 +227,8 @@ class TestHttpParser(unittest.TestCase): self.assertEqual(self.parser.buffer, b'') def test_connect_without_host_header_request_parse(self): - """Some clients sends CONNECT requests without a Host header field. + """Case where clients can send CONNECT request without a Host header field. + Example: 1. pip3 --proxy http://localhost:8899 install Uses HTTP/1.0, Host header missing with CONNECT requests @@ -219,6 +241,26 @@ class TestHttpParser(unittest.TestCase): self.assertEqual(self.parser.version, b'HTTP/1.0') self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_HEADERS) + def test_response_parse_without_content_length(self): + """Case when server response doesn't contain a content-length header for non-chunk response types. + + HttpParser by itself has no way to know if more data should be expected. + In example below, parser reaches state HTTP_PARSER_STATE_HEADERS_COMPLETE + and it is responsibility of callee to change state to HTTP_PARSER_STATE_COMPLETE + when server stream closes. + """ + self.parser.type = HTTP_RESPONSE_PARSER + self.parser.parse(b'HTTP/1.0 200 OK' + CRLF) + self.assertEqual(self.parser.code, b'200') + self.assertEqual(self.parser.version, b'HTTP/1.0') + self.assertEqual(self.parser.state, HTTP_PARSER_STATE_LINE_RCVD) + self.parser.parse(CRLF.join([ + b'Server: BaseHTTP/0.3 Python/2.7.10', + b'Date: Thu, 13 Dec 2018 16:24:09 GMT', + CRLF + ])) + self.assertEqual(self.parser.state, HTTP_PARSER_STATE_HEADERS_COMPLETE) + def test_response_parse(self): self.parser.type = HTTP_RESPONSE_PARSER self.parser.parse(b''.join([ @@ -313,40 +355,76 @@ class MockConnection(object): self.buffer += data +class MockHTTPRequestHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_response(200) + if self.path != '/no-content-length': + self.send_header('content-length', 2) + self.end_headers() + self.wfile.write(b'OK') + + class TestProxy(unittest.TestCase): + mock_server = None + mock_server_port = None + mock_server_thread = None + + @staticmethod + def get_available_port(): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.bind(('', 0)) + _, port = sock.getsockname() + return port + + @classmethod + def setUpClass(cls): + cls.mock_server_port = cls.get_available_port() + cls.mock_server = HTTPServer(('127.0.0.1', cls.mock_server_port), MockHTTPRequestHandler) + cls.mock_server_thread = Thread(target=cls.mock_server.serve_forever) + cls.mock_server_thread.setDaemon(True) + cls.mock_server_thread.start() + + @classmethod + def tearDownClass(cls): + cls.mock_server.shutdown() + cls.mock_server.server_close() + cls.mock_server_thread.join() + def setUp(self): self._conn = MockConnection() self._addr = ('127.0.0.1', 54382) self.proxy = Proxy(Client(self._conn, self._addr)) def test_http_get(self): - self.proxy.client.conn.queue(b'GET http://httpbin.org/get HTTP/1.1' + CRLF) + # Send request line + self.proxy.client.conn.queue(bytes_('GET http://localhost:%d/get HTTP/1.1' % self.mock_server_port) + CRLF) self.proxy._process_request(self.proxy.client.recv()) self.assertNotEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE) - + # Send headers and blank line, thus completing HTTP request self.proxy.client.conn.queue(CRLF.join([ b'User-Agent: curl/7.27.0', - b'Host: httpbin.org', + bytes_('Host: localhost:%d' % self.mock_server_port), b'Accept: */*', b'Proxy-Connection: Keep-Alive', CRLF ])) - self.proxy._process_request(self.proxy.client.recv()) self.assertEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE) - self.assertEqual(self.proxy.server.addr, (b'httpbin.org', 80)) - + self.assertEqual(self.proxy.server.addr, (b'localhost', self.mock_server_port)) + # Flush data queued for server self.proxy.server.flush() self.assertEqual(self.proxy.server.buffer_size(), 0) - + # Receive full response from server data = self.proxy.server.recv() while data: self.proxy._process_response(data) + logging.info(self.proxy.response.state) if self.proxy.response.state == HTTP_PARSER_STATE_COMPLETE: break data = self.proxy.server.recv() - + # Verify 200 success response code self.assertEqual(self.proxy.response.state, HTTP_PARSER_STATE_COMPLETE) self.assertEqual(int(self.proxy.response.code), 200) @@ -399,15 +477,9 @@ class TestProxy(unittest.TestCase): CRLF ])) - -class TestAuthenticatedProxy(unittest.TestCase): - - def setUp(self): - self._conn = MockConnection() - self._addr = ('127.0.0.1', 54382) - self.proxy = Proxy(Client(self._conn, self._addr), bytes_('Basic %s' % base64.b64encode('user:pass'))) - def test_proxy_authentication_failed(self): + self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(bytes_('user:pass'))) + with self.assertRaises(ProxyAuthenticationFailed): self.proxy._process_request(CRLF.join([ b'GET http://abhinavsingh.com HTTP/1.1', @@ -415,14 +487,16 @@ class TestAuthenticatedProxy(unittest.TestCase): CRLF ])) - def test_http_get(self): - self.proxy.client.conn.queue(b'GET http://httpbin.org/get HTTP/1.1' + CRLF) + def test_authenticated_proxy_http_get(self): + self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(bytes_('user:pass'))) + + self.proxy.client.conn.queue(bytes_('GET http://localhost:%d/get HTTP/1.1' % self.mock_server_port) + CRLF) self.proxy._process_request(self.proxy.client.recv()) self.assertNotEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE) self.proxy.client.conn.queue(CRLF.join([ b'User-Agent: curl/7.27.0', - b'Host: httpbin.org', + bytes_('Host: localhost:%d' % self.mock_server_port), b'Accept: */*', b'Proxy-Connection: Keep-Alive', b'Proxy-Authorization: Basic dXNlcjpwYXNz', @@ -431,7 +505,7 @@ class TestAuthenticatedProxy(unittest.TestCase): self.proxy._process_request(self.proxy.client.recv()) self.assertEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE) - self.assertEqual(self.proxy.server.addr, (b'httpbin.org', 80)) + self.assertEqual(self.proxy.server.addr, (b'localhost', self.mock_server_port)) self.proxy.server.flush() self.assertEqual(self.proxy.server.buffer_size(), 0)