Rooba
/
proxy.py
mirror of https://github.com/abhinavsingh/proxy.py.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove dependency from httpbin.org for tests. 

1. Add explicit state matching in HttpParser class
2. Highlight bug in HttpParser when content-length is missing
3. Add more test cases
This commit is contained in:
Abhinav Singh 2018-12-13 23:43:07 +05:30
parent e23444b766
commit 14655aa022
2 changed files with 101 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
proxy.py
View File

@ -174,7 +174,9 @@ class HttpParser(object):
self.buffer = data
def process(self, data):
if self.state >= HTTP_PARSER_STATE_HEADERS_COMPLETE and \
if self.state in (HTTP_PARSER_STATE_HEADERS_COMPLETE,
HTTP_PARSER_STATE_RCVING_BODY,
HTTP_PARSER_STATE_COMPLETE) and \
(self.method == b'POST' or self.type == HTTP_RESPONSE_PARSER):
if not self.body:
self.body = b''
@ -198,9 +200,9 @@ class HttpParser(object):
if line is False:
return line, data
if self.state < HTTP_PARSER_STATE_LINE_RCVD:
if self.state == HTTP_PARSER_STATE_INITIALIZED:
self.process_line(line)
elif self.state < HTTP_PARSER_STATE_HEADERS_COMPLETE:
elif self.state in (HTTP_PARSER_STATE_LINE_RCVD, HTTP_PARSER_STATE_RCVING_HEADERS):
self.process_header(line)
if self.state == HTTP_PARSER_STATE_HEADERS_COMPLETE and \

118
tests.py
View File

@ -8,9 +8,30 @@
:copyright: (c) 2013-2018 by Abhinav Singh.
:license: BSD, see LICENSE for more details.
"""
import sys
import base64
import socket
import logging
import unittest
from proxy import *
from threading import Thread
from contextlib import closing
from proxy import Proxy, ChunkParser, HttpParser, Client
from proxy import ProxyAuthenticationFailed, ProxyConnectionFailed
from proxy import CRLF, text_, bytes_
from proxy import HTTP_PARSER_STATE_COMPLETE, CHUNK_PARSER_STATE_COMPLETE, \
CHUNK_PARSER_STATE_WAITING_FOR_SIZE, CHUNK_PARSER_STATE_WAITING_FOR_DATA, \
HTTP_PARSER_STATE_INITIALIZED, HTTP_PARSER_STATE_LINE_RCVD, HTTP_PARSER_STATE_RCVING_HEADERS, \
HTTP_PARSER_STATE_HEADERS_COMPLETE, HTTP_PARSER_STATE_RCVING_BODY, HTTP_RESPONSE_PARSER, \
PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT
# logging.basicConfig(level=logging.DEBUG,
# format='%(asctime)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s')
# True if we are running on Python 3.
if sys.version_info[0] == 3:
from http.server import HTTPServer, BaseHTTPRequestHandler
else:
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
class TestChunkParser(unittest.TestCase):
@ -206,7 +227,8 @@ class TestHttpParser(unittest.TestCase):
self.assertEqual(self.parser.buffer, b'')
def test_connect_without_host_header_request_parse(self):
"""Some clients sends CONNECT requests without a Host header field.
"""Case where clients can send CONNECT request without a Host header field.
Example:
1. pip3 --proxy http://localhost:8899 install <package name>
Uses HTTP/1.0, Host header missing with CONNECT requests
@ -219,6 +241,26 @@ class TestHttpParser(unittest.TestCase):
self.assertEqual(self.parser.version, b'HTTP/1.0')
self.assertEqual(self.parser.state, HTTP_PARSER_STATE_RCVING_HEADERS)
def test_response_parse_without_content_length(self):
"""Case when server response doesn't contain a content-length header for non-chunk response types.
HttpParser by itself has no way to know if more data should be expected.
In example below, parser reaches state HTTP_PARSER_STATE_HEADERS_COMPLETE
and it is responsibility of callee to change state to HTTP_PARSER_STATE_COMPLETE
when server stream closes.
"""
self.parser.type = HTTP_RESPONSE_PARSER
self.parser.parse(b'HTTP/1.0 200 OK' + CRLF)
self.assertEqual(self.parser.code, b'200')
self.assertEqual(self.parser.version, b'HTTP/1.0')
self.assertEqual(self.parser.state, HTTP_PARSER_STATE_LINE_RCVD)
self.parser.parse(CRLF.join([
b'Server: BaseHTTP/0.3 Python/2.7.10',
b'Date: Thu, 13 Dec 2018 16:24:09 GMT',
CRLF
]))
self.assertEqual(self.parser.state, HTTP_PARSER_STATE_HEADERS_COMPLETE)
def test_response_parse(self):
self.parser.type = HTTP_RESPONSE_PARSER
self.parser.parse(b''.join([
@ -313,40 +355,76 @@ class MockConnection(object):
self.buffer += data
class MockHTTPRequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
if self.path != '/no-content-length':
self.send_header('content-length', 2)
self.end_headers()
self.wfile.write(b'OK')
class TestProxy(unittest.TestCase):
mock_server = None
mock_server_port = None
mock_server_thread = None
@staticmethod
def get_available_port():
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
sock.bind(('', 0))
_, port = sock.getsockname()
return port
@classmethod
def setUpClass(cls):
cls.mock_server_port = cls.get_available_port()
cls.mock_server = HTTPServer(('127.0.0.1', cls.mock_server_port), MockHTTPRequestHandler)
cls.mock_server_thread = Thread(target=cls.mock_server.serve_forever)
cls.mock_server_thread.setDaemon(True)
cls.mock_server_thread.start()
@classmethod
def tearDownClass(cls):
cls.mock_server.shutdown()
cls.mock_server.server_close()
cls.mock_server_thread.join()
def setUp(self):
self._conn = MockConnection()
self._addr = ('127.0.0.1', 54382)
self.proxy = Proxy(Client(self._conn, self._addr))
def test_http_get(self):
self.proxy.client.conn.queue(b'GET http://httpbin.org/get HTTP/1.1' + CRLF)
# Send request line
self.proxy.client.conn.queue(bytes_('GET http://localhost:%d/get HTTP/1.1' % self.mock_server_port) + CRLF)
self.proxy._process_request(self.proxy.client.recv())
self.assertNotEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE)
# Send headers and blank line, thus completing HTTP request
self.proxy.client.conn.queue(CRLF.join([
b'User-Agent: curl/7.27.0',
b'Host: httpbin.org',
bytes_('Host: localhost:%d' % self.mock_server_port),
b'Accept: */*',
b'Proxy-Connection: Keep-Alive',
CRLF
]))
self.proxy._process_request(self.proxy.client.recv())
self.assertEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE)
self.assertEqual(self.proxy.server.addr, (b'httpbin.org', 80))
self.assertEqual(self.proxy.server.addr, (b'localhost', self.mock_server_port))
# Flush data queued for server
self.proxy.server.flush()
self.assertEqual(self.proxy.server.buffer_size(), 0)
# Receive full response from server
data = self.proxy.server.recv()
while data:
self.proxy._process_response(data)
logging.info(self.proxy.response.state)
if self.proxy.response.state == HTTP_PARSER_STATE_COMPLETE:
break
data = self.proxy.server.recv()
# Verify 200 success response code
self.assertEqual(self.proxy.response.state, HTTP_PARSER_STATE_COMPLETE)
self.assertEqual(int(self.proxy.response.code), 200)
@ -399,15 +477,9 @@ class TestProxy(unittest.TestCase):
CRLF
]))
class TestAuthenticatedProxy(unittest.TestCase):
def setUp(self):
self._conn = MockConnection()
self._addr = ('127.0.0.1', 54382)
self.proxy = Proxy(Client(self._conn, self._addr), bytes_('Basic %s' % base64.b64encode('user:pass')))
def test_proxy_authentication_failed(self):
self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(bytes_('user:pass')))
with self.assertRaises(ProxyAuthenticationFailed):
self.proxy._process_request(CRLF.join([
b'GET http://abhinavsingh.com HTTP/1.1',
@ -415,14 +487,16 @@ class TestAuthenticatedProxy(unittest.TestCase):
CRLF
]))
def test_http_get(self):
self.proxy.client.conn.queue(b'GET http://httpbin.org/get HTTP/1.1' + CRLF)
def test_authenticated_proxy_http_get(self):
self.proxy = Proxy(Client(self._conn, self._addr), b'Basic %s' % base64.b64encode(bytes_('user:pass')))
self.proxy.client.conn.queue(bytes_('GET http://localhost:%d/get HTTP/1.1' % self.mock_server_port) + CRLF)
self.proxy._process_request(self.proxy.client.recv())
self.assertNotEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE)
self.proxy.client.conn.queue(CRLF.join([
b'User-Agent: curl/7.27.0',
b'Host: httpbin.org',
bytes_('Host: localhost:%d' % self.mock_server_port),
b'Accept: */*',
b'Proxy-Connection: Keep-Alive',
b'Proxy-Authorization: Basic dXNlcjpwYXNz',
@ -431,7 +505,7 @@ class TestAuthenticatedProxy(unittest.TestCase):
self.proxy._process_request(self.proxy.client.recv())
self.assertEqual(self.proxy.request.state, HTTP_PARSER_STATE_COMPLETE)
self.assertEqual(self.proxy.server.addr, (b'httpbin.org', 80))
self.assertEqual(self.proxy.server.addr, (b'localhost', self.mock_server_port))
self.proxy.server.flush()
self.assertEqual(self.proxy.server.buffer_size(), 0)