Rooba
/
proxy.py
mirror of https://github.com/abhinavsingh/proxy.py.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor (#213) 

* Add DEFAULT_HTTP_PORT constant

* Use DEFAULT_HTTP_PORT in tests

* Refactor into exception module

* Refactor into inspector module

* Refactor into server module

* Refactor into proxy module
This commit is contained in:
Abhinav Singh 2019-12-01 22:46:00 -08:00 committed by GitHub
parent 64192250ee
commit 6137fd6f82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 603 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
proxy/common/constants.py
View File

@ -72,3 +72,4 @@ DEFAULT_STATIC_SERVER_DIR = PROXY_PY_DIR
DEFAULT_THREADLESS = False
DEFAULT_TIMEOUT = 10
DEFAULT_VERSION = False
DEFAULT_HTTP_PORT = 80

95
proxy/http/exception.py
View File

@ -1,95 +0,0 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from typing import Optional, Dict
from .parser import HttpParser
from .codes import httpStatusCodes
from ..common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY
from ..common.utils import build_http_response
class HttpProtocolException(Exception):
"""Top level HttpProtocolException exception class.
All exceptions raised during execution of Http request lifecycle MUST
inherit HttpProtocolException base class. Implement response() method
to optionally return custom response to client."""
def response(self, request: HttpParser) -> Optional[memoryview]:
return None # pragma: no cover
class HttpRequestRejected(HttpProtocolException):
"""Generic exception that can be used to reject the client requests.
Connections can either be dropped/closed or optionally an
HTTP status code can be returned."""
def __init__(self,
status_code: Optional[int] = None,
reason: Optional[bytes] = None,
headers: Optional[Dict[bytes, bytes]] = None,
body: Optional[bytes] = None):
self.status_code: Optional[int] = status_code
self.reason: Optional[bytes] = reason
self.headers: Optional[Dict[bytes, bytes]] = headers
self.body: Optional[bytes] = body
def response(self, _request: HttpParser) -> Optional[memoryview]:
if self.status_code:
return memoryview(build_http_response(
status_code=self.status_code,
reason=self.reason,
headers=self.headers,
body=self.body
))
return None
class ProxyConnectionFailed(HttpProtocolException):
"""Exception raised when HttpProxyPlugin is unable to establish connection to upstream server."""
RESPONSE_PKT = memoryview(build_http_response(
httpStatusCodes.BAD_GATEWAY,
reason=b'Bad Gateway',
headers={
PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
b'Connection': b'close'
},
body=b'Bad Gateway'
))
def __init__(self, host: str, port: int, reason: str):
self.host: str = host
self.port: int = port
self.reason: str = reason
def response(self, _request: HttpParser) -> memoryview:
return self.RESPONSE_PKT
class ProxyAuthenticationFailed(HttpProtocolException):
"""Exception raised when Http Proxy auth is enabled and
incoming request doesn't present necessary credentials."""
RESPONSE_PKT = memoryview(build_http_response(
httpStatusCodes.PROXY_AUTH_REQUIRED,
reason=b'Proxy Authentication Required',
headers={
PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
b'Proxy-Authenticate': b'Basic',
b'Connection': b'close',
},
body=b'Proxy Authentication Required'))
def response(self, _request: HttpParser) -> memoryview:
return self.RESPONSE_PKT

21
proxy/http/exception/__init__.py Normal file
View File

@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from .base import HttpProtocolException
from .http_request_rejected import HttpRequestRejected
from .proxy_auth_failed import ProxyAuthenticationFailed
from .proxy_conn_failed import ProxyConnectionFailed
__all__ = [
'HttpProtocolException',
'HttpRequestRejected',
'ProxyAuthenticationFailed',
'ProxyConnectionFailed',
]

24
proxy/http/exception/base.py Normal file
View File

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from typing import Optional
from ..parser import HttpParser
class HttpProtocolException(Exception):
"""Top level HttpProtocolException exception class.
All exceptions raised during execution of Http request lifecycle MUST
inherit HttpProtocolException base class. Implement response() method
to optionally return custom response to client."""
def response(self, request: HttpParser) -> Optional[memoryview]:
return None # pragma: no cover

42
proxy/http/exception/http_request_rejected.py Normal file
View File

@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from typing import Optional, Dict
from .base import HttpProtocolException
from ..parser import HttpParser
from ...common.utils import build_http_response
class HttpRequestRejected(HttpProtocolException):
"""Generic exception that can be used to reject the client requests.
Connections can either be dropped/closed or optionally an
HTTP status code can be returned."""
def __init__(self,
status_code: Optional[int] = None,
reason: Optional[bytes] = None,
headers: Optional[Dict[bytes, bytes]] = None,
body: Optional[bytes] = None):
self.status_code: Optional[int] = status_code
self.reason: Optional[bytes] = reason
self.headers: Optional[Dict[bytes, bytes]] = headers
self.body: Optional[bytes] = body
def response(self, _request: HttpParser) -> Optional[memoryview]:
if self.status_code:
return memoryview(build_http_response(
status_code=self.status_code,
reason=self.reason,
headers=self.headers,
body=self.body
))
return None

34
proxy/http/exception/proxy_auth_failed.py Normal file
View File

@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from .base import HttpProtocolException
from ..parser import HttpParser
from ..codes import httpStatusCodes
from ...common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY
from ...common.utils import build_http_response
class ProxyAuthenticationFailed(HttpProtocolException):
"""Exception raised when Http Proxy auth is enabled and
incoming request doesn't present necessary credentials."""
RESPONSE_PKT = memoryview(build_http_response(
httpStatusCodes.PROXY_AUTH_REQUIRED,
reason=b'Proxy Authentication Required',
headers={
PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
b'Proxy-Authenticate': b'Basic',
b'Connection': b'close',
},
body=b'Proxy Authentication Required'))
def response(self, _request: HttpParser) -> memoryview:
return self.RESPONSE_PKT

38
proxy/http/exception/proxy_conn_failed.py Normal file
View File

@ -0,0 +1,38 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from .base import HttpProtocolException
from ..parser import HttpParser
from ..codes import httpStatusCodes
from ...common.constants import PROXY_AGENT_HEADER_VALUE, PROXY_AGENT_HEADER_KEY
from ...common.utils import build_http_response
class ProxyConnectionFailed(HttpProtocolException):
"""Exception raised when HttpProxyPlugin is unable to establish connection to upstream server."""
RESPONSE_PKT = memoryview(build_http_response(
httpStatusCodes.BAD_GATEWAY,
reason=b'Bad Gateway',
headers={
PROXY_AGENT_HEADER_KEY: PROXY_AGENT_HEADER_VALUE,
b'Connection': b'close'
},
body=b'Bad Gateway'
))
def __init__(self, host: str, port: int, reason: str):
self.host: str = host
self.port: int = port
self.reason: str = reason
def response(self, _request: HttpParser) -> memoryview:
return self.RESPONSE_PKT

15
proxy/http/inspector/__init__.py Normal file
View File

@ -0,0 +1,15 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from .devtools import DevtoolsProtocolPlugin
__all__ = [
'DevtoolsProtocolPlugin',
]

110
proxy/http/inspector/devtools.py Normal file
View File

@ -0,0 +1,110 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
import json
import logging
from typing import List, Tuple, Any, Dict
from .transformer import CoreEventsToDevtoolsProtocol
from ..parser import HttpParser
from ..websocket import WebsocketFrame, websocketOpcodes
from ..server import HttpWebServerBasePlugin, httpProtocolTypes
from ...common.utils import bytes_, text_
from ...core.event import EventSubscriber
logger = logging.getLogger(__name__)
class DevtoolsProtocolPlugin(HttpWebServerBasePlugin):
"""Speaks DevTools protocol with client over websocket.
- It responds to DevTools client request methods and also
relay proxy.py core events to the client.
- Core events are transformed into DevTools protocol format before
dispatching to client.
- Core events unrelated to DevTools protocol are dropped.
"""
DOC_URL = 'http://dashboard.proxy.py'
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.subscriber = EventSubscriber(self.event_queue)
def routes(self) -> List[Tuple[int, str]]:
return [
(httpProtocolTypes.WEBSOCKET, text_(self.flags.devtools_ws_path))
]
def handle_request(self, request: HttpParser) -> None:
raise NotImplementedError('This should have never been called')
def on_websocket_open(self) -> None:
self.subscriber.subscribe(
lambda event: CoreEventsToDevtoolsProtocol.transformer(self.client, event))
def on_websocket_message(self, frame: WebsocketFrame) -> None:
try:
assert frame.data
message = json.loads(frame.data)
except UnicodeDecodeError:
logger.error(frame.data)
logger.info(frame.opcode)
return
self.handle_devtools_message(message)
def on_websocket_close(self) -> None:
self.subscriber.unsubscribe()
def handle_devtools_message(self, message: Dict[str, Any]) -> None:
frame = WebsocketFrame()
frame.fin = True
frame.opcode = websocketOpcodes.TEXT_FRAME
# logger.info(message)
method = message['method']
if method in (
'Page.canScreencast',
'Network.canEmulateNetworkConditions',
'Emulation.canEmulate',
):
data: Dict[str, Any] = {
'result': False
}
elif method == 'Page.getResourceTree':
data = {
'result': {
'frameTree': {
'frame': {
'id': 1,
'url': DevtoolsProtocolPlugin.DOC_URL,
'mimeType': 'other',
},
'childFrames': [],
'resources': []
}
}
}
elif method == 'Network.getResponseBody':
connection_id = message['params']['requestId']
data = {
'result': {
'body': text_(CoreEventsToDevtoolsProtocol.RESPONSES[connection_id]),
'base64Encoded': False,
}
}
else:
logging.warning('Unhandled devtools method %s', method)
data = {}
data['id'] = message['id']
frame.data = bytes_(json.dumps(data))
self.client.queue(memoryview(frame.build()))

115
proxy/http/inspector.py → proxy/http/inspector/transformer.py
View File

@ -9,114 +9,23 @@
:license: BSD, see LICENSE for more details.
"""
import json
import logging
import secrets
import time
from typing import List, Tuple, Any, Dict
from typing import Any, Dict
from .parser import HttpParser
from .websocket import WebsocketFrame, websocketOpcodes
from .server import HttpWebServerBasePlugin, httpProtocolTypes
from ..common.constants import PROXY_PY_START_TIME
from ..common.utils import bytes_, text_
from ..core.connection import TcpClientConnection
from ..core.event import EventSubscriber, eventNames
logger = logging.getLogger(__name__)
from ..websocket import WebsocketFrame
from ...common.constants import PROXY_PY_START_TIME
from ...common.utils import bytes_
from ...core.connection import TcpClientConnection
from ...core.event import eventNames
class DevtoolsProtocolPlugin(HttpWebServerBasePlugin):
"""Speaks DevTools protocol with client over websocket.
- It responds to DevTools client request methods and also
relay proxy.py core events to the client.
- Core events are transformed into DevTools protocol format before
dispatching to client.
- Core events unrelated to DevTools protocol are dropped.
"""
class CoreEventsToDevtoolsProtocol:
DOC_URL = 'http://dashboard.proxy.py'
FRAME_ID = secrets.token_hex(8)
LOADER_ID = secrets.token_hex(8)
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.subscriber = EventSubscriber(self.event_queue)
def routes(self) -> List[Tuple[int, str]]:
return [
(httpProtocolTypes.WEBSOCKET, text_(self.flags.devtools_ws_path))
]
def handle_request(self, request: HttpParser) -> None:
raise NotImplementedError('This should have never been called')
def on_websocket_open(self) -> None:
self.subscriber.subscribe(
lambda event: CoreEventsToDevtoolsProtocol.transformer(self.client, event))
def on_websocket_message(self, frame: WebsocketFrame) -> None:
try:
assert frame.data
message = json.loads(frame.data)
except UnicodeDecodeError:
logger.error(frame.data)
logger.info(frame.opcode)
return
self.handle_devtools_message(message)
def on_websocket_close(self) -> None:
self.subscriber.unsubscribe()
def handle_devtools_message(self, message: Dict[str, Any]) -> None:
frame = WebsocketFrame()
frame.fin = True
frame.opcode = websocketOpcodes.TEXT_FRAME
# logger.info(message)
method = message['method']
if method in (
'Page.canScreencast',
'Network.canEmulateNetworkConditions',
'Emulation.canEmulate',
):
data: Dict[str, Any] = {
'result': False
}
elif method == 'Page.getResourceTree':
data = {
'result': {
'frameTree': {
'frame': {
'id': 1,
'url': DevtoolsProtocolPlugin.DOC_URL,
'mimeType': 'other',
},
'childFrames': [],
'resources': []
}
}
}
elif method == 'Network.getResponseBody':
connection_id = message['params']['requestId']
data = {
'result': {
'body': text_(CoreEventsToDevtoolsProtocol.RESPONSES[connection_id]),
'base64Encoded': False,
}
}
else:
logging.warning('Unhandled devtools method %s', method)
data = {}
data['id'] = message['id']
frame.data = bytes_(json.dumps(data))
self.client.queue(memoryview(frame.build()))
class CoreEventsToDevtoolsProtocol:
RESPONSES: Dict[str, bytes] = {}
@staticmethod
@ -145,9 +54,9 @@ class CoreEventsToDevtoolsProtocol:
now = time.time()
return {
'requestId': event['request_id'],
'frameId': DevtoolsProtocolPlugin.FRAME_ID,
'loaderId': DevtoolsProtocolPlugin.LOADER_ID,
'documentURL': DevtoolsProtocolPlugin.DOC_URL,
'frameId': CoreEventsToDevtoolsProtocol.FRAME_ID,
'loaderId': CoreEventsToDevtoolsProtocol.LOADER_ID,
'documentURL': CoreEventsToDevtoolsProtocol.DOC_URL,
'timestamp': now - PROXY_PY_START_TIME,
'wallTime': now,
'hasUserGesture': False,
@ -172,8 +81,8 @@ class CoreEventsToDevtoolsProtocol:
def response_headers_complete(event: Dict[str, Any]) -> Dict[str, Any]:
return {
'requestId': event['request_id'],
'frameId': DevtoolsProtocolPlugin.FRAME_ID,
'loaderId': DevtoolsProtocolPlugin.LOADER_ID,
'frameId': CoreEventsToDevtoolsProtocol.FRAME_ID,
'loaderId': CoreEventsToDevtoolsProtocol.LOADER_ID,
'timestamp': time.time(),
'type': event['event_payload']['headers']['content-type']
if event['event_payload']['headers'].has_header('content-type')

4
proxy/http/parser.py
View File

@ -14,7 +14,7 @@ from typing import TypeVar, NamedTuple, Optional, Dict, Type, Tuple, List
from .methods import httpMethods
from .chunk_parser import ChunkParser, chunkParserStates
from ..common.constants import DEFAULT_DISABLE_HEADERS, COLON, CRLF, WHITESPACE, HTTP_1_1
from ..common.constants import DEFAULT_DISABLE_HEADERS, COLON, CRLF, WHITESPACE, HTTP_1_1, DEFAULT_HTTP_PORT
from ..common.utils import build_http_request, find_http_line, text_
@ -115,7 +115,7 @@ class HttpParser:
self.host, self.port = u.hostname, u.port
elif self.url:
self.host, self.port = self.url.hostname, self.url.port \
if self.url.port else 80
if self.url.port else DEFAULT_HTTP_PORT
else:
raise KeyError(
'Invalid request. Method: %r, Url: %r' %

17
proxy/http/proxy/__init__.py Normal file
View File

@ -0,0 +1,17 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from .plugin import HttpProxyBasePlugin
from .server import HttpProxyPlugin
__all__ = [
'HttpProxyBasePlugin',
'HttpProxyPlugin',
]

84
proxy/http/proxy/plugin.py Normal file
View File

@ -0,0 +1,84 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from abc import ABC, abstractmethod
from typing import Optional
from ..parser import HttpParser
from ...common.flags import Flags
from ...core.event import EventQueue
from ...core.connection import TcpClientConnection
class HttpProxyBasePlugin(ABC):
"""Base HttpProxyPlugin Plugin class.
Implement various lifecycle event methods to customize behavior."""
def __init__(
self,
uid: str,
flags: Flags,
client: TcpClientConnection,
event_queue: EventQueue) -> None:
self.uid = uid # pragma: no cover
self.flags = flags # pragma: no cover
self.client = client # pragma: no cover
self.event_queue = event_queue # pragma: no cover
def name(self) -> str:
"""A unique name for your plugin.
Defaults to name of the class. This helps plugin developers to directly
access a specific plugin by its name."""
return self.__class__.__name__ # pragma: no cover
@abstractmethod
def before_upstream_connection(
self, request: HttpParser) -> Optional[HttpParser]:
"""Handler called just before Proxy upstream connection is established.
Return optionally modified request object.
Raise HttpRequestRejected or HttpProtocolException directly to drop the connection."""
return request # pragma: no cover
@abstractmethod
def handle_client_request(
self, request: HttpParser) -> Optional[HttpParser]:
"""Handler called before dispatching client request to upstream.
Note: For pipelined (keep-alive) connections, this handler can be
called multiple times, for each request sent to upstream.
Note: If TLS interception is enabled, this handler can
be called multiple times if client exchanges multiple
requests over same SSL session.
Return optionally modified request object to dispatch to upstream.
Return None to drop the request data, e.g. in case a response has already been queued.
Raise HttpRequestRejected or HttpProtocolException directly to
teardown the connection with client.
"""
return request # pragma: no cover
@abstractmethod
def handle_upstream_chunk(self, chunk: memoryview) -> memoryview:
"""Handler called right after receiving raw response from upstream server.
For HTTPS connections, chunk will be encrypted unless
TLS interception is also enabled."""
return chunk # pragma: no cover
@abstractmethod
def on_upstream_connection_close(self) -> None:
"""Handler called right after upstream connection has been closed."""
pass # pragma: no cover

88
proxy/http/proxy.py → proxy/http/proxy/server.py
View File

@ -16,91 +16,25 @@ import socket
import time
import errno
import logging
from abc import ABC, abstractmethod
from typing import Optional, List, Union, Dict, cast, Any, Tuple
from proxy.core.event import eventNames, EventQueue
from .handler import HttpProtocolHandlerPlugin
from .exception import HttpProtocolException, ProxyConnectionFailed, ProxyAuthenticationFailed
from .codes import httpStatusCodes
from .parser import HttpParser, httpParserStates, httpParserTypes
from .methods import httpMethods
from .plugin import HttpProxyBasePlugin
from ..handler import HttpProtocolHandlerPlugin
from ..exception import HttpProtocolException, ProxyConnectionFailed, ProxyAuthenticationFailed
from ..codes import httpStatusCodes
from ..parser import HttpParser, httpParserStates, httpParserTypes
from ..methods import httpMethods
from ..common.types import HasFileno
from ..common.flags import Flags
from ..common.constants import PROXY_AGENT_HEADER_VALUE
from ..common.utils import build_http_response, text_
from ...common.types import HasFileno
from ...common.constants import PROXY_AGENT_HEADER_VALUE
from ...common.utils import build_http_response, text_
from ..core.connection import TcpClientConnection, TcpServerConnection, TcpConnectionUninitializedException
from ...core.event import eventNames
from ...core.connection import TcpServerConnection, TcpConnectionUninitializedException
logger = logging.getLogger(__name__)
class HttpProxyBasePlugin(ABC):
"""Base HttpProxyPlugin Plugin class.
Implement various lifecycle event methods to customize behavior."""
def __init__(
self,
uid: str,
flags: Flags,
client: TcpClientConnection,
event_queue: EventQueue) -> None:
self.uid = uid # pragma: no cover
self.flags = flags # pragma: no cover
self.client = client # pragma: no cover
self.event_queue = event_queue # pragma: no cover
def name(self) -> str:
"""A unique name for your plugin.
Defaults to name of the class. This helps plugin developers to directly
access a specific plugin by its name."""
return self.__class__.__name__ # pragma: no cover
@abstractmethod
def before_upstream_connection(
self, request: HttpParser) -> Optional[HttpParser]:
"""Handler called just before Proxy upstream connection is established.
Return optionally modified request object.
Raise HttpRequestRejected or HttpProtocolException directly to drop the connection."""
return request # pragma: no cover
@abstractmethod
def handle_client_request(
self, request: HttpParser) -> Optional[HttpParser]:
"""Handler called before dispatching client request to upstream.
Note: For pipelined (keep-alive) connections, this handler can be
called multiple times, for each request sent to upstream.
Note: If TLS interception is enabled, this handler can
be called multiple times if client exchanges multiple
requests over same SSL session.
Return optionally modified request object to dispatch to upstream.
Return None to drop the request data, e.g. in case a response has already been queued.
Raise HttpRequestRejected or HttpProtocolException directly to
teardown the connection with client.
"""
return request # pragma: no cover
@abstractmethod
def handle_upstream_chunk(self, chunk: memoryview) -> memoryview:
"""Handler called right after receiving raw response from upstream server.
For HTTPS connections, chunk will be encrypted unless
TLS interception is also enabled."""
return chunk # pragma: no cover
@abstractmethod
def on_upstream_connection_close(self) -> None:
"""Handler called right after upstream connection has been closed."""
pass # pragma: no cover
class HttpProxyPlugin(HttpProtocolHandlerPlugin):
"""HttpProtocolHandler plugin which implements HttpProxy specifications."""

21
proxy/http/server/__init__.py Normal file
View File

@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from .web import HttpWebServerPlugin
from .pac_plugin import HttpWebServerPacFilePlugin
from .plugin import HttpWebServerBasePlugin
from .protocols import httpProtocolTypes
__all__ = [
'HttpWebServerPlugin',
'HttpWebServerPacFilePlugin',
'HttpWebServerBasePlugin',
'httpProtocolTypes',
]

61
proxy/http/server/pac_plugin.py Normal file
View File

@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
import gzip
from typing import List, Tuple, Optional, Any
from .plugin import HttpWebServerBasePlugin
from .protocols import httpProtocolTypes
from ..websocket import WebsocketFrame
from ..parser import HttpParser
from ...common.utils import bytes_, text_, build_http_response
class HttpWebServerPacFilePlugin(HttpWebServerBasePlugin):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.pac_file_response: Optional[memoryview] = None
self.cache_pac_file_response()
def routes(self) -> List[Tuple[int, str]]:
if self.flags.pac_file_url_path:
return [
(httpProtocolTypes.HTTP, text_(self.flags.pac_file_url_path)),
(httpProtocolTypes.HTTPS, text_(self.flags.pac_file_url_path)),
]
return [] # pragma: no cover
def handle_request(self, request: HttpParser) -> None:
if self.flags.pac_file and self.pac_file_response:
self.client.queue(self.pac_file_response)
def on_websocket_open(self) -> None:
pass # pragma: no cover
def on_websocket_message(self, frame: WebsocketFrame) -> None:
pass # pragma: no cover
def on_websocket_close(self) -> None:
pass # pragma: no cover
def cache_pac_file_response(self) -> None:
if self.flags.pac_file:
try:
with open(self.flags.pac_file, 'rb') as f:
content = f.read()
except IOError:
content = bytes_(self.flags.pac_file)
self.pac_file_response = memoryview(build_http_response(
200, reason=b'OK', headers={
b'Content-Type': b'application/x-ns-proxy-autoconfig',
b'Content-Encoding': b'gzip',
}, body=gzip.compress(content)
))

59
proxy/http/server/plugin.py Normal file
View File

@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from abc import ABC, abstractmethod
from typing import List, Tuple
from ..websocket import WebsocketFrame
from ..parser import HttpParser
from ...common.flags import Flags
from ...core.connection import TcpClientConnection
from ...core.event import EventQueue
class HttpWebServerBasePlugin(ABC):
"""Web Server Plugin for routing of requests."""
def __init__(
self,
uid: str,
flags: Flags,
client: TcpClientConnection,
event_queue: EventQueue):
self.uid = uid
self.flags = flags
self.client = client
self.event_queue = event_queue
@abstractmethod
def routes(self) -> List[Tuple[int, str]]:
"""Return List(protocol, path) that this plugin handles."""
raise NotImplementedError() # pragma: no cover
@abstractmethod
def handle_request(self, request: HttpParser) -> None:
"""Handle the request and serve response."""
raise NotImplementedError() # pragma: no cover
@abstractmethod
def on_websocket_open(self) -> None:
"""Called when websocket handshake has finished."""
raise NotImplementedError() # pragma: no cover
@abstractmethod
def on_websocket_message(self, frame: WebsocketFrame) -> None:
"""Handle websocket frame."""
raise NotImplementedError() # pragma: no cover
@abstractmethod
def on_websocket_close(self) -> None:
"""Called when websocket connection has been closed."""
raise NotImplementedError() # pragma: no cover

18
proxy/http/server/protocols.py Normal file
View File

@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
"""
proxy.py
~~~~~~~~
⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
Network monitoring, controls & Application development, testing, debugging.
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from typing import NamedTuple
HttpProtocolTypes = NamedTuple('HttpProtocolTypes', [
('HTTP', int),
('HTTPS', int),
('WEBSOCKET', int),
])
httpProtocolTypes = HttpProtocolTypes(1, 2, 3)

115
proxy/http/server.py → proxy/http/server/web.py
View File

@ -15,116 +15,23 @@ import logging
import os
import mimetypes
import socket
from abc import ABC, abstractmethod
from typing import List, Tuple, Optional, NamedTuple, Dict, Union, Any, Pattern
from typing import List, Tuple, Optional, Dict, Union, Any, Pattern
from .exception import HttpProtocolException
from .websocket import WebsocketFrame, websocketOpcodes
from .codes import httpStatusCodes
from .parser import HttpParser, httpParserStates, httpParserTypes
from .handler import HttpProtocolHandlerPlugin
from .plugin import HttpWebServerBasePlugin
from .protocols import httpProtocolTypes
from ..exception import HttpProtocolException
from ..websocket import WebsocketFrame, websocketOpcodes
from ..codes import httpStatusCodes
from ..parser import HttpParser, httpParserStates, httpParserTypes
from ..handler import HttpProtocolHandlerPlugin
from ..common.utils import bytes_, text_, build_http_response, build_websocket_handshake_response
from ..common.flags import Flags
from ..common.constants import PROXY_AGENT_HEADER_VALUE
from ..common.types import HasFileno
from ..core.connection import TcpClientConnection
from ..core.event import EventQueue
from ...common.utils import bytes_, text_, build_http_response, build_websocket_handshake_response
from ...common.constants import PROXY_AGENT_HEADER_VALUE
from ...common.types import HasFileno
logger = logging.getLogger(__name__)
HttpProtocolTypes = NamedTuple('HttpProtocolTypes', [
('HTTP', int),
('HTTPS', int),
('WEBSOCKET', int),
])
httpProtocolTypes = HttpProtocolTypes(1, 2, 3)
class HttpWebServerBasePlugin(ABC):
"""Web Server Plugin for routing of requests."""
def __init__(
self,
uid: str,
flags: Flags,
client: TcpClientConnection,
event_queue: EventQueue):
self.uid = uid
self.flags = flags
self.client = client
self.event_queue = event_queue
@abstractmethod
def routes(self) -> List[Tuple[int, str]]:
"""Return List(protocol, path) that this plugin handles."""
raise NotImplementedError() # pragma: no cover
@abstractmethod
def handle_request(self, request: HttpParser) -> None:
"""Handle the request and serve response."""
raise NotImplementedError() # pragma: no cover
@abstractmethod
def on_websocket_open(self) -> None:
"""Called when websocket handshake has finished."""
raise NotImplementedError() # pragma: no cover
@abstractmethod
def on_websocket_message(self, frame: WebsocketFrame) -> None:
"""Handle websocket frame."""
raise NotImplementedError() # pragma: no cover
@abstractmethod
def on_websocket_close(self) -> None:
"""Called when websocket connection has been closed."""
raise NotImplementedError() # pragma: no cover
class HttpWebServerPacFilePlugin(HttpWebServerBasePlugin):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.pac_file_response: Optional[memoryview] = None
self.cache_pac_file_response()
def routes(self) -> List[Tuple[int, str]]:
if self.flags.pac_file_url_path:
return [
(httpProtocolTypes.HTTP, text_(self.flags.pac_file_url_path)),
(httpProtocolTypes.HTTPS, text_(self.flags.pac_file_url_path)),
]
return [] # pragma: no cover
def handle_request(self, request: HttpParser) -> None:
if self.flags.pac_file and self.pac_file_response:
self.client.queue(self.pac_file_response)
def on_websocket_open(self) -> None:
pass # pragma: no cover
def on_websocket_message(self, frame: WebsocketFrame) -> None:
pass # pragma: no cover
def on_websocket_close(self) -> None:
pass # pragma: no cover
def cache_pac_file_response(self) -> None:
if self.flags.pac_file:
try:
with open(self.flags.pac_file, 'rb') as f:
content = f.read()
except IOError:
content = bytes_(self.flags.pac_file)
self.pac_file_response = memoryview(build_http_response(
200, reason=b'OK', headers={
b'Content-Type': b'application/x-ns-proxy-autoconfig',
b'Content-Encoding': b'gzip',
}, body=gzip.compress(content)
))
class HttpWebServerPlugin(HttpProtocolHandlerPlugin):
"""HttpProtocolHandler plugin which handles incoming requests to local web server."""

4
proxy/plugin/reverse_proxy.py
View File

@ -12,7 +12,7 @@ import random
from typing import List, Tuple
from urllib import parse as urlparse
from ..common.constants import DEFAULT_BUFFER_SIZE
from ..common.constants import DEFAULT_BUFFER_SIZE, DEFAULT_HTTP_PORT
from ..common.utils import socket_connection, text_
from ..http.parser import HttpParser
from ..http.websocket import WebsocketFrame
@ -58,7 +58,7 @@ class ReverseProxyPlugin(HttpWebServerBasePlugin):
upstream = random.choice(ReverseProxyPlugin.REVERSE_PROXY_PASS)
url = urlparse.urlsplit(upstream)
assert url.hostname
with socket_connection((text_(url.hostname), url.port if url.port else 80)) as conn:
with socket_connection((text_(url.hostname), url.port if url.port else DEFAULT_HTTP_PORT)) as conn:
conn.send(request.build())
self.client.queue(memoryview(conn.recv(DEFAULT_BUFFER_SIZE)))

3
tests/common/test_utils.py
View File

@ -13,6 +13,7 @@ import unittest
from unittest import mock
from proxy.common.constants import DEFAULT_IPV6_HOSTNAME, DEFAULT_IPV4_HOSTNAME, DEFAULT_PORT, DEFAULT_TIMEOUT
from proxy.common.constants import DEFAULT_HTTP_PORT
from proxy.common.utils import new_socket_connection, socket_connection
@ -21,7 +22,7 @@ class TestSocketConnectionUtils(unittest.TestCase):
def setUp(self) -> None:
self.addr_ipv4 = (str(DEFAULT_IPV4_HOSTNAME), DEFAULT_PORT)
self.addr_ipv6 = (str(DEFAULT_IPV6_HOSTNAME), DEFAULT_PORT)
self.addr_dual = ('httpbin.org', 80)
self.addr_dual = ('httpbin.org', DEFAULT_HTTP_PORT)
@mock.patch('socket.socket')
def test_new_socket_connection_ipv4(self, mock_socket: mock.Mock) -> None:

7
tests/http/test_http_proxy.py
View File

@ -12,6 +12,7 @@ import unittest
import selectors
from unittest import mock
from proxy.common.constants import DEFAULT_HTTP_PORT
from proxy.common.flags import Flags
from proxy.http.proxy import HttpProxyPlugin
from proxy.http.handler import HttpProtocolHandler
@ -45,7 +46,7 @@ class TestHttpProxyPlugin(unittest.TestCase):
def test_proxy_plugin_initialized(self) -> None:
self.plugin.assert_called()
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_proxy_plugin_on_and_before_upstream_connection(
self,
mock_server_conn: mock.Mock) -> None:
@ -65,11 +66,11 @@ class TestHttpProxyPlugin(unittest.TestCase):
data=None), selectors.EVENT_READ)], ]
self.protocol_handler.run_once()
mock_server_conn.assert_called_with('upstream.host', 80)
mock_server_conn.assert_called_with('upstream.host', DEFAULT_HTTP_PORT)
self.plugin.return_value.before_upstream_connection.assert_called()
self.plugin.return_value.handle_client_request.assert_called()
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_proxy_plugin_before_upstream_connection_can_teardown(
self,
mock_server_conn: mock.Mock) -> None:

2
tests/http/test_http_proxy_tls_interception.py
View File

@ -28,7 +28,7 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
@mock.patch('ssl.wrap_socket')
@mock.patch('ssl.create_default_context')
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
@mock.patch('subprocess.Popen')
@mock.patch('selectors.DefaultSelector')
@mock.patch('socket.fromfd')

8
tests/http/test_protocol_handler.py
View File

@ -47,7 +47,7 @@ class TestHttpProtocolHandler(unittest.TestCase):
self.fileno, self._addr, flags=self.flags)
self.protocol_handler.initialize()
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_http_get(self, mock_server_connection: mock.Mock) -> None:
server = mock_server_connection.return_value
server.connect.return_value = True
@ -99,7 +99,7 @@ class TestHttpProtocolHandler(unittest.TestCase):
assert parser.code is not None
self.assertEqual(int(parser.code), 200)
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_http_tunnel(self, mock_server_connection: mock.Mock) -> None:
server = mock_server_connection.return_value
server.connect.return_value = True
@ -189,7 +189,7 @@ class TestHttpProtocolHandler(unittest.TestCase):
@mock.patch('selectors.DefaultSelector')
@mock.patch('socket.fromfd')
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_authenticated_proxy_http_get(
self, mock_server_connection: mock.Mock,
mock_fromfd: mock.Mock,
@ -237,7 +237,7 @@ class TestHttpProtocolHandler(unittest.TestCase):
@mock.patch('selectors.DefaultSelector')
@mock.patch('socket.fromfd')
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_authenticated_proxy_http_tunnel(
self, mock_server_connection: mock.Mock,
mock_fromfd: mock.Mock,

16
tests/plugin/test_http_proxy_plugins.py
View File

@ -20,7 +20,7 @@ from proxy.common.flags import Flags
from proxy.http.handler import HttpProtocolHandler
from proxy.http.proxy import HttpProxyPlugin
from proxy.common.utils import build_http_request, bytes_, build_http_response
from proxy.common.constants import PROXY_AGENT_HEADER_VALUE
from proxy.common.constants import PROXY_AGENT_HEADER_VALUE, DEFAULT_HTTP_PORT
from proxy.http.codes import httpStatusCodes
from proxy.plugin import ProposedRestApiPlugin, RedirectToCustomServerPlugin
@ -54,7 +54,7 @@ class TestHttpProxyPluginExamples(unittest.TestCase):
self.fileno, self._addr, flags=self.flags)
self.protocol_handler.initialize()
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_modify_post_data_plugin(
self, mock_server_conn: mock.Mock) -> None:
original = b'{"key": "value"}'
@ -77,7 +77,7 @@ class TestHttpProxyPluginExamples(unittest.TestCase):
data=None), selectors.EVENT_READ)], ]
self.protocol_handler.run_once()
mock_server_conn.assert_called_with('httpbin.org', 80)
mock_server_conn.assert_called_with('httpbin.org', DEFAULT_HTTP_PORT)
mock_server_conn.return_value.queue.assert_called_with(
build_http_request(
b'POST', b'/post',
@ -91,7 +91,7 @@ class TestHttpProxyPluginExamples(unittest.TestCase):
)
)
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_proposed_rest_api_plugin(
self, mock_server_conn: mock.Mock) -> None:
path = b'/v1/users/'
@ -121,7 +121,7 @@ class TestHttpProxyPluginExamples(unittest.TestCase):
ProposedRestApiPlugin.REST_API_SPEC[path]))
))
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_redirect_to_custom_server_plugin(
self, mock_server_conn: mock.Mock) -> None:
request = build_http_request(
@ -152,7 +152,7 @@ class TestHttpProxyPluginExamples(unittest.TestCase):
)
)
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_filter_by_upstream_host_plugin(
self, mock_server_conn: mock.Mock) -> None:
request = build_http_request(
@ -182,7 +182,7 @@ class TestHttpProxyPluginExamples(unittest.TestCase):
)
)
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
def test_man_in_the_middle_plugin(
self, mock_server_conn: mock.Mock) -> None:
request = build_http_request(
@ -224,7 +224,7 @@ class TestHttpProxyPluginExamples(unittest.TestCase):
# Client read
self.protocol_handler.run_once()
mock_server_conn.assert_called_with('super.secure', 80)
mock_server_conn.assert_called_with('super.secure', DEFAULT_HTTP_PORT)
server.connect.assert_called_once()
queued_request = \
build_http_request(

2
tests/plugin/test_http_proxy_plugins_with_tls_interception.py
View File

@ -31,7 +31,7 @@ class TestHttpProxyPluginExamplesWithTlsInterception(unittest.TestCase):
@mock.patch('ssl.wrap_socket')
@mock.patch('ssl.create_default_context')
@mock.patch('proxy.http.proxy.TcpServerConnection')
@mock.patch('proxy.http.proxy.server.TcpServerConnection')
@mock.patch('subprocess.Popen')
@mock.patch('selectors.DefaultSelector')
@mock.patch('socket.fromfd')