From 2a9db3a2dc9904bdafbf090a288823909d936814 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Fri, 5 Nov 2021 03:11:22 +0530 Subject: [PATCH] Update outdated sections of the codebase (#670) * Remove autopep8, is redundant now after recent CI changes * Add pyenv .python-version to .gitignore * Update year * Add lib-pytest target so that pytest can run in isolation * Add git-push hook which will also run the lint. By default now git-pre-commit hook will only run pytest. * Update outdated sections of README * Update requirement to match setup.cfg install_requires * Deprecate proxy.start and TestCase.PROXY_PORT Proxy port during test is now available as self.PROXY.pool.flags.port. Also now TestCase utilize ephemeral port strategy instead of calling get_available_port utility method. * Rename to git-pre-push * Ideally public repo dont require CODECOV_TOKEN but codecov integration is broken since introduction of codecov-action@v2 (instead of codecov binary invocation) * Issue is possibly with codecov@v2 action, fallback to codecov. See https://github.com/abhinavsingh/proxy.py/runs/4110423084\?check_suite_focus\=true and https://github.com/codecov/uploader/issues/223 * Revert back to v2 --- .gitignore | 1 + LICENSE | 2 +- Makefile | 21 +++-- README.md | 152 ++++++++++++++++++++++-------------- git-pre-commit | 2 +- git-pre-push | 3 + proxy/__init__.py | 6 +- proxy/proxy.py | 18 +---- proxy/testing/test_case.py | 11 +-- requirements.txt | 2 +- tests/testing/test_embed.py | 12 +-- tox.ini | 2 +- 12 files changed, 125 insertions(+), 107 deletions(-) create mode 100755 git-pre-push diff --git a/.gitignore b/.gitignore index 343de585..2c194fa6 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ .mypy_cache .hypothesis .tox +.python-version coverage.xml proxy.py.iml diff --git a/LICENSE b/LICENSE index d607876f..35873c12 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013-2020 by Abhinav Singh and contributors. +Copyright (c) 2013-2022 by Abhinav Singh and contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/Makefile b/Makefile index 66dcd365..d4f0e6d3 100644 --- a/Makefile +++ b/Makefile @@ -15,22 +15,14 @@ CA_KEY_FILE_PATH := ca-key.pem CA_CERT_FILE_PATH := ca-cert.pem CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem -.PHONY: all https-certificates ca-certificates autopep8 devtools -.PHONY: lib-version lib-clean lib-test lib-package lib-coverage lib-lint +.PHONY: all https-certificates sign-https-certificates ca-certificates +.PHONY: lib-version lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest .PHONY: lib-release-test lib-release lib-profile .PHONY: container container-run container-release -.PHONY: dashboard dashboard-clean +.PHONY: devtools dashboard dashboard-clean all: lib-test -devtools: - pushd dashboard && npm run devtools && popd - -autopep8: - autopep8 --recursive --in-place --aggressive examples - autopep8 --recursive --in-place --aggressive proxy - autopep8 --recursive --in-place --aggressive tests - https-certificates: # Generate server key python -m proxy.common.pki gen_private_key \ @@ -91,9 +83,11 @@ lib-clean: lib-lint: python -m tox -e lint -lib-test: lib-clean lib-version lib-lint +lib-pytest: python -m tox -e python -- -v +lib-test: lib-clean lib-version lib-lint lib-pytest + lib-package: lib-clean lib-version python -m tox -e cleanup-dists,build-dists,metadata-validation @@ -110,6 +104,9 @@ lib-coverage: lib-profile: sudo py-spy record -o profile.svg -t -F -s -- python -m proxy +devtools: + pushd dashboard && npm run devtools && popd + dashboard: pushd dashboard && npm run build && popd diff --git a/README.md b/README.md index 13869df9..dbc567a2 100644 --- a/README.md +++ b/README.md @@ -69,18 +69,19 @@ - [Embed proxy.py](#embed-proxypy) - [Blocking Mode](#blocking-mode) - [Non-blocking Mode](#non-blocking-mode) + - [Ephemeral Port](#ephemeral-port) - [Loading Plugins](#loading-plugins) - [Unit testing with proxy.py](#unit-testing-with-proxypy) - [proxy.TestCase](#proxytestcase) - [Override Startup Flags](#override-startup-flags) - [With unittest.TestCase](#with-unittesttestcase) - [Plugin Developer and Contributor Guide](#plugin-developer-and-contributor-guide) + - [High level architecture](#high-level-architecture) - [Everything is a plugin](#everything-is-a-plugin) - - [Internal Architecture](#internal-architecture) - [Internal Documentation](#internal-documentation) - [Development Guide](#development-guide) - [Setup Local Environment](#setup-local-environment) - - [Setup pre-commit hook](#setup-pre-commit-hook) + - [Setup Git Hooks](#setup-git-hooks) - [Sending a Pull Request](#sending-a-pull-request) - [Utilities](#utilities) - [TCP](#tcp-sockets) @@ -307,8 +308,7 @@ To start `proxy.py` from source code follow these instructions: - Install deps ```bash - ❯ pip install -r requirements.txt - ❯ pip install -r requirements-testing.txt + ❯ pip install -rrequirements.txt -rrequirements-testing.txt -rrequirements-tunnel.txt ``` - Run tests @@ -1149,24 +1149,40 @@ by using `start` method: Example: import proxy if __name__ == '__main__': - with proxy.start([]): + with proxy.Proxy([]) as p: # ... your logic here ... ``` Note that: -1. `start` is similar to `main`, except `start` won't block. -1. `start` is a context manager. +1. `Proxy` is similar to `main`, except `Proxy` does not block. +1. Internally `Proxy` is a context manager. It will start `proxy.py` when called and will shut it down - once scope ends. -1. Just like `main`, startup flags with `start` method + once the scope ends. +1. Just like `main`, startup flags with `Proxy` can be customized by either passing flags as list of - input arguments e.g. `start(['--port', '8899'])` or - by using passing flags as kwargs e.g. `start(port=8899)`. + input arguments e.g. `Proxy(['--port', '8899'])` or + by using passing flags as kwargs e.g. `Proxy(port=8899)`. + +## Ephemeral Port + +Use `--port=0` to bind `proxy.py` on a random port allocated by the kernel. + +In embedded mode, you can access this port. Example: + +```python +import proxy + +if __name__ == '__main__': + with proxy.Proxy([]) as p: + print(p.pool.flags.port) +``` + +`pool.flags.port` will give you access to the random port allocated by the kernel. ## Loading Plugins -You can, of course, list plugins to load in the input arguments list of `proxy.main`, `proxy.start` or the `Proxy` constructor. Use the `--plugins` flag as when starting from command line: +You can, of course, list plugins to load in the input arguments list of `proxy.main` or `Proxy` constructor. Use the `--plugins` flag when starting from command line: ```python import proxy @@ -1177,7 +1193,7 @@ if __name__ == '__main__': ]) ``` -However, for simplicity you can pass the list of plugins to load as a keyword argument to `proxy.main`, `proxy.start` or the `Proxy` constructor: +For simplicity you can pass the list of plugins to load as a keyword argument to `proxy.main` or the `Proxy` constructor: ```python import proxy @@ -1193,20 +1209,19 @@ if __name__ == '__main__': Note that it supports: 1. The fully-qualified name of a class as `bytes` -2. Any `type` instance for a Proxy.py plugin class. This is especially useful for custom plugins defined locally. +2. Any `type` instance of a plugin class. This is especially useful for plugins defined at runtime # Unit testing with proxy.py ## proxy.TestCase -To setup and teardown `proxy.py` for your Python unittest classes, +To setup and teardown `proxy.py` for your Python `unittest` classes, simply use `proxy.TestCase` instead of `unittest.TestCase`. Example: ```python import proxy - class TestProxyPyEmbedded(proxy.TestCase): def test_my_application_with_proxy(self) -> None: @@ -1217,7 +1232,7 @@ Note that: 1. `proxy.TestCase` overrides `unittest.TestCase.run()` method to setup and teardown `proxy.py`. 2. `proxy.py` server will listen on a random available port on the system. - This random port is available as `self.PROXY_PORT` within your test cases. + This random port is available as `self.PROXY.pool.flags.port` within your test cases. 3. Only a single worker is started by default (`--num-workers 1`) for faster setup and teardown. 4. Most importantly, `proxy.TestCase` also ensures `proxy.py` server is up and running before proceeding with execution of tests. By default, @@ -1272,52 +1287,63 @@ or simply setup / teardown `proxy.py` within # Plugin Developer and Contributor Guide +## High level architecture + +```bash + +-------------+ + | Proxy([]) | + +------+------+ + | + | + +-----------v--------------+ + | AcceptorPool(...) | + +------------+-------------+ + | + | ++-----------------+ | +-----------------+ +| Acceptor(..) <-------------+-----------> Acceptor(..) | ++-----------------+ +-----------------+ +``` + +`proxy.py` is made with performance in mind. By default, `proxy.py` +will try to utilize all available CPU cores to it for accepting new +client connections. This is achieved by starting `AcceptorPool` which +listens on configured server port. Then, `AcceptorPool` starts `Acceptor` +processes (`--num-workers`) to accept incoming client connections. + +Each `Acceptor` process delegates the accepted client connection +to a `Work` class. Currently, `HttpProtocolHandler` is the default +work klass hardcoded into the code. + +`HttpProtocolHandler` simply assumes that incoming clients will follow +HTTP specification. Specific HTTP proxy and HTTP server implementations +are written as plugins of `HttpProtocolHandler`. + +See documentation of `HttpProtocolHandlerPlugin` for available lifecycle hooks. +Use `HttpProtocolHandlerPlugin` to add new features for http(s) clients. Example, +See `HttpWebServerPlugin`. + ## Everything is a plugin -As you might have guessed by now, in `proxy.py` everything is a plugin. +Within `proxy.py` everything is a plugin. -- We enabled proxy server plugins using `--plugins` flag. - All the [plugin examples](#plugin-examples) were implementing - `HttpProxyBasePlugin`. See documentation of - [HttpProxyBasePlugin](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L894-L938) - for available lifecycle hooks. Use `HttpProxyBasePlugin` to modify - behavior of http(s) proxy protocol between client and upstream server. - Example, [FilterByUpstreamHostPlugin](#filterbyupstreamhostplugin). +- We enabled `proxy server` plugins using `--plugins` flag. + Proxy server `HttpProxyPlugin` is a plugin of `HttpProtocolHandler`. + Further, Proxy server allows plugin through `HttpProxyBasePlugin` specification. -- We also enabled inbuilt web server using `--enable-web-server`. - Inbuilt web server implements `HttpProtocolHandlerPlugin` plugin. - See documentation of [HttpProtocolHandlerPlugin](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L793-L850) - for available lifecycle hooks. Use `HttpProtocolHandlerPlugin` to add - new features for http(s) clients. Example, - [HttpWebServerPlugin](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L1185-L1260). +- All the proxy server [plugin examples](#plugin-examples) were implementing + `HttpProxyBasePlugin`. See documentation of `HttpProxyBasePlugin` for available + lifecycle hooks. Use `HttpProxyBasePlugin` to modify behavior of http(s) proxy protocol + between client and upstream server. Example, + [FilterByUpstreamHostPlugin](#filterbyupstreamhostplugin). + +- We also enabled inbuilt `web server` using `--enable-web-server`. + Web server `HttpWebServerPlugin` is a plugin of `HttpProtocolHandler` + and implements `HttpProtocolHandlerPlugin` specification. - There also is a `--disable-http-proxy` flag. It disables inbuilt proxy server. Use this flag with `--enable-web-server` flag to run `proxy.py` as a programmable - http(s) server. [HttpProxyPlugin](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L941-L1182) - also implements `HttpProtocolHandlerPlugin`. - -## Internal Architecture - -- [HttpProtocolHandler](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L1263-L1440) - thread is started with the accepted [TcpClientConnection](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L230-L237). - `HttpProtocolHandler` is responsible for parsing incoming client request and invoking - `HttpProtocolHandlerPlugin` lifecycle hooks. - -- `HttpProxyPlugin` which implements `HttpProtocolHandlerPlugin` also has its own plugin - mechanism. Its responsibility is to establish connection between client and - upstream [TcpServerConnection](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L204-L227) - and invoke `HttpProxyBasePlugin` lifecycle hooks. - -- `HttpProtocolHandler` threads are started by [Acceptor](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L424-L472) - processes. - -- `--num-workers` `Acceptor` processes are started by - [AcceptorPool](https://github.com/abhinavsingh/proxy.py/blob/b03629fa0df1595eb4995427bc601063be7fdca9/proxy.py#L368-L421) - on start-up. - -- `AcceptorPool` listens on server socket and pass the handler to `Acceptor` processes. - Workers are responsible for accepting new client connections and starting - `HttpProtocolHandler` thread. + http(s) server. ## Development Guide @@ -1327,13 +1353,23 @@ Contributors must start `proxy.py` from source to verify and develop new feature See [Run proxy.py from command line using repo source](#from-command-line-using-repo-source) for details. -### Setup pre-commit hook +[![WARNING](https://img.shields.io/static/v1?label=MacOS&message=warning&color=red)] +(https://github.com/abhinavsingh/proxy.py/issues/642#issuecomment-960819271) On `macOS` +you must install `Python` using `pyenv`, as `Python` installed via `homebrew` tends +to be problematic. See linked thread for more details. -Pre-commit hook ensures lint checking and tests execution. +### Setup Git Hooks + +Pre-commit hook ensures tests are passing. 1. `cd /path/to/proxy.py` 2. `ln -s $(PWD)/git-pre-commit .git/hooks/pre-commit` +Pre-push hook ensures lint and tests are passing. + +1. `cd /path/to/proxy.py` +2. `ln -s $(PWD)/git-pre-push .git/hooks/pre-push` + ### Sending a Pull Request Every pull request is tested using GitHub actions. diff --git a/git-pre-commit b/git-pre-commit index 2aad4864..aeef83fa 100755 --- a/git-pre-commit +++ b/git-pre-commit @@ -1,3 +1,3 @@ #!/bin/bash -make +make lib-pytest diff --git a/git-pre-push b/git-pre-push new file mode 100755 index 00000000..2aad4864 --- /dev/null +++ b/git-pre-push @@ -0,0 +1,3 @@ +#!/bin/bash + +make diff --git a/proxy/__init__.py b/proxy/__init__.py index ca7c5269..16cacc41 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -8,9 +8,7 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from .proxy import entry_point -from .proxy import main, start -from .proxy import Proxy +from .proxy import entry_point, main, Proxy from .testing.test_case import TestCase __all__ = [ @@ -19,7 +17,7 @@ __all__ = [ 'entry_point', # Embed proxy.py. See # https://github.com/abhinavsingh/proxy.py#embed-proxypy - 'main', 'start', + 'main', # Unit testing with proxy.py. See # https://github.com/abhinavsingh/proxy.py#unit-testing-with-proxypy 'TestCase', diff --git a/proxy/proxy.py b/proxy/proxy.py index f6f3e2e2..330c7a9c 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -12,7 +12,6 @@ import abc import argparse import base64 import collections -import contextlib import ipaddress import multiprocessing import os @@ -24,7 +23,7 @@ import importlib import inspect from types import TracebackType -from typing import Dict, List, Optional, Generator, Any, Tuple, Type, Union, cast +from typing import Dict, List, Optional, Any, Tuple, Type, Union, cast from proxy.core.acceptor.work import Work @@ -497,21 +496,6 @@ class Proxy: ) -@contextlib.contextmanager -def start( - input_args: Optional[List[str]] = None, - **opts: Any, -) -> Generator[Proxy, None, None]: - """Deprecated. Kept for backward compatibility. - - New users must directly use proxy.Proxy context manager class.""" - try: - with Proxy(input_args, **opts) as p: - yield p - except KeyboardInterrupt: - pass - - def main( input_args: Optional[List[str]] = None, **opts: Any, diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py index 573ffa46..6416f82f 100644 --- a/proxy/testing/test_case.py +++ b/proxy/testing/test_case.py @@ -15,7 +15,7 @@ from typing import Optional, List, Generator, Any from ..proxy import Proxy from ..common.constants import DEFAULT_TIMEOUT -from ..common.utils import get_available_port, new_socket_connection +from ..common.utils import new_socket_connection from ..plugin import CacheResponsesPlugin @@ -27,21 +27,18 @@ class TestCase(unittest.TestCase): '--threadless', ] - PROXY_PORT: int = 8899 PROXY: Optional[Proxy] = None INPUT_ARGS: Optional[List[str]] = None @classmethod def setUpClass(cls) -> None: - cls.PROXY_PORT = get_available_port() - cls.INPUT_ARGS = getattr(cls, 'PROXY_PY_STARTUP_FLAGS') \ if hasattr(cls, 'PROXY_PY_STARTUP_FLAGS') \ else cls.DEFAULT_PROXY_PY_STARTUP_FLAGS cls.INPUT_ARGS.append('--hostname') cls.INPUT_ARGS.append('0.0.0.0') cls.INPUT_ARGS.append('--port') - cls.INPUT_ARGS.append(str(cls.PROXY_PORT)) + cls.INPUT_ARGS.append('0') cls.PROXY = Proxy(input_args=cls.INPUT_ARGS) cls.PROXY.flags.plugins[b'HttpProxyBasePlugin'].append( @@ -49,7 +46,8 @@ class TestCase(unittest.TestCase): ) cls.PROXY.__enter__() - cls.wait_for_server(cls.PROXY_PORT) + assert cls.PROXY.pool + cls.wait_for_server(cls.PROXY.pool.flags.port) @staticmethod def wait_for_server( @@ -77,7 +75,6 @@ class TestCase(unittest.TestCase): def tearDownClass(cls) -> None: assert cls.PROXY cls.PROXY.__exit__(None, None, None) - cls.PROXY_PORT = 8899 cls.PROXY = None cls.INPUT_ARGS = None diff --git a/requirements.txt b/requirements.txt index 71f75de8..50d48949 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -typing-extensions==3.10.0.2; python_version < "3.8" +typing-extensions; python_version < "3.8" diff --git a/tests/testing/test_embed.py b/tests/testing/test_embed.py index 87609c86..2f9f7d03 100644 --- a/tests/testing/test_embed.py +++ b/tests/testing/test_embed.py @@ -34,12 +34,13 @@ class TestProxyPyEmbedded(TestCase): def test_with_proxy(self) -> None: """Makes a HTTP request to in-build web server via proxy server.""" - with socket_connection(('localhost', self.PROXY_PORT)) as conn: + assert self.PROXY and self.PROXY.pool + with socket_connection(('localhost', self.PROXY.pool.flags.port)) as conn: conn.send( build_http_request( - httpMethods.GET, b'http://localhost:%d/' % self.PROXY_PORT, + httpMethods.GET, b'http://localhost:%d/' % self.PROXY.pool.flags.port, headers={ - b'Host': b'localhost:%d' % self.PROXY_PORT, + b'Host': b'localhost:%d' % self.PROXY.pool.flags.port, }, ), ) @@ -72,14 +73,15 @@ class TestProxyPyEmbedded(TestCase): self.make_http_request_using_proxy() def make_http_request_using_proxy(self) -> None: + assert self.PROXY and self.PROXY.pool proxy_handler = urllib.request.ProxyHandler({ - 'http': 'http://localhost:%d' % self.PROXY_PORT, + 'http': 'http://localhost:%d' % self.PROXY.pool.flags.port, }) opener = urllib.request.build_opener(proxy_handler) with self.assertRaises(urllib.error.HTTPError): r: http.client.HTTPResponse = opener.open( 'http://localhost:%d/' % - self.PROXY_PORT, timeout=10, + self.PROXY.pool.flags.port, timeout=10, ) self.assertEqual(r.status, 404) self.assertEqual(r.headers.get('server'), PROXY_AGENT_HEADER_VALUE) diff --git a/tox.ini b/tox.ini index 370e1293..9f2f42cb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py35,py36,py37,py38 +envlist = py36,py37,py38,py39,py310 isolated_build = true minversion = 3.21.0