diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 489b01b0..21b2f994 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -169,6 +169,7 @@ repos: - --strict-optional - benchmark/ - examples/ + - skeleton/ - proxy/ - tests/ pass_filenames: false diff --git a/README.md b/README.md index 3b24f3ba..b85d1d7e 100644 --- a/README.md +++ b/README.md @@ -1784,7 +1784,9 @@ Listed below are a few strategies for using `proxy.py` in your private/productio > You MUST `avoid forking` the repository *"just"* to put your plugin code in `proxy/plugin` directory. Forking is recommended workflow for project contributors, NOT for project users. -Instead, use one of the suggested approaches from below. Then load your plugins using `--plugin`, `--plugins` flags or `plugin` kwargs. +- Instead, use one of the suggested approaches from below. +- Then load your plugins using `--plugin`, `--plugins` flags or `plugin` kwargs. +- See [skeleton](https://github.com/abhinavsingh/proxy.py/tree/develop/skeleton) app for example standalone project using `proxy.py`. ### Via Requirements @@ -2243,33 +2245,33 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--tunnel-ssh-key-passphrase TUNNEL_SSH_KEY_PASSPHRASE] [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--enable-events] [--threadless] [--threaded] [--num-workers NUM_WORKERS] - [--backlog BACKLOG] [--hostname HOSTNAME] [--port PORT] - [--port-file PORT_FILE] [--unix-socket-path UNIX_SOCKET_PATH] - [--local-executor LOCAL_EXECUTOR] [--num-acceptors NUM_ACCEPTORS] - [--version] [--log-level LOG_LEVEL] [--log-file LOG_FILE] - [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT] + [--local-executor LOCAL_EXECUTOR] [--backlog BACKLOG] + [--hostname HOSTNAME] [--port PORT] [--port-file PORT_FILE] + [--unix-socket-path UNIX_SOCKET_PATH] + [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL] + [--log-file LOG_FILE] [--log-format LOG_FORMAT] + [--open-file-limit OPEN_FILE_LIMIT] [--plugins PLUGINS [PLUGINS ...]] [--enable-dashboard] - [--enable-ssh-tunnel] [--work-klass WORK_KLASS] - [--pid-file PID_FILE] [--enable-conn-pool] [--key-file KEY_FILE] + [--basic-auth BASIC_AUTH] [--enable-ssh-tunnel] + [--work-klass WORK_KLASS] [--pid-file PID_FILE] + [--enable-proxy-protocol] [--enable-conn-pool] [--key-file KEY_FILE] [--cert-file CERT_FILE] [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--server-recvbuf-size SERVER_RECVBUF_SIZE] [--timeout TIMEOUT] - [--enable-proxy-protocol] [--disable-http-proxy] - [--disable-headers DISABLE_HEADERS] [--ca-key-file CA_KEY_FILE] - [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE] - [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE] - [--auth-plugin AUTH_PLUGIN] [--basic-auth BASIC_AUTH] - [--cache-dir CACHE_DIR] - [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS] - [--enable-web-server] [--enable-static-server] - [--static-server-dir STATIC_SERVER_DIR] + [--disable-http-proxy] [--disable-headers DISABLE_HEADERS] + [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR] + [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE] + [--ca-signing-key-file CA_SIGNING_KEY_FILE] + [--auth-plugin AUTH_PLUGIN] [--cache-dir CACHE_DIR] + [--proxy-pool PROXY_POOL] [--enable-web-server] + [--enable-static-server] [--static-server-dir STATIC_SERVER_DIR] [--min-compression-length MIN_COMPRESSION_LENGTH] [--pac-file PAC_FILE] [--pac-file-url-path PAC_FILE_URL_PATH] - [--proxy-pool PROXY_POOL] + [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] + [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] - [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.4.0rc7.dev12+gd234339.d20220116 +proxy.py v2.4.0rc7.dev28+gfbd7b46.d20220120 options: -h, --help show this help message and exit @@ -2299,6 +2301,14 @@ options: handle each client connection. --num-workers NUM_WORKERS Defaults to number of CPU cores. + --local-executor LOCAL_EXECUTOR + Default: 1. Enabled by default. Use 0 to disable. When + enabled acceptors will make use of local (same + process) executor instead of distributing load across + remote (other process) executors. Enable this option + to achieve CPU affinity between acceptors and + executors, instead of using underlying OS kernel + scheduling algorithm. --backlog BACKLOG Default: 100. Maximum number of pending connections to proxy server --hostname HOSTNAME Default: 127.0.0.1. Server IP address. @@ -2309,14 +2319,6 @@ options: --unix-socket-path UNIX_SOCKET_PATH Default: None. Unix socket path to use. When provided --host and --port flags are ignored - --local-executor LOCAL_EXECUTOR - Default: 1. Enabled by default. Use 0 to disable. When - enabled acceptors will make use of local (same - process) executor instead of distributing load across - remote (other process) executors. Enable this option - to achieve CPU affinity between acceptors and - executors, instead of using underlying OS kernel - scheduling algorithm. --num-acceptors NUM_ACCEPTORS Defaults to number of CPU cores. --version, -v Prints proxy.py version. @@ -2335,11 +2337,17 @@ options: Comma separated plugins. You may use --plugins flag multiple times. --enable-dashboard Default: False. Enables proxy.py dashboard. + --basic-auth BASIC_AUTH + Default: No authentication. Specify colon separated + user:password to enable basic authentication. --enable-ssh-tunnel Default: False. Enable SSH tunnel. --work-klass WORK_KLASS Default: proxy.http.HttpProtocolHandler. Work klass to use for work execution. --pid-file PID_FILE Default: None. Save "parent" process ID to a file. + --enable-proxy-protocol + Default: False. If used, will enable proxy protocol. + Only version 1 is currently supported. --enable-conn-pool Default: False. (WIP) Enable upstream connection pooling. --key-file KEY_FILE Default: None. Server key file to enable end-to-end @@ -2358,9 +2366,6 @@ options: --timeout TIMEOUT Default: 10.0. Number of seconds after which an inactive connection must be dropped. Inactivity is defined by no data sent or received by the client. - --enable-proxy-protocol - Default: False. If used, will enable proxy protocol. - Only version 1 is currently supported. --disable-http-proxy Default: False. Whether to disable proxy.HttpProxyPlugin. --disable-headers DISABLE_HEADERS @@ -2388,17 +2393,13 @@ options: generation of HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file --auth-plugin AUTH_PLUGIN - Default: proxy.http.proxy.AuthPlugin. Auth plugin to - use instead of default basic auth plugin. - --basic-auth BASIC_AUTH - Default: No authentication. Specify colon separated - user:password to enable basic authentication. + Default: proxy.http.proxy.auth.AuthPlugin. Auth plugin + to use instead of default basic auth plugin. --cache-dir CACHE_DIR Default: A temporary directory. Flag only applicable when cache plugin is used with on-disk storage. - --filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS - Default: Blocks Facebook. Comma separated list of IPv4 - and IPv6 addresses. + --proxy-pool PROXY_POOL + List of upstream proxies to use in the pool --enable-web-server Default: False. Whether to enable proxy.HttpWebServerPlugin. --enable-static-server @@ -2419,18 +2420,19 @@ options: this option enables proxy.HttpWebServerPlugin. --pac-file-url-path PAC_FILE_URL_PATH Default: /. Web server path to serve the PAC file. - --proxy-pool PROXY_POOL - List of upstream proxies to use in the pool + --cloudflare-dns-mode CLOUDFLARE_DNS_MODE + Default: security. Either "security" (for malware + protection) or "family" (for malware and adult content + protection) + --filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS + Default: Blocks Facebook. Comma separated list of IPv4 + and IPv6 addresses. --filtered-client-ips FILTERED_CLIENT_IPS Default: 127.0.0.1,::1. Comma separated list of IPv4 and IPv6 addresses. --filtered-url-regex-config FILTERED_URL_REGEX_CONFIG Default: No config. Comma separated list of IPv4 and IPv6 addresses. - --cloudflare-dns-mode CLOUDFLARE_DNS_MODE - Default: security. Either "security" (for malware - protection) or "family" (for malware and adult content - protection) Proxy.py not working? Report at: https://github.com/abhinavsingh/proxy.py/issues/new diff --git a/check.py b/check.py index 37320f79..40c6aa97 100644 --- a/check.py +++ b/check.py @@ -37,6 +37,7 @@ ALL_PY_FILES = ( list(REPO_ROOT.glob('*.py')) + list((REPO_ROOT / 'proxy').rglob('*.py')) + list((REPO_ROOT / 'examples').rglob('*.py')) + + list((REPO_ROOT / 'skeleton').rglob('*.py')) + list((REPO_ROOT / 'benchmark').rglob('*.py')) + list((REPO_ROOT / 'tests').rglob('*.py')) ) diff --git a/skeleton/README.md b/skeleton/README.md new file mode 100644 index 00000000..3a0655e1 --- /dev/null +++ b/skeleton/README.md @@ -0,0 +1,37 @@ +# Skeleton App + +This directory contains a sample standalone application structure which uses `proxy.py` +via `requirements.txt` file. + +## Setup + +```console +$ git clone https://github.com/abhinavsingh/proxy.py.git +$ cd proxy.py/skeleton +$ python3 -m venv .venv +$ source .venv/bin/activate +$ pip install -r requirements.txt +``` + +## Run It + +Start your app and make a web request to `/` and a proxy request via the instance. You will +see log lines like this: + +```console +$ python -m app +...[redacted]... - Loaded plugin proxy.http.proxy.HttpProxyPlugin +...[redacted]... - Loaded plugin proxy.http.server.HttpWebServerPlugin +...[redacted]... - Loaded plugin app.plugins.MyWebServerPlugin +...[redacted]... - Loaded plugin app.plugins.MyProxyPlugin +...[redacted]... - Listening on 127.0.0.1:9000 +...[redacted]... - Started 16 acceptors in threadless (local) mode +...[redacted]... - HttpProtocolException: HttpRequestRejected b"I'm a tea pot" +...[redacted]... - 127.0.0.1:64601 - GET None:None/get - None None - 0 bytes - 0.64ms +...[redacted]... - 127.0.0.1:64622 - GET / - curl/7.77.0 - 0.95ms +``` + +Voila!!! + +That is your custom app skeleton structure built on top of `proxy.py`. Now copy the `app` directory +outside of `proxy.py` repo and create your own git repo. Customize the `app` for your project needs diff --git a/skeleton/app/__init__.py b/skeleton/app/__init__.py new file mode 100755 index 00000000..09b790c2 --- /dev/null +++ b/skeleton/app/__init__.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from .app import entry_point + + +__all__ = [ + 'entry_point', +] diff --git a/skeleton/app/__main__.py b/skeleton/app/__main__.py new file mode 100644 index 00000000..ddbd774d --- /dev/null +++ b/skeleton/app/__main__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from app import entry_point + + +if __name__ == '__main__': + entry_point() diff --git a/skeleton/app/app.py b/skeleton/app/app.py new file mode 100644 index 00000000..857f253a --- /dev/null +++ b/skeleton/app/app.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import proxy + + +def entry_point() -> None: + with proxy.Proxy( + enable_web_server=True, + port=9000, + # NOTE: Pass plugins via *args if you define custom flags. + # Currently plugins passed via **kwargs are not discovered for + # custom flags by proxy.py + # + # See https://github.com/abhinavsingh/proxy.py/issues/871 + plugins=[ + 'app.plugins.MyWebServerPlugin', + 'app.plugins.MyProxyPlugin', + ], + ) as _: + proxy.sleep_loop() diff --git a/skeleton/app/plugins/__init__.py b/skeleton/app/plugins/__init__.py new file mode 100755 index 00000000..42f36060 --- /dev/null +++ b/skeleton/app/plugins/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from .my_web_plugin import MyWebServerPlugin +from .my_proxy_plugin import MyProxyPlugin + + +__all__ = [ + 'MyWebServerPlugin', + 'MyProxyPlugin', +] diff --git a/skeleton/app/plugins/my_proxy_plugin.py b/skeleton/app/plugins/my_proxy_plugin.py new file mode 100644 index 00000000..26566757 --- /dev/null +++ b/skeleton/app/plugins/my_proxy_plugin.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. + + .. spelling:: + + ip +""" +from typing import Optional + +from proxy.http import httpStatusCodes +from proxy.http.proxy import HttpProxyBasePlugin +from proxy.http.parser import HttpParser +from proxy.http.exception import HttpRequestRejected + + +class MyProxyPlugin(HttpProxyBasePlugin): + """Drop traffic by inspecting incoming client IP address.""" + + def before_upstream_connection( + self, request: HttpParser, + ) -> Optional[HttpParser]: + assert not self.flags.unix_socket_path and self.client.addr + if self.client.addr[0] in '127.0.0.1,::1'.split(','): + raise HttpRequestRejected( + status_code=httpStatusCodes.I_AM_A_TEAPOT, + reason=b'I\'m a tea pot', + ) + return request diff --git a/skeleton/app/plugins/my_web_plugin.py b/skeleton/app/plugins/my_web_plugin.py new file mode 100644 index 00000000..892b3746 --- /dev/null +++ b/skeleton/app/plugins/my_web_plugin.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import logging +from typing import List, Tuple + +from proxy.http.parser import HttpParser +from proxy.http.server import HttpWebServerBasePlugin, httpProtocolTypes +from proxy.http.responses import okResponse + + +logger = logging.getLogger(__name__) + + +class MyWebServerPlugin(HttpWebServerBasePlugin): + """Demonstrates inbuilt web server routing using plugin.""" + + def routes(self) -> List[Tuple[int, str]]: + return [ + (httpProtocolTypes.HTTP, r'/$'), + ] + + def handle_request(self, request: HttpParser) -> None: + self.client.queue(okResponse(content=b'Hello World')) diff --git a/skeleton/requirements.txt b/skeleton/requirements.txt new file mode 100644 index 00000000..2430cfd1 --- /dev/null +++ b/skeleton/requirements.txt @@ -0,0 +1 @@ +proxy.py @ git+https://github.com/abhinavsingh/proxy.py.git@develop