diff --git a/.gitignore b/.gitignore index ae8c07d0..3611a6c9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ .tox .python-version +__pycache__ + coverage.xml proxy.py.iml diff --git a/benchmark/_proxy.py b/benchmark/_proxy.py index 4a5ec0cf..f47b0039 100644 --- a/benchmark/_proxy.py +++ b/benchmark/_proxy.py @@ -22,7 +22,7 @@ if __name__ == '__main__': backlog=65536, open_file_limit=65536, enable_web_server=True, - disable_proxy_server=False, + disable_proxy_server=True, num_acceptors=10, local_executor=1, log_file='/dev/null', diff --git a/proxy/common/flag.py b/proxy/common/flag.py index edd07b7c..6786456f 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -98,17 +98,13 @@ class FlagParser: print(PY2_DEPRECATION_MESSAGE) sys.exit(1) - # Dirty hack to always discover --basic-auth flag - # defined by proxy auth plugin. - in_args = input_args + ['--plugin', PLUGIN_PROXY_AUTH] - # Discover flags from requested plugin. # This will also surface external plugin flags # under --help. - Plugins.discover(in_args) + Plugins.discover(input_args) # Parse flags - args = flags.parse_args(in_args) + args = flags.parse_args(input_args) # Print version and exit if args.version: diff --git a/proxy/http/proxy/auth.py b/proxy/http/proxy/auth.py index f2632d79..0238d212 100644 --- a/proxy/http/proxy/auth.py +++ b/proxy/http/proxy/auth.py @@ -18,18 +18,7 @@ from typing import Optional from ...http import httpHeaders from ..exception import ProxyAuthenticationFailed from ...http.proxy import HttpProxyBasePlugin -from ...common.flag import flags from ...http.parser import HttpParser -from ...common.constants import DEFAULT_BASIC_AUTH - - -flags.add_argument( - '--basic-auth', - type=str, - default=DEFAULT_BASIC_AUTH, - help='Default: No authentication. Specify colon separated user:password ' - 'to enable basic authentication.', -) class AuthPlugin(HttpProxyBasePlugin): diff --git a/proxy/proxy.py b/proxy/proxy.py index 8f04b2af..5725ffb7 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -11,6 +11,7 @@ import os import sys import time +import pprint import signal import logging from typing import Any, List, Optional @@ -23,9 +24,9 @@ from .common.utils import bytes_ from .core.acceptor import Listener, AcceptorPool from .common.constants import ( IS_WINDOWS, DEFAULT_PLUGINS, DEFAULT_VERSION, DEFAULT_LOG_FILE, - DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT, - DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_ENABLE_DASHBOARD, - DEFAULT_ENABLE_SSH_TUNNEL, + DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_BASIC_AUTH, + DEFAULT_LOG_FORMAT, DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT, + DEFAULT_ENABLE_DASHBOARD, DEFAULT_ENABLE_SSH_TUNNEL, ) @@ -40,7 +41,6 @@ flags.add_argument( help='Prints proxy.py version.', ) -# TODO: Convert me into 1-letter choices # TODO: Add --verbose option which also # starts to log traffic flowing between # clients and upstream servers. @@ -98,6 +98,16 @@ flags.add_argument( help='Default: False. Enables proxy.py dashboard.', ) +# NOTE: Same reason as mention above. +# Ideally this flag belongs to proxy auth plugin. +flags.add_argument( + '--basic-auth', + type=str, + default=DEFAULT_BASIC_AUTH, + help='Default: No authentication. Specify colon separated user:password ' + 'to enable basic authentication.', +) + flags.add_argument( '--enable-ssh-tunnel', action='store_true', @@ -133,6 +143,10 @@ class Proxy: Executor pool receives newly accepted work by :class:`~proxy.core.acceptor.Acceptor` and creates an instance of work class for processing the received work. + In ``--threadless`` mode and with ``--local-executor 0``, + acceptors will start a companion thread to handle accepted + client connections. + Optionally, Proxy class also initializes the EventManager. A multi-process safe pubsub system which can be used to build various patterns for message sharing and/or signaling. @@ -156,18 +170,17 @@ class Proxy: def setup(self) -> None: # TODO: Introduce cron feature - # https://github.com/abhinavsingh/proxy.py/issues/392 + # https://github.com/abhinavsingh/proxy.py/discussions/808 # - # TODO: Introduce ability to publish - # adhoc events which can modify behaviour of server - # at runtime. Example, updating flags, plugin - # configuration etc. + # TODO: Introduce ability to change flags dynamically + # https://github.com/abhinavsingh/proxy.py/discussions/1020 # - # TODO: Python shell within running proxy.py environment? + # TODO: Python shell within running proxy.py environment + # https://github.com/abhinavsingh/proxy.py/discussions/1021 + # + # TODO: Near realtime resource / stats monitoring + # https://github.com/abhinavsingh/proxy.py/discussions/1023 # - # TODO: Pid watcher which watches for processes started - # by proxy.py core. May be alert or restart those processes - # on failure. self._write_pid_file() # We setup listeners first because of flags.port override # in case of ephemeral port being used @@ -263,10 +276,15 @@ class Proxy: os.remove(self.flags.port_file) def _register_signals(self) -> None: - # TODO: Handle SIGINFO, SIGUSR1, SIGUSR2 + # TODO: Define SIGUSR1, SIGUSR2 signal.signal(signal.SIGINT, self._handle_exit_signal) signal.signal(signal.SIGTERM, self._handle_exit_signal) if not IS_WINDOWS: + if hasattr(signal, 'SIGINFO'): + signal.signal( # pragma: no cover + signal.SIGINFO, # pylint: disable=E1101 + self._handle_siginfo, + ) signal.signal(signal.SIGHUP, self._handle_exit_signal) # TODO: SIGQUIT is ideally meant to terminate with core dumps signal.signal(signal.SIGQUIT, self._handle_exit_signal) @@ -276,6 +294,9 @@ class Proxy: logger.info('Received signal %d' % signum) sys.exit(0) + def _handle_siginfo(self, _signum: int, _frame: Any) -> None: + pprint.pprint(self.flags.__dict__) # pragma: no cover + def sleep_loop() -> None: while True: diff --git a/pytest.ini b/pytest.ini index adf8bedc..1af93373 100644 --- a/pytest.ini +++ b/pytest.ini @@ -51,7 +51,7 @@ doctest_optionflags = ALLOW_UNICODE ELLIPSIS # Marks tests with an empty parameterset as xfail(run=False) empty_parameter_set_mark = xfail -faulthandler_timeout = 30 +faulthandler_timeout = 120 filterwarnings = error diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 93a88b73..bbe2fdec 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -23,7 +23,7 @@ import pytest from proxy.common.constants import IS_WINDOWS -def check_output(args: List[Any]) -> bytes: +def check_output(args: List[Any]) -> bytes: # pragma: no cover args = args if not IS_WINDOWS else ['powershell'] + args return _check_output(args) diff --git a/tests/integration/test_integration.sh b/tests/integration/test_integration.sh index 06cafd2d..428abdcc 100755 --- a/tests/integration/test_integration.sh +++ b/tests/integration/test_integration.sh @@ -23,6 +23,8 @@ if [[ -z "$PROXY_PY_PORT" ]]; then exit 1 fi +CURL="curl -v --connect-timeout 20 --max-time 120 --retry-connrefused --retry-delay 5 --retry 3" + PROXY_URL="http://localhost:$PROXY_PY_PORT" TEST_URL="$PROXY_URL/http-route-example" CURL_EXTRA_FLAGS="" @@ -50,7 +52,7 @@ while true; do done # Wait for http proxy and web server to start -CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL $TEST_URL" +CMD="$CURL $CURL_EXTRA_FLAGS -x $PROXY_URL $TEST_URL" while true; do RESPONSE=$($CMD 2> /dev/null) if [[ $? == 0 ]]; then @@ -90,13 +92,13 @@ Disallow: /deny EOM echo "[Test HTTP Request via Proxy]" -CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL http://httpbin.org/robots.txt" +CMD="$CURL $CURL_EXTRA_FLAGS -x $PROXY_URL http://httpbin.org/robots.txt" RESPONSE=$($CMD 2> /dev/null) verify_response "$RESPONSE" "$ROBOTS_RESPONSE" VERIFIED1=$? echo "[Test HTTPS Request via Proxy]" -CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL https://httpbin.org/robots.txt" +CMD="$CURL $CURL_EXTRA_FLAGS -x $PROXY_URL https://httpbin.org/robots.txt" RESPONSE=$($CMD 2> /dev/null) verify_response "$RESPONSE" "$ROBOTS_RESPONSE" VERIFIED2=$? @@ -105,7 +107,7 @@ if $USE_HTTPS; then VERIFIED3=0 else echo "[Test Internal Web Server via Proxy]" - curl -v \ + $CURL \ $CURL_EXTRA_FLAGS \ -x $PROXY_URL \ "$PROXY_URL" @@ -121,7 +123,7 @@ fi echo "[Test Download File Hash Verifies 1]" touch downloaded.hash echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc -" > downloaded.hash -curl -vL \ +$CURL -L \ $CURL_EXTRA_FLAGS \ -o downloaded.whl \ -x $PROXY_URL \ @@ -133,7 +135,7 @@ rm downloaded.whl downloaded.hash echo "[Test Download File Hash Verifies 2]" touch downloaded.hash echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 -" > downloaded.hash -curl -vL \ +$CURL -L \ $CURL_EXTRA_FLAGS \ -o downloaded.whl \ -x $PROXY_URL \ diff --git a/tests/test_circular_imports.py b/tests/test_circular_imports.py index 32763491..9fd74ebe 100644 --- a/tests/test_circular_imports.py +++ b/tests/test_circular_imports.py @@ -85,7 +85,15 @@ def _discover_path_importables( 'import_path', _find_all_importables(proxy), ) -def test_no_warnings(import_path: str) -> None: +# Marked as disabled_ because: +# 1. This test case was added when isort integration was problematic +# 2. This test case never found a real circular import scenario +# 3. This test case consumes 60% of test suite runtime +# 4. Kept in the repo because we might still want to enable +# this in future, conditionally. Example, we can run +# this only on a single OS and Python version combination +# instead of running it across entire matrix. +def disabled_test_no_warnings(import_path: str) -> None: """Verify that exploding importables doesn't explode. This is seeking for any import errors including ones caused