Handle `SIGINFO`. Try `kill -s INFO <pid>`. (#1024)

* Handle `SIGINFO`. Try `kill -s INFO <pid>`. Also remove dirty hack added in flags to incorporate `--basic-auth` flag. Add `__pycache__` to ignore list. Disable http proxy during acceptor benchmark. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * not on windows * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * no cover * # pylint: disable=E1101 * Curl retry on error and check_output 2 minute timeout * check output timeout None * Update `faulthandler_timeout` to 2 minutes * Disable `test_circular_imports`, `isort` integration now works * Fix curl flags * Revert back to older flags * SIGINFO attribute might not even exist Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
2022-01-20 23:54:44 +05:30 · 2022-01-20 23:54:44 +05:30 · 627b42f923
parent 85ad44b46e
commit 627b42f923
9 changed files with 59 additions and 41 deletions
--- a/.gitignore
+++ b/.gitignore
@ -10,6 +10,8 @@
 .tox
 .python-version

+__pycache__
+
 coverage.xml
 proxy.py.iml

--- a/benchmark/_proxy.py
+++ b/benchmark/_proxy.py
@ -22,7 +22,7 @@ if __name__ == '__main__':
            backlog=65536,
            open_file_limit=65536,
            enable_web_server=True,
-            disable_proxy_server=False,
+            disable_proxy_server=True,
            num_acceptors=10,
            local_executor=1,
            log_file='/dev/null',
--- a/proxy/common/flag.py
+++ b/proxy/common/flag.py
@ -98,17 +98,13 @@ class FlagParser:
            print(PY2_DEPRECATION_MESSAGE)
            sys.exit(1)

-        # Dirty hack to always discover --basic-auth flag
-        # defined by proxy auth plugin.
-        in_args = input_args + ['--plugin', PLUGIN_PROXY_AUTH]
-
        # Discover flags from requested plugin.
        # This will also surface external plugin flags
        # under --help.
-        Plugins.discover(in_args)
+        Plugins.discover(input_args)

        # Parse flags
-        args = flags.parse_args(in_args)
+        args = flags.parse_args(input_args)

        # Print version and exit
        if args.version:
--- a/proxy/http/proxy/auth.py
+++ b/proxy/http/proxy/auth.py
@ -18,18 +18,7 @@ from typing import Optional
 from ...http import httpHeaders
 from ..exception import ProxyAuthenticationFailed
 from ...http.proxy import HttpProxyBasePlugin
-from ...common.flag import flags
 from ...http.parser import HttpParser
-from ...common.constants import DEFAULT_BASIC_AUTH
-
-
-flags.add_argument(
-    '--basic-auth',
-    type=str,
-    default=DEFAULT_BASIC_AUTH,
-    help='Default: No authentication. Specify colon separated user:password '
-    'to enable basic authentication.',
-)


 class AuthPlugin(HttpProxyBasePlugin):
--- a/proxy/proxy.py
+++ b/proxy/proxy.py
@ -11,6 +11,7 @@
 import os
 import sys
 import time
+import pprint
 import signal
 import logging
 from typing import Any, List, Optional
@ -23,9 +24,9 @@ from .common.utils import bytes_
 from .core.acceptor import Listener, AcceptorPool
 from .common.constants import (
    IS_WINDOWS, DEFAULT_PLUGINS, DEFAULT_VERSION, DEFAULT_LOG_FILE,
-    DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT,
-    DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_ENABLE_DASHBOARD,
-    DEFAULT_ENABLE_SSH_TUNNEL,
+    DEFAULT_PID_FILE, DEFAULT_LOG_LEVEL, DEFAULT_BASIC_AUTH,
+    DEFAULT_LOG_FORMAT, DEFAULT_WORK_KLASS, DEFAULT_OPEN_FILE_LIMIT,
+    DEFAULT_ENABLE_DASHBOARD, DEFAULT_ENABLE_SSH_TUNNEL,
 )


@ -40,7 +41,6 @@ flags.add_argument(
    help='Prints proxy.py version.',
 )

-# TODO: Convert me into 1-letter choices
 # TODO: Add --verbose option which also
 # starts to log traffic flowing between
 # clients and upstream servers.
@ -98,6 +98,16 @@ flags.add_argument(
    help='Default: False.  Enables proxy.py dashboard.',
 )

+# NOTE: Same reason as mention above.
+# Ideally this flag belongs to proxy auth plugin.
+flags.add_argument(
+    '--basic-auth',
+    type=str,
+    default=DEFAULT_BASIC_AUTH,
+    help='Default: No authentication. Specify colon separated user:password '
+    'to enable basic authentication.',
+)
+
 flags.add_argument(
    '--enable-ssh-tunnel',
    action='store_true',
@ -133,6 +143,10 @@ class Proxy:
    Executor pool receives newly accepted work by :class:`~proxy.core.acceptor.Acceptor`
    and creates an instance of work class for processing the received work.

+    In ``--threadless`` mode and with ``--local-executor 0``,
+    acceptors will start a companion thread to handle accepted
+    client connections.
+
    Optionally, Proxy class also initializes the EventManager.
    A multi-process safe pubsub system which can be used to build various
    patterns for message sharing and/or signaling.
@ -156,18 +170,17 @@ class Proxy:

    def setup(self) -> None:
        # TODO: Introduce cron feature
-        # https://github.com/abhinavsingh/proxy.py/issues/392
+        # https://github.com/abhinavsingh/proxy.py/discussions/808
        #
-        # TODO: Introduce ability to publish
-        # adhoc events which can modify behaviour of server
-        # at runtime.  Example, updating flags, plugin
-        # configuration etc.
+        # TODO: Introduce ability to change flags dynamically
+        # https://github.com/abhinavsingh/proxy.py/discussions/1020
        #
-        # TODO: Python shell within running proxy.py environment?
+        # TODO: Python shell within running proxy.py environment
+        # https://github.com/abhinavsingh/proxy.py/discussions/1021
+        #
+        # TODO: Near realtime resource / stats monitoring
+        # https://github.com/abhinavsingh/proxy.py/discussions/1023
        #
-        # TODO: Pid watcher which watches for processes started
-        # by proxy.py core.  May be alert or restart those processes
-        # on failure.
        self._write_pid_file()
        # We setup listeners first because of flags.port override
        # in case of ephemeral port being used
@ -263,10 +276,15 @@ class Proxy:
            os.remove(self.flags.port_file)

    def _register_signals(self) -> None:
-        # TODO: Handle SIGINFO, SIGUSR1, SIGUSR2
+        # TODO: Define SIGUSR1, SIGUSR2
        signal.signal(signal.SIGINT, self._handle_exit_signal)
        signal.signal(signal.SIGTERM, self._handle_exit_signal)
        if not IS_WINDOWS:
+            if hasattr(signal, 'SIGINFO'):
+                signal.signal(      # pragma: no cover
+                    signal.SIGINFO,       # pylint: disable=E1101
+                    self._handle_siginfo,
+                )
            signal.signal(signal.SIGHUP, self._handle_exit_signal)
            # TODO: SIGQUIT is ideally meant to terminate with core dumps
            signal.signal(signal.SIGQUIT, self._handle_exit_signal)
@ -276,6 +294,9 @@ class Proxy:
        logger.info('Received signal %d' % signum)
        sys.exit(0)

+    def _handle_siginfo(self, _signum: int, _frame: Any) -> None:
+        pprint.pprint(self.flags.__dict__)  # pragma: no cover
+

 def sleep_loop() -> None:
    while True:
--- a/pytest.ini
+++ b/pytest.ini
@ -51,7 +51,7 @@ doctest_optionflags = ALLOW_UNICODE ELLIPSIS
 # Marks tests with an empty parameterset as xfail(run=False)
 empty_parameter_set_mark = xfail

-faulthandler_timeout = 30
+faulthandler_timeout = 120

 filterwarnings =
  error
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@ -23,7 +23,7 @@ import pytest
 from proxy.common.constants import IS_WINDOWS


-def check_output(args: List[Any]) -> bytes:
+def check_output(args: List[Any]) -> bytes:     # pragma: no cover
    args = args if not IS_WINDOWS else ['powershell'] + args
    return _check_output(args)

--- a/tests/integration/test_integration.sh
+++ b/tests/integration/test_integration.sh
@ -23,6 +23,8 @@ if [[ -z "$PROXY_PY_PORT" ]]; then
  exit 1
 fi

+CURL="curl -v --connect-timeout 20 --max-time 120 --retry-connrefused --retry-delay 5 --retry 3"
+
 PROXY_URL="http://localhost:$PROXY_PY_PORT"
 TEST_URL="$PROXY_URL/http-route-example"
 CURL_EXTRA_FLAGS=""
@ -50,7 +52,7 @@ while true; do
 done

 # Wait for http proxy and web server to start
-CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL $TEST_URL"
+CMD="$CURL $CURL_EXTRA_FLAGS -x $PROXY_URL $TEST_URL"
 while true; do
    RESPONSE=$($CMD 2> /dev/null)
    if [[ $? == 0 ]]; then
@ -90,13 +92,13 @@ Disallow: /deny
 EOM

 echo "[Test HTTP Request via Proxy]"
-CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL http://httpbin.org/robots.txt"
+CMD="$CURL $CURL_EXTRA_FLAGS -x $PROXY_URL http://httpbin.org/robots.txt"
 RESPONSE=$($CMD 2> /dev/null)
 verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
 VERIFIED1=$?

 echo "[Test HTTPS Request via Proxy]"
-CMD="curl -v $CURL_EXTRA_FLAGS -x $PROXY_URL https://httpbin.org/robots.txt"
+CMD="$CURL $CURL_EXTRA_FLAGS -x $PROXY_URL https://httpbin.org/robots.txt"
 RESPONSE=$($CMD 2> /dev/null)
 verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
 VERIFIED2=$?
@ -105,7 +107,7 @@ if $USE_HTTPS; then
    VERIFIED3=0
 else
    echo "[Test Internal Web Server via Proxy]"
-    curl -v \
+    $CURL \
        $CURL_EXTRA_FLAGS \
        -x $PROXY_URL \
        "$PROXY_URL"
@ -121,7 +123,7 @@ fi
 echo "[Test Download File Hash Verifies 1]"
 touch downloaded.hash
 echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc  -" > downloaded.hash
-curl -vL \
+$CURL -L \
    $CURL_EXTRA_FLAGS \
    -o downloaded.whl \
    -x $PROXY_URL \
@ -133,7 +135,7 @@ rm downloaded.whl downloaded.hash
 echo "[Test Download File Hash Verifies 2]"
 touch downloaded.hash
 echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8  -" > downloaded.hash
-curl -vL \
+$CURL -L \
    $CURL_EXTRA_FLAGS \
    -o downloaded.whl \
    -x $PROXY_URL \
--- a/tests/test_circular_imports.py
+++ b/tests/test_circular_imports.py
@ -85,7 +85,15 @@ def _discover_path_importables(
    'import_path',
    _find_all_importables(proxy),
 )
-def test_no_warnings(import_path: str) -> None:
+# Marked as disabled_ because:
+# 1. This test case was added when isort integration was problematic
+# 2. This test case never found a real circular import scenario
+# 3. This test case consumes 60% of test suite runtime
+# 4. Kept in the repo because we might still want to enable
+#    this in future, conditionally.  Example, we can run
+#    this only on a single OS and Python version combination
+#    instead of running it across entire matrix.
+def disabled_test_no_warnings(import_path: str) -> None:
    """Verify that exploding importables doesn't explode.

    This is seeking for any import errors including ones caused