From 25b7952b1d8b49e431ed7f9b53012c3fdf360b27 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 6 Nov 2021 17:25:32 +0530 Subject: [PATCH] Fix python 3.10 @ ubuntu pytest ignore (#685) * Use suggested fix in #683 to remove pytest ignore * Handle should not flush now which can block, instead let base tcp handler do its magic * test speed up, doc string, logging enhancements * Move macOS to the end of workflows * Fix mypy warnings --- .github/workflows/test-dashboard.yml | 2 +- .github/workflows/test-library.yml | 2 +- proxy/common/constants.py | 2 +- proxy/common/utils.py | 2 +- proxy/core/acceptor/threadless.py | 2 +- proxy/http/handler.py | 13 +++---------- proxy/plugin/cloudflare_dns.py | 4 +++- proxy/testing/test_case.py | 7 +++---- pytest.ini | 2 +- tests/test_circular_imports.py | 26 ++++++++++++++++++-------- tests/testing/test_test_case.py | 4 ++-- 11 files changed, 35 insertions(+), 31 deletions(-) diff --git a/.github/workflows/test-dashboard.yml b/.github/workflows/test-dashboard.yml index 7965993c..172e27a4 100644 --- a/.github/workflows/test-dashboard.yml +++ b/.github/workflows/test-dashboard.yml @@ -9,7 +9,7 @@ jobs: name: Node ${{ matrix.node }} @ ${{ matrix.os }} strategy: matrix: - os: [macOS, ubuntu, windows] + os: [ubuntu, windows, macOS] node: ['10.x', '11.x', '12.x'] max-parallel: 4 fail-fast: false diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index c026523e..07c4decf 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -21,7 +21,7 @@ jobs: - build strategy: matrix: - os: [macOS, Ubuntu] + os: [Ubuntu, macOS] python: ['3.6', '3.7', '3.8', '3.9', '3.10'] max-parallel: 4 fail-fast: false diff --git a/proxy/common/constants.py b/proxy/common/constants.py index ef413c4a..25c6ee54 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -80,7 +80,7 @@ DEFAULT_PORT = 8899 DEFAULT_SERVER_RECVBUF_SIZE = DEFAULT_BUFFER_SIZE DEFAULT_STATIC_SERVER_DIR = os.path.join(PROXY_PY_DIR, "public") DEFAULT_THREADLESS = False -DEFAULT_TIMEOUT = 10 +DEFAULT_TIMEOUT = 10.0 DEFAULT_VERSION = False DEFAULT_HTTP_PORT = 80 DEFAULT_HTTPS_PORT = 443 diff --git a/proxy/common/utils.py b/proxy/common/utils.py index 2fb802bf..a325226c 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -181,7 +181,7 @@ def wrap_socket( def new_socket_connection( - addr: Tuple[str, int], timeout: int = DEFAULT_TIMEOUT, source_address: Optional[Tuple[str, int]] = None, + addr: Tuple[str, int], timeout: float = DEFAULT_TIMEOUT, source_address: Optional[Tuple[str, int]] = None, ) -> socket.socket: conn = None try: diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py index 7d6e770e..2e25673a 100644 --- a/proxy/core/acceptor/threadless.py +++ b/proxy/core/acceptor/threadless.py @@ -195,7 +195,7 @@ class Threadless(multiprocessing.Process): try: self.selector = selectors.DefaultSelector() self.selector.register(self.client_queue, selectors.EVENT_READ) - self.loop = asyncio.get_event_loop() + self.loop = asyncio.get_event_loop_policy().get_event_loop() while not self.running.is_set(): self.run_once() except KeyboardInterrupt: diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 07715c95..1c2dd831 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -114,18 +114,11 @@ class HttpProtocolHandler(BaseTcpServerHandler): def shutdown(self) -> None: try: - # Flush pending buffer if any - # TODO: This can block as it register, select, unregister. - # _flush is only for use with threaded mode. In threadless - # mode BaseTcpServerHandler provides must_flush_before_shutdown - # implementation. + # Flush pending buffer in threaded mode only. + # For threadless mode, BaseTcpServerHandler implements + # the must_flush_before_shutdown logic automagically. if self.selector: self._flush() - else: - # Call flush for threadless mode assuming client is write ready - # This may fail or block. Using BaseTcpServerHandler will - # solve this issue. - self.client.flush() # Invoke plugin.on_client_connection_close for plugin in self.plugins.values(): diff --git a/proxy/plugin/cloudflare_dns.py b/proxy/plugin/cloudflare_dns.py index eaa45147..da188563 100644 --- a/proxy/plugin/cloudflare_dns.py +++ b/proxy/plugin/cloudflare_dns.py @@ -74,7 +74,9 @@ class CloudflareDnsResolverPlugin(HttpProxyBasePlugin): # instead of making a DNS query repeatedly for the same host. return answers[0]['data'], None except Exception as e: - logger.exception('Unable to resolve DNS-over-HTTPS', exc_info=e) + logger.info( + 'Unable to resolve DNS-over-HTTPS for host {0} : {1}'.format(host, str(e)), + ) return None, None def before_upstream_connection( diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py index 6416f82f..cb33d61a 100644 --- a/proxy/testing/test_case.py +++ b/proxy/testing/test_case.py @@ -52,16 +52,15 @@ class TestCase(unittest.TestCase): @staticmethod def wait_for_server( proxy_port: int, - wait_for_seconds: int = DEFAULT_TIMEOUT, + wait_for_seconds: float = DEFAULT_TIMEOUT, ) -> None: """Wait for proxy.py server to come up.""" start_time = time.time() while True: try: - conn = new_socket_connection( + new_socket_connection( ('localhost', proxy_port), - ) - conn.close() + ).close() break except ConnectionRefusedError: time.sleep(0.1) diff --git a/pytest.ini b/pytest.ini index 1557bcfd..720d3180 100644 --- a/pytest.ini +++ b/pytest.ini @@ -57,7 +57,7 @@ filterwarnings = error # FIXME: Address the deprecation warning in Python 3.10 and revert this ignore - ignore:There is no current event loop:DeprecationWarning:proxy.core.acceptor.threadless + # ignore:There is no current event loop:DeprecationWarning:proxy.core.acceptor.threadless junit_duration_report = call # xunit1 contains more metadata than xunit2 so it's better for CI UIs: diff --git a/tests/test_circular_imports.py b/tests/test_circular_imports.py index ba119860..4e6f8d6f 100644 --- a/tests/test_circular_imports.py +++ b/tests/test_circular_imports.py @@ -1,22 +1,32 @@ -"""Tests for circular imports in all local packages and modules. +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. -This ensures all internal packages can be imported right away without -any need to import some other module before doing so. + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. -This module is based on an idea that pytest uses for self-testing: -* https://github.com/sanitizers/octomachinery/blob/be18b54/tests/circular_imports_test.py -* https://github.com/pytest-dev/pytest/blob/d18c75b/testing/test_meta.py -* https://twitter.com/codewithanthony/status/1229445110510735361 + Tests for circular imports in all local packages and modules. + + This ensures all internal packages can be imported right away without + any need to import some other module before doing so. + + This module is based on an idea that pytest uses for self-testing: + * https://github.com/sanitizers/octomachinery/blob/be18b54/tests/circular_imports_test.py + * https://github.com/pytest-dev/pytest/blob/d18c75b/testing/test_meta.py + * https://twitter.com/codewithanthony/status/1229445110510735361 """ from itertools import chain from pathlib import Path from types import ModuleType from typing import Generator, List + import os import pkgutil import subprocess import sys - import pytest import proxy diff --git a/tests/testing/test_test_case.py b/tests/testing/test_test_case.py index 0d4ee869..daa28d0d 100644 --- a/tests/testing/test_test_case.py +++ b/tests/testing/test_test_case.py @@ -16,8 +16,8 @@ from proxy.common.utils import get_available_port class TestTestCase(unittest.TestCase): - def test_wait_for_server(self) -> None: + def test_wait_for_server_raises_timeout_error(self) -> None: with self.assertRaises(TimeoutError): proxy.TestCase.wait_for_server( - get_available_port(), wait_for_seconds=1, + get_available_port(), wait_for_seconds=0.1, )