From 2fa320d03f06995cc3a0822e8efccc5704e019d7 Mon Sep 17 00:00:00 2001 From: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com> Date: Sat, 13 Apr 2024 13:22:25 +0530 Subject: [PATCH] Python 3.11 support (#1384) * Changes for Python 3.11 support * Updated README.md for versioning info * Update `httpx==0.27.0` to avoid `cgi` deprecation warning from pytest on Python 3.11 * Make tests work for 3.11 * Declare support for 3.11 * Use 3.11-alpine for Docker images * Preserve pylint version for `python_version <= 3.10` * Preserve httpx version for <= 3.10 * `httpx` usage fix in tests for <=3.10 * Adjust pylint and pytest for >= 3.11 * Use 3.11.8, bad-option-value and httpx proxies fix * tox for 3.11 * Fix for `TOXENV: py` * -vv for pytest * Downgrade to `pytest-asyncio==0.21.1` * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove asyncio_mode=strict * try with `pytest-cov==4.1.0` for 3.11 * bump coverage for 3.11 * Try `3.11` in GitHub workflow which installs >3.11.8 unavailable via pyenv yet * Revert back to `-v` --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/test-library.yml | 15 ++++++++++++--- .pylintrc | 12 ++++++++---- Dockerfile | 2 +- Makefile | 2 +- README.md | 8 ++++---- benchmark/requirements.txt | 2 +- docs/conf.py | 2 ++ examples/web_scraper.py | 7 +++++++ proxy/common/types.py | 8 ++++---- proxy/core/base/tcp_server.py | 5 +++++ proxy/core/work/fd/fd.py | 15 +++++++++++++++ proxy/core/work/local.py | 5 +++++ proxy/core/work/remote.py | 5 +++++ proxy/http/server/plugin.py | 2 +- proxy/plugin/proxy_pool.py | 1 + proxy/testing/test_case.py | 1 + requirements-testing.txt | 29 ++++++++++++++++++++--------- setup.cfg | 1 + tests/http/proxy/test_http2.py | 18 +++++++++++++++--- tox.ini | 1 - 20 files changed, 109 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index a9fcad51..b01f93b0 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -446,8 +446,9 @@ jobs: # NOTE: The latest and the lowest supported Pythons are prioritized # NOTE: to improve the responsiveness. It's nice to see the most # NOTE: important results first. - - '3.10' + - '3.11' - 3.6 + - '3.10' - 3.9 - 3.8 - 3.7 @@ -463,7 +464,7 @@ jobs: env: PY_COLORS: 1 TOX_PARALLEL_NO_SPINNER: 1 - TOXENV: python + TOXENV: py steps: - name: Switch to using Python v${{ matrix.python }} @@ -500,7 +501,15 @@ jobs: steps.calc-cache-key-py.outputs.py-hash-key }}- ${{ runner.os }}-pip- - - name: Install tox + - name: Install tox for >= 3.11 + if: matrix.python == '3.11' + run: >- + python -m + pip install + --user + tox==4.14.2 + - name: Install tox for < 3.11 + if: matrix.python != '3.11' run: >- python -m pip install diff --git a/.pylintrc b/.pylintrc index 25434cea..c705e362 100644 --- a/.pylintrc +++ b/.pylintrc @@ -131,6 +131,10 @@ disable=raw-checker-failed, useless-return, useless-super-delegation, wrong-import-order, + # Required because to support 3.11 + # we added unnecessary-dunder-call which is not supported for <=3.11 + # see https://github.com/abhinavsingh/proxy.py/actions/runs/8671404475/job/23780537848?pr=1384 + bad-option-value # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -419,7 +423,7 @@ contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=os,io # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). @@ -446,7 +450,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. -ignored-modules= +ignored-modules=abc # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. @@ -605,5 +609,5 @@ preferred-modules= # Exceptions that will emit a warning when being caught. Defaults to # "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +overgeneral-exceptions=builtins.BaseException, + builtins.Exception diff --git a/Dockerfile b/Dockerfile index dcfb3611..5277069a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-alpine as base +FROM python:3.11-alpine as base LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ diff --git a/Makefile b/Makefile index f111b2f9..e4255176 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ lib-mypy: tox -e lint -- mypy --all-files lib-pytest: - $(PYTHON) -m tox -e python -- -v + $(PYTHON) -m tox -e py -- -v lib-test: lib-clean lib-check lib-lint lib-pytest diff --git a/README.md b/README.md index ab37c7e8..0ea85446 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ [![iOS, iOS Simulator](https://img.shields.io/static/v1?label=tested%20with&message=iOS%20%F0%9F%93%B1%20%7C%20iOS%20Simulator%20%F0%9F%93%B1&color=darkgreen&style=for-the-badge)](https://abhinavsingh.com/proxy-py-a-lightweight-single-file-http-proxy-server-in-python/) [![pypi version](https://img.shields.io/pypi/v/proxy.py?style=flat-square)](https://pypi.org/project/proxy.py/) -[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10&color=blue&style=flat-square)](https://www.python.org/) +[![Python 3.x](https://img.shields.io/static/v1?label=Python&message=3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9%20%7C%203.10%20%7C%203.11&color=blue&style=flat-square)](https://www.python.org/) [![Checked with mypy](https://img.shields.io/static/v1?label=MyPy&message=checked&color=blue&style=flat-square)](http://mypy-lang.org/) [![doc](https://img.shields.io/readthedocs/proxypy/latest?style=flat-square&color=darkgreen)](https://proxypy.readthedocs.io/) @@ -2366,7 +2366,7 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT] [--filtered-client-ips FILTERED_CLIENT_IPS] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] -proxy.py v2.4.4rc5.dev36+g6c9d0315.d20240411 +proxy.py v2.4.4rc6.dev11+gac1f05d7.d20240413 options: -h, --help show this help message and exit @@ -2489,8 +2489,8 @@ options: Default: None. Signing certificate to use for signing dynamically generated HTTPS certificates. If used, must also pass --ca-key-file and --ca-signing-key-file - --ca-file CA_FILE Default: /Users/abhinavsingh/Dev/proxy.py/.venv/lib/py - thon3.10/site-packages/certifi/cacert.pem. Provide + --ca-file CA_FILE Default: /Users/abhinavsingh/Dev/proxy.py/.venv3118/li + b/python3.11/site-packages/certifi/cacert.pem. Provide path to custom CA bundle for peer certificate verification --ca-signing-key-file CA_SIGNING_KEY_FILE diff --git a/benchmark/requirements.txt b/benchmark/requirements.txt index c9fbac1f..1b0d1fe2 100644 --- a/benchmark/requirements.txt +++ b/benchmark/requirements.txt @@ -1,4 +1,4 @@ -aiohttp==3.8.1 +aiohttp==3.8.2 # Blacksheep depends upon essentials_openapi which is pinned to pyyaml==5.4.1 # and pyyaml>5.3.1 is broken for cython 3 # See https://github.com/yaml/pyyaml/issues/724#issuecomment-1638587228 diff --git a/docs/conf.py b/docs/conf.py index 40573d5e..0388bd31 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -321,6 +321,8 @@ nitpick_ignore = [ (_py_class_role, 'HostPort'), (_py_class_role, 'TcpOrTlsSocket'), (_py_class_role, 're.Pattern'), + (_py_class_role, 'proxy.core.base.tcp_server.T'), + (_py_class_role, 'proxy.common.types.RePattern'), (_py_obj_role, 'proxy.core.work.threadless.T'), (_py_obj_role, 'proxy.core.work.work.T'), (_py_obj_role, 'proxy.core.base.tcp_server.T'), diff --git a/examples/web_scraper.py b/examples/web_scraper.py index daf31d61..618b7bf7 100644 --- a/examples/web_scraper.py +++ b/examples/web_scraper.py @@ -9,6 +9,8 @@ :license: BSD, see LICENSE for more details. """ import time +from abc import abstractmethod +from typing import Any from proxy import Proxy from proxy.core.work import Work @@ -52,6 +54,11 @@ class WebScraper(Work[TcpClientConnection]): Return True to shutdown work.""" return False + @staticmethod + @abstractmethod + def create(*args: Any) -> TcpClientConnection: + raise NotImplementedError() + if __name__ == '__main__': with Proxy( diff --git a/proxy/common/types.py b/proxy/common/types.py index 984cc3bd..78f3bd50 100644 --- a/proxy/common/types.py +++ b/proxy/common/types.py @@ -14,7 +14,7 @@ import sys import queue import socket import ipaddress -from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union, TypeVar if TYPE_CHECKING: # pragma: no cover @@ -34,8 +34,8 @@ TcpOrTlsSocket = Union[ssl.SSLSocket, socket.socket] HostPort = Tuple[str, int] if sys.version_info.minor == 6: - RePattern = Any + RePattern = TypeVar('RePattern', bound=Any) elif sys.version_info.minor in (7, 8): - RePattern = re.Pattern # type: ignore + RePattern = TypeVar('RePattern', bound=re.Pattern) # type: ignore else: - RePattern = re.Pattern[Any] # type: ignore + RePattern = TypeVar('RePattern', bound=re.Pattern[Any]) # type: ignore diff --git a/proxy/core/base/tcp_server.py b/proxy/core/base/tcp_server.py index 842e255a..4d32949d 100644 --- a/proxy/core/base/tcp_server.py +++ b/proxy/core/base/tcp_server.py @@ -240,3 +240,8 @@ class BaseTcpServerHandler(Work[T]): conn = wrap_socket(conn, self.flags.keyfile, self.flags.certfile) self.work._conn = conn return conn + + @staticmethod + @abstractmethod + def create(*args: Any) -> T: + raise NotImplementedError() diff --git a/proxy/core/work/fd/fd.py b/proxy/core/work/fd/fd.py index 73001912..a1302476 100644 --- a/proxy/core/work/fd/fd.py +++ b/proxy/core/work/fd/fd.py @@ -9,7 +9,9 @@ :license: BSD, see LICENSE for more details. """ import socket +import asyncio import logging +from abc import abstractmethod from typing import Any, TypeVar, Optional from ...event import eventNames @@ -47,3 +49,16 @@ class ThreadlessFdExecutor(Threadless[T]): exc_info=e, ) self._cleanup(fileno) + + @property + @abstractmethod + def loop(self) -> Optional[asyncio.AbstractEventLoop]: + raise NotImplementedError() + + @abstractmethod + def receive_from_work_queue(self) -> bool: + raise NotImplementedError() + + @abstractmethod + def work_queue_fileno(self) -> Optional[int]: + raise NotImplementedError() diff --git a/proxy/core/work/local.py b/proxy/core/work/local.py index 0745e817..6b3ae993 100644 --- a/proxy/core/work/local.py +++ b/proxy/core/work/local.py @@ -11,6 +11,7 @@ import queue import asyncio import contextlib +from abc import abstractmethod from typing import Any, Optional from .threadless import Threadless @@ -40,3 +41,7 @@ class BaseLocalExecutor(Threadless[NonBlockingQueue]): return True self.work(work) return False + + @abstractmethod + def work(self, *args: Any) -> None: + raise NotImplementedError() diff --git a/proxy/core/work/remote.py b/proxy/core/work/remote.py index afac2ebe..57344385 100644 --- a/proxy/core/work/remote.py +++ b/proxy/core/work/remote.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for more details. """ import asyncio +from abc import abstractmethod from typing import Any, Optional from multiprocessing import connection @@ -37,3 +38,7 @@ class BaseRemoteExecutor(Threadless[connection.Connection]): def receive_from_work_queue(self) -> bool: self.work(self.work_queue.recv()) return False + + @abstractmethod + def work(self, *args: Any) -> None: + raise NotImplementedError() diff --git a/proxy/http/server/plugin.py b/proxy/http/server/plugin.py index 434fba24..48cc5eb2 100644 --- a/proxy/http/server/plugin.py +++ b/proxy/http/server/plugin.py @@ -169,7 +169,7 @@ class ReverseProxyBasePlugin(ABC): def handle_route(self, request: HttpParser, pattern: RePattern) -> Url: """Implement this method if you have configured dynamic routes.""" - pass + raise NotImplementedError() def regexes(self) -> List[str]: """Helper method to return list of route regular expressions.""" diff --git a/proxy/plugin/proxy_pool.py b/proxy/plugin/proxy_pool.py index 6e040347..c244aded 100644 --- a/proxy/plugin/proxy_pool.py +++ b/proxy/plugin/proxy_pool.py @@ -186,6 +186,7 @@ class ProxyPoolPlugin(TcpUpstreamConnectionHandler, HttpProxyBasePlugin): """Will never be called since we didn't establish an upstream connection.""" if not self.upstream: return chunk + # pylint: disable=broad-exception-raised raise Exception("This should have never been called") def on_upstream_connection_close(self) -> None: diff --git a/proxy/testing/test_case.py b/proxy/testing/test_case.py index afd13fe1..4c2740e0 100644 --- a/proxy/testing/test_case.py +++ b/proxy/testing/test_case.py @@ -42,6 +42,7 @@ class TestCase(unittest.TestCase): cls.PROXY.flags.plugins[b'HttpProxyBasePlugin'].append( CacheResponsesPlugin, ) + # pylint: disable=unnecessary-dunder-call cls.PROXY.__enter__() assert cls.PROXY.acceptors cls.wait_for_server(cls.PROXY.flags.port) diff --git a/requirements-testing.txt b/requirements-testing.txt index 13eba0f9..61153d0d 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -1,21 +1,32 @@ wheel==0.37.1 python-coveralls==2.9.3 -coverage==6.2 +coverage==6.2; python_version < '3.11' +coverage==7.4.4; python_version >= '3.11' flake8==4.0.1 -pytest==7.0.1 -pytest-cov==3.0.0 -pytest-xdist == 2.5.0 -pytest-mock==3.6.1 -pytest-asyncio==0.16.0 +# pytest for Python<3.11 +pytest==7.0.1; python_version < '3.11' +pytest-cov==3.0.0; python_version < '3.11' +pytest-xdist==2.5.0; python_version < '3.11' +pytest-mock==3.6.1; python_version < '3.11' +pytest-asyncio==0.16.0; python_version < '3.11' +# pytest for Python>=3.11 +pytest==8.1.1; python_version >= '3.11' +pytest-cov==5.0.0; python_version >= '3.11' +pytest-xdist==3.5.0; python_version >= '3.11' +pytest-mock==3.14.0; python_version >= '3.11' +pytest-asyncio==0.21.1; python_version >= '3.11' autopep8==1.6.0 mypy==0.971 py-spy==0.3.12 -tox==3.28.0 +tox==3.28.0; python_version < '3.11' +tox==4.14.2; python_version >= '3.11' mccabe==0.6.1 -pylint==2.13.7 +pylint==2.13.7; python_version < '3.11' +pylint==3.1.0; python_version >= '3.11' rope==1.1.1 # Required by test_http2.py -httpx==0.22.0 +httpx==0.22.0; python_version < '3.11' +httpx==0.27.0; python_version >= '3.11' h2==4.1.0 hpack==4.0.0 hyperframe==6.0.1 diff --git a/setup.cfg b/setup.cfg index 124681dd..fb14a651 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,6 +65,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Internet Topic :: Internet :: Proxy Servers diff --git a/tests/http/proxy/test_http2.py b/tests/http/proxy/test_http2.py index a8f1eeed..2ef95eaa 100644 --- a/tests/http/proxy/test_http2.py +++ b/tests/http/proxy/test_http2.py @@ -8,6 +8,9 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import sys +from typing import Any, Dict + import httpx from proxy import TestCase @@ -17,14 +20,23 @@ class TestHttp2WithProxy(TestCase): def test_http2_via_proxy(self) -> None: assert self.PROXY + proxy_url = 'http://localhost:%d' % self.PROXY.flags.port + proxies: Dict[str, Any] = ( + { + 'proxies': { + 'all://': proxy_url, + }, + } + # For Python>=3.11, proxies keyword is deprecated by httpx + if sys.version_info < (3, 11, 0) + else {'proxy': proxy_url} + ) response = httpx.get( 'https://www.google.com', headers={'accept': 'application/json'}, verify=httpx.create_ssl_context(http2=True), timeout=httpx.Timeout(timeout=5.0), - proxies={ - 'all://': 'http://localhost:%d' % self.PROXY.flags.port, - }, + **proxies, ) self.assertEqual(response.status_code, 200) diff --git a/tox.ini b/tox.ini index 2a777f9c..5e7d706a 100644 --- a/tox.ini +++ b/tox.ini @@ -262,7 +262,6 @@ deps = pre-commit pylint >= 2.5.3 pylint-pytest < 1.1.0 - pytest-mock == 3.6.1 -r docs/requirements.in -r requirements-tunnel.txt -r requirements-testing.txt