tests: enabling mypy on untyped defs (#2204)

This commit is contained in:
Henry Schreiner 2022-02-23 17:10:53 -05:00 committed by GitHub
parent 66856a7427
commit 8385df36a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 123 additions and 99 deletions

View File

@ -1,4 +1,4 @@
exclude: '^.*patches|.*\.cgi$'
exclude: (^.*patches|.*\.cgi$|^packages/micropip/src/micropip/externals|^src/py/lib/pystone.py$)
default_language_version:
python: "3.9"
repos:
@ -14,14 +14,12 @@ repos:
rev: "5.10.1"
hooks:
- id: isort
exclude: ^packages/micropip/src/micropip/externals
- repo: https://github.com/asottile/pyupgrade
rev: "v2.31.0"
hooks:
- id: pyupgrade
args: ["--py39-plus"]
exclude: ^packages/micropip/src/micropip/externals
- repo: https://github.com/hadialqattan/pycln
rev: "v1.1.0"

View File

@ -113,7 +113,7 @@ class SeleniumWrapper:
self.server_hostname = server_hostname
self.base_url = f"http://{self.server_hostname}:{self.server_port}"
self.server_log = server_log
self.driver = self.get_driver()
self.driver = self.get_driver() # type: ignore[attr-defined]
self.set_script_timeout(script_timeout)
self.script_timeout = script_timeout
self.prepare_driver()
@ -562,6 +562,7 @@ def selenium_common(request, web_server_main, load_pyodide=True):
"""
server_hostname, server_port, server_log = web_server_main
cls: type[SeleniumWrapper]
if request.param == "firefox":
cls = FirefoxWrapper
elif request.param == "chrome":
@ -569,7 +570,7 @@ def selenium_common(request, web_server_main, load_pyodide=True):
elif request.param == "node":
cls = NodeWrapper
else:
assert False
raise AssertionError(f"Unknown browser: {request.param}")
selenium = cls(
server_port=server_port,
server_hostname=server_hostname,
@ -687,7 +688,7 @@ def spawn_web_server(build_dir=None):
tmp_dir = tempfile.mkdtemp()
log_path = pathlib.Path(tmp_dir) / "http-server.log"
q = multiprocessing.Queue()
q: multiprocessing.Queue[str] = multiprocessing.Queue()
p = multiprocessing.Process(target=run_web_server, args=(q, log_path, build_dir))
try:
@ -740,8 +741,8 @@ def run_web_server(q, log_filepath, build_dir):
with socketserver.TCPServer(("", 0), Handler) as httpd:
host, port = httpd.server_address
print(f"Starting webserver at http://{host}:{port}")
httpd.server_name = "test-server"
httpd.server_port = port
httpd.server_name = "test-server" # type: ignore[attr-defined]
httpd.server_port = port # type: ignore[attr-defined]
q.put(port)
def service_actions():
@ -752,7 +753,7 @@ def run_web_server(q, log_filepath, build_dir):
except queue.Empty:
pass
httpd.service_actions = service_actions
httpd.service_actions = service_actions # type: ignore[assignment]
httpd.serve_forever()

View File

@ -214,9 +214,9 @@ def flatten_suffix_tree(tree):
suffix tree, but the suffix tree is inconveniently shaped. So we flatten
it...
"""
result = {}
path = []
iters = []
result: dict[tuple[str, ...], Any] = {}
path: list[str] = []
iters: list[Any] = []
cur_iter = iter(tree.items())
while True:
try:
@ -268,11 +268,11 @@ class PyodideAnalyzer:
self.doclets = flatten_suffix_tree(self._objects_by_path._tree)
def get_val():
return OrderedDict([["attribute", []], ["function", []], ["class", []]])
return OrderedDict([("attribute", []), ("function", []), ("class", [])])
modules = ["globalThis", "pyodide", "PyProxy"]
self.js_docs = {key: get_val() for key in modules}
items = {key: [] for key in modules}
items = {key: list[Any]() for key in modules}
for (key, doclet) in self.doclets.items():
if getattr(doclet.value, "is_private", False):
continue

View File

@ -61,7 +61,7 @@ def get_packages_summary_directive(app):
)
def format_packages_table(
self, packages: dict[str, Any], columns: tuple[str]
self, packages: dict[str, Any], columns: tuple[str, ...]
) -> list[Any]:
table_spec = addnodes.tabular_col_spec()
table_spec["spec"] = r"\X{1}{2}\X{1}{2}"

View File

@ -3,6 +3,7 @@ import io
import sys
import zipfile
from pathlib import Path
from typing import Any
import pytest
@ -190,7 +191,10 @@ def test_add_requirement(web_server_tst_data):
base_url = f"http://{server_hostname}:{server_port}/"
url = base_url + "snowballstemmer-2.0.0-py2.py3-none-any.whl"
transaction = {"wheels": [], "locked": {}}
transaction: dict[str, Any] = {
"wheels": [],
"locked": {},
}
asyncio.get_event_loop().run_until_complete(
_micropip.PACKAGE_MANAGER.add_requirement(url, {}, transaction)
)
@ -258,7 +262,7 @@ def test_install_non_pure_python_wheel():
msg = "not a pure Python 3 wheel"
with pytest.raises(ValueError, match=msg):
url = "http://scikit_learn-0.22.2.post1-cp35-cp35m-macosx_10_9_intel.whl"
transaction = {"wheels": [], "locked": {}}
transaction = {"wheels": list[Any](), "locked": dict[str, Any]()}
asyncio.get_event_loop().run_until_complete(
_micropip.PACKAGE_MANAGER.add_requirement(url, {}, transaction)
)

View File

@ -3,6 +3,7 @@ from pyodide_build.testing import run_in_pyodide
@run_in_pyodide(packages=["wrapt"])
def test_wrapt():
import inspect
import unittest
import wrapt

View File

@ -371,7 +371,9 @@ def build_from_graph(pkg_map: dict[str, BasePackage], outputdir: Path, args) ->
built_queue: Queue = Queue()
thread_lock = Lock()
queue_idx = 1
package_set = {}
package_set: dict[
str, None
] = {} # using dict keys for insertion order preservation
def builder(n):
nonlocal queue_idx

View File

@ -18,7 +18,7 @@ from contextlib import contextmanager
from datetime import datetime
from pathlib import Path
from textwrap import dedent
from typing import Any
from typing import Any, Optional
from urllib import request
from . import pywasmcross
@ -55,6 +55,9 @@ class BashRunnerWithSharedEnvironment:
directly to adjust the environment, or read to get variables.
"""
_fd_read: Optional[int]
_fd_write: Optional[int]
def __init__(self, env=None):
if env is None:
env = dict(os.environ)
@ -81,7 +84,7 @@ class BashRunnerWithSharedEnvironment:
def close(self):
"""Free the file descriptors."""
if self._fd_read:
if self._fd_read and self._fd_write:
os.close(self._fd_read)
os.close(self._fd_write)
self._fd_read = None

View File

@ -1,5 +1,5 @@
from pathlib import Path
from typing import Any, Optional
from typing import Any, Optional, Union
# TODO: support more complex types for validation
@ -46,7 +46,9 @@ PACKAGE_CONFIG_SPEC: dict[str, dict[str, Any]] = {
def check_package_config(
config: dict[str, Any], raise_errors: bool = True, file_path: Optional[Path] = None
config: dict[str, Any],
raise_errors: bool = True,
file_path: Optional[Union[Path, str]] = None,
) -> list[str]:
"""Check the validity of a loaded meta.yaml file
@ -118,7 +120,7 @@ def check_package_config(
return errors_msg
def parse_package_config(path: Path, check: bool = True) -> dict[str, Any]:
def parse_package_config(path: Union[Path, str], check: bool = True) -> dict[str, Any]:
"""Load a meta.yaml file
Parameters

View File

@ -1,7 +1,7 @@
import contextlib
import inspect
from base64 import b64encode
from typing import Callable, Optional, Union
from typing import Callable, Optional
import pytest
@ -33,7 +33,7 @@ def run_in_pyodide(
module_scope: bool = False,
packages: list[str] = [],
xfail_browsers: dict[str, str] = {},
driver_timeout: Optional[Union[str, int]] = None,
driver_timeout: Optional[float] = None,
) -> Callable:
"""
This decorator can be called in two ways --- with arguments and without
@ -49,7 +49,7 @@ def run_in_pyodide(
Whether to use a standalone selenium instance to run the test or not
packages : List[str]
List of packages to load before running the test
driver_timeout : Optional[Union[str, int]]
driver_timeout : Optional[float]
selenium driver timeout (in seconds)
"""
@ -127,7 +127,7 @@ def run_in_pyodide(
@contextlib.contextmanager
def set_webdriver_script_timeout(selenium, script_timeout: Optional[Union[int, float]]):
def set_webdriver_script_timeout(selenium, script_timeout: Optional[float]):
"""Set selenium script timeout
Parameters
@ -145,7 +145,7 @@ def set_webdriver_script_timeout(selenium, script_timeout: Optional[Union[int, f
selenium.set_script_timeout(selenium.script_timeout)
def parse_driver_timeout(request) -> Optional[Union[int, float]]:
def parse_driver_timeout(request) -> Optional[float]:
"""Parse driver timeout value from pytest request object"""
mark = request.node.get_closest_marker("driver_timeout")
if mark is None:

View File

@ -62,7 +62,7 @@ def test_build_dependencies(n_jobs, monkeypatch):
pkg_map = buildall.generate_dependency_graph(PACKAGES_DIR, {"lxml", "micropip"})
Args = namedtuple("args", ["n_jobs", "force_rebuild"])
Args = namedtuple("Args", ["n_jobs", "force_rebuild"])
buildall.build_from_graph(
pkg_map, Path("."), Args(n_jobs=n_jobs, force_rebuild=True)
)
@ -105,7 +105,7 @@ def test_build_all_dependencies(n_jobs, monkeypatch):
pkg_map = buildall.generate_dependency_graph(PACKAGES_DIR, packages={"*"})
Args = namedtuple("args", ["n_jobs", "force_rebuild"])
Args = namedtuple("Args", ["n_jobs", "force_rebuild"])
buildall.build_from_graph(
pkg_map, Path("."), Args(n_jobs=n_jobs, force_rebuild=False)
)
@ -124,7 +124,7 @@ def test_build_error(n_jobs, monkeypatch):
pkg_map = buildall.generate_dependency_graph(PACKAGES_DIR, {"lxml"})
with pytest.raises(ValueError, match="Failed build"):
Args = namedtuple("args", ["n_jobs", "force_rebuild"])
Args = namedtuple("Args", ["n_jobs", "force_rebuild"])
buildall.build_from_graph(
pkg_map, Path("."), Args(n_jobs=n_jobs, force_rebuild=True)
)

View File

@ -1,5 +1,6 @@
import os
from pathlib import Path
from typing import Union
import pytest
import yaml
@ -45,7 +46,7 @@ def test_mkpkg_update(tmpdir, monkeypatch, old_dist_type, new_dist_type):
old_ext = ".tar.gz" if old_dist_type == "sdist" else ".whl"
old_url = "https://<some>/idna-2.0" + old_ext
db_init = {
db_init: dict[str, dict[str, Union[str, list[str]]]] = {
"package": {"name": "idna", "version": "2.0"},
"source": {
"sha256": "b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
@ -61,12 +62,12 @@ def test_mkpkg_update(tmpdir, monkeypatch, old_dist_type, new_dist_type):
source_fmt = new_dist_type
if new_dist_type == "same":
source_fmt = None
pyodide_build.mkpkg.update_package("idna", None, source_fmt)
pyodide_build.mkpkg.update_package("idna", False, source_fmt)
db = parse_package_config(meta_path)
assert list(db.keys()) == list(db_init.keys())
assert parse_version(db["package"]["version"]) > parse_version(
db_init["package"]["version"]
db_init["package"]["version"] # type: ignore[arg-type]
)
if new_dist_type == "wheel":
assert db["source"]["url"].endswith(".whl")

View File

@ -2,6 +2,7 @@
warn_unused_configs = true
python_version = "3.9"
mypy_path = "src/py"
check_untyped_defs = true
[[tool.mypy.overrides]]
module = [
@ -14,6 +15,12 @@ module = [
]
ignore_missing_imports = true
[[tool.mypy.overrides]]
module = [
"micropip.externals.*",
]
check_untyped_defs = false
[tool.pycln]
all = true

View File

@ -184,7 +184,7 @@ class Console:
globals : ``dict``
The global namespace in which to evaluate the code. Defaults to a new empty dictionary.
stdin_callback : ``Callable[[str], None]``
stdin_callback : ``Callable[[], str]``
Function to call at each read from ``sys.stdin``. Defaults to ``None``.
stdout_callback : ``Callable[[str], None]``
@ -205,7 +205,7 @@ class Console:
globals : ``Dict[str, Any]``
The namespace used as the global
stdin_callback : ``Callback[[str], None]``
stdin_callback : ``Callback[[], str]``
Function to call at each read from ``sys.stdin``.
stdout_callback : ``Callback[[str], None]``
@ -225,7 +225,7 @@ class Console:
self,
globals: Optional[dict] = None,
*,
stdin_callback: Optional[Callable[[str], None]] = None,
stdin_callback: Optional[Callable[[], str]] = None,
stdout_callback: Optional[Callable[[str], None]] = None,
stderr_callback: Optional[Callable[[str], None]] = None,
persistent_stream_redirection: bool = False,
@ -259,6 +259,7 @@ class Console:
if self._stream_generator:
return
self._stream_generator = self._stdstreams_redirections_inner()
assert self._stream_generator is not None
next(self._stream_generator) # trigger stream redirection
# streams will be reverted to normal when self._stream_generator is destroyed.
@ -280,7 +281,7 @@ class Console:
if self._streams_redirected:
yield
return
redirects = []
redirects: list[Any] = []
if self.stdin_callback:
stdin_name = getattr(sys.stdin, "name", "<stdin>")
stdin_stream = _ReadStream(self.stdin_callback, name=stdin_name)
@ -456,7 +457,7 @@ def repr_shorten(
if necessary.
If it is longer than ``limit`` then return the firsts ``split``
characters and the last ``split`` characters seperated by '...'.
characters and the last ``split`` characters separated by '...'.
Default value for ``split`` is `limit // 2`.
"""
if split is None:

View File

@ -42,7 +42,7 @@ def get_cmeth_docstring(func):
sig = signature(func)
# remove param and return annotations and
for param in sig.parameters.values():
param._annotation = _empty
sig._return_annotation = _empty
param._annotation = _empty # type: ignore[attr-defined]
sig._return_annotation = _empty # type: ignore[attr-defined]
return func.__name__ + str(sig) + "\n--\n\n" + dedent_docstring(func.__doc__)

View File

@ -216,14 +216,14 @@ class WebLoop(asyncio.AbstractEventLoop):
self._check_closed()
if self._task_factory is None:
task = asyncio.tasks.Task(coro, loop=self, name=name)
if task._source_traceback:
if task._source_traceback: # type: ignore[attr-defined]
# Added comment:
# this only happens if get_debug() returns True.
# In that case, remove create_task from _source_traceback.
del task._source_traceback[-1]
del task._source_traceback[-1] # type: ignore[attr-defined]
else:
task = self._task_factory(self, coro)
asyncio.tasks._set_task_name(task, name)
asyncio.tasks._set_task_name(task, name) # type: ignore[attr-defined]
return task
@ -389,7 +389,7 @@ class WebLoopPolicy(asyncio.DefaultEventLoopPolicy): # type: ignore
def new_event_loop(self):
"""Create a new event loop"""
self._default_loop = WebLoop()
self._default_loop = WebLoop() # type: ignore[abstract]
return self._default_loop
def set_event_loop(self, loop: asyncio.AbstractEventLoop):

View File

@ -23,15 +23,15 @@ def test_command_compiler():
)
assert isinstance(c("1<>2", "<input>", "single"), CodeRunner)
c = _CommandCompiler()
assert c("def test():\n 1", "<input>", "single") is None
assert isinstance(c("def test():\n 1\n", "<input>", "single"), CodeRunner)
c2 = _CommandCompiler()
assert c2("def test():\n 1", "<input>", "single") is None
assert isinstance(c2("def test():\n 1\n", "<input>", "single"), CodeRunner)
with pytest.raises(SyntaxError, match="invalid syntax"):
c("1<>2", "<input>", "single")
c2("1<>2", "<input>", "single")
assert isinstance(
c("from __future__ import barry_as_FLUFL", "<input>", "single"), CodeRunner
c2("from __future__ import barry_as_FLUFL", "<input>", "single"), CodeRunner
)
assert isinstance(c("1<>2", "<input>", "single"), CodeRunner)
assert isinstance(c2("1<>2", "<input>", "single"), CodeRunner)
def test_write_stream():
@ -142,7 +142,7 @@ def test_interactive_console():
def test_top_level_await():
from asyncio import Queue, sleep
q = Queue()
q: Queue[int] = Queue()
shell = Console(locals())
fut = shell.push("await q.get()")
@ -229,14 +229,14 @@ def test_nonpersistent_redirection(safe_sys_redirections):
my_stdout = ""
my_stderr = ""
def stdin_callback():
pass
def stdin_callback() -> str:
return ""
def stdout_callback(string):
def stdout_callback(string: str) -> None:
nonlocal my_stdout
my_stdout += string
def stderr_callback(string):
def stderr_callback(string: str) -> None:
nonlocal my_stderr
my_stderr += string
@ -425,7 +425,7 @@ def test_console_html(console_html_fixture):
>>> Test()
[[;;;terminal-error]Traceback (most recent call last):
File \"/lib/python3.9/site-packages/_pyodide/console.py\", line 464, in repr_shorten
File \"/lib/python3.9/site-packages/_pyodide/console.py\", line 465, in repr_shorten
text = repr(value)
File \"<console>\", line 3, in __repr__
TypeError: hi]

View File

@ -270,8 +270,10 @@ def test_test_unvendoring(selenium_standalone):
def test_install_archive(selenium):
build_dir = Path(__file__).parents[2] / "build"
test_dir = Path(__file__).parent
# TODO: first arguement actually works as a path due to implementation,
# maybe it can be proposed to typeshed?
shutil.make_archive(
test_dir / "test_pkg", "gztar", root_dir=test_dir, base_dir="test_pkg"
str(test_dir / "test_pkg"), "gztar", root_dir=test_dir, base_dir="test_pkg"
)
build_test_pkg = build_dir / "test_pkg.tar.gz"
if not build_test_pkg.exists():
@ -335,9 +337,9 @@ def test_get_dynlibs():
t.flush()
assert sorted(get_dynlibs(t, Path("/p"))) == so_files
with NamedTemporaryFile(suffix=".zip") as t:
x = ZipFile(t, mode="w")
x2 = ZipFile(t, mode="w")
for file in files:
x.writestr(file, "")
x.close()
x2.writestr(file, "")
x2.close()
t.flush()
assert sorted(get_dynlibs(t, Path("/p"))) == so_files

View File

@ -1,5 +1,6 @@
import re
from textwrap import dedent
from typing import Any
import pytest
@ -57,8 +58,8 @@ def test_code_runner():
# Ast transform
import ast
l = cr.ast.body[0].value.left
cr.ast.body[0].value.left = ast.BinOp(
l = cr.ast.body[0].value.left # type: ignore[attr-defined]
cr.ast.body[0].value.left = ast.BinOp( # type: ignore[attr-defined]
left=l, op=ast.Mult(), right=ast.Constant(value=2)
)
assert cr.compile().run({"x": 3}) == 13
@ -69,7 +70,7 @@ def test_code_runner():
def test_code_runner_mode():
from codeop import PyCF_DONT_IMPLY_DEDENT
from codeop import PyCF_DONT_IMPLY_DEDENT # type: ignore[attr-defined]
assert CodeRunner("1+1\n1+1", mode="exec").compile().run() == 2
with pytest.raises(SyntaxError, match="invalid syntax"):
@ -86,7 +87,7 @@ def test_code_runner_mode():
def test_eval_code():
ns = {}
ns: dict[str, Any] = {}
assert (
eval_code(
"""
@ -150,12 +151,12 @@ def test_eval_code():
def test_eval_code_locals():
globals = {}
globals: dict[str, Any] = {}
eval_code("x=2", globals, {})
with pytest.raises(NameError):
eval_code("x", globals, {})
locals = {}
locals: dict[str, Any] = {}
eval_code("import sys; sys.getrecursionlimit()", globals, locals)
with pytest.raises(NameError):
eval_code("sys.getrecursionlimit()", globals, {})
@ -958,17 +959,16 @@ def test_custom_stdin_stdout(selenium_standalone_noload):
globalThis.pyodide = pyodide;
"""
)
outstrings = sum((s.removesuffix("\n").split("\n") for s in strings), [])
outstrings: list[str] = sum((s.removesuffix("\n").split("\n") for s in strings), [])
print(outstrings)
assert (
selenium.run_js(
"""
f"""
return pyodide.runPython(`
[input() for x in range(%s)]
[input() for x in range({len(outstrings)})]
# ... test more stuff
`).toJs();
"""
% len(outstrings)
)
== outstrings
)

View File

@ -35,7 +35,7 @@ def dummy_decorator(*args, **kwargs):
"chrome": "nlopt set_min_objective triggers a fatal runtime error in chrome 89 see #1493",
},
)
def some_func():
def some_func(f):
import nlopt
import numpy as np
@ -49,7 +49,7 @@ def test_run_in_pyodide_multiline_decorator():
_run_in_pyodide_get_source(some_func).strip()
== dedent(
"""
def some_func():
def some_func(f):
import nlopt
import numpy as np

View File

@ -1,4 +1,6 @@
# See also test_pyproxy, test_jsproxy, and test_python.
from typing import Any
import pytest
from hypothesis import assume, given, settings, strategies
from hypothesis.strategies import from_type, text
@ -667,7 +669,7 @@ def assert_py_to_js_to_py(selenium, name):
@run_in_pyodide
def test_recursive_list_to_js():
x = []
x: Any = []
x.append(x)
from pyodide import to_js
@ -676,7 +678,7 @@ def test_recursive_list_to_js():
@run_in_pyodide
def test_recursive_dict_to_js():
x = {}
x: Any = {}
x[0] = x
from pyodide import to_js
@ -1236,25 +1238,25 @@ def test_buffer_format_string(selenium):
)
format_tests = [
["c", "Uint8"],
["b", "Int8"],
["B", "Uint8"],
["?", "Uint8"],
["h", "Int16"],
["H", "Uint16"],
["i", "Int32"],
["I", "Uint32"],
["l", "Int32"],
["L", "Uint32"],
["n", "Int32"],
["N", "Uint32"],
["q", "BigInt64"],
["Q", "BigUint64"],
["f", "Float32"],
["d", "Float64"],
["s", "Uint8"],
["p", "Uint8"],
["P", "Uint32"],
("c", "Uint8"),
("b", "Int8"),
("B", "Uint8"),
("?", "Uint8"),
("h", "Int16"),
("H", "Uint16"),
("i", "Int32"),
("I", "Uint32"),
("l", "Int32"),
("L", "Uint32"),
("n", "Int32"),
("N", "Uint32"),
("q", "BigInt64"),
("Q", "BigUint64"),
("f", "Float32"),
("d", "Float64"),
("s", "Uint8"),
("p", "Uint8"),
("P", "Uint32"),
]
def process_fmt_string(fmt):
@ -1275,11 +1277,11 @@ def test_buffer_format_string(selenium):
assert array_name == expected_array_name
endian_tests = [
["@h", "Int16", False],
["=H", "Uint16", False],
["<i", "Int32", False],
[">I", "Uint32", True],
["!l", "Int32", True],
("@h", "Int16", False),
("=H", "Uint16", False),
("<i", "Int32", False),
(">I", "Uint32", True),
("!l", "Int32", True),
]
for fmt, expected_array_name, expected_is_big_endian in endian_tests: