Rooba
/
pyodide
mirror of https://github.com/pyodide/pyodide.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TST Make `pyodide-test-runner` installable (#2742)

This commit is contained in:
Gyeongjae Choi 2022-07-04 16:07:01 +09:00 committed by GitHub
parent bfd364b28b
commit 7d7b7e899d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 252 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
.circleci/config.yml
View File

@ -25,6 +25,8 @@ jobs:
name: Test docs
command: |
mkdir test-results
pip install ./pyodide-test-runner
npm install -g node-fetch@2
pytest docs/sphinx_pyodide/tests --junitxml=test-results/junit.xml
- store_test_results:
path: test-results
@ -214,6 +216,8 @@ jobs:
command: |
make npm-link
mkdir test-results
pip install ./pyodide-test-runner
npm install -g node-fetch@2
tools/pytest_wrapper.py \
--junitxml=test-results/junit.xml \
--verbose \
@ -231,6 +235,8 @@ jobs:
name: stack-size
command: |
make npm-link
pip install ./pyodide-test-runner
npm install -g node-fetch@2
pytest -s benchmark/stack_usage.py | sed -n 's/## //pg'
test-python:
@ -243,17 +249,31 @@ jobs:
name: test
command: |
mkdir test-results
python3 -m pip install ./pyodide-test-runner
python3 -m pip install -e ./pyodide-build
npm install -g node-fetch@2
PYODIDE_ROOT=. pytest \
--junitxml=test-results/junit.xml \
--verbose \
-k 'not (chrome or firefox or node)' \
--cov=pyodide_build --cov=pyodide \
src pyodide-build packages/micropip/ pyodide-test-runner
src pyodide-build packages/micropip/
- store_test_results:
path: test-results
test-test-runner:
<<: *defaults
resource_class: medium+
steps:
- attach_workspace:
at: .
- run:
name: test
command: |
npm install -g node-fetch@2
cd pyodide-test-runner && pytest -v .
test-js:
<<: *defaults
resource_class: small
@ -291,6 +311,8 @@ jobs:
- run:
name: benchmark
command: |
pip install ./pyodide-test-runner
npm install -g node-fetch@2
python benchmark/benchmark.py all --output dist/benchmarks.json
- store_artifacts:
@ -482,7 +504,7 @@ workflows:
- test-main:
name: test-core-chrome
test-params: -k "chrome and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-test-runner
test-params: -k "chrome and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
requires:
- build-core
filters:
@ -491,7 +513,7 @@ workflows:
- test-main:
name: test-core-firefox
test-params: -k "firefox and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-test-runner
test-params: -k "firefox and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
requires:
- build-core
filters:
@ -500,7 +522,7 @@ workflows:
- test-main:
name: test-core-node
test-params: -k node src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-test-runner
test-params: -k node src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
requires:
- build-core
filters:
@ -552,6 +574,13 @@ workflows:
tags:
only: /.*/
- test-test-runner:
requires:
- build-core
filters:
tags:
only: /.*/
- test-js:
requires:
- build-core

1
.github/workflows/main.yml vendored
View File

@ -128,6 +128,7 @@ jobs:
shell: bash -l {0}
run: |
pip install -r requirements.txt
pip install ./pyodide-test-runner
# FIXME: playwright 1.23.0 has unknown performance issue on firefox
pip install "playwright<1.23.0" && python -m playwright install

1
benchmark/benchmark.py
View File

@ -7,7 +7,6 @@ from pathlib import Path
from time import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.append(str(Path(__file__).resolve().parents[1] / "pyodide-test-runner"))
from pyodide_test_runner import ( # noqa: E402
SeleniumChromeWrapper,

79
conftest.py
View File

@ -6,31 +6,12 @@ import sys
import pytest
pytest_plugins = ("pytest_asyncio",)
ROOT_PATH = pathlib.Path(__file__).parents[0].resolve()
DIST_PATH = ROOT_PATH / "dist"
sys.path.append(str(ROOT_PATH / "pyodide-test-runner"))
sys.path.append(str(ROOT_PATH / "pyodide-build"))
sys.path.append(str(ROOT_PATH / "src" / "py"))
from pyodide_test_runner.fixture import ( # noqa: F401
console_html_fixture,
playwright_browsers,
script_type,
selenium,
selenium_common,
selenium_context_manager,
selenium_esm,
selenium_module_scope,
selenium_standalone,
selenium_standalone_noload,
selenium_standalone_noload_common,
selenium_webworker_standalone,
web_server_main,
web_server_secondary,
)
from pyodide_test_runner.utils import maybe_skip_test
from pyodide_test_runner.utils import package_is_built as _package_is_built
from pyodide_test_runner.utils import parse_xfail_browsers
@ -38,23 +19,11 @@ from pyodide_test_runner.utils import parse_xfail_browsers
def pytest_addoption(parser):
group = parser.getgroup("general")
group.addoption(
"--dist-dir",
action="store",
default=DIST_PATH,
help="Path to the dist directory",
)
group.addoption(
"--run-xfail",
action="store_true",
help="If provided, tests marked as xfail will be run",
)
group.addoption(
"--runner",
default="selenium",
choices=["selenium", "playwright"],
help="Select testing frameworks, selenium or playwright (default: %(default)s)",
)
def pytest_configure(config):
@ -165,51 +134,3 @@ def extra_checks_test_wrapper(browser, trace_hiwire_refs, trace_pyproxies):
def package_is_built(package_name):
return _package_is_built(package_name, pytest.pyodide_dist_dir)
import ast
from copy import deepcopy
from typing import Any
from _pytest.assertion.rewrite import AssertionRewritingHook, rewrite_asserts
from _pytest.python import (
pytest_pycollect_makemodule as orig_pytest_pycollect_makemodule,
)
# Handling for pytest assertion rewrites
# First we find the pytest rewrite config. It's an attribute of the pytest
# assertion rewriting meta_path_finder, so we locate that to get the config.
def _get_pytest_rewrite_config() -> Any:
for meta_path_finder in sys.meta_path:
if isinstance(meta_path_finder, AssertionRewritingHook):
break
else:
return None
return meta_path_finder.config
# Now we need to parse the ast of the files, rewrite the ast, and store the
# original and rewritten ast into dictionaries. `run_in_pyodide` will look the
# ast up in the appropriate dictionary depending on whether or not it is using
# pytest assert rewrites.
REWRITE_CONFIG = _get_pytest_rewrite_config()
del _get_pytest_rewrite_config
ORIGINAL_MODULE_ASTS: dict[str, ast.Module] = {}
REWRITTEN_MODULE_ASTS: dict[str, ast.Module] = {}
def pytest_pycollect_makemodule(
module_path: pathlib.Path, path: Any, parent: Any
) -> None:
source = module_path.read_bytes()
strfn = str(module_path)
tree = ast.parse(source, filename=strfn)
ORIGINAL_MODULE_ASTS[strfn] = tree
tree2 = deepcopy(tree)
rewrite_asserts(tree2, source, strfn, REWRITE_CONFIG)
REWRITTEN_MODULE_ASTS[strfn] = tree2
orig_pytest_pycollect_makemodule(module_path, parent)

1
pyodide-test-runner/README.md Normal file
View File

@ -0,0 +1 @@
# pyodide-test-runner

4
pyodide-test-runner/conftest.py Normal file
View File

@ -0,0 +1,4 @@
pytest_plugins = [
"pyodide_test_runner.hook",
"pyodide_test_runner.fixture",
]

10
pyodide-test-runner/pyodide_test_runner/decorator.py
View File

@ -5,16 +5,16 @@ from base64 import b64decode, b64encode
from copy import deepcopy
from typing import Any, Callable, Collection
from pyodide_test_runner.utils import package_is_built as _package_is_built
import pytest
from .hook import ORIGINAL_MODULE_ASTS, REWRITTEN_MODULE_ASTS
from .utils import package_is_built as _package_is_built
def package_is_built(package_name):
return _package_is_built(package_name, pytest.pyodide_dist_dir)
import pytest
class SeleniumType:
JavascriptException: type
browser: str
@ -152,8 +152,6 @@ class run_in_pyodide:
when an assertion fails, but requires us to load pytest.
"""
from conftest import ORIGINAL_MODULE_ASTS, REWRITTEN_MODULE_ASTS
self._pkgs = list(packages)
self._pytest_not_built = False
if (

4
pyodide-test-runner/pyodide_test_runner/fixture.py
View File

@ -1,4 +1,6 @@
import contextlib
import os
from pathlib import Path
import pytest
@ -80,7 +82,7 @@ def selenium_common(
f"Unknown runner or browser: {runner_type} / {request.param}"
)
dist_dir = request.config.getoption("--dist-dir")
dist_dir = Path(os.getcwd(), request.config.getoption("--dist-dir"))
runner = cls(
server_port=server_port,
server_hostname=server_hostname,

91
pyodide-test-runner/pyodide_test_runner/hook.py Normal file
View File

@ -0,0 +1,91 @@
import ast
import sys
from copy import deepcopy
from pathlib import Path
from typing import Any
import pytest
from _pytest.assertion.rewrite import AssertionRewritingHook, rewrite_asserts
from _pytest.python import (
pytest_pycollect_makemodule as orig_pytest_pycollect_makemodule,
)
def pytest_configure(config):
config.addinivalue_line(
"markers",
"skip_refcount_check: Don't run refcount checks",
)
config.addinivalue_line(
"markers",
"skip_pyproxy_check: Don't run pyproxy allocation checks",
)
config.addinivalue_line(
"markers",
"driver_timeout: Set script timeout in WebDriver",
)
config.addinivalue_line(
"markers",
"xfail_browsers: xfail a test in specific browsers",
)
pytest.pyodide_dist_dir = config.getoption("--dist-dir")
@pytest.hookimpl(tryfirst=True)
def pytest_addoption(parser):
group = parser.getgroup("general")
group.addoption(
"--dist-dir",
action="store",
default="pyodide",
help="Path to the pyodide dist directory",
type=Path,
)
group.addoption(
"--runner",
default="selenium",
choices=["selenium", "playwright"],
help="Select testing frameworks, selenium or playwright (default: %(default)s)",
)
# Handling for pytest assertion rewrites
# First we find the pytest rewrite config. It's an attribute of the pytest
# assertion rewriting meta_path_finder, so we locate that to get the config.
def _get_pytest_rewrite_config() -> Any:
for meta_path_finder in sys.meta_path:
if isinstance(meta_path_finder, AssertionRewritingHook):
break
else:
return None
return meta_path_finder.config
# Now we need to parse the ast of the files, rewrite the ast, and store the
# original and rewritten ast into dictionaries. `run_in_pyodide` will look the
# ast up in the appropriate dictionary depending on whether or not it is using
# pytest assert rewrites.
REWRITE_CONFIG = _get_pytest_rewrite_config()
del _get_pytest_rewrite_config
ORIGINAL_MODULE_ASTS: dict[str, ast.Module] = {}
REWRITTEN_MODULE_ASTS: dict[str, ast.Module] = {}
def pytest_pycollect_makemodule(module_path: Path, path: Any, parent: Any) -> None:
source = module_path.read_bytes()
strfn = str(module_path)
tree = ast.parse(source, filename=strfn)
ORIGINAL_MODULE_ASTS[strfn] = tree
tree2 = deepcopy(tree)
rewrite_asserts(tree2, source, strfn, REWRITE_CONFIG)
REWRITTEN_MODULE_ASTS[strfn] = tree2
orig_pytest_pycollect_makemodule(module_path, parent)

3
pyodide-test-runner/pyodide_test_runner/node_test_driver.js
View File

@ -3,7 +3,6 @@ const readline = require("readline");
const path = require("path");
const util = require("util");
const node_fetch = require("node-fetch");
const base64 = require("base-64");
let baseUrl = process.argv[2];
let distDir = process.argv[3];
@ -26,8 +25,6 @@ const context = {
TextDecoder: util.TextDecoder,
TextEncoder: util.TextEncoder,
URL,
atob: base64.decode,
btoa: base64.encode,
clearInterval,
clearTimeout,
setInterval,

126
pyodide-test-runner/pyodide_test_runner/tests/test_decorator.py
View File

@ -1,83 +1,53 @@
import asyncio
import pytest
from hypothesis import given, settings
from pyodide_test_runner.decorator import run_in_pyodide
from pyodide_test_runner.hypothesis import any_strategy, std_hypothesis_settings
from pyodide_test_runner.utils import parse_driver_timeout
from pyodide import eval_code_async
@run_in_pyodide
def example_func(selenium):
pass
@run_in_pyodide(_force_assert_rewrites=True)
def example_func1(selenium):
x = 6
y = 7
assert x == y
def test_selenium1(selenium):
import pytest
with pytest.raises(AssertionError, match="assert 6 == 7"):
x = 6
y = 7
assert x == y
run_in_pyodide_alias = run_in_pyodide(_force_assert_rewrites=True)
@run_in_pyodide_alias
def example_func2(selenium):
def test_selenium2(selenium):
import pytest
x = 6
y = 7
assert x == y
run_in_pyodide_inner = run_in_pyodide()
with pytest.raises(AssertionError, match="assert 6 == 7"):
assert x == y
@run_in_pyodide(_force_assert_rewrites=True)
async def async_example_func(selenium):
async def test_selenium3(selenium):
from asyncio import sleep
import pytest
await sleep(0.01)
x = 6
await sleep(0.01)
y = 7
assert x == y
class selenium_mock:
JavascriptException = Exception
browser = "none"
@staticmethod
def load_package(*args, **kwargs):
pass
@staticmethod
def run_async(code: str):
return asyncio.new_event_loop().run_until_complete(eval_code_async(code))
def test_local1():
with pytest.raises(AssertionError, match="assert 6 == 7"):
example_func1(selenium_mock)
assert x == y
def test_local2():
with pytest.raises(AssertionError, match="assert 6 == 7"):
example_func2(selenium_mock)
def test_local3():
with pytest.raises(AssertionError, match="assert 6 == 7"):
async_example_func(selenium_mock)
def test_local_inner_function():
@run_in_pyodide
def inner_function(selenium, x):
assert x == 6
return 7
assert inner_function(selenium_mock, 6) == 7
def test_local_inner_function_closure_error():
def test_inner_function_closure_error(selenium):
x = 6
@run_in_pyodide
@ -86,7 +56,7 @@ def test_local_inner_function_closure_error():
return 7
with pytest.raises(NameError, match="'x' is not defined"):
inner_function(selenium_mock)
inner_function(selenium)
def test_inner_function(selenium):
@ -126,25 +96,26 @@ def example_decorator_func(selenium):
pass
def test_local4():
example_decorator_func(selenium_mock)
assert example_decorator_func.dec_info == [
def test_selenium4(selenium_standalone):
example_decorator_func(selenium_standalone)
assert example_decorator_func.dec_info[-3:] == [
("testdec1", "a"),
("testdec2", "b"),
("testdec1", "c"),
]
class selenium_mock_fail_load_package(selenium_mock):
@staticmethod
def load_package(*args, **kwargs):
def test_local_fail_load_package(selenium_standalone):
selenium = selenium_standalone
def _load_package_error(*args, **kwargs):
raise OSError("STOP!")
selenium.load_package = _load_package_error
def test_local_fail_load_package():
exc = None
try:
example_func1(selenium_mock_fail_load_package)
example_func(selenium)
except OSError:
exc = pytest.ExceptionInfo.from_current()
@ -161,14 +132,6 @@ def test_local_fail_load_package():
)
def test_selenium(selenium):
with pytest.raises(AssertionError, match="assert 6 == 7"):
example_func1(selenium)
with pytest.raises(AssertionError, match="assert 6 == 7"):
example_func2(selenium)
@run_in_pyodide
def test_trivial1(selenium):
x = 7
@ -220,39 +183,16 @@ async def test_run_in_pyodide_async(selenium):
max_examples=25,
)
@run_in_pyodide
def test_hypothesis(selenium, obj):
def test_hypothesis(selenium_standalone, obj):
from pyodide import to_js
to_js(obj)
run_in_pyodide_inner = run_in_pyodide()
run_in_pyodide_alias2 = pytest.mark.driver_timeout(40)(run_in_pyodide_inner)
@run_in_pyodide_alias2
def test_run_in_pyodide_alias(request):
assert parse_driver_timeout(request.node) == 40
@run_in_pyodide
def test_pickle_jsexception(selenium):
import pickle
from pyodide.code import run_js
pickle.dumps(run_js("new Error('hi');"))
def test_raises_jsexception(selenium):
import pytest
from pyodide import JsException
@run_in_pyodide
def raise_jsexception(selenium):
from pyodide.code import run_js
run_js("throw new Error('hi');")
with pytest.raises(JsException, match="Error: hi"):
raise_jsexception(selenium)

4
pyodide-test-runner/pyproject.toml Normal file
View File

@ -0,0 +1,4 @@
[build-system]
requires = ["setuptools>=42", "wheel"]
build-backend = "setuptools.build_meta"

49
pyodide-test-runner/setup.cfg Normal file
View File

@ -0,0 +1,49 @@
[metadata]
name = pyodide-test-runner
version = 0.21.0.dev0
author = Pyodide developers
description = "Pytest plugin for testing Pyodide and third-party applications that use Pyodide"
long_description = file: README.md
long_description_content_type = text/markdown
url = https://github.com/pyodide/pyodide
project_urls =
Bug Tracker = https://github.com/pyodide/pyodide/issues
Documentation = https://pyodide.org/en/stable/
classifiers =
Programming Language :: Python :: 3
License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
Operating System :: OS Independent
Framework :: Pytest
[options]
package_dir =
= .
packages = find:
python_requires = >=3.10
include_package_data = True
install_requires =
pexpect
pytest
pytest-asyncio
selenium
tblib
# This is required to add node driver code to the package.
[options.package_data]
pyodide_test_runner = *.js
# pytest will look up `pytest11` entrypoints to find plugins
# See: https://docs.pytest.org/en/7.1.x/how-to/writing_plugins.html#making-your-plugin-installable-by-others
[options.entry_points]
pytest11 =
pyodide_test_runner = pyodide_test_runner.fixture
pyodide_test_runner_hook = pyodide_test_runner.hook
[options.packages.find]
where = .
[tool:pytest]
asyncio_mode = strict
addopts =
--tb=short
--dist-dir=../dist

6
requirements.txt
View File

@ -6,13 +6,7 @@
# testing
build==0.7.0
hypothesis
pexpect
pytest
pytest-asyncio
pytest-cov
pytest-httpserver
pytest-instafail
pytest-rerunfailures
pytest-xdist
selenium==4.1.0
tblib

7
setup.cfg
View File

@ -7,12 +7,9 @@ norecursedirs =
addopts =
--doctest-modules
--ignore="packages/matplotlib/src"
--ignore="pyodide-test-runner"
--tb=short
markers =
skip_refcount_check: Dont run refcount checks
skip_pyproxy_check: Dont run pyproxy allocation checks
driver_timeout: Set script timeout in WebDriver
xfail_browsers: xfail a test in specific browsers
--dist-dir=dist
testpaths =
src

22
src/tests/test_pyodide.py
View File

@ -1146,6 +1146,28 @@ def test_run_js(selenium):
assert x == 77
@run_in_pyodide
def test_pickle_jsexception(selenium):
import pickle
from pyodide.code import run_js
pickle.dumps(run_js("new Error('hi');"))
def test_raises_jsexception(selenium):
from pyodide.ffi import JsException
@run_in_pyodide
def raise_jsexception(selenium):
from pyodide.code import run_js
run_js("throw new Error('hi');")
with pytest.raises(JsException, match="Error: hi"):
raise_jsexception(selenium)
@run_in_pyodide(packages=["pytest"])
def test_moved_deprecation_warnings(selenium_standalone):
import pytest

5
tools/bump_version.py
View File

@ -64,6 +64,11 @@ PYTHON_TARGETS = [
build_version_pattern(r"version\s*=\s*{{{python_version}}}"),
prerelease=False,
),
Target(
ROOT / "pyodide-test-runner/setup.cfg",
build_version_pattern("version = {python_version}"),
prerelease=False,
),
]
JS_TARGETS = [