mirror of https://github.com/pyodide/pyodide.git
391 lines
13 KiB
Python
391 lines
13 KiB
Python
# flake8: noqa
|
|
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
from textwrap import dedent
|
|
from threading import Event, Thread
|
|
from typing import Any
|
|
|
|
import pytest
|
|
import typer
|
|
from typer.testing import CliRunner
|
|
|
|
from pyodide_build.cli import build
|
|
from .fixture import reset_cache, reset_env_vars, xbuildenv
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
def _make_fake_package(
|
|
root: Path, name: str, ver: str, requires: list[str], wheel: bool
|
|
) -> None:
|
|
canonical_name = re.sub("[_.]", "-", name)
|
|
module_name = re.sub("-", "_", name)
|
|
packageDir = root / canonical_name
|
|
packageDir.mkdir(exist_ok=True)
|
|
with tempfile.TemporaryDirectory() as td:
|
|
build_path = Path(td)
|
|
src_path = build_path / "src" / module_name
|
|
src_path.mkdir(exist_ok=True, parents=True)
|
|
with open(build_path / "pyproject.toml", "w") as cf:
|
|
requirements = []
|
|
extras_requirements: dict[str, list[str]] = {}
|
|
for x in requires:
|
|
extras = []
|
|
if x.find(";") != -1:
|
|
requirement, marker_part = x.split(";")
|
|
extras = re.findall(r"extra\s*==\s*[\"'](\w+)[\"']", marker_part)
|
|
else:
|
|
requirement = x
|
|
if len(extras) > 0:
|
|
for match in extras:
|
|
if match not in extras_requirements:
|
|
extras_requirements[match] = []
|
|
extras_requirements[match].append(requirement)
|
|
else:
|
|
requirements.append(requirement)
|
|
extras_requirements_text = ""
|
|
for e in extras_requirements.keys():
|
|
extras_requirements_text += f"{e} = [\n"
|
|
for r in extras_requirements[e]:
|
|
extras_requirements_text += f"'{r}',\n"
|
|
extras_requirements_text += "]\n"
|
|
template = dedent(
|
|
"""
|
|
[project]
|
|
name = "{name}"
|
|
version = "{version}"
|
|
authors = [{{name = "Your Name", email = "you@yourdomain.com"}}]
|
|
description = "Example project {name}"
|
|
readme = "README.md"
|
|
requires-python = ">=3.8"
|
|
classifiers = [
|
|
"Development Status :: 3 - Alpha",
|
|
"License :: OSI Approved :: MIT License",
|
|
"Natural Language :: English",
|
|
"Operating System :: OS Independent",
|
|
"Programming Language :: Python",
|
|
"Programming Language :: Python :: 3.8",
|
|
"Programming Language :: Python :: 3.9",
|
|
]
|
|
dependencies = {requirements}
|
|
|
|
[build-system]
|
|
build-backend = "setuptools.build_meta"
|
|
requires = ["setuptools >= 65.0","wheel","cython >= 0.29.0"]
|
|
|
|
[project.optional-dependencies]
|
|
{optional_deps_text}
|
|
"""
|
|
)
|
|
config_str = template.format(
|
|
name=canonical_name,
|
|
version=ver,
|
|
requirements=str(requirements),
|
|
optional_deps_text=extras_requirements_text,
|
|
)
|
|
cf.write(config_str)
|
|
with open(build_path / "README.md", "w") as rf:
|
|
rf.write("\n")
|
|
if wheel:
|
|
with open(src_path / "__init__.py", "w") as f:
|
|
f.write(f'print("Hello from {name} module")\n')
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"build",
|
|
"--wheel",
|
|
build_path,
|
|
"--outdir",
|
|
packageDir,
|
|
]
|
|
)
|
|
else:
|
|
# make cython sdist
|
|
# i.e. create pyc + setup.cfg (needs cython) in folder and run python -m build
|
|
with open(src_path / "__init__.py", "w") as f:
|
|
f.write("from .compiled_mod import *")
|
|
with open(src_path / "compiled_mod.pyx", "w") as f:
|
|
f.write(f'print("Hello from compiled module {name}")')
|
|
with open(build_path / "setup.py", "w") as sf:
|
|
sf.write(
|
|
f"""
|
|
from setuptools import setup
|
|
from Cython.Build import cythonize
|
|
setup(ext_modules=cythonize("src/{module_name}/*.pyx",language_level=3))
|
|
"""
|
|
)
|
|
with open(build_path / "MANIFEST.in", "w") as mf:
|
|
mf.write("global-include *.pyx\n")
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"build",
|
|
"--sdist",
|
|
build_path,
|
|
"--outdir",
|
|
packageDir,
|
|
]
|
|
)
|
|
|
|
|
|
# module scope fixture that makes a fake pypi
|
|
@pytest.fixture(scope="module")
|
|
def fake_pypi_server():
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
root = Path(tmpdir)
|
|
simple_root = root / "simple"
|
|
if not simple_root.exists():
|
|
simple_root.mkdir(exist_ok=True, parents=True)
|
|
# top package resolves_package that should resolve okay - nb:
|
|
# this package depends on micropip which is in pyodide already
|
|
# and should not be rebuilt
|
|
_make_fake_package(
|
|
simple_root,
|
|
"resolves-package",
|
|
"1.0.0",
|
|
["pkg-a", "pkg-b[docs]", "micropip"],
|
|
True,
|
|
)
|
|
_make_fake_package(simple_root, "pkg-a", "1.0.0", ["pkg-c"], True)
|
|
_make_fake_package(
|
|
simple_root, "pkg-b", "1.0.0", ['pkg-d==2.0.0;extra=="docs"'], False
|
|
)
|
|
_make_fake_package(simple_root, "pkg-c", "1.0.0", [], True)
|
|
_make_fake_package(simple_root, "pkg-c", "2.0.0", [], True)
|
|
_make_fake_package(simple_root, "pkg-d", "1.0.0", [], False)
|
|
_make_fake_package(simple_root, "pkg-d", "2.0.0", [], False)
|
|
|
|
# top package doesn't resolve package that requires
|
|
_make_fake_package(
|
|
simple_root,
|
|
"fails_package",
|
|
"1.0.0",
|
|
["pkg_a", "pkg_b[docs]", "pkg_d==1.0.0"],
|
|
True,
|
|
)
|
|
|
|
# spawn webserver
|
|
def server_thread(
|
|
root_path: Path, server_evt: Event, ret_values: list[Any]
|
|
) -> None:
|
|
class PathRequestHandler(SimpleHTTPRequestHandler):
|
|
def __init__(self, *args, **argv):
|
|
argv["directory"] = root_path.resolve()
|
|
super().__init__(*args, **argv)
|
|
|
|
server = ThreadingHTTPServer(
|
|
("127.0.0.1", 0), RequestHandlerClass=PathRequestHandler
|
|
)
|
|
ret_values.append(server)
|
|
server_evt.set()
|
|
server.serve_forever(poll_interval=0.05)
|
|
|
|
server_evt = Event()
|
|
ret_values: list[ThreadingHTTPServer] = []
|
|
running_thread = Thread(
|
|
target=server_thread,
|
|
kwargs={
|
|
"root_path": root,
|
|
"server_evt": server_evt,
|
|
"ret_values": ret_values,
|
|
},
|
|
)
|
|
running_thread.start()
|
|
server_evt.wait()
|
|
# now ret_values[0] should be server
|
|
server = ret_values[0]
|
|
addr = f"http://{server.server_address[0]}:{server.server_address[1]}/simple" # type: ignore[str-bytes-safe]
|
|
|
|
yield (addr, f"{server.server_address[0]}:{server.server_address[1]}") # type: ignore[str-bytes-safe]
|
|
# cleanup
|
|
server.shutdown()
|
|
|
|
|
|
# fixture to redirect a single test to use fake pypi in resolution
|
|
@pytest.fixture
|
|
def fake_pypi_url(fake_pypi_server):
|
|
import pyodide_build.out_of_tree.pypi
|
|
|
|
pypi_old = pyodide_build.out_of_tree.pypi._PYPI_INDEX
|
|
pyodide_build.out_of_tree.pypi._PYPI_TRUSTED_HOSTS = [fake_pypi_server[1]]
|
|
pyodide_build.out_of_tree.pypi._PYPI_INDEX = [fake_pypi_server[0]]
|
|
yield fake_pypi_server[0]
|
|
pyodide_build.out_of_tree.pypi._PYPI_INDEX = pypi_old
|
|
|
|
|
|
def test_fetch_or_build_pypi(xbuildenv):
|
|
# TODO: - make test run without pyodide
|
|
output_dir = xbuildenv / "dist"
|
|
# one pure-python package (doesn't need building) and one sdist package (needs building)
|
|
pkgs = ["pytest-pyodide", "pycryptodome==3.15.0"]
|
|
|
|
app = typer.Typer()
|
|
app.command()(build.main)
|
|
|
|
for p in pkgs:
|
|
result = runner.invoke(
|
|
app,
|
|
[p],
|
|
)
|
|
assert result.exit_code == 0, result.stdout
|
|
|
|
built_wheels = set(output_dir.glob("*.whl"))
|
|
assert len(built_wheels) == len(pkgs)
|
|
|
|
|
|
def test_fetch_or_build_pypi_with_deps_and_extras(xbuildenv):
|
|
# TODO: - make test run without pyodide
|
|
output_dir = xbuildenv / "dist"
|
|
# one pure-python package (doesn't need building) which depends on one sdist package (needs building)
|
|
pkgs = ["eth-hash[pycryptodome]==0.5.1", "safe-pysha3 (>=1.0.0)"]
|
|
|
|
app = typer.Typer()
|
|
app.command()(build.main)
|
|
|
|
for p in pkgs:
|
|
result = runner.invoke(
|
|
app,
|
|
[p, "--build-dependencies"],
|
|
)
|
|
assert result.exit_code == 0, result.stdout
|
|
|
|
built_wheels = set(output_dir.glob("*.whl"))
|
|
assert len(built_wheels) == 3
|
|
|
|
|
|
def test_fake_pypi_succeed(xbuildenv, fake_pypi_url):
|
|
# TODO: - make test run without pyodide
|
|
output_dir = xbuildenv / "dist"
|
|
# build package that resolves right
|
|
app = typer.Typer()
|
|
app.command()(build.main)
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
["resolves-package", "--build-dependencies"],
|
|
)
|
|
|
|
assert result.exit_code == 0, str(result.stdout) + str(result)
|
|
|
|
built_wheels = set(output_dir.glob("*.whl"))
|
|
assert len(built_wheels) == 5
|
|
# make sure built in package micropip is not rebuilt
|
|
assert len(set(output_dir.glob("micropip*.whl"))) == 0
|
|
|
|
|
|
def test_fake_pypi_resolve_fail(xbuildenv, fake_pypi_url):
|
|
output_dir = xbuildenv / "dist"
|
|
|
|
# build package that resolves right
|
|
|
|
app = typer.Typer()
|
|
app.command()(build.main)
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
["fails-package", "--build-dependencies"],
|
|
)
|
|
|
|
# this should fail and should not build any wheels
|
|
assert result.exit_code != 0, result.stdout
|
|
built_wheels = set(output_dir.glob("*.whl"))
|
|
assert len(built_wheels) == 0
|
|
|
|
|
|
def test_fake_pypi_extras_build(xbuildenv, fake_pypi_url):
|
|
# TODO: - make test run without pyodide
|
|
output_dir = xbuildenv / "dist"
|
|
# build package that resolves right
|
|
app = typer.Typer()
|
|
app.command()(build.main)
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
["pkg-b[docs]", "--build-dependencies"],
|
|
)
|
|
|
|
# this should work
|
|
assert result.exit_code == 0, result.stdout
|
|
built_wheels = set(output_dir.glob("*.whl"))
|
|
assert len(built_wheels) == 2
|
|
|
|
|
|
def test_fake_pypi_repeatable_build(xbuildenv, fake_pypi_url):
|
|
output_dir = xbuildenv / "dist"
|
|
|
|
# build package that resolves right
|
|
app = typer.Typer()
|
|
app.command()(build.main)
|
|
|
|
# override a dependency version and build
|
|
# pkg-a
|
|
with open("requirements.txt", "w") as req_file:
|
|
req_file.write(
|
|
"""
|
|
# Whole line comment
|
|
pkg-c~=1.0.0 # end of line comment
|
|
pkg-a
|
|
"""
|
|
)
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"-r",
|
|
"requirements.txt",
|
|
"--build-dependencies",
|
|
"--output-lockfile",
|
|
"lockfile.txt",
|
|
],
|
|
)
|
|
# this should work
|
|
assert result.exit_code == 0, result.stdout
|
|
built_wheels = list(output_dir.glob("*.whl"))
|
|
assert len(built_wheels) == 2, result.stdout
|
|
|
|
# should have built version 1.0.0 of pkg-c
|
|
for x in built_wheels:
|
|
if x.name.startswith("pkg_c"):
|
|
assert x.name.find("1.0.0") != -1, x.name
|
|
x.unlink()
|
|
|
|
# rebuild from package-versions lockfile and
|
|
# check it outputs the same version number
|
|
result = runner.invoke(
|
|
app,
|
|
["-r", "lockfile.txt"],
|
|
)
|
|
|
|
# should still have built 1.0.0 of pkg-c
|
|
built_wheels = list(output_dir.glob("*.whl"))
|
|
for x in built_wheels:
|
|
if x.name.startswith("pkg_c"):
|
|
assert x.name.find("1.0.0") != -1, x.name
|
|
|
|
assert len(built_wheels) == 2, result.stdout
|
|
|
|
|
|
def test_bad_requirements_text(xbuildenv):
|
|
app = typer.Typer()
|
|
app.command()(build.main)
|
|
# test 1 - error on URL location in requirements
|
|
# test 2 - error on advanced options
|
|
# test 3 - error on editable install of package
|
|
bad_lines = [" pkg-c@http://www.pkg-c.org", " -r bob.txt", " -e pkg-c"]
|
|
for line in bad_lines:
|
|
with open("requirements.txt", "w") as req_file:
|
|
req_file.write(line + "\n")
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
["-r", "requirements.txt"],
|
|
)
|
|
assert result.exit_code != 0 and line.strip() in str(result)
|