From ae4492a1fd9d6eeded7a47bfa7fdf1779be30956 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Sat, 14 Jan 2023 22:59:42 +0900 Subject: [PATCH] Add a CLI command to create a zipfile of Python libraries (#3411) Co-authored-by: Roman Yurchak --- docs/project/changelog.md | 4 + pyodide-build/pyodide_build/cli/build.py | 2 +- .../pyodide_build/cli/build_recipes.py | 2 +- pyodide-build/pyodide_build/cli/config.py | 2 +- .../pyodide_build/cli/create_zipfile.py | 26 ++++ pyodide-build/pyodide_build/cli/py_compile.py | 2 +- pyodide-build/pyodide_build/cli/skeleton.py | 2 +- pyodide-build/pyodide_build/cli/venv.py | 2 +- pyodide-build/pyodide_build/pyzip.py | 137 ++++++++++++++++++ pyodide-build/pyodide_build/tests/__init__.py | 0 pyodide-build/pyodide_build/tests/fixture.py | 22 +++ pyodide-build/pyodide_build/tests/test_cli.py | 58 +++++++- .../pyodide_build/tests/test_pypi.py | 2 +- .../pyodide_build/tests/test_pyzip.py | 45 ++++++ pyodide-build/setup.cfg | 1 + pyproject.toml | 1 + 16 files changed, 299 insertions(+), 9 deletions(-) create mode 100644 pyodide-build/pyodide_build/cli/create_zipfile.py create mode 100644 pyodide-build/pyodide_build/pyzip.py create mode 100644 pyodide-build/pyodide_build/tests/__init__.py create mode 100644 pyodide-build/pyodide_build/tests/fixture.py create mode 100644 pyodide-build/pyodide_build/tests/test_pyzip.py diff --git a/docs/project/changelog.md b/docs/project/changelog.md index 8bb07d059..99ea352f6 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -21,6 +21,10 @@ myst: to .pyc files {pr}`3253` +- Added `pyodide create-zipfile` CLI command that creates a zip file of a directory. + This command is hidden by default since it is not intended for use by end users. + {pr}`3411` + - {{ Fix }} Fixed a bug where `pyodide build` would fail on package that use CMake, when run multiple times. {pr}`3445` diff --git a/pyodide-build/pyodide_build/cli/build.py b/pyodide-build/pyodide_build/cli/build.py index fdd4f1a70..998ec8c66 100644 --- a/pyodide-build/pyodide_build/cli/build.py +++ b/pyodide-build/pyodide_build/cli/build.py @@ -7,7 +7,7 @@ from typing import Optional from urllib.parse import urlparse import requests -import typer # type: ignore[import] +import typer from .. import common from ..out_of_tree import build diff --git a/pyodide-build/pyodide_build/cli/build_recipes.py b/pyodide-build/pyodide_build/cli/build_recipes.py index c10e1ea71..1bf999b5c 100644 --- a/pyodide-build/pyodide_build/cli/build_recipes.py +++ b/pyodide-build/pyodide_build/cli/build_recipes.py @@ -1,7 +1,7 @@ import argparse from pathlib import Path -import typer # type: ignore[import] +import typer from .. import buildall diff --git a/pyodide-build/pyodide_build/cli/config.py b/pyodide-build/pyodide_build/cli/config.py index df2b38573..aaeb3a0f5 100644 --- a/pyodide-build/pyodide_build/cli/config.py +++ b/pyodide-build/pyodide_build/cli/config.py @@ -1,4 +1,4 @@ -import typer # type: ignore[import] +import typer from ..common import get_make_environment_vars from ..out_of_tree.utils import initialize_pyodide_root diff --git a/pyodide-build/pyodide_build/cli/create_zipfile.py b/pyodide-build/pyodide_build/cli/create_zipfile.py new file mode 100644 index 000000000..f12eddcb4 --- /dev/null +++ b/pyodide-build/pyodide_build/cli/create_zipfile.py @@ -0,0 +1,26 @@ +from pathlib import Path + +import typer + +from ..pyzip import create_zipfile + + +def main( + libdir: Path = typer.Argument( + ..., help="Path to the directory containing the Python standard library." + ), + pycompile: bool = typer.Option( + False, help="Whether to compile the .py files into .pyc." + ), + output: Path = typer.Option( + "python.zip", help="Path to the output zip file. Defaults to python.zip." + ), +) -> None: + """ + Bundle Python standard libraries into a zip file. + """ + create_zipfile(libdir, output, pycompile=pycompile, filterfunc=None) + typer.echo(f"Zip file created at {output}") + + +main.typer_kwargs = {"hidden": True} # type: ignore[attr-defined] diff --git a/pyodide-build/pyodide_build/cli/py_compile.py b/pyodide-build/pyodide_build/cli/py_compile.py index cfc1b6446..7bb8a8f68 100644 --- a/pyodide-build/pyodide_build/cli/py_compile.py +++ b/pyodide-build/pyodide_build/cli/py_compile.py @@ -1,7 +1,7 @@ import sys from pathlib import Path -import typer # type: ignore[import] +import typer from pyodide_build._py_compile import _py_compile_wheel diff --git a/pyodide-build/pyodide_build/cli/skeleton.py b/pyodide-build/pyodide_build/cli/skeleton.py index 80e3bbf7d..d7f7b5c31 100644 --- a/pyodide-build/pyodide_build/cli/skeleton.py +++ b/pyodide-build/pyodide_build/cli/skeleton.py @@ -3,7 +3,7 @@ from pathlib import Path -import typer # type: ignore[import] +import typer from .. import common, mkpkg diff --git a/pyodide-build/pyodide_build/cli/venv.py b/pyodide-build/pyodide_build/cli/venv.py index d063fbf61..d7656f52f 100644 --- a/pyodide-build/pyodide_build/cli/venv.py +++ b/pyodide-build/pyodide_build/cli/venv.py @@ -1,6 +1,6 @@ from pathlib import Path -import typer # type: ignore[import] +import typer from ..out_of_tree import venv from ..out_of_tree.utils import initialize_pyodide_root diff --git a/pyodide-build/pyodide_build/pyzip.py b/pyodide-build/pyodide_build/pyzip.py new file mode 100644 index 000000000..d8f60b59f --- /dev/null +++ b/pyodide-build/pyodide_build/pyzip.py @@ -0,0 +1,137 @@ +import shutil +from collections.abc import Callable +from pathlib import Path +from tempfile import TemporaryDirectory + +# This files are removed from the stdlib by default +REMOVED_FILES = ( + # package management + "ensurepip/", + "venv/", + # build system + "lib2to3/", + # other platforms + "_osx_support.py", + # Not supported by browser + "curses/", + "dbm/", + "idlelib/", + "tkinter/", + "turtle.py", + "turtledemo", + "webbrowser.py", +) + +# This files are unvendored from the stdlib by default +UNVENDORED_FILES = ( + "test/", + "distutils/", + "sqlite3", + "ssl.py", + "lzma.py", +) + +# TODO: These modules have test directory which we unvendors it separately. +# So we should not pack them into the zip file in order to make e.g. import ctypes.test work. +# Note that all these tests are moved to the subdirectory of `test` module in upstream CPython 3.12.0a1. +# So we don't need this after we upgrade to 3.12.0 +NOT_ZIPPED_FILES = ("ctypes/", "unittest/") + + +def default_filterfunc( + root: Path, verbose: bool = False +) -> Callable[[str, list[str]], set[str]]: + """ + The default filter function used by `create_zipfile`. + + This function filters out several modules that are: + + - not supported in Pyodide due to browser limitations (e.g. `tkinter`) + - unvendored from the standard library (e.g. `sqlite3`) + """ + + def filterfunc(path: Path | str, names: list[str]) -> set[str]: + filtered_files = { + (root / f).resolve() + for f in REMOVED_FILES + UNVENDORED_FILES + NOT_ZIPPED_FILES + } + + path = Path(path).resolve() + + if path.name == "__pycache__": + return set(names) + + _names = [] + for name in names: + + fullpath = path / name + + if fullpath.name == "__pycache__" or fullpath in filtered_files: + if verbose: + print(f"Skipping {fullpath}") + + _names.append(name) + + return set(_names) + + return filterfunc + + +def create_zipfile( + libdir: Path | str, + output: Path | str = "python", + pycompile: bool = False, + filterfunc: Callable[[str, list[str]], set[str]] | None = None, +) -> None: + """ + Bundle Python standard libraries into a zip file. + + The basic idea of this function is similar to the standard library's + {ref}`zipfile.PyZipFile` class. + + However, we need some additional functionality for Pyodide. For example: + + - We need to remove some unvendored modules, e.g. `sqlite3` + - We need an option to "not" compile the files in the zip file + + hence this function. + + Parameters + ---------- + libdir + Path to the directory containing the Python standard library. + + output + Path to the output zip file. Defaults to python.zip. + + pycompile + Whether to compile the .py files into .pyc, by default False + + filterfunc + A function that filters the files to be included in the zip file. + This function will be passed to {ref}`shutil.copytree` 's ignore argument. + By default, Pyodide's default filter function is used. + + Returns + ------- + BytesIO + A BytesIO object containing the zip file. + """ + + if pycompile: + raise NotImplementedError( + "TODO: implement after https://github.com/pyodide/pyodide/pull/3253 is merged" + ) + + libdir = Path(libdir) + output = Path(output) + output = output.with_name(output.name.rstrip(".zip")) + + if filterfunc is None: + filterfunc = default_filterfunc(libdir) + + with TemporaryDirectory() as temp_dir_str: + temp_dir = Path(temp_dir_str) + shutil.copytree(libdir, temp_dir, ignore=filterfunc, dirs_exist_ok=True) + + shutil.make_archive(str(output), "zip", temp_dir) diff --git a/pyodide-build/pyodide_build/tests/__init__.py b/pyodide-build/pyodide_build/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyodide-build/pyodide_build/tests/fixture.py b/pyodide-build/pyodide_build/tests/fixture.py new file mode 100644 index 000000000..2c17dcfdb --- /dev/null +++ b/pyodide-build/pyodide_build/tests/fixture.py @@ -0,0 +1,22 @@ +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def temp_python_lib(tmp_path_factory): + libdir = tmp_path_factory.mktemp("python") + + path = Path(libdir) + + (path / "test").mkdir() + (path / "test" / "test_blah.py").touch() + (path / "distutils").mkdir() + (path / "turtle.py").touch() + + (path / "module1.py").touch() + (path / "module2.py").touch() + + (path / "hello_pyodide.py").write_text("def hello(): return 'hello'") + + yield libdir diff --git a/pyodide-build/pyodide_build/tests/test_cli.py b/pyodide-build/pyodide_build/tests/test_cli.py index 611509d2f..8ba3339d7 100644 --- a/pyodide-build/pyodide_build/tests/test_cli.py +++ b/pyodide-build/pyodide_build/tests/test_cli.py @@ -1,12 +1,17 @@ +# flake8: noqa + +import os import shutil from pathlib import Path import pytest -import typer # type: ignore[import] +import typer from typer.testing import CliRunner # type: ignore[import] from pyodide_build import common -from pyodide_build.cli import build_recipes, config, skeleton +from pyodide_build.cli import build, build_recipes, config, create_zipfile, skeleton + +from .fixture import temp_python_lib only_node = pytest.mark.xfail_browsers( chrome="node only", firefox="node only", safari="node only" @@ -127,3 +132,52 @@ def test_config_get(cfg_name, env_var): ) assert result.stdout.strip() == common.get_make_flag(env_var) + + +def test_fetch_or_build_pypi(selenium, tmp_path): + # TODO: Run this test without building Pyodide + + output_dir = tmp_path / "dist" + # one pure-python package (doesn't need building) and one sdist package (needs building) + pkgs = ["pytest-pyodide", "pycryptodome==3.15.0"] + + os.chdir(tmp_path) + + app = typer.Typer() + app.command()(build.main) + + for p in pkgs: + result = runner.invoke( + app, + [p], + ) + assert result.exit_code == 0, result.stdout + + built_wheels = set(output_dir.glob("*.whl")) + assert len(built_wheels) == len(pkgs) + + +def test_create_zipfile(temp_python_lib, tmp_path): + from zipfile import ZipFile + + output = tmp_path / "python.zip" + + app = typer.Typer() + app.command()(create_zipfile.main) + + result = runner.invoke( + app, + [ + str(temp_python_lib), + "--output", + str(output), + ], + ) + + assert result.exit_code == 0, result.stdout + assert "Zip file created" in result.stdout + assert output.exists() + + with ZipFile(output) as zf: + assert "module1.py" in zf.namelist() + assert "module2.py" in zf.namelist() diff --git a/pyodide-build/pyodide_build/tests/test_pypi.py b/pyodide-build/pyodide_build/tests/test_pypi.py index c5926d39f..b25afc993 100644 --- a/pyodide-build/pyodide_build/tests/test_pypi.py +++ b/pyodide-build/pyodide_build/tests/test_pypi.py @@ -10,7 +10,7 @@ from threading import Event, Thread from typing import Any import pytest -import typer # type: ignore[import] +import typer from typer.testing import CliRunner # type: ignore[import] from pyodide_build.cli import build diff --git a/pyodide-build/pyodide_build/tests/test_pyzip.py b/pyodide-build/pyodide_build/tests/test_pyzip.py new file mode 100644 index 000000000..9b5537efd --- /dev/null +++ b/pyodide-build/pyodide_build/tests/test_pyzip.py @@ -0,0 +1,45 @@ +# flake8: noqa + +from pyodide_build.pyzip import create_zipfile, default_filterfunc + +from .fixture import temp_python_lib + + +def test_defaultfilterfunc(temp_python_lib): + filterfunc = default_filterfunc(temp_python_lib, verbose=True) + + ignored = ["test", "distutils", "turtle.py"] + assert set(ignored) == filterfunc(str(temp_python_lib), ignored) + + assert set() == filterfunc(str(temp_python_lib), ["hello.py", "world.py"]) + + +def test_create_zip(temp_python_lib, tmp_path): + from zipfile import ZipFile + + output = tmp_path / "python.zip" + + create_zipfile(temp_python_lib, output, pycompile=False, filterfunc=None) + + assert output.exists() + + with ZipFile(output) as zf: + assert "module1.py" in zf.namelist() + assert "module2.py" in zf.namelist() + + +def test_import_from_zip(temp_python_lib, tmp_path, monkeypatch): + output = tmp_path / "python.zip" + + create_zipfile(temp_python_lib, output, pycompile=False, filterfunc=None) + + assert output.exists() + + import sys + + monkeypatch.setattr(sys, "path", [str(output)]) + + import hello_pyodide # type: ignore[import] + + assert hello_pyodide.__file__.startswith(str(output)) + assert hello_pyodide.hello() == "hello" diff --git a/pyodide-build/setup.cfg b/pyodide-build/setup.cfg index 7280bee3b..6cc46acfd 100644 --- a/pyodide-build/setup.cfg +++ b/pyodide-build/setup.cfg @@ -50,6 +50,7 @@ pyodide.cli = skeleton = pyodide_build.cli.skeleton:app py-compile = pyodide_build.cli.py_compile:main config = pyodide_build.cli.config:app + create-zipfile = pyodide_build.cli.create_zipfile:main [options.extras_require] test = diff --git a/pyproject.toml b/pyproject.toml index 59aab3244..f012b7643 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ module = [ "virtualenv", "termcolor", "test", + "typer", ] ignore_missing_imports = true