Add a CLI command to create a zipfile of Python libraries (#3411)

Co-authored-by: Roman Yurchak <rth.yurchak@gmail.com>

docs/project/changelog.md

@@ -21,6 +21,10 @@ myst:
   to .pyc files
   {pr}`3253`
 
+- Added `pyodide create-zipfile` CLI command that creates a zip file of a directory.
+  This command is hidden by default since it is not intended for use by end users.
+  {pr}`3411`
+
 - {{ Fix }} Fixed a bug where `pyodide build` would fail on package that use CMake,
   when run multiple times.
   {pr}`3445`

pyodide-build/pyodide_build/cli/build.py

@@ -7,7 +7,7 @@ from typing import Optional
 from urllib.parse import urlparse
 
 import requests
-import typer  # type: ignore[import]
+import typer
 
 from .. import common
 from ..out_of_tree import build

pyodide-build/pyodide_build/cli/build_recipes.py

@@ -1,7 +1,7 @@
 import argparse
 from pathlib import Path
 
-import typer  # type: ignore[import]
+import typer
 
 from .. import buildall
 

pyodide-build/pyodide_build/cli/config.py

@@ -1,4 +1,4 @@
-import typer  # type: ignore[import]
+import typer
 
 from ..common import get_make_environment_vars
 from ..out_of_tree.utils import initialize_pyodide_root

pyodide-build/pyodide_build/cli/create_zipfile.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+import typer
+
+from ..pyzip import create_zipfile
+
+
+def main(
+    libdir: Path = typer.Argument(
+        ..., help="Path to the directory containing the Python standard library."
+    ),
+    pycompile: bool = typer.Option(
+        False, help="Whether to compile the .py files into .pyc."
+    ),
+    output: Path = typer.Option(
+        "python.zip", help="Path to the output zip file. Defaults to python.zip."
+    ),
+) -> None:
+    """
+    Bundle Python standard libraries into a zip file.
+    """
+    create_zipfile(libdir, output, pycompile=pycompile, filterfunc=None)
+    typer.echo(f"Zip file created at {output}")
+
+
+main.typer_kwargs = {"hidden": True}  # type: ignore[attr-defined]

pyodide-build/pyodide_build/cli/py_compile.py

@@ -1,7 +1,7 @@
 import sys
 from pathlib import Path
 
-import typer  # type: ignore[import]
+import typer
 
 from pyodide_build._py_compile import _py_compile_wheel
 

pyodide-build/pyodide_build/cli/skeleton.py

@@ -3,7 +3,7 @@
 
 from pathlib import Path
 
-import typer  # type: ignore[import]
+import typer
 
 from .. import common, mkpkg
 

pyodide-build/pyodide_build/cli/venv.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-import typer  # type: ignore[import]
+import typer
 
 from ..out_of_tree import venv
 from ..out_of_tree.utils import initialize_pyodide_root

pyodide-build/pyodide_build/pyzip.py

@@ -0,0 +1,137 @@
+import shutil
+from collections.abc import Callable
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+# This files are removed from the stdlib by default
+REMOVED_FILES = (
+    # package management
+    "ensurepip/",
+    "venv/",
+    # build system
+    "lib2to3/",
+    # other platforms
+    "_osx_support.py",
+    # Not supported by browser
+    "curses/",
+    "dbm/",
+    "idlelib/",
+    "tkinter/",
+    "turtle.py",
+    "turtledemo",
+    "webbrowser.py",
+)
+
+# This files are unvendored from the stdlib by default
+UNVENDORED_FILES = (
+    "test/",
+    "distutils/",
+    "sqlite3",
+    "ssl.py",
+    "lzma.py",
+)
+
+# TODO: These modules have test directory which we unvendors it separately.
+#       So we should not pack them into the zip file in order to make e.g. import ctypes.test work.
+#       Note that all these tests are moved to the subdirectory of `test` module in upstream CPython 3.12.0a1.
+#       So we don't need this after we upgrade to 3.12.0
+NOT_ZIPPED_FILES = ("ctypes/", "unittest/")
+
+
+def default_filterfunc(
+    root: Path, verbose: bool = False
+) -> Callable[[str, list[str]], set[str]]:
+    """
+    The default filter function used by `create_zipfile`.
+
+    This function filters out several modules that are:
+
+    - not supported in Pyodide due to browser limitations (e.g. `tkinter`)
+    - unvendored from the standard library (e.g. `sqlite3`)
+    """
+
+    def filterfunc(path: Path | str, names: list[str]) -> set[str]:
+        filtered_files = {
+            (root / f).resolve()
+            for f in REMOVED_FILES + UNVENDORED_FILES + NOT_ZIPPED_FILES
+        }
+
+        path = Path(path).resolve()
+
+        if path.name == "__pycache__":
+            return set(names)
+
+        _names = []
+        for name in names:
+
+            fullpath = path / name
+
+            if fullpath.name == "__pycache__" or fullpath in filtered_files:
+                if verbose:
+                    print(f"Skipping {fullpath}")
+
+                _names.append(name)
+
+        return set(_names)
+
+    return filterfunc
+
+
+def create_zipfile(
+    libdir: Path | str,
+    output: Path | str = "python",
+    pycompile: bool = False,
+    filterfunc: Callable[[str, list[str]], set[str]] | None = None,
+) -> None:
+    """
+    Bundle Python standard libraries into a zip file.
+
+    The basic idea of this function is similar to the standard library's
+    {ref}`zipfile.PyZipFile` class.
+
+    However, we need some additional functionality for Pyodide. For example:
+
+    - We need to remove some unvendored modules, e.g. `sqlite3`
+    - We need an option to "not" compile the files in the zip file
+
+    hence this function.
+
+    Parameters
+    ----------
+    libdir
+        Path to the directory containing the Python standard library.
+
+    output
+        Path to the output zip file. Defaults to python.zip.
+
+    pycompile
+        Whether to compile the .py files into .pyc, by default False
+
+    filterfunc
+        A function that filters the files to be included in the zip file.
+        This function will be passed to {ref}`shutil.copytree` 's ignore argument.
+        By default, Pyodide's default filter function is used.
+
+    Returns
+    -------
+    BytesIO
+        A BytesIO object containing the zip file.
+    """
+
+    if pycompile:
+        raise NotImplementedError(
+            "TODO: implement after https://github.com/pyodide/pyodide/pull/3253 is merged"
+        )
+
+    libdir = Path(libdir)
+    output = Path(output)
+    output = output.with_name(output.name.rstrip(".zip"))
+
+    if filterfunc is None:
+        filterfunc = default_filterfunc(libdir)
+
+    with TemporaryDirectory() as temp_dir_str:
+        temp_dir = Path(temp_dir_str)
+        shutil.copytree(libdir, temp_dir, ignore=filterfunc, dirs_exist_ok=True)
+
+        shutil.make_archive(str(output), "zip", temp_dir)

pyodide-build/pyodide_build/tests/fixture.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture(scope="module")
+def temp_python_lib(tmp_path_factory):
+    libdir = tmp_path_factory.mktemp("python")
+
+    path = Path(libdir)
+
+    (path / "test").mkdir()
+    (path / "test" / "test_blah.py").touch()
+    (path / "distutils").mkdir()
+    (path / "turtle.py").touch()
+
+    (path / "module1.py").touch()
+    (path / "module2.py").touch()
+
+    (path / "hello_pyodide.py").write_text("def hello(): return 'hello'")
+
+    yield libdir

pyodide-build/pyodide_build/tests/test_cli.py

@@ -1,12 +1,17 @@
+# flake8: noqa
+
+import os
 import shutil
 from pathlib import Path
 
 import pytest
-import typer  # type: ignore[import]
+import typer
 from typer.testing import CliRunner  # type: ignore[import]
 
 from pyodide_build import common
-from pyodide_build.cli import build_recipes, config, skeleton
+from pyodide_build.cli import build, build_recipes, config, create_zipfile, skeleton
+
+from .fixture import temp_python_lib
 
 only_node = pytest.mark.xfail_browsers(
     chrome="node only", firefox="node only", safari="node only"
@@ -127,3 +132,52 @@ def test_config_get(cfg_name, env_var):
     )
 
     assert result.stdout.strip() == common.get_make_flag(env_var)
+
+
+def test_fetch_or_build_pypi(selenium, tmp_path):
+    # TODO: Run this test without building Pyodide
+
+    output_dir = tmp_path / "dist"
+    # one pure-python package (doesn't need building) and one sdist package (needs building)
+    pkgs = ["pytest-pyodide", "pycryptodome==3.15.0"]
+
+    os.chdir(tmp_path)
+
+    app = typer.Typer()
+    app.command()(build.main)
+
+    for p in pkgs:
+        result = runner.invoke(
+            app,
+            [p],
+        )
+        assert result.exit_code == 0, result.stdout
+
+    built_wheels = set(output_dir.glob("*.whl"))
+    assert len(built_wheels) == len(pkgs)
+
+
+def test_create_zipfile(temp_python_lib, tmp_path):
+    from zipfile import ZipFile
+
+    output = tmp_path / "python.zip"
+
+    app = typer.Typer()
+    app.command()(create_zipfile.main)
+
+    result = runner.invoke(
+        app,
+        [
+            str(temp_python_lib),
+            "--output",
+            str(output),
+        ],
+    )
+
+    assert result.exit_code == 0, result.stdout
+    assert "Zip file created" in result.stdout
+    assert output.exists()
+
+    with ZipFile(output) as zf:
+        assert "module1.py" in zf.namelist()
+        assert "module2.py" in zf.namelist()

pyodide-build/pyodide_build/tests/test_pypi.py

@@ -10,7 +10,7 @@ from threading import Event, Thread
 from typing import Any
 
 import pytest
-import typer  # type: ignore[import]
+import typer
 from typer.testing import CliRunner  # type: ignore[import]
 
 from pyodide_build.cli import build

pyodide-build/pyodide_build/tests/test_pyzip.py

@@ -0,0 +1,45 @@
+# flake8: noqa
+
+from pyodide_build.pyzip import create_zipfile, default_filterfunc
+
+from .fixture import temp_python_lib
+
+
+def test_defaultfilterfunc(temp_python_lib):
+    filterfunc = default_filterfunc(temp_python_lib, verbose=True)
+
+    ignored = ["test", "distutils", "turtle.py"]
+    assert set(ignored) == filterfunc(str(temp_python_lib), ignored)
+
+    assert set() == filterfunc(str(temp_python_lib), ["hello.py", "world.py"])
+
+
+def test_create_zip(temp_python_lib, tmp_path):
+    from zipfile import ZipFile
+
+    output = tmp_path / "python.zip"
+
+    create_zipfile(temp_python_lib, output, pycompile=False, filterfunc=None)
+
+    assert output.exists()
+
+    with ZipFile(output) as zf:
+        assert "module1.py" in zf.namelist()
+        assert "module2.py" in zf.namelist()
+
+
+def test_import_from_zip(temp_python_lib, tmp_path, monkeypatch):
+    output = tmp_path / "python.zip"
+
+    create_zipfile(temp_python_lib, output, pycompile=False, filterfunc=None)
+
+    assert output.exists()
+
+    import sys
+
+    monkeypatch.setattr(sys, "path", [str(output)])
+
+    import hello_pyodide  # type: ignore[import]
+
+    assert hello_pyodide.__file__.startswith(str(output))
+    assert hello_pyodide.hello() == "hello"

pyodide-build/setup.cfg

@@ -50,6 +50,7 @@ pyodide.cli =
     skeleton = pyodide_build.cli.skeleton:app
     py-compile = pyodide_build.cli.py_compile:main
     config = pyodide_build.cli.config:app
+    create-zipfile = pyodide_build.cli.create_zipfile:main
 
 [options.extras_require]
 test =

pyproject.toml

@@ -34,6 +34,7 @@ module = [
   "virtualenv",
   "termcolor",
   "test",
+  "typer",
 ]
 ignore_missing_imports = true