Update `pyodide create-zipfile` command to accept list of modules to exclude (#4723)

Makefile

@@ -228,7 +228,7 @@ rwildcard=$(wildcard $1) $(foreach d,$1,$(call rwildcard,$(addsuffix /$(notdir $
 
 dist/python_stdlib.zip: $(call rwildcard,src/py/*) $(CPYTHONLIB)
 	make pyodide_build
-	pyodide create-zipfile $(CPYTHONLIB) src/py --compression-level "$(PYODIDE_ZIP_COMPRESSION_LEVEL)" --output $@
+	pyodide create-zipfile $(CPYTHONLIB) src/py --exclude "$(PYZIP_EXCLUDE_FILES)" --stub "$(PYZIP_JS_STUBS)" --compression-level "$(PYODIDE_ZIP_COMPRESSION_LEVEL)" --output $@
 
 dist/test.html: src/templates/test.html
 	cp $< $@

Makefile.envs

@@ -52,6 +52,30 @@ export PYODIDE_ZIP_COMPRESSION_LEVEL?=6
 
 export PIP_CONSTRAINT=$(PYODIDE_ROOT)/tools/constraints.txt
 
+# List of modules to exclude from the zipped standard library
+export PYZIP_EXCLUDE_FILES=\
+	ensurepip/ \
+	venv/ \
+	lib2to3/ \
+	_osx_support.py \
+	_aix_support.py \
+	curses/ \
+	dbm/ \
+	idlelib/ \
+	tkinter/ \
+	turtle.py \
+	turtledemo/ \
+	test/ \
+	sqlite3/ \
+	ssl.py \
+	lzma.py \
+	_pydecimal.py \
+	pydoc_data/
+
+# List of modules that we replace with a stub in the zipped standard library
+export PYZIP_JS_STUBS=\
+	webbrowser.py
+
 export DBGFLAGS_NODEBUG=-g0
 export DBGFLAGS_WASMDEBUG=-g2
 export DBGFLAGS_SOURCEMAPDEBUG=-g3

pyodide-build/pyodide_build/cli/create_zipfile.py

@@ -1,3 +1,4 @@
+import re
 from pathlib import Path
 
 import typer
@@ -10,6 +11,14 @@ def main(
         ...,
         help="List of paths to the directory containing the Python standard library or extra packages.",
     ),
+    exclude: str = typer.Option(
+        "",
+        help="List of files to exclude from the zip file. Defaults to no files.",
+    ),
+    stub: str = typer.Option(
+        "",
+        help="List of files that are replaced by JS implementations. Defaults to no files.",
+    ),
     pycompile: bool = typer.Option(
         False, help="Whether to compile the .py files into .pyc."
     ),
@@ -23,8 +32,17 @@ def main(
     """
     Bundle Python standard libraries into a zip file.
     """
+
+    # Convert the comma / space separated strings to lists
+    excludes = [
+        item.strip() for item in re.split(r",|\s", exclude) if item.strip() != ""
+    ]
+    stubs = [item.strip() for item in re.split(r",|\s", stub) if item.strip() != ""]
+
     create_zipfile(
         libdir,
+        excludes,
+        stubs,
         output,
         pycompile=pycompile,
         filterfunc=None,

pyodide-build/pyodide_build/pyzip.py

@@ -6,41 +6,9 @@ from tempfile import TemporaryDirectory
 from ._py_compile import _compile
 from .common import make_zip_archive
 
-# These files are removed from the stdlib
-REMOVED_FILES = (
-    # package management
-    "ensurepip/",
-    "venv/",
-    # build system
-    "lib2to3/",
-    # other platforms
-    "_osx_support.py",
-    "_aix_support.py",
-    # Not supported by browser
-    "curses/",
-    "dbm/",
-    "idlelib/",
-    "tkinter/",
-    "turtle.py",
-    "turtledemo",
-)
-
-# These files are unvendored from the stdlib and can be loaded with `loadPackage`
-UNVENDORED_FILES = (
-    "test/",
-    "sqlite3",
-    "ssl.py",
-    "lzma.py",
-    "_pydecimal.py",
-    "pydoc_data",
-)
-
-# We have JS implementations of these modules
-JS_STUB_FILES = ("webbrowser.py",)
-
 
 def default_filterfunc(
-    root: Path, verbose: bool = False
+    root: Path, excludes: list[str], stubs: list[str], verbose: bool = False
 ) -> Callable[[str, list[str]], set[str]]:
     """
     The default filter function used by `create_zipfile`.
@@ -75,15 +43,13 @@ def default_filterfunc(
         return False
 
     def filterfunc(path: Path | str, names: list[str]) -> set[str]:
-        filtered_files = {
-            (root / f).resolve() for f in REMOVED_FILES + UNVENDORED_FILES
-        }
+        filtered_files = {(root / f).resolve() for f in excludes}
 
         # We have JS implementations of these modules, so we don't need to
         # include the Python ones. Checking the name of the root directory
         # is a bit of a hack, but it works...
         if root.name.startswith("python3"):
-            filtered_files.update({root / f for f in JS_STUB_FILES})
+            filtered_files.update({root / f for f in stubs})
 
         path = Path(path).resolve()
 
@@ -107,6 +73,8 @@ def default_filterfunc(
 
 def create_zipfile(
     libdirs: list[Path],
+    excludes: list[str] | None = None,
+    stubs: list[str] | None = None,
     output: Path | str = "python",
     pycompile: bool = False,
     filterfunc: Callable[[str, list[str]], set[str]] | None = None,
@@ -130,6 +98,12 @@ def create_zipfile(
     libdirs
         List of paths to the directory containing the Python standard library or extra packages.
 
+    excludes
+        List of files to exclude from the zip file.
+
+    stubs
+        List of files that are replaced by JS implementations.
+
     output
         Path to the output zip file. Defaults to python.zip.
 
@@ -152,6 +126,8 @@ def create_zipfile(
     """
 
     archive = Path(output)
+    excludes = excludes or []
+    stubs = stubs or []
 
     with TemporaryDirectory() as temp_dir_str:
         temp_dir = Path(temp_dir_str)
@@ -160,7 +136,7 @@ def create_zipfile(
             libdir = Path(libdir)
 
             if filterfunc is None:
-                _filterfunc = default_filterfunc(libdir)
+                _filterfunc = default_filterfunc(libdir, excludes, stubs)
 
             shutil.copytree(libdir, temp_dir, ignore=_filterfunc, dirs_exist_ok=True)
 

pyodide-build/pyodide_build/tests/test_pyzip.py

@@ -6,9 +6,11 @@ from .fixture import temp_python_lib, temp_python_lib2
 
 
 def test_defaultfilterfunc(temp_python_lib):
-    filterfunc = default_filterfunc(temp_python_lib, verbose=True)
-
     ignored = ["test", "turtle.py"]
+    filterfunc = default_filterfunc(
+        temp_python_lib, excludes=ignored, stubs=[], verbose=True
+    )
+
     assert set(ignored) == filterfunc(str(temp_python_lib), ignored)
 
     assert set() == filterfunc(str(temp_python_lib), ["hello.py", "world.py"])
@@ -19,7 +21,14 @@ def test_create_zip(temp_python_lib, tmp_path):
 
     output = tmp_path / "python.zip"
 
-    create_zipfile([temp_python_lib], output, pycompile=False, filterfunc=None)
+    create_zipfile(
+        [temp_python_lib],
+        excludes=[],
+        stubs=[],
+        output=output,
+        pycompile=False,
+        filterfunc=None,
+    )
 
     assert output.exists()
 
@@ -33,7 +42,14 @@ def test_create_zip_compile(temp_python_lib, tmp_path):
 
     output = tmp_path / "python.zip"
 
-    create_zipfile([temp_python_lib], output, pycompile=True, filterfunc=None)
+    create_zipfile(
+        [temp_python_lib],
+        excludes=[],
+        stubs=[],
+        output=output,
+        pycompile=True,
+        filterfunc=None,
+    )
 
     assert output.exists()
 
@@ -46,7 +62,12 @@ def test_import_from_zip(temp_python_lib, temp_python_lib2, tmp_path, monkeypatc
     output = tmp_path / "python.zip"
 
     create_zipfile(
-        [temp_python_lib, temp_python_lib2], output, pycompile=False, filterfunc=None
+        [temp_python_lib, temp_python_lib2],
+        excludes=[],
+        stubs=[],
+        output=output,
+        pycompile=False,
+        filterfunc=None,
     )
 
     assert output.exists()