diff --git a/Makefile b/Makefile index e42b8d98f..d1a082806 100644 --- a/Makefile +++ b/Makefile @@ -228,7 +228,7 @@ rwildcard=$(wildcard $1) $(foreach d,$1,$(call rwildcard,$(addsuffix /$(notdir $ dist/python_stdlib.zip: $(call rwildcard,src/py/*) $(CPYTHONLIB) make pyodide_build - pyodide create-zipfile $(CPYTHONLIB) src/py --compression-level "$(PYODIDE_ZIP_COMPRESSION_LEVEL)" --output $@ + pyodide create-zipfile $(CPYTHONLIB) src/py --exclude "$(PYZIP_EXCLUDE_FILES)" --stub "$(PYZIP_JS_STUBS)" --compression-level "$(PYODIDE_ZIP_COMPRESSION_LEVEL)" --output $@ dist/test.html: src/templates/test.html cp $< $@ diff --git a/Makefile.envs b/Makefile.envs index 582dcdcc8..9166d1550 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -52,6 +52,30 @@ export PYODIDE_ZIP_COMPRESSION_LEVEL?=6 export PIP_CONSTRAINT=$(PYODIDE_ROOT)/tools/constraints.txt +# List of modules to exclude from the zipped standard library +export PYZIP_EXCLUDE_FILES=\ + ensurepip/ \ + venv/ \ + lib2to3/ \ + _osx_support.py \ + _aix_support.py \ + curses/ \ + dbm/ \ + idlelib/ \ + tkinter/ \ + turtle.py \ + turtledemo/ \ + test/ \ + sqlite3/ \ + ssl.py \ + lzma.py \ + _pydecimal.py \ + pydoc_data/ + +# List of modules that we replace with a stub in the zipped standard library +export PYZIP_JS_STUBS=\ + webbrowser.py + export DBGFLAGS_NODEBUG=-g0 export DBGFLAGS_WASMDEBUG=-g2 export DBGFLAGS_SOURCEMAPDEBUG=-g3 diff --git a/pyodide-build/pyodide_build/cli/create_zipfile.py b/pyodide-build/pyodide_build/cli/create_zipfile.py index 3b9ab4b3d..2593ef848 100644 --- a/pyodide-build/pyodide_build/cli/create_zipfile.py +++ b/pyodide-build/pyodide_build/cli/create_zipfile.py @@ -1,3 +1,4 @@ +import re from pathlib import Path import typer @@ -10,6 +11,14 @@ def main( ..., help="List of paths to the directory containing the Python standard library or extra packages.", ), + exclude: str = typer.Option( + "", + help="List of files to exclude from the zip file. Defaults to no files.", + ), + stub: str = typer.Option( + "", + help="List of files that are replaced by JS implementations. Defaults to no files.", + ), pycompile: bool = typer.Option( False, help="Whether to compile the .py files into .pyc." ), @@ -23,8 +32,17 @@ def main( """ Bundle Python standard libraries into a zip file. """ + + # Convert the comma / space separated strings to lists + excludes = [ + item.strip() for item in re.split(r",|\s", exclude) if item.strip() != "" + ] + stubs = [item.strip() for item in re.split(r",|\s", stub) if item.strip() != ""] + create_zipfile( libdir, + excludes, + stubs, output, pycompile=pycompile, filterfunc=None, diff --git a/pyodide-build/pyodide_build/pyzip.py b/pyodide-build/pyodide_build/pyzip.py index f537d4664..67a55f2a9 100644 --- a/pyodide-build/pyodide_build/pyzip.py +++ b/pyodide-build/pyodide_build/pyzip.py @@ -6,41 +6,9 @@ from tempfile import TemporaryDirectory from ._py_compile import _compile from .common import make_zip_archive -# These files are removed from the stdlib -REMOVED_FILES = ( - # package management - "ensurepip/", - "venv/", - # build system - "lib2to3/", - # other platforms - "_osx_support.py", - "_aix_support.py", - # Not supported by browser - "curses/", - "dbm/", - "idlelib/", - "tkinter/", - "turtle.py", - "turtledemo", -) - -# These files are unvendored from the stdlib and can be loaded with `loadPackage` -UNVENDORED_FILES = ( - "test/", - "sqlite3", - "ssl.py", - "lzma.py", - "_pydecimal.py", - "pydoc_data", -) - -# We have JS implementations of these modules -JS_STUB_FILES = ("webbrowser.py",) - def default_filterfunc( - root: Path, verbose: bool = False + root: Path, excludes: list[str], stubs: list[str], verbose: bool = False ) -> Callable[[str, list[str]], set[str]]: """ The default filter function used by `create_zipfile`. @@ -75,15 +43,13 @@ def default_filterfunc( return False def filterfunc(path: Path | str, names: list[str]) -> set[str]: - filtered_files = { - (root / f).resolve() for f in REMOVED_FILES + UNVENDORED_FILES - } + filtered_files = {(root / f).resolve() for f in excludes} # We have JS implementations of these modules, so we don't need to # include the Python ones. Checking the name of the root directory # is a bit of a hack, but it works... if root.name.startswith("python3"): - filtered_files.update({root / f for f in JS_STUB_FILES}) + filtered_files.update({root / f for f in stubs}) path = Path(path).resolve() @@ -107,6 +73,8 @@ def default_filterfunc( def create_zipfile( libdirs: list[Path], + excludes: list[str] | None = None, + stubs: list[str] | None = None, output: Path | str = "python", pycompile: bool = False, filterfunc: Callable[[str, list[str]], set[str]] | None = None, @@ -130,6 +98,12 @@ def create_zipfile( libdirs List of paths to the directory containing the Python standard library or extra packages. + excludes + List of files to exclude from the zip file. + + stubs + List of files that are replaced by JS implementations. + output Path to the output zip file. Defaults to python.zip. @@ -152,6 +126,8 @@ def create_zipfile( """ archive = Path(output) + excludes = excludes or [] + stubs = stubs or [] with TemporaryDirectory() as temp_dir_str: temp_dir = Path(temp_dir_str) @@ -160,7 +136,7 @@ def create_zipfile( libdir = Path(libdir) if filterfunc is None: - _filterfunc = default_filterfunc(libdir) + _filterfunc = default_filterfunc(libdir, excludes, stubs) shutil.copytree(libdir, temp_dir, ignore=_filterfunc, dirs_exist_ok=True) diff --git a/pyodide-build/pyodide_build/tests/test_pyzip.py b/pyodide-build/pyodide_build/tests/test_pyzip.py index 572f89d4f..bf0f6e32b 100644 --- a/pyodide-build/pyodide_build/tests/test_pyzip.py +++ b/pyodide-build/pyodide_build/tests/test_pyzip.py @@ -6,9 +6,11 @@ from .fixture import temp_python_lib, temp_python_lib2 def test_defaultfilterfunc(temp_python_lib): - filterfunc = default_filterfunc(temp_python_lib, verbose=True) - ignored = ["test", "turtle.py"] + filterfunc = default_filterfunc( + temp_python_lib, excludes=ignored, stubs=[], verbose=True + ) + assert set(ignored) == filterfunc(str(temp_python_lib), ignored) assert set() == filterfunc(str(temp_python_lib), ["hello.py", "world.py"]) @@ -19,7 +21,14 @@ def test_create_zip(temp_python_lib, tmp_path): output = tmp_path / "python.zip" - create_zipfile([temp_python_lib], output, pycompile=False, filterfunc=None) + create_zipfile( + [temp_python_lib], + excludes=[], + stubs=[], + output=output, + pycompile=False, + filterfunc=None, + ) assert output.exists() @@ -33,7 +42,14 @@ def test_create_zip_compile(temp_python_lib, tmp_path): output = tmp_path / "python.zip" - create_zipfile([temp_python_lib], output, pycompile=True, filterfunc=None) + create_zipfile( + [temp_python_lib], + excludes=[], + stubs=[], + output=output, + pycompile=True, + filterfunc=None, + ) assert output.exists() @@ -46,7 +62,12 @@ def test_import_from_zip(temp_python_lib, temp_python_lib2, tmp_path, monkeypatc output = tmp_path / "python.zip" create_zipfile( - [temp_python_lib, temp_python_lib2], output, pycompile=False, filterfunc=None + [temp_python_lib, temp_python_lib2], + excludes=[], + stubs=[], + output=output, + pycompile=False, + filterfunc=None, ) assert output.exists()