Implement PEP 658: .whl.metadata files (#3981)

docs/project/changelog.md

@@ -68,6 +68,13 @@ myst:
 - Upgraded scikit-learn to 1.3.0 {pr}`3976`
 - Upgraded pyodide-http to 0.2.1
 
+### CLI
+
+- {{ Enhancement }} `pyodide build-recipes` now accepts a `--metadata-files`
+  option to install `*.whl.metadata` files as specified in
+  [PEP 658](https://peps.python.org/pep-0658/).
+  {pr}`3981`
+
 ## Version 0.23.4
 
 _July 6, 2023_

packages/Makefile

@@ -10,6 +10,7 @@ all:
 	--recipe-dir=./ \
 	--install \
 	--install-dir=../dist \
+	--metadata-files \
 	--n-jobs $${PYODIDE_JOBS:-4} \
 	--log-dir=./build-logs \
 	--compression-level "$(PYODIDE_ZIP_COMPRESSION_LEVEL)"

pyodide-build/pyodide_build/buildall.py

@@ -29,6 +29,7 @@ from rich.table import Table
 from . import build_env, recipe
 from .buildpkg import needs_rebuild
 from .common import (
+    extract_wheel_metadata_file,
     find_matching_wheels,
     find_missing_executables,
     repack_zip_archive,
@@ -706,7 +707,10 @@ def generate_lockfile(
 
 
 def copy_packages_to_dist_dir(
-    packages: Iterable[BasePackage], output_dir: Path, compression_level: int = 6
+    packages: Iterable[BasePackage],
+    output_dir: Path,
+    compression_level: int = 6,
+    metadata_files: bool = False,
 ) -> None:
     for pkg in packages:
         if pkg.package_type == "static_library":
@@ -719,6 +723,12 @@ def copy_packages_to_dist_dir(
             output_dir / dist_artifact_path.name, compression_level=compression_level
         )
 
+        if metadata_files and dist_artifact_path.suffix == ".whl":
+            extract_wheel_metadata_file(
+                dist_artifact_path,
+                output_dir / f"{dist_artifact_path.name}.metadata",
+            )
+
         test_path = pkg.tests_path()
         if test_path:
             shutil.copy(test_path, output_dir)
@@ -773,7 +783,10 @@ def copy_logs(pkg_map: dict[str, BasePackage], log_dir: Path) -> None:
 
 
 def install_packages(
-    pkg_map: dict[str, BasePackage], output_dir: Path, compression_level: int = 6
+    pkg_map: dict[str, BasePackage],
+    output_dir: Path,
+    compression_level: int = 6,
+    metadata_files: bool = False,
 ) -> None:
     """
     Install packages into the output directory.
@@ -792,7 +805,10 @@ def install_packages(
 
     logger.info(f"Copying built packages to {output_dir}")
     copy_packages_to_dist_dir(
-        pkg_map.values(), output_dir, compression_level=compression_level
+        pkg_map.values(),
+        output_dir,
+        compression_level=compression_level,
+        metadata_files=metadata_files,
     )
 
     lockfile_path = output_dir / "pyodide-lock.json"

pyodide-build/pyodide_build/cli/build_recipes.py

@@ -30,6 +30,12 @@ def recipe(
         help="Path to install built packages and pyodide-lock.json. "
         "If not specified, the default is `./dist`.",
     ),
+    metadata_files: bool = typer.Option(
+        False,
+        help="If true, extract the METADATA file from the built wheels "
+        "to a matching *.whl.metadata file. "
+        "If false, no *.whl.metadata file is produced.",
+    ),
     cflags: str = typer.Option(
         None, help="Extra compiling flags. Default: SIDE_MODULE_CFLAGS"
     ),
@@ -91,9 +97,9 @@ def recipe(
     build_args = buildall.set_default_build_args(build_args)
 
     if no_deps:
-        if install or log_dir_:
+        if install or log_dir_ or metadata_files:
             logger.warning(
-                "WARNING: when --no-deps is set, --install and --log-dir parameters are ignored",
+                "WARNING: when --no-deps is set, the --install, --log-dir, and --metadata-files parameters are ignored",
             )
 
         # TODO: use multiprocessing?
@@ -118,5 +124,12 @@ def recipe(
 
         if install:
             buildall.install_packages(
-                pkg_map, install_dir_, compression_level=compression_level
+                pkg_map,
+                install_dir_,
+                compression_level=compression_level,
+                metadata_files=metadata_files,
+            )
+        elif metadata_files:
+            logger.warning(
+                "WARNING: when --install is not set, the --metadata-files parameter is ignored",
             )

pyodide-build/pyodide_build/common.py

@@ -16,8 +16,10 @@ from contextlib import contextmanager
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, NoReturn
+from zipfile import ZipFile
 
 from packaging.tags import Tag
+from packaging.utils import canonicalize_name as canonicalize_package_name
 from packaging.utils import parse_wheel_filename
 
 from .logger import logger
@@ -312,3 +314,60 @@ def modify_wheel(wheel: Path) -> Iterator[Path]:
         yield wheel_dir
         wheel.unlink()
         pack_wheel(wheel_dir, wheel.parent)
+
+
+def extract_wheel_metadata_file(wheel_path: Path, output_path: Path) -> None:
+    """Extracts the METADATA file from the given wheel and writes it to the
+    output path.
+
+    Raises an exception if the METADATA file does not exist.
+
+    For a wheel called "NAME-VERSION-...", the METADATA file is expected to be
+    found in a directory inside the wheel archive, whose name starts with NAME
+    and ends with ".dist-info". See:
+    https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-contents
+    """
+    with ZipFile(wheel_path, mode="r") as wheel:
+        pkg_name = wheel_path.name.split("-", 1)[0]
+        dist_info_dir = get_wheel_dist_info_dir(wheel, pkg_name)
+        metadata_path = f"{dist_info_dir}/METADATA"
+        try:
+            wheel.getinfo(metadata_path).filename = output_path.name
+            wheel.extract(metadata_path, output_path.parent)
+        except KeyError as err:
+            raise Exception(f"METADATA file not found for {pkg_name}") from err
+
+
+def get_wheel_dist_info_dir(wheel: ZipFile, pkg_name: str) -> str:
+    """Returns the path of the contained .dist-info directory.
+
+    Raises an Exception if the directory is not found, more than
+    one is found, or it does not match the provided `pkg_name`.
+
+    Adapted from:
+    https://github.com/pypa/pip/blob/ea727e4d6ab598f34f97c50a22350febc1214a97/src/pip/_internal/utils/wheel.py#L38
+    """
+
+    # Zip file path separators must be /
+    subdirs = {name.split("/", 1)[0] for name in wheel.namelist()}
+    info_dirs = [subdir for subdir in subdirs if subdir.endswith(".dist-info")]
+
+    if len(info_dirs) == 0:
+        raise Exception(f".dist-info directory not found for {pkg_name}")
+
+    if len(info_dirs) > 1:
+        raise Exception(
+            f"multiple .dist-info directories found for {pkg_name}: {', '.join(info_dirs)}"
+        )
+
+    (info_dir,) = info_dirs
+
+    info_dir_name = canonicalize_package_name(info_dir)
+    canonical_name = canonicalize_package_name(pkg_name)
+
+    if not info_dir_name.startswith(canonical_name):
+        raise Exception(
+            f".dist-info directory {info_dir!r} does not start with {canonical_name!r}"
+        )
+
+    return info_dir

pyodide-build/pyodide_build/tests/test_common.py

@@ -4,6 +4,7 @@ import pytest
 
 from pyodide_build.common import (
     environment_substitute_args,
+    extract_wheel_metadata_file,
     find_missing_executables,
     get_num_cores,
     make_zip_archive,
@@ -123,3 +124,31 @@ def test_repack_zip_archive(
         assert fh.namelist() == ["a/b.txt", "a/b/c.txt"]
         assert fh.getinfo("a/b.txt").compress_type == expected_compression_type
     assert input_path.stat().st_size == expected_size
+
+
+def test_extract_wheel_metadata_file(tmp_path):
+    # Test extraction if metadata exists
+
+    input_path = tmp_path / "pkg-0.1-abc.whl"
+    metadata_path = "pkg-0.1.dist-info/METADATA"
+    metadata_str = "This is METADATA"
+
+    with zipfile.ZipFile(input_path, "w") as fh:
+        fh.writestr(metadata_path, metadata_str)
+
+    output_path = tmp_path / f"{input_path.name}.metadata"
+
+    extract_wheel_metadata_file(input_path, output_path)
+    assert output_path.read_text() == metadata_str
+
+    # Test extraction if metadata is missing
+
+    input_path_empty = tmp_path / "pkg-0.2-abc.whl"
+
+    with zipfile.ZipFile(input_path_empty, "w") as fh:
+        pass
+
+    output_path_empty = tmp_path / f"{input_path_empty.name}.metadata"
+
+    with pytest.raises(Exception):
+        extract_wheel_metadata_file(input_path_empty, output_path_empty)

tools/deploy_s3.py

@@ -137,6 +137,8 @@ def deploy_to_s3_main(
                 # However, JsDelivr will currently not serve .ts file in the
                 # custom CDN configuration, so it does not really matter.
                 content_type = "text/x.typescript"
+            elif file_path.name.endswith(".whl.metadata"):
+                content_type = "text/plain"
             else:
                 content_type = mimetypes.guess_type(file_path)[0]
                 if content_type is None: