From 0980b5167737aae2b6e3f7183eec14cb1ee6138b Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Mon, 17 Jul 2023 15:17:24 +0300 Subject: [PATCH] Implement PEP 658: .whl.metadata files (#3981) --- docs/project/changelog.md | 7 +++ packages/Makefile | 1 + pyodide-build/pyodide_build/buildall.py | 22 ++++++- .../pyodide_build/cli/build_recipes.py | 19 +++++- pyodide-build/pyodide_build/common.py | 59 +++++++++++++++++++ .../pyodide_build/tests/test_common.py | 29 +++++++++ tools/deploy_s3.py | 2 + 7 files changed, 133 insertions(+), 6 deletions(-) diff --git a/docs/project/changelog.md b/docs/project/changelog.md index ddef7c969..6605afbbe 100644 --- a/docs/project/changelog.md +++ b/docs/project/changelog.md @@ -68,6 +68,13 @@ myst: - Upgraded scikit-learn to 1.3.0 {pr}`3976` - Upgraded pyodide-http to 0.2.1 +### CLI + +- {{ Enhancement }} `pyodide build-recipes` now accepts a `--metadata-files` + option to install `*.whl.metadata` files as specified in + [PEP 658](https://peps.python.org/pep-0658/). + {pr}`3981` + ## Version 0.23.4 _July 6, 2023_ diff --git a/packages/Makefile b/packages/Makefile index e7040ea7b..f6e8cf816 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -10,6 +10,7 @@ all: --recipe-dir=./ \ --install \ --install-dir=../dist \ + --metadata-files \ --n-jobs $${PYODIDE_JOBS:-4} \ --log-dir=./build-logs \ --compression-level "$(PYODIDE_ZIP_COMPRESSION_LEVEL)" diff --git a/pyodide-build/pyodide_build/buildall.py b/pyodide-build/pyodide_build/buildall.py index d90a430f4..347ae3b3a 100755 --- a/pyodide-build/pyodide_build/buildall.py +++ b/pyodide-build/pyodide_build/buildall.py @@ -29,6 +29,7 @@ from rich.table import Table from . import build_env, recipe from .buildpkg import needs_rebuild from .common import ( + extract_wheel_metadata_file, find_matching_wheels, find_missing_executables, repack_zip_archive, @@ -706,7 +707,10 @@ def generate_lockfile( def copy_packages_to_dist_dir( - packages: Iterable[BasePackage], output_dir: Path, compression_level: int = 6 + packages: Iterable[BasePackage], + output_dir: Path, + compression_level: int = 6, + metadata_files: bool = False, ) -> None: for pkg in packages: if pkg.package_type == "static_library": @@ -719,6 +723,12 @@ def copy_packages_to_dist_dir( output_dir / dist_artifact_path.name, compression_level=compression_level ) + if metadata_files and dist_artifact_path.suffix == ".whl": + extract_wheel_metadata_file( + dist_artifact_path, + output_dir / f"{dist_artifact_path.name}.metadata", + ) + test_path = pkg.tests_path() if test_path: shutil.copy(test_path, output_dir) @@ -773,7 +783,10 @@ def copy_logs(pkg_map: dict[str, BasePackage], log_dir: Path) -> None: def install_packages( - pkg_map: dict[str, BasePackage], output_dir: Path, compression_level: int = 6 + pkg_map: dict[str, BasePackage], + output_dir: Path, + compression_level: int = 6, + metadata_files: bool = False, ) -> None: """ Install packages into the output directory. @@ -792,7 +805,10 @@ def install_packages( logger.info(f"Copying built packages to {output_dir}") copy_packages_to_dist_dir( - pkg_map.values(), output_dir, compression_level=compression_level + pkg_map.values(), + output_dir, + compression_level=compression_level, + metadata_files=metadata_files, ) lockfile_path = output_dir / "pyodide-lock.json" diff --git a/pyodide-build/pyodide_build/cli/build_recipes.py b/pyodide-build/pyodide_build/cli/build_recipes.py index b8140cfcc..d42e62c7f 100644 --- a/pyodide-build/pyodide_build/cli/build_recipes.py +++ b/pyodide-build/pyodide_build/cli/build_recipes.py @@ -30,6 +30,12 @@ def recipe( help="Path to install built packages and pyodide-lock.json. " "If not specified, the default is `./dist`.", ), + metadata_files: bool = typer.Option( + False, + help="If true, extract the METADATA file from the built wheels " + "to a matching *.whl.metadata file. " + "If false, no *.whl.metadata file is produced.", + ), cflags: str = typer.Option( None, help="Extra compiling flags. Default: SIDE_MODULE_CFLAGS" ), @@ -91,9 +97,9 @@ def recipe( build_args = buildall.set_default_build_args(build_args) if no_deps: - if install or log_dir_: + if install or log_dir_ or metadata_files: logger.warning( - "WARNING: when --no-deps is set, --install and --log-dir parameters are ignored", + "WARNING: when --no-deps is set, the --install, --log-dir, and --metadata-files parameters are ignored", ) # TODO: use multiprocessing? @@ -118,5 +124,12 @@ def recipe( if install: buildall.install_packages( - pkg_map, install_dir_, compression_level=compression_level + pkg_map, + install_dir_, + compression_level=compression_level, + metadata_files=metadata_files, + ) + elif metadata_files: + logger.warning( + "WARNING: when --install is not set, the --metadata-files parameter is ignored", ) diff --git a/pyodide-build/pyodide_build/common.py b/pyodide-build/pyodide_build/common.py index 77eb168e5..534b55920 100644 --- a/pyodide-build/pyodide_build/common.py +++ b/pyodide-build/pyodide_build/common.py @@ -16,8 +16,10 @@ from contextlib import contextmanager from pathlib import Path from tempfile import TemporaryDirectory from typing import Any, NoReturn +from zipfile import ZipFile from packaging.tags import Tag +from packaging.utils import canonicalize_name as canonicalize_package_name from packaging.utils import parse_wheel_filename from .logger import logger @@ -312,3 +314,60 @@ def modify_wheel(wheel: Path) -> Iterator[Path]: yield wheel_dir wheel.unlink() pack_wheel(wheel_dir, wheel.parent) + + +def extract_wheel_metadata_file(wheel_path: Path, output_path: Path) -> None: + """Extracts the METADATA file from the given wheel and writes it to the + output path. + + Raises an exception if the METADATA file does not exist. + + For a wheel called "NAME-VERSION-...", the METADATA file is expected to be + found in a directory inside the wheel archive, whose name starts with NAME + and ends with ".dist-info". See: + https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-contents + """ + with ZipFile(wheel_path, mode="r") as wheel: + pkg_name = wheel_path.name.split("-", 1)[0] + dist_info_dir = get_wheel_dist_info_dir(wheel, pkg_name) + metadata_path = f"{dist_info_dir}/METADATA" + try: + wheel.getinfo(metadata_path).filename = output_path.name + wheel.extract(metadata_path, output_path.parent) + except KeyError as err: + raise Exception(f"METADATA file not found for {pkg_name}") from err + + +def get_wheel_dist_info_dir(wheel: ZipFile, pkg_name: str) -> str: + """Returns the path of the contained .dist-info directory. + + Raises an Exception if the directory is not found, more than + one is found, or it does not match the provided `pkg_name`. + + Adapted from: + https://github.com/pypa/pip/blob/ea727e4d6ab598f34f97c50a22350febc1214a97/src/pip/_internal/utils/wheel.py#L38 + """ + + # Zip file path separators must be / + subdirs = {name.split("/", 1)[0] for name in wheel.namelist()} + info_dirs = [subdir for subdir in subdirs if subdir.endswith(".dist-info")] + + if len(info_dirs) == 0: + raise Exception(f".dist-info directory not found for {pkg_name}") + + if len(info_dirs) > 1: + raise Exception( + f"multiple .dist-info directories found for {pkg_name}: {', '.join(info_dirs)}" + ) + + (info_dir,) = info_dirs + + info_dir_name = canonicalize_package_name(info_dir) + canonical_name = canonicalize_package_name(pkg_name) + + if not info_dir_name.startswith(canonical_name): + raise Exception( + f".dist-info directory {info_dir!r} does not start with {canonical_name!r}" + ) + + return info_dir diff --git a/pyodide-build/pyodide_build/tests/test_common.py b/pyodide-build/pyodide_build/tests/test_common.py index 45982713a..2dd8fffdd 100644 --- a/pyodide-build/pyodide_build/tests/test_common.py +++ b/pyodide-build/pyodide_build/tests/test_common.py @@ -4,6 +4,7 @@ import pytest from pyodide_build.common import ( environment_substitute_args, + extract_wheel_metadata_file, find_missing_executables, get_num_cores, make_zip_archive, @@ -123,3 +124,31 @@ def test_repack_zip_archive( assert fh.namelist() == ["a/b.txt", "a/b/c.txt"] assert fh.getinfo("a/b.txt").compress_type == expected_compression_type assert input_path.stat().st_size == expected_size + + +def test_extract_wheel_metadata_file(tmp_path): + # Test extraction if metadata exists + + input_path = tmp_path / "pkg-0.1-abc.whl" + metadata_path = "pkg-0.1.dist-info/METADATA" + metadata_str = "This is METADATA" + + with zipfile.ZipFile(input_path, "w") as fh: + fh.writestr(metadata_path, metadata_str) + + output_path = tmp_path / f"{input_path.name}.metadata" + + extract_wheel_metadata_file(input_path, output_path) + assert output_path.read_text() == metadata_str + + # Test extraction if metadata is missing + + input_path_empty = tmp_path / "pkg-0.2-abc.whl" + + with zipfile.ZipFile(input_path_empty, "w") as fh: + pass + + output_path_empty = tmp_path / f"{input_path_empty.name}.metadata" + + with pytest.raises(Exception): + extract_wheel_metadata_file(input_path_empty, output_path_empty) diff --git a/tools/deploy_s3.py b/tools/deploy_s3.py index 9b6be7937..859da7159 100644 --- a/tools/deploy_s3.py +++ b/tools/deploy_s3.py @@ -137,6 +137,8 @@ def deploy_to_s3_main( # However, JsDelivr will currently not serve .ts file in the # custom CDN configuration, so it does not really matter. content_type = "text/x.typescript" + elif file_path.name.endswith(".whl.metadata"): + content_type = "text/plain" else: content_type = mimetypes.guess_type(file_path)[0] if content_type is None: