Implement more accurate wheel finding using the packaging tags module (#2275)

This adds a find_matching_wheels function that correctly determines
which wheels the Pyodide interpreter supports according to the platform
compatibility tags spec. This should fix problems with extra host platform
wheels lying around or wheels built for different versions of Python (e.g., 
cp39 vs cp310).

pyodide-build/pyodide_build/buildall.py

@@ -19,7 +19,7 @@ from typing import Any, Optional
 
 from . import common
 from .buildpkg import needs_rebuild
-from .common import UNVENDORED_STDLIB_MODULES
+from .common import UNVENDORED_STDLIB_MODULES, find_matching_wheels
 from .io import parse_package_config
 
 
@@ -103,14 +103,12 @@ class Package(BasePackage):
 
     def wheel_path(self) -> Path:
         dist_dir = self.pkgdir / "dist"
-        wheels = list(dist_dir.glob("*emscripten_wasm32.whl")) + list(
-            dist_dir.glob("*py3-none-any.whl")
-        )
-        if len(wheels) != 1:
+        wheel, *rest = find_matching_wheels(dist_dir.glob("*.whl"))
+        if rest:
             raise Exception(
-                f"Unexpected number of wheels {len(wheels)} when building {self.name}"
+                f"Unexpected number of wheels {len(rest) + 1} when building {self.name}"
             )
-        return wheels[0]
+        return wheel
 
     def tests_path(self) -> Optional[Path]:
         tests = list((self.pkgdir / "dist").glob("*-tests.tar"))

pyodide-build/pyodide_build/buildpkg.py

@@ -23,6 +23,7 @@ from typing import Any, NoReturn, Optional, TextIO
 from urllib import request
 
 from . import pywasmcross
+from .common import find_matching_wheels
 
 
 @contextmanager
@@ -466,11 +467,16 @@ def package_wheel(
         return
 
     distdir = srcpath / "dist"
-    wheel_paths = list(distdir.glob("*.whl"))
-    assert len(wheel_paths) == 1
-    unpack_wheel(wheel_paths[0])
-    wheel_paths[0].unlink()
-    wheel_dir = next(p for p in distdir.glob("*") if p.is_dir())
+    wheel, *rest = find_matching_wheels(distdir.glob("*.whl"))
+    if rest:
+        raise Exception(
+            f"Unexpected number of wheels {len(rest) + 1} when building {pkg_name}"
+        )
+    unpack_wheel(wheel)
+    wheel.unlink()
+    name, ver, _ = wheel.name.split("-", 2)
+    wheel_dir_name = f"{name}-{ver}"
+    wheel_dir = distdir / wheel_dir_name
 
     post = build_metadata.get("post")
     if post:

pyodide-build/pyodide_build/common.py

@@ -1,7 +1,48 @@
 import functools
 import subprocess
 from pathlib import Path
-from typing import Optional
+from typing import Iterable, Iterator, Optional
+
+from packaging.tags import Tag, compatible_tags, cpython_tags
+from packaging.utils import parse_wheel_filename
+
+PLATFORM = "emscripten_wasm32"
+
+
+def pyodide_tags() -> Iterator[Tag]:
+    """
+    Returns the sequence of tag triples for the Pyodide interpreter.
+
+    The sequence is ordered in decreasing specificity.
+    """
+
+    yield from cpython_tags(platforms=[PLATFORM])
+    yield from compatible_tags(platforms=[PLATFORM])
+
+
+def find_matching_wheels(wheel_paths: Iterable[Path]) -> Iterator[Path]:
+    """
+    Returns the sequence wheels whose tags match the Pyodide interpreter.
+
+    Parameters
+    ----------
+    wheel_paths
+        A list of paths to wheels
+
+    Returns
+    -------
+    The subset of wheel_paths that have tags that match the Pyodide interpreter.
+    """
+    wheel_paths = list(wheel_paths)
+    wheel_tags_list: list[frozenset[Tag]] = []
+    for wheel in wheel_paths:
+        _, _, _, tags = parse_wheel_filename(wheel.name)
+        wheel_tags_list.append(tags)
+    for supported_tag in pyodide_tags():
+        for wheel_path, wheel_tags in zip(wheel_paths, wheel_tags_list):
+            if supported_tag in wheel_tags:
+                yield wheel_path
+
 
 UNVENDORED_STDLIB_MODULES = {"test", "distutils"}
 

pyodide-build/pyodide_build/pywasmcross.py

@@ -131,7 +131,7 @@ def compile(env, **kwargs):
     make_command_wrapper_symlinks(env)
     args["builddir"] = str(Path(".").absolute())
     env["PYWASMCROSS_ARGS"] = json.dumps(args)
-    env["_PYTHON_HOST_PLATFORM"] = "emscripten_wasm32"
+    env["_PYTHON_HOST_PLATFORM"] = common.PLATFORM
 
     try:
         subprocess.check_call([sys.executable, "setup.py", "bdist_wheel"], env=env)

pyodide-build/pyodide_build/tests/test_common.py

@@ -1,9 +1,10 @@
-from pyodide_build.common import (
+from pyodide_build.common import (  # type: ignore[import]
     ALWAYS_PACKAGES,
     CORE_PACKAGES,
     CORE_SCIPY_PACKAGES,
     UNVENDORED_STDLIB_MODULES,
     _parse_package_subset,
+    find_matching_wheels,
     get_make_environment_vars,
     get_make_flag,
 )
@@ -77,3 +78,36 @@ def test_get_make_environment_vars():
     assert "SIDE_MODULE_CFLAGS" in vars
     assert "SIDE_MODULE_CXXFLAGS" in vars
     assert "TOOLSDIR" in vars
+
+
+def test_wheel_paths():
+    from pathlib import Path
+
+    old_version = "cp38"
+    current_version = "cp39"
+    future_version = "cp317"
+    strings = []
+
+    for interp in [
+        old_version,
+        current_version,
+        future_version,
+        "py3",
+        "py2",
+        "py2.py3",
+    ]:
+        for abi in [interp, "abi3", "none"]:
+            for arch in ["emscripten_wasm32", "linux_x86_64", "any"]:
+                strings.append(f"wrapt-1.13.3-{interp}-{abi}-{arch}.whl")
+
+    paths = [Path(x) for x in strings]
+    assert [x.stem.split("-", 2)[-1] for x in find_matching_wheels(paths)] == [
+        "cp39-cp39-emscripten_wasm32",
+        "cp39-abi3-emscripten_wasm32",
+        "cp39-none-emscripten_wasm32",
+        "cp38-abi3-emscripten_wasm32",
+        "py3-none-emscripten_wasm32",
+        "py2.py3-none-emscripten_wasm32",
+        "py3-none-any",
+        "py2.py3-none-any",
+    ]