CI Fill out more information to pyodide-cross-build-environments.json automatically (#4855)

tools/tests/test_update_cross_build_releases.py

@@ -1,10 +1,11 @@
+import json
 import sys
 from pathlib import Path
 
 from pyodide_build.xbuildenv_releases import CrossBuildEnvMetaSpec
 
 sys.path.append(str(Path(__file__).parents[1]))
-from update_cross_build_releases import add_version
+from update_cross_build_releases import add_version, parse_env_var
 
 
 def test_add_version():
@@ -92,3 +93,99 @@ def test_add_version():
     assert list(new_metadata.releases.keys())[0] == "0.17.0"
     assert list(new_metadata.releases.keys())[1] == "0.17.0a1"
     assert list(new_metadata.releases.keys())[2] == "0.16.0"
+
+
+def test_add_version_full():
+    metadata = CrossBuildEnvMetaSpec.parse_raw(
+        """
+{
+    "releases": {
+        "0.17.0": {
+            "version": "0.17.0",
+            "url": "https://example.com/xbuildenv-0.17.0.tar.bz2",
+            "sha256": "1234567890abcdef",
+            "python_version": "3.8.10",
+            "emscripten_version": "2.0.10",
+            "min_pyodide_build_version": "0.17.0",
+            "max_pyodide_build_version": "0.17.0"
+        },
+        "0.16.0": {
+            "version": "0.16.0",
+            "url": "https://example.com/xbuildenv-0.16.0.tar.bz2",
+            "sha256": "abcdef1234567890",
+            "python_version": "3.8.10",
+            "emscripten_version": "2.0.10",
+            "min_pyodide_build_version": "0.16.0",
+            "max_pyodide_build_version": "0.16.0"
+        }
+    }
+}
+        """.strip()
+    )
+
+    new_metadata_raw = add_version(
+        metadata.json(),
+        "0.18.0",
+        "https://example.com/xbuildenv-0.18.0.tar.bz2",
+        "abcdef1234567890",
+        python_version="3.8.11",
+        emscripten_version="2.0.11",
+        min_pyodide_build_version="0.18.0",
+        max_pyodide_build_version="0.18.0",
+    )
+
+    new_metadata = CrossBuildEnvMetaSpec.parse_raw(new_metadata_raw)
+    assert new_metadata.releases["0.18.0"].version == "0.18.0"
+    assert (
+        new_metadata.releases["0.18.0"].url
+        == "https://example.com/xbuildenv-0.18.0.tar.bz2"
+    )
+    assert new_metadata.releases["0.18.0"].sha256 == "abcdef1234567890"
+    assert new_metadata.releases["0.18.0"].python_version == "3.8.11"
+    assert new_metadata.releases["0.18.0"].emscripten_version == "2.0.11"
+    assert new_metadata.releases["0.18.0"].min_pyodide_build_version == "0.18.0"
+    assert new_metadata.releases["0.18.0"].max_pyodide_build_version == "0.18.0"
+
+
+def test_exclude_none():
+    metadata = CrossBuildEnvMetaSpec.parse_raw(
+        """
+{
+    "releases": {
+        "0.17.0": {
+            "version": "0.17.0",
+            "url": "https://example.com/xbuildenv-0.17.0.tar.bz2",
+            "sha256": "1234567890abcdef",
+            "python_version": "3.8.10",
+            "emscripten_version": "2.0.10"
+        }
+    }
+}
+        """.strip()
+    )
+
+    new_metadata_raw = add_version(
+        metadata.json(),
+        "0.18.0",
+        "https://example.com/xbuildenv-0.18.0.tar.bz2",
+        "abcdef1234567890",
+    )
+
+    metadata = json.loads(new_metadata_raw)
+    assert "releases" in metadata
+    assert "0.17.0" in metadata["releases"]
+    assert "python_version" in metadata["releases"]["0.17.0"]
+    # unspecified fields should be excluded
+    assert "min_pyodide_build_version" not in metadata["releases"]["0.17.0"]
+
+
+def test_parse_env_var():
+    testdata = """
+export PYVERSION ?= 3.45.67
+export PYODIDE_EMSCRIPTEN_VERSION ?= 1.23.45
+export SOMETHING_ELSE ?= hello
+"""
+
+    assert parse_env_var(testdata, "PYVERSION") == "3.45.67"
+    assert parse_env_var(testdata, "PYODIDE_EMSCRIPTEN_VERSION") == "1.23.45"
+    assert parse_env_var(testdata, "SOMETHING_ELSE") == "hello"

tools/update_cross_build_releases.py

@@ -1,6 +1,10 @@
 import argparse
 import hashlib
 import json
+import shutil
+import tempfile
+from collections.abc import Generator
+from contextlib import contextmanager
 from pathlib import Path
 
 import requests
@@ -15,6 +19,10 @@ METADATA_FILE = Path(__file__).parents[1] / "pyodide-cross-build-environments.js
 
 BASE_URL = "https://github.com/pyodide/pyodide/releases/download/{version}/xbuildenv-{version}.tar.bz2"
 
+# Pyodide build version that is compatible with the latest cross-build environment
+# Note for maintainers: update this value when there is a breaking changes in the cross-build environment
+MIN_COMPATIBLE_PYODIDE_BUILD_VERSION = "0.26.0"
+
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser("Update cross-build environments files")
@@ -30,16 +38,48 @@ def get_archive(url: str) -> bytes:
     return resp.content
 
 
-def add_version(raw_metadata: str, version: str, url: str, digest: str) -> str:
+def parse_env_var(content: str, var_name: str) -> str:
+    # A very dummy parser for env vars.
+    for line in content.splitlines():
+        if line.startswith(f"export {var_name}"):
+            return line.split("=")[1].strip()
+
+    return ""
+
+
+@contextmanager
+def extract_archive(archive: bytes) -> Generator[Path, None, None]:
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_dir_path = Path(tmp_dir)
+        archive_path = tmp_dir_path / "xbuildenv.tar.bz2"
+        archive_path.write_bytes(archive)
+
+        # Extract the archive
+        shutil.unpack_archive(str(archive_path), extract_dir=tmp_dir)
+
+        yield tmp_dir_path
+
+
+def add_version(
+    raw_metadata: str,
+    version: str,
+    url: str,
+    digest: str,
+    python_version: str | None = None,
+    emscripten_version: str | None = None,
+    min_pyodide_build_version: str | None = None,
+    max_pyodide_build_version: str | None = None,
+) -> str:
     metadata = CrossBuildEnvMetaSpec.parse_raw(raw_metadata)
     new_release = CrossBuildEnvReleaseSpec(
         version=version,
         url=url,
         sha256=digest,
-        python_version="FIXME",
-        emscripten_version="FIXME",
-        min_pyodide_build_version="FIXME",
-        max_pyodide_build_version="FIXME",
+        python_version=python_version or "FIXME",
+        emscripten_version=emscripten_version or "FIXME",
+        min_pyodide_build_version=min_pyodide_build_version or "FIXME",
+        # Max version is optional, and maintainers should update it when needed.
+        max_pyodide_build_version=max_pyodide_build_version or None,
     )
 
     metadata.releases[version] = new_release
@@ -48,7 +88,7 @@ def add_version(raw_metadata: str, version: str, url: str, digest: str) -> str:
     metadata.releases = dict(
         sorted(metadata.releases.items(), reverse=True, key=lambda x: Version(x[0]))
     )
-    dictionary = metadata.dict()
+    dictionary = metadata.dict(exclude_none=True)
     return json.dumps(dictionary, indent=2)
 
 
@@ -61,8 +101,24 @@ def main():
     content = get_archive(full_url)
     digest = hashlib.sha256(content).hexdigest()
 
+    with extract_archive(content) as extracted:
+        makefile_path = extracted / "xbuildenv" / "pyodide-root" / "Makefile.envs"
+        makefile_content = makefile_path.read_text()
+        python_version = parse_env_var(makefile_content, "PYVERSION")
+        emscripten_version = parse_env_var(
+            makefile_content, "PYODIDE_EMSCRIPTEN_VERSION"
+        )
+
     metadata = METADATA_FILE.read_text()
-    new_metadata = add_version(metadata, version, full_url, digest)
+    new_metadata = add_version(
+        metadata,
+        version,
+        full_url,
+        digest,
+        python_version=python_version,
+        emscripten_version=emscripten_version,
+        min_pyodide_build_version=MIN_COMPATIBLE_PYODIDE_BUILD_VERSION,
+    )
 
     METADATA_FILE.write_text(new_metadata + "\n")