From c2a797530db073d3c904d6bffb112f657ecb69d7 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Mon, 10 Jun 2024 20:53:55 +0900 Subject: [PATCH] CI Fill out more information to pyodide-cross-build-environments.json automatically (#4855) --- .../tests/test_update_cross_build_releases.py | 99 ++++++++++++++++++- tools/update_cross_build_releases.py | 70 +++++++++++-- 2 files changed, 161 insertions(+), 8 deletions(-) diff --git a/tools/tests/test_update_cross_build_releases.py b/tools/tests/test_update_cross_build_releases.py index 80e6df06a..cf7ef2cac 100644 --- a/tools/tests/test_update_cross_build_releases.py +++ b/tools/tests/test_update_cross_build_releases.py @@ -1,10 +1,11 @@ +import json import sys from pathlib import Path from pyodide_build.xbuildenv_releases import CrossBuildEnvMetaSpec sys.path.append(str(Path(__file__).parents[1])) -from update_cross_build_releases import add_version +from update_cross_build_releases import add_version, parse_env_var def test_add_version(): @@ -92,3 +93,99 @@ def test_add_version(): assert list(new_metadata.releases.keys())[0] == "0.17.0" assert list(new_metadata.releases.keys())[1] == "0.17.0a1" assert list(new_metadata.releases.keys())[2] == "0.16.0" + + +def test_add_version_full(): + metadata = CrossBuildEnvMetaSpec.parse_raw( + """ +{ + "releases": { + "0.17.0": { + "version": "0.17.0", + "url": "https://example.com/xbuildenv-0.17.0.tar.bz2", + "sha256": "1234567890abcdef", + "python_version": "3.8.10", + "emscripten_version": "2.0.10", + "min_pyodide_build_version": "0.17.0", + "max_pyodide_build_version": "0.17.0" + }, + "0.16.0": { + "version": "0.16.0", + "url": "https://example.com/xbuildenv-0.16.0.tar.bz2", + "sha256": "abcdef1234567890", + "python_version": "3.8.10", + "emscripten_version": "2.0.10", + "min_pyodide_build_version": "0.16.0", + "max_pyodide_build_version": "0.16.0" + } + } +} + """.strip() + ) + + new_metadata_raw = add_version( + metadata.json(), + "0.18.0", + "https://example.com/xbuildenv-0.18.0.tar.bz2", + "abcdef1234567890", + python_version="3.8.11", + emscripten_version="2.0.11", + min_pyodide_build_version="0.18.0", + max_pyodide_build_version="0.18.0", + ) + + new_metadata = CrossBuildEnvMetaSpec.parse_raw(new_metadata_raw) + assert new_metadata.releases["0.18.0"].version == "0.18.0" + assert ( + new_metadata.releases["0.18.0"].url + == "https://example.com/xbuildenv-0.18.0.tar.bz2" + ) + assert new_metadata.releases["0.18.0"].sha256 == "abcdef1234567890" + assert new_metadata.releases["0.18.0"].python_version == "3.8.11" + assert new_metadata.releases["0.18.0"].emscripten_version == "2.0.11" + assert new_metadata.releases["0.18.0"].min_pyodide_build_version == "0.18.0" + assert new_metadata.releases["0.18.0"].max_pyodide_build_version == "0.18.0" + + +def test_exclude_none(): + metadata = CrossBuildEnvMetaSpec.parse_raw( + """ +{ + "releases": { + "0.17.0": { + "version": "0.17.0", + "url": "https://example.com/xbuildenv-0.17.0.tar.bz2", + "sha256": "1234567890abcdef", + "python_version": "3.8.10", + "emscripten_version": "2.0.10" + } + } +} + """.strip() + ) + + new_metadata_raw = add_version( + metadata.json(), + "0.18.0", + "https://example.com/xbuildenv-0.18.0.tar.bz2", + "abcdef1234567890", + ) + + metadata = json.loads(new_metadata_raw) + assert "releases" in metadata + assert "0.17.0" in metadata["releases"] + assert "python_version" in metadata["releases"]["0.17.0"] + # unspecified fields should be excluded + assert "min_pyodide_build_version" not in metadata["releases"]["0.17.0"] + + +def test_parse_env_var(): + testdata = """ +export PYVERSION ?= 3.45.67 +export PYODIDE_EMSCRIPTEN_VERSION ?= 1.23.45 +export SOMETHING_ELSE ?= hello +""" + + assert parse_env_var(testdata, "PYVERSION") == "3.45.67" + assert parse_env_var(testdata, "PYODIDE_EMSCRIPTEN_VERSION") == "1.23.45" + assert parse_env_var(testdata, "SOMETHING_ELSE") == "hello" diff --git a/tools/update_cross_build_releases.py b/tools/update_cross_build_releases.py index b99a1ff7d..34c135432 100644 --- a/tools/update_cross_build_releases.py +++ b/tools/update_cross_build_releases.py @@ -1,6 +1,10 @@ import argparse import hashlib import json +import shutil +import tempfile +from collections.abc import Generator +from contextlib import contextmanager from pathlib import Path import requests @@ -15,6 +19,10 @@ METADATA_FILE = Path(__file__).parents[1] / "pyodide-cross-build-environments.js BASE_URL = "https://github.com/pyodide/pyodide/releases/download/{version}/xbuildenv-{version}.tar.bz2" +# Pyodide build version that is compatible with the latest cross-build environment +# Note for maintainers: update this value when there is a breaking changes in the cross-build environment +MIN_COMPATIBLE_PYODIDE_BUILD_VERSION = "0.26.0" + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser("Update cross-build environments files") @@ -30,16 +38,48 @@ def get_archive(url: str) -> bytes: return resp.content -def add_version(raw_metadata: str, version: str, url: str, digest: str) -> str: +def parse_env_var(content: str, var_name: str) -> str: + # A very dummy parser for env vars. + for line in content.splitlines(): + if line.startswith(f"export {var_name}"): + return line.split("=")[1].strip() + + return "" + + +@contextmanager +def extract_archive(archive: bytes) -> Generator[Path, None, None]: + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir_path = Path(tmp_dir) + archive_path = tmp_dir_path / "xbuildenv.tar.bz2" + archive_path.write_bytes(archive) + + # Extract the archive + shutil.unpack_archive(str(archive_path), extract_dir=tmp_dir) + + yield tmp_dir_path + + +def add_version( + raw_metadata: str, + version: str, + url: str, + digest: str, + python_version: str | None = None, + emscripten_version: str | None = None, + min_pyodide_build_version: str | None = None, + max_pyodide_build_version: str | None = None, +) -> str: metadata = CrossBuildEnvMetaSpec.parse_raw(raw_metadata) new_release = CrossBuildEnvReleaseSpec( version=version, url=url, sha256=digest, - python_version="FIXME", - emscripten_version="FIXME", - min_pyodide_build_version="FIXME", - max_pyodide_build_version="FIXME", + python_version=python_version or "FIXME", + emscripten_version=emscripten_version or "FIXME", + min_pyodide_build_version=min_pyodide_build_version or "FIXME", + # Max version is optional, and maintainers should update it when needed. + max_pyodide_build_version=max_pyodide_build_version or None, ) metadata.releases[version] = new_release @@ -48,7 +88,7 @@ def add_version(raw_metadata: str, version: str, url: str, digest: str) -> str: metadata.releases = dict( sorted(metadata.releases.items(), reverse=True, key=lambda x: Version(x[0])) ) - dictionary = metadata.dict() + dictionary = metadata.dict(exclude_none=True) return json.dumps(dictionary, indent=2) @@ -61,8 +101,24 @@ def main(): content = get_archive(full_url) digest = hashlib.sha256(content).hexdigest() + with extract_archive(content) as extracted: + makefile_path = extracted / "xbuildenv" / "pyodide-root" / "Makefile.envs" + makefile_content = makefile_path.read_text() + python_version = parse_env_var(makefile_content, "PYVERSION") + emscripten_version = parse_env_var( + makefile_content, "PYODIDE_EMSCRIPTEN_VERSION" + ) + metadata = METADATA_FILE.read_text() - new_metadata = add_version(metadata, version, full_url, digest) + new_metadata = add_version( + metadata, + version, + full_url, + digest, + python_version=python_version, + emscripten_version=emscripten_version, + min_pyodide_build_version=MIN_COMPATIBLE_PYODIDE_BUILD_VERSION, + ) METADATA_FILE.write_text(new_metadata + "\n")