DOC Use pregenerated lockfile to make package list (#5036)

Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Makefile.envs

@@ -5,6 +5,12 @@ export PYODIDE_ABI_VERSION ?= 2024_0
 
 export PYTHON_ARCHIVE_SHA256=73ac8fe780227bf371add8373c3079f42a0dc62deff8d612cd15a618082ab623
 
+# URL to the prebuilt packages
+export PYODIDE_PREBUILT_PACKAGES_BASE=https://github.com/pyodide/pyodide-recipes/releases/download/20250107
+export PYODIDE_PREBUILT_PACKAGES_URL=$(PYODIDE_PREBUILT_PACKAGES_BASE)/packages.tar.bz2
+export PYODIDE_PREBUILT_PACKAGES_LOCKFILE=$(PYODIDE_PREBUILT_PACKAGES_BASE)/pyodide-lock.json
+export ENABLE_PREBUILT_PACKAGES ?= 0
+
 ifdef CPYTHON_DEBUG
 	export CPYTHON_ABI_FLAGS=d
 endif

docs/sphinx_pyodide/sphinx_pyodide/packages.py

@@ -1,17 +1,16 @@
+import json
 import pathlib
-import sys
+import re
+import subprocess
 from typing import Any
+from urllib.request import urlopen
 
 from docutils import nodes
 from docutils.parsers.rst import Directive
+from pyodide_lock import PackageSpec, PyodideLockSpec
 from sphinx import addnodes
 
 base_dir = pathlib.Path(__file__).resolve().parents[3]
-sys.path.append(str(base_dir / "pyodide-build"))
-
-from pyodide_build.io import MetaConfig
-
-PYODIDE_TESTONLY = "pyodide.test"
 
 
 def get_packages_summary_directive(app):
@@ -21,56 +20,64 @@ def get_packages_summary_directive(app):
         required_arguments = 1
 
         def run(self):
-            packages_root = base_dir / self.arguments[0]
-            packages_list = self.get_package_metadata_list(packages_root)
+            url = self.parse_lockfile_url()
+            resp = urlopen(url)
+            lockfile_json = resp.read().decode("utf-8")
 
-            packages = {}
-            for package in packages_list:
+            lockfile = PyodideLockSpec(**json.loads(lockfile_json))
+            lockfile_packages = lockfile.packages
+
+            python_packages = {}
+            for package in lockfile_packages.values():
                 try:
-                    name, version, is_package, tag, disabled = self.parse_package_info(
-                        package
-                    )
+                    name, version, is_package = self.parse_package_info(package)
                 except Exception:
                     print(f"Warning: failed to parse package config for {package}")
 
-                # skip libraries (e.g. libxml, libyaml, ...) and test only packages
-                if not is_package or PYODIDE_TESTONLY in tag or disabled:
+                if not is_package or name.endswith("-tests"):
                     continue
 
-                packages[name] = {
+                python_packages[name] = {
                     "name": name,
                     "version": version,
                 }
 
             result = []
             columns = ("name", "version")
-            table_markup = self.format_packages_table(packages, columns)
+            table_markup = self.format_packages_table(python_packages, columns)
             result.extend(table_markup)
 
             return result
 
-        def parse_package_info(
-            self, config: pathlib.Path
-        ) -> tuple[str, str, bool, list[str], bool]:
-            yaml_data = MetaConfig.from_yaml(config)
-
-            name = yaml_data.package.name
-            version = yaml_data.package.version
-            tag = yaml_data.package.tag
-            is_package = yaml_data.build.package_type == "package"
-            disabled = yaml_data.package.disabled
-
-            return name, version, is_package, tag, disabled
-
-        def get_package_metadata_list(
-            self, directory: pathlib.Path
-        ) -> list[pathlib.Path]:
-            """Return metadata files of packages in alphabetical order (case insensitive)"""
-            return sorted(
-                directory.glob("**/meta.yaml"),
-                key=lambda path: path.parent.name.lower(),
+        def parse_lockfile_url(self) -> str:
+            envs = subprocess.run(
+                ["make", "-f", str(base_dir / "Makefile.envs"), ".output_vars"],
+                capture_output=True,
+                text=True,
+                env={"PYODIDE_ROOT": str(base_dir)},
+                check=False,
             )
 
+            if envs.returncode != 0:
+                raise RuntimeError("Failed to parse Makefile.envs")
+
+            pattern = re.search(r"PYODIDE_PREBUILT_PACKAGES_LOCKFILE=(.*)", envs.stdout)
+            if not pattern:
+                raise RuntimeError("Failed to find lockfile URL in Makefile.envs")
+
+            url = pattern.group(1)
+            return url
+
+        def parse_package_info(
+            self,
+            package: PackageSpec,
+        ) -> tuple[str, str, bool]:
+            name = package.name
+            version = package.version
+            is_package = package.package_type == "package"
+
+            return name, version, is_package
+
         def format_packages_table(
             self, packages: dict[str, Any], columns: tuple[str, ...]
         ) -> list[Any]: