Use pyodide-build submodule (#5152)

Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>

.circleci/config.yml

@@ -29,6 +29,8 @@ jobs:
       - DISABLE_DYLINK: << parameters.disable_dylink >>
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - restore_cache:
           keys:
@@ -132,6 +134,8 @@ jobs:
     resource_class: large
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -190,6 +194,8 @@ jobs:
     resource_class: large
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -219,6 +225,8 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .
@@ -246,6 +254,8 @@ jobs:
     <<: *defaults
     steps:
       - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
 
       - attach_workspace:
           at: .

.github/workflows/main.yml

@@ -27,6 +27,8 @@ jobs:
     needs: get_python_version
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ needs.get_python_version.outputs.PYVERSION }}
@@ -34,8 +36,8 @@ jobs:
         shell: bash -l {0}
         run: |
           mkdir test-results
-          python3 -m pip install -r requirements.txt -r requirements-deploy.txt
           make pyodide_build
+          python3 -m pip install -r requirements.txt -r requirements-deploy.txt
       - name: Run tests
         shell: bash -l {0}
         run: |
@@ -43,7 +45,7 @@ jobs:
               --junitxml=test-results/junit.xml \
               --verbose \
               --runtime=host \
-              --cov=pyodide_build --cov=pyodide \
+              --cov=pyodide \
               src packages/_tests tools/
       - uses: codecov/codecov-action@v4
         with:
@@ -68,7 +70,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-
+        with:
+          submodules: recursive
       - name: Cache ccache output
         uses: actions/cache@v4
         with:
@@ -169,6 +172,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Download build artifact
         uses: actions/download-artifact@v4

.github/workflows/update_cross_build_releases.yml

@@ -15,6 +15,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "pyodide-build"]
+	path = pyodide-build
+	url = https://github.com/pyodide/pyodide-build

Makefile

@@ -225,8 +225,8 @@ $(eval $(call preprocess-js,js2python.js))
 
 .PHONY: pyodide_build
 pyodide_build:
-	@echo "Ensuring required pyodide-build version is installed"
-	./tools/check_and_install_pyodide_build.py "$(PYODIDE_BUILD_COMMIT)" --repo "$(PYODIDE_BUILD_REPO)"
+	@echo "Ensuring pyodide-build is installed"
+	pip install -e ./pyodide-build
 	@which pyodide >/dev/null
 
 

Makefile.envs

@@ -5,15 +5,6 @@ export PYODIDE_ABI_VERSION ?= 2024_0
 
 export PYTHON_ARCHIVE_SHA256=d01ec6a33bc10009b09c17da95cc2759af5a580a7316b3a446eb4190e13f97b2
 
-# Update the following variables when you want to update the version of the
-# pyodide-build version used in the build process. If you want to make breaking
-# changes in pyodide-build, it is also useful to change the repository URL
-# to your fork to test the changes are working as expected.
-
-# v0.27.3
-export PYODIDE_BUILD_COMMIT=fac0109aa2acf14469320b049d710dd42639bf94
-export PYODIDE_BUILD_REPO=https://github.com/pyodide/pyodide-build
-
 ifdef CPYTHON_DEBUG
 	export CPYTHON_ABI_FLAGS=d
 endif

docs/development/contributing.md

@@ -29,7 +29,7 @@ To contribute code, see the following steps,
 
 3. Clone your fork of Pyodide
    ```
-   git clone https://github.com/<your-username>/pyodide.git
+   git clone --recursive https://github.com/<your-username>/pyodide.git
    ```
    and add the upstream remote,
    ```

docs/development/maintainers.md

@@ -193,8 +193,27 @@ note to update them independently.
 
 ## Updating pyodide-build
 
-to change the version of pyodide-build, update the PYODIDE_BUILD_REPO and PYODIDE_BUILD_COMMIT
-variables in `Makefile.envs`.
+to change the version of pyodide-build, change the commit of the pyodide-build submodule.
+
+```bash
+cd pyodide-build
+git checkout "<COMMIT HASH>"
+```
+
+to test with the fork of pyodide-build, change the `.gitmodules` file to point to your fork and update the commit hash
+
+```ini
+# .gitmodules
+[submodule "pyodide-build"]
+	path = pyodide-build
+	url = https://github.com/<yourfork>/pyodide-build
+```
+
+```bash
+git submodule sync
+cd pyodide-build
+git checkout "<COMMIT HASH"
+```
 
 ## Upgrading pyodide to a new version of CPython
 

docs/development/new-packages.md

@@ -60,7 +60,7 @@ folder](https://github.com/pyodide/pyodide/tree/main/packages).
 First clone the Pyodide git repository:
 
 ```bash
-git clone https://github.com/pyodide/pyodide
+git clone --recursive https://github.com/pyodide/pyodide
 cd pyodide
 ```
 

pyodide-build

@@ -0,0 +1 @@
+Subproject commit fac0109aa2acf14469320b049d710dd42639bf94

src/tests/test_cmdline_runner.py

@@ -445,7 +445,7 @@ def test_package_index(tmp_path):
     version = "0.26.0"  # just need some version that already exists + contains pyodide-lock.json
 
     mgr = CrossBuildEnvManager(path)
-    mgr.install(version, skip_install_cross_build_packages=True)
+    mgr.install(version, skip_install_cross_build_packages=True, force_install=True)
 
     env_path = mgr.symlink_dir.resolve()
 

tools/check_and_install_pyodide_build.py

@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import subprocess as sp
-import sys
-
-
-def get_pyodide_build_install_url() -> str | None:
-    """
-    Return the version of the pyodide-build package or the URL to the repository.
-    """
-    freeze_result = sp.check_output(
-        [
-            sys.executable,
-            "-m",
-            "pip",
-            "freeze",
-        ]
-    )
-
-    for line in freeze_result.decode().split("\n"):
-        if line.startswith("pyodide-build"):
-            try:
-                return line.split(" @ ")[1]
-            except IndexError:
-                print("pyodide-build is not installed from a VCS: ", line)
-                return None
-
-    return None
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("commit", type=str)
-    parser.add_argument(
-        "--repo", type=str, default="https://github.com/pyodide/pyodide-build"
-    )
-
-    return parser.parse_args()
-
-
-def main():
-    args = parse_args()
-    install_url = f"git+{args.repo}@{args.commit}"
-    installed_url = get_pyodide_build_install_url()
-
-    if not installed_url or installed_url != install_url:
-        sp.check_call(
-            [
-                sys.executable,
-                "-m",
-                "pip",
-                "install",
-                install_url,
-            ]
-        )
-
-
-if __name__ == "__main__":
-    main()