From 1c82138a272e77d5aae420adbc522f11e1fe4608 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 7 Nov 2024 21:04:14 +0900 Subject: [PATCH] Use pyodide-build submodule (#5152) Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- .circleci/config.yml | 10 ++++ .github/workflows/main.yml | 11 +++- .../workflows/update_cross_build_releases.yml | 2 + .gitmodules | 3 + Makefile | 4 +- Makefile.envs | 9 --- docs/development/contributing.md | 2 +- docs/development/maintainers.md | 23 +++++++- docs/development/new-packages.md | 2 +- pyodide-build | 1 + src/tests/test_cmdline_runner.py | 2 +- tools/check_and_install_pyodide_build.py | 59 ------------------- 12 files changed, 50 insertions(+), 78 deletions(-) create mode 100644 .gitmodules create mode 160000 pyodide-build delete mode 100755 tools/check_and_install_pyodide_build.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d98476b2..6a72acdc3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,6 +29,8 @@ jobs: - DISABLE_DYLINK: << parameters.disable_dylink >> steps: - checkout + - run: git submodule sync + - run: git submodule update --init - restore_cache: keys: @@ -132,6 +134,8 @@ jobs: resource_class: large steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . @@ -190,6 +194,8 @@ jobs: resource_class: large steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . @@ -219,6 +225,8 @@ jobs: <<: *defaults steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . @@ -246,6 +254,8 @@ jobs: <<: *defaults steps: - checkout + - run: git submodule sync + - run: git submodule update --init - attach_workspace: at: . diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2327f9dc5..fcbb88f8d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,6 +27,8 @@ jobs: needs: get_python_version steps: - uses: actions/checkout@v4 + with: + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ needs.get_python_version.outputs.PYVERSION }} @@ -34,8 +36,8 @@ jobs: shell: bash -l {0} run: | mkdir test-results - python3 -m pip install -r requirements.txt -r requirements-deploy.txt make pyodide_build + python3 -m pip install -r requirements.txt -r requirements-deploy.txt - name: Run tests shell: bash -l {0} run: | @@ -43,7 +45,7 @@ jobs: --junitxml=test-results/junit.xml \ --verbose \ --runtime=host \ - --cov=pyodide_build --cov=pyodide \ + --cov=pyodide \ src packages/_tests tools/ - uses: codecov/codecov-action@v4 with: @@ -68,7 +70,8 @@ jobs: steps: - uses: actions/checkout@v4 - + with: + submodules: recursive - name: Cache ccache output uses: actions/cache@v4 with: @@ -169,6 +172,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Download build artifact uses: actions/download-artifact@v4 diff --git a/.github/workflows/update_cross_build_releases.yml b/.github/workflows/update_cross_build_releases.yml index bcfbda5f1..4d75d8a03 100644 --- a/.github/workflows/update_cross_build_releases.yml +++ b/.github/workflows/update_cross_build_releases.yml @@ -15,6 +15,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: recursive - uses: actions/setup-python@v5 with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..b32d197f2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "pyodide-build"] + path = pyodide-build + url = https://github.com/pyodide/pyodide-build diff --git a/Makefile b/Makefile index 218619f2c..f4f10870b 100644 --- a/Makefile +++ b/Makefile @@ -225,8 +225,8 @@ $(eval $(call preprocess-js,js2python.js)) .PHONY: pyodide_build pyodide_build: - @echo "Ensuring required pyodide-build version is installed" - ./tools/check_and_install_pyodide_build.py "$(PYODIDE_BUILD_COMMIT)" --repo "$(PYODIDE_BUILD_REPO)" + @echo "Ensuring pyodide-build is installed" + pip install -e ./pyodide-build @which pyodide >/dev/null diff --git a/Makefile.envs b/Makefile.envs index 2b626695d..b48c6b352 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -5,15 +5,6 @@ export PYODIDE_ABI_VERSION ?= 2024_0 export PYTHON_ARCHIVE_SHA256=d01ec6a33bc10009b09c17da95cc2759af5a580a7316b3a446eb4190e13f97b2 -# Update the following variables when you want to update the version of the -# pyodide-build version used in the build process. If you want to make breaking -# changes in pyodide-build, it is also useful to change the repository URL -# to your fork to test the changes are working as expected. - -# v0.27.3 -export PYODIDE_BUILD_COMMIT=fac0109aa2acf14469320b049d710dd42639bf94 -export PYODIDE_BUILD_REPO=https://github.com/pyodide/pyodide-build - ifdef CPYTHON_DEBUG export CPYTHON_ABI_FLAGS=d endif diff --git a/docs/development/contributing.md b/docs/development/contributing.md index 92283fc80..b1c752930 100644 --- a/docs/development/contributing.md +++ b/docs/development/contributing.md @@ -29,7 +29,7 @@ To contribute code, see the following steps, 3. Clone your fork of Pyodide ``` - git clone https://github.com//pyodide.git + git clone --recursive https://github.com//pyodide.git ``` and add the upstream remote, ``` diff --git a/docs/development/maintainers.md b/docs/development/maintainers.md index f4caa870b..0d0e2033d 100644 --- a/docs/development/maintainers.md +++ b/docs/development/maintainers.md @@ -193,8 +193,27 @@ note to update them independently. ## Updating pyodide-build -to change the version of pyodide-build, update the PYODIDE_BUILD_REPO and PYODIDE_BUILD_COMMIT -variables in `Makefile.envs`. +to change the version of pyodide-build, change the commit of the pyodide-build submodule. + +```bash +cd pyodide-build +git checkout "" +``` + +to test with the fork of pyodide-build, change the `.gitmodules` file to point to your fork and update the commit hash + +```ini +# .gitmodules +[submodule "pyodide-build"] + path = pyodide-build + url = https://github.com//pyodide-build +``` + +```bash +git submodule sync +cd pyodide-build +git checkout " str | None: - """ - Return the version of the pyodide-build package or the URL to the repository. - """ - freeze_result = sp.check_output( - [ - sys.executable, - "-m", - "pip", - "freeze", - ] - ) - - for line in freeze_result.decode().split("\n"): - if line.startswith("pyodide-build"): - try: - return line.split(" @ ")[1] - except IndexError: - print("pyodide-build is not installed from a VCS: ", line) - return None - - return None - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("commit", type=str) - parser.add_argument( - "--repo", type=str, default="https://github.com/pyodide/pyodide-build" - ) - - return parser.parse_args() - - -def main(): - args = parse_args() - install_url = f"git+{args.repo}@{args.commit}" - installed_url = get_pyodide_build_install_url() - - if not installed_url or installed_url != install_url: - sp.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - install_url, - ] - ) - - -if __name__ == "__main__": - main()