version: 2.1 defaults: &defaults working_directory: ~/repo docker: # Note: when updating the docker image version, # make sure there are no extra old versions lying around. # (e.g. `rg -F --hidden `) - image: pyodide/pyodide-env:20240127-chrome114-firefox122-py312 environment: - EMSDK_NUM_CORES: 3 EMCC_CORES: 3 PYODIDE_JOBS: 3 # Make sure the ccache dir is consistent between core and package builds # (it's not the case otherwise) CCACHE_DIR: /root/.ccache/ # Disable the compression of wheels, so they are better compressed by the CDN PYODIDE_ZIP_COMPRESSION_LEVEL: 0 orbs: macos: circleci/macos@2.3.6 jobs: build-core: parameters: disable_dylink: description: Should we disable dynamic linking? 0 means no, 1 means yes. type: integer default: 0 <<: *defaults environment: - DISABLE_DYLINK: << parameters.disable_dylink >> steps: - checkout - restore_cache: keys: - -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }} - -core-v20220915-{{ checksum "cpython/Makefile" }} - -core-v20220915 - run: name: calculate build hash for ccache command: | pip install pathspec ./tools/calculate_build_cache_key.py > build_hash.txt - run: name: build emsdk no_output_timeout: 20m command: | # This is necessary to use the ccache from emsdk source pyodide_env.sh ccache -z make -C emsdk ccache -s # Set mtime for EM_CONFIG to avoid ccache cache misses touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten - run: name: build cpython no_output_timeout: 20m command: | # This is necessary to use the ccache from emsdk source pyodide_env.sh ccache -z make -C cpython ccache -s - run: name: build pyodide core no_output_timeout: 20m command: | # This is necessary to use the ccache from emsdk source pyodide_env.sh ccache -z PYODIDE_PACKAGES="tag:core" make ccache -s - run: name: check-size command: | du dist/ -abh --max-depth 1 | sort -k 2 pip install brotli ./tools/check_compressed_size.py dist/pyodide.asm.* dist/python_stdlib* - save_cache: paths: - /root/.ccache key: -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }} - run: name: Clean up workspace command: | rm -rf cpython/{build,downloads} - persist_to_workspace: root: . paths: - . - run: name: Zip build directory command: | tar cjf pyodide.tar.gz dist tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm} dist/{package,pyodide-lock}.json dist/python_stdlib.zip tar cjf pyodide-static-libs.tar.gz -C cpython/installs . - store_artifacts: path: /root/repo/dist/ - store_artifacts: path: /root/repo/pyodide.tar.gz - store_artifacts: path: /root/repo/pyodide-core.tar.gz - store_artifacts: path: /root/repo/pyodide-static-libs.tar.gz - store_artifacts: path: /root/repo/packages/build-logs build-packages: parameters: packages: description: The packages to be built. type: string <<: *defaults resource_class: large steps: - checkout - attach_workspace: at: . - restore_cache: keys: - -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >> - run: name: build packages no_output_timeout: 60m command: | source pyodide_env.sh # Set mtime for EM_CONFIG to avoid ccache cache misses touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten ccache -z PYODIDE_PACKAGES='<< parameters.packages >>' make dist/pyodide-lock.json ccache -s environment: PYODIDE_JOBS: 5 - run: name: check-size command: du dist/ -abh --max-depth 1 | sort -k 2 - run: name: Zip build directory command: | tar cjf pyodide.tar.gz dist tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm} dist/{package,pyodide-lock}.json dist/python_stdlib.zip tar cjf build-logs.tar.gz packages/build-logs - run: name: Clean up package source files command: | cd packages && find **/build ! -name '.packaged' -type f -exec rm -f {} + - store_artifacts: path: /root/repo/pyodide.tar.gz - store_artifacts: path: /root/repo/pyodide-core.tar.gz - store_artifacts: path: /root/repo/build-logs.tar.gz - save_cache: paths: - /root/.ccache key: -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >> build-pyodide-debug: <<: *defaults resource_class: large steps: - checkout - attach_workspace: at: . - run: name: build pyodide debug command: | cp -r dist dist-release rm dist/pyodide.asm.js source pyodide_env.sh ccache -z PYODIDE_DEBUG=1 make dist/pyodide.asm.js ccache -s cd dist npx prettier -w pyodide.asm.js npx prettier -w pyodide.js cd .. mv dist dist-debug mv dist-release dist - persist_to_workspace: root: . paths: - . create-xbuild-env: <<: *defaults steps: - checkout - attach_workspace: at: . - run: name: create xbuild environment command: | pip install -e ./pyodide-build pyodide xbuildenv create . - run: name: Zip xbuild environment command: | tar cjf xbuildenv.tar.gz ./xbuildenv/ - store_artifacts: path: /root/repo/xbuildenv.tar.gz - persist_to_workspace: root: . paths: - ./xbuildenv build-test-pyc-packages: <<: *defaults steps: - checkout - attach_workspace: at: . - run: name: Install requirements command: | pip install -r requirements.txt pip install -e ./pyodide-build - run: name: Py-compile packages command: | pyodide py-compile --compression-level 0 dist/ - run: name: Test import of py-compiled packages command: | tools/pytest_wrapper.py \ --junitxml=test-results/junit.xml \ --verbose \ --durations 50 \ --benchmark-json=benchmark-time.json \ "packages/_tests/test_packages_common.py::test_import" -k "node" mv dist/ dist-pyc/ - persist_to_workspace: root: . paths: - ./dist-pyc test-main: parameters: test-params: description: The tests to run. type: string cache-dir: description: pytest-cache-dir. type: string default: "" <<: *defaults resource_class: medium+ steps: - attach_workspace: at: . - run: name: test command: | make npm-link mkdir test-results pip install -r requirements.txt pip install -e ./pyodide-build if [ -z "<< parameters.cache-dir >>" ]; then export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)" else export CACHE_DIR=".test_cache/<< parameters.cache-dir >>" fi echo "pytest cache dir: $CACHE_DIR" source pyodide_env.sh tools/pytest_wrapper.py \ --junitxml=test-results/junit.xml \ --verbose \ --durations 50 \ --benchmark-json=benchmark-time.json \ --benchmark-columns=mean,min,max,stddev \ << parameters.test-params >> \ -o cache_dir=$CACHE_DIR - store_test_results: path: test-results test-main-macos: parameters: test-params: description: The tests to run. type: string cache-dir: description: pytest-cache-dir. type: string default: "" resource_class: macos.x86.medium.gen2 macos: xcode: 15.0.0 working_directory: ~/repo steps: - attach_workspace: at: . # The standard way of enabling safaridriver no longer works for Safari 14+ # https://blog.bytesguy.com/enabling-remote-automation-in-safari-14 - macos/add-safari-permissions - run: name: install miniforge command: | curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh bash Miniforge3-MacOSX-x86_64.sh -b && rm -f Miniforge3-MacOSX-x86_64.sh ~/miniforge3/bin/conda create -n pyodide python=3.12 -y - run: name: install dependencies command: | export PATH="$HOME/miniforge3/bin:$PATH" conda create -n pyodide python=3.12 -y source activate pyodide python -m pip install -r requirements.txt pip install -e ./pyodide-build - run: name: show safari version command: | safari_version=$(defaults read /Applications/Safari.app/Contents/Info.plist CFBundleShortVersionString) echo "Safari Version: $safari_version" - run: name: test safari command: | export PATH="$HOME/miniforge3/bin:$PATH" source activate pyodide mkdir test-results if [ -z "<< parameters.cache-dir >>" ]; then export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)" else export CACHE_DIR=".test_cache/<< parameters.cache-dir >>" fi echo "pytest cache dir: $CACHE_DIR" tools/pytest_wrapper.py \ --junitxml=test-results/junit.xml \ --verbose \ --durations 50 \ << parameters.test-params >> \ -o cache_dir=$CACHE_DIR - store_test_results: path: test-results benchmark-stack-size: <<: *defaults steps: - attach_workspace: at: . - run: name: stack-size command: | make npm-link pip install -r requirements.txt pytest -s benchmark/stack_usage.py --rt node,chrome,firefox | sed -n 's/## //pg' test-js: <<: *defaults resource_class: small steps: - attach_workspace: at: . - run: name: test-types command: | cd src/test-js npm ci npm link ../../dist npm run test-types npm test - run: name: test-unit command: | cd src/js npm run test:unit # - run: # name: check if Pyodide works with Webpack # command: | # git clone https://github.com/pyodide/pyodide-webpack-example.git # export DEV_PYODIDE_PATH=`realpath dist` # cd pyodide-webpack-example # git checkout 164054a9c6fbd2176f386b6552ed8d079c6bcc04 # ./build.sh - run: name: test npm deploy (dry run) command: | DRY_RUN=1 ./tools/deploy_to_npm.sh test-cmdline-runner: <<: *defaults steps: - attach_workspace: at: . - run: name: test command: | set -x export PYODIDE_ROOT=$(pwd) echo $PYODIDE_ROOT source pyodide_env.sh cd ~ mkdir test cd test git clone https://github.com/python-attrs/attrs --depth 1 --branch 22.1.0 git clone https://github.com/zopefoundation/zope.interface.git --depth 1 --branch 5.4.0 python -m venv .venv-host source .venv-host/bin/activate pip install $PYODIDE_ROOT/pyodide-build pyodide venv .venv-pyodide source .venv-pyodide/bin/activate cd zope.interface pyodide build pip install dist/*.whl cd ../attrs pip install .[tests] python -m pytest -k 'not mypy' benchmark: <<: *defaults resource_class: medium+ steps: - checkout - attach_workspace: at: . - run: name: install requirements command: | pip3 install numpy matplotlib pandas pip install -r requirements.txt - run: name: benchmark command: | python benchmark/benchmark.py all --output dist/benchmarks.json - store_artifacts: path: /root/repo/dist/benchmarks.json check-release-version: # Check that the tag matches the version numbers in the repo and fail the # release if not <<: *defaults steps: - checkout - run: name: check-release-version command: | ./tools/bump_version.py --check --new-version ${CIRCLE_TAG} deploy-release: # To reduce chance of deployment issues, try to keep the steps here as # similar as possible to the steps in deploy-dev! resource_class: small <<: *defaults steps: - checkout - attach_workspace: at: . - run: name: Install requirements command: | python3 -m pip install -e "./pyodide-build[deploy]" wget https://github.com/tcnksm/ghr/releases/download/v0.16.2/ghr_v0.16.2_linux_amd64.tar.gz tar xzf ghr_v0.16.2_linux_amd64.tar.gz mv ghr_v0.16.2_linux_amd64/ghr /tmp/ghr-bin - run: name: Deploy Github Releases command: | mkdir -p /tmp/ghr/dist cp -r dist /tmp/ghr/pyodide cp -r xbuildenv /tmp/ghr/xbuildenv cp -r cpython/installs /tmp/ghr/static-libraries cd /tmp/ghr tar cjf dist/pyodide-${CIRCLE_TAG}.tar.bz2 pyodide/ tar cjf dist/pyodide-core-${CIRCLE_TAG}.tar.bz2 pyodide/pyodide{.js,.mjs,.asm.js,.asm.wasm} pyodide/{package,pyodide-lock}.json pyodide/python_stdlib.zip tar cjf dist/xbuildenv-${CIRCLE_TAG}.tar.bz2 xbuildenv/ tar cjf dist/static-libraries-${CIRCLE_TAG}.tar.bz2 static-libraries/ # If it has an 'a' in the tag it's an alpha so mark it as a prelease [[ "${CIRCLE_TAG}" =~ a ]] && export MAYBE_PRE_RELEASE=-prerelease # Options have to come first, last two lines have to be # ${CIRCLE_TAG} and dist. # TODO: Should we get rid of -delete? /tmp/ghr-bin \ -t "${GITHUB_TOKEN}" \ -u "${CIRCLE_PROJECT_USERNAME}" \ -r "${CIRCLE_PROJECT_REPONAME}" \ -c "${CIRCLE_SHA1}" \ -delete \ ${MAYBE_PRE_RELEASE} \ "${CIRCLE_TAG}" \ dist - run: name: Deploy to npm command: | ./tools/deploy_to_npm.sh - run: name: Set PYODIDE_BASE_URL command: | PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/pyc/" make dist/console.html cp dist/console.html dist-pyc/console.html PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/debug/" make dist/console.html cp dist/console.html dist-debug/console.html PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make dist/console.html - run: name: Deploy to S3 command: | python3 tools/deploy_s3.py dist/ "v${CIRCLE_TAG}/full/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=30758400, immutable, public' - run: name: Deploy debug version to S3 command: | python3 tools/deploy_s3.py dist-debug/ "v${CIRCLE_TAG}/debug/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=30758400, immutable, public' - run: name: Deploy the pyc version to S3 command: | python3 tools/deploy_s3.py dist-pyc/ "v${CIRCLE_TAG}/pyc/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=30758400, immutable, public' deploy-dev: # To reduce chance of deployment issues, try to keep the steps here as # similar as possible to the steps in deploy-release! resource_class: small <<: *defaults steps: - checkout - attach_workspace: at: . - run: name: Install requirements command: | python3 -m pip install -e "./pyodide-build[deploy]" - run: name: Set PYODIDE_BASE_URL command: | PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/pyc/" make dist/console.html cp dist/console.html dist-pyc/console.html PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/debug/" make dist/console.html cp dist/console.html dist-debug/console.html PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make dist/console.html - run: name: Deploy to S3 command: | python3 tools/deploy_s3.py dist/ "dev/full/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=3600, public' --rm-remote-prefix - run: name: Deploy debug version to S3 command: | python3 tools/deploy_s3.py dist-debug/ "dev/debug/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=3600, public' --rm-remote-prefix - run: name: Deploy pyc version to S3 command: | python3 tools/deploy_s3.py dist-pyc/ "dev/pyc/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=3600, public' --rm-remote-prefix - run: # Unlike the release version, we upload the dev version to S3 not to GitHub. name: Deploy release files to S3 command: | mkdir -p /tmp/ghr/dist cp -r dist /tmp/ghr/pyodide cp -r xbuildenv /tmp/ghr/xbuildenv cp -r cpython/installs /tmp/ghr/static-libraries cd /tmp/ghr tar cjf dist/pyodide-core.tar.bz2 pyodide/pyodide{.js,.mjs,.asm.js,.asm.wasm} pyodide/{package,pyodide-lock}.json pyodide/python_stdlib.zip tar cjf dist/xbuildenv.tar.bz2 xbuildenv/ tar cjf dist/static-libraries.tar.bz2 static-libraries/ cd - python3 tools/deploy_s3.py /tmp/ghr/dist/ "xbuildenv/dev" --bucket "pyodide-cache" --cache-control 'max-age=3600, public' --overwrite \ --access-key-env "AWS_ACCESS_KEY_ID_CACHE" --secret-key-env "AWS_SECRET_ACCESS_KEY_CACHE" workflows: version: 2 build-and-deploy: jobs: - build-core: filters: tags: only: /.*/ - build-core: name: "build-core-nodylink" disable_dylink: 1 filters: tags: only: /.*/ - build-packages: name: build-libraries packages: "tag:library" requires: - build-core filters: tags: only: /.*/ post-steps: - persist_to_workspace: root: . paths: - ./packages - ./dist - build-packages: name: build-packages-no-numpy-dependents packages: "*,no-numpy-dependents" requires: - build-core - build-libraries filters: tags: only: /.*/ post-steps: - persist_to_workspace: root: . paths: - ./packages - ./dist - build-packages: name: build-packages-opencv-python packages: opencv-python requires: - build-packages-no-numpy-dependents filters: tags: only: /.*/ post-steps: - persist_to_workspace: root: . paths: - ./packages/opencv-python/build - ./packages/opencv-python/dist - ./packages/build-logs/opencv* - ./dist/opencv* - build-packages: name: build-packages-numpy-dependents packages: "*,!opencv-python" requires: - build-packages-no-numpy-dependents filters: tags: only: /.*/ post-steps: - persist_to_workspace: root: . paths: - ./packages - ./dist - build-packages: name: build-packages packages: "*" requires: - build-packages-numpy-dependents - build-packages-opencv-python filters: tags: only: /.*/ post-steps: - persist_to_workspace: root: . paths: - ./packages/.artifacts - ./dist - test-main: name: test-core-chrome-nodylink test-params: --runtime=chrome-no-host -m 'not requires_dynamic_linking' -k "not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ requires: - build-core-nodylink filters: tags: only: /.*/ - test-main: name: test-core-firefox-nodylink test-params: --runtime=firefox-no-host -m 'not requires_dynamic_linking' -k "not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ requires: - build-core-nodylink filters: tags: only: /.*/ - test-main: name: test-core-node-nodylink test-params: --runtime=node-no-host -m 'not requires_dynamic_linking' -k "not cmdline_runner" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-build/pyodide_build/tests requires: - build-core-nodylink filters: tags: only: /.*/ - test-main: name: test-core-chrome test-params: --runtime=chrome-no-host -k "not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ requires: - build-core filters: tags: only: /.*/ - test-main: name: test-core-firefox test-params: --runtime=firefox-no-host -k "not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ requires: - build-core filters: tags: only: /.*/ - test-main: name: test-core-node test-params: --runtime=node-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-build/pyodide_build/tests requires: - build-core filters: tags: only: /.*/ - test-main-macos: name: test-core-safari test-params: --runtime=safari-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ requires: - build-core filters: tags: only: /.*/ - test-main: name: test-core-chrome-webworker test-params: --runtime=chrome-no-host src/tests/test_webworker.py requires: - test-core-chrome filters: tags: only: /.*/ - test-main: name: test-core-firefox-webworker test-params: --runtime=firefox-no-host src/tests/test_webworker.py requires: - test-core-firefox filters: tags: only: /.*/ - test-main: name: test-packages-chrome-no-numpy-dependents test-params: --runtime=chrome-no-host packages/*/test*.py cache-dir: .pytest_cache_chrome requires: - build-packages-no-numpy-dependents filters: tags: only: /.*/ post-steps: - persist_to_workspace: root: . paths: - ./.test_cache - test-main: name: test-packages-chrome test-params: --runtime=chrome-no-host packages/*/test*.py --skip-passed cache-dir: .pytest_cache_chrome requires: - test-packages-chrome-no-numpy-dependents - build-packages filters: tags: only: /.*/ - test-main: name: test-packages-firefox-no-numpy-dependents test-params: --runtime=firefox-no-host packages/*/test*.py cache-dir: .pytest_cache_firefox requires: - build-packages-no-numpy-dependents filters: tags: only: /.*/ post-steps: - persist_to_workspace: root: . paths: - ./.test_cache - test-main: name: test-packages-firefox test-params: --runtime=firefox-no-host packages/*/test*.py --skip-passed cache-dir: .pytest_cache_firefox requires: - test-packages-firefox-no-numpy-dependents - build-packages filters: tags: only: /.*/ - test-main: name: test-packages-node-no-numpy-dependents test-params: --runtime=node-no-host packages/*/test*.py cache-dir: .pytest_cache_node requires: - build-packages-no-numpy-dependents filters: tags: only: /.*/ post-steps: - persist_to_workspace: root: . paths: - ./.test_cache - test-main: name: test-packages-node test-params: --runtime=node-no-host packages/*/test*.py --skip-passed cache-dir: .pytest_cache_node requires: - test-packages-node-no-numpy-dependents - build-packages filters: tags: only: /.*/ - test-main-macos: name: test-packages-safari-no-numpy-dependents test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py cache-dir: .pytest_cache_safari requires: - build-packages-no-numpy-dependents filters: tags: only: /.*/ post-steps: - persist_to_workspace: root: . paths: - ./.test_cache - test-main-macos: name: test-packages-safari test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py --skip-passed cache-dir: .pytest_cache_safari requires: - test-packages-safari-no-numpy-dependents - build-packages filters: tags: only: /.*/ - test-js: requires: - build-core filters: tags: only: /.*/ - benchmark-stack-size: requires: - build-core filters: tags: only: /.*/ - benchmark: requires: - build-packages filters: tags: only: /.*/ - create-xbuild-env: requires: - build-packages filters: tags: only: /.*/ - test-cmdline-runner: requires: - build-packages filters: tags: only: /.*/ - build-pyodide-debug: requires: - build-packages filters: tags: only: /.*/ - build-test-pyc-packages: requires: - build-packages filters: tags: only: /.*/ - check-release-version: filters: branches: ignore: /.*/ tags: only: /^\d+\.\d+\.\w+$/ - deploy-release: requires: - check-release-version - test-core-firefox - test-packages-firefox - build-pyodide-debug - create-xbuild-env - build-test-pyc-packages filters: branches: ignore: /.*/ tags: only: /^\d+\.\d+\.\w+$/ context: - s3-deployment - deploy-dev: requires: - test-core-firefox - test-packages-firefox - build-pyodide-debug - create-xbuild-env - build-test-pyc-packages filters: branches: only: main context: - s3-deployment