pyodide/.circleci/config.yml

898 lines
26 KiB
YAML

version: 2.1
defaults: &defaults
working_directory: ~/repo
docker:
# Note: when updating the docker image version,
# make sure there are no extra old versions lying around.
# (e.g. `rg -F --hidden <old_tag>`)
- image: pyodide/pyodide-env:20230506-chrome112-firefox112-py311
environment:
- EMSDK_NUM_CORES: 3
EMCC_CORES: 3
PYODIDE_JOBS: 3
# Make sure the ccache dir is consistent between core and package builds
# (it's not the case otherwise)
CCACHE_DIR: /root/.ccache/
# Disable the compression of wheels, so they are better compressed by the CDN
PYODIDE_ZIP_COMPRESSION_LEVEL: 0
orbs:
macos: circleci/macos@2.3.6
jobs:
build-core:
<<: *defaults
steps:
- checkout
- restore_cache:
keys:
- -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
- -core-v20220915-{{ checksum "cpython/Makefile" }}
- -core-v20220915
- run:
name: calculate build hash for ccache
command: |
pip install pathspec
./tools/calculate_build_cache_key.py > build_hash.txt
- run:
name: build emsdk
no_output_timeout: 20m
command: |
# This is necessary to use the ccache from emsdk
source pyodide_env.sh
ccache -z
make -C emsdk
ccache -s
# Set mtime for EM_CONFIG to avoid ccache cache misses
touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten
- run:
name: build cpython
no_output_timeout: 20m
command: |
# This is necessary to use the ccache from emsdk
source pyodide_env.sh
ccache -z
make -C cpython
ccache -s
- run:
name: build pyodide core
no_output_timeout: 20m
command: |
# This is necessary to use the ccache from emsdk
source pyodide_env.sh
ccache -z
PYODIDE_PACKAGES="tag:core" make
ccache -s
- run:
name: check-size
command: |
du dist/ -abh --max-depth 1 | sort -k 2
pip install brotli
./tools/check_compressed_size.py dist/pyodide.asm.* dist/python_stdlib*
- save_cache:
paths:
- /root/.ccache
key: -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
- run:
name: Clean up workspace
command: |
rm -rf cpython/{build,downloads}
rm -rf emsdk/emsdk/binaryen
- persist_to_workspace:
root: .
paths:
- .
- run:
name: Zip build directory
command: |
tar cjf pyodide.tar.gz dist
tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm} dist/{package,pyodide-lock}.json dist/python_stdlib.zip
- store_artifacts:
path: /root/repo/dist/
- store_artifacts:
path: /root/repo/pyodide.tar.gz
- store_artifacts:
path: /root/repo/pyodide-core.tar.gz
- store_artifacts:
path: /root/repo/packages/build-logs
build-packages:
parameters:
packages:
description: The packages to be built.
type: string
<<: *defaults
resource_class: large
steps:
- checkout
- attach_workspace:
at: .
- restore_cache:
keys:
- -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >>
- run:
name: build packages
no_output_timeout: 60m
command: |
source pyodide_env.sh
# Set mtime for EM_CONFIG to avoid ccache cache misses
touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten
ccache -z
PYODIDE_PACKAGES='<< parameters.packages >>' make dist/pyodide-lock.json
ccache -s
environment:
PYODIDE_JOBS: 5
- run:
name: check-size
command: du dist/ -abh --max-depth 1 | sort -k 2
- run:
name: Zip build directory
command: |
tar cjf pyodide.tar.gz dist
tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm} dist/{package,pyodide-lock}.json dist/python_stdlib.zip
tar cjf build-logs.tar.gz packages/build-logs
- run:
name: Clean up package source files
command: |
cd packages && find **/build ! -name '.packaged' -type f -exec rm -f {} +
- store_artifacts:
path: /root/repo/pyodide.tar.gz
- store_artifacts:
path: /root/repo/pyodide-core.tar.gz
- store_artifacts:
path: /root/repo/build-logs.tar.gz
- save_cache:
paths:
- /root/.ccache
key: -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >>
build-pyodide-debug:
<<: *defaults
resource_class: large
steps:
- checkout
- attach_workspace:
at: .
- run:
name: build pyodide debug
command: |
cp -r dist dist-release
rm dist/pyodide.asm.js
source pyodide_env.sh
ccache -z
PYODIDE_DEBUG=1 make dist/pyodide.asm.js
ccache -s
cd dist
npx prettier -w pyodide.asm.js
npx prettier -w pyodide.js
cd ..
mv dist dist-debug
mv dist-release dist
- persist_to_workspace:
root: .
paths:
- .
create-xbuild-env:
<<: *defaults
steps:
- checkout
- attach_workspace:
at: .
- run:
name: create xbuild environment
command: |
pip install -e ./pyodide-build
pyodide xbuildenv create .
- run:
name: Zip xbuild environment
command: |
tar cjf xbuildenv.tar.gz ./xbuildenv/
- store_artifacts:
path: /root/repo/xbuildenv.tar.gz
- persist_to_workspace:
root: .
paths:
- ./xbuildenv
build-test-pyc-packages:
<<: *defaults
steps:
- checkout
- attach_workspace:
at: .
- run:
name: Install requirements
command: |
pip install -r requirements.txt
pip install -e ./pyodide-build
npm install -g node-fetch@2
- run:
name: Py-compile packages
command: |
pyodide py-compile --compression-level 0 dist/
- run:
name: Test import of py-compiled packages
command: |
tools/pytest_wrapper.py \
--junitxml=test-results/junit.xml \
--verbose \
--durations 50 \
--benchmark-json=benchmark-time.json \
"packages/_tests/test_packages_common.py::test_import" -k "node"
mv dist/ dist-pyc/
- persist_to_workspace:
root: .
paths:
- ./dist-pyc
test-main:
parameters:
test-params:
description: The tests to run.
type: string
cache-dir:
description: pytest-cache-dir.
type: string
default: ""
<<: *defaults
resource_class: medium+
steps:
- attach_workspace:
at: .
- run:
name: test
command: |
make npm-link
mkdir test-results
pip install -r requirements.txt
pip install -e ./pyodide-build
npm install -g node-fetch@2
if [ -z "<< parameters.cache-dir >>" ]; then
export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
else
export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
fi
echo "pytest cache dir: $CACHE_DIR"
source pyodide_env.sh
tools/pytest_wrapper.py \
--junitxml=test-results/junit.xml \
--verbose \
--durations 50 \
--benchmark-json=benchmark-time.json \
--benchmark-columns=mean,min,max,stddev \
<< parameters.test-params >> \
-o cache_dir=$CACHE_DIR
- store_test_results:
path: test-results
test-main-macos:
parameters:
test-params:
description: The tests to run.
type: string
cache-dir:
description: pytest-cache-dir.
type: string
default: ""
resource_class: macos.x86.medium.gen2
macos:
xcode: 14.3.1
working_directory: ~/repo
steps:
- attach_workspace:
at: .
# The standard way of enabling safaridriver no longer works for Safari 14+
# https://blog.bytesguy.com/enabling-remote-automation-in-safari-14
- macos/add-safari-permissions
- run:
name: install miniforge
command: |
curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
bash Miniforge3-MacOSX-x86_64.sh -b && rm -f Miniforge3-MacOSX-x86_64.sh
~/miniforge3/bin/conda create -n pyodide python=3.11 -y
- run:
name: install dependencies
command: |
export PATH="$HOME/miniforge3/bin:$PATH"
conda create -n pyodide python=3.11 -y
source activate pyodide
python -m pip install -r requirements.txt
pip install -e ./pyodide-build
- run:
name: test safari
command: |
export PATH="$HOME/miniforge3/bin:$PATH"
source activate pyodide
mkdir test-results
if [ -z "<< parameters.cache-dir >>" ]; then
export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
else
export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
fi
echo "pytest cache dir: $CACHE_DIR"
tools/pytest_wrapper.py \
--junitxml=test-results/junit.xml \
--verbose \
--durations 50 \
<< parameters.test-params >> \
-o cache_dir=$CACHE_DIR
- store_test_results:
path: test-results
benchmark-stack-size:
<<: *defaults
steps:
- attach_workspace:
at: .
- run:
name: stack-size
command: |
make npm-link
pip install -r requirements.txt
npm install -g node-fetch@2
pytest -s benchmark/stack_usage.py --rt node,chrome,firefox | sed -n 's/## //pg'
test-js:
<<: *defaults
resource_class: small
steps:
- attach_workspace:
at: .
- run:
name: test
command: |
cd src/test-js
npm ci
npm link ../../dist
npm run test-types
npm test
# - run:
# name: check if Pyodide works with Webpack
# command: |
# git clone https://github.com/pyodide/pyodide-webpack-example.git
# export DEV_PYODIDE_PATH=`realpath dist`
# cd pyodide-webpack-example
# git checkout 164054a9c6fbd2176f386b6552ed8d079c6bcc04
# ./build.sh
- run:
name: test npm deploy (dry run)
command: |
DRY_RUN=1 ./tools/deploy_to_npm.sh
test-cmdline-runner:
<<: *defaults
steps:
- attach_workspace:
at: .
- run:
name: test
command: |
set -x
export PYODIDE_ROOT=$(pwd)
echo $PYODIDE_ROOT
source pyodide_env.sh
cd ~
mkdir test
cd test
git clone https://github.com/python-attrs/attrs --depth 1 --branch 22.1.0
git clone https://github.com/zopefoundation/zope.interface.git --depth 1 --branch 5.4.0
python -m venv .venv-host
source .venv-host/bin/activate
pip install $PYODIDE_ROOT/pyodide-build
pyodide venv .venv-pyodide
source .venv-pyodide/bin/activate
cd zope.interface
pyodide build
pip install dist/*.whl
cd ../attrs
pip install .[tests]
python -m pytest -k 'not mypy'
benchmark:
<<: *defaults
resource_class: medium+
steps:
- checkout
- attach_workspace:
at: .
- run:
name: install requirements
command: |
pip3 install numpy matplotlib pandas
pip install -r requirements.txt
npm install -g node-fetch@2
- run:
name: benchmark
command: |
python benchmark/benchmark.py all --output dist/benchmarks.json
- store_artifacts:
path: /root/repo/dist/benchmarks.json
deploy-release:
# To reduce chance of deployment issues, try to keep the steps here as
# similar as possible to the steps in deploy-dev!
resource_class: small
<<: *defaults
steps:
- checkout
- attach_workspace:
at: .
- run:
name: Install requirements
command: |
python3 -m pip install -e "./pyodide-build[deploy]"
wget https://github.com/tcnksm/ghr/releases/download/v0.16.0/ghr_v0.16.0_linux_amd64.tar.gz
tar xzf ghr_v0.16.0_linux_amd64.tar.gz
mv ghr_v0.16.0_linux_amd64/ghr /tmp/ghr-bin
- run:
name: Deploy Github Releases
command: |
mkdir -p /tmp/ghr/dist
cp -r dist /tmp/ghr/pyodide
cp -r xbuildenv /tmp/ghr/xbuildenv
cd /tmp/ghr
tar cjf dist/pyodide-${CIRCLE_TAG}.tar.bz2 pyodide/
tar cjf dist/pyodide-core-${CIRCLE_TAG}.tar.bz2 pyodide/pyodide{.js,.mjs,.asm.js,.asm.wasm} pyodide/{package,pyodide-lock}.json pyodide/python_stdlib.zip
tar cjf dist/xbuildenv-${CIRCLE_TAG}.tar.bz2 xbuildenv/
/tmp/ghr-bin -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" \
-r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" \
-delete "${CIRCLE_TAG}" \
dist
- run:
name: Deploy to npm
command: |
./tools/deploy_to_npm.sh
- run:
name: Set PYODIDE_BASE_URL
command: |
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/pyc/" make dist/console.html
cp dist/console.html dist-pyc/console.html
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/debug/" make dist/console.html
cp dist/console.html dist-debug/console.html
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make dist/console.html
- run:
name: Deploy to S3
command: |
python3 tools/deploy_s3.py dist/ "v${CIRCLE_TAG}/full/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=30758400, immutable, public'
- run:
name: Deploy debug version to S3
command: |
python3 tools/deploy_s3.py dist-debug/ "v${CIRCLE_TAG}/debug/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=30758400, immutable, public'
- run:
name: Deploy the pyc version to S3
command: |
python3 tools/deploy_s3.py dist-pyc/ "v${CIRCLE_TAG}/pyc/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=30758400, immutable, public'
deploy-dev:
# To reduce chance of deployment issues, try to keep the steps here as
# similar as possible to the steps in deploy-release!
resource_class: small
<<: *defaults
steps:
- checkout
- attach_workspace:
at: .
- run:
name: Install requirements
command: |
python3 -m pip install -e "./pyodide-build[deploy]"
- run:
name: Set PYODIDE_BASE_URL
command: |
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/pyc/" make dist/console.html
cp dist/console.html dist-pyc/console.html
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/debug/" make dist/console.html
cp dist/console.html dist-debug/console.html
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make dist/console.html
- run:
name: Deploy to S3
command: |
python3 tools/deploy_s3.py dist/ "dev/full/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=3600, public' --rm-remote-prefix
- run:
name: Deploy debug version to S3
command: |
python3 tools/deploy_s3.py dist-debug/ "dev/debug/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=3600, public' --rm-remote-prefix
- run:
name: Deploy pyc version to S3
command: |
python3 tools/deploy_s3.py dist-pyc/ "dev/pyc/" --bucket "pyodide-cdn2.iodide.io" --cache-control 'max-age=3600, public' --rm-remote-prefix
- run:
# Unlike the release version, we upload the dev version to S3 not to GitHub.
name: Deploy release files to S3
command: |
mkdir -p /tmp/ghr/dist
cp -r dist /tmp/ghr/pyodide
cp -r xbuildenv /tmp/ghr/xbuildenv
cd /tmp/ghr
tar cjf dist/pyodide-core.tar.bz2 pyodide/pyodide{.js,.mjs,.asm.js,.asm.wasm} pyodide/{package,pyodide-lock}.json pyodide/python_stdlib.zip
tar cjf dist/xbuildenv.tar.bz2 xbuildenv/
cd -
python3 tools/deploy_s3.py /tmp/ghr/dist/ "xbuildenv/dev" --bucket "pyodide-cache" --cache-control 'max-age=3600, public' --overwrite \
--access-key-env "AWS_ACCESS_KEY_ID_CACHE" --secret-key-env "AWS_SECRET_ACCESS_KEY_CACHE"
workflows:
version: 2
build-and-deploy:
jobs:
- build-core:
filters:
tags:
only: /.*/
- build-packages:
name: build-libraries
packages: "tag:library"
requires:
- build-core
filters:
tags:
only: /.*/
post-steps:
- persist_to_workspace:
root: .
paths:
- ./packages
- ./dist
- build-packages:
name: build-packages-no-numpy-dependents
packages: "*,no-numpy-dependents"
requires:
- build-core
- build-libraries
filters:
tags:
only: /.*/
post-steps:
- persist_to_workspace:
root: .
paths:
- ./packages
- ./dist
- build-packages:
name: build-packages-opencv-python
packages: opencv-python
requires:
- build-packages-no-numpy-dependents
filters:
tags:
only: /.*/
post-steps:
- persist_to_workspace:
root: .
paths:
- ./packages/opencv-python/build
- ./packages/opencv-python/dist
- ./packages/build-logs/opencv*
- ./dist/opencv*
- build-packages:
name: build-packages-numpy-dependents
packages: "*,!opencv-python"
requires:
- build-packages-no-numpy-dependents
filters:
tags:
only: /.*/
post-steps:
- persist_to_workspace:
root: .
paths:
- ./packages
- ./dist
- build-packages:
name: build-packages
packages: "*"
requires:
- build-packages-numpy-dependents
- build-packages-opencv-python
filters:
tags:
only: /.*/
post-steps:
- persist_to_workspace:
root: .
paths:
- ./packages/.artifacts
- ./dist
- test-main:
name: test-core-chrome
test-params: --runtime=chrome-no-host -k "not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
requires:
- build-core
filters:
tags:
only: /.*/
- test-main:
name: test-core-firefox
test-params: --runtime=firefox-no-host -k "not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
requires:
- build-core
filters:
tags:
only: /.*/
- test-main:
name: test-core-node
test-params: --runtime=node-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-build/pyodide_build/tests
requires:
- build-core
filters:
tags:
only: /.*/
- test-main-macos:
name: test-core-safari
test-params: --runtime=safari-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
requires:
- build-core
filters:
tags:
only: /.*/
- test-main:
name: test-core-chrome-webworker
test-params: --runtime=chrome-no-host src/tests/test_webworker.py
requires:
- test-core-chrome
filters:
tags:
only: /.*/
- test-main:
name: test-core-firefox-webworker
test-params: --runtime=firefox-no-host src/tests/test_webworker.py
requires:
- test-core-firefox
filters:
tags:
only: /.*/
- test-main:
name: test-packages-chrome-no-numpy-dependents
test-params: --runtime=chrome-no-host packages/*/test*.py
cache-dir: .pytest_cache_chrome
requires:
- build-packages-no-numpy-dependents
filters:
tags:
only: /.*/
post-steps:
- persist_to_workspace:
root: .
paths:
- ./.test_cache
- test-main:
name: test-packages-chrome
test-params: --runtime=chrome-no-host packages/*/test*.py --skip-passed
cache-dir: .pytest_cache_chrome
requires:
- test-packages-chrome-no-numpy-dependents
- build-packages
filters:
tags:
only: /.*/
- test-main:
name: test-packages-firefox-no-numpy-dependents
test-params: --runtime=firefox-no-host packages/*/test*.py
cache-dir: .pytest_cache_firefox
requires:
- build-packages-no-numpy-dependents
filters:
tags:
only: /.*/
post-steps:
- persist_to_workspace:
root: .
paths:
- ./.test_cache
- test-main:
name: test-packages-firefox
test-params: --runtime=firefox-no-host packages/*/test*.py --skip-passed
cache-dir: .pytest_cache_firefox
requires:
- test-packages-firefox-no-numpy-dependents
- build-packages
filters:
tags:
only: /.*/
- test-main:
name: test-packages-node-no-numpy-dependents
test-params: --runtime=node-no-host packages/*/test*.py
cache-dir: .pytest_cache_node
requires:
- build-packages-no-numpy-dependents
filters:
tags:
only: /.*/
post-steps:
- persist_to_workspace:
root: .
paths:
- ./.test_cache
- test-main:
name: test-packages-node
test-params: --runtime=node-no-host packages/*/test*.py --skip-passed
cache-dir: .pytest_cache_node
requires:
- test-packages-node-no-numpy-dependents
- build-packages
filters:
tags:
only: /.*/
- test-main-macos:
name: test-packages-safari-no-numpy-dependents
test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py
cache-dir: .pytest_cache_safari
requires:
- build-packages-no-numpy-dependents
filters:
tags:
only: /.*/
post-steps:
- persist_to_workspace:
root: .
paths:
- ./.test_cache
- test-main-macos:
name: test-packages-safari
test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py --skip-passed
cache-dir: .pytest_cache_safari
requires:
- test-packages-safari-no-numpy-dependents
- build-packages
filters:
tags:
only: /.*/
- test-js:
requires:
- build-core
filters:
tags:
only: /.*/
- benchmark-stack-size:
requires:
- build-core
filters:
tags:
only: /.*/
- benchmark:
requires:
- build-packages
filters:
tags:
only: /.*/
- create-xbuild-env:
requires:
- build-packages
filters:
tags:
only: /.*/
- test-cmdline-runner:
requires:
- build-packages
filters:
tags:
only: /.*/
- build-pyodide-debug:
requires:
- build-packages
filters:
tags:
only: /.*/
- build-test-pyc-packages:
requires:
- build-packages
filters:
tags:
only: /.*/
- deploy-release:
requires:
- test-core-firefox
- test-packages-firefox
- build-pyodide-debug
- create-xbuild-env
- build-test-pyc-packages
filters:
branches:
ignore: /.*/
tags:
only: /^\d+\.\d+\.\w+$/
context:
- s3-deployment
- deploy-dev:
requires:
- test-core-firefox
- test-packages-firefox
- build-pyodide-debug
- create-xbuild-env
- build-test-pyc-packages
filters:
branches:
only: main
context:
- s3-deployment