mirror of https://github.com/pyodide/pyodide.git
603 lines
18 KiB
YAML
603 lines
18 KiB
YAML
version: 2.1
|
|
|
|
defaults: &defaults
|
|
working_directory: ~/repo
|
|
docker:
|
|
# Note: when updating the docker image version,
|
|
# make sure there are no extra old versions lying around.
|
|
# (e.g. `rg -F --hidden <old_tag>`)
|
|
- image: pyodide/pyodide-env:20220525-py310-chrome102-firefox100
|
|
environment:
|
|
- EMSDK_NUM_CORES: 3
|
|
EMCC_CORES: 3
|
|
PYODIDE_JOBS: 3
|
|
# Make sure the ccache dir is consistent between core and package builds
|
|
# (it's not the case otherwise)
|
|
CCACHE_DIR: /root/.ccache/
|
|
|
|
jobs:
|
|
test-docs:
|
|
<<: *defaults
|
|
resource_class: small
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Test docs
|
|
command: |
|
|
mkdir test-results
|
|
pytest docs/sphinx_pyodide/tests --junitxml=test-results/junit.xml
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
build-core:
|
|
<<: *defaults
|
|
steps:
|
|
- checkout
|
|
|
|
- restore_cache:
|
|
keys:
|
|
- -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
|
|
- -core-v20220303-{{ checksum "cpython/Makefile" }}
|
|
- -core-v20220303
|
|
|
|
- run:
|
|
name: build emsdk
|
|
no_output_timeout: 20m
|
|
command: |
|
|
# This is necessary to use the ccache from emsdk
|
|
source pyodide_env.sh
|
|
|
|
ccache -z
|
|
make -C emsdk
|
|
ccache -s
|
|
|
|
# Set mtime for EM_CONFIG to avoid ccache cache misses
|
|
touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten
|
|
|
|
- run:
|
|
name: build cpython
|
|
no_output_timeout: 20m
|
|
command: |
|
|
# This is necessary to use the ccache from emsdk
|
|
source pyodide_env.sh
|
|
|
|
ccache -z
|
|
make -C cpython
|
|
ccache -s
|
|
|
|
- run:
|
|
name: build pyodide core
|
|
no_output_timeout: 20m
|
|
command: |
|
|
# This is necessary to use the ccache from emsdk
|
|
source pyodide_env.sh
|
|
|
|
ccache -z
|
|
PYODIDE_PACKAGES="core" make
|
|
ccache -s
|
|
|
|
- run:
|
|
name: check-size
|
|
command: du dist/ -abh --max-depth 1 | sort -k 2
|
|
|
|
- save_cache:
|
|
paths:
|
|
- /root/.ccache
|
|
key: -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
|
|
|
|
- run:
|
|
name: Clean up workspace
|
|
command: |
|
|
rm -rf cpython/{build,downloads}
|
|
rm -rf emsdk/emsdk/binaryen
|
|
|
|
- run:
|
|
name: Zip build directory
|
|
command: |
|
|
tar cjf pyodide-build.tar.gz dist
|
|
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- .
|
|
|
|
- store_artifacts:
|
|
path: /root/repo/dist/
|
|
|
|
- store_artifacts:
|
|
path: /root/repo/pyodide-build.tar.gz
|
|
|
|
- store_artifacts:
|
|
path: /root/repo/packages/build-logs
|
|
|
|
build-packages:
|
|
parameters:
|
|
packages:
|
|
description: The packages to be built.
|
|
type: string
|
|
<<: *defaults
|
|
resource_class: large
|
|
steps:
|
|
- checkout
|
|
|
|
- attach_workspace:
|
|
at: .
|
|
|
|
- restore_cache:
|
|
keys:
|
|
- -pkg2-v20220303-{{ checksum "Makefile.envs" }}
|
|
- -pkg2-v20220303
|
|
|
|
- run:
|
|
name: install rust
|
|
command: |
|
|
make rust
|
|
|
|
- run:
|
|
name: build packages
|
|
no_output_timeout: 60m
|
|
command: |
|
|
source pyodide_env.sh
|
|
|
|
# Set mtime for EM_CONFIG to avoid ccache cache misses
|
|
touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten
|
|
|
|
ccache -z
|
|
PYODIDE_PACKAGES='<< parameters.packages >>' make -C packages
|
|
ccache -s
|
|
environment:
|
|
PYODIDE_JOBS: 5
|
|
|
|
- run:
|
|
name: check-size
|
|
command: du dist/ -abh --max-depth 1 | sort -k 2
|
|
|
|
- save_cache:
|
|
paths:
|
|
- /root/.ccache
|
|
key: -pkg2-v20220303-{{ checksum "Makefile.envs" }}
|
|
|
|
- run:
|
|
name: Zip build directory
|
|
command: |
|
|
tar --exclude="node_modules" -cjf pyodide-build.tar.gz dist
|
|
tar cjf build-logs.tar.gz packages/build-logs
|
|
|
|
- store_artifacts:
|
|
path: /root/repo/pyodide-build.tar.gz
|
|
|
|
- store_artifacts:
|
|
path: /root/repo/build-logs.tar.gz
|
|
|
|
build-pyodide-debug:
|
|
<<: *defaults
|
|
resource_class: large
|
|
steps:
|
|
- checkout
|
|
|
|
- attach_workspace:
|
|
at: .
|
|
|
|
- run:
|
|
name: build pyodide debug
|
|
command: |
|
|
cp -r dist dist-release
|
|
rm dist/pyodide.asm.js
|
|
source pyodide_env.sh
|
|
ccache -z
|
|
PYODIDE_SYMBOLS=1 make dist/pyodide.asm.js
|
|
ccache -s
|
|
cd dist
|
|
npx prettier -w pyodide.asm.js
|
|
npx prettier -w pyodide.js
|
|
cd ..
|
|
mv dist dist-debug
|
|
mv dist-release dist
|
|
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- .
|
|
|
|
test-main:
|
|
parameters:
|
|
test-params:
|
|
description: The tests to run.
|
|
type: string
|
|
<<: *defaults
|
|
resource_class: medium+
|
|
steps:
|
|
- attach_workspace:
|
|
at: .
|
|
- run:
|
|
name: test
|
|
command: |
|
|
make npm-link
|
|
mkdir test-results
|
|
tools/pytest_wrapper.py \
|
|
--junitxml=test-results/junit.xml \
|
|
--verbose \
|
|
--durations 50 \
|
|
<< parameters.test-params >>
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
benchmark-stack-size:
|
|
<<: *defaults
|
|
steps:
|
|
- attach_workspace:
|
|
at: .
|
|
- run:
|
|
name: stack-size
|
|
command: |
|
|
make npm-link
|
|
pytest -s benchmark/stack_usage.py | sed -n 's/## //pg'
|
|
|
|
test-python:
|
|
# Unlike test-main this uses "resource_class small" and doesn't use "pytest_wrapper."
|
|
<<: *defaults
|
|
resource_class: small
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: test
|
|
command: |
|
|
mkdir test-results
|
|
python3 -m pip install -e ./pyodide-build
|
|
PYODIDE_ROOT=. pytest \
|
|
--junitxml=test-results/junit.xml \
|
|
--verbose \
|
|
-k 'not (chrome or firefox or node)' \
|
|
--cov=pyodide_build --cov=pyodide \
|
|
src pyodide-build packages/micropip/ pyodide-test-runner
|
|
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
test-js:
|
|
<<: *defaults
|
|
resource_class: small
|
|
steps:
|
|
- attach_workspace:
|
|
at: .
|
|
- run:
|
|
name: test
|
|
command: |
|
|
cd src/test-js
|
|
npm ci
|
|
npm link ../../dist
|
|
npm run test-types
|
|
npm test
|
|
# - run:
|
|
# name: check if webpack cli works well with load-pyodide.js
|
|
# command: |
|
|
# git clone https://github.com/pyodide/pyodide-webpack-example.git
|
|
# cd pyodide-webpack-example
|
|
# npm ci
|
|
# cp ../src/js/*.js node_modules/pyodide/
|
|
# npx webpack
|
|
|
|
benchmark:
|
|
<<: *defaults
|
|
resource_class: medium+
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: .
|
|
- run:
|
|
name: install requirements
|
|
command: |
|
|
pip3 install numpy matplotlib pandas
|
|
- run:
|
|
name: benchmark
|
|
command: |
|
|
python benchmark/benchmark.py all --output dist/benchmarks.json
|
|
|
|
- store_artifacts:
|
|
path: /root/repo/dist/benchmarks.json
|
|
|
|
deploy-release:
|
|
# To reduce chance of deployment issues, try to keep the steps here as
|
|
# similar as possible to the steps in deploy-dev!
|
|
resource_class: small
|
|
|
|
docker:
|
|
- image: cibuilds/github:0.13
|
|
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: .
|
|
- run:
|
|
name: Install requirements
|
|
command: |
|
|
apk add --no-cache --update python3 make npm
|
|
python3 -m pip install awscli
|
|
- run:
|
|
name: Deploy Github Releases
|
|
command: |
|
|
cp -r dist /tmp/pyodide
|
|
cd /tmp/
|
|
tar cjf pyodide-build-${CIRCLE_TAG}.tar.bz2 pyodide/
|
|
ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" -delete "${CIRCLE_TAG}" ./pyodide-build-${CIRCLE_TAG}.tar.bz2
|
|
|
|
- run:
|
|
name: Deploy to npm
|
|
command: |
|
|
./tools/deploy_to_npm.sh
|
|
|
|
- run:
|
|
name: Deploy to PyPI
|
|
command: |
|
|
# Ask github actions to deploy for us
|
|
curl \
|
|
-X POST \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
-H "Authorization: token ${GITHUB_TOKEN}" \
|
|
https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/dispatches \
|
|
-d '{"event_type":"deploy-release","client_payload":{}}'
|
|
|
|
- run:
|
|
name: Set PYODIDE_BASE_URL
|
|
command: |
|
|
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/debug/" make dist/console.html
|
|
cp dist/console.html dist-debug/console.html
|
|
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make dist/console.html
|
|
- run:
|
|
name: Deploy to pyodide-cdn2.iodide.io
|
|
command: |
|
|
find dist/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
|
|
aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*.data' --cache-control 'max-age=30758400, immutable, public' --content-encoding 'gzip' # 1 year cache
|
|
aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, immutable, public' --content-type 'application/wasm' --content-encoding 'gzip' # 1 year
|
|
- run:
|
|
name: Deploy debug version to pyodide-cdn2.iodide.io
|
|
command: |
|
|
find dist-debug/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
|
|
aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/debug/" --exclude '*.data' --cache-control 'max-age=30758400, public' --content-encoding 'gzip' # 1 year cache
|
|
aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/debug/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, public' --content-type 'application/wasm' --content-encoding 'gzip' # 1 year cache
|
|
- run:
|
|
name: update 301 redirect for the /latest/* route.
|
|
command: |
|
|
aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json
|
|
|
|
deploy-dev:
|
|
# To reduce chance of deployment issues, try to keep the steps here as
|
|
# similar as possible to the steps in deploy-release!
|
|
resource_class: small
|
|
|
|
docker:
|
|
- image: cibuilds/github:0.13
|
|
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: .
|
|
- run:
|
|
name: Install requirements
|
|
command: |
|
|
apk add --no-cache --update python3 make
|
|
python3 -m pip install awscli
|
|
- run:
|
|
name: Set PYODIDE_BASE_URL
|
|
command: |
|
|
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/debug/" make dist/console.html
|
|
cp dist/console.html dist-debug/console.html
|
|
PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make dist/console.html
|
|
- run:
|
|
name: Deploy to pyodide-cdn2.iodide.io
|
|
command: |
|
|
find dist/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
|
|
aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/full/"
|
|
aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip' # 1 hour cache
|
|
aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public' --content-type 'application/wasm' --content-encoding 'gzip' # 1 hour cache
|
|
- run:
|
|
name: Deploy debug version to pyodide-cdn2.iodide.io
|
|
command: |
|
|
find dist-debug/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
|
|
aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/debug/"
|
|
aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/dev/debug/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip' # 1 hour cache
|
|
aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/dev/debug/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public' --content-type 'application/wasm' --content-encoding 'gzip' # 1 hour cache
|
|
|
|
workflows:
|
|
version: 2
|
|
build-and-deploy:
|
|
jobs:
|
|
- test-docs:
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- test-python:
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- build-core:
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- build-packages:
|
|
name: build-packages-no-numpy-dependents
|
|
packages: "*,no-numpy-dependents"
|
|
requires:
|
|
- build-core
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
post-steps:
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- ./packages
|
|
- ./dist
|
|
|
|
- build-packages:
|
|
name: build-packages-opencv-python
|
|
packages: opencv-python
|
|
requires:
|
|
- build-packages-no-numpy-dependents
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
post-steps:
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- ./packages/opencv-python/build
|
|
- ./packages/opencv-python/dist
|
|
- ./packages/build-logs/opencv*
|
|
- ./dist/opencv*
|
|
|
|
- build-packages:
|
|
name: build-packages-numpy-dependents
|
|
packages: "*,!opencv-python"
|
|
requires:
|
|
- build-packages-no-numpy-dependents
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
post-steps:
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- ./packages
|
|
- ./dist
|
|
|
|
- build-packages:
|
|
name: build-packages
|
|
packages: "*"
|
|
requires:
|
|
- build-packages-numpy-dependents
|
|
- build-packages-opencv-python
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
post-steps:
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- ./packages/.artifacts
|
|
- ./dist
|
|
|
|
- test-main:
|
|
name: test-core-chrome
|
|
test-params: -k "chrome and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-test-runner
|
|
requires:
|
|
- build-core
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- test-main:
|
|
name: test-core-firefox
|
|
test-params: -k "firefox and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-test-runner
|
|
requires:
|
|
- build-core
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- test-main:
|
|
name: test-core-node
|
|
test-params: -k node src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-test-runner
|
|
requires:
|
|
- build-core
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- test-main:
|
|
name: test-core-chrome-webworker
|
|
test-params: -k chrome src/tests/test_webworker.py
|
|
requires:
|
|
- test-core-chrome
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- test-main:
|
|
name: test-core-firefox-webworker
|
|
test-params: -k firefox src/tests/test_webworker.py
|
|
requires:
|
|
- test-core-firefox
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- test-main:
|
|
name: test-packages-chrome
|
|
test-params: -k chrome packages/test* packages/*/test*
|
|
requires:
|
|
- build-packages
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- test-main:
|
|
name: test-packages-firefox
|
|
test-params: -k firefox packages/test* packages/*/test*
|
|
requires:
|
|
- build-packages
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- test-main:
|
|
name: test-packages-node
|
|
test-params: -k node packages/test* packages/*/test*
|
|
requires:
|
|
- build-packages
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- test-js:
|
|
requires:
|
|
- build-core
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- benchmark-stack-size:
|
|
requires:
|
|
- build-core
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- benchmark:
|
|
requires:
|
|
- build-packages
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- build-pyodide-debug:
|
|
requires:
|
|
- build-packages
|
|
filters:
|
|
tags:
|
|
only: /.*/
|
|
|
|
- deploy-release:
|
|
requires:
|
|
- test-python
|
|
- test-core-firefox
|
|
- test-packages-firefox
|
|
- build-pyodide-debug
|
|
filters:
|
|
branches:
|
|
ignore: /.*/
|
|
tags:
|
|
only: /^\d+\.\d+\.\w+$/
|
|
- deploy-dev:
|
|
requires:
|
|
- test-python
|
|
- test-core-firefox
|
|
- test-packages-firefox
|
|
- build-pyodide-debug
|
|
filters:
|
|
branches:
|
|
only: main
|