version: 2.1 defaults: &defaults working_directory: ~/repo docker: - image: pyodide/pyodide-env:latest environment: - EMSDK_NUM_CORES: 3 EMCC_CORES: 3 PYODIDE_JOBS: 3 # Make sure the ccache dir is consistent between core and package builds # (it's not the case otherwise) CCACHE_DIR: /root/.ccache/ jobs: lint: <<: *defaults resource_class: small steps: - checkout - run: name: lint command: make lint test-docs: <<: *defaults resource_class: small steps: - checkout - run: name: Test docs command: | mkdir test-results pytest docs/sphinx_pyodide/tests --junitxml=test-results/junit.xml - store_test_results: path: test-results build-core: <<: *defaults steps: - checkout - restore_cache: keys: - -core-{{ checksum "Makefile.envs" }}-{{ checksum "cpython/Makefile" }}-v20210910- - run: name: build emsdk no_output_timeout: 1200 command: | # This is necessary to use the ccache from emsdk source pyodide_env.sh ccache -z make -C emsdk ccache -s # Set mtime for EM_CONFIG to avoid ccache cache misses touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten - run: name: build cpython no_output_timeout: 1200 command: | # This is necessary to use the ccache from emsdk source pyodide_env.sh ccache -z make -C cpython ccache -s - run: name: build pyodide core no_output_timeout: 1200 command: | # This is necessary to use the ccache from emsdk source pyodide_env.sh ccache -z PYODIDE_PACKAGES="core" make ccache -s - run: name: check-size command: ls -lh build/ - save_cache: paths: - /root/.ccache key: -core-{{ checksum "Makefile.envs" }}-{{ checksum "cpython/Makefile" }}-v20210910- - run: name: Clean up workspace command: | rm -rf cpython/{build,downloads} rm -rf emsdk/emsdk/binaryen - run: name: Zip build directory command: | tar cjf pyodide-build.tar.gz build - persist_to_workspace: root: . paths: - . - store_artifacts: path: /root/repo/build/ - store_artifacts: path: /root/repo/pyodide-build.tar.gz - store_artifacts: path: /root/repo/packages/build-logs build-packages-no-numpy-dependents: <<: *defaults resource_class: large steps: - checkout - attach_workspace: at: . - restore_cache: keys: - -pkg-{{ checksum "Makefile.envs" }}-v20210911- - run: name: build packages no_output_timeout: 1800 command: | source pyodide_env.sh # Set mtime for EM_CONFIG to avoid ccache cache misses touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten ccache -z PYODIDE_PACKAGES='*, no-numpy-dependents' make -C packages ccache -s environment: PYODIDE_JOBS: 5 - run: name: check-size command: ls -lh build/ - save_cache: paths: - /root/.ccache key: -pkg-{{ checksum "Makefile.envs" }}-v20210911- - persist_to_workspace: root: . paths: - ./packages - ./build - run: name: Zip build directory command: | tar cjf pyodide-build.tar.gz build - persist_to_workspace: root: . paths: - . - store_artifacts: path: /root/repo/build/ - store_artifacts: path: /root/repo/pyodide-build.tar.gz - store_artifacts: path: /root/repo/packages/build-logs build-packages: <<: *defaults resource_class: large steps: - checkout - attach_workspace: at: . - restore_cache: keys: - -pkg-{{ checksum "Makefile.envs" }}-v20210911- - run: name: build packages no_output_timeout: 1800 command: | source pyodide_env.sh # Set mtime for EM_CONFIG to avoid ccache cache misses touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten ccache -z PYODIDE_PACKAGES='*' make -C packages ccache -s environment: PYODIDE_JOBS: 5 - run: name: check-size command: ls -lh build/ - save_cache: paths: - /root/.ccache key: -pkg-{{ checksum "Makefile.envs" }}-v20210911- - run: name: Zip build directory command: | tar cjf pyodide-build.tar.gz build - persist_to_workspace: root: . paths: - ./packages/.artifacts - ./build - store_artifacts: path: /root/repo/build/ - store_artifacts: path: /root/repo/pyodide-build.tar.gz - store_artifacts: path: /root/repo/packages/build-logs test-main: parameters: test-params: description: The tests to run. type: string <<: *defaults resource_class: medium+ steps: - attach_workspace: at: . - run: name: test command: | mkdir test-results tools/pytest_wrapper.py \ --junitxml=test-results/junit.xml \ --verbose \ --durations 10 \ << parameters.test-params >> - store_test_results: path: test-results benchmark-stack-size: <<: *defaults steps: - attach_workspace: at: . - run: name: stack-size command: | pytest -s benchmark/stack_usage.py | sed -n 's/## //pg' test-python: # Unlike test-main this uses "resource_class small" and doesn't use "pytest_wrapper." <<: *defaults resource_class: small steps: - checkout - run: name: test command: | mkdir test-results pytest \ --junitxml=test-results/junit.xml \ --verbose \ -k 'not (chrome or firefox or node)' \ --cov=pyodide_build --cov=pyodide \ src pyodide-build packages/micropip/ - store_test_results: path: test-results test-js: <<: *defaults resource_class: small steps: - attach_workspace: at: . - run: name: test command: | cd src/js npx tsd npm ci npm test # - run: # name: check if webpack cli works well with load-pyodide.js # command: | # git clone https://github.com/pyodide/pyodide-webpack-example.git # cd pyodide-webpack-example # npm ci # cp ../src/js/*.js node_modules/pyodide/ # npx webpack benchmark: <<: *defaults resource_class: medium+ steps: - checkout - attach_workspace: at: . - run: name: benchmark command: | python benchmark/benchmark.py /usr/local/bin/python3 build/benchmarks.json - store_artifacts: path: /root/repo/build/benchmarks.json deploy-release: # To reduce chance of deployment issues, try to keep the steps here as # similar as possible to the steps in deploy-dev! resource_class: small docker: - image: cibuilds/github:0.13 steps: - checkout - attach_workspace: at: . - run: name: Install requirements command: | apk add --no-cache --update python3 make python3 -m pip install awscli - run: name: Deploy Github Releases command: | cp -r build /tmp/pyodide cd /tmp/ tar cjf pyodide-build-${CIRCLE_TAG}.tar.bz2 pyodide/ ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" -delete "${CIRCLE_TAG}" ./pyodide-build-${CIRCLE_TAG}.tar.bz2 - run: name: Set PYODIDE_BASE_URL command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make update_base_url - run: name: Deploy to pyodide-cdn2.iodide.io command: | find build/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;" aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*.data' --cache-control 'max-age=30758400, immutable, public' --content-encoding 'gzip' # 1 year cache aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, immutable, public' --content-type 'application/wasm' --content-encoding 'gzip' # 1 year # update 301 redirect for the /latest/* route. aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json deploy-dev: # To reduce chance of deployment issues, try to keep the steps here as # similar as possible to the steps in deploy-release! resource_class: small docker: - image: cibuilds/github:0.13 steps: - checkout - attach_workspace: at: . - run: name: Install requirements command: | apk add --no-cache --update python3 make python3 -m pip install awscli - run: name: Set PYODIDE_BASE_URL command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make update_base_url - run: name: Deploy to pyodide-cdn2.iodide.io command: | find build/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;" aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/full/" aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip' # 1 hour cache aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public' --content-type 'application/wasm' --content-encoding 'gzip' # 1 hour cache workflows: version: 2 build-and-deploy: jobs: - lint: filters: tags: only: /.*/ - test-docs: filters: tags: only: /.*/ - test-python: filters: tags: only: /.*/ - build-core: filters: tags: only: /.*/ - build-packages-no-numpy-dependents: requires: - build-core filters: tags: only: /.*/ - build-packages: requires: - build-packages-no-numpy-dependents filters: tags: only: /.*/ - test-main: name: test-core-chrome test-params: -k "chrome and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ requires: - build-core filters: tags: only: /.*/ - test-main: name: test-core-firefox test-params: -k "firefox and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ requires: - build-core filters: tags: only: /.*/ - test-main: name: test-core-node test-params: -k node src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ requires: - build-core filters: tags: only: /.*/ - test-main: name: test-core-chrome-webworker test-params: -k chrome src/tests/test_webworker.py requires: - test-core-chrome filters: tags: only: /.*/ - test-main: name: test-core-firefox-webworker test-params: -k firefox src/tests/test_webworker.py requires: - test-core-firefox filters: tags: only: /.*/ - test-main: name: test-packages-chrome test-params: -k chrome packages/test* packages/*/test* requires: - build-packages filters: tags: only: /.*/ - test-main: name: test-packages-firefox test-params: -k firefox packages/test* packages/*/test* requires: - build-packages filters: tags: only: /.*/ - test-main: name: test-packages-node test-params: -k "node and not numpy" packages/test* packages/*/test* requires: - build-packages filters: tags: only: /.*/ - test-js: requires: - build-core filters: tags: only: /.*/ - benchmark-stack-size: requires: - build-core filters: tags: only: /.*/ - benchmark: requires: - build-packages filters: tags: only: /.*/ - deploy-release: requires: - lint - test-python - test-core-firefox - test-packages-firefox filters: branches: ignore: /.*/ tags: only: /^\d+\.\d+\.\w+$/ - deploy-dev: requires: - lint - test-python - test-core-firefox - test-packages-firefox filters: branches: only: main