pyodide/.circleci/config.yml

version: 2.1

defaults: &defaults
  working_directory: ~/repo
  docker:
    # Note: when updating the docker image version,
    #       make sure there are no extra old versions lying around.
    #       (e.g. `rg -F --hidden <old_tag>`)
    - image: pyodide/pyodide-env:20220629-py310-chrome102-firefox100
  environment:
    - EMSDK_NUM_CORES: 3
      EMCC_CORES: 3
      PYODIDE_JOBS: 3
      # Make sure the ccache dir is consistent between core and package builds
      # (it's not the case otherwise)
      CCACHE_DIR: /root/.ccache/

orbs:
  macos: circleci/macos@2.2.0

jobs:
  test-docs:
    <<: *defaults
    resource_class: small
    steps:
      - checkout
      - run:
          name: Test docs
          command: |
            mkdir test-results
            pip install pytest-pyodide
            pip install -e ./pyodide-build
            npm install -g node-fetch@2
            pytest docs/sphinx_pyodide/tests --junitxml=test-results/junit.xml
      - store_test_results:
          path: test-results

  build-core:
    <<: *defaults
    steps:
      - checkout

      - restore_cache:
          keys:
            - -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
            - -core-v20220303-{{ checksum "cpython/Makefile" }}
            - -core-v20220303

      - run:
          name: build emsdk
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C emsdk
            ccache -s

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

      - run:
          name: build cpython
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C cpython
            ccache -s

      - run:
          name: build pyodide core
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            PYODIDE_PACKAGES="core" make
            ccache -s

      - run:
          name: check-size
          command: du dist/ -abh --max-depth 1  | sort -k 2

      - save_cache:
          paths:
            - /root/.ccache
          key: -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}

      - run:
          name: Clean up workspace
          command: |
            rm -rf cpython/{build,downloads}
            rm -rf emsdk/emsdk/binaryen

      - persist_to_workspace:
          root: .
          paths:
            - .

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide.tar.gz dist
            tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm,.asm.data,_py.tar} dist/{package,repodata}.json

      - store_artifacts:
          path: /root/repo/dist/

      - store_artifacts:
          path: /root/repo/pyodide.tar.gz

      - store_artifacts:
          path: /root/repo/pyodide-core.tar.gz

      - store_artifacts:
          path: /root/repo/packages/build-logs

  build-packages:
    parameters:
      packages:
        description: The packages to be built.
        type: string
    <<: *defaults
    resource_class: large
    steps:
      - checkout

      - attach_workspace:
          at: .

      - restore_cache:
          keys:
            - -pkg2-v20220303-{{ checksum "Makefile.envs" }}
            - -pkg2-v20220303

      - run:
          name: install rust
          command: |
            make rust

      - run:
          name: build packages
          no_output_timeout: 60m
          command: |
            source pyodide_env.sh

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

            ccache -z
            PYODIDE_PACKAGES='<< parameters.packages >>' make -C packages
            ccache -s
          environment:
            PYODIDE_JOBS: 5

      - run:
          name: check-size
          command: du dist/ -abh --max-depth 1  | sort -k 2

      - save_cache:
          paths:
            - /root/.ccache
          key: -pkg2-v20220303-{{ checksum "Makefile.envs" }}

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide.tar.gz dist
            tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm,.asm.data,_py.tar} dist/{package,repodata}.json
            tar cjf build-logs.tar.gz  packages/build-logs

      - run:
          name: Clean up package source files
          command: |
            cd packages && find **/build ! -name '.packaged' -type f -exec rm -f {} +

      - store_artifacts:
          path: /root/repo/pyodide.tar.gz

      - store_artifacts:
          path: /root/repo/pyodide-core.tar.gz

      - store_artifacts:
          path: /root/repo/build-logs.tar.gz

  build-pyodide-debug:
    <<: *defaults
    resource_class: large
    steps:
      - checkout

      - attach_workspace:
          at: .

      - run:
          name: build pyodide debug
          command: |
            cp -r dist dist-release
            rm dist/pyodide.asm.js
            source pyodide_env.sh
            ccache -z
            PYODIDE_SYMBOLS=1 make dist/pyodide.asm.js
            ccache -s
            cd dist
            npx prettier -w pyodide.asm.js
            npx prettier -w pyodide.js
            cd ..
            mv dist dist-debug
            mv dist-release dist

      - persist_to_workspace:
          root: .
          paths:
            - .

  create-xbuild-env:
    <<: *defaults
    steps:
      - checkout

      - attach_workspace:
          at: .

      - run:
          name: create xbuild environment
          command: |
            pip install -e ./pyodide-build
            python -m pyodide_build create_xbuildenv

      - run:
          name: Zip xbuild environment
          command: |
            tar cjf xbuildenv.tar.gz  xbuildenv/

      - store_artifacts:
          path: /root/repo/xbuildenv.tar.gz

      - persist_to_workspace:
          root: .
          paths:
            - ./xbuildenv

  test-main:
    parameters:
      test-params:
        description: The tests to run.
        type: string
      cache-dir:
        description: pytest-cache-dir.
        type: string
        default: ""
    <<: *defaults
    resource_class: medium+
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            make npm-link
            mkdir test-results
            pip install pytest-pyodide
            pip install -r requirements.txt
            pip install -e pyodide-build
            npm install -g node-fetch@2
            if [ -z "<< parameters.cache-dir >>" ]; then
              export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
            else
              export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
            fi
            echo "pytest cache dir: $CACHE_DIR"
            tools/pytest_wrapper.py \
              --junitxml=test-results/junit.xml \
              --verbose \
              --durations 50 \
              --benchmark-json=benchmark-time.json \
              --benchmark-columns=mean,min,max,stddev \
              << parameters.test-params >> \
              -o cache_dir=$CACHE_DIR
      - store_test_results:
          path: test-results

  test-main-macos:
    parameters:
      test-params:
        description: The tests to run.
        type: string
      cache-dir:
        description: pytest-cache-dir.
        type: string
        default: ""
    resource_class: medium
    macos:
      xcode: 13.3.1

    working_directory: ~/repo
    steps:
      - attach_workspace:
          at: .
      # The standard way of enabling safaridriver no longer works for Safari 14+
      # https://blog.bytesguy.com/enabling-remote-automation-in-safari-14
      - macos/add-safari-permissions
      - run:
          name: install miniforge
          command: |
            curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
            bash Miniforge3-MacOSX-x86_64.sh -b && rm -f Miniforge3-MacOSX-x86_64.sh
            ~/miniforge3/bin/conda create -n pyodide python=3.10 -y
      - run:
          name: install dependencies
          command: |
            export PATH="$HOME/miniforge3/bin:$PATH"
            conda create -n pyodide python=3.10 -y
            source activate pyodide
            python -m pip install -r requirements.txt
            python -m pip install pytest-pyodide
      - run:
          name: test safari
          command: |
            export PATH="$HOME/miniforge3/bin:$PATH"
            source activate pyodide
            mkdir test-results
            pip install pytest-pyodide
            pip install -e ./pyodide-build
            if [ -z "<< parameters.cache-dir >>" ]; then
              export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
            else
              export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
            fi
            echo "pytest cache dir: $CACHE_DIR"
            tools/pytest_wrapper.py \
              --junitxml=test-results/junit.xml \
              --verbose \
              --durations 50 \
              << parameters.test-params >> \
              -o cache_dir=$CACHE_DIR
      - store_test_results:
          path: test-results

  benchmark-stack-size:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .
      - run:
          name: stack-size
          command: |
            make npm-link
            pip install pytest-pyodide
            npm install -g node-fetch@2
            pytest -s benchmark/stack_usage.py  | sed -n 's/## //pg'

  test-js:
    <<: *defaults
    resource_class: small
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            cd src/test-js
            npm ci
            npm link ../../dist
            npm run test-types
            npm test
      - run:
          name: check if Pyodide works with Webpack
          command: |
            git clone https://github.com/pyodide/pyodide-webpack-example.git
            export DEV_PYODIDE_PATH=`realpath dist`
            cd pyodide-webpack-example
            git checkout 164054a9c6fbd2176f386b6552ed8d079c6bcc04
            ./build.sh

  test-cmdline-runner:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .

      - run:
          name: test
          command: |
            set -x
            export PYODIDE_ROOT=$(pwd)
            echo $PYODIDE_ROOT
            cd ~
            mkdir test
            cd test
            git clone https://github.com/python-attrs/attrs --depth 1 --branch 22.1.0
            git clone https://github.com/zopefoundation/zope.interface.git --depth 1 --branch 5.4.0
            git clone https://github.com/nedbat/coveragepy.git --depth 1 --branch coverage-5.5

            python3.10 -m venv .venv-host
            source .venv-host/bin/activate
            pip install $PYODIDE_ROOT/pyodide-build
            pyodide venv .venv-pyodide
            source .venv-pyodide/bin/activate

            cd zope.interface
            pyodide build
            pip install dist/*.whl
            cd ../coveragepy/
            pyodide build
            pip install dist/*.whl
            cd ../attrs
            pyodide build
            WHEEL_NAME=$(echo dist/*.whl)
            pip install $WHEEL_NAME[tests]
            python -m pytest -k 'not mypy'

  benchmark:
    <<: *defaults
    resource_class: medium+
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: install requirements
          command: |
            pip3 install numpy matplotlib pandas
      - run:
          name: benchmark
          command: |
            pip install pytest-pyodide
            npm install -g node-fetch@2
            python benchmark/benchmark.py all --output dist/benchmarks.json

      - store_artifacts:
          path: /root/repo/dist/benchmarks.json

  deploy-release:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-dev!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make npm
            python3 -m pip install awscli

      - run:
          name: Deploy Github Releases
          command: |
            mkdir -p /tmp/ghr/dist
            cp -r dist /tmp/ghr/pyodide
            cp -r xbuildenv /tmp/ghr/xbuildenv
            cd /tmp/ghr
            tar cjf dist/pyodide-${CIRCLE_TAG}.tar.bz2  pyodide/
            tar cjf dist/pyodide-core-${CIRCLE_TAG}.tar.bz2 pyodide/pyodide{.js,.mjs,.asm.js,.asm.wasm,.asm.data,_py.tar} pyodide/{package,repodata}.json
            tar cjf dist/xbuildenv-${CIRCLE_TAG}.tar.bz2  xbuildenv/
            ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" \
              -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" \
              -delete "${CIRCLE_TAG}" \
              dist

      - run:
          name: Deploy to npm
          command: |
            ./tools/deploy_to_npm.sh

      - run:
          name: Deploy to PyPI
          command: |
            # Ask github actions to deploy for us
            export PAYLOAD={"ref":"${CIRCLE_TAG}","inputs":{}}
            curl \
              -X POST \
              -H "Accept: application/vnd.github.v3+json" \
              -H "Authorization: Bearer ${GITHUB_TOKEN}" \
              https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/dispatches/deploy-release \
              -d "$PAYLOAD"

      - run:
          name: Set PYODIDE_BASE_URL
          command: |
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/debug/" make dist/console.html
            cp dist/console.html dist-debug/console.html
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make dist/console.html
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find dist/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*.data' --cache-control 'max-age=30758400, immutable, public' --content-encoding 'gzip'    # 1 year cache
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, immutable, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year
      - run:
          name: Deploy debug version to pyodide-cdn2.iodide.io
          command: |
            find dist-debug/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/debug/" --exclude '*.data' --cache-control 'max-age=30758400, public' --content-encoding 'gzip'  # 1 year cache
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/debug/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year cache
      - run:
          name: update 301 redirect for the /latest/* route.
          command: |
            aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json

  deploy-dev:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-release!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make
            python3 -m pip install awscli
      - run:
          name: Set PYODIDE_BASE_URL
          command: |
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/debug/" make dist/console.html
            cp dist/console.html dist-debug/console.html
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make dist/console.html
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find dist/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/full/"
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache
      - run:
          name: Deploy debug version to pyodide-cdn2.iodide.io
          command: |
            find dist-debug/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/debug/"
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/dev/debug/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/dev/debug/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache

workflows:
  version: 2
  build-and-deploy:
    jobs:
      - test-docs:
          filters:
            tags:
              only: /.*/

      - build-core:
          filters:
            tags:
              only: /.*/

      - build-packages:
          name: build-packages-no-numpy-dependents
          packages: "*,no-numpy-dependents"
          requires:
            - build-core
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages
                  - ./dist

      - build-packages:
          name: build-packages-opencv-python
          packages: opencv-python
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages/opencv-python/build
                  - ./packages/opencv-python/dist
                  - ./packages/build-logs/opencv*
                  - ./dist/opencv*

      - build-packages:
          name: build-packages-numpy-dependents
          packages: "*,!opencv-python"
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages
                  - ./dist

      - build-packages:
          name: build-packages
          packages: "*"
          requires:
            - build-packages-numpy-dependents
            - build-packages-opencv-python
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages/.artifacts
                  - ./dist

      - test-main:
          name: test-core-chrome
          test-params: --runtime=chrome-no-host -k "not webworker" src packages/micropip  packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox
          test-params: --runtime=firefox-no-host -k "not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-node
          test-params: --runtime=node-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main-macos:
          name: test-core-safari
          test-params: --runtime=safari-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-chrome-webworker
          test-params: --runtime=chrome-no-host src/tests/test_webworker.py
          requires:
            - test-core-chrome
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox-webworker
          test-params: --runtime=firefox-no-host src/tests/test_webworker.py
          requires:
            - test-core-firefox
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-chrome-no-numpy-dependents
          test-params: --runtime=chrome-no-host packages/*/test*.py
          cache-dir: .pytest_cache_chrome
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main:
          name: test-packages-chrome
          test-params: --runtime=chrome-no-host packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_chrome
          requires:
            - test-packages-chrome-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-firefox-no-numpy-dependents
          test-params: --runtime=firefox-no-host packages/*/test*.py
          cache-dir: .pytest_cache_firefox
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main:
          name: test-packages-firefox
          test-params: --runtime=firefox-no-host packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_firefox
          requires:
            - test-packages-firefox-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-node-no-numpy-dependents
          test-params: --runtime=node-no-host packages/*/test*.py
          cache-dir: .pytest_cache_node
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main:
          name: test-packages-node
          test-params: --runtime=node-no-host packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_node
          requires:
            - test-packages-node-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main-macos:
          name: test-packages-safari-no-numpy-dependents
          test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py
          cache-dir: .pytest_cache_safari
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main-macos:
          name: test-packages-safari
          test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_safari
          requires:
            - test-packages-safari-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-js:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark-stack-size:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - create-xbuild-env:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-cmdline-runner:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - build-pyodide-debug:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - deploy-release:
          requires:
            - test-core-firefox
            - test-packages-firefox
            - build-pyodide-debug
            - create-xbuild-env
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /^\d+\.\d+\.\w+$/
          context:
            - s3-deployment
      - deploy-dev:
          requires:
            - test-core-firefox
            - test-packages-firefox
            - build-pyodide-debug
          filters:
            branches:
              only: main
          context:
            - s3-deployment