pyodide/.circleci/config.yml

version: 2.1

defaults: &defaults
  working_directory: ~/repo
  docker:
    - image: pyodide/pyodide-env:19
  environment:
    - EMSDK_NUM_CORES: 3
      EMCC_CORES: 3
      PYODIDE_JOBS: 3
      # Make sure the ccache dir is consistent between core and package builds
      # (it's not the case otherwise)
      CCACHE_DIR: /root/.ccache/

jobs:
  lint:
    <<: *defaults
    resource_class: small
    steps:
      - checkout

      - run:
          name: lint
          command: make lint

  test-docs:
    <<: *defaults
    resource_class: small
    steps:
      - checkout

      - run:
          name: Test docs
          command: |
            mkdir test-results
            pytest docs/sphinx_pyodide/tests --junitxml=test-results/junit.xml
      - store_test_results:
          path: test-results

  build-core:
    <<: *defaults
    steps:
      - checkout

      - restore_cache:
          keys:
            - -core-{{ checksum "Makefile.envs" }}-{{ checksum "cpython/Makefile" }}-v20210910-

      - run:
          name: build emsdk
          no_output_timeout: 1200
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C emsdk
            ccache -s

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

      - run:
          name: build cpython
          no_output_timeout: 1200
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C cpython
            ccache -s

      - run:
          name: build pyodide core
          no_output_timeout: 1200
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            PYODIDE_PACKAGES="core" make
            ccache -s

      - run:
          name: check-size
          command: ls -lh build/

      - save_cache:
          paths:
            - /root/.ccache
          key: -core-{{ checksum "Makefile.envs" }}-{{ checksum "cpython/Makefile" }}-v20210910-

      - run:
          name: Clean up workspace
          command: |
            rm -rf cpython/{build,downloads}
            rm -rf emsdk/emsdk/binaryen

      - persist_to_workspace:
          root: .
          paths:
            - .

      - store_artifacts:
          path: /root/repo/build/

  build-packages:
    <<: *defaults
    resource_class: large

    steps:
      - checkout

      - attach_workspace:
          at: .

      - restore_cache:
          keys:
            - -pkg-{{ checksum "Makefile.envs" }}-v20210911-

      - run:
          name: build packages
          no_output_timeout: 1800
          command: |
            source pyodide_env.sh

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

            ccache -z
            PYODIDE_PACKAGES='*' make -C packages
            ccache -s
          environment:
            PYODIDE_JOBS: 5

      - run:
          name: check-size
          command: ls -lh build/

      - save_cache:
          paths:
            - /root/.ccache
          key: -pkg-{{ checksum "Makefile.envs" }}-v20210911-

      - persist_to_workspace:
          root: .
          paths:
            - ./packages/.artifacts
            - ./build

      - store_artifacts:
          path: /root/repo/build/

      - store_artifacts:
          path: /root/repo/packages/build-logs

  test-main:
    parameters:
      test-params:
        description: The tests to run.
        type: string
    <<: *defaults
    resource_class: medium+
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            mkdir test-results
            tools/pytest_wrapper.py \
              --junitxml=test-results/junit.xml \
              --verbose \
              << parameters.test-params >>
      - store_test_results:
          path: test-results

  benchmark-stack-size:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .
      - run:
          name: stack-size
          command: |
            pytest -s benchmark/stack_usage.py  | sed -n 's/## //pg'

  test-emsdk:
    <<: *defaults
    resource_class: small
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: make -C emsdk test

  test-python:
    # Unlike test-main this uses "resource_class small" and doesn't use "pytest_wrapper."
    <<: *defaults
    resource_class: small
    steps:
      - checkout
      - run:
          name: test
          command: |
            mkdir test-results
            pytest \
              --junitxml=test-results/junit.xml \
              --verbose \
              -k 'not (chrome or firefox or node)' \
              --cov=pyodide_build --cov=pyodide \
              src pyodide-build packages/micropip/

      - store_test_results:
          path: test-results

  test-js:
    <<: *defaults
    resource_class: small
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            cd src/js
            npx tsd
            npm install
            npm test
      - run:     
          name: check if webpack cli works well with load-pyodide.js
          command: | 
            git clone https://github.com/pyodide/pyodide-webpack-example.git
            cd pyodide-webpack-example
            npm install     
            cp ../src/js/load-pyodide.js node_modules/pyodide/load-pyodide.js                  
            head -20 node_modules/pyodide/load-pyodide.js
            npx webpack
   

  benchmark:
    <<: *defaults
    resource_class: medium+
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: benchmark
          command: |
            python benchmark/benchmark.py /usr/local/bin/python3 build/benchmarks.json
      - store_artifacts:
          path: /root/repo/build/benchmarks.json

  deploy-release:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-dev!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make
            python3 -m pip install awscli
      - run:
          name: Deploy Github Releases
          command: |
            cp -r build /tmp/pyodide
            cd /tmp/
            tar cjf pyodide-build-${CIRCLE_TAG}.tar.bz2  pyodide/
            ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" -delete  "${CIRCLE_TAG}" ./pyodide-build-${CIRCLE_TAG}.tar.bz2

      - run:
          name: Set PYODIDE_BASE_URL
          command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make update_base_url
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find build/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*.data' --cache-control 'max-age=30758400, immutable, public' --content-encoding 'gzip'    # 1 year cache
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, immutable, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year
            # update 301 redirect for the /latest/* route.
            aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json

  deploy-dev:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-release!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make
            python3 -m pip install awscli
      - run:
          name: Set PYODIDE_BASE_URL
          command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make update_base_url
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find build/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/full/"
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache

workflows:
  version: 2
  build-and-deploy:
    jobs:
      - lint:
          filters:
            tags:
              only: /.*/

      - test-docs:
          filters:
            tags:
              only: /.*/

      - test-python:
          filters:
            tags:
              only: /.*/

      - build-core:
          filters:
            tags:
              only: /.*/

      - build-packages:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-chrome
          test-params: -k "chrome and not webworker" src packages/micropip
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox
          test-params: -k "firefox and not webworker" src packages/micropip
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-node
          test-params: -k node src packages/micropip
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-chrome-webworker
          test-params: -k chrome src/tests/test_webworker.py
          requires:
            - test-core-chrome
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox-webworker
          test-params: -k firefox src/tests/test_webworker.py
          requires:
            - test-core-firefox
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-chrome
          test-params: -k chrome packages/test* packages/*/test*
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-firefox
          test-params: -k firefox packages/test* packages/*/test*
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-node
          test-params: -k "node and not numpy" packages/test* packages/*/test*
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-emsdk:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-js:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark-stack-size:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - deploy-release:
          requires:
            - lint
            - test-python
            - test-core-firefox
            - test-packages-firefox
            - test-emsdk
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /^\d+\.\d+\.\w+$/
      - deploy-dev:
          requires:
            - lint
            - test-python
            - test-core-firefox
            - test-packages-firefox
            - test-emsdk
          filters:
            branches:
              only: main