pyodide/.circleci/config.yml

version: 2

defaults: &defaults
  working_directory: ~/repo
  docker:
    - image: iodide/pyodide-env:0.16.1
  environment:
    - EMSDK_NUM_CORES: 4
      EMCC_CORES: 4

jobs:
  lint:
    <<: *defaults
    steps:
      - checkout

      - run:
          name: lint
          command: |
            pip install black mypy
            # TODO: investigate why clang-format without version is not available
            sudo ln -s /usr/bin/clang-format-6.0 /usr/bin/clang-format
            make lint
            black --check --exclude tools/file_packager.py .
            mypy --ignore-missing-imports pyodide_build/ src/ packages/micropip/micropip/ packages/*/test*
  build:
    <<: *defaults
    steps:
      - checkout

      - restore_cache:
          keys:
          - v1-emsdk-{{ checksum "emsdk/Makefile" }}-v20201203-

      - run:
          name: dependencies
          command: sudo apt install -y libtinfo5

      - run:
          name: build
          no_output_timeout: 1200
          command: |
            ccache -z
            make
            ccache -s

      - run:
          name: check-size
          command: ls -lh build/

      - save_cache:
          paths:
            - ./emsdk/emsdk
            - ~/.ccache
          key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v20201203-{{ .BuildNum }}

      - persist_to_workspace:
          root: .
          paths:
            - ./build
            - ./cpython/build/3.8.2/host

      - store_artifacts:
          path: /home/circleci/repo/build/

  build-test-minimal:
    <<: *defaults
    steps:
      - checkout

      # this cache is generated by the main build job, we never store it here
      - restore_cache:
          keys:
          - v1-emsdk-{{ checksum "emsdk/Makefile" }}-v20201203-

      - run:
          name: dependencies
          command: sudo apt install -y libtinfo5

      - run:
          name: build
          no_output_timeout: 1200
          command: |
            ccache -z
            PYODIDE_PACKAGES='micropip' make
            ccache -s

      - run:
          name: check-size
          command: ls -lh build/

      - run:
          name: test
          command: |
            export PYODIDE_PACKAGES='micropip'
            # only check common tests, all other are checked in the main test jobs
            pytest src/tests/test_common.py -v -k firefox

      - store_artifacts:
          path: /home/circleci/repo/build/

  test-firefox:
    <<: *defaults
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            pytest src packages/*/test* packages/micropip/micropip/ pyodide_build -v -k firefox

  test-chrome:
    <<: *defaults
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            pytest src packages/*/test* packages/micropip/micropip/ pyodide_build -v -k chrome

  test-python:
    <<: *defaults
    steps:
      - checkout
      - run:
          name: deps
          command: |
            sudo pip install pytest-cov
      - run:
          name: test
          command: |
            pytest src pyodide_build packages/micropip/test_micropip.py -v -k 'not (chrome or firefox)' --cov=pyodide_build --cov=pyodide

  benchmark:
    <<: *defaults
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: benchmark
          command: |
            python benchmark/benchmark.py cpython/build/3.8.2/host/bin/python3 build/benchmarks.json
      - store_artifacts:
          path: /home/circleci/repo/build/benchmarks.json

  deploy-release:
    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3
            python3 -m pip install awscli
      - run:
          name: Deploy Github Releases
          command: |
            cp -r build /tmp/pyodide
            cd /tmp/
            tar cjf pyodide-build-${CIRCLE_TAG}.tar.bz2  pyodide/
            ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" -delete  "${CIRCLE_TAG}" ./pyodide-build-${CIRCLE_TAG}.tar.bz2

      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find build/ -type f -exec sh -c "echo \"Compressing {}\"; gzip {}; mv {}.gz {};" \;
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*.data' --cache-control 'max-age=30758400, immutable, public' --content-encoding 'gzip'    # 1 year cache
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, immutable, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year
            # update 301 redirect for the /latest/* route.
            aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json

  deploy-s3:
    docker:
      - image: circleci/python:3.7.7

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            sudo apt-get update
            sudo apt-get install -y groff
            python3 -m pip install awscli
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find build/ -type f -exec sh -c "echo \"Compressing {}\"; gzip {}; mv {}.gz {};" \;
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/full/"
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache

workflows:
  version: 2
  build-and-deploy:
    jobs:
      - lint
      - build:
          requires:
            - lint
          filters:
            tags:
              only: /.*/
      - build-test-minimal:
          requires:
            - lint
      - test-chrome:
          requires:
            - build
      - test-firefox:
          requires:
            - build
          filters:
            tags:
              only: /.*/
      - test-python:
          filters:
            tags:
              only: /.*/
      - benchmark:
          requires:
            - build
      - deploy-release:
          requires:
            - test-firefox
            - test-python
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /^\d+\.\d+\.\d+$/
      - deploy-s3:
          requires:
            - test-firefox
            - test-python
          filters:
            branches:
              only: master