lightning/.azure/ipu-tests.yml

trigger:
  tags:
    include:
      - '*'
  branches:
    include:
      - master
      - release/*
      - refs/tags/*

pr:
  branches:
    include:
      - "master"
      - "release/*"
  paths:
    include:
      - ".actions/**"
      - ".azure/ipu-tests.yml"
      - "requirements/fabric/**"
      - "src/lightning/__about__.py"
      - "src/lightning/__init__.py"
      - "src/lightning/__main__.py"
      - "src/lightning/__setup__.py"
      - "src/lightning/__version__.py"
      - "src/lightning/fabric/**"
      - "src/lightning_fabric/*"
      - "requirements/pytorch/**"
      - "src/lightning/pytorch/**"
      - "src/pytorch_lightning/*"
      - "tests/tests_pytorch/**"
      - "pyproject.toml"  # includes pytest config
    exclude:
      - "requirements/*/docs.txt"
      - "*.md"
      - "**/*.md"

variables:
  - name: poplar_sdk
    # https://docs.graphcore.ai/projects/poptorch-user-guide/en/latest/installation.html#version-compatibility
    value: "poplar_sdk-ubuntu_20_04-3.1.0+1205-58b501c780"

jobs:
  - job: testing
    # how long to run the job before automatically cancelling
    timeoutInMinutes: "20"
    pool: graphcore-ipus
    workspace:
      clean: all

    steps:

    - script: |
        set -eux
        # ls -la /mnt/public/packages
        ls -la /opt/poplar
        tar -xzf /opt/poplar/${{ variables.poplar_sdk }}.tar.gz
        pip debug --verbose
        pip install ${{ variables.poplar_sdk }}/poptorch-*ubuntu*.whl
      displayName: "Poplar SDK: Extract & Install"

    - bash: |
        # enable scripts basically just set/modify some environment variables
        source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
        gc-info --list-devices
        printenv
        python --version
        pip --version
        pip list
      displayName: 'Image info & GraphCore'

    - script: |
        set -eux
        source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
        NUM_IPUS=$(gc-info --ipu-count)
        if [[ -z "${NUM_IPUS}" ]] || [[ "${NUM_IPUS}" -eq 0 ]]; then
            echo "No IPUs found to reset. Exiting"
            exit 1
        fi
        echo "Resetting parity on ${NUM_IPUS} IPU devices"
        i=0
        while [[ i -lt  "${NUM_IPUS}" ]]; do
            gc-reset -d "${i}"
            i=$((i + 1))
        done
      displayName: "Reset IPU devices"

    - bash: |
        for fpath in `ls requirements/pytorch/*.txt`; do \
          python ./requirements/pytorch/adjust-versions.py $fpath; \
        done

        pip install .[test] -f /mnt/public/packages
        pip uninstall -y neptune-client  # it is not clear what version is compatible
        pip list
      env:
        PACKAGE_NAME: "pytorch"
        FREEZE_REQUIREMENTS: "1"
        GIT_TERMINAL_PROMPT: "1"
      displayName: 'Install package & dependencies'

    - bash: |
        python requirements/collect_env_details.py
        python -c "import torch"
      displayName: 'Env details'

    - script: |
        set -eux
        source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
        source ${{ variables.poplar_sdk }}/popart-ubuntu*/enable.sh
        python -c "import poptorch; print(poptorch.__version__)"
      displayName: "Check poptorch installation"

    - bash: |
        pip install -q -r .actions/requirements.txt
        python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
          --source_import="lightning.fabric,lightning.pytorch" \
          --target_import="lightning_fabric,pytorch_lightning"
      displayName: 'Adjust tests'

    - bash: |
        source ${{ variables.poplar_sdk }}/poplar-ubuntu*/enable.sh
        source ${{ variables.poplar_sdk }}/popart-ubuntu*/enable.sh
        python -m coverage run --source pytorch_lightning -m pytest tests/tests_pytorch -vv --durations=50
      env:
        MKL_THREADING_LAYER: "GNU"
        POPTORCH_WAIT_FOR_IPU: "1"
        PL_RUN_IPU_TESTS: "1"
      displayName: 'Testing: PyTorch standard'