lightning/.azure/app-cloud-e2e.yml

# Python package
# Create and test a Python package on multiple Python versions.
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
# https://docs.microsoft.com/azure/devops/pipelines/languages/python

trigger:
  tags:
    include:
      - '*'
  branches:
    include:
      - "master"
      - "release/*"
      - "refs/tags/*"

schedules:
  - cron: "0 0 * * *" # At the end of every day
    displayName: Daily midnight testing
    branches:
      include:
        - "master"

pr:
  branches:
    include:
      - "master"
      - "release/*"
  paths:
    include:
      - ".actions/**"
      - ".azure/app-cloud-e2e.yml"
      - "src/lightning_app/**"
      - "requirements/app/**"
      - "tests/tests_examples_app/**"
      - "setup.py"
    exclude:
      - "requirements/*/docs.txt"
      - "*.md"
      - "**/*.md"

# variables are automatically exported as environment variables so this will override pip's default cache dir
variables:
  - name: pip_cache_dir
    value: $(Pipeline.Workspace)/.pip
  - name: local_id
    value: $(Build.BuildId)
  - name: video_artifact_dir
    value: ./videos

jobs:
  - job: App_cloud_e2e_testing
    pool: azure-cpus
    container:
      image: mcr.microsoft.com/playwright/python:v1.27.1-focal
      options: "--shm-size=4gb"
    strategy:
      matrix:
        'App: v0_app':
          name: "v0_app"
          dir: "public"
        'App: boring_app':
          name: "boring_app"
          dir: "public"
        'App: template_streamlit_ui':
          name: "template_streamlit_ui"
          dir: "public"
        'App: template_react_ui':
          name: "template_react_ui"
          dir: "public"
        # 'App: template_jupyterlab':  # TODO: clarify where these files lives
        #   name: "template_jupyterlab"
        'App: installation_commands_app':
          name: "installation_commands_app"
          dir: "public"
        'App: drive':
          name: "drive"
          dir: "public"
        'App: payload':
          name: "payload"
          dir: "public"
        'App: commands_and_api':
          name: "commands_and_api"
          dir: "public"
        'App: quick_start':
          name: "quick_start"
          dir: "public"
        'App: idle_timeout':
          name: "idle_timeout"
          dir: "local"
        'App: collect_failures':
          name: "collect_failures"
          dir: "local"
        'App: custom_work_dependencies':
          name: "custom_work_dependencies"
          dir: "local"
    timeoutInMinutes: "10"
    cancelTimeoutInMinutes: "1"
    # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
    workspace:
      clean: all
    steps:

    - script: echo '##vso[task.setvariable variable=local_id]$(System.PullRequest.PullRequestNumber)'
      displayName: "Set id for this PR"
      condition: eq(variables['Build.Reason'], 'PullRequest')

    - bash: |
        whoami
        mkdir -p $(video_artifact_dir)
        printf "local id: $(local_id)\n"
        python --version
        pip --version
      displayName: 'Info'

    # TODO: we are testing it as `lightning`, so add also version for `lightning_app`
    - bash: git restore . && python -m pip install -e . -r requirements/app/test.txt -r requirements/app/ui.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
      displayName: 'Install Lightning & dependencies'

    - bash: |
        python -m pip install playwright
        python -m playwright install  # --with-deps
      displayName: 'Install Playwright system dependencies'

    # The magic happens here it doesn't need to install the quick start dependencies.
    # This test is very important to test the main user story of lightning app.
    # It also e2e tests running on cloud without installing dependencies.
    - bash: |
        rm -rf examples/lightning-quick-start || true
        git clone https://github.com/Lightning-AI/lightning-quick-start examples/lightning-quick-start
      condition: eq(variables['name'], 'quick_start')
      displayName: 'Clone Quick start Repo'


    # - bash: |
    #     rm -rf examples/app_template_jupyterlab || true
    #     git clone https://github.com/Lightning-AI/LAI-lightning-template-jupyterlab-App examples/app_template_jupyterlab
    #     cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_examples_app/test_template_jupyterlab.py
    #   condition: eq(variables['name'], 'template_jupyterlab')
    #   displayName: 'Clone Template Jupyter Lab Repo'

    - bash: |
        rm -rf examples/app_template_react_ui || true
        git clone https://github.com/Lightning-AI/lightning-template-react examples/app_template_react_ui
      condition: eq(variables['name'], 'template_react_ui')
      displayName: 'Clone Template React UI Repo'

    - bash: python -m pip install -q -r .actions/requirements.txt
      displayName: 'Install assistant dependencies'

    # Fix imports to use `lightning` instead of `lightning_app` since we install lightning only ATM
    # TODO: fixme when installing `lightning_app` as well
    - bash: |
        python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning_app" --target_import="lightning.app"
        python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
      displayName: 'Adjust examples'

    - bash: |
        pip --version
        pip list
      displayName: 'List pip dependency'

    - bash: |
        mkdir -p ${VIDEO_LOCATION}
        ls -l examples/${TEST_APP_NAME}
        ls -l tests/tests_examples_app/public
        python -m pytest tests/tests_examples_app/${TEST_APP_FOLDER}/test_${TEST_APP_NAME}.py::test_${TEST_APP_NAME}_example_cloud \
          --timeout=540 --capture=no -v --color=yes
      env:
        HEADLESS: '1'
        PACKAGE_LIGHTNING: '1'
        CLOUD: '1'
        VIDEO_LOCATION: $(video_artifact_dir)
        PR_NUMBER: $(local_id)
        TEST_APP_NAME: $(name)
        TEST_APP_FOLDER: $(dir)
        HAR_LOCATION: './artifacts/hars'
        SLOW_MO: '50'
        #LAI_USER: $(LAI_USER)  # for STAGING
        #LAI_PASS: $(LAI_PASS)  # for STAGING
        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
        LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
        LIGHTNING_DEBUG: '1'
      displayName: 'Run the tests'

    - task: PublishPipelineArtifact@1
      condition: failed()
      inputs:
        path: $(video_artifact_dir)
        artifactName: $(name)
      displayName: 'Publish videos'

    - bash: |
        time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
      condition: always()
      env:
        #LAI_USER: $(LAI_USER)  # for STAGING
        #LAI_PASS: $(LAI_PASS)  # for STAGING
        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
        LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
        PR_NUMBER: $(local_id)
        TEST_APP_NAME: $(name)
        TEST_APP_FOLDER: $(dir)
        # GRID_USER_ID: $(LIGHTNING_USER_ID)  # TODO: clarify the meaning
        # GRID_USER_KEY: $(LIGHTNING_API_KEY)  # TODO: clarify the meaning
        # GRID_URL: $(LIGHTNING_CLOUD_URL)
        # _GRID_USERNAME: $(LIGHTNING_USERNAME)
      displayName: 'Clean Previous Apps'