lightning/.azure/app-cloud-e2e.yml

# Python package
# Create and test a Python package on multiple Python versions.
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
# https://docs.microsoft.com/azure/devops/pipelines/languages/python

trigger:
  tags:
    include:
      - '*'
  branches:
    include:
      - "master"
      - "release/*"
      - "refs/tags/*"

schedules:
  - cron: "0 0 * * *" # At the end of every day
    displayName: Daily midnight testing
    branches:
      include:
        - "master"

pr:
  branches:
    include:
      - "master"
      - "release/*"
  paths:
    include:
      - ".actions/*"
      - ".azure/app-cloud-e2e.yml"
      - "src/lightning/__about__.py"
      - "src/lightning/__init__.py"
      - "src/lightning/__main__.py"
      - "src/lightning/__setup__.py"
      - "src/lightning/__version__.py"
      - "src/lightning/app/**"
      - "src/lightning_app/*"
      - "examples/app/**"
      - "requirements/app/**"
      - "tests/integrations_app/**"
      - "setup.py"
    exclude:
      - "!tests/integrations_app/flagship/**"
      - "requirements/*/docs.txt"
      - "*.md"
      - "**/*.md"

# variables are automatically exported as environment variables so this will override pip's default cache dir
variables:
  - name: pip_cache_dir
    value: $(Pipeline.Workspace)/.pip
  - name: local_id
    value: $(Build.BuildId)
  - name: video_artifact_dir
    value: ./videos

jobs:
  - job: test_e2e
    pool: "azure-cpus"
    container:
      # see all available tags: https://mcr.microsoft.com/en-us/product/playwright/python/tags
      image: mcr.microsoft.com/playwright/python:v1.32.1-focal
      options: "--shm-size=4gb"
    strategy:
      matrix:
        'App: v0_app':
          name: "v0_app"
          dir: "public"
        'App: boring_app':
          name: "boring_app"
          dir: "public"
        'App: template_streamlit_ui':
          name: "template_streamlit_ui"
          dir: "public"
        'App: template_react_ui':
          name: "template_react_ui"
          dir: "public"
        # 'App: template_jupyterlab':  # TODO: clarify where these files lives
        #   name: "template_jupyterlab"
        'App: installation_commands_app':
          name: "installation_commands_app"
          dir: "public"
        'App: drive':
          name: "drive"
          dir: "public"
        'App: payload':
          name: "payload"
          dir: "public"
        'App: commands_and_api':
          name: "commands_and_api"
          dir: "public"
        'App: quick_start':
          name: "quick_start"
          dir: "public"
        'App: idle_timeout':
          name: "idle_timeout"
          dir: "local"
        'App: collect_failures':
          name: "collect_failures"
          dir: "local"
        'App: custom_work_dependencies':
          name: "custom_work_dependencies"
          dir: "local"
    timeoutInMinutes: "15"
    cancelTimeoutInMinutes: "1"
    # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
    workspace:
      clean: all
    variables:
      FREEZE_REQUIREMENTS: "1"
      HEADLESS: '1'
      PACKAGE_LIGHTNING: '1'
      CLOUD: '1'
      VIDEO_LOCATION: $(video_artifact_dir)
      PR_NUMBER: $(local_id)
      TEST_APP_NAME: $(name)
      TEST_APP_FOLDER: $(dir)
      HAR_LOCATION: './artifacts/hars'
      SLOW_MO: '50'
      LIGHTNING_DEBUG: '1'
    steps:

    - script: echo '##vso[task.setvariable variable=local_id]$(System.PullRequest.PullRequestNumber)'
      displayName: "Set id for this PR"
      condition: eq(variables['Build.Reason'], 'PullRequest')

    - bash: |
        whoami
        mkdir -p "$(video_artifact_dir)/$(name)"
        printf "local id: $(local_id)\n"
        python --version
        pip --version
        echo "allow fail: ${{ in(variables['name'], 'quick_start', 'template_react_ui') }}"        
      displayName: 'Info'

    # TODO: we are testing it as `lightning`, so add also version for `lightning_app`
    - bash: |
        pip install -e .[app-dev] \
          -f https://download.pytorch.org/whl/cpu/torch_stable.html        
      displayName: 'Install Lightning & dependencies'

    - bash: python -m playwright install  # --with-deps
      displayName: 'Install Playwright system dependencies'

    # The magic happens here it doesn't need to install the quick start dependencies.
    # This test is very important to test the main user story of lightning app.
    # It also e2e tests running on cloud without installing dependencies.
    - bash: |
        git clone https://github.com/Lightning-AI/lightning-quick-start examples/app/quick-start        
      # without succeeded this could run even if the job has already failed
      condition: and(succeeded(), eq(variables['name'], 'quick_start'))
      displayName: 'Clone Quick start Repo'
    - bash: |
        git clone https://github.com/Lightning-AI/lightning-template-react examples/app/template_react_ui        
      # without succeeded this could run even if the job has already failed
      condition: and(succeeded(), eq(variables['name'], 'template_react_ui'))
      displayName: 'Clone Template React UI Repo'

    # Replace imports to use `lightning` instead of `lightning_app` since we install lightning only ATM
    - bash: |
        pip install -q -r .actions/requirements.txt
        python .actions/assistant.py copy_replace_imports \
          --source_dir="./examples" --source_import="lightning_app" --target_import="lightning.app"        
      displayName: 'Adjust examples'

    - bash: pip --version && pip list
      displayName: 'List pip dependency'

    - bash: |
        ls -l examples/app/$(TEST_APP_NAME)
        echo ${TEST_FILE}
        python -m pytest ${TEST_FILE}::test_$(TEST_APP_NAME)_example_cloud \
          --timeout=360 --capture=no -v --color=yes        
      env:
        TEST_FILE: tests/integrations_app/$(TEST_APP_FOLDER)/test_$(TEST_APP_NAME).py
        #LAI_USER: $(LAI_USER)  # for STAGING
        #LAI_PASS: $(LAI_PASS)  # for STAGING
        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
        LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
      # Todo: investigate why these apps are failing
      continueOnError: ${{ in(variables['name'], 'quick_start', 'template_react_ui') }}
      displayName: 'Run the tests'

    - task: PublishPipelineArtifact@1
      condition: failed()
      inputs:
        path: "$(video_artifact_dir)/$(name)"
        artifactName: $(name)
        publishLocation: 'pipeline'
      displayName: 'Publish videos'

    - bash: |
        time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"        
      condition: always()
      continueOnError: "true"
      timeoutInMinutes: "3"
      env:
        #LAI_USER: $(LAI_USER)  # for STAGING
        #LAI_PASS: $(LAI_PASS)  # for STAGING
        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
        LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
      displayName: 'Clean Previous Apps'