ci: move Flagships to GH (#16420)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.azure/app-cloud-e2e.yml

@@ -34,6 +34,7 @@ pr:
       - "tests/integrations_app/**"
       - "setup.py"
     exclude:
+      - "!tests/integrations_app/flagship/**"
       - "requirements/*/docs.txt"
       - "*.md"
       - "**/*.md"
@@ -150,7 +151,6 @@ jobs:
       displayName: 'Install Lightning & dependencies'
 
     - bash: |
-        pip install playwright
         python -m playwright install  # --with-deps
       displayName: 'Install Playwright system dependencies'
 

.azure/app-flagships.yml

@@ -1,154 +0,0 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
-trigger:
-  tags:
-    include:
-      - '*'
-  branches:
-    include:
-      - "release/*"
-      - "refs/tags/*"
-      # TODO: just for debugging this PR
-      - "ci/flagship-*"
-
-schedules:
-  - cron: "0 0 * * *" # At the end of every day
-    displayName: Daily midnight testing
-    branches:
-      include:
-        - "release/*"
-
-pr: none
-
-# variables are automatically exported as environment variables so this will override pip's default cache dir
-variables:
-  - name: pip_cache_dir
-    value: $(Pipeline.Workspace)/.pip
-  - name: local_id
-    value: $(Build.BuildId)
-  - name: video_artifact_dir
-    value: ./videos
-
-jobs:
-  - job: test_flagships
-    pool: azure-cpus
-    container:
-      image: mcr.microsoft.com/playwright/python:v1.29.1-focal
-      options: "--shm-size=4gb"
-
-    # TODO:
-    # - Training Studio
-    # - Echo
-    # - StreamLit / Gradio
-    # - All homepage & docs apps
-
-    strategy:
-      matrix:
-        'App: Flashy':
-          name: "flashy"
-          repo: "https://github.com/Lightning-AI/LAI-Flashy-App.git"
-        'App: Muse':
-          name: "stable-diffusion"
-          repo: "https://github.com/Lightning-AI/stable-diffusion-deploy.git"
-        'Component: Jupyter':
-          name: "jupyter"
-          repo: "https://github.com/Lightning-AI/LAI-Jupyter-Component.git"
-    timeoutInMinutes: "25"
-    cancelTimeoutInMinutes: "1"
-    # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
-    workspace:
-      clean: all
-    variables:
-      HEADLESS: '1'
-      PACKAGE_LIGHTNING: '1'
-      CLOUD: '1'
-      VIDEO_LOCATION: $(video_artifact_dir)
-      HAR_LOCATION: './artifacts/hars'
-      SLOW_MO: '50'
-      LIGHTNING_DEBUG: '1'
-    steps:
-
-    - bash: |
-        whoami
-        mkdir -p tests/_flagships
-        mkdir -p $(video_artifact_dir)/$(name)
-        printf "local id: $(local_id)\n"
-        python --version
-        pip --version
-      displayName: 'Info'
-
-    - script: |
-        pip install -q -r .actions/requirements.txt
-        python .actions/assistant.py copy_replace_imports \
-          --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
-      displayName: 'Adjust testing'
-
-    - script: |
-        pip install playwright
-        python -m playwright install  # --with-deps
-      displayName: 'Install Playwright system dependencies'
-
-    - script: git clone $(repo) tests/_flagships/$(name)
-      displayName: 'Clone the Repo/App'
-
-    - script: |
-        ls -l .
-        pip install -r requirements-dev.txt
-        pip install -e .
-      workingDirectory: tests/_flagships/$(name)
-      condition: eq(variables['name'], 'flashy')
-      displayName: 'adjust env -> Flashy'
-
-    - script: |
-        pip install -e _flagships/$(name)
-        cp _flagships/$(name)/tests/test_app.py integrations_app/flagship/test_stable-diffusion.py
-      workingDirectory: tests/
-      condition: eq(variables['name'], 'stable-diffusion')
-      displayName: 'adjust env -> Muse'
-
-    - script: |
-        pip install -e _flagships/$(name)
-        # pip install -r _flagships/$(name)/tests/requirements-dev.txt
-        cp _flagships/$(name)/tests/test_jupyter_app.py integrations_app/flagship/test_jupyter.py
-      workingDirectory: tests/
-      condition: eq(variables['name'], 'jupyter')
-      displayName: 'adjust env -> Jupyter'
-
-    - script: pip install -e .[cloud,test] -f https://download.pytorch.org/whl/cpu/torch_stable.html
-      displayName: 'Install Lightning & dependencies'
-
-    - bash: pip --version && pip list
-      displayName: 'List pip dependency'
-
-    - script: |
-        ls -l tests/_flagships
-        python -m pytest tests/integrations_app/flagship/test_$(name).py \
-          --timeout=540 --capture=no -v --color=yes
-      env:
-        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
-        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
-        LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
-        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
-      displayName: 'Run the tests'
-
-    - task: PublishPipelineArtifact@1
-      condition: failed()
-      inputs:
-        path: "$(video_artifact_dir)/$(name)"
-        artifactName: $(name)
-        publishLocation: 'pipeline'
-      displayName: 'Publish videos'
-
-    - script: |
-        time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
-      condition: always()
-      timeoutInMinutes: "3"
-      env:
-        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
-        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
-        LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
-        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
-      displayName: 'Clean Previous Apps'

.github/actions/prep-apps/action.yml

@@ -0,0 +1,39 @@
+name: Adjust App environment
+description: make adjustment specific for selected App
+
+inputs:
+  name:
+    description: application name
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+
+  - name: adjust env -> Flashy
+    if: inputs.name == 'flashy'
+    working-directory: tests/_flagship-app
+    run: |
+      ls -l .
+      pip install -r requirements-dev.txt -f $TORCH_URL
+      pip install -e . -f $TORCH_URL
+    shell: bash
+
+  - name: adjust env -> Muse
+    if: inputs.name == 'muse'
+    working-directory: tests/
+    run: |
+      pip install -e _flagship-app -f $TORCH_URL
+      cp _flagship-app/tests/test_app.py \
+        integrations_app/flagship/test_${{ inputs.name }}.py
+    shell: bash
+
+  - name: adjust env -> Jupyter
+    if: inputs.name == 'jupyter'
+    working-directory: tests/
+    run: |
+      pip install -e _flagship-app -f $TORCH_URL
+      # pip install -r _flagship-app/tests/requirements-dev.txt
+      cp _flagship-app/tests/test_jupyter_app.py \
+        integrations_app/flagship/test_${{ inputs.name }}.py
+    shell: bash

.github/checkgroup.yml

@@ -246,6 +246,7 @@ subprojects:
       - ".github/workflows/ci-examples-app.yml"
       - "src/lightning_app/**"
       - "tests/integrations_app/**"
+      - "!tests/integrations_app/flagship/**"
       - "examples/app_*/**"
       - "requirements/app/**"
       - "setup.py"
@@ -263,6 +264,13 @@ subprojects:
       - "app-examples (windows-2022, app, 3.9, oldest)"
       - "app-examples (windows-2022, lightning, 3.9, latest)"
 
+  - id: "lightning: Flagships"
+    paths:
+      - ".github/workflows/ci-flagship-apps.yml"
+      - "tests/integrations_app/flagship/**"
+    checks:
+      - "app-flagships (flashy, Lightning-AI/LAI-Flashy-App)"
+
   - id: "lightning_app: Azure"
     paths:
       - ".actions/**"
@@ -270,6 +278,7 @@ subprojects:
       - "src/lightning_app/**"
       - "requirements/app/**"
       - "tests/integrations_app/**"
+      - "!tests/integrations_app/flagship/**"
       - "setup.py"
       - "!requirements/*/docs.txt"
       - "!*.md"

.github/workflows/ci-examples-app.yml

@@ -12,6 +12,7 @@ on:
       - ".github/workflows/ci-examples-app.yml"
       - "src/lightning_app/**"
       - "tests/integrations_app/**"
+      - "!tests/integrations_app/flagship/**"
       - "examples/app_*/**"
       - "requirements/app/**"
       - "setup.py"

.github/workflows/ci-flagship-apps.yml

@@ -0,0 +1,119 @@
+name: Test App - flagships
+
+# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
+on:
+  push:
+    branches: ["release/*"]
+  pull_request:
+    branches: [master, "release/*"]
+    types: [opened, reopened, ready_for_review, synchronize]
+    paths:
+      - ".github/workflows/ci-flagship-apps.yml"
+      - "tests/integrations_app/flagship/**"
+  workflow_dispatch: {}
+    # TODO: inputs specifying lightning branch and switch prod/staging
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  app-flagships:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    container:
+      image: mcr.microsoft.com/playwright/python:v1.29.1-focal
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {app: "flashy", repo: "Lightning-AI/LAI-Flashy-App"}
+          - {app: "muse", repo: "Lightning-AI/stable-diffusion-deploy"}
+          - {app: "jupyter", repo: "Lightning-AI/LAI-Jupyter-Component"}
+
+          # TODO:
+          # - Training Studio
+          # - Echo
+          # - StreamLit / Gradio
+          # - All homepage & docs apps
+
+    env:
+      HEADLESS: '1'
+      PACKAGE_LIGHTNING: '1'
+      CLOUD: '1'
+      VIDEO_LOCATION: './videos'
+      HAR_LOCATION: './artifacts/hars'
+      SLOW_MO: '50'
+      LIGHTNING_DEBUG: '1'
+      TORCH_URL: 'https://download.pytorch.org/whl/cpu/torch_stable.html'
+    # Timeout: https://stackoverflow.com/a/59076067/4521646
+    timeout-minutes: 20
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: basic setup
+        run: |
+          mkdir -p tests/_flagships
+          mkdir -p $VIDEO_LOCATION
+          python -m playwright install  # --with-deps
+
+      - name: Adjust testing
+        run: |
+          pip install -q -r .actions/requirements.txt
+          python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
+            --source_import="lightning_app" --target_import="lightning.app"
+
+      - name: Clone the Repo/App
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ matrix.repo }}
+          path: tests/_flagship-app
+
+      - name: Adjust env. for this App
+        uses: ./.github/actions/prep-apps
+        with:
+          name: ${{ matrix.app }}
+
+      - name: Install Lightning package
+        run: pip install -e .[cloud,test] -f $TORCH_URL
+      - name: List pip dependency
+        run: pip --version && pip list
+
+      - name: Run the tests
+        working-directory: tests/
+        env:
+          LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID_PROD }}
+          LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY_PROD }}
+          LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME_PROD }}
+          LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL_PROD }}
+        run: |
+          ls -l _flagship-app
+          python -m pytest integrations_app/flagship/test_${{ matrix.app }}.py \
+            --capture=no -v --color=yes
+
+      - name: Upload recordings
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: flahship-app-${{ matrix.app }}
+          path: ${{ env.VIDEO_LOCATION }}
+
+      - name: Clean Previous Apps
+        if: always()
+        timeout-minutes: 3
+        env:
+          LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID_PROD }}
+          LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY_PROD }}
+          LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME_PROD }}
+          LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL_PROD }}
+        run: |
+          time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"

pyproject.toml

@@ -209,7 +209,13 @@ norecursedirs = [
     "build",
     "docs",
 ]
-addopts = "--strict-markers --doctest-modules --color=yes --disable-pytest-warnings --ignore=legacy/checkpoints"
+addopts = [
+    "--strict-markers",
+    "--doctest-modules",
+    "--color=yes",
+    "--disable-pytest-warnings",
+    "--ignore=legacy/checkpoints",
+]
 markers = [
     "cloud:Run the cloud tests for example",
 ]

tests/integrations_app/flagship/__init__.py

@@ -2,4 +2,4 @@ import os.path
 
 from integrations_app import _PATH_TESTS_DIR
 
-_PATH_INTEGRATIONS_DIR = os.path.join(_PATH_TESTS_DIR, "_flagships")
+_PATH_INTEGRATIONS_DIR = os.path.join(_PATH_TESTS_DIR, "_flagship-app")

tests/integrations_app/flagship/test_flashy.py

@@ -1,4 +1,3 @@
-import os
 from time import sleep
 
 import pytest
@@ -74,6 +73,6 @@ def validate_app_functionalities(app_page: "Page") -> None:
 
 @pytest.mark.cloud
 def test_app_cloud() -> None:
-    with run_app_in_cloud(os.path.join(_PATH_INTEGRATIONS_DIR, "flashy")) as (admin_page, view_page, fetch_logs, _):
+    with run_app_in_cloud(_PATH_INTEGRATIONS_DIR) as (admin_page, view_page, fetch_logs, _):
 
         validate_app_functionalities(view_page)