ci: flagship apps 1/n (#16304)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.azure/app-cloud-e2e.yml

@@ -48,7 +48,7 @@ variables:
     value: ./videos
 
 jobs:
-  - job: App_e2e_cloud
+  - job: test_e2e
     pool: "azure-cpus"
     container:
       image: mcr.microsoft.com/playwright/python:v1.28.0-focal
@@ -134,7 +134,7 @@ jobs:
 
     - bash: |
         whoami
-        mkdir -p $(video_artifact_dir)
+        mkdir -p $(VIDEO_LOCATION)
         printf "local id: $(local_id)\n"
         python --version
         pip --version
@@ -178,14 +178,13 @@ jobs:
         python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
       displayName: 'Adjust examples'
 
-    - bash: |
-        pip --version
-        pip list
+    - bash: pip --version && pip list
       displayName: 'List pip dependency'
 
     - bash: |
-        ls -l examples/${TEST_APP_NAME}
-        python -m pytest ${TEST_FILE}::test_${TEST_APP_NAME}_example_cloud \
+        ls -l examples/$(TEST_APP_NAME)
+        echo ${TEST_FILE}
+        python -m pytest ${TEST_FILE}::test_$(TEST_APP_NAME)_example_cloud \
           --timeout=540 --capture=no -v --color=yes
       env:
         TEST_FILE: tests/integrations_app/$(TEST_APP_FOLDER)/test_$(TEST_APP_NAME).py
@@ -208,6 +207,7 @@ jobs:
     - bash: |
         time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
       condition: always()
+      timeoutInMinutes: "3"
       env:
         #LAI_USER: $(LAI_USER)  # for STAGING
         #LAI_PASS: $(LAI_PASS)  # for STAGING
@@ -215,5 +215,4 @@ jobs:
         LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
         LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
         LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
-      timeoutInMinutes: "3"
       displayName: 'Clean Previous Apps'

.azure/app-flagships.yml

@@ -0,0 +1,127 @@
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+
+trigger:
+  tags:
+    include:
+      - '*'
+  branches:
+    include:
+      - "release/*"
+      - "refs/tags/*"
+      # TODO: just for debugging this PR
+      - "ci/flagship-*"
+
+schedules:
+  - cron: "0 0 * * *" # At the end of every day
+    displayName: Daily midnight testing
+    branches:
+      include:
+        - "release/*"
+
+# variables are automatically exported as environment variables so this will override pip's default cache dir
+variables:
+  - name: pip_cache_dir
+    value: $(Pipeline.Workspace)/.pip
+  - name: local_id
+    value: $(Build.BuildId)
+  - name: video_artifact_dir
+    value: ./videos
+
+jobs:
+  - job: test_flagships
+    pool: azure-cpus
+    container:
+      image: mcr.microsoft.com/playwright/python:v1.28.0-focal
+      options: "--shm-size=4gb"
+
+    #- Training Studio
+    #- Flashy
+    #- Muse
+    #- Echo
+    #- StreamLit / Gradio
+    #- Jupyter Notebook Component
+    #- All homepage & docs apps
+
+    strategy:
+      matrix:
+        'App: Flashy':
+          name: "flashy"
+          repo: "https://github.com/Lightning-AI/LAI-Flashy-App.git"
+    timeoutInMinutes: "25"
+    cancelTimeoutInMinutes: "1"
+    # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
+    workspace:
+      clean: all
+    variables:
+      HEADLESS: '1'
+      PACKAGE_LIGHTNING: '1'
+      CLOUD: '1'
+      VIDEO_LOCATION: $(video_artifact_dir)
+      HAR_LOCATION: './artifacts/hars'
+      SLOW_MO: '50'
+      LIGHTNING_DEBUG: '1'
+    steps:
+
+    - bash: |
+        whoami
+        mkdir -p tests/_flagships
+        mkdir -p $(video_artifact_dir)
+        printf "local id: $(local_id)\n"
+        python --version
+        pip --version
+      displayName: 'Info'
+
+    - script: pip install -e .[cloud,test] -f https://download.pytorch.org/whl/cpu/torch_stable.html
+      displayName: 'Install Lightning & dependencies'
+
+    - script: |
+        pip install playwright
+        python -m playwright install  # --with-deps
+      displayName: 'Install Playwright system dependencies'
+
+    - script: git clone $(repo) tests/_flagships/$(name)
+      displayName: 'Clone the Repo/App'
+
+    - script: |
+        cd tests/_flagships/$(name)
+        ls -l .
+        pip install -r requirements-dev.txt
+        pip install -e .
+      condition: eq(variables['name'], 'flashy')
+      displayName: 'adjust env for Flashy'
+
+    - bash: pip --version && pip list
+      displayName: 'List pip dependency'
+
+    - script: |
+        ls -l tests/_flagships
+        python -m pytest tests/integrations_app/flagship/test_$(name).py \
+          --timeout=540 --capture=no -v --color=yes
+      env:
+        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
+        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
+        LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
+        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
+      displayName: 'Run the tests'
+
+    - task: PublishPipelineArtifact@1
+      condition: failed()
+      inputs:
+        path: "$(video_artifact_dir)/$(name)"
+        artifactName: $(name)
+        publishLocation: 'pipeline'
+      displayName: 'Publish videos'
+
+    - script: |
+        time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
+      condition: always()
+      timeoutInMinutes: "3"
+      env:
+        LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
+        LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
+        LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
+        LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
+      displayName: 'Clean Previous Apps'

src/lightning_app/testing/helpers.py

@@ -3,6 +3,7 @@ import sys
 from queue import Empty
 from typing import List, Optional, Tuple
 
+import pytest
 from packaging.version import Version
 
 from lightning_app import LightningFlow, LightningWork
@@ -63,8 +64,6 @@ class _RunIf:
         cloud: bool = False,
         **kwargs,
     ):
-        import pytest
-
         """
         Args:
             *args: Any :class:`pytest.mark.skipif` arguments.

src/lightning_app/testing/testing.py

@@ -261,9 +261,12 @@ def run_app_in_cloud(
     if url.endswith("/"):
         url = url[:-1]
     payload = {"apiKey": _Config.api_key, "username": _Config.username}
-    res = requests.post(url + "/v1/auth/login", data=json.dumps(payload))
+    url_login = url + "/v1/auth/login"
+    res = requests.post(url_login, data=json.dumps(payload))
     if "token" not in res.json():
-        raise Exception("You haven't properly setup your environment variables.")
+        raise RuntimeError(
+            f"You haven't properly setup your environment variables with {url_login} and data: \n{payload}"
+        )
 
     token = res.json()["token"]
 
@@ -473,13 +476,13 @@ def wait_for(page, callback: Callable, *args, **kwargs) -> Any:
             res = callback(*args, **kwargs)
             if res:
                 return res
-        except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError) as e:
-            print(e)
+        except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError) as err:
+            print(err)
             try:
                 sleep(7)
                 page.reload()
-            except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError) as e:
-                print(e)
+            except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError) as err:
+                print(err)
                 pass
             sleep(3)
 

tests/integrations_app/__init__.py

@@ -0,0 +1,3 @@
+from os.path import dirname
+
+_PATH_TESTS_DIR = dirname(dirname(__file__))

tests/integrations_app/flagship/__init__.py

@@ -0,0 +1,5 @@
+import os.path
+
+from integrations_app import _PATH_TESTS_DIR
+
+_PATH_INTEGRATIONS_DIR = os.path.join(_PATH_TESTS_DIR, "_flagships")

tests/integrations_app/flagship/test_flashy.py

@@ -0,0 +1,79 @@
+import os
+from time import sleep
+
+import pytest
+from integrations_app.flagship import _PATH_INTEGRATIONS_DIR
+
+from lightning_app.testing.testing import run_app_in_cloud
+from lightning_app.utilities.imports import _is_playwright_available
+
+if _is_playwright_available():
+    import playwright
+    from playwright.sync_api import expect, Page
+
+
+# TODO: when this function is moved to the app itself we can just import it, so to keep better aligned
+def validate_app_functionalities(app_page: "Page") -> None:
+    """Validate the page after app starts.
+
+    this is direct copy-paste of validation living in the app repository:
+    https://github.com/Lightning-AI/LAI-Flashy-App/blob/main/tests/test_app_gallery.py#L205
+
+    app_page: The UI page of the app to be validated.
+    """
+
+    while True:
+        try:
+            app_page.reload()
+            sleep(5)
+            app_label = app_page.frame_locator("iframe").locator("text=Choose your AI task")
+            app_label.wait_for(timeout=30 * 1000)
+            break
+        except (
+            playwright._impl._api_types.Error,
+            playwright._impl._api_types.TimeoutError,
+        ):
+            pass
+
+    input_field = app_page.frame_locator("iframe").locator('input:below(:text("Data URL"))').first
+    input_field.wait_for(timeout=1000)
+    input_field.type("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip")
+    sleep(1)
+    upload_btn = app_page.frame_locator("iframe").locator('button:has-text("Upload")')
+    upload_btn.wait_for(timeout=1000)
+    upload_btn.click()
+
+    sleep(10)
+
+    train_folder_dropdown = app_page.frame_locator("iframe").locator("#mui-2")
+    train_folder_dropdown.click()
+
+    train_folder = app_page.frame_locator("iframe").locator('text="hymenoptera_data/train"')
+    train_folder.scroll_into_view_if_needed()
+    train_folder.click()
+
+    val_folder_dropdown = app_page.frame_locator("iframe").locator("#mui-3")
+    val_folder_dropdown.click()
+
+    val_folder = app_page.frame_locator("iframe").locator('text="hymenoptera_data/val"')
+    val_folder.scroll_into_view_if_needed()
+    val_folder.click()
+
+    train_btn = app_page.frame_locator("iframe").locator('button:has-text("Start training!")')
+    train_btn.click()
+
+    # Sometimes the results don't show until we refresh the page
+    sleep(10)
+
+    app_page.reload()
+
+    app_page.frame_locator("iframe").locator('button:has-text("RESULTS")').click()
+    runs = app_page.frame_locator("iframe").locator("table tbody tr")
+    expect(runs).to_have_count(1, timeout=120000)
+
+
+@pytest.mark.cloud
+def test_app_cloud() -> None:
+    with run_app_in_cloud(os.path.join(_PATH_INTEGRATIONS_DIR, "flashy")) as (admin_page, view_page, fetch_logs, _):
+
+        validate_app_functionalities(view_page)