diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml index ea09bc1b83..b59a861202 100644 --- a/.azure/app-cloud-e2e.yml +++ b/.azure/app-cloud-e2e.yml @@ -48,7 +48,7 @@ variables: value: ./videos jobs: - - job: App_e2e_cloud + - job: test_e2e pool: "azure-cpus" container: image: mcr.microsoft.com/playwright/python:v1.28.0-focal @@ -134,7 +134,7 @@ jobs: - bash: | whoami - mkdir -p $(video_artifact_dir) + mkdir -p $(VIDEO_LOCATION) printf "local id: $(local_id)\n" python --version pip --version @@ -178,14 +178,13 @@ jobs: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app" displayName: 'Adjust examples' - - bash: | - pip --version - pip list + - bash: pip --version && pip list displayName: 'List pip dependency' - bash: | - ls -l examples/${TEST_APP_NAME} - python -m pytest ${TEST_FILE}::test_${TEST_APP_NAME}_example_cloud \ + ls -l examples/$(TEST_APP_NAME) + echo ${TEST_FILE} + python -m pytest ${TEST_FILE}::test_$(TEST_APP_NAME)_example_cloud \ --timeout=540 --capture=no -v --color=yes env: TEST_FILE: tests/integrations_app/$(TEST_APP_FOLDER)/test_$(TEST_APP_NAME).py @@ -208,6 +207,7 @@ jobs: - bash: | time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()" condition: always() + timeoutInMinutes: "3" env: #LAI_USER: $(LAI_USER) # for STAGING #LAI_PASS: $(LAI_PASS) # for STAGING @@ -215,5 +215,4 @@ jobs: LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD) LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD) LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) - timeoutInMinutes: "3" displayName: 'Clean Previous Apps' diff --git a/.azure/app-flagships.yml b/.azure/app-flagships.yml new file mode 100644 index 0000000000..21c8955458 --- /dev/null +++ b/.azure/app-flagships.yml @@ -0,0 +1,127 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: + tags: + include: + - '*' + branches: + include: + - "release/*" + - "refs/tags/*" + # TODO: just for debugging this PR + - "ci/flagship-*" + +schedules: + - cron: "0 0 * * *" # At the end of every day + displayName: Daily midnight testing + branches: + include: + - "release/*" + +# variables are automatically exported as environment variables so this will override pip's default cache dir +variables: + - name: pip_cache_dir + value: $(Pipeline.Workspace)/.pip + - name: local_id + value: $(Build.BuildId) + - name: video_artifact_dir + value: ./videos + +jobs: + - job: test_flagships + pool: azure-cpus + container: + image: mcr.microsoft.com/playwright/python:v1.28.0-focal + options: "--shm-size=4gb" + + #- Training Studio + #- Flashy + #- Muse + #- Echo + #- StreamLit / Gradio + #- Jupyter Notebook Component + #- All homepage & docs apps + + strategy: + matrix: + 'App: Flashy': + name: "flashy" + repo: "https://github.com/Lightning-AI/LAI-Flashy-App.git" + timeoutInMinutes: "25" + cancelTimeoutInMinutes: "1" + # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace + workspace: + clean: all + variables: + HEADLESS: '1' + PACKAGE_LIGHTNING: '1' + CLOUD: '1' + VIDEO_LOCATION: $(video_artifact_dir) + HAR_LOCATION: './artifacts/hars' + SLOW_MO: '50' + LIGHTNING_DEBUG: '1' + steps: + + - bash: | + whoami + mkdir -p tests/_flagships + mkdir -p $(video_artifact_dir) + printf "local id: $(local_id)\n" + python --version + pip --version + displayName: 'Info' + + - script: pip install -e .[cloud,test] -f https://download.pytorch.org/whl/cpu/torch_stable.html + displayName: 'Install Lightning & dependencies' + + - script: | + pip install playwright + python -m playwright install # --with-deps + displayName: 'Install Playwright system dependencies' + + - script: git clone $(repo) tests/_flagships/$(name) + displayName: 'Clone the Repo/App' + + - script: | + cd tests/_flagships/$(name) + ls -l . + pip install -r requirements-dev.txt + pip install -e . + condition: eq(variables['name'], 'flashy') + displayName: 'adjust env for Flashy' + + - bash: pip --version && pip list + displayName: 'List pip dependency' + + - script: | + ls -l tests/_flagships + python -m pytest tests/integrations_app/flagship/test_$(name).py \ + --timeout=540 --capture=no -v --color=yes + env: + LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD) + LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD) + LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD) + LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) + displayName: 'Run the tests' + + - task: PublishPipelineArtifact@1 + condition: failed() + inputs: + path: "$(video_artifact_dir)/$(name)" + artifactName: $(name) + publishLocation: 'pipeline' + displayName: 'Publish videos' + + - script: | + time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()" + condition: always() + timeoutInMinutes: "3" + env: + LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD) + LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD) + LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD) + LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) + displayName: 'Clean Previous Apps' diff --git a/src/lightning_app/testing/helpers.py b/src/lightning_app/testing/helpers.py index cb21d31414..0d323789ab 100644 --- a/src/lightning_app/testing/helpers.py +++ b/src/lightning_app/testing/helpers.py @@ -3,6 +3,7 @@ import sys from queue import Empty from typing import List, Optional, Tuple +import pytest from packaging.version import Version from lightning_app import LightningFlow, LightningWork @@ -63,8 +64,6 @@ class _RunIf: cloud: bool = False, **kwargs, ): - import pytest - """ Args: *args: Any :class:`pytest.mark.skipif` arguments. diff --git a/src/lightning_app/testing/testing.py b/src/lightning_app/testing/testing.py index aaefe25598..d223a8ce07 100644 --- a/src/lightning_app/testing/testing.py +++ b/src/lightning_app/testing/testing.py @@ -261,9 +261,12 @@ def run_app_in_cloud( if url.endswith("/"): url = url[:-1] payload = {"apiKey": _Config.api_key, "username": _Config.username} - res = requests.post(url + "/v1/auth/login", data=json.dumps(payload)) + url_login = url + "/v1/auth/login" + res = requests.post(url_login, data=json.dumps(payload)) if "token" not in res.json(): - raise Exception("You haven't properly setup your environment variables.") + raise RuntimeError( + f"You haven't properly setup your environment variables with {url_login} and data: \n{payload}" + ) token = res.json()["token"] @@ -473,13 +476,13 @@ def wait_for(page, callback: Callable, *args, **kwargs) -> Any: res = callback(*args, **kwargs) if res: return res - except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError) as e: - print(e) + except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError) as err: + print(err) try: sleep(7) page.reload() - except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError) as e: - print(e) + except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError) as err: + print(err) pass sleep(3) diff --git a/tests/integrations_app/__init__.py b/tests/integrations_app/__init__.py index e69de29bb2..a3c9eb29e7 100644 --- a/tests/integrations_app/__init__.py +++ b/tests/integrations_app/__init__.py @@ -0,0 +1,3 @@ +from os.path import dirname + +_PATH_TESTS_DIR = dirname(dirname(__file__)) diff --git a/tests/integrations_app/flagship/__init__.py b/tests/integrations_app/flagship/__init__.py new file mode 100644 index 0000000000..03b5efcea4 --- /dev/null +++ b/tests/integrations_app/flagship/__init__.py @@ -0,0 +1,5 @@ +import os.path + +from integrations_app import _PATH_TESTS_DIR + +_PATH_INTEGRATIONS_DIR = os.path.join(_PATH_TESTS_DIR, "_flagships") diff --git a/tests/integrations_app/flagship/test_flashy.py b/tests/integrations_app/flagship/test_flashy.py new file mode 100644 index 0000000000..2217794b1f --- /dev/null +++ b/tests/integrations_app/flagship/test_flashy.py @@ -0,0 +1,79 @@ +import os +from time import sleep + +import pytest +from integrations_app.flagship import _PATH_INTEGRATIONS_DIR + +from lightning_app.testing.testing import run_app_in_cloud +from lightning_app.utilities.imports import _is_playwright_available + +if _is_playwright_available(): + import playwright + from playwright.sync_api import expect, Page + + +# TODO: when this function is moved to the app itself we can just import it, so to keep better aligned +def validate_app_functionalities(app_page: "Page") -> None: + """Validate the page after app starts. + + this is direct copy-paste of validation living in the app repository: + https://github.com/Lightning-AI/LAI-Flashy-App/blob/main/tests/test_app_gallery.py#L205 + + app_page: The UI page of the app to be validated. + """ + + while True: + try: + app_page.reload() + sleep(5) + app_label = app_page.frame_locator("iframe").locator("text=Choose your AI task") + app_label.wait_for(timeout=30 * 1000) + break + except ( + playwright._impl._api_types.Error, + playwright._impl._api_types.TimeoutError, + ): + pass + + input_field = app_page.frame_locator("iframe").locator('input:below(:text("Data URL"))').first + input_field.wait_for(timeout=1000) + input_field.type("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip") + sleep(1) + upload_btn = app_page.frame_locator("iframe").locator('button:has-text("Upload")') + upload_btn.wait_for(timeout=1000) + upload_btn.click() + + sleep(10) + + train_folder_dropdown = app_page.frame_locator("iframe").locator("#mui-2") + train_folder_dropdown.click() + + train_folder = app_page.frame_locator("iframe").locator('text="hymenoptera_data/train"') + train_folder.scroll_into_view_if_needed() + train_folder.click() + + val_folder_dropdown = app_page.frame_locator("iframe").locator("#mui-3") + val_folder_dropdown.click() + + val_folder = app_page.frame_locator("iframe").locator('text="hymenoptera_data/val"') + val_folder.scroll_into_view_if_needed() + val_folder.click() + + train_btn = app_page.frame_locator("iframe").locator('button:has-text("Start training!")') + train_btn.click() + + # Sometimes the results don't show until we refresh the page + sleep(10) + + app_page.reload() + + app_page.frame_locator("iframe").locator('button:has-text("RESULTS")').click() + runs = app_page.frame_locator("iframe").locator("table tbody tr") + expect(runs).to_have_count(1, timeout=120000) + + +@pytest.mark.cloud +def test_app_cloud() -> None: + with run_app_in_cloud(os.path.join(_PATH_INTEGRATIONS_DIR, "flashy")) as (admin_page, view_page, fetch_logs, _): + + validate_app_functionalities(view_page)