From 4586d118298cea2fe080197f79cb21dd8d324cbc Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Aug 2022 17:50:30 +0200 Subject: [PATCH] adding CI for e2e on Azure (#14282) * start CI * wip * matrix * name * wip * prune * rm * ls * cache * dir * ls * name * cleaning * clone * git * git * if * private * . * local_id * clean * var * group check * ci --- .azure/app-cloud-e2e.yml | 140 +++++++++++++++ .github/checkgroup.yml | 14 +- .github/workflows/ci-app-cloud-e2e-test.yml | 187 -------------------- 3 files changed, 142 insertions(+), 199 deletions(-) create mode 100644 .azure/app-cloud-e2e.yml delete mode 100644 .github/workflows/ci-app-cloud-e2e-test.yml diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml new file mode 100644 index 0000000000..d5aeee88c0 --- /dev/null +++ b/.azure/app-cloud-e2e.yml @@ -0,0 +1,140 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: + tags: + include: + - '*' + branches: + include: + - "master" + - "release/*" + - "refs/tags/*" + +pr: + - "master" + - "release/*" + +variables: + # variables are automatically exported as environment variables so this will override pip's default cache dir + - name: pip_cache_dir + value: $(Pipeline.Workspace)/.pip + +jobs: + - job: App_cloud_e2e_testing + pool: + vmImage: 'ubuntu-latest' + timeoutInMinutes: "30" + cancelTimeoutInMinutes: "2" + strategy: + matrix: + 'App: v0_app': + name: "v0_app" + 'App: boring_app': + name: "boring_app" + 'App: template_streamlit_ui': + name: "template_streamlit_ui" +# 'App: template_react_ui': # TODO: clarify visibility private/public +# name: "template_react_ui" + 'App: template_jupyterlab': # TODO: clarify where these files lives + name: "template_jupyterlab" + 'App: idle_timeout': + name: "idle_timeout" + 'App: collect_failures': + name: "collect_failures" + 'App: custom_work_dependencies': + name: "custom_work_dependencies" + 'App: drive': + name: "drive" + 'App: payload': + name: "payload" + 'App: commands_and_api': + name: "commands_and_api" + workspace: + clean: all + steps: + - bash: | + python --version + pip --version + displayName: 'Info' + + # TODO: parse the PR number + - bash: | + ID=$(date +%s) + echo "##vso[task.setvariable variable=local_id]$ID" + + - task: Cache@2 + inputs: + key: 'pip | "$(name)" | requirements/app/base.txt' + restoreKeys: | + pip | "$(Agent.OS)" + path: $(pip_cache_dir) + displayName: Cache pip + + - bash: python -m pip install -r requirements/app/devel.txt --quiet --find-links ${TORCH_URL} + env: + TORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html + displayName: 'Install dependencies' + + - bash: | + python -m pip install playwright + python -m playwright install --with-deps + displayName: 'Install Playwright system dependencies' + + - bash: pip install -e . + displayName: 'Install lightning' + + - bash: | + git clone https://github.com/Lightning-AI/LAI-lightning-template-jupyterlab-App examples/app_template_jupyterlab + cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py + condition: eq(variables['name'], 'template_jupyterlab') + displayName: 'Clone Template Jupyter Lab Repo' + + - bash: git clone https://github.com/Lightning-AI/lightning-template-react examples/app_template_react_ui + condition: eq(variables['name'], 'template_react_ui') + displayName: 'Clone Template React UI Repo' + + - bash: | + mkdir -p ${VIDEO_LOCATION} + ls -l examples/${TEST_APP_NAME} + ls -l tests/tests_app_examples + python -m pytest tests/tests_app_examples/test_${TEST_APP_NAME}.py::test_${TEST_APP_NAME}_example_cloud --timeout=900 --capture=no -v --color=yes + env: + HEADLESS: '1' + PACKAGE_LIGHTNING: '1' + CLOUD: '1' + VIDEO_LOCATION: '$(Build.ArtifactStagingDirectory)/videos' + PR_NUMBER: $(local_id) + TEST_APP_NAME: $(name) + HAR_LOCATION: './artifacts/hars' + SLOW_MO: '50' + LAI_USER: $(LAI_USER) + LAI_PASS: $(LAI_PASS) + LIGHTNING_USER_ID: $(LIGHTNING_USER_ID) + LIGHTNING_API_KEY: $(LIGHTNING_API_KEY) + LIGHTNING_USERNAME: $(LIGHTNING_USERNAME) + LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL) + displayName: 'Run the tests' + + - publish: '$(Build.ArtifactStagingDirectory)/videos' + displayName: 'Publish videos' + artifact: $(name) + + - bash: | + time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()" + env: + LAI_USER: $(LAI_USER) + LAI_PASS: $(LAI_PASS) + LIGHTNING_USER_ID: $(LIGHTNING_USER_ID) + LIGHTNING_API_KEY: $(LIGHTNING_API_KEY) + LIGHTNING_USERNAME: $(LIGHTNING_USERNAME) + LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL) + PR_NUMBER: $(local_id) + TEST_APP_NAME: $(name) + GRID_USER_ID: $(LIGHTNING_USER_ID) # TODO: clarify the meaning + GRID_USER_KEY: $(LIGHTNING_API_KEY) # TODO: clarify the meaning + GRID_URL: $(LIGHTNING_CLOUD_URL) + _GRID_USERNAME: $(LIGHTNING_USERNAME) + displayName: 'Clean Previous Apps' diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index c2654eddd7..6389fdf71e 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -108,7 +108,7 @@ subprojects: - id: "lightning_app" paths: - - ".github/workflows/ci-app*.yml" + - ".azure/app-cloud-e2e.yml" - "requirements/app/**" - "src/lightning_app/**" - "tests/tests_app/**" @@ -117,17 +117,7 @@ subprojects: # the examples are used in the app CI - "examples/app_*" checks: - - "Cloud Test (boring_app)" - - "Cloud Test (collect_failures)" - - "Cloud Test (commands_and_api)" - - "Cloud Test (custom_work_dependencies)" - - "Cloud Test (drive)" - - "Cloud Test (idle_timeout)" - - "Cloud Test (payload)" - - "Cloud Test (template_jupyterlab)" - - "Cloud Test (template_react_ui)" - - "Cloud Test (template_streamlit_ui)" - - "Cloud Test (v0_app)" + - "App.cloud-e2e" - "doctest (app)" - "make-docs (app)" - "pytest (macOS-11, 3.8, latest)" diff --git a/.github/workflows/ci-app-cloud-e2e-test.yml b/.github/workflows/ci-app-cloud-e2e-test.yml deleted file mode 100644 index 07b253f5d6..0000000000 --- a/.github/workflows/ci-app-cloud-e2e-test.yml +++ /dev/null @@ -1,187 +0,0 @@ -name: cloud-testing - -# Used to run the e2e tests on lightning.ai - -on: # Trigger the workflow on push or pull request, but only for the master branch - push: - branches: [master, "release/*"] - pull_request: - branches: [master, "release/*"] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} - -jobs: - # This is job should once only once per PR to detect file changes so run required jobs. - # see .github/file-filters.yml to define file filters and run the jobs based on the output of each filter. - # More info: https://github.com/marketplace/actions/paths-changes-filter - - changes: - runs-on: ubuntu-latest - # Set job outputs to the values from filter step - outputs: - app_examples: ${{ steps.filter.outputs.app_examples }} - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: .github/file-filters.yml - - cloud-test: - name: Cloud Test - needs: changes - if: ${{ needs.changes.outputs.app_examples == 'true' }} - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - app_name: - - v0_app - - boring_app -# - quick_start # TODO: fix this - - template_streamlit_ui - - template_react_ui - - template_jupyterlab - - idle_timeout - - collect_failures - - custom_work_dependencies - - drive - - payload - - commands_and_api - timeout-minutes: 35 - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Get PR ID - id: PR - run: | - if [ -z ${{github.event.number}} ]; then - echo "::set-output name=ID::$(date +%s)" - else - echo "::set-output name=ID::${{github.event.number}}" - fi - -# TODO: Enable cache -# - name: Cache virtualenv -# id: cache-venv -# uses: actions/cache@v3 -# with: -# path: ./.venv/ -# key: ${{ runner.os }}-pip-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }} -# restore-keys: ${{ runner.os }}-venv-${{ matrix.app_name }}- - - - name: Install dependencies - shell: bash - run: | - pip --version - python -m pip install -r requirements/app/devel.txt --no-cache --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html -# if: steps.cache-venv.outputs.cache-hit != 'true' # TODO: Enable cache - - - name: Cache Playwright dependencies - id: playwright-cache - uses: actions/cache@v3 - with: - path: ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }} - restore-keys: ${{ runner.os }}-playwright-${{ matrix.app_name }}- - - - name: Install Playwright system dependencies - shell: bash - run: | - python -m pip install playwright - python -m playwright install --with-deps -# if: steps.playwright-cache.outputs.cache-hit != 'true' # TODO: Enable cache - - - name: Install lightning - run: | - pip install -e . - shell: bash - - - name: Lightning Install quick-start - if: ${{ (matrix.app_name == 'quick_start') }} - shell: bash - run: | - python -m lightning install app lightning/quick-start -y - - - name: Clone Template React UI Repo - uses: actions/checkout@v3 - with: - repository: Lightning-AI/lightning-template-react - token: ${{ secrets.PAT_GHOST }} - ref: 'master' - path: examples/app_template_react_ui - - - name: Clone Template Jupyter Lab Repo - uses: actions/checkout@v3 - with: - repository: Lightning-AI/lightning-template-jupyterlab - token: ${{ secrets.PAT_GHOST }} - ref: 'master' - path: examples/app_template_jupyterlab - - - name: Copy Template Jupyter Lab Repo tests - shell: bash - run: cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py - - - name: List pip dependency - shell: bash - run: | - pip list - - - name: Run the tests - env: - LAI_USER: ${{ secrets.LAI_USER }} - LAI_PASS: ${{ secrets.LAI_PASS }} - LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }} - LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }} - LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }} - LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }} - CLOUD: "1" - VIDEO_LOCATION: ./artifacts/videos - PR_NUMBER: ${{ steps.PR.outputs.ID }} - TEST_APP_NAME: ${{ matrix.app_name }} - HAR_LOCATION: ./artifacts/hars - SLOW_MO: 50 - shell: bash - run: | - mkdir -p ${VIDEO_LOCATION} - HEADLESS=1 PACKAGE_LIGHTNING=1 python -m pytest tests/tests_app_examples/test_${{ matrix.app_name }}.py::test_${{ matrix.app_name }}_example_cloud --timeout=900 --capture=no -v --color=yes - # Delete the artifacts if successful - rm -r ${VIDEO_LOCATION}/${{ matrix.app_name }} - - - uses: actions/upload-artifact@v3 - - if: ${{ always() }} - with: - name: test-artifacts - path: ./artifacts/videos - - - name: Clean Previous Apps - if: ${{ always() }} - env: - LAI_USER: ${{ secrets.LAI_USER }} - LAI_PASS: ${{ secrets.LAI_PASS }} - LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }} - LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }} - LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }} - LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }} - PR_NUMBER: ${{ steps.PR.outputs.ID }} - TEST_APP_NAME: ${{ matrix.app_name }} - GRID_USER_ID: ${{ secrets.LIGHTNING_USER_ID }} - GRID_USER_KEY: ${{ secrets.LIGHTNING_API_KEY }} - GRID_URL: ${{ secrets.LIGHTNING_CLOUD_URL }} - _GRID_USERNAME: ${{ secrets.LIGHTNING_USERNAME }} - shell: bash - run: | - time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"