adding CI for e2e on Azure (#14282)
* start CI * wip * matrix * name * wip * prune * rm * ls * cache * dir * ls * name * cleaning * clone * git * git * if * private * . * local_id * clean * var * group check * ci
This commit is contained in:
Jirka Borovec
GitHub
parent
4c47619f38
commit
4586d11829
|
|
@ -0,0 +1,140 @@
|
||||||
|
# Python package
|
||||||
|
# Create and test a Python package on multiple Python versions.
|
||||||
|
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
|
||||||
|
# https://docs.microsoft.com/azure/devops/pipelines/languages/python
|
||||||
|
|
||||||
|
trigger:
|
||||||
|
tags:
|
||||||
|
include:
|
||||||
|
- '*'
|
||||||
|
branches:
|
||||||
|
include:
|
||||||
|
- "master"
|
||||||
|
- "release/*"
|
||||||
|
- "refs/tags/*"
|
||||||
|
|
||||||
|
pr:
|
||||||
|
- "master"
|
||||||
|
- "release/*"
|
||||||
|
|
||||||
|
variables:
|
||||||
|
# variables are automatically exported as environment variables so this will override pip's default cache dir
|
||||||
|
- name: pip_cache_dir
|
||||||
|
value: $(Pipeline.Workspace)/.pip
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
- job: App_cloud_e2e_testing
|
||||||
|
pool:
|
||||||
|
vmImage: 'ubuntu-latest'
|
||||||
|
timeoutInMinutes: "30"
|
||||||
|
cancelTimeoutInMinutes: "2"
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
'App: v0_app':
|
||||||
|
name: "v0_app"
|
||||||
|
'App: boring_app':
|
||||||
|
name: "boring_app"
|
||||||
|
'App: template_streamlit_ui':
|
||||||
|
name: "template_streamlit_ui"
|
||||||
|
# 'App: template_react_ui': # TODO: clarify visibility private/public
|
||||||
|
# name: "template_react_ui"
|
||||||
|
'App: template_jupyterlab': # TODO: clarify where these files lives
|
||||||
|
name: "template_jupyterlab"
|
||||||
|
'App: idle_timeout':
|
||||||
|
name: "idle_timeout"
|
||||||
|
'App: collect_failures':
|
||||||
|
name: "collect_failures"
|
||||||
|
'App: custom_work_dependencies':
|
||||||
|
name: "custom_work_dependencies"
|
||||||
|
'App: drive':
|
||||||
|
name: "drive"
|
||||||
|
'App: payload':
|
||||||
|
name: "payload"
|
||||||
|
'App: commands_and_api':
|
||||||
|
name: "commands_and_api"
|
||||||
|
workspace:
|
||||||
|
clean: all
|
||||||
|
steps:
|
||||||
|
- bash: |
|
||||||
|
python --version
|
||||||
|
pip --version
|
||||||
|
displayName: 'Info'
|
||||||
|
|
||||||
|
# TODO: parse the PR number
|
||||||
|
- bash: |
|
||||||
|
ID=$(date +%s)
|
||||||
|
echo "##vso[task.setvariable variable=local_id]$ID"
|
||||||
|
|
||||||
|
- task: Cache@2
|
||||||
|
inputs:
|
||||||
|
key: 'pip | "$(name)" | requirements/app/base.txt'
|
||||||
|
restoreKeys: |
|
||||||
|
pip | "$(Agent.OS)"
|
||||||
|
path: $(pip_cache_dir)
|
||||||
|
displayName: Cache pip
|
||||||
|
|
||||||
|
- bash: python -m pip install -r requirements/app/devel.txt --quiet --find-links ${TORCH_URL}
|
||||||
|
env:
|
||||||
|
TORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html
|
||||||
|
displayName: 'Install dependencies'
|
||||||
|
|
||||||
|
- bash: |
|
||||||
|
python -m pip install playwright
|
||||||
|
python -m playwright install --with-deps
|
||||||
|
displayName: 'Install Playwright system dependencies'
|
||||||
|
|
||||||
|
- bash: pip install -e .
|
||||||
|
displayName: 'Install lightning'
|
||||||
|
|
||||||
|
- bash: |
|
||||||
|
git clone https://github.com/Lightning-AI/LAI-lightning-template-jupyterlab-App examples/app_template_jupyterlab
|
||||||
|
cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py
|
||||||
|
condition: eq(variables['name'], 'template_jupyterlab')
|
||||||
|
displayName: 'Clone Template Jupyter Lab Repo'
|
||||||
|
|
||||||
|
- bash: git clone https://github.com/Lightning-AI/lightning-template-react examples/app_template_react_ui
|
||||||
|
condition: eq(variables['name'], 'template_react_ui')
|
||||||
|
displayName: 'Clone Template React UI Repo'
|
||||||
|
|
||||||
|
- bash: |
|
||||||
|
mkdir -p ${VIDEO_LOCATION}
|
||||||
|
ls -l examples/${TEST_APP_NAME}
|
||||||
|
ls -l tests/tests_app_examples
|
||||||
|
python -m pytest tests/tests_app_examples/test_${TEST_APP_NAME}.py::test_${TEST_APP_NAME}_example_cloud --timeout=900 --capture=no -v --color=yes
|
||||||
|
env:
|
||||||
|
HEADLESS: '1'
|
||||||
|
PACKAGE_LIGHTNING: '1'
|
||||||
|
CLOUD: '1'
|
||||||
|
VIDEO_LOCATION: '$(Build.ArtifactStagingDirectory)/videos'
|
||||||
|
PR_NUMBER: $(local_id)
|
||||||
|
TEST_APP_NAME: $(name)
|
||||||
|
HAR_LOCATION: './artifacts/hars'
|
||||||
|
SLOW_MO: '50'
|
||||||
|
LAI_USER: $(LAI_USER)
|
||||||
|
LAI_PASS: $(LAI_PASS)
|
||||||
|
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID)
|
||||||
|
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY)
|
||||||
|
LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
|
||||||
|
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL)
|
||||||
|
displayName: 'Run the tests'
|
||||||
|
|
||||||
|
- publish: '$(Build.ArtifactStagingDirectory)/videos'
|
||||||
|
displayName: 'Publish videos'
|
||||||
|
artifact: $(name)
|
||||||
|
|
||||||
|
- bash: |
|
||||||
|
time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
|
||||||
|
env:
|
||||||
|
LAI_USER: $(LAI_USER)
|
||||||
|
LAI_PASS: $(LAI_PASS)
|
||||||
|
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID)
|
||||||
|
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY)
|
||||||
|
LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
|
||||||
|
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL)
|
||||||
|
PR_NUMBER: $(local_id)
|
||||||
|
TEST_APP_NAME: $(name)
|
||||||
|
GRID_USER_ID: $(LIGHTNING_USER_ID) # TODO: clarify the meaning
|
||||||
|
GRID_USER_KEY: $(LIGHTNING_API_KEY) # TODO: clarify the meaning
|
||||||
|
GRID_URL: $(LIGHTNING_CLOUD_URL)
|
||||||
|
_GRID_USERNAME: $(LIGHTNING_USERNAME)
|
||||||
|
displayName: 'Clean Previous Apps'
|
||||||
|
|
@ -108,7 +108,7 @@ subprojects:
|
||||||
|
|
||||||
- id: "lightning_app"
|
- id: "lightning_app"
|
||||||
paths:
|
paths:
|
||||||
- ".github/workflows/ci-app*.yml"
|
- ".azure/app-cloud-e2e.yml"
|
||||||
- "requirements/app/**"
|
- "requirements/app/**"
|
||||||
- "src/lightning_app/**"
|
- "src/lightning_app/**"
|
||||||
- "tests/tests_app/**"
|
- "tests/tests_app/**"
|
||||||
|
|
@ -117,17 +117,7 @@ subprojects:
|
||||||
# the examples are used in the app CI
|
# the examples are used in the app CI
|
||||||
- "examples/app_*"
|
- "examples/app_*"
|
||||||
checks:
|
checks:
|
||||||
- "Cloud Test (boring_app)"
|
- "App.cloud-e2e"
|
||||||
- "Cloud Test (collect_failures)"
|
|
||||||
- "Cloud Test (commands_and_api)"
|
|
||||||
- "Cloud Test (custom_work_dependencies)"
|
|
||||||
- "Cloud Test (drive)"
|
|
||||||
- "Cloud Test (idle_timeout)"
|
|
||||||
- "Cloud Test (payload)"
|
|
||||||
- "Cloud Test (template_jupyterlab)"
|
|
||||||
- "Cloud Test (template_react_ui)"
|
|
||||||
- "Cloud Test (template_streamlit_ui)"
|
|
||||||
- "Cloud Test (v0_app)"
|
|
||||||
- "doctest (app)"
|
- "doctest (app)"
|
||||||
- "make-docs (app)"
|
- "make-docs (app)"
|
||||||
- "pytest (macOS-11, 3.8, latest)"
|
- "pytest (macOS-11, 3.8, latest)"
|
||||||
|
|
|
||||||
|
|
@ -1,187 +0,0 @@
|
||||||
name: cloud-testing
|
|
||||||
|
|
||||||
# Used to run the e2e tests on lightning.ai
|
|
||||||
|
|
||||||
on: # Trigger the workflow on push or pull request, but only for the master branch
|
|
||||||
push:
|
|
||||||
branches: [master, "release/*"]
|
|
||||||
pull_request:
|
|
||||||
branches: [master, "release/*"]
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
|
|
||||||
cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
# This is job should once only once per PR to detect file changes so run required jobs.
|
|
||||||
# see .github/file-filters.yml to define file filters and run the jobs based on the output of each filter.
|
|
||||||
# More info: https://github.com/marketplace/actions/paths-changes-filter
|
|
||||||
|
|
||||||
changes:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
# Set job outputs to the values from filter step
|
|
||||||
outputs:
|
|
||||||
app_examples: ${{ steps.filter.outputs.app_examples }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2
|
|
||||||
- name: Set up Python 3.8
|
|
||||||
uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: "3.8"
|
|
||||||
|
|
||||||
- uses: dorny/paths-filter@v2
|
|
||||||
id: filter
|
|
||||||
with:
|
|
||||||
filters: .github/file-filters.yml
|
|
||||||
|
|
||||||
cloud-test:
|
|
||||||
name: Cloud Test
|
|
||||||
needs: changes
|
|
||||||
if: ${{ needs.changes.outputs.app_examples == 'true' }}
|
|
||||||
runs-on: ubuntu-20.04
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
app_name:
|
|
||||||
- v0_app
|
|
||||||
- boring_app
|
|
||||||
# - quick_start # TODO: fix this
|
|
||||||
- template_streamlit_ui
|
|
||||||
- template_react_ui
|
|
||||||
- template_jupyterlab
|
|
||||||
- idle_timeout
|
|
||||||
- collect_failures
|
|
||||||
- custom_work_dependencies
|
|
||||||
- drive
|
|
||||||
- payload
|
|
||||||
- commands_and_api
|
|
||||||
timeout-minutes: 35
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2
|
|
||||||
- name: Set up Python 3.8
|
|
||||||
uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: "3.8"
|
|
||||||
|
|
||||||
- name: Get PR ID
|
|
||||||
id: PR
|
|
||||||
run: |
|
|
||||||
if [ -z ${{github.event.number}} ]; then
|
|
||||||
echo "::set-output name=ID::$(date +%s)"
|
|
||||||
else
|
|
||||||
echo "::set-output name=ID::${{github.event.number}}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# TODO: Enable cache
|
|
||||||
# - name: Cache virtualenv
|
|
||||||
# id: cache-venv
|
|
||||||
# uses: actions/cache@v3
|
|
||||||
# with:
|
|
||||||
# path: ./.venv/
|
|
||||||
# key: ${{ runner.os }}-pip-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }}
|
|
||||||
# restore-keys: ${{ runner.os }}-venv-${{ matrix.app_name }}-
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
pip --version
|
|
||||||
python -m pip install -r requirements/app/devel.txt --no-cache --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
|
|
||||||
# if: steps.cache-venv.outputs.cache-hit != 'true' # TODO: Enable cache
|
|
||||||
|
|
||||||
- name: Cache Playwright dependencies
|
|
||||||
id: playwright-cache
|
|
||||||
uses: actions/cache@v3
|
|
||||||
with:
|
|
||||||
path: ~/.cache/ms-playwright
|
|
||||||
key: ${{ runner.os }}-playwright-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }}
|
|
||||||
restore-keys: ${{ runner.os }}-playwright-${{ matrix.app_name }}-
|
|
||||||
|
|
||||||
- name: Install Playwright system dependencies
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
python -m pip install playwright
|
|
||||||
python -m playwright install --with-deps
|
|
||||||
# if: steps.playwright-cache.outputs.cache-hit != 'true' # TODO: Enable cache
|
|
||||||
|
|
||||||
- name: Install lightning
|
|
||||||
run: |
|
|
||||||
pip install -e .
|
|
||||||
shell: bash
|
|
||||||
|
|
||||||
- name: Lightning Install quick-start
|
|
||||||
if: ${{ (matrix.app_name == 'quick_start') }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
python -m lightning install app lightning/quick-start -y
|
|
||||||
|
|
||||||
- name: Clone Template React UI Repo
|
|
||||||
uses: actions/checkout@v3
|
|
||||||
with:
|
|
||||||
repository: Lightning-AI/lightning-template-react
|
|
||||||
token: ${{ secrets.PAT_GHOST }}
|
|
||||||
ref: 'master'
|
|
||||||
path: examples/app_template_react_ui
|
|
||||||
|
|
||||||
- name: Clone Template Jupyter Lab Repo
|
|
||||||
uses: actions/checkout@v3
|
|
||||||
with:
|
|
||||||
repository: Lightning-AI/lightning-template-jupyterlab
|
|
||||||
token: ${{ secrets.PAT_GHOST }}
|
|
||||||
ref: 'master'
|
|
||||||
path: examples/app_template_jupyterlab
|
|
||||||
|
|
||||||
- name: Copy Template Jupyter Lab Repo tests
|
|
||||||
shell: bash
|
|
||||||
run: cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py
|
|
||||||
|
|
||||||
- name: List pip dependency
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
pip list
|
|
||||||
|
|
||||||
- name: Run the tests
|
|
||||||
env:
|
|
||||||
LAI_USER: ${{ secrets.LAI_USER }}
|
|
||||||
LAI_PASS: ${{ secrets.LAI_PASS }}
|
|
||||||
LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
|
|
||||||
LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }}
|
|
||||||
LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }}
|
|
||||||
LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }}
|
|
||||||
CLOUD: "1"
|
|
||||||
VIDEO_LOCATION: ./artifacts/videos
|
|
||||||
PR_NUMBER: ${{ steps.PR.outputs.ID }}
|
|
||||||
TEST_APP_NAME: ${{ matrix.app_name }}
|
|
||||||
HAR_LOCATION: ./artifacts/hars
|
|
||||||
SLOW_MO: 50
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
mkdir -p ${VIDEO_LOCATION}
|
|
||||||
HEADLESS=1 PACKAGE_LIGHTNING=1 python -m pytest tests/tests_app_examples/test_${{ matrix.app_name }}.py::test_${{ matrix.app_name }}_example_cloud --timeout=900 --capture=no -v --color=yes
|
|
||||||
# Delete the artifacts if successful
|
|
||||||
rm -r ${VIDEO_LOCATION}/${{ matrix.app_name }}
|
|
||||||
|
|
||||||
- uses: actions/upload-artifact@v3
|
|
||||||
|
|
||||||
if: ${{ always() }}
|
|
||||||
with:
|
|
||||||
name: test-artifacts
|
|
||||||
path: ./artifacts/videos
|
|
||||||
|
|
||||||
- name: Clean Previous Apps
|
|
||||||
if: ${{ always() }}
|
|
||||||
env:
|
|
||||||
LAI_USER: ${{ secrets.LAI_USER }}
|
|
||||||
LAI_PASS: ${{ secrets.LAI_PASS }}
|
|
||||||
LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
|
|
||||||
LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }}
|
|
||||||
LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }}
|
|
||||||
LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }}
|
|
||||||
PR_NUMBER: ${{ steps.PR.outputs.ID }}
|
|
||||||
TEST_APP_NAME: ${{ matrix.app_name }}
|
|
||||||
GRID_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
|
|
||||||
GRID_USER_KEY: ${{ secrets.LIGHTNING_API_KEY }}
|
|
||||||
GRID_URL: ${{ secrets.LIGHTNING_CLOUD_URL }}
|
|
||||||
_GRID_USERNAME: ${{ secrets.LIGHTNING_USERNAME }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
|
|
||||||
Loading…
Reference in New Issue