adding CI for e2e on Azure (#14282)
* start CI * wip * matrix * name * wip * prune * rm * ls * cache * dir * ls * name * cleaning * clone * git * git * if * private * . * local_id * clean * var * group check * ci
This commit is contained in:
Jirka Borovec
GitHub
parent
4c47619f38
commit
4586d11829
|
|
@ -0,0 +1,140 @@
|
|||
# Python package
|
||||
# Create and test a Python package on multiple Python versions.
|
||||
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
|
||||
# https://docs.microsoft.com/azure/devops/pipelines/languages/python
|
||||
|
||||
trigger:
|
||||
tags:
|
||||
include:
|
||||
- '*'
|
||||
branches:
|
||||
include:
|
||||
- "master"
|
||||
- "release/*"
|
||||
- "refs/tags/*"
|
||||
|
||||
pr:
|
||||
- "master"
|
||||
- "release/*"
|
||||
|
||||
variables:
|
||||
# variables are automatically exported as environment variables so this will override pip's default cache dir
|
||||
- name: pip_cache_dir
|
||||
value: $(Pipeline.Workspace)/.pip
|
||||
|
||||
jobs:
|
||||
- job: App_cloud_e2e_testing
|
||||
pool:
|
||||
vmImage: 'ubuntu-latest'
|
||||
timeoutInMinutes: "30"
|
||||
cancelTimeoutInMinutes: "2"
|
||||
strategy:
|
||||
matrix:
|
||||
'App: v0_app':
|
||||
name: "v0_app"
|
||||
'App: boring_app':
|
||||
name: "boring_app"
|
||||
'App: template_streamlit_ui':
|
||||
name: "template_streamlit_ui"
|
||||
# 'App: template_react_ui': # TODO: clarify visibility private/public
|
||||
# name: "template_react_ui"
|
||||
'App: template_jupyterlab': # TODO: clarify where these files lives
|
||||
name: "template_jupyterlab"
|
||||
'App: idle_timeout':
|
||||
name: "idle_timeout"
|
||||
'App: collect_failures':
|
||||
name: "collect_failures"
|
||||
'App: custom_work_dependencies':
|
||||
name: "custom_work_dependencies"
|
||||
'App: drive':
|
||||
name: "drive"
|
||||
'App: payload':
|
||||
name: "payload"
|
||||
'App: commands_and_api':
|
||||
name: "commands_and_api"
|
||||
workspace:
|
||||
clean: all
|
||||
steps:
|
||||
- bash: |
|
||||
python --version
|
||||
pip --version
|
||||
displayName: 'Info'
|
||||
|
||||
# TODO: parse the PR number
|
||||
- bash: |
|
||||
ID=$(date +%s)
|
||||
echo "##vso[task.setvariable variable=local_id]$ID"
|
||||
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: 'pip | "$(name)" | requirements/app/base.txt'
|
||||
restoreKeys: |
|
||||
pip | "$(Agent.OS)"
|
||||
path: $(pip_cache_dir)
|
||||
displayName: Cache pip
|
||||
|
||||
- bash: python -m pip install -r requirements/app/devel.txt --quiet --find-links ${TORCH_URL}
|
||||
env:
|
||||
TORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html
|
||||
displayName: 'Install dependencies'
|
||||
|
||||
- bash: |
|
||||
python -m pip install playwright
|
||||
python -m playwright install --with-deps
|
||||
displayName: 'Install Playwright system dependencies'
|
||||
|
||||
- bash: pip install -e .
|
||||
displayName: 'Install lightning'
|
||||
|
||||
- bash: |
|
||||
git clone https://github.com/Lightning-AI/LAI-lightning-template-jupyterlab-App examples/app_template_jupyterlab
|
||||
cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py
|
||||
condition: eq(variables['name'], 'template_jupyterlab')
|
||||
displayName: 'Clone Template Jupyter Lab Repo'
|
||||
|
||||
- bash: git clone https://github.com/Lightning-AI/lightning-template-react examples/app_template_react_ui
|
||||
condition: eq(variables['name'], 'template_react_ui')
|
||||
displayName: 'Clone Template React UI Repo'
|
||||
|
||||
- bash: |
|
||||
mkdir -p ${VIDEO_LOCATION}
|
||||
ls -l examples/${TEST_APP_NAME}
|
||||
ls -l tests/tests_app_examples
|
||||
python -m pytest tests/tests_app_examples/test_${TEST_APP_NAME}.py::test_${TEST_APP_NAME}_example_cloud --timeout=900 --capture=no -v --color=yes
|
||||
env:
|
||||
HEADLESS: '1'
|
||||
PACKAGE_LIGHTNING: '1'
|
||||
CLOUD: '1'
|
||||
VIDEO_LOCATION: '$(Build.ArtifactStagingDirectory)/videos'
|
||||
PR_NUMBER: $(local_id)
|
||||
TEST_APP_NAME: $(name)
|
||||
HAR_LOCATION: './artifacts/hars'
|
||||
SLOW_MO: '50'
|
||||
LAI_USER: $(LAI_USER)
|
||||
LAI_PASS: $(LAI_PASS)
|
||||
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID)
|
||||
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY)
|
||||
LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
|
||||
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL)
|
||||
displayName: 'Run the tests'
|
||||
|
||||
- publish: '$(Build.ArtifactStagingDirectory)/videos'
|
||||
displayName: 'Publish videos'
|
||||
artifact: $(name)
|
||||
|
||||
- bash: |
|
||||
time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
|
||||
env:
|
||||
LAI_USER: $(LAI_USER)
|
||||
LAI_PASS: $(LAI_PASS)
|
||||
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID)
|
||||
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY)
|
||||
LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
|
||||
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL)
|
||||
PR_NUMBER: $(local_id)
|
||||
TEST_APP_NAME: $(name)
|
||||
GRID_USER_ID: $(LIGHTNING_USER_ID) # TODO: clarify the meaning
|
||||
GRID_USER_KEY: $(LIGHTNING_API_KEY) # TODO: clarify the meaning
|
||||
GRID_URL: $(LIGHTNING_CLOUD_URL)
|
||||
_GRID_USERNAME: $(LIGHTNING_USERNAME)
|
||||
displayName: 'Clean Previous Apps'
|
||||
|
|
@ -108,7 +108,7 @@ subprojects:
|
|||
|
||||
- id: "lightning_app"
|
||||
paths:
|
||||
- ".github/workflows/ci-app*.yml"
|
||||
- ".azure/app-cloud-e2e.yml"
|
||||
- "requirements/app/**"
|
||||
- "src/lightning_app/**"
|
||||
- "tests/tests_app/**"
|
||||
|
|
@ -117,17 +117,7 @@ subprojects:
|
|||
# the examples are used in the app CI
|
||||
- "examples/app_*"
|
||||
checks:
|
||||
- "Cloud Test (boring_app)"
|
||||
- "Cloud Test (collect_failures)"
|
||||
- "Cloud Test (commands_and_api)"
|
||||
- "Cloud Test (custom_work_dependencies)"
|
||||
- "Cloud Test (drive)"
|
||||
- "Cloud Test (idle_timeout)"
|
||||
- "Cloud Test (payload)"
|
||||
- "Cloud Test (template_jupyterlab)"
|
||||
- "Cloud Test (template_react_ui)"
|
||||
- "Cloud Test (template_streamlit_ui)"
|
||||
- "Cloud Test (v0_app)"
|
||||
- "App.cloud-e2e"
|
||||
- "doctest (app)"
|
||||
- "make-docs (app)"
|
||||
- "pytest (macOS-11, 3.8, latest)"
|
||||
|
|
|
|||
|
|
@ -1,187 +0,0 @@
|
|||
name: cloud-testing
|
||||
|
||||
# Used to run the e2e tests on lightning.ai
|
||||
|
||||
on: # Trigger the workflow on push or pull request, but only for the master branch
|
||||
push:
|
||||
branches: [master, "release/*"]
|
||||
pull_request:
|
||||
branches: [master, "release/*"]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
|
||||
cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
|
||||
|
||||
jobs:
|
||||
# This is job should once only once per PR to detect file changes so run required jobs.
|
||||
# see .github/file-filters.yml to define file filters and run the jobs based on the output of each filter.
|
||||
# More info: https://github.com/marketplace/actions/paths-changes-filter
|
||||
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Set job outputs to the values from filter step
|
||||
outputs:
|
||||
app_examples: ${{ steps.filter.outputs.app_examples }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.8
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.8"
|
||||
|
||||
- uses: dorny/paths-filter@v2
|
||||
id: filter
|
||||
with:
|
||||
filters: .github/file-filters.yml
|
||||
|
||||
cloud-test:
|
||||
name: Cloud Test
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.app_examples == 'true' }}
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
app_name:
|
||||
- v0_app
|
||||
- boring_app
|
||||
# - quick_start # TODO: fix this
|
||||
- template_streamlit_ui
|
||||
- template_react_ui
|
||||
- template_jupyterlab
|
||||
- idle_timeout
|
||||
- collect_failures
|
||||
- custom_work_dependencies
|
||||
- drive
|
||||
- payload
|
||||
- commands_and_api
|
||||
timeout-minutes: 35
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.8
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.8"
|
||||
|
||||
- name: Get PR ID
|
||||
id: PR
|
||||
run: |
|
||||
if [ -z ${{github.event.number}} ]; then
|
||||
echo "::set-output name=ID::$(date +%s)"
|
||||
else
|
||||
echo "::set-output name=ID::${{github.event.number}}"
|
||||
fi
|
||||
|
||||
# TODO: Enable cache
|
||||
# - name: Cache virtualenv
|
||||
# id: cache-venv
|
||||
# uses: actions/cache@v3
|
||||
# with:
|
||||
# path: ./.venv/
|
||||
# key: ${{ runner.os }}-pip-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }}
|
||||
# restore-keys: ${{ runner.os }}-venv-${{ matrix.app_name }}-
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
pip --version
|
||||
python -m pip install -r requirements/app/devel.txt --no-cache --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
|
||||
# if: steps.cache-venv.outputs.cache-hit != 'true' # TODO: Enable cache
|
||||
|
||||
- name: Cache Playwright dependencies
|
||||
id: playwright-cache
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-playwright-${{ matrix.app_name }}-${{ hashFiles('requirements/app/base.txt', 'requirements/app/*.txt', 'src/lightning_app/__version__.py') }}
|
||||
restore-keys: ${{ runner.os }}-playwright-${{ matrix.app_name }}-
|
||||
|
||||
- name: Install Playwright system dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
python -m pip install playwright
|
||||
python -m playwright install --with-deps
|
||||
# if: steps.playwright-cache.outputs.cache-hit != 'true' # TODO: Enable cache
|
||||
|
||||
- name: Install lightning
|
||||
run: |
|
||||
pip install -e .
|
||||
shell: bash
|
||||
|
||||
- name: Lightning Install quick-start
|
||||
if: ${{ (matrix.app_name == 'quick_start') }}
|
||||
shell: bash
|
||||
run: |
|
||||
python -m lightning install app lightning/quick-start -y
|
||||
|
||||
- name: Clone Template React UI Repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
repository: Lightning-AI/lightning-template-react
|
||||
token: ${{ secrets.PAT_GHOST }}
|
||||
ref: 'master'
|
||||
path: examples/app_template_react_ui
|
||||
|
||||
- name: Clone Template Jupyter Lab Repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
repository: Lightning-AI/lightning-template-jupyterlab
|
||||
token: ${{ secrets.PAT_GHOST }}
|
||||
ref: 'master'
|
||||
path: examples/app_template_jupyterlab
|
||||
|
||||
- name: Copy Template Jupyter Lab Repo tests
|
||||
shell: bash
|
||||
run: cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py
|
||||
|
||||
- name: List pip dependency
|
||||
shell: bash
|
||||
run: |
|
||||
pip list
|
||||
|
||||
- name: Run the tests
|
||||
env:
|
||||
LAI_USER: ${{ secrets.LAI_USER }}
|
||||
LAI_PASS: ${{ secrets.LAI_PASS }}
|
||||
LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
|
||||
LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }}
|
||||
LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }}
|
||||
LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }}
|
||||
CLOUD: "1"
|
||||
VIDEO_LOCATION: ./artifacts/videos
|
||||
PR_NUMBER: ${{ steps.PR.outputs.ID }}
|
||||
TEST_APP_NAME: ${{ matrix.app_name }}
|
||||
HAR_LOCATION: ./artifacts/hars
|
||||
SLOW_MO: 50
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p ${VIDEO_LOCATION}
|
||||
HEADLESS=1 PACKAGE_LIGHTNING=1 python -m pytest tests/tests_app_examples/test_${{ matrix.app_name }}.py::test_${{ matrix.app_name }}_example_cloud --timeout=900 --capture=no -v --color=yes
|
||||
# Delete the artifacts if successful
|
||||
rm -r ${VIDEO_LOCATION}/${{ matrix.app_name }}
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: test-artifacts
|
||||
path: ./artifacts/videos
|
||||
|
||||
- name: Clean Previous Apps
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
LAI_USER: ${{ secrets.LAI_USER }}
|
||||
LAI_PASS: ${{ secrets.LAI_PASS }}
|
||||
LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
|
||||
LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }}
|
||||
LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }}
|
||||
LIGHTNING_CLOUD_URL: ${{ secrets.LIGHTNING_CLOUD_URL }}
|
||||
PR_NUMBER: ${{ steps.PR.outputs.ID }}
|
||||
TEST_APP_NAME: ${{ matrix.app_name }}
|
||||
GRID_USER_ID: ${{ secrets.LIGHTNING_USER_ID }}
|
||||
GRID_USER_KEY: ${{ secrets.LIGHTNING_API_KEY }}
|
||||
GRID_URL: ${{ secrets.LIGHTNING_CLOUD_URL }}
|
||||
_GRID_USERNAME: ${{ secrets.LIGHTNING_USERNAME }}
|
||||
shell: bash
|
||||
run: |
|
||||
time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
|
||||
Loading…
Reference in New Issue