Merge the slow and regular test workflows (#15331)
* extend matrix * drop * group-check * groups * cat * typo * cat2type * cat2type * env vars * '' * Rename to slow. Fix timeout * Examples are GPU only * str * Extra step * '' Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com> Co-authored-by: Akihiro Nitta <nitta@akihironitta.com> Co-authored-by: Carlos Mocholí <carlossmocholi@gmail.com>
This commit is contained in:
Jirka Borovec
GitHub
parent
d5003b1c07
commit
175603ca3f
|
|
@ -51,9 +51,9 @@ subprojects:
|
|||
- "pytorch-lightning (GPUs)"
|
||||
- "pytorch-lightning (HPUs)"
|
||||
- "pytorch-lightning (IPUs)"
|
||||
- "pl-slow (macOS-11, 3.7, 1.11)"
|
||||
- "pl-slow (ubuntu-20.04, 3.7, 1.11)"
|
||||
- "pl-slow (windows-2022, 3.7, 1.11)"
|
||||
- "pl-cpu (slow, macOS-11, pytorch, 3.7, 1.11)"
|
||||
- "pl-cpu (slow, ubuntu-20.04, pytorch, 3.7, 1.11)"
|
||||
- "pl-cpu (slow, windows-2022, pytorch, 3.7, 1.11)"
|
||||
# TODO: since this job has intermittent availability, it cannot be required or it will block all PL PRs from forks
|
||||
#- "test-on-tpus"
|
||||
|
||||
|
|
@ -88,9 +88,9 @@ subprojects:
|
|||
paths:
|
||||
- ".github/workflows/ci-pytorch-tests-slow.yml"
|
||||
checks:
|
||||
- "pl-slow (macOS-11, 3.7, 1.11)"
|
||||
- "pl-slow (ubuntu-20.04, 3.7, 1.11)"
|
||||
- "pl-slow (windows-2022, 3.7, 1.11)"
|
||||
- "pl-cpu (slow, macOS-11, pytorch, 3.7, 1.11)"
|
||||
- "pl-cpu (slow, ubuntu-20.04, pytorch, 3.7, 1.11)"
|
||||
- "pl-cpu (slow, windows-2022, pytorch, 3.7, 1.11)"
|
||||
|
||||
- id: "pytorch_lightning: Azure GPU"
|
||||
paths:
|
||||
|
|
@ -205,9 +205,9 @@ subprojects:
|
|||
- "pytorch-lightning (GPUs)"
|
||||
- "pytorch-lightning (HPUs)"
|
||||
- "pytorch-lightning (IPUs)"
|
||||
- "pl-slow (macOS-11, 3.7, 1.11)"
|
||||
- "pl-slow (ubuntu-20.04, 3.7, 1.11)"
|
||||
- "pl-slow (windows-2022, 3.7, 1.11)"
|
||||
- "pl-cpu (slow, macOS-11, pytorch, 3.7, 1.11)"
|
||||
- "pl-cpu (slow, ubuntu-20.04, pytorch, 3.7, 1.11)"
|
||||
- "pl-cpu (slow, windows-2022, pytorch, 3.7, 1.11)"
|
||||
#- "test-on-tpus"
|
||||
|
||||
- id: "lightning_lite: Tests"
|
||||
|
|
|
|||
|
|
@ -1,99 +0,0 @@
|
|||
name: Test PyTorch - slow
|
||||
|
||||
# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
|
||||
on:
|
||||
push:
|
||||
branches: [master, "release/*"]
|
||||
pull_request:
|
||||
branches: [master, "release/*"]
|
||||
types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
|
||||
paths:
|
||||
- ".github/workflows/ci-pytorch-tests-slow.yml"
|
||||
- "requirements/pytorch/*"
|
||||
- "src/pytorch_lightning/**"
|
||||
- "tests/tests_pytorch/**"
|
||||
- "setup.cfg" # includes pytest config
|
||||
- "requirements/lite/*"
|
||||
- "src/lightning_lite/**"
|
||||
- ".actions/**"
|
||||
- "!requirements/*/docs.txt"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
|
||||
cancel-in-progress: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}
|
||||
|
||||
jobs:
|
||||
pl-slow:
|
||||
runs-on: ${{ matrix.os }}
|
||||
if: github.event.pull_request.draft == false
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-20.04, windows-2022, macOS-11]
|
||||
# TODO: package parametrization
|
||||
# same config as '.azure-pipelines/gpu-tests-pytorch.yml'
|
||||
python-version: ["3.7"]
|
||||
pytorch-version: ["1.11"]
|
||||
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Get pip cache
|
||||
id: pip-cache
|
||||
run: |
|
||||
python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
|
||||
|
||||
- name: Cache pip
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('requirements/pytorch/base.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-py${{ matrix.python-version }}-
|
||||
|
||||
- name: Install package & dependencies
|
||||
env:
|
||||
PACKAGE_NAME: "pytorch"
|
||||
FREEZE_REQUIREMENTS: 1
|
||||
run: |
|
||||
# adjust versions according installed Torch version
|
||||
python ./requirements/pytorch/adjust-versions.py requirements.txt ${{ matrix.pytorch-version }}
|
||||
pip install -e .[test] --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
|
||||
pip list
|
||||
shell: bash
|
||||
|
||||
- name: Testing PyTorch
|
||||
working-directory: tests/tests_pytorch
|
||||
run: coverage run --source pytorch_lightning -m pytest -v --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}.xml
|
||||
env:
|
||||
PL_RUN_SLOW_TESTS: 1
|
||||
|
||||
- name: Upload pytest test results
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }}
|
||||
path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}.xml
|
||||
|
||||
- name: Statistics
|
||||
if: success()
|
||||
working-directory: tests/tests_pytorch
|
||||
run: |
|
||||
coverage report
|
||||
coverage xml
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
# see: https://github.com/actions/toolkit/issues/399
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
file: tests/tests_pytorch/coverage.xml
|
||||
flags: cpu,pytest-slow,torch${{ matrix.pytorch-version }}
|
||||
name: CPU-coverage
|
||||
fail_ci_if_error: false
|
||||
|
|
@ -67,6 +67,10 @@ jobs:
|
|||
- {os: "macOS-11", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.13", release: "pre"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.13", release: "pre"}
|
||||
- {os: "windows-2022", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.13", release: "pre"}
|
||||
# run test under SLOW label
|
||||
- {type: "slow", os: "macOS-11", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.11"}
|
||||
- {type: "slow", os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.11"}
|
||||
- {type: "slow", os: "windows-2022", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.11"}
|
||||
|
||||
timeout-minutes: 60
|
||||
|
||||
|
|
@ -99,6 +103,7 @@ jobs:
|
|||
python .actions/assistant.py replace_oldest_ver
|
||||
|
||||
- name: Pull legacy checkpoints
|
||||
if: ${{ matrix.type != 'slow' }}
|
||||
run: bash .actions/pull_legacy_checkpoints.sh
|
||||
|
||||
- name: Adjust PyTorch versions in requirements files
|
||||
|
|
@ -121,8 +126,16 @@ jobs:
|
|||
restore-keys: |
|
||||
${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.release }}-${{ matrix.requires }}-
|
||||
|
||||
- name: Switch PyTorch URL
|
||||
run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
|
||||
- name: Env. variables
|
||||
run: |
|
||||
# Switch PyTorch URL
|
||||
python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
|
||||
# Switch coverage scope
|
||||
python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'pytorch_lightning'))" >> $GITHUB_ENV
|
||||
# Whether to run slow tests
|
||||
python -c "print('PL_RUN_SLOW_TESTS=' + str(int('${{ matrix.type }}' == 'slow')))" >> $GITHUB_ENV
|
||||
# Decide timeout
|
||||
python -c "print('TEST_TIMEOUT=' + str(120 if '${{ matrix.type }}' == 'slow' else 60))" >> $GITHUB_ENV
|
||||
|
||||
- name: Install package & dependencies
|
||||
env:
|
||||
|
|
@ -170,13 +183,10 @@ jobs:
|
|||
# needs to run outside of `pytest`
|
||||
run: python utilities/test_warnings.py
|
||||
|
||||
- name: Switch coverage scope
|
||||
run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'pytorch_lightning'))" >> $GITHUB_ENV
|
||||
|
||||
- name: Testing PyTorch
|
||||
working-directory: tests/tests_pytorch
|
||||
# NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
|
||||
run: coverage run --source ${COVERAGE_SCOPE} -m pytest -v --timeout=120 --durations=50 --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
|
||||
run: coverage run --source ${COVERAGE_SCOPE} -m pytest -v --timeout=${TEST_TIMEOUT} --durations=50 --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
|
||||
|
||||
- name: Upload pytest results
|
||||
if: failure()
|
||||
|
|
@ -185,14 +195,6 @@ jobs:
|
|||
name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}
|
||||
path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
|
||||
|
||||
- name: Adjust examples
|
||||
if: ${{ matrix.pkg-name == 'lightning' }}
|
||||
run: python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="pytorch_lightning,lightning_lite" --target_import="lightning.pytorch,lightning.lite"
|
||||
|
||||
- name: Run Examples
|
||||
working-directory: ./examples
|
||||
run: python -m pytest test_pl_examples.py -v --durations=10
|
||||
|
||||
- name: Statistics
|
||||
if: success()
|
||||
working-directory: tests/tests_pytorch
|
||||
|
|
|
|||
Loading…
Reference in New Issue