From 175603ca3f68481500056036f232bbab0e3fffa8 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Tue, 8 Nov 2022 16:00:08 +0100 Subject: [PATCH] Merge the slow and regular test workflows (#15331) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * extend matrix * drop * group-check * groups * cat * typo * cat2type * cat2type * env vars * '' * Rename to slow. Fix timeout * Examples are GPU only * str * Extra step * '' Co-authored-by: Jirka Borovec Co-authored-by: Akihiro Nitta Co-authored-by: Carlos Mocholí --- .github/checkgroup.yml | 18 ++-- .github/workflows/ci-pytorch-tests-slow.yml | 99 --------------------- .github/workflows/ci-pytorch-tests.yml | 30 ++++--- 3 files changed, 25 insertions(+), 122 deletions(-) delete mode 100644 .github/workflows/ci-pytorch-tests-slow.yml diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 4634f55ca2..b3b0ac8e8a 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -51,9 +51,9 @@ subprojects: - "pytorch-lightning (GPUs)" - "pytorch-lightning (HPUs)" - "pytorch-lightning (IPUs)" - - "pl-slow (macOS-11, 3.7, 1.11)" - - "pl-slow (ubuntu-20.04, 3.7, 1.11)" - - "pl-slow (windows-2022, 3.7, 1.11)" + - "pl-cpu (slow, macOS-11, pytorch, 3.7, 1.11)" + - "pl-cpu (slow, ubuntu-20.04, pytorch, 3.7, 1.11)" + - "pl-cpu (slow, windows-2022, pytorch, 3.7, 1.11)" # TODO: since this job has intermittent availability, it cannot be required or it will block all PL PRs from forks #- "test-on-tpus" @@ -88,9 +88,9 @@ subprojects: paths: - ".github/workflows/ci-pytorch-tests-slow.yml" checks: - - "pl-slow (macOS-11, 3.7, 1.11)" - - "pl-slow (ubuntu-20.04, 3.7, 1.11)" - - "pl-slow (windows-2022, 3.7, 1.11)" + - "pl-cpu (slow, macOS-11, pytorch, 3.7, 1.11)" + - "pl-cpu (slow, ubuntu-20.04, pytorch, 3.7, 1.11)" + - "pl-cpu (slow, windows-2022, pytorch, 3.7, 1.11)" - id: "pytorch_lightning: Azure GPU" paths: @@ -205,9 +205,9 @@ subprojects: - "pytorch-lightning (GPUs)" - "pytorch-lightning (HPUs)" - "pytorch-lightning (IPUs)" - - "pl-slow (macOS-11, 3.7, 1.11)" - - "pl-slow (ubuntu-20.04, 3.7, 1.11)" - - "pl-slow (windows-2022, 3.7, 1.11)" + - "pl-cpu (slow, macOS-11, pytorch, 3.7, 1.11)" + - "pl-cpu (slow, ubuntu-20.04, pytorch, 3.7, 1.11)" + - "pl-cpu (slow, windows-2022, pytorch, 3.7, 1.11)" #- "test-on-tpus" - id: "lightning_lite: Tests" diff --git a/.github/workflows/ci-pytorch-tests-slow.yml b/.github/workflows/ci-pytorch-tests-slow.yml deleted file mode 100644 index b6f2bd1884..0000000000 --- a/.github/workflows/ci-pytorch-tests-slow.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: Test PyTorch - slow - -# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: - push: - branches: [master, "release/*"] - pull_request: - branches: [master, "release/*"] - types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped - paths: - - ".github/workflows/ci-pytorch-tests-slow.yml" - - "requirements/pytorch/*" - - "src/pytorch_lightning/**" - - "tests/tests_pytorch/**" - - "setup.cfg" # includes pytest config - - "requirements/lite/*" - - "src/lightning_lite/**" - - ".actions/**" - - "!requirements/*/docs.txt" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} - -jobs: - pl-slow: - runs-on: ${{ matrix.os }} - if: github.event.pull_request.draft == false - strategy: - fail-fast: false - matrix: - os: [ubuntu-20.04, windows-2022, macOS-11] - # TODO: package parametrization - # same config as '.azure-pipelines/gpu-tests-pytorch.yml' - python-version: ["3.7"] - pytorch-version: ["1.11"] - - timeout-minutes: 20 - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Get pip cache - id: pip-cache - run: | - python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" - - - name: Cache pip - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('requirements/pytorch/base.txt') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}- - - - name: Install package & dependencies - env: - PACKAGE_NAME: "pytorch" - FREEZE_REQUIREMENTS: 1 - run: | - # adjust versions according installed Torch version - python ./requirements/pytorch/adjust-versions.py requirements.txt ${{ matrix.pytorch-version }} - pip install -e .[test] --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade - pip list - shell: bash - - - name: Testing PyTorch - working-directory: tests/tests_pytorch - run: coverage run --source pytorch_lightning -m pytest -v --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}.xml - env: - PL_RUN_SLOW_TESTS: 1 - - - name: Upload pytest test results - if: failure() - uses: actions/upload-artifact@v3 - with: - name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }} - path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}.xml - - - name: Statistics - if: success() - working-directory: tests/tests_pytorch - run: | - coverage report - coverage xml - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - # see: https://github.com/actions/toolkit/issues/399 - continue-on-error: true - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: tests/tests_pytorch/coverage.xml - flags: cpu,pytest-slow,torch${{ matrix.pytorch-version }} - name: CPU-coverage - fail_ci_if_error: false diff --git a/.github/workflows/ci-pytorch-tests.yml b/.github/workflows/ci-pytorch-tests.yml index 7ec6a89686..4da920c5e8 100644 --- a/.github/workflows/ci-pytorch-tests.yml +++ b/.github/workflows/ci-pytorch-tests.yml @@ -67,6 +67,10 @@ jobs: - {os: "macOS-11", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.13", release: "pre"} - {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.13", release: "pre"} - {os: "windows-2022", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.13", release: "pre"} + # run test under SLOW label + - {type: "slow", os: "macOS-11", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.11"} + - {type: "slow", os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.11"} + - {type: "slow", os: "windows-2022", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.11"} timeout-minutes: 60 @@ -99,6 +103,7 @@ jobs: python .actions/assistant.py replace_oldest_ver - name: Pull legacy checkpoints + if: ${{ matrix.type != 'slow' }} run: bash .actions/pull_legacy_checkpoints.sh - name: Adjust PyTorch versions in requirements files @@ -121,8 +126,16 @@ jobs: restore-keys: | ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.release }}-${{ matrix.requires }}- - - name: Switch PyTorch URL - run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV + - name: Env. variables + run: | + # Switch PyTorch URL + python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV + # Switch coverage scope + python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'pytorch_lightning'))" >> $GITHUB_ENV + # Whether to run slow tests + python -c "print('PL_RUN_SLOW_TESTS=' + str(int('${{ matrix.type }}' == 'slow')))" >> $GITHUB_ENV + # Decide timeout + python -c "print('TEST_TIMEOUT=' + str(120 if '${{ matrix.type }}' == 'slow' else 60))" >> $GITHUB_ENV - name: Install package & dependencies env: @@ -170,13 +183,10 @@ jobs: # needs to run outside of `pytest` run: python utilities/test_warnings.py - - name: Switch coverage scope - run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'pytorch_lightning'))" >> $GITHUB_ENV - - name: Testing PyTorch working-directory: tests/tests_pytorch # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003 - run: coverage run --source ${COVERAGE_SCOPE} -m pytest -v --timeout=120 --durations=50 --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml + run: coverage run --source ${COVERAGE_SCOPE} -m pytest -v --timeout=${TEST_TIMEOUT} --durations=50 --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml - name: Upload pytest results if: failure() @@ -185,14 +195,6 @@ jobs: name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }} path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml - - name: Adjust examples - if: ${{ matrix.pkg-name == 'lightning' }} - run: python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="pytorch_lightning,lightning_lite" --target_import="lightning.pytorch,lightning.lite" - - - name: Run Examples - working-directory: ./examples - run: python -m pytest test_pl_examples.py -v --durations=10 - - name: Statistics if: success() working-directory: tests/tests_pytorch