From dbb4482ecc157553d80eee654c4e0a18c1b9c1bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Wed, 21 Sep 2022 16:28:35 +0200 Subject: [PATCH] Set path filters in favor of the required-job action (#14294) Co-authored-by: Jirka --- .azure/gpu-tests.yml | 77 ++++++++++++--------- .azure/ipu-tests.yml | 18 ++++- .github/workflows/ci-pytorch-test-conda.yml | 10 +-- .github/workflows/ci-pytorch-test-full.yml | 15 ++-- 4 files changed, 75 insertions(+), 45 deletions(-) diff --git a/.azure/gpu-tests.yml b/.azure/gpu-tests.yml index 67e4f3d0ba..1e589e708c 100644 --- a/.azure/gpu-tests.yml +++ b/.azure/gpu-tests.yml @@ -12,14 +12,50 @@ trigger: - "master" - "release/*" - "refs/tags/*" + paths: + include: + - ".azure/gpu-tests.yml" + - "examples/run_ddp_examples.sh" + - "examples/convert_from_pt_to_pl/**" + - "examples/run_pl_examples.sh" + - "examples/pl_basics/backbone_image_classifier.py" + - "examples/pl_basics/autoencoder.py" + - "examples/pl_loops/mnist_lite.py" + - "examples/pl_fault_tolerant/automatic.py" + - "examples/test_pl_examples.py" + - "examples/pl_integrations/dali_image_classifier.py" + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config + - "requirements/lite/**" + - "src/lightning_lite/**" + - "tests/tests_lite/**" pr: - - "master" - - "release/*" - -variables: - - name: continue - value: '1' + branches: + include: + - "master" + - "release/*" + paths: + include: + - ".azure/gpu-tests.yml" + - "examples/run_ddp_examples.sh" + - "examples/convert_from_pt_to_pl/**" + - "examples/run_pl_examples.sh" + - "examples/pl_basics/backbone_image_classifier.py" + - "examples/pl_basics/autoencoder.py" + - "examples/pl_loops/mnist_lite.py" + - "examples/pl_fault_tolerant/automatic.py" + - "examples/test_pl_examples.py" + - "examples/pl_integrations/dali_image_classifier.py" + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config + - "requirements/lite/**" + - "src/lightning_lite/**" + - "tests/tests_lite/**" jobs: - job: testing @@ -41,22 +77,6 @@ jobs: clean: all steps: - - - bash: | - CHANGED_FILES=$(git diff --name-status origin/master -- . | awk '{print $2}') - FILTER='.azure/gpu_*|src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_*' - echo $CHANGED_FILES > changed_files.txt - MATCHES=$(cat changed_files.txt | grep -E $FILTER) - echo $MATCHES - if [ -z "$MATCHES" ]; then - echo "Skip" - echo "##vso[task.setvariable variable=continue]0" - else - echo "Continue" - echo "##vso[task.setvariable variable=continue]1" - fi - displayName: Skipper - - bash: | lspci | egrep 'VGA|3D' whereis nvidia @@ -66,7 +86,6 @@ jobs: pip --version pip list displayName: 'Image info & NVIDIA' - condition: eq(variables['continue'], '1') - bash: | set -e @@ -86,7 +105,6 @@ jobs: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 displayName: 'Install dependencies' - condition: eq(variables['continue'], '1') - bash: | set -e @@ -95,16 +113,13 @@ jobs: python requirements/pytorch/check-avail-strategies.py python requirements/pytorch/check-avail-extras.py displayName: 'Env details' - condition: eq(variables['continue'], '1') - bash: bash .actions/pull_legacy_checkpoints.sh displayName: 'Get legacy checkpoints' - condition: eq(variables['continue'], '1') - bash: python -m coverage run --source pytorch_lightning -m pytest workingDirectory: src/pytorch_lightning displayName: 'Testing: PyTorch doctests' - condition: eq(variables['continue'], '1') - bash: python -m coverage run --source pytorch_lightning -m pytest --ignore benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50 env: @@ -112,7 +127,6 @@ jobs: workingDirectory: tests/tests_pytorch displayName: 'Testing: PyTorch standard' timeoutInMinutes: "35" - condition: eq(variables['continue'], '1') - bash: bash run_standalone_tests.sh workingDirectory: tests/tests_pytorch @@ -122,7 +136,6 @@ jobs: PL_STANDALONE_TESTS_SOURCE: "pytorch_lightning" displayName: 'Testing: PyTorch standalone tests' timeoutInMinutes: "35" - condition: eq(variables['continue'], '1') - bash: bash run_standalone_tasks.sh workingDirectory: tests/tests_pytorch @@ -131,7 +144,6 @@ jobs: PL_RUN_CUDA_TESTS: "1" displayName: 'Testing: PyTorch standalone tasks' timeoutInMinutes: "10" - condition: eq(variables['continue'], '1') - bash: | python -m coverage report @@ -141,14 +153,13 @@ jobs: ls -l workingDirectory: tests/tests_pytorch displayName: 'Statistics' - condition: eq(variables['continue'], '1') - task: PublishTestResults@2 displayName: 'Publish test results' inputs: testResultsFiles: '$(Build.StagingDirectory)/test-results.xml' testRunTitle: '$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)' - condition: and(succeededOrFailed(), eq(variables['continue'], '1')) + condition: succeededOrFailed() - script: | set -e @@ -160,11 +171,9 @@ jobs: env: PL_USE_MOCKED_MNIST: "1" displayName: 'Testing: PyTorch examples' - condition: eq(variables['continue'], '1') - bash: python -m pytest benchmarks -v --maxfail=2 --durations=0 workingDirectory: tests/tests_pytorch env: PL_RUN_CUDA_TESTS: "1" displayName: 'Testing: PyTorch benchmarks' - condition: eq(variables['continue'], '1') diff --git a/.azure/ipu-tests.yml b/.azure/ipu-tests.yml index a4d6831883..e6b6fcb246 100644 --- a/.azure/ipu-tests.yml +++ b/.azure/ipu-tests.yml @@ -7,10 +7,24 @@ trigger: - master - release/* - refs/tags/* + paths: + include: + - ".azure/ipu-tests.yml" + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" pr: - - master - - release/* + branches: + include: + - "master" + - "release/*" + paths: + include: + - ".azure/ipu-tests.yml" + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" variables: - name: poplar_sdk diff --git a/.github/workflows/ci-pytorch-test-conda.yml b/.github/workflows/ci-pytorch-test-conda.yml index c05772ed34..0f40d074dc 100644 --- a/.github/workflows/ci-pytorch-test-conda.yml +++ b/.github/workflows/ci-pytorch-test-conda.yml @@ -6,6 +6,12 @@ on: branches: [master, "release/*"] pull_request: branches: [master, "release/*"] + paths: + - ".github/workflows/ci-pytorch-test-conda.yml" + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} @@ -35,10 +41,6 @@ jobs: - uses: actions/checkout@v3 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v29.0.4 - - name: Update base dependencies env: PACKAGE_NAME: pytorch diff --git a/.github/workflows/ci-pytorch-test-full.yml b/.github/workflows/ci-pytorch-test-full.yml index d17e23dba1..f0989721a2 100644 --- a/.github/workflows/ci-pytorch-test-full.yml +++ b/.github/workflows/ci-pytorch-test-full.yml @@ -7,6 +7,15 @@ on: pull_request: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] # add `ready_for_review` since draft is skipped + paths: + - "requirements/pytorch/**" + - "src/pytorch_lightning/**" + - "tests/tests_pytorch/**" + - "setup.cfg" # includes pytest config + - ".github/workflows/ci-pytorch-test-full.yml" + - "requirements/lite/**" + - "src/lightning_lite/**" + - "tests/tests_lite/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} @@ -37,10 +46,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v29.0.4 - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -138,7 +143,7 @@ jobs: - name: Testing Warnings # the stacklevel can only be set on >=3.7 - if: ${{ matrix.python-version != '3.7' }} + if: matrix.python-version != '3.7' working-directory: tests/tests_pytorch # needs to run outside of `pytest` run: python utilities/test_warnings.py