From 5788789f0164b984fdcb9d6390b3babbf5997fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Fri, 19 Nov 2021 03:07:33 +0100 Subject: [PATCH] Move benchmarks into the test directory (#10614) --- .azure-pipelines/gpu-benchmark.yml | 2 +- .azure-pipelines/gpu-tests.yml | 4 ++-- benchmarks/__init__.py | 18 ------------------ pyproject.toml | 6 +----- tests/benchmarks/__init__.py | 0 .../benchmarks}/generate_comparison.py | 2 +- .../benchmarks}/test_basic_parity.py | 0 .../benchmarks}/test_sharded_parity.py | 0 tests/special_tests.sh | 2 +- 9 files changed, 6 insertions(+), 28 deletions(-) delete mode 100644 benchmarks/__init__.py create mode 100644 tests/benchmarks/__init__.py rename {benchmarks => tests/benchmarks}/generate_comparison.py (97%) rename {benchmarks => tests/benchmarks}/test_basic_parity.py (100%) rename {benchmarks => tests/benchmarks}/test_sharded_parity.py (100%) diff --git a/.azure-pipelines/gpu-benchmark.yml b/.azure-pipelines/gpu-benchmark.yml index f8b9593d72..6d45cc2f45 100644 --- a/.azure-pipelines/gpu-benchmark.yml +++ b/.azure-pipelines/gpu-benchmark.yml @@ -36,7 +36,7 @@ jobs: steps: - bash: | - python -m pytest benchmarks -v --durations=0 + python -m pytest tests/benchmarks -v --durations=0 displayName: 'Testing: benchmarks' env: PL_RUNNING_BENCHMARKS: 1 diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml index f1af36a609..71332a840f 100644 --- a/.azure-pipelines/gpu-tests.yml +++ b/.azure-pipelines/gpu-tests.yml @@ -68,7 +68,7 @@ jobs: displayName: 'Get legacy checkpoints' - bash: | - python -m coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50 + python -m coverage run --source pytorch_lightning -m pytest pytorch_lightning tests --ignore tests/benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50 displayName: 'Testing: standard' - bash: | @@ -113,5 +113,5 @@ jobs: displayName: 'Testing: examples' - bash: | - python -m pytest benchmarks -v --maxfail=2 --durations=0 + python -m pytest tests/benchmarks -v --maxfail=2 --durations=0 displayName: 'Testing: benchmarks' diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py deleted file mode 100644 index b4a3da40d4..0000000000 --- a/benchmarks/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright The PyTorch Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -_BENCHMARK_ROOT = os.path.dirname(__file__) -_PROJECT_ROOT = os.path.dirname(_BENCHMARK_ROOT) -_PATH_DATASETS = os.path.join(_PROJECT_ROOT, "Datasets") diff --git a/pyproject.toml b/pyproject.toml index 6546d96e3d..08b7b50eee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,6 @@ requires = [ [tool.isort] known_first_party = [ - "benchmarks", "docs", "pl_examples", "pytorch_lightning", @@ -24,7 +23,7 @@ line-length = 120 [tool.mypy] -files = ["pytorch_lightning", "pl_examples", "benchmarks"] +files = ["pytorch_lightning"] disallow_untyped_defs = "True" ignore_missing_imports = "True" show_error_codes = "True" @@ -53,9 +52,6 @@ module = [ "pytorch_lightning.distributed.*", "pytorch_lightning.tuner.*", "pytorch_lightning.utilities.*", - "pl_examples.*", - "benchmarks.*", - "tests.helpers.*" ] ignore_errors = "True" diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/benchmarks/generate_comparison.py b/tests/benchmarks/generate_comparison.py similarity index 97% rename from benchmarks/generate_comparison.py rename to tests/benchmarks/generate_comparison.py index 5a9cde0d80..bc95b5d9cf 100644 --- a/benchmarks/generate_comparison.py +++ b/tests/benchmarks/generate_comparison.py @@ -16,7 +16,7 @@ import os import matplotlib.pylab as plt import pandas as pd -from benchmarks.test_basic_parity import measure_loops +from tests.benchmarks.test_basic_parity import measure_loops from tests.helpers.advanced_models import ParityModuleMNIST, ParityModuleRNN NUM_EPOCHS = 20 diff --git a/benchmarks/test_basic_parity.py b/tests/benchmarks/test_basic_parity.py similarity index 100% rename from benchmarks/test_basic_parity.py rename to tests/benchmarks/test_basic_parity.py diff --git a/benchmarks/test_sharded_parity.py b/tests/benchmarks/test_sharded_parity.py similarity index 100% rename from benchmarks/test_sharded_parity.py rename to tests/benchmarks/test_sharded_parity.py diff --git a/tests/special_tests.sh b/tests/special_tests.sh index 4143ec5930..6a3701a7ee 100755 --- a/tests/special_tests.sh +++ b/tests/special_tests.sh @@ -20,7 +20,7 @@ export PL_RUNNING_SPECIAL_TESTS=1 defaults='-m coverage run --source pytorch_lightning --append -m pytest --capture=no' # find tests marked as `@RunIf(special=True)`. done manually instead of with pytest because it is faster -grep_output=$(grep --recursive --word-regexp 'tests' 'benchmarks' --regexp 'special=True' --include '*.py' --exclude 'tests/conftest.py') +grep_output=$(grep --recursive --word-regexp 'tests' --regexp 'special=True' --include '*.py' --exclude 'tests/conftest.py') # file paths, remove duplicates files=$(echo "$grep_output" | cut -f1 -d: | sort | uniq)