From c438d0dd90eedfb6f91cd086ccb4948977cf4eb3 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 3 Jun 2020 14:28:19 +0200 Subject: [PATCH] increase acc (#2039) * increase acc * try 0.45 * @pytest * @pytest * try .50 * duration * pytest --- .circleci/config.yml | 4 ++-- .drone.yml | 2 +- .github/workflows/ci-testing.yml | 2 +- .run_local_tests.sh | 2 +- setup.cfg | 6 +++++- tests/README.md | 2 +- tests/base/utils.py | 4 ++-- tests/trainer/test_dataloaders.py | 15 +++++---------- tests/trainer/test_lr_finder.py | 3 +-- 9 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1cd6ac7a4d..7bd1de8f6c 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ references: name: Testing command: | python --version ; pip --version ; pip list - py.test pytorch_lightning tests -v --doctest-modules --junitxml=test-reports/pytest_junit.xml + py.test pytorch_lightning tests -v --junitxml=test-reports/pytest_junit.xml no_output_timeout: 15m examples: &examples @@ -30,7 +30,7 @@ references: command: | pip install -r ./pl_examples/requirements.txt --user python --version ; pip --version ; pip list - py.test pl_examples -v --doctest-modules --junitxml=test-reports/pytest_junit.xml + py.test pl_examples -v --junitxml=test-reports/pytest_junit.xml no_output_timeout: 20m install_pkg: &install_pkg diff --git a/.drone.yml b/.drone.yml index 88e2d76a52..49fda10c56 100644 --- a/.drone.yml +++ b/.drone.yml @@ -38,7 +38,7 @@ steps: #- pip install -r ./docs/requirements.txt --user -q - pip list - python -c "import torch ; print(' & '.join([torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]) if torch.cuda.is_available() else 'only CPU')" - - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests benchmarks -v --doctest-modules # --flake8 + - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests benchmarks -v # --flake8 #- cd docs; make doctest; make coverage - coverage report - codecov --token $CODECOV_TOKEN # --pr $DRONE_PULL_REQUEST --build $DRONE_BUILD_NUMBER --branch $DRONE_BRANCH --commit $DRONE_COMMIT --tag $DRONE_TAG diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 15447c5d5b..13e599dd21 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -112,7 +112,7 @@ jobs: run: | # tox --sitepackages # flake8 . - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --doctest-modules --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml + coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml coverage report - name: Upload pytest test results diff --git a/.run_local_tests.sh b/.run_local_tests.sh index 77003cc396..c0c030a78e 100644 --- a/.run_local_tests.sh +++ b/.run_local_tests.sh @@ -12,7 +12,7 @@ rm -rf ./tests/cometruns* rm -rf ./tests/wandb* rm -rf ./tests/tests/* rm -rf ./lightning_logs -python -m coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules --flake8 --durations=0 +python -m coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --flake8 python -m coverage report -m # specific file diff --git a/setup.cfg b/setup.cfg index aab7a580c7..827b820f7c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,12 +6,16 @@ norecursedirs = python_files = test_*.py # doctest_plus = disabled -addopts = --strict +addopts = + --strict + --doctest-modules + --durations=0 markers = slow remote_data filterwarnings gpus_param_tests +junit_duration_report = call [coverage:report] exclude_lines = diff --git a/tests/README.md b/tests/README.md index 51f1fde2c3..4f35a29603 100644 --- a/tests/README.md +++ b/tests/README.md @@ -40,7 +40,7 @@ Make sure to run coverage on a GPU machine with at least 2 GPUs and NVIDIA apex cd pytorch-lightning # generate coverage (coverage is also installed as part of dev dependencies under tests/requirements-devel.txt) -coverage run --source pytorch_lightning -m py.test pytorch_lightning tests examples -v --doctest-modules +coverage run --source pytorch_lightning -m py.test pytorch_lightning tests examples -v # print coverage stats coverage report -m diff --git a/tests/base/utils.py b/tests/base/utils.py index 28c32a707f..4cb7e4cd67 100644 --- a/tests/base/utils.py +++ b/tests/base/utils.py @@ -25,7 +25,7 @@ def assert_speed_parity(pl_times, pt_times, num_epochs): f"lightning was slower than PT (threshold {max_diff_per_epoch})" -def run_model_test_without_loggers(trainer_options, model, min_acc=0.30): +def run_model_test_without_loggers(trainer_options, model, min_acc=0.50): reset_seed() # fit model @@ -155,7 +155,7 @@ def load_model_from_checkpoint(root_weights_dir, module_class=EvalModelTemplate) return trained_model -def run_prediction(dataloader, trained_model, dp=False, min_acc=0.3): +def run_prediction(dataloader, trained_model, dp=False, min_acc=0.50): # run prediction on 1 batch for batch in dataloader: break diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py index 21783686e7..4dde52536e 100644 --- a/tests/trainer/test_dataloaders.py +++ b/tests/trainer/test_dataloaders.py @@ -248,9 +248,8 @@ def test_mixing_of_dataloader_options(tmpdir): f'`test_dataloaders` not initiated properly, got {trainer.test_dataloaders}' +@pytest.mark.skip('TODO: speed up this test') def test_train_inf_dataloader_error(tmpdir): - pytest.skip('TODO: fix speed of this test') - """Test inf train data loader (e.g. IterableDataset)""" model = EvalModelTemplate() model.train_dataloader = model.train_dataloader__infinite @@ -261,9 +260,8 @@ def test_train_inf_dataloader_error(tmpdir): trainer.fit(model) +@pytest.mark.skip('TODO: speed up this test') def test_val_inf_dataloader_error(tmpdir): - pytest.skip('TODO: fix speed of this test') - """Test inf train data loader (e.g. IterableDataset)""" model = EvalModelTemplate() model.val_dataloader = model.val_dataloader__infinite @@ -274,9 +272,8 @@ def test_val_inf_dataloader_error(tmpdir): trainer.fit(model) +@pytest.mark.skip('TODO: speed up this test') def test_test_inf_dataloader_error(tmpdir): - pytest.skip('TODO: fix speed of this test') - """Test inf train data loader (e.g. IterableDataset)""" model = EvalModelTemplate() model.test_dataloader = model.test_dataloader__infinite @@ -288,9 +285,8 @@ def test_test_inf_dataloader_error(tmpdir): @pytest.mark.parametrize('check_interval', [50, 1.0]) +@pytest.mark.skip('TODO: speed up this test') def test_inf_train_dataloader(tmpdir, check_interval): - pytest.skip('TODO: fix speed of this test') - """Test inf train data loader (e.g. IterableDataset)""" model = EvalModelTemplate() @@ -307,9 +303,8 @@ def test_inf_train_dataloader(tmpdir, check_interval): @pytest.mark.parametrize('check_interval', [1.0]) +@pytest.mark.skip('TODO: speed up this test') def test_inf_val_dataloader(tmpdir, check_interval): - pytest.skip('TODO: fix speed of this test') - """Test inf val data loader (e.g. IterableDataset)""" model = EvalModelTemplate() diff --git a/tests/trainer/test_lr_finder.py b/tests/trainer/test_lr_finder.py index 0760d6389d..e7733ae20f 100755 --- a/tests/trainer/test_lr_finder.py +++ b/tests/trainer/test_lr_finder.py @@ -134,9 +134,8 @@ def test_call_to_trainer_method(tmpdir): 'Learning rate was not altered after running learning rate finder' +@pytest.mark.skip('TODO: speed up this test') def test_accumulation_and_early_stopping(tmpdir): - pytest.skip('TODO: speed up this test') - """ Test that early stopping of learning rate finder works, and that accumulation also works for this feature """