From 4e13e419ea80889f41cf00319fd855230cdad165 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sat, 27 Jun 2020 15:13:29 +0200 Subject: [PATCH] add CLI test for examples (#2285) * cli examples * ddp * CI * CI * req * tests * skip DDP Co-authored-by: William Falcon --- .drone.yml | 4 +- pl_examples/basic_examples/cpu_template.py | 17 +++--- pl_examples/basic_examples/gpu_template.py | 52 +++++-------------- .../basic_examples/multi_node_ddp2_demo.py | 34 ++++++------ .../basic_examples/multi_node_ddp_demo.py | 34 ++++++------ pl_examples/test_examples.py | 47 +++++++++++++++++ 6 files changed, 108 insertions(+), 80 deletions(-) create mode 100644 pl_examples/test_examples.py diff --git a/.drone.yml b/.drone.yml index 2ae15a0d33..04dc9857dc 100644 --- a/.drone.yml +++ b/.drone.yml @@ -36,9 +36,11 @@ steps: - pip install -r ./requirements/base.txt --user -q - pip install -r ./requirements/devel.txt --user -q #- pip install -r ./requirements/docs.txt --user -q + - pip install -r ./requirements/examples.txt --user -q - pip list - python -c "import torch ; print(' & '.join([torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]) if torch.cuda.is_available() else 'only CPU')" - - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests benchmarks -v --durations=25 # --flake8 + - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --durations=25 # --flake8 + - python -m py.test benchmarks pl_examples -v --maxfail=2 --durations=0 # --flake8 #- cd docs; make doctest; make coverage - coverage report - codecov --token $CODECOV_TOKEN # --pr $DRONE_PULL_REQUEST --build $DRONE_BUILD_NUMBER --branch $DRONE_BRANCH --commit $DRONE_COMMIT --tag $DRONE_TAG diff --git a/pl_examples/basic_examples/cpu_template.py b/pl_examples/basic_examples/cpu_template.py index 6aa2e820bc..73a75c345a 100644 --- a/pl_examples/basic_examples/cpu_template.py +++ b/pl_examples/basic_examples/cpu_template.py @@ -4,13 +4,10 @@ Runs a model on the CPU on a single node. import os from argparse import ArgumentParser -import numpy as np -import torch - -import pytorch_lightning as pl +from pytorch_lightning import Trainer, seed_everything from pl_examples.models.lightning_template import LightningTemplateModel -pl.seed_everything(234) +seed_everything(234) def main(args): @@ -26,7 +23,7 @@ def main(args): # ------------------------ # 2 INIT TRAINER # ------------------------ - trainer = pl.Trainer.from_argparse_args(args) + trainer = Trainer.from_argparse_args(args) # ------------------------ # 3 START TRAINING @@ -34,7 +31,7 @@ def main(args): trainer.fit(model) -if __name__ == '__main__': +def run_cli(): # ------------------------ # TRAINING ARGUMENTS # ------------------------ @@ -44,10 +41,14 @@ if __name__ == '__main__': # each LightningModule defines arguments relevant to it parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir) - parser = pl.Trainer.add_argparse_args(parser) + parser = Trainer.add_argparse_args(parser) args = parser.parse_args() # --------------------- # RUN TRAINING # --------------------- main(args) + + +if __name__ == '__main__': + run_cli() diff --git a/pl_examples/basic_examples/gpu_template.py b/pl_examples/basic_examples/gpu_template.py index 2edf89f935..2a811f83f3 100644 --- a/pl_examples/basic_examples/gpu_template.py +++ b/pl_examples/basic_examples/gpu_template.py @@ -4,18 +4,13 @@ Runs a model on a single node across multiple gpus. import os from argparse import ArgumentParser -import numpy as np -import torch - -import pytorch_lightning as pl +from pytorch_lightning import Trainer, seed_everything from pl_examples.models.lightning_template import LightningTemplateModel -SEED = 2334 -torch.manual_seed(SEED) -np.random.seed(SEED) +seed_everything(234) -def main(hparams): +def main(args): """ Main training routine specific for this project :param hparams: @@ -23,17 +18,12 @@ def main(hparams): # ------------------------ # 1 INIT LIGHTNING MODEL # ------------------------ - model = LightningTemplateModel(**vars(hparams)) + model = LightningTemplateModel(**vars(args)) # ------------------------ # 2 INIT TRAINER # ------------------------ - trainer = pl.Trainer( - max_epochs=hparams.epochs, - gpus=hparams.gpus, - distributed_backend=hparams.distributed_backend, - precision=16 if hparams.use_16bit else 32, - ) + trainer = Trainer.from_argparse_args(args) # ------------------------ # 3 START TRAINING @@ -41,7 +31,7 @@ def main(hparams): trainer.fit(model) -if __name__ == '__main__': +def run_cli(): # ------------------------ # TRAINING ARGUMENTS # ------------------------ @@ -50,32 +40,16 @@ if __name__ == '__main__': root_dir = os.path.dirname(os.path.realpath(__file__)) parent_parser = ArgumentParser(add_help=False) - # gpu args - parent_parser.add_argument( - '--gpus', - type=int, - default=2, - help='how many gpus' - ) - parent_parser.add_argument( - '--distributed_backend', - type=str, - default='dp', - help='supports four options dp, ddp, ddp2, ddp_spawn, ...', - choices=['dp', 'ddp', 'ddp2', 'ddp_spawn', 'ddp_cpu'], - ) - parent_parser.add_argument( - '--use_16bit', - dest='use_16bit', - action='store_true', - help='if true uses 16 bit precision' - ) - # each LightningModule defines arguments relevant to it parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir) - hyperparams = parser.parse_args() + parser = Trainer.add_argparse_args(parser) + args = parser.parse_args() # --------------------- # RUN TRAINING # --------------------- - main(hyperparams) + main(args) + + +if __name__ == '__main__': + run_cli() diff --git a/pl_examples/basic_examples/multi_node_ddp2_demo.py b/pl_examples/basic_examples/multi_node_ddp2_demo.py index ca9c986a17..1dc73613fb 100644 --- a/pl_examples/basic_examples/multi_node_ddp2_demo.py +++ b/pl_examples/basic_examples/multi_node_ddp2_demo.py @@ -4,31 +4,28 @@ Multi-node example (GPU) import os from argparse import ArgumentParser -import numpy as np -import torch - -import pytorch_lightning as pl +from pytorch_lightning import Trainer, seed_everything from pl_examples.models.lightning_template import LightningTemplateModel -SEED = 2334 -torch.manual_seed(SEED) -np.random.seed(SEED) +seed_everything(234) -def main(hparams): +def main(args): """Main training routine specific for this project.""" # ------------------------ # 1 INIT LIGHTNING MODEL # ------------------------ - model = LightningTemplateModel(hparams) + model = LightningTemplateModel(args) # ------------------------ # 2 INIT TRAINER # ------------------------ - trainer = pl.Trainer( - gpus=2, - num_nodes=2, - distributed_backend='ddp2' + trainer = Trainer( + gpus=args.gpus, + num_nodes=args.num_nodes, + distributed_backend='ddp2', + max_epochs=args.max_epochs, + max_steps=args.max_steps, ) # ------------------------ @@ -37,15 +34,20 @@ def main(hparams): trainer.fit(model) -if __name__ == '__main__': +def run_cli(): root_dir = os.path.dirname(os.path.realpath(__file__)) parent_parser = ArgumentParser(add_help=False) # each LightningModule defines arguments relevant to it parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir) - hyperparams = parser.parse_args() + parser = Trainer.add_argparse_args(parser) + args = parser.parse_args() # --------------------- # RUN TRAINING # --------------------- - main(hyperparams) + main(args) + + +if __name__ == '__main__': + run_cli() diff --git a/pl_examples/basic_examples/multi_node_ddp_demo.py b/pl_examples/basic_examples/multi_node_ddp_demo.py index 518a9f39cc..84f241391e 100644 --- a/pl_examples/basic_examples/multi_node_ddp_demo.py +++ b/pl_examples/basic_examples/multi_node_ddp_demo.py @@ -4,31 +4,28 @@ Multi-node example (GPU) import os from argparse import ArgumentParser -import numpy as np -import torch - -import pytorch_lightning as pl from pl_examples.models.lightning_template import LightningTemplateModel +from pytorch_lightning import Trainer, seed_everything -SEED = 2334 -torch.manual_seed(SEED) -np.random.seed(SEED) +seed_everything(234) -def main(hparams): +def main(args): """Main training routine specific for this project.""" # ------------------------ # 1 INIT LIGHTNING MODEL # ------------------------ - model = LightningTemplateModel(hparams) + model = LightningTemplateModel(args) # ------------------------ # 2 INIT TRAINER # ------------------------ - trainer = pl.Trainer( - gpus=2, - num_nodes=2, - distributed_backend='ddp' + trainer = Trainer( + gpus=args.gpus, + num_nodes=args.num_nodes, + distributed_backend='ddp', + max_epochs=args.max_epochs, + max_steps=args.max_steps, ) # ------------------------ @@ -37,15 +34,20 @@ def main(hparams): trainer.fit(model) -if __name__ == '__main__': +def run_cli(): root_dir = os.path.dirname(os.path.realpath(__file__)) parent_parser = ArgumentParser(add_help=False) # each LightningModule defines arguments relevant to it parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir) - hyperparams = parser.parse_args() + parser = Trainer.add_argparse_args(parser) + args = parser.parse_args() # --------------------- # RUN TRAINING # --------------------- - main(hyperparams) + main(args) + + +if __name__ == '__main__': + run_cli() diff --git a/pl_examples/test_examples.py b/pl_examples/test_examples.py new file mode 100644 index 0000000000..cfc2fc8391 --- /dev/null +++ b/pl_examples/test_examples.py @@ -0,0 +1,47 @@ +from unittest import mock + +import pytest +import torch + + +@pytest.mark.parametrize('cli_args', ['--max_epochs 1 --max_steps 3']) +def test_cpu_template(cli_args): + """Test running CLI for an example with default params.""" + from pl_examples.basic_examples.cpu_template import run_cli + + cli_args = cli_args.split(' ') if cli_args else [] + with mock.patch("argparse._sys.argv", ["any.py"] + cli_args): + run_cli() + + +@pytest.mark.parametrize('cli_args', ['--max_epochs 1 --max_steps 3']) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine") +def test_gpu_template(cli_args): + """Test running CLI for an example with default params.""" + from pl_examples.basic_examples.gpu_template import run_cli + + cli_args = cli_args.split(' ') if cli_args else [] + with mock.patch("argparse._sys.argv", ["any.py"] + cli_args): + run_cli() + + +# @pytest.mark.parametrize('cli_args', ['--max_epochs 1 --max_steps 3 --num_nodes 1 --gpus 2']) +# @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") +# def test_multi_node_ddp(cli_args): +# """Test running CLI for an example with default params.""" +# from pl_examples.basic_examples.multi_node_ddp_demo import run_cli +# +# cli_args = cli_args.split(' ') if cli_args else [] +# with mock.patch("argparse._sys.argv", ["any.py"] + cli_args): +# run_cli() + + +# @pytest.mark.parametrize('cli_args', ['--max_epochs 1 --max_steps 3 --num_nodes 1 --gpus 2']) +# @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") +# def test_multi_node_ddp2(cli_args): +# """Test running CLI for an example with default params.""" +# from pl_examples.basic_examples.multi_node_ddp2_demo import run_cli +# +# cli_args = cli_args.split(' ') if cli_args else [] +# with mock.patch("argparse._sys.argv", ["any.py"] + cli_args): +# run_cli()