From ba806c8ee0ff9a038471e151d703e38f357d234a Mon Sep 17 00:00:00 2001
From: Jirka Borovec <Borda@users.noreply.github.com>
Date: Mon, 15 Feb 2021 16:36:13 +0100
Subject: [PATCH] enable testing DDP examples (#4995)

* enable testing DDP examples

* args

* ddp_spawn

* ddp as extra script

* path

# Conflicts:
#	.drone.yml

* install

* -u

* q
---
 azure-pipelines.yml            |  6 ++++++
 pl_examples/run_ddp-example.sh | 12 ++++++++++++
 pl_examples/test_examples.py   | 32 ++------------------------------
 3 files changed, 20 insertions(+), 30 deletions(-)
 create mode 100644 pl_examples/run_ddp-example.sh

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 0bb4f5cfff..8a6f132452 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -88,3 +88,9 @@ jobs:
     - script: |
         python -m pytest benchmarks pl_examples -v --maxfail=2 --durations=0
       displayName: 'Testing: extended'
+
+    - script: |
+        python setup.py install --user --quiet
+        bash pl_examples/run_ddp-example.sh
+        pip uninstall -y pytorch-lightning
+      displayName: 'Examples'
diff --git a/pl_examples/run_ddp-example.sh b/pl_examples/run_ddp-example.sh
new file mode 100644
index 0000000000..f0c7695e76
--- /dev/null
+++ b/pl_examples/run_ddp-example.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+ARGS_DEFAULT=" --default_root_dir %(tmpdir)s --max_epochs 1 --batch_size 32 --limit_train_batches 2 --limit_val_batches 2"
+ARGS_EXTRA_DDP=" --gpus 2 --accelerator ddp"
+ARGS_EXTRA_AMP=" --precision 16"
+
+python pl_examples/basic_examples/simple_image_classifier.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP}
+python pl_examples/basic_examples/simple_image_classifier.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP} ${ARGS_EXTRA_AMP}
+python pl_examples/basic_examples/backbone_image_classifier.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP}
+python pl_examples/basic_examples/backbone_image_classifier.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP} ${ARGS_EXTRA_AMP}
+python pl_examples/basic_examples/autoencoder.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP}
+python pl_examples/basic_examples/autoencoder.py ${ARGS_DEFAULT} ${ARGS_EXTRA_DDP} ${ARGS_EXTRA_AMP}
diff --git a/pl_examples/test_examples.py b/pl_examples/test_examples.py
index 96fca5bd20..b930957a26 100644
--- a/pl_examples/test_examples.py
+++ b/pl_examples/test_examples.py
@@ -38,17 +38,7 @@ ARGS_DP = ARGS_DEFAULT + """
 --accelerator dp \
 """
 
-ARGS_DP_AMP = ARGS_DP + """
---precision 16 \
-"""
-
-ARGS_DDP = ARGS_DEFAULT + """
---gpus 2 \
---accelerator ddp \
---precision 16 \
-"""
-
-ARGS_DDP_AMP = ARGS_DEFAULT + """
+ARGS_AMP = """
 --precision 16 \
 """
 
@@ -61,7 +51,7 @@ ARGS_DDP_AMP = ARGS_DEFAULT + """
     ]
 )
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
-@pytest.mark.parametrize('cli_args', [ARGS_DP, ARGS_DP_AMP])
+@pytest.mark.parametrize('cli_args', [ARGS_DP, ARGS_DP + ARGS_AMP])
 def test_examples_dp(tmpdir, import_cli, cli_args):
 
     module = importlib.import_module(import_cli)
@@ -72,24 +62,6 @@ def test_examples_dp(tmpdir, import_cli, cli_args):
         module.cli_main()
 
 
-# ToDo: fix this failing example
-# @pytest.mark.parametrize('import_cli', [
-#     'pl_examples.basic_examples.simple_image_classifier',
-#     'pl_examples.basic_examples.backbone_image_classifier',
-#     'pl_examples.basic_examples.autoencoder',
-# ])
-# @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
-# @pytest.mark.parametrize('cli_args', [ARGS_DDP, ARGS_DDP_AMP])
-# def test_examples_ddp(tmpdir, import_cli, cli_args):
-#
-#     module = importlib.import_module(import_cli)
-#     # update the temp dir
-#     cli_args = cli_args % {'tmpdir': tmpdir}
-#
-#     with mock.patch("argparse._sys.argv", ["any.py"] + cli_args.strip().split()):
-#         module.cli_main()
-
-
 @pytest.mark.parametrize(
     'import_cli', [
         'pl_examples.basic_examples.simple_image_classifier',