Run on_validation_end only on main process in DDP (#1125)

Co-authored-by: xingzhaolee <xingzhaolee@users.noreply.github.com>
2020-03-12 22:50:00 +08:00 · 2020-03-12 22:50:00 +08:00 · b4d4e489bf
parent b29613b758
commit b4d4e489bf
1 changed files with 4 additions and 0 deletions
--- a/pytorch_lightning/callbacks/model_checkpoint.py
+++ b/pytorch_lightning/callbacks/model_checkpoint.py
@ -175,6 +175,10 @@ class ModelCheckpoint(Callback):
        return filepath

    def on_validation_end(self, trainer, pl_module):
+        # only run on main process
+        if trainer.proc_rank != 0:
+            return
+
        metrics = trainer.callback_metrics
        epoch = trainer.current_epoch
        self.epochs_since_last_check += 1