From b4d4e489bf413ebf3288d29c5905d2292ce18d58 Mon Sep 17 00:00:00 2001 From: Xing Zhao LEE <49442025+xingzhaolee@users.noreply.github.com> Date: Thu, 12 Mar 2020 22:50:00 +0800 Subject: [PATCH] Run on_validation_end only on main process in DDP (#1125) Co-authored-by: xingzhaolee --- pytorch_lightning/callbacks/model_checkpoint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py index 832a56654e..c3d096eb75 100644 --- a/pytorch_lightning/callbacks/model_checkpoint.py +++ b/pytorch_lightning/callbacks/model_checkpoint.py @@ -175,6 +175,10 @@ class ModelCheckpoint(Callback): return filepath def on_validation_end(self, trainer, pl_module): + # only run on main process + if trainer.proc_rank != 0: + return + metrics = trainer.callback_metrics epoch = trainer.current_epoch self.epochs_since_last_check += 1