From 495ffbd028ae860528c719544cf0409b41d5ef5a Mon Sep 17 00:00:00 2001 From: areshytko Date: Tue, 7 Apr 2020 13:39:54 +0300 Subject: [PATCH] Tensorboard logger check if lightning_logs directory exists (#1377) * tensorboard logger version if root_dir not exist * update changelog * resolve comments Co-authored-by: Alexander Reshytko Co-authored-by: William Falcon --- CHANGELOG.md | 1 + pytorch_lightning/loggers/tensorboard.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ae8550fdf..f198a1870b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Fixed running `on_validation_end` only on main process in DDP ([#1125](https://github.com/PyTorchLightning/pytorch-lightning/pull/1125)) - Fixes `use_amp` issue ([#1145](https://github.com/PyTorchLightning/pytorch-lightning/pull/1145)) - Fixes using deprecated `use_amp` attribute ([#1145](https://github.com/PyTorchLightning/pytorch-lightning/pull/1145)) +- Fixed Tensorboard logger error: lightning_logs directory not exists in multi-node DDP on nodes with rank != 0 ([#1375](https://github.com/PyTorchLightning/pytorch-lightning/issues/1375)). - Fixed `Unimplemented backend XLA` error on TPU ([#1387](https://github.com/PyTorchLightning/pytorch-lightning/pull/1387)) ## [0.7.1] - 2020-03-07 diff --git a/pytorch_lightning/loggers/tensorboard.py b/pytorch_lightning/loggers/tensorboard.py index 2ef6a55bca..7ae3ec85ed 100644 --- a/pytorch_lightning/loggers/tensorboard.py +++ b/pytorch_lightning/loggers/tensorboard.py @@ -9,6 +9,7 @@ from pkg_resources import parse_version from torch.utils.tensorboard import SummaryWriter from pytorch_lightning.loggers.base import LightningLoggerBase, rank_zero_only +from pytorch_lightning import _logger as log class TensorBoardLogger(LightningLoggerBase): @@ -163,6 +164,11 @@ class TensorBoardLogger(LightningLoggerBase): def _get_next_version(self): root_dir = os.path.join(self.save_dir, self.name) + + if not os.path.isdir(root_dir): + log.warning('Missing logger folder: %s', root_dir) + return 0 + existing_versions = [] for d in os.listdir(root_dir): if os.path.isdir(os.path.join(root_dir, d)) and d.startswith("version_"):