From b14c4d4c70ee0b0fead51fc08f8ed323aea8bf10 Mon Sep 17 00:00:00 2001 From: Brendan Fahy Date: Sat, 3 Oct 2020 02:35:42 +0000 Subject: [PATCH] handle fsspec inconsistency in PyArrowHDFS (#3805) --- pytorch_lightning/loggers/tensorboard.py | 3 ++- pytorch_lightning/trainer/connectors/checkpoint_connector.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/loggers/tensorboard.py b/pytorch_lightning/loggers/tensorboard.py index b695476cf5..177e314ebf 100644 --- a/pytorch_lightning/loggers/tensorboard.py +++ b/pytorch_lightning/loggers/tensorboard.py @@ -232,7 +232,8 @@ class TensorBoardLogger(LightningLoggerBase): return 0 existing_versions = [] - for d in self._fs.ls(root_dir): + for listing in self._fs.listdir(root_dir): + d = listing["name"] bn = os.path.basename(d) if self._fs.isdir(d) and bn.startswith("version_"): dir_ver = bn.split("_")[1].replace('/', '') diff --git a/pytorch_lightning/trainer/connectors/checkpoint_connector.py b/pytorch_lightning/trainer/connectors/checkpoint_connector.py index 12ab5eeb23..caf20a888a 100644 --- a/pytorch_lightning/trainer/connectors/checkpoint_connector.py +++ b/pytorch_lightning/trainer/connectors/checkpoint_connector.py @@ -194,7 +194,7 @@ class CheckpointConnector: folderpath = str(self.trainer.weights_save_path) fs = get_filesystem(folderpath) if fs.exists(folderpath): - files = [os.path.basename(f) for f in fs.ls(folderpath)] + files = [os.path.basename(f['name']) for f in fs.listdir(folderpath)] hpc_weight_paths = [x for x in files if 'hpc_ckpt' in x] # if hpc weights exist restore model @@ -333,7 +333,7 @@ class CheckpointConnector: def max_ckpt_in_folder(self, path, name_key='ckpt_'): fs = get_filesystem(path) - files = [os.path.basename(f) for f in fs.ls(path)] + files = [os.path.basename(f["name"]) for f in fs.listdir(path)] files = [x for x in files if name_key in x] if len(files) == 0: return 0