Rooba
/
lightning
mirror of https://github.com/Lightning-AI/lightning.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

handle fsspec inconsistency in PyArrowHDFS (#3805)

This commit is contained in:
Brendan Fahy 2020-10-03 02:35:42 +00:00 committed by GitHub
parent 74484edecd
commit b14c4d4c70
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pytorch_lightning/loggers/tensorboard.py
View File

@ -232,7 +232,8 @@ class TensorBoardLogger(LightningLoggerBase):
return 0
existing_versions = []
for d in self._fs.ls(root_dir):
for listing in self._fs.listdir(root_dir):
d = listing["name"]
bn = os.path.basename(d)
if self._fs.isdir(d) and bn.startswith("version_"):
dir_ver = bn.split("_")[1].replace('/', '')

4
pytorch_lightning/trainer/connectors/checkpoint_connector.py
View File

@ -194,7 +194,7 @@ class CheckpointConnector:
folderpath = str(self.trainer.weights_save_path)
fs = get_filesystem(folderpath)
if fs.exists(folderpath):
files = [os.path.basename(f) for f in fs.ls(folderpath)]
files = [os.path.basename(f['name']) for f in fs.listdir(folderpath)]
hpc_weight_paths = [x for x in files if 'hpc_ckpt' in x]
# if hpc weights exist restore model
@ -333,7 +333,7 @@ class CheckpointConnector:
def max_ckpt_in_folder(self, path, name_key='ckpt_'):
fs = get_filesystem(path)
files = [os.path.basename(f) for f in fs.ls(path)]
files = [os.path.basename(f["name"]) for f in fs.listdir(path)]
files = [x for x in files if name_key in x]
if len(files) == 0:
return 0