Rooba
/
lightning
mirror of https://github.com/Lightning-AI/lightning.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Downloader: Resolve race condition (#19348)

This commit is contained in:
thomas chaton 2024-01-25 15:36:42 +00:00 committed by GitHub
parent c10fd22c74
commit ee9f17eb3c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/lightning/data/streaming/downloader.py
View File

@ -20,6 +20,7 @@ from urllib import parse
from filelock import FileLock, Timeout
from lightning.data.streaming.client import S3Client
from lightning.data.streaming.constants import _INDEX_FILENAME
class Downloader(ABC):
@ -56,7 +57,7 @@ class S3Downloader(Downloader):
return
try:
with FileLock(local_filepath + ".lock", timeout=0):
with FileLock(local_filepath + ".lock", timeout=1 if obj.path.endswith(_INDEX_FILENAME) else 0):
if self._s5cmd_available:
proc = subprocess.Popen(
f"s5cmd cp {remote_filepath} {local_filepath}",