From ee9f17eb3ceb98c256d62128e07e61d4a97a9911 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 25 Jan 2024 15:36:42 +0000 Subject: [PATCH] Downloader: Resolve race condition (#19348) --- src/lightning/data/streaming/downloader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lightning/data/streaming/downloader.py b/src/lightning/data/streaming/downloader.py index 005f6cb0fc..982b4d2514 100644 --- a/src/lightning/data/streaming/downloader.py +++ b/src/lightning/data/streaming/downloader.py @@ -20,6 +20,7 @@ from urllib import parse from filelock import FileLock, Timeout from lightning.data.streaming.client import S3Client +from lightning.data.streaming.constants import _INDEX_FILENAME class Downloader(ABC): @@ -56,7 +57,7 @@ class S3Downloader(Downloader): return try: - with FileLock(local_filepath + ".lock", timeout=0): + with FileLock(local_filepath + ".lock", timeout=1 if obj.path.endswith(_INDEX_FILENAME) else 0): if self._s5cmd_available: proc = subprocess.Popen( f"s5cmd cp {remote_filepath} {local_filepath}",