From 61a286d62ba687c08158d8b1f99a68e5991d909e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 18 Nov 2023 12:24:30 -0500 Subject: [PATCH] Make input dir in DataProcessor required (#18910) --- src/lightning/data/streaming/data_processor.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lightning/data/streaming/data_processor.py b/src/lightning/data/streaming/data_processor.py index f2f79f7346..4b213a2a3f 100644 --- a/src/lightning/data/streaming/data_processor.py +++ b/src/lightning/data/streaming/data_processor.py @@ -731,7 +731,7 @@ class DataTransformRecipe(DataRecipe): class DataProcessor: def __init__( self, - input_dir: Optional[Union[str, Dir]] = None, + input_dir: Union[str, Dir], output_dir: Optional[Union[str, Dir]] = None, num_workers: Optional[int] = None, num_downloaders: Optional[int] = None, @@ -771,9 +771,8 @@ class DataProcessor: self.stop_queues: List[Queue] = [] self.reorder_files = reorder_files - if self.input_dir: - # Ensure the input dir is the same across all nodes - self.input_dir = self._broadcast_object(self.input_dir) + # Ensure the input dir is the same across all nodes + self.input_dir = self._broadcast_object(self.input_dir) if self.output_dir: # Ensure the output dir is the same across all nodes