Invalid cache before listing drive when collecting component names (#13971)

Co-authored-by: thomas chaton <thomas@grid.ai>
Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com>

src/lightning_app/storage/drive.py

@@ -260,6 +260,9 @@ class Drive:
     def _collect_component_names(self) -> List[str]:
         sep = "/"
         if self.fs.exists(self.drive_root):
+            # Invalidate cache before running ls in case new directories have been added
+            # TODO: Re-evaluate this - may lead to performance issues
+            self.fs.invalidate_cache()
             return [str(p.split(sep)[-1]) for p in self.fs.ls(self.drive_root)]
         return []