From 20419089b6179a5a5d2d880d03cd50437576fb06 Mon Sep 17 00:00:00 2001 From: Rick Izzo Date: Wed, 7 Dec 2022 10:11:52 -0500 Subject: [PATCH] ENG-627: Docs for CloudCompute Mount Argument (#15182) fixed conflicts --- docs/source-app/api_reference/storage.rst | 9 ++ docs/source-app/api_references.rst | 16 ++ docs/source-app/glossary/index.rst | 7 + docs/source-app/glossary/mount.rst | 1 + docs/source-app/workflows/index.rst | 8 + .../workflows/mount_cloud_object_store.rst | 141 ++++++++++++++++++ examples/app_mount/app.py | 2 +- 7 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 docs/source-app/glossary/mount.rst create mode 100644 docs/source-app/workflows/mount_cloud_object_store.rst diff --git a/docs/source-app/api_reference/storage.rst b/docs/source-app/api_reference/storage.rst index 5bcdb0973d..4d125b80ae 100644 --- a/docs/source-app/api_reference/storage.rst +++ b/docs/source-app/api_reference/storage.rst @@ -20,6 +20,7 @@ ______________ ~path.Path ~drive.Drive ~payload.Payload + ~mount.Mount ---- @@ -56,6 +57,14 @@ Learn more about Storage :height: 180 :tag: Intermediate +.. displayitem:: + :header: The Mount Object. + :description: Mount an AWS S3 Bucket When Running on the Cloud. + :col_css: col-md-4 + :button_link: ../workflows/mount_aws_s3_bucket.html + :height: 180 + :tag: Intermediate + .. raw:: html diff --git a/docs/source-app/api_references.rst b/docs/source-app/api_references.rst index 808d2cb0a3..34cae0694b 100644 --- a/docs/source-app/api_references.rst +++ b/docs/source-app/api_references.rst @@ -72,6 +72,7 @@ _______ ~path.Path ~drive.Drive ~payload.Payload + ~mount.Mount Learn more about :ref:`Storage `. @@ -90,3 +91,18 @@ _______ ~cloud.CloudRuntime ~singleprocess.SingleProcessRuntime ~multiprocess.MultiProcessRuntime + +---- + +lightning_app.utilities.packaging +_________________________________ + +.. currentmodule:: lightning_app.utilities.packaging + +.. autosummary:: + :toctree: generated/ + :nosignatures: + :template: classtemplate_no_index.rst + + ~cloud_compute.CloudCompute + ~build_config.BuildConfig diff --git a/docs/source-app/glossary/index.rst b/docs/source-app/glossary/index.rst index 9652106e2a..a46da20ddd 100644 --- a/docs/source-app/glossary/index.rst +++ b/docs/source-app/glossary/index.rst @@ -112,6 +112,13 @@ Glossary :button_link: ../core_api/lightning_app/index.html :height: 100 +.. displayitem:: + :header: Mounts + :description: Mount Cloud Data + :col_css: col-md-6 + :button_link: mount.html + :height: 180 + .. displayitem:: :header: Sharing Components :description: Let's create an ecosystem altogether diff --git a/docs/source-app/glossary/mount.rst b/docs/source-app/glossary/mount.rst new file mode 100644 index 0000000000..a62d72b5b7 --- /dev/null +++ b/docs/source-app/glossary/mount.rst @@ -0,0 +1 @@ +.. include:: ../workflows/mount_cloud_object_store.rst diff --git a/docs/source-app/workflows/index.rst b/docs/source-app/workflows/index.rst index 314593c2aa..801c6ccecb 100644 --- a/docs/source-app/workflows/index.rst +++ b/docs/source-app/workflows/index.rst @@ -187,6 +187,14 @@ How to: :button_link: ssh/index.html :height: 180 +.. displayitem:: + :header: Mount Cloud Data + :description: Learn how Lightning Mounts are used to make the contents of an cloud object store bucket available on disk when running in the cloud. + :col_css: col-md-4 + :button_link: mount_cloud_object_store.html + :height: 180 + + .. raw:: html diff --git a/docs/source-app/workflows/mount_cloud_object_store.rst b/docs/source-app/workflows/mount_cloud_object_store.rst new file mode 100644 index 0000000000..4dfbe7bb37 --- /dev/null +++ b/docs/source-app/workflows/mount_cloud_object_store.rst @@ -0,0 +1,141 @@ +:orphan: + +############## +Add Cloud Data +############## + +**Audience:** Users who want to read files stored in a Cloud Object Bucket in an app. + +****************************** +Mounting Public AWS S3 Buckets +****************************** + +=================== +Add Mount to a Work +=================== + +To mount data from a cloud bucket to your app compute, initialize a :class:`~lightning_app.storage.mount.Mount` +object with the source path of the s3 bucket and the absolute directory path where it should be mounted and +pass the :class:`~lightning_app.storage.mount.Mount` to the :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute` +of the :class:`~lightning_app.core.work.LightningWork` it should be mounted on. + +In this example, we will mount an S3 bucket: ``s3://ryft-public-sample-data/esRedditJson/`` to ``/content/esRedditJson/``. + +.. code-block:: python + + from lightning_app import CloudCompute + from lightning_app.storage import Mount + + self.my_work = MyWorkClass( + cloud_compute=CloudCompute( + mounts=Mount( + source="s3://ryft-public-sample-data/esRedditJson/", + mount_path="/content/esRedditJson/", + ), + ) + ) + +You can also pass multiple mounts to a single work by passing a ``List[Mount(...), ...]`` to the +``CloudCompute(mounts=...)`` argument. + +.. note:: + + * Mounts supported up to 1 Million files, 5GB per file. Need larger mounts? Contact support@lightning.ai + * When adding multiple mounts, each one should have a unique ``mount_path``. + * A maximum of 10 :class:`~lightning_app.storage.mount.Mount`\s can be added to a :class:`~lightning_app.core.work.LightningWork`. + +======================= +Read Files From a Mount +======================= + +Once a :class:`~lightning_app.storage.mount.Mount` object is passed to :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute`, +you can access, list, or read any file from the mount under the specified ``mount_path``, just like you would if it +was on your local machine. + +Assuming your ``mount_path`` is ``"/content/esRedditJson/"`` you can do the following: + +---------- +Read Files +---------- + +.. code-block:: python + + with open("/content/esRedditJson/esRedditJson1", "r") as f: + some_data = f.read() + + # do something with "some_data"... + +---------- +List Files +---------- + +.. code-block:: python + + files = os.listdir("/content/esRedditJson/") + +-------------------- +See the Full Example +-------------------- + +.. code-block:: python + :emphasize-lines: 10,15 + + import os + + import lightning as L + from lightning_app import CloudCompute + from lightning_app.storage import Mount + + class ReadMount(L.LightningWork): + def run(self): + # Print a list of files stored in the mounted S3 Bucket. + files = os.listdir("/content/esRedditJson/") + for file in files: + print(file) + + # Read the contents of a particular file in the bucket "esRedditJson1" + with open("/content/esRedditJson/esRedditJson1", "r") as f: + some_data = f.read() + # do something with "some_data"... + + class Flow(L.LightningFlow): + def __init__(self): + super().__init__() + self.my_work = ReadMount( + cloud_compute=CloudCompute( + mounts=Mount( + source="s3://ryft-public-sample-data/esRedditJson/", + mount_path="/content/esRedditJson/", + ), + ) + ) + + def run(self): + self.my_work.run() + +.. note:: + + When running a Lighting App on your local machine, any :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute` + configuration (including a :class:`~lightning_app.storage.mount.Mount`) is ignored at runtime. If you need access to + these files on your local disk, you should download a copy of them to your machine. + +.. note:: + + Mounted files from an S3 bucket are ``read-only``. Any modifications, additions, or deletions + to files in the mounted directory will not be reflected in the cloud object store. + +---- + +********************************************** +Mounting Private AWS S3 Buckets - Coming Soon! +********************************************** + +We'll Let you know when this feature is ready! + +---- + +************************************************ +Mounting Google Cloud GCS Buckets - Coming Soon! +************************************************ + +We'll Let you know when this feature is ready! diff --git a/examples/app_mount/app.py b/examples/app_mount/app.py index 11da2f0255..d0d2adf3e0 100644 --- a/examples/app_mount/app.py +++ b/examples/app_mount/app.py @@ -32,4 +32,4 @@ class Flow(L.LightningFlow): self.work_1.run() -app = L.LightningApp(Flow(), log_level="debug") +app = L.LightningApp(Flow())