From eec862ef2fa97489c8f8c7ebe698cf1460c29326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Thu, 14 Jul 2022 15:45:46 +0200 Subject: [PATCH] Revert "Update Lightning App docs (#13537)" (#13655) This reverts commit cd31ba3f87d51b1e09e466c34b9b530809b38ddd. --- .actions/app/delete_cloud_lightning_apps.py | 34 -- .actions/app/download_frontend.py | 5 - .azure/gpu-tests.yml | 14 - .azure/hpu-tests.yml | 1 - .azure/ipu-tests.yml | 1 - .circleci/config.yml | 1 - .github/workflows/ci-app_block.yml | 5 +- .github/workflows/ci-pytorch_test-conda.yml | 34 +- .github/workflows/ci-pytorch_test-full.yml | 57 +--- .github/workflows/ci-pytorch_test-slow.yml | 37 +- .github/workflows/docs-checks.yml | 26 +- .github/workflows/docs-deploy.yml | 75 ---- .pre-commit-config.yaml | 3 - docs/{source-pytorch => }/Makefile | 4 +- docs/{source-pytorch => }/make.bat | 4 +- docs/source-app/Makefile | 19 -- docs/source-app/_templates/classtemplate.rst | 4 + .../_templates/classtemplate_no_index.rst | 12 - docs/source-app/_templates/layout.html | 10 - .../_templates/theme_variables.jinja | 8 +- .../api_reference/api_references.rst | 90 ----- docs/source-app/code_samples/basics/0.py | 4 +- docs/source-app/code_samples/basics/1.py | 4 +- .../code_samples/convert_pl_to_app/app.py | 17 - .../convert_pl_to_app/requirements.py | 3 - .../code_samples/convert_pl_to_app/train.py | 46 --- .../code_samples/quickstart/app/app_0.py | 8 +- .../code_samples/quickstart/app/app_1.py | 12 +- .../code_samples/quickstart/app_01.py | 12 +- .../code_samples/quickstart/app_02.py | 10 +- .../code_samples/quickstart/app_03.py | 10 +- .../code_samples/quickstart/app_comp.py | 10 +- .../quickstart/hello_world/app.py | 6 +- .../quickstart/hello_world/app_ui.py | 14 +- docs/source-app/conf.py | 35 +- docs/source-app/core_api/core_api.rst | 40 --- docs/source-app/core_api/lightning_app/app.py | 27 -- .../core_api/lightning_app/communication.rst | 135 +++++++- .../lightning_app/communication_content.rst | 160 --------- .../core_api/lightning_app/dynamic_work.rst | 188 +++++++++- .../lightning_app/dynamic_work_content.rst | 202 ----------- .../core_api/lightning_app/lightning_app.rst | 3 +- docs/source-app/core_api/lightning_flow.rst | 5 +- .../core_api/lightning_work/compute.rst | 99 +++++- .../lightning_work/compute_content.rst | 100 ------ .../lightning_work/handling_app_exception.rst | 80 ++++- .../handling_app_exception_content.rst | 74 ---- .../lightning_work/lightning_work.rst | 3 +- .../core_api/lightning_work/payload.rst | 84 ++++- .../lightning_work/payload_content.rst | 75 ---- .../core_api/lightning_work/status.rst | 204 ++++++++++- .../lightning_work/status_content.rst | 197 ----------- docs/source-app/examples/dag/dag.rst | 29 +- .../examples/dag/dag_from_scratch.rst | 11 +- docs/source-app/examples/data_explore_app.rst | 2 - docs/source-app/examples/etl_app.rst | 2 - docs/source-app/examples/file_server/app.py | 232 ------------- .../examples/file_server/file_server.rst | 12 - .../file_server/file_server_content.rst | 82 ----- .../file_server/file_server_step_1.rst | 11 - .../file_server/file_server_step_2.rst | 37 -- .../file_server/file_server_step_3.rst | 16 - .../file_server/file_server_step_4.rst | 86 ----- .../examples/github_repo_runner/.lightning | 1 - .../examples/github_repo_runner/app.py | 299 ---------------- .../github_repo_runner/github_repo_runner.rst | 13 - .../github_repo_runner_content.rst | 98 ------ .../github_repo_runner_step_1.rst | 62 ---- .../github_repo_runner_step_2.rst | 68 ---- .../github_repo_runner_step_3.rst | 62 ---- .../github_repo_runner_step_4.rst | 93 ----- .../github_repo_runner_step_5.rst | 77 ----- docs/source-app/examples/hands_on_example.rst | 50 --- .../examples/hpo/build_from_scratch.rst | 41 --- docs/source-app/examples/hpo/hpo.rst | 80 ----- docs/source-app/examples/hpo/hpo_wi.rst | 57 ---- docs/source-app/examples/hpo/hpo_wo.rst | 57 ---- .../source-app/examples/hpo/lightning_hpo.rst | 99 ------ .../examples/hpo/lightning_hpo_target.py | 53 --- .../examples/hpo/optuna_reference.py | 36 -- docs/source-app/examples/model_deploy_app.rst | 3 + .../examples/model_server_app/app.py | 34 -- .../model_server_app/load_testing.rst | 57 ---- .../model_server_app/locust_component.py | 43 --- .../examples/model_server_app/locustfile.py | 41 --- .../examples/model_server_app/model_server.py | 90 ----- .../model_server_app/model_server.rst | 48 --- .../model_server_app/model_server_app.rst | 15 - .../model_server_app_content.rst | 84 ----- .../putting_everything_together.rst | 80 ----- .../examples/model_server_app/train.py | 42 --- .../examples/model_server_app/train.rst | 49 --- .../source-app/examples/research_demo_app.rst | 2 - .../get_started/add_an_interactive_demo.rst | 18 - docs/source-app/get_started/build_model.rst | 76 ----- .../get_started/go_beyond_training.rst | 18 - .../jumpstart_from_app_gallery.rst | 123 ------- .../jumpstart_from_component_gallery.rst | 155 --------- .../get_started/lightning_apps_intro.rst | 14 - .../get_started/training_with_apps.rst | 136 -------- .../get_started/what_app_can_do.rst | 196 ----------- docs/source-app/glossary/app_tree.rst | 45 +-- .../build_config/build_config_basic.rst | 11 +- docs/source-app/glossary/dag.rst | 38 +-- docs/source-app/glossary/debug_app.rst | 2 - docs/source-app/glossary/distributed_fe.rst | 2 - .../glossary/distributed_hardware.rst | 2 - docs/source-app/glossary/event_loop.rst | 8 +- docs/source-app/glossary/fault_tolerance.rst | 2 - docs/source-app/glossary/index.rst | 80 ----- .../glossary/lightning_app_overview/index.rst | 1 - docs/source-app/glossary/scheduling.rst | 60 ++-- .../glossary/sharing_components.rst | 8 +- docs/source-app/glossary/storage/drive.rst | 201 ++++++++++- .../glossary/storage/drive_content.rst | 199 ----------- docs/source-app/glossary/storage/path.rst | 4 +- docs/source-app/glossary/storage/storage.rst | 27 -- docs/source-app/index.rst | 323 ++++++------------ docs/source-app/installation.rst | 41 --- docs/source-app/levels/advanced/index.rst | 74 ---- docs/source-app/levels/advanced/level_16.rst | 10 - docs/source-app/levels/advanced/level_17.rst | 10 - docs/source-app/levels/advanced/level_18.rst | 10 - docs/source-app/levels/advanced/level_19.rst | 11 - docs/source-app/levels/advanced/level_20.rst | 11 - docs/source-app/levels/basic/index.rst | 92 ----- docs/source-app/levels/basic/level_1.rst | 99 ------ docs/source-app/levels/basic/level_2.rst | 36 -- docs/source-app/levels/basic/level_3.rst | 56 --- docs/source-app/levels/basic/level_4.rst | 122 ------- docs/source-app/levels/basic/level_5.rst | 31 -- docs/source-app/levels/basic/level_6.rst | 78 ----- docs/source-app/levels/basic/level_7.rst | 36 -- docs/source-app/levels/intermediate/index.rst | 100 ------ .../levels/intermediate/level_10.rst | 14 - .../levels/intermediate/level_11.rst | 12 - .../levels/intermediate/level_12.rst | 10 - .../levels/intermediate/level_13.rst | 10 - .../levels/intermediate/level_14.rst | 10 - .../levels/intermediate/level_15.rst | 10 - .../levels/intermediate/level_8.rst | 10 - .../levels/intermediate/level_9.rst | 10 - docs/source-app/make.bat | 35 -- docs/source-app/{ => pages}/basics.rst | 52 +-- .../source-app/{ => pages}/contribute_app.rst | 0 .../{ => pages}/install_beginner.rst | 24 +- docs/source-app/pages/installation.rst | 26 ++ .../{intro.rst => pages/introduction.rst} | 19 +- .../lightning_apps_intro.rst} | 215 +++++++----- .../{ => pages}/moving_to_the_cloud.rst | 34 +- docs/source-app/{ => pages}/quickstart.rst | 32 +- docs/source-app/{ => pages}/testing.rst | 27 +- .../{ => pages}/ui_and_frontends.rst | 4 +- .../{examples => tutorials}/hpo/hpo.py | 19 +- docs/source-app/tutorials/hpo/hpo.rst | 146 ++++++++ .../{examples => tutorials}/hpo/objective.py | 12 +- .../source-app/workflows/access_app_state.rst | 61 ++++ .../workflows/add_components/index.rst | 4 +- .../workflows/add_server/any_server.rst | 26 +- .../workflows/add_server/flask_basic.rst | 36 +- docs/source-app/workflows/add_web_link.rst | 24 +- .../add_web_ui/angular_js_intermediate.rst | 2 - .../workflows/add_web_ui/dash/basic.rst | 90 ++--- .../workflows/add_web_ui/dash/index.rst | 2 - .../add_web_ui/dash/intermediate.rst | 26 +- .../add_web_ui/dash/intermediate_plot.py | 88 ----- .../add_web_ui/dash/intermediate_state.py | 39 --- .../workflows/add_web_ui/gradio/basic.rst | 48 +-- .../workflows/add_web_ui/gradio/index.rst | 2 - .../add_web_ui/gradio/intermediate.rst | 25 +- .../workflows/add_web_ui/html/basic.rst | 36 +- .../workflows/add_web_ui/html/index.rst | 2 - .../source-app/workflows/add_web_ui/index.rst | 4 +- .../workflows/add_web_ui/index_content.rst | 18 + .../integrate_any_javascript_framework.rst | 2 - .../workflows/add_web_ui/jupyter_basic.rst | 2 - ...ommunicate_between_react_and_lightning.rst | 12 +- .../react/connect_react_and_lightning.rst | 26 +- .../workflows/add_web_ui/react/index.rst | 2 - .../workflows/add_web_ui/streamlit/basic.rst | 55 +-- .../workflows/add_web_ui/streamlit/index.rst | 2 - .../add_web_ui/streamlit/intermediate.rst | 38 +-- .../add_web_ui/vue_js_intermediate.rst | 2 - .../arrange_tabs/arrange_app_basic.rst | 45 ++- .../from_pytorch_lightning_script.rst | 6 +- .../build_lightning_app/from_scratch.rst | 128 ++++++- .../from_scratch_content.rst | 121 ------- .../build_lightning_component/basic.rst | 205 ++++++++++- .../from_scratch_component_content.rst | 200 ----------- .../index_content.rst | 8 +- .../intermediate.rst | 14 +- .../publish_a_component.rst | 12 +- docs/source-app/workflows/debug_locally.rst | 2 - .../workflows/enable_fault_tolerance.rst | 2 - docs/source-app/workflows/extend_app.rst | 18 +- docs/source-app/workflows/index.rst | 114 ------- .../run_app_on_cloud/cloud_files.rst | 58 ---- .../workflows/run_app_on_cloud/index.rst | 5 - .../run_app_on_cloud/index_content.rst | 115 ------- .../run_app_on_cloud/lightning_cloud.rst | 67 ---- .../workflows/run_app_on_cloud/on_prem.rst | 6 - .../run_app_on_cloud/on_your_own_machine.rst | 23 -- .../run_components_on_different_hardware.rst | 2 - .../workflows/run_work_in_parallel.rst | 59 +++- .../run_work_in_parallel_content.rst | 53 --- docs/source-app/workflows/run_work_once.rst | 125 ++++++- .../workflows/run_work_once_content.rst | 151 -------- docs/source-app/workflows/schedule_apps.rst | 2 - docs/source-app/workflows/share_app.rst | 2 +- .../share_files_between_components.rst | 65 ++-- .../share_files_between_components/app.py | 48 --- docs/source-app/workflows/test_an_app.rst | 2 - docs/source-pytorch/conf.py | 2 +- src/lightning_app/CHANGELOG.md | 17 - 214 files changed, 2557 insertions(+), 7756 deletions(-) delete mode 100644 .actions/app/delete_cloud_lightning_apps.py delete mode 100644 .actions/app/download_frontend.py delete mode 100644 .github/workflows/docs-deploy.yml rename docs/{source-pytorch => }/Makefile (91%) rename docs/{source-pytorch => }/make.bat (93%) delete mode 100644 docs/source-app/Makefile delete mode 100644 docs/source-app/_templates/classtemplate_no_index.rst delete mode 100644 docs/source-app/_templates/layout.html delete mode 100644 docs/source-app/api_reference/api_references.rst delete mode 100644 docs/source-app/code_samples/convert_pl_to_app/app.py delete mode 100644 docs/source-app/code_samples/convert_pl_to_app/requirements.py delete mode 100644 docs/source-app/code_samples/convert_pl_to_app/train.py delete mode 100644 docs/source-app/core_api/core_api.rst delete mode 100644 docs/source-app/core_api/lightning_app/app.py delete mode 100644 docs/source-app/core_api/lightning_app/communication_content.rst delete mode 100644 docs/source-app/core_api/lightning_app/dynamic_work_content.rst delete mode 100644 docs/source-app/core_api/lightning_work/compute_content.rst delete mode 100644 docs/source-app/core_api/lightning_work/handling_app_exception_content.rst delete mode 100644 docs/source-app/core_api/lightning_work/payload_content.rst delete mode 100644 docs/source-app/core_api/lightning_work/status_content.rst delete mode 100644 docs/source-app/examples/file_server/app.py delete mode 100644 docs/source-app/examples/file_server/file_server.rst delete mode 100644 docs/source-app/examples/file_server/file_server_content.rst delete mode 100644 docs/source-app/examples/file_server/file_server_step_1.rst delete mode 100644 docs/source-app/examples/file_server/file_server_step_2.rst delete mode 100644 docs/source-app/examples/file_server/file_server_step_3.rst delete mode 100644 docs/source-app/examples/file_server/file_server_step_4.rst delete mode 100644 docs/source-app/examples/github_repo_runner/.lightning delete mode 100644 docs/source-app/examples/github_repo_runner/app.py delete mode 100644 docs/source-app/examples/github_repo_runner/github_repo_runner.rst delete mode 100644 docs/source-app/examples/github_repo_runner/github_repo_runner_content.rst delete mode 100644 docs/source-app/examples/github_repo_runner/github_repo_runner_step_1.rst delete mode 100644 docs/source-app/examples/github_repo_runner/github_repo_runner_step_2.rst delete mode 100644 docs/source-app/examples/github_repo_runner/github_repo_runner_step_3.rst delete mode 100644 docs/source-app/examples/github_repo_runner/github_repo_runner_step_4.rst delete mode 100644 docs/source-app/examples/github_repo_runner/github_repo_runner_step_5.rst delete mode 100644 docs/source-app/examples/hands_on_example.rst delete mode 100644 docs/source-app/examples/hpo/build_from_scratch.rst delete mode 100644 docs/source-app/examples/hpo/hpo.rst delete mode 100644 docs/source-app/examples/hpo/hpo_wi.rst delete mode 100644 docs/source-app/examples/hpo/hpo_wo.rst delete mode 100644 docs/source-app/examples/hpo/lightning_hpo.rst delete mode 100644 docs/source-app/examples/hpo/lightning_hpo_target.py delete mode 100644 docs/source-app/examples/hpo/optuna_reference.py create mode 100644 docs/source-app/examples/model_deploy_app.rst delete mode 100644 docs/source-app/examples/model_server_app/app.py delete mode 100644 docs/source-app/examples/model_server_app/load_testing.rst delete mode 100644 docs/source-app/examples/model_server_app/locust_component.py delete mode 100644 docs/source-app/examples/model_server_app/locustfile.py delete mode 100644 docs/source-app/examples/model_server_app/model_server.py delete mode 100644 docs/source-app/examples/model_server_app/model_server.rst delete mode 100644 docs/source-app/examples/model_server_app/model_server_app.rst delete mode 100644 docs/source-app/examples/model_server_app/model_server_app_content.rst delete mode 100644 docs/source-app/examples/model_server_app/putting_everything_together.rst delete mode 100644 docs/source-app/examples/model_server_app/train.py delete mode 100644 docs/source-app/examples/model_server_app/train.rst delete mode 100644 docs/source-app/get_started/add_an_interactive_demo.rst delete mode 100644 docs/source-app/get_started/build_model.rst delete mode 100644 docs/source-app/get_started/go_beyond_training.rst delete mode 100644 docs/source-app/get_started/jumpstart_from_app_gallery.rst delete mode 100644 docs/source-app/get_started/jumpstart_from_component_gallery.rst delete mode 100644 docs/source-app/get_started/lightning_apps_intro.rst delete mode 100644 docs/source-app/get_started/training_with_apps.rst delete mode 100644 docs/source-app/get_started/what_app_can_do.rst delete mode 100644 docs/source-app/glossary/index.rst delete mode 100644 docs/source-app/glossary/storage/drive_content.rst delete mode 100644 docs/source-app/installation.rst delete mode 100644 docs/source-app/levels/advanced/index.rst delete mode 100644 docs/source-app/levels/advanced/level_16.rst delete mode 100644 docs/source-app/levels/advanced/level_17.rst delete mode 100644 docs/source-app/levels/advanced/level_18.rst delete mode 100644 docs/source-app/levels/advanced/level_19.rst delete mode 100644 docs/source-app/levels/advanced/level_20.rst delete mode 100644 docs/source-app/levels/basic/index.rst delete mode 100644 docs/source-app/levels/basic/level_1.rst delete mode 100644 docs/source-app/levels/basic/level_2.rst delete mode 100644 docs/source-app/levels/basic/level_3.rst delete mode 100644 docs/source-app/levels/basic/level_4.rst delete mode 100644 docs/source-app/levels/basic/level_5.rst delete mode 100644 docs/source-app/levels/basic/level_6.rst delete mode 100644 docs/source-app/levels/basic/level_7.rst delete mode 100644 docs/source-app/levels/intermediate/index.rst delete mode 100644 docs/source-app/levels/intermediate/level_10.rst delete mode 100644 docs/source-app/levels/intermediate/level_11.rst delete mode 100644 docs/source-app/levels/intermediate/level_12.rst delete mode 100644 docs/source-app/levels/intermediate/level_13.rst delete mode 100644 docs/source-app/levels/intermediate/level_14.rst delete mode 100644 docs/source-app/levels/intermediate/level_15.rst delete mode 100644 docs/source-app/levels/intermediate/level_8.rst delete mode 100644 docs/source-app/levels/intermediate/level_9.rst delete mode 100644 docs/source-app/make.bat rename docs/source-app/{ => pages}/basics.rst (91%) rename docs/source-app/{ => pages}/contribute_app.rst (100%) rename docs/source-app/{ => pages}/install_beginner.rst (91%) create mode 100644 docs/source-app/pages/installation.rst rename docs/source-app/{intro.rst => pages/introduction.rst} (95%) rename docs/source-app/{get_started/go_beyond_training_content.rst => pages/lightning_apps_intro.rst} (64%) rename docs/source-app/{ => pages}/moving_to_the_cloud.rst (85%) rename docs/source-app/{ => pages}/quickstart.rst (90%) rename docs/source-app/{ => pages}/testing.rst (95%) rename docs/source-app/{ => pages}/ui_and_frontends.rst (82%) rename docs/source-app/{examples => tutorials}/hpo/hpo.py (70%) create mode 100644 docs/source-app/tutorials/hpo/hpo.rst rename docs/source-app/{examples => tutorials}/hpo/objective.py (55%) create mode 100644 docs/source-app/workflows/access_app_state.rst delete mode 100644 docs/source-app/workflows/add_web_ui/dash/intermediate_plot.py delete mode 100644 docs/source-app/workflows/add_web_ui/dash/intermediate_state.py delete mode 100644 docs/source-app/workflows/build_lightning_app/from_scratch_content.rst delete mode 100644 docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst delete mode 100644 docs/source-app/workflows/index.rst delete mode 100644 docs/source-app/workflows/run_app_on_cloud/cloud_files.rst delete mode 100644 docs/source-app/workflows/run_app_on_cloud/index.rst delete mode 100644 docs/source-app/workflows/run_app_on_cloud/index_content.rst delete mode 100644 docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst delete mode 100644 docs/source-app/workflows/run_app_on_cloud/on_prem.rst delete mode 100644 docs/source-app/workflows/run_app_on_cloud/on_your_own_machine.rst delete mode 100644 docs/source-app/workflows/run_work_in_parallel_content.rst delete mode 100644 docs/source-app/workflows/run_work_once_content.rst delete mode 100644 docs/source-app/workflows/share_files_between_components/app.py delete mode 100644 src/lightning_app/CHANGELOG.md diff --git a/.actions/app/delete_cloud_lightning_apps.py b/.actions/app/delete_cloud_lightning_apps.py deleted file mode 100644 index f682847f1a..0000000000 --- a/.actions/app/delete_cloud_lightning_apps.py +++ /dev/null @@ -1,34 +0,0 @@ -import os - -from lightning_cloud.openapi.rest import ApiException - -from lightning_app.utilities.cloud import _get_project -from lightning_app.utilities.network import LightningClient - -client = LightningClient() - -try: - PR_NUMBER = int(os.getenv("PR_NUMBER", None)) -except (TypeError, ValueError): - # Failed when the PR is running master or 'PR_NUMBER' isn't defined. - PR_NUMBER = "" - -APP_NAME = os.getenv("TEST_APP_NAME", "") - -project = _get_project(client) -list_lightningapps = client.lightningapp_instance_service_list_lightningapp_instances(project.project_id) - -print([lightningapp.name for lightningapp in list_lightningapps.lightningapps]) - -for lightningapp in list_lightningapps.lightningapps: - if PR_NUMBER and APP_NAME and not lightningapp.name.startswith(f"test-{PR_NUMBER}-{APP_NAME}-"): - continue - print(f"Deleting {lightningapp.name}") - try: - res = client.lightningapp_instance_service_delete_lightningapp_instance( - project_id=project.project_id, - id=lightningapp.id, - ) - assert res == {} - except ApiException as e: - print(f"Failed to delete {lightningapp.name}. Exception {e}") diff --git a/.actions/app/download_frontend.py b/.actions/app/download_frontend.py deleted file mode 100644 index 44d3886580..0000000000 --- a/.actions/app/download_frontend.py +++ /dev/null @@ -1,5 +0,0 @@ -import lightning_app -from lightning_app.utilities.packaging.lightning_utils import download_frontend - -if __name__ == "__main__": - download_frontend(lightning_app._PROJECT_ROOT) diff --git a/.azure/gpu-tests.yml b/.azure/gpu-tests.yml index 0f29eca5dd..f84463a661 100644 --- a/.azure/gpu-tests.yml +++ b/.azure/gpu-tests.yml @@ -12,7 +12,6 @@ trigger: - "master" - "release/*" - "refs/tags/*" - pr: - "master" - "release/*" @@ -38,19 +37,6 @@ jobs: steps: - - bash: | - CHANGED_FILES=$(git diff --name-status master | awk '{print $2}') - echo $CHANGED_FILES > changed_files.txt - MATCHES=$(cat changed_files.txt | grep -E $FILTER) - echo $MATCHES - if [ -z "$MATCHES" ]; then - echo "Skip" - else - echo "Continue" - fi - - displayName: Decide if skipping should be done. - - bash: | lspci | egrep 'VGA|3D' whereis nvidia diff --git a/.azure/hpu-tests.yml b/.azure/hpu-tests.yml index e530ace901..a3041ce32d 100644 --- a/.azure/hpu-tests.yml +++ b/.azure/hpu-tests.yml @@ -9,7 +9,6 @@ trigger: - "master" - "release/*" - "refs/tags/*" - pr: - "master" - "release/*" diff --git a/.azure/ipu-tests.yml b/.azure/ipu-tests.yml index dccde83869..418a70d6fa 100644 --- a/.azure/ipu-tests.yml +++ b/.azure/ipu-tests.yml @@ -7,7 +7,6 @@ trigger: - master - release/* - refs/tags/* - pr: - master - release/* diff --git a/.circleci/config.yml b/.circleci/config.yml index 29fe5fcc1a..c608680e1c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,6 @@ trigger: - "master" - "release/*" - "refs/tags/*" - pr: - "master" - "release/*" diff --git a/.github/workflows/ci-app_block.yml b/.github/workflows/ci-app_block.yml index fa582e99ac..8c2dd772aa 100644 --- a/.github/workflows/ci-app_block.yml +++ b/.github/workflows/ci-app_block.yml @@ -13,9 +13,12 @@ jobs: - name: Get changed files using defaults id: changed-files uses: tj-actions/changed-files@v23 - - name: List all added files run: | for file in ${{ steps.changed-files.outputs.all_changed_and_modified_files }}; do echo "$file" done + + - name: Block edits in docs/source-app + if: contains(steps.changed-files.outputs.all_changed_and_modified_files, 'docs/source-app') + run: exit 1 diff --git a/.github/workflows/ci-pytorch_test-conda.yml b/.github/workflows/ci-pytorch_test-conda.yml index 241bdaa5b8..43cf73052b 100644 --- a/.github/workflows/ci-pytorch_test-conda.yml +++ b/.github/workflows/ci-pytorch_test-conda.yml @@ -6,6 +6,11 @@ on: # Trigger the workflow on push or pull request, but only for the master bra branches: [master, "release/*"] pull_request: branches: [master, "release/*"] + paths-ignore: + - "src/lightning_app/**" # todo: implement job skip + - "tests/tests_app/**" # todo: implement job skip + - "tests/tests_app_examples/**" # todo: implement job skip + - "examples/app_*" # todo: implement job skip concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} @@ -29,34 +34,13 @@ jobs: - {python-version: "3.9", pytorch-version: "1.11"} timeout-minutes: 30 - steps: - name: Workaround for https://github.com/actions/checkout/issues/760 run: git config --global --add safe.directory /__w/lightning/lightning - uses: actions/checkout@v2 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v23.1 - - - name: Decide if the test should be skipped - id: skip - run: | - FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_' - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr " " "\n" > changed_files.txt - MATCHES=$(cat changed_files.txt | grep -E $FILTER) - echo $MATCHES - if [ -z "$MATCHES" ]; then - echo "Skip" - echo "::set-output name=continue::'0'" - else - echo "Continue" - echo "::set-output name=continue::'1'" - fi - - name: Update base dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} env: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 @@ -66,12 +50,10 @@ jobs: pip install -e .[test] - name: DocTests - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: ./src run: pytest pytorch_lightning --cov=pytorch_lightning - name: Update all dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} env: HOROVOD_BUILD_ARCH_FLAGS: "-mfma" HOROVOD_WITHOUT_MXNET: 1 @@ -90,11 +72,9 @@ jobs: python requirements/pytorch/check-avail-extras.py - name: Pull legacy checkpoints - if: ${{ (steps.skip.outputs.continue == '1') }} run: bash .actions/pull_legacy_checkpoints.sh - name: Testing PyTorch - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: tests/tests_pytorch run: coverage run --source pytorch_lightning -m pytest -v --timeout 150 --durations=50 --junitxml=results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml @@ -106,7 +86,7 @@ jobs: if: failure() - name: Statistics - if: ${{ success() && (steps.skip.outputs.continue == '1') }} + if: success() working-directory: tests/tests_pytorch run: | coverage report @@ -114,7 +94,7 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - if: ${{ success() && (steps.skip.outputs.continue == '1') }} + if: always() # see: https://github.com/actions/toolkit/issues/399 continue-on-error: true with: diff --git a/.github/workflows/ci-pytorch_test-full.yml b/.github/workflows/ci-pytorch_test-full.yml index c488cfef9f..2db59a30f5 100644 --- a/.github/workflows/ci-pytorch_test-full.yml +++ b/.github/workflows/ci-pytorch_test-full.yml @@ -1,4 +1,4 @@ -name: Test PyTorch full +name: Test PyTorch full # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows on: # Trigger the workflow on push or pull request, but only for the master branch @@ -7,6 +7,11 @@ on: # Trigger the workflow on push or pull request, but only for the master bra pull_request: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] + paths-ignore: + - "src/lightning_app/**" # todo: implement job skip + - "tests/tests_app/**" # todo: implement job skip + - "tests/tests_app_examples/**" # todo: implement job skip + - "examples/app_*" # todo: implement job skip concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} @@ -32,67 +37,40 @@ jobs: steps: - uses: actions/checkout@v2 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v23.1 - - - name: Decide if the test should be skipped - id: skip - run: | - FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_' - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr " " "\n" > changed_files.txt - MATCHES=$(cat changed_files.txt | grep -E $FILTER) - echo $MATCHES - if [ -z "$MATCHES" ]; then - echo "Skip" - echo "::set-output name=continue::'0'" - else - echo "Continue" - echo "::set-output name=continue::'1'" - fi - - name: Set up Python ${{ matrix.python-version }} - if: ${{ (steps.skip.outputs.continue == '1') }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Reset caching - if: ${{ (steps.skip.outputs.continue == '1') }} run: python -c "import time; days = time.time() / 60 / 60 / 24; print(f'TIME_PERIOD=d{int(days / 2) * 2}')" >> $GITHUB_ENV - name: basic setup - if: ${{ (steps.skip.outputs.continue == '1') }} run: | pip --version pip install -q fire # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646 - name: Setup macOS - if: ${{ (runner.os == 'macOS') && (steps.skip.outputs.continue == '1') }} + if: runner.os == 'macOS' run: | brew install openmpi libuv # Horovod on macOS requires OpenMPI, Gloo not currently supported - - name: Setup Windows - if: ${{ (runner.os == 'windows') && (steps.skip.outputs.continue == '1') }} + if: runner.os == 'windows' run: | python .actions/assistant.py requirements_prune_pkgs horovod - - name: Set min. dependencies - if: ${{ (matrix.requires == 'oldest') && (steps.skip.outputs.continue == '1') }} + if: matrix.requires == 'oldest' run: | python .actions/assistant.py replace_oldest_ver # Note: This uses an internal pip API and may not always work # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - name: Get pip cache dir - if: ${{ (steps.skip.outputs.continue == '1') }} id: pip-cache run: echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - if: ${{ (steps.skip.outputs.continue == '1') }} uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} @@ -101,11 +79,9 @@ jobs: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}- - name: Pull legacy checkpoints - if: ${{ (steps.skip.outputs.continue == '1') }} run: bash .actions/pull_legacy_checkpoints.sh - name: Install dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} env: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 @@ -117,12 +93,10 @@ jobs: shell: bash - name: DocTests - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: ./src run: pytest pytorch_lightning --cov=pytorch_lightning - name: Install extra dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} run: | # adjust versions according installed Torch version python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt @@ -131,7 +105,7 @@ jobs: shell: bash - name: Reinstall Horovod if necessary - if: ${{ (runner.os != 'windows') && (steps.skip.outputs.continue == '1') }} + if: runner.os != 'windows' env: HOROVOD_BUILD_ARCH_FLAGS: "-mfma" HOROVOD_WITHOUT_MXNET: 1 @@ -148,43 +122,38 @@ jobs: shell: bash - name: Cache datasets - if: ${{ (steps.skip.outputs.continue == '1') }} uses: actions/cache@v2 with: path: Datasets key: pl-dataset - name: Sanity check - if: ${{ (steps.skip.outputs.continue == '1') }} run: python requirements/pytorch/check-avail-extras.py - name: Testing PyTorch - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: tests/tests_pytorch # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003 run: coverage run --source pytorch_lightning -m pytest -v --durations=50 --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml - name: Upload pytest results - if: ${{ (failure()) && (steps.skip.outputs.continue == '1') }} uses: actions/upload-artifact@v3 with: name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }} path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml + if: failure() - name: Prepare Examples - if: ${{ (steps.skip.outputs.continue == '1') }} run: | # adjust versions according installed Torch version python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt pip install -r requirements/pytorch/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade - name: Run Examples - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: ./examples run: python -m pytest test_pl_examples.py -v --durations=10 - name: Statistics - if: ${{ (success()) && (steps.skip.outputs.continue == '1') }} + if: success() working-directory: tests/tests_pytorch run: | coverage report @@ -192,7 +161,7 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - if: ${{ (always()) && (steps.skip.outputs.continue == '1') }} + if: always() # see: https://github.com/actions/toolkit/issues/399 continue-on-error: true with: diff --git a/.github/workflows/ci-pytorch_test-slow.yml b/.github/workflows/ci-pytorch_test-slow.yml index b42e529dce..79cede5b59 100644 --- a/.github/workflows/ci-pytorch_test-slow.yml +++ b/.github/workflows/ci-pytorch_test-slow.yml @@ -7,6 +7,11 @@ on: # Trigger the workflow on push or pull request, but only for the master bra pull_request: branches: [master, "release/*"] types: [opened, reopened, ready_for_review, synchronize] + paths-ignore: + - "src/lightning_app/**" # todo: implement job skip + - "tests/tests_app/**" # todo: implement job skip + - "tests/tests_app_examples/**" # todo: implement job skip + - "examples/app_*" # todo: implement job skip concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} @@ -27,43 +32,19 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v2 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v23.1 - - - name: Decide if the test should be skipped - id: skip - run: | - FILTER='src/pytorch_lightning|requirements/pytorch|tests/tests_pytorch|examples/pl_' - echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr " " "\n" > changed_files.txt - MATCHES=$(cat changed_files.txt | grep -E $FILTER) - echo $MATCHES - if [ -z "$MATCHES" ]; then - echo "Skip" - echo "::set-output name=continue::'0'" - else - echo "Continue" - echo "::set-output name=continue::'1'" - fi - - uses: actions/setup-python@v2 - if: ${{ (steps.skip.outputs.continue == '1') }} with: python-version: ${{ matrix.python-version }} - name: Reset caching - if: ${{ (steps.skip.outputs.continue == '1') }} run: python -c "import time; days = time.time() / 60 / 60 / 24; print(f'TIME_PERIOD=d{int(days / 2) * 2}')" >> $GITHUB_ENV - name: Get pip cache - if: ${{ (steps.skip.outputs.continue == '1') }} id: pip-cache run: | python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" - name: Cache pip - if: ${{ (steps.skip.outputs.continue == '1') }} uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} @@ -72,7 +53,6 @@ jobs: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}- - name: Install dependencies - if: ${{ (steps.skip.outputs.continue == '1') }} env: PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 @@ -84,21 +64,20 @@ jobs: shell: bash - name: Testing PyTorch - if: ${{ (steps.skip.outputs.continue == '1') }} working-directory: tests/tests_pytorch run: coverage run --source pytorch_lightning -m pytest -v --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}.xml env: PL_RUN_SLOW_TESTS: 1 - name: Upload pytest test results - if: ${{ (failure()) && (steps.skip.outputs.continue == '1') }} uses: actions/upload-artifact@v3 with: name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }} path: tests/tests_pytorch/results-${{ runner.os }}-py${{ matrix.python-version }}.xml + if: failure() - name: Statistics - if: ${{ (success()) && (steps.skip.outputs.continue == '1') }} + if: success() working-directory: tests/tests_pytorch run: | coverage report @@ -106,7 +85,7 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - if: ${{ (success()) && (steps.skip.outputs.continue == '1') }} + if: success() # see: https://github.com/actions/toolkit/issues/399 continue-on-error: true with: diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index c5b44eedb4..eadf11a568 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -14,10 +14,6 @@ concurrency: jobs: doctest: runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - pkg: ["app", "pytorch"] steps: - uses: actions/checkout@v2 with: @@ -35,12 +31,13 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-docs-test-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements/${{ matrix.pkg }}/*.txt') }} + key: ${{ runner.os }}-docs-test-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements/pytorch/*.txt') }} restore-keys: | ${{ runner.os }}-docs-test-pip-td${{ env.TIME_PERIOD }}- - name: Install dependencies env: + PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 run: | sudo apt-get update @@ -48,27 +45,22 @@ jobs: pip --version pip install -q fire # python -m pip install --upgrade --user pip - pip install -e . --quiet -r requirements/${{ matrix.pkg }}/base.txt -r requirements/${{ matrix.pkg }}/docs.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html - pip install -r requirements/${{ matrix.pkg }}/devel.txt + pip install -e . --quiet -r requirements/pytorch/docs.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html + pip install -r requirements/pytorch/devel.txt pip list shell: bash - name: Test Documentation env: SPHINX_MOCK_REQUIREMENTS: 0 - working-directory: ./docs/source-${{ matrix.pkg }} + working-directory: ./docs run: | - # ToDo: proper parametrize # First run the same pipeline as Read-The-Docs make doctest make coverage make-docs: runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - pkg: ["app", "pytorch"] steps: - uses: actions/checkout@v2 with: @@ -84,27 +76,27 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-docs-make-pip-${{ hashFiles('requirements/${{ matrix.pkg }}/*.txt') }} + key: ${{ runner.os }}-docs-make-pip-${{ hashFiles('requirements/pytorch/base.txt') }} restore-keys: | ${{ runner.os }}-docs-make-pip- - name: Install dependencies env: + PACKAGE_NAME: pytorch FREEZE_REQUIREMENTS: 1 run: | sudo apt-get update sudo apt-get install -y cmake pandoc pip --version - pip install -e . --quiet -r requirements/${{ matrix.pkg }}/base.txt -r requirements/${{ matrix.pkg }}/docs.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html + pip install -e . --quiet -r requirements/pytorch/docs.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux sudo apt-get update && sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures pip list shell: bash - name: Make Documentation - working-directory: ./docs/source-${{ matrix.pkg }} + working-directory: ./docs run: | - # ToDo: rather use python cmd # First run the same pipeline as Read-The-Docs make html --debug --jobs $(nproc) SPHINXOPTS="-W --keep-going" diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml deleted file mode 100644 index cb6e090e5a..0000000000 --- a/.github/workflows/docs-deploy.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: "Deploy Docs" -on: - push: - branches: [master] - -jobs: - # https://github.com/marketplace/actions/deploy-to-github-pages - build-docs-deploy: - runs-on: ubuntu-20.04 - steps: - - name: Checkout 🛎️ - uses: actions/checkout@v2 - # If you're using actions/checkout@v2 you must set persist-credentials to false in most cases for the deployment to work correctly. - with: - persist-credentials: false - - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - id: 'auth' - name: 'Authenticate to Google Cloud' - uses: 'google-github-actions/auth@v0' - with: - credentials_json: ${{ secrets.GCS_SA_KEY }} - - - name: Setup gcloud - uses: 'google-github-actions/setup-gcloud@v0' - with: - project_id: ${{ secrets.GCS_PROJECT }} - - # Note: This uses an internal pip API and may not always work - # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - - name: Cache pip - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-deploy-docs-pip-${{ hashFiles('requirements/app/*.txt') }} - restore-keys: | - ${{ runner.os }}-deploy-docs-pip- - - - name: Install dependencies - env: - FREEZE_REQUIREMENTS: 1 - run: | - sudo apt-get update - sudo apt-get install -y cmake pandoc - pip --version - pip install -e . --quiet -r requirements/app/docs.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html - sudo apt-get update && sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures - pip list - shell: bash - - - name: Make Documentation - working-directory: ./docs/source-app - run: | - # First run the same pipeline as Read-The-Docs - make clean - make html --jobs 2 - - - name: Deploy 🚀 - uses: JamesIves/github-pages-deploy-action@4.1.4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages # The branch the action should deploy to. - folder: docs/build/html # The folder the action should deploy. - clean: true # Automatically remove deleted files from the deploy branch - target-folder: docs # If you'd like to push the contents of the deployment folder into a specific directory - single-commit: true # you'd prefer to have a single commit on the deployment branch instead of full history - if: success() - - # Uploading docs to GCS so they can be served on lightning.ai - - name: Upload to GCS 🪣 - run: |- - gsutil -m rsync -d -R docs/build/html/ gs://${{ secrets.GCS_BUCKET }} - if: success() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86d301d111..0c96c9ceea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -77,7 +77,6 @@ repos: hooks: - id: black name: Format code - exclude: docs/source-app - repo: https://github.com/asottile/blacken-docs rev: v1.12.1 @@ -85,7 +84,6 @@ repos: - id: blacken-docs args: [--line-length=120] additional_dependencies: [black==21.12b0] - exclude: docs/source-app - repo: https://github.com/executablebooks/mdformat rev: 0.7.14 @@ -102,4 +100,3 @@ repos: hooks: - id: flake8 name: Check PEP8 - exclude: docs/source-app diff --git a/docs/source-pytorch/Makefile b/docs/Makefile similarity index 91% rename from docs/source-pytorch/Makefile rename to docs/Makefile index 68be4c930e..fe8ff0e5fb 100644 --- a/docs/source-pytorch/Makefile +++ b/docs/Makefile @@ -4,8 +4,8 @@ # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build -SOURCEDIR = . -BUILDDIR = ../build +SOURCEDIR = source-pytorch +BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: diff --git a/docs/source-pytorch/make.bat b/docs/make.bat similarity index 93% rename from docs/source-pytorch/make.bat rename to docs/make.bat index 9b565142ae..09a132f69a 100644 --- a/docs/source-pytorch/make.bat +++ b/docs/make.bat @@ -7,8 +7,8 @@ REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) -set SOURCEDIR=. -set BUILDDIR=../build +set SOURCEDIR=source-pytorch +set BUILDDIR=build if "%1" == "" goto help diff --git a/docs/source-app/Makefile b/docs/source-app/Makefile deleted file mode 100644 index 68be4c930e..0000000000 --- a/docs/source-app/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SOURCEDIR = . -BUILDDIR = ../build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source-app/_templates/classtemplate.rst b/docs/source-app/_templates/classtemplate.rst index 5b7f465516..b99cdb3426 100644 --- a/docs/source-app/_templates/classtemplate.rst +++ b/docs/source-app/_templates/classtemplate.rst @@ -7,3 +7,7 @@ .. autoclass:: {{ name }} :members: + +.. + autogenerated from source/_templates/classtemplate.rst + note it does not have :inherited-members: diff --git a/docs/source-app/_templates/classtemplate_no_index.rst b/docs/source-app/_templates/classtemplate_no_index.rst deleted file mode 100644 index 858c37b515..0000000000 --- a/docs/source-app/_templates/classtemplate_no_index.rst +++ /dev/null @@ -1,12 +0,0 @@ -:orphan: - -.. role:: hidden - :class: hidden-section -.. currentmodule:: {{ module }} - - -{{ name | underline }} - -.. autoclass:: {{ name }} - :members: - :noindex: diff --git a/docs/source-app/_templates/layout.html b/docs/source-app/_templates/layout.html deleted file mode 100644 index dfb2c269ed..0000000000 --- a/docs/source-app/_templates/layout.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "!layout.html" %} - - -{% block footer %} -{{ super() }} - - -{% endblock %} diff --git a/docs/source-app/_templates/theme_variables.jinja b/docs/source-app/_templates/theme_variables.jinja index 447390ecc9..da4cf094f8 100644 --- a/docs/source-app/_templates/theme_variables.jinja +++ b/docs/source-app/_templates/theme_variables.jinja @@ -1,8 +1,8 @@ {%- set external_urls = { - 'github': 'https://github.com/Lightning-AI/lightning', - 'github_issues': 'https://github.com/Lightning-AI/lightning/issues', - 'contributing': 'https://github.com/Lightning-AI/lightning/blob/master/.github/CONTRIBUTING.md', - 'governance': 'https://github.com/Lightning-AI/lightning/blob/master/docs/source-pytorch/governance.rst', + 'github': 'https://github.com/PytorchLightning/lightning', + 'github_issues': 'https://github.com/PytorchLightning/lightning/issues', + 'contributing': 'https://github.com/PytorchLightning/pytorch-lightning/blob/master/CONTRIBUTING.md', + 'governance': 'https://github.com/PytorchLightning/pytorch-lightning/blob/master/governance.md', 'docs': 'https://lightning.rtfd.io/en/latest', 'twitter': 'https://twitter.com/PyTorchLightnin', 'discuss': 'https://pytorch-lightning.slack.com', diff --git a/docs/source-app/api_reference/api_references.rst b/docs/source-app/api_reference/api_references.rst deleted file mode 100644 index 42cfcb7aed..0000000000 --- a/docs/source-app/api_reference/api_references.rst +++ /dev/null @@ -1,90 +0,0 @@ -:orphan: - -############################## -Lightning App - API References -############################## - -Core ----- - -.. currentmodule:: lightning_app.core - -.. autosummary:: - :toctree: api - :nosignatures: - :template: classtemplate_no_index.rst - - LightningApp - LightningFlow - LightningWork - -Learn more about :ref:`Lightning Core `. - ----- - -Built-in Components -___________________ - -.. currentmodule:: lightning_app.components - -.. autosummary:: - :toctree: generated/ - :nosignatures: - :template: classtemplate_no_index.rst - - ~serve.serve.ModelInferenceAPI - ~python.popen.PopenPythonScript - ~serve.gradio.ServeGradio - ~python.tracer.TracerPythonScript - ----- - -Frontend's -__________ - -.. currentmodule:: lightning_app.frontend - -.. autosummary:: - :toctree: generated/ - :nosignatures: - :template: classtemplate_no_index.rst - - ~frontend.Frontend - ~web.StaticWebFrontend - ~stream_lit.StreamlitFrontend - -Learn more about :ref:`Frontend's `. - ----- - -Storage -_______ - -.. currentmodule:: lightning_app.storage - -.. autosummary:: - :toctree: generated/ - :nosignatures: - :template: classtemplate_no_index.rst - - ~drive.Drive - ~path.Path - ~payload.Payload - -Learn more about :ref:`Storage `. - ----- - -Runners -_______ - -.. currentmodule:: lightning_app.runners - -.. autosummary:: - :toctree: generated/ - :nosignatures: - :template: classtemplate_no_index.rst - - ~cloud.CloudRuntime - ~multiprocess.MultiProcessRuntime - ~singleprocess.SingleProcessRuntime diff --git a/docs/source-app/code_samples/basics/0.py b/docs/source-app/code_samples/basics/0.py index 75959d9fc5..273ced0186 100644 --- a/docs/source-app/code_samples/basics/0.py +++ b/docs/source-app/code_samples/basics/0.py @@ -1,7 +1,7 @@ -import lightning as L +from lightning_app import LightningWork -class ExampleWork(L.LightningWork): +class ExampleWork(LightningWork): def run(self, *args, **kwargs): print(f"I received the following props: args: {args} kwargs: {kwargs}") diff --git a/docs/source-app/code_samples/basics/1.py b/docs/source-app/code_samples/basics/1.py index 3a249c8868..81f6f16cde 100644 --- a/docs/source-app/code_samples/basics/1.py +++ b/docs/source-app/code_samples/basics/1.py @@ -1,7 +1,7 @@ -import lightning as L +from lightning_app import LightningWork -class ExampleWork(L.LightningWork): +class ExampleWork(LightningWork): def __init__(self): super().__init__(cache_calls=False) diff --git a/docs/source-app/code_samples/convert_pl_to_app/app.py b/docs/source-app/code_samples/convert_pl_to_app/app.py deleted file mode 100644 index fe003dbb43..0000000000 --- a/docs/source-app/code_samples/convert_pl_to_app/app.py +++ /dev/null @@ -1,17 +0,0 @@ -import lightning as L -from lightning.app.components.python import TracerPythonScript - - -class RootFlow(L.LightningFlow): - def __init__(self): - super().__init__() - self.runner = TracerPythonScript( - "train.py", - cloud_compute=L.CloudCompute("gpu"), - ) - - def run(self): - self.runner.run() - - -app = L.LightningApp(RootFlow()) diff --git a/docs/source-app/code_samples/convert_pl_to_app/requirements.py b/docs/source-app/code_samples/convert_pl_to_app/requirements.py deleted file mode 100644 index e8fb43ef7d..0000000000 --- a/docs/source-app/code_samples/convert_pl_to_app/requirements.py +++ /dev/null @@ -1,3 +0,0 @@ -torch -torchvision -pytorch_lightning diff --git a/docs/source-app/code_samples/convert_pl_to_app/train.py b/docs/source-app/code_samples/convert_pl_to_app/train.py deleted file mode 100644 index 6dd3042577..0000000000 --- a/docs/source-app/code_samples/convert_pl_to_app/train.py +++ /dev/null @@ -1,46 +0,0 @@ -import os - -import torch -import torch.nn.functional as F -from torch import nn -from torch.utils.data import DataLoader, random_split -from torchvision import transforms as T -from torchvision.datasets import MNIST - -import pytorch_lightning as pl - - -class LitAutoEncoder(pl.LightningModule): - def __init__(self): - super().__init__() - self.encoder = nn.Sequential(nn.Linear(28 * 28, 128), nn.ReLU(), nn.Linear(128, 3)) - self.decoder = nn.Sequential(nn.Linear(3, 128), nn.ReLU(), nn.Linear(128, 28 * 28)) - - def forward(self, x): - # in lightning, - # forward defines the prediction/inference actions - embedding = self.encoder(x) - return embedding - - def training_step(self, batch, batch_idx): - # training_step defines the train loop. - # It is independent of forward - x, y = batch - x = x.view(x.size(0), -1) - z = self.encoder(x) - x_hat = self.decoder(z) - loss = F.mse_loss(x_hat, x) - self.log("train_loss", loss) - return loss - - def configure_optimizers(self): - optimizer = torch.optim.Adam(self.parameters(), lr=1e-3) - return optimizer - - -dataset = MNIST(os.getcwd(), download=True, transform=T.ToTensor()) -train, val = random_split(dataset, [55000, 5000]) - -autoencoder = LitAutoEncoder() -trainer = pl.Trainer(accelerator="auto") -trainer.fit(autoencoder, DataLoader(train), DataLoader(val)) diff --git a/docs/source-app/code_samples/quickstart/app/app_0.py b/docs/source-app/code_samples/quickstart/app/app_0.py index 3952cafc95..c12d2e87c1 100644 --- a/docs/source-app/code_samples/quickstart/app/app_0.py +++ b/docs/source-app/code_samples/quickstart/app/app_0.py @@ -1,10 +1,10 @@ from docs.quickstart.app_02 import HourLongWork -import lightning as L +from lightning_app import LightningApp, LightningFlow, LightningWork -class RootFlow(L.LightningFlow): - def __init__(self, child_work_1: L.LightningWork, child_work_2: L.LightningWork): +class RootFlow(LightningFlow): + def __init__(self, child_work_1: LightningWork, child_work_2: LightningWork): super().__init__() self.child_work_1 = child_work_1 self.child_work_2 = child_work_2 @@ -19,4 +19,4 @@ class RootFlow(L.LightningFlow): print("1 hour later `child_work_2` started!") -app = L.LightningApp(RootFlow(HourLongWork(parallel=True), HourLongWork(parallel=True))) +app = LightningApp(RootFlow(HourLongWork(parallel=True), HourLongWork(parallel=True))) diff --git a/docs/source-app/code_samples/quickstart/app/app_1.py b/docs/source-app/code_samples/quickstart/app/app_1.py index ac41c5ef83..1fe011882f 100644 --- a/docs/source-app/code_samples/quickstart/app/app_1.py +++ b/docs/source-app/code_samples/quickstart/app/app_1.py @@ -2,14 +2,14 @@ import flash from flash.core.data.utils import download_data from flash.image import ImageClassificationData, ImageClassifier -import lightning as L +from lightning_app import CloudCompute, LightningApp, LightningFlow, LightningWork from pytorch_lightning.callbacks import ModelCheckpoint # Step 1: Create a training LightningWork component that gets a backbone as input # and saves the best model and its score -class ImageClassifierTrainWork(L.LightningWork): - def __init__(self, max_epochs: int, backbone: str, cloud_compute: L.CloudCompute): +class ImageClassifierTrainWork(LightningWork): + def __init__(self, max_epochs: int, backbone: str, cloud_compute: CloudCompute): # parallel is set to True to run asynchronously super().__init__(parallel=True, cloud_compute=cloud_compute) # Number of epochs to run @@ -44,7 +44,7 @@ class ImageClassifierTrainWork(L.LightningWork): # Step 2: Create a serving LightningWork component that gets a model input and serves it -class ImageClassifierServeWork(L.LightningWork): +class ImageClassifierServeWork(LightningWork): def run(self, best_model_path: str): # Load the model from the model path model = ImageClassifier.load_from_checkpoint(best_model_path) @@ -53,7 +53,7 @@ class ImageClassifierServeWork(L.LightningWork): # Step 3: Create a root LightningFlow component that gets number of epochs and a path to # a dataset as inputs, initialize 2 training components and serves the best model -class RootFlow(L.LightningFlow): +class RootFlow(LightningFlow): def __init__(self, max_epochs: int, data_dir: str): super().__init__() self.data_dir = data_dir @@ -89,4 +89,4 @@ class RootFlow(L.LightningFlow): download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip", "./data") # Initalize your Lightning app with 5 epochs -app = L.LightningApp(RootFlow(5, "./data/hymenoptera_data")) +app = LightningApp(RootFlow(5, "./data/hymenoptera_data")) diff --git a/docs/source-app/code_samples/quickstart/app_01.py b/docs/source-app/code_samples/quickstart/app_01.py index 0a435c61c9..d4a4e6ee92 100644 --- a/docs/source-app/code_samples/quickstart/app_01.py +++ b/docs/source-app/code_samples/quickstart/app_01.py @@ -1,19 +1,19 @@ -import lightning as L -from lightning.app.utilities.app_helpers import pretty_state +from lightning_app import LightningApp, LightningFlow, LightningWork +from lightning_app.utilities.app_helpers import pretty_state -class Work(L.LightningWork): +class Work(LightningWork): def __init__(self): super().__init__(cache_calls=False) # Attributes are registered automatically in the state. self.counter = 0 def run(self): - # Incrementing an attribute gets reflected in the `Flow` state. + # Incrementing an attribute gets reflected in the `RootFlow` state. self.counter += 1 -class Flow(L.LightningFlow): +class Flow(LightningFlow): def __init__(self): super().__init__() self.w = Work() @@ -24,4 +24,4 @@ class Flow(L.LightningFlow): self.w.run() -app = L.LightningApp(Flow()) +app = LightningApp(Flow()) diff --git a/docs/source-app/code_samples/quickstart/app_02.py b/docs/source-app/code_samples/quickstart/app_02.py index e756019b92..bc2fe3d549 100644 --- a/docs/source-app/code_samples/quickstart/app_02.py +++ b/docs/source-app/code_samples/quickstart/app_02.py @@ -1,10 +1,10 @@ from time import sleep -import lightning as L +from lightning_app import LightningApp, LightningFlow, LightningWork # This work takes an hour to run -class HourLongWork(L.LightningWork): +class HourLongWork(LightningWork): def __init__(self, parallel: bool = False): super().__init__(parallel=parallel) self.progress = 0.0 @@ -16,8 +16,8 @@ class HourLongWork(L.LightningWork): sleep(1) -class RootFlow(L.LightningFlow): - def __init__(self, child_work: L.LightningWork): +class RootFlow(LightningFlow): + def __init__(self, child_work: LightningWork): super().__init__() self.child_work = child_work @@ -29,4 +29,4 @@ class RootFlow(L.LightningFlow): print("1 hour later!") -app = L.LightningApp(RootFlow(HourLongWork())) +app = LightningApp(RootFlow(HourLongWork())) diff --git a/docs/source-app/code_samples/quickstart/app_03.py b/docs/source-app/code_samples/quickstart/app_03.py index ee2047ae98..e841088b02 100644 --- a/docs/source-app/code_samples/quickstart/app_03.py +++ b/docs/source-app/code_samples/quickstart/app_03.py @@ -1,9 +1,9 @@ from time import sleep -import lightning as L +from lightning_app import LightningApp, LightningFlow, LightningWork -class HourLongWork(L.LightningWork): +class HourLongWork(LightningWork): def __init__(self): super().__init__(cache_calls=False) self.progress = 0.0 @@ -15,8 +15,8 @@ class HourLongWork(L.LightningWork): sleep(1) -class RootFlow(L.LightningFlow): - def __init__(self, child_work: L.LightningWork): +class RootFlow(LightningFlow): + def __init__(self, child_work: LightningWork): super().__init__() self.child_work = child_work @@ -28,4 +28,4 @@ class RootFlow(L.LightningFlow): print("1 hour later!") -app = L.LightningApp(RootFlow(HourLongWork())) +app = LightningApp(RootFlow(HourLongWork())) diff --git a/docs/source-app/code_samples/quickstart/app_comp.py b/docs/source-app/code_samples/quickstart/app_comp.py index 600badbd0d..35a0d4fc87 100644 --- a/docs/source-app/code_samples/quickstart/app_comp.py +++ b/docs/source-app/code_samples/quickstart/app_comp.py @@ -1,8 +1,8 @@ -import lightning as L -from lightning.app.testing.helpers import EmptyFlow, EmptyWork +from lightning_app import LightningApp, LightningFlow +from lightning_app.testing.helpers import EmptyFlow, EmptyWork -class FlowB(L.LightningFlow): +class FlowB(LightningFlow): def __init__(self): super().__init__() self.flow_d = EmptyFlow() @@ -12,7 +12,7 @@ class FlowB(L.LightningFlow): ... -class FlowA(L.LightningFlow): +class FlowA(LightningFlow): def __init__(self): super().__init__() self.flow_b = FlowB() @@ -23,4 +23,4 @@ class FlowA(L.LightningFlow): ... -app = L.LightningApp(FlowA()) +app = LightningApp(FlowA()) diff --git a/docs/source-app/code_samples/quickstart/hello_world/app.py b/docs/source-app/code_samples/quickstart/hello_world/app.py index 7514d90856..de82b5f351 100644 --- a/docs/source-app/code_samples/quickstart/hello_world/app.py +++ b/docs/source-app/code_samples/quickstart/hello_world/app.py @@ -1,8 +1,8 @@ -import lightning as L +from lightning_app import LightningApp, LightningFlow # Step 1: Subclass LightningFlow component to define the app flow. -class HelloWorld(L.LightningFlow): +class HelloWorld(LightningFlow): # Step 2: Add the app logic to the LightningFlow run method to # ``print("Hello World!")`. @@ -13,4 +13,4 @@ class HelloWorld(L.LightningFlow): # Step 3: Initalize a LightningApp with the LightningFlow you defined (in step 1) -app = L.LightningApp(HelloWorld()) +app = LightningApp(HelloWorld()) diff --git a/docs/source-app/code_samples/quickstart/hello_world/app_ui.py b/docs/source-app/code_samples/quickstart/hello_world/app_ui.py index 8ed669d81f..f19ede3113 100644 --- a/docs/source-app/code_samples/quickstart/hello_world/app_ui.py +++ b/docs/source-app/code_samples/quickstart/hello_world/app_ui.py @@ -1,12 +1,12 @@ import os -import lightning as L -from lightning.app.frontend import StaticWebFrontend, StreamlitFrontend -from lightning.app.utilities.state import AppState +from lightning_app import LightningApp, LightningFlow +from lightning_app.frontend import StaticWebFrontend, StreamlitFrontend +from lightning_app.utilities.state import AppState # Step 1: Define your LightningFlow component with the app UI -class UIStreamLit(L.LightningFlow): +class UIStreamLit(LightningFlow): def __init__(self): super().__init__() self.should_print = False @@ -31,7 +31,7 @@ def render_fn(state: AppState): # Step 4: Implement a Static Web Frontend. This could be react, vue, etc. -class UIStatic(L.LightningFlow): +class UIStatic(LightningFlow): # Step 5: def configure_layout(self): @@ -39,7 +39,7 @@ class UIStatic(L.LightningFlow): # Step 6: Implement the root flow. -class HelloWorld(L.LightningFlow): +class HelloWorld(LightningFlow): def __init__(self): super().__init__() self.static_ui = UIStatic() @@ -55,4 +55,4 @@ class HelloWorld(L.LightningFlow): ] -app = L.LightningApp(HelloWorld()) +app = LightningApp(HelloWorld()) diff --git a/docs/source-app/conf.py b/docs/source-app/conf.py index 18b5dcf0ff..be95b0d2b3 100644 --- a/docs/source-app/conf.py +++ b/docs/source-app/conf.py @@ -15,28 +15,32 @@ import inspect import os import shutil import sys +from importlib.util import module_from_spec, spec_from_file_location import pt_lightning_sphinx_theme -import lightning_app - _PATH_HERE = os.path.abspath(os.path.dirname(__file__)) _PATH_ROOT = os.path.realpath(os.path.join(_PATH_HERE, "..", "..")) sys.path.insert(0, os.path.abspath(_PATH_ROOT)) SPHINX_MOCK_REQUIREMENTS = int(os.environ.get("SPHINX_MOCK_REQUIREMENTS", True)) +# alternative https://stackoverflow.com/a/67692/4521646 +spec = spec_from_file_location("lightning_app/__about__.py", os.path.join(_PATH_ROOT, "lightning_app", "__about__.py")) +about = module_from_spec(spec) +spec.loader.exec_module(about) + # -- Project information ----------------------------------------------------- # this name shall match the project name in Github as it is used for linking to code project = "lightning" -copyright = lightning_app.__copyright__ -author = lightning_app.__author__ +copyright = about.__copyright__ +author = about.__author__ # The short X.Y version -version = lightning_app.__version__ +version = about.__version__ # The full version, including alpha/beta/rc tags -release = lightning_app.__version__ +release = about.__version__ # Options for the linkcode extension # ---------------------------------- @@ -156,8 +160,8 @@ html_theme_path = [pt_lightning_sphinx_theme.get_html_theme_path()] # documentation. html_theme_options = { - "pytorch_project": lightning_app.__homepage__, - "canonical_url": lightning_app.__homepage__, + "pytorch_project": about.__homepage__, + "canonical_url": about.__homepage__, "collapse_navigation": False, "display_version": True, "logo_only": False, @@ -223,7 +227,7 @@ texinfo_documents = [ project + " Documentation", author, project, - lightning_app.__docs__, + about.__docs__, "Miscellaneous", ), ] @@ -277,15 +281,6 @@ for path_ipynb in glob.glob(os.path.join(_PATH_ROOT, "notebooks", "*.ipynb")): path_ipynb2 = os.path.join(path_nbs, os.path.basename(path_ipynb)) shutil.copy(path_ipynb, path_ipynb2) -# copy all examples to local folder -path_examples = os.path.join(_PATH_HERE, "..", "examples") -if not os.path.isdir(path_examples): - os.mkdir(path_examples) -for path_app_example in glob.glob(os.path.join(_PATH_ROOT, "examples", "app_*")): - path_app_example2 = os.path.join(path_examples, os.path.basename(path_app_example)) - if not os.path.isdir(path_app_example2): - shutil.copytree(path_app_example, path_app_example2, dirs_exist_ok=True) - # Ignoring Third-party packages # https://stackoverflow.com/questions/15889621/sphinx-how-to-exclude-imports-in-automodule @@ -319,7 +314,7 @@ autodoc_mock_imports = MOCK_PACKAGES def linkcode_resolve(domain, info): def find_source(): # try to find the file and line number, based on code from numpy: - # https://github.com/numpy/numpy/blob/master/doc/source-app/conf.py#L286 + # https://github.com/numpy/numpy/blob/master/doc/source/conf.py#L286 obj = sys.modules[info["module"]] for part in info["fullname"].split("."): obj = getattr(obj, part) @@ -386,6 +381,6 @@ doctest_test_doctest_blocks = "" doctest_global_setup = """ import importlib import os -import lightning as L +import lightning_app """ coverage_skip_undoc_in_source = True diff --git a/docs/source-app/core_api/core_api.rst b/docs/source-app/core_api/core_api.rst deleted file mode 100644 index 594433acce..0000000000 --- a/docs/source-app/core_api/core_api.rst +++ /dev/null @@ -1,40 +0,0 @@ -:orphan: - -.. _core_api: - -############################### -Learn more about Lightning Core -############################### - -.. raw:: html - -
-
- -.. displayitem:: - :header: Level-up with Lightning Apps - :description: From Basics to Advanced Skills - :col_css: col-md-6 - :button_link: ../levels/basic/index.html - :height: 180 - -.. displayitem:: - :header: Understand Lightning App - :description: Detailed description - :col_css: col-md-6 - :button_link: lightning_app/index.html - :height: 180 - -.. displayitem:: - :header: Understand Lightning Flow - :description: Detailed description - :col_css: col-md-6 - :button_link: lightning_flow.html - :height: 180 - -.. displayitem:: - :header: Understand Lightning Work - :description: Detailed description - :col_css: col-md-6 - :button_link: lightning_work/index.html - :height: 180 diff --git a/docs/source-app/core_api/lightning_app/app.py b/docs/source-app/core_api/lightning_app/app.py deleted file mode 100644 index 0a435c61c9..0000000000 --- a/docs/source-app/core_api/lightning_app/app.py +++ /dev/null @@ -1,27 +0,0 @@ -import lightning as L -from lightning.app.utilities.app_helpers import pretty_state - - -class Work(L.LightningWork): - def __init__(self): - super().__init__(cache_calls=False) - # Attributes are registered automatically in the state. - self.counter = 0 - - def run(self): - # Incrementing an attribute gets reflected in the `Flow` state. - self.counter += 1 - - -class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.w = Work() - - def run(self): - if self.w.has_started: - print(f"State: {pretty_state(self.state)} \n") - self.w.run() - - -app = L.LightningApp(Flow()) diff --git a/docs/source-app/core_api/lightning_app/communication.rst b/docs/source-app/core_api/lightning_app/communication.rst index 9a823b0844..a8872585e5 100644 --- a/docs/source-app/core_api/lightning_app/communication.rst +++ b/docs/source-app/core_api/lightning_app/communication.rst @@ -1,15 +1,138 @@ :orphan: -########################################## -Communication between Lightning Components -########################################## +################################ +Communication Between Components +################################ **Audience:** Users that want to create interactive applications. -**Level:** Intermediate +**Level:** Advanced -**Prerequisite**: Read the `Communication in Lightning Apps article <../../access_app_state.html>`_. +**Prerequisite**: Read the :ref:`access_app_state` guide. ---- -.. include:: ../../core_api/lightning_app/communication_content.rst +*********************************** +Why should components communicate ? +*********************************** + +When creating interactive apps with multiple components, you might want your components to share information with each other. You might to rely on that information to control their execution, share progress in the UI, trigger a sequence of operations, etc. + +By design, the :class:`~lightning_app.core.flow.LightningFlow` communicates to all :class:`~lightning_app.core.flow.LightningWork` within the application, but :class:`~lightning_app.core.flow.LightningWork` can't communicate between each other directly, they need the flow as a proxy to do so. + +Once a ``LightningWork`` is running, any updates to its state is automatically communicated to the flow as a delta (using `DeepDiff `_). The state communication isn't bi-directional, it is only done from work to flow. + +Internally, the Lightning App is alternatively collecting deltas sent from all the registered ``LightningWorks`` and/or UI, and running the root flow run method of the app. + +******************************* +Communication From Work to Flow +******************************* + +Below, find an example to better understand this behavior. + +The ``WorkCounter`` increments a counter until 1 million and the ``Flow`` prints the work counter. + +As the work is running into its own process, its state changes is sent to the Flow which contains the latest value of the counter. + +.. code-block:: python + + import lightning_app as la + + + class WorkCounter(lapp.LightningWork): + def __init__(self): + super().__init__(parallel=True) + self.counter = 0 + + def run(self): + for _ in range(int(10e6)): + self.counter += 1 + + + class Flow(lapp.LightningFlow): + def __init__(self): + super().__init__() + self.w = WorkCounter() + + def run(self): + self.w.run() + print(self.w.counter) + + + app = lapp.LightningApp(Flow()) + + +A delta sent from the work to the flow looks like this: + +.. code-block:: python + + {"values_changed": {"root['works']['w']['vars']['counter']": {"new_value": 425}}} + +Here is the associated illustration: + +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/deltas.gif + :alt: Mechanism showing how delta are sent. + :width: 100 % + + +******************************* +Communication From From to Work +******************************* + +Communication from the flow to the work while running isn't support yet. If your application requires this feature, please open an issue on Github. + +.. code-block:: python + + import lightning_app as la + from time import sleep + + + class WorkCounter(lapp.LightningWork): + def __init__(self): + super().__init__(parallel=True) + self.counter = 0 + + def run(self): + while True: + sleep(1) + print(f"Work {self.counter}") + + + class Flow(lapp.LightningFlow): + def __init__(self): + super().__init__() + self.w = WorkCounter() + + def run(self): + self.w.run() + sleep(1) + print(f"Flow {self.w.counter}") + self.w.counter += 1 + + + app = lapp.LightningApp(Flow()) + +As you can observe, there is a divergence between the value within the Work and the Flow. + +.. code-block:: console + + Flow 0 + Flow 1 + Flow 2 + Flow 3 + Work 0 + Flow 4 + Work 0 + Flow 5 + Work 0 + Flow 6 + Work 0 + Flow 7 + Work 0 + Flow 8 + Work 0 + Flow 9 + Work 0 + Flow 10 + +.. note:: Technically, the flow and works relies on queues to share data (multiprocessing locally and redis lists in the cloud). diff --git a/docs/source-app/core_api/lightning_app/communication_content.rst b/docs/source-app/core_api/lightning_app/communication_content.rst deleted file mode 100644 index 8e251412f3..0000000000 --- a/docs/source-app/core_api/lightning_app/communication_content.rst +++ /dev/null @@ -1,160 +0,0 @@ - -******************************** -Communication Between Components -******************************** - -When creating interactive Lightning Apps (App) with multiple components, you may need your components to share information with each other and rely on that information to control their execution, share progress in the UI, trigger a sequence of operations, etc. - -To accomplish that, Lightning components can communicate using the App State. The App State is composed of all attributes defined within each component's **__init__** method e.g anything attached to the component with **self.x = y**. - -All attributes of all **LightningWork (Work)** components are accessible in the **LightningFlow (Flow)** components in real-time. - -By design, the Flows communicate to all **Works** within the application. However, Works can't communicate with each other directly, they must use Flows as a proxy to communicate. - -Once a Work is running, any updates to the Work's state is automatically communicated to the Flow, as a delta (using `DeepDiff `_). The state communication isn't bi-directional, communication is only done from Work to Flow. - -Internally, the App is alternatively collecting deltas sent from all the registered Works and/or UI, and running the root Flow run method of the App. - ----- - -************************************************* -Communication from LightningWork to LightningFlow -************************************************* - -LightningFlow (Flow) can access their children's LightningWork (Work) state. - -When a running Work attribute gets updated inside its method (separate process locally or remote machine), the app re-executes Flow's run method once it receives the state update from the Work. - -Here's an example to better understand communication from Work to Flow. - -The ``WorkCounter`` increments a counter until 1 million and the ``Flow`` prints the work counter. - -As the Work is running its own process, its state changes are sent to the Flow which contains the latest value of the counter. - -.. code-block:: python - - import lightning as L - - - class WorkCounter(L.LightningWork): - def __init__(self): - super().__init__(parallel=True) - self.counter = 0 - - def run(self): - for _ in range(int(10e6)): - self.counter += 1 - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.w = WorkCounter() - - def run(self): - self.w.run() - print(self.w.counter) - - - app = L.LightningApp(Flow()) - - -A delta sent from the Work to the Flow looks like this: - -.. code-block:: python - - {"values_changed": {"root['works']['w']['vars']['counter']": {"new_value": 425}}} - -Here is the associated illustration: - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/deltas.gif - :alt: Mechanism showing how delta are sent. - :width: 100 % - -Here's another example that is slightly different. Here we define a Flow and Work, where the Work increments a counter indefinitely and the Flow prints its state which contain the Work. - -You can easily check the state of your entire app as follows: - -.. literalinclude:: ../../core_api/lightning_app/app.py - -Run the app with: - -.. code-block:: bash - - lightning run app docs/source-app/core_api/lightning_app/app.py - -And here's the output you get when running the App using the **Lightning CLI**: - -.. code-block:: console - - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - State: {'works': {'w': {'vars': {'counter': 1}}}} - State: {'works': {'w': {'vars': {'counter': 2}}}} - State: {'works': {'w': {'vars': {'counter': 3}}}} - State: {'works': {'w': {'vars': {'counter': 3}}}} - State: {'works': {'w': {'vars': {'counter': 4}}}} - ... - ----- - -************************************************* -Communication from LightningFlow to LightningWork -************************************************* - -Communication from the LightningFlow (Flow) to the LightningWork (Work) while running **isn't supported yet**. If your application requires this feature, please open an issue on Github. - -Here's an example of what would happen if you try to have the Flow communicate with the Work: - -.. code-block:: python - - import lightning as L - from time import sleep - - - class WorkCounter(L.LightningWork): - def __init__(self): - super().__init__(parallel=True) - self.counter = 0 - - def run(self): - while True: - sleep(1) - print(f"Work {self.counter}") - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.w = WorkCounter() - - def run(self): - self.w.run() - sleep(1) - print(f"Flow {self.w.counter}") - self.w.counter += 1 - - - app = L.LightningApp(Flow()) - -As you can see, there is a divergence between the values within the Work and the Flow. - -.. code-block:: console - - Flow 0 - Flow 1 - Flow 2 - Flow 3 - Work 0 - Flow 4 - Work 0 - Flow 5 - Work 0 - Flow 6 - Work 0 - Flow 7 - Work 0 - Flow 8 - Work 0 - Flow 9 - Work 0 - Flow 10 diff --git a/docs/source-app/core_api/lightning_app/dynamic_work.rst b/docs/source-app/core_api/lightning_app/dynamic_work.rst index bf202aa590..b4ad9d1144 100644 --- a/docs/source-app/core_api/lightning_app/dynamic_work.rst +++ b/docs/source-app/core_api/lightning_app/dynamic_work.rst @@ -1,15 +1,187 @@ :orphan: -.. _dynamic_work: +############ +Dynamic Work +############ -##################### -Dynamic LightningWork -##################### +**Audience:** Users who want to learn how to create application which adapts to user demands. -**Audience:** Users who want to create applications that adapt to user demands. - -**Level:** Advanced +**Level:** Intermediate ---- -.. include:: dynamic_work_content.rst +*************************************************** +Why should I care about creating work dynamically ? +*************************************************** + +Imagine you want to create a research notebook app for your team, where every member can create multiple `JupyterLab `_ session on their hardware of choice. + +To allow every notebook to choose hardware, it needs to be set up in it's own :class:`~lightning_app.core.work.LightningWork`, but you can't know the number of notebooks user will need in advance. In this case you'll need to add ``LightningWorks`` dynamically at run time. + +This is what **dynamic works** enables. + +*************************** +When to use dynamic works ? +*************************** + +Dynamic works should be used anytime you want change the resources your application is using at runtime. + +******************* +How to add a work ? +******************* + +You can simply attach your components in the **run** method of a flow using python **hasattr**, **setattr** and **getattr** functions. + +.. code-block:: python + + class RootFlow(lapp.LightningFlow): + def run(self): + + if not hasattr(self, "work"): + setattr(self, "work", Work()) # The `Work` component is created and attached here. + getattr(self, "work").run() # Run the `Work` component. + +But it is usually more readable to use Lightning built-in :class:`~lightning_app.structures.Dict` or :class:`~lightning_app.structures.List` as follows: + +.. code-block:: python + + from lightning_app.structures import Dict + + + class RootFlow(lapp.LightningFlow): + def __init__(self): + super().__init__() + self.dict = Dict() + + def run(self): + if "work" not in self.dict: + self.dict["work"] = Work() # The `Work` component is attached here. + self.dict["work"].run() + + +******************** +How to stop a work ? +******************** + +In order to stop a work, simply use the work ``stop`` method as follows: + +.. code-block:: python + + class RootFlow(lapp.LightningFlow): + def __init__(self): + super().__init__() + self.work = Work() + + def run(self): + self.work.stop() + + +********************************** +Application Example with StreamLit +********************************** + +.. + The entire application can be found `here `_. + +The Notebook Manager +^^^^^^^^^^^^^^^^^^^^ + +In the component below, we are dynamically creating ``JupyterLabWork`` every time as user clicks the ``Create Jupyter Notebook`` button. + +To do so, we are iterating over the list of ``jupyter_config_requests`` infinitely. + +.. code-block:: python + + import lightning_app as la + + + class JupyterLabManager(lapp.LightningFlow): + """This flow manages the users notebooks running within works.""" + + def __init__(self): + super().__init__() + self.jupyter_works = lapp.structures.Dict() + self.jupyter_config_requests = [] + + def run(self): + for idx, jupyter_config in enumerate(self.jupyter_config_requests): + + # The Jupyter Config has this form is: + # {"use_gpu": False/True, "token": None, "username": ..., "stop": False} + + # Step 1: Check if JupyterWork already exists for this username + username = jupyter_config["username"] + if username not in self.jupyter_works: + jupyter_config["ready"] = False + + # Set the hardware selected by the user: GPU or CPU. + cloud_compute = lapp.CloudCompute("gpu" if jupyter_config["use_gpu"] else "cpu-small") + + # Step 2: Create new JupyterWork dynamically ! + self.jupyter_works[username] = JupyterLabWork(cloud_compute=cloud_compute) + + # Step 3: Run the JupyterWork + self.jupyter_works[username].run() + + # Step 4: Store the notebook token in the associated config. + # We are using this to know when the notebook is ready + # and display the stop button on the UI. + if self.jupyter_works[username].token: + jupyter_config["token"] = self.jupyter_works[username].token + + # Step 5: Stop the work if the user requested it. + if jupyter_config["stop"]: + self.jupyter_works[username].stop() + self.jupyter_config_requests.pop(idx) + + def configure_layout(self): + return StreamlitFrontend(render_fn=render_fn) + + +The StreamLit UI +^^^^^^^^^^^^^^^^ + +In the UI below, we receive the **state** of the Jupyter Manager and it can be modified directly from the UI interaction. + +.. code-block:: python + + def render_fn(state): + import streamlit as st + + # Step 1: Enable users to select their notebooks and create them + column_1, column_2, column_3 = st.columns(3) + with column_1: + create_jupyter = st.button("Create Jupyter Notebook") + with column_2: + username = st.text_input("Enter your username", "tchaton") + assert username + with column_3: + use_gpu = st.checkbox("Use GPU") + + # Step 2: If a user clicked the button, add an element to the list of configs + # Note: state.jupyter_config_requests = ... will sent the state update to the component. + if create_jupyter: + new_config = [{"use_gpu": use_gpu, "token": None, "username": username, "stop": False}] + state.jupyter_config_requests = state.jupyter_config_requests + new_config + + # Step 3: List of running notebooks. + for idx, config in enumerate(state.jupyter_config_requests): + column_1, column_2, column_3 = st.columns(3) + with column_1: + if not idx: + st.write(f"Idx") + st.write(f"{idx}") + with column_2: + if not idx: + st.write(f"Use GPU") + st.write(config["use_gpu"]) + with column_3: + if not idx: + st.write(f"Stop") + if config["token"]: + should_stop = st.button("Stop this notebook") + + # Step 4: Change stop if the user clicked the button + if should_stop: + config["stop"] = should_stop + state.jupyter_config_requests = state.jupyter_config_requests diff --git a/docs/source-app/core_api/lightning_app/dynamic_work_content.rst b/docs/source-app/core_api/lightning_app/dynamic_work_content.rst deleted file mode 100644 index d1f76d4b1f..0000000000 --- a/docs/source-app/core_api/lightning_app/dynamic_work_content.rst +++ /dev/null @@ -1,202 +0,0 @@ -*************************************** -What Dynamic LightningWork does for you -*************************************** - -Dynamic LightningWork (Work) changes the resources your application uses while the application is running (aka at runtime). - -For example, imagine you want to create a research notebook app for your team. You want every member to be able to create multiple `JupyterLab `_ sessions on their hardware of choice. - -To allow every notebook to choose hardware, it needs to be set up in it's own :class:`~lightning_app.core.work.LightningWork`, but you can't know the number of notebooks user will need in advance. In this case you'll need to add ``LightningWorks`` dynamically at run time. - ----- - -***************** -Use Dynamic Works -***************** - -Dynamic Works should be used anytime you want change the resources your application is using while it is running (aka at runtime). - -You're usually going to use the ``start`` and ``stop`` methods together. - ----- - -Add a Dynamic Work -^^^^^^^^^^^^^^^^^^ - -There are a couple of ways you can add a dynamic Work: - -- Option 1: Attach your components in the **run** method using the Python functions. -- Option 2: Use the Lightning built-in classes :class:`~lightning.structures.Dict` or :class:`~lightning.structures.List`. - -.. note:: Using the Lightning built-in classes is usually easier to read. - ----- - -**OPTION 1:** Attach your components in the run method of a flow using the Python functions **hasattr**, **setattr**, and **getattr**: - -.. code-block:: python - - class RootFlow(lapp.LightningFlow): - - def run(self): - - if not hasattr(self, "work"): - # The `Work` component is created and attached here. - setattr(self, "work", Work()) - # Run the `Work` component. - getattr(self, "work").run() - -**OPTION 2:** Use the built-in Lightning classes :class:`~lightning_app.structures.Dict` or :class:`~lightning_app.structures.List` - -.. code-block:: python - - from lightning_app.structures import Dict - - class RootFlow(lapp.LightningFlow): - - def __init__(self): - super().__init__() - self.dict = Dict() - - def run(self): - if "work" not in self.dict: - # The `Work` component is attached here. - self.dict["work"] = Work() - self.dict["work"].run() - ----- - -Stop a Work -^^^^^^^^^^^ -Stop a work when you are concerned about cost. - -To stop a work, use the work ``stop`` method: - -.. code-block:: python - - class RootFlow(L.LightningFlow): - - def __init__(self): - super().__init__() - self.work = Work() - - def run(self): - self.work.stop() - ----- - -********************* -Dynamic Work Examples -********************* - -.. - The entire application can be found `here `_. - ----- - -Dynamic Work with Jupyter Notebooks -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In this example, we are dynamically creating ``JupyterLabWork`` every time a user clicks the **Create Jupyter Notebook** button. - -In order to do that, we are iterating over the list of ``jupyter_config_requests`` infinitely. - -.. code-block:: python - - import lightning as L - - - class JupyterLabManager(L.LightningFlow): - - """This flow manages the users notebooks running within works."""" - - def __init__(self): - super().__init__() - self.jupyter_works = L.structures.Dict() - self.jupyter_config_requests = [] - - def run(self): - for idx, jupyter_config in enumerate(self.jupyter_config_requests): - - # The Jupyter Config has this form is: - # {"use_gpu": False/True, "token": None, "username": ..., "stop": False} - - # Step 1: Check if JupyterWork already exists for this username - username = jupyter_config["username"] - if username not in self.jupyter_works: - jupyter_config["ready"] = False - - # Set the hardware selected by the user: GPU or CPU. - cloud_compute = L.CloudCompute("gpu" if jupyter_config["use_gpu"] else "cpu-small") - - # Step 2: Create new JupyterWork dynamically ! - self.jupyter_works[username] = JupyterLabWork(cloud_compute=cloud_compute) - - # Step 3: Run the JupyterWork - self.jupyter_works[username].run() - - # Step 4: Store the notebook token in the associated config. - # We are using this to know when the notebook is ready - # and display the stop button on the UI. - if self.jupyter_works[username].token: - jupyter_config["token"] = self.jupyter_works[username].token - - # Step 5: Stop the work if the user requested it. - if jupyter_config['stop']: - self.jupyter_works[username].stop() - self.jupyter_config_requests.pop(idx) - - def configure_layout(self): - return L.app.frontend.StreamlitFrontend(render_fn=render_fn) - ----- - -Dynamic Works with StreamLit UI -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Continuing from the Jupyter Notebook example, in the UI, we receive the **state** of the Jupyter Manager and the state can be modified directly from the UI. - -.. code-block:: python - - import streamlit as st - - - def render_fn(state): - - # Step 1: Enable users to select their notebooks and create them - column_1, column_2, column_3 = st.columns(3) - with column_1: - create_jupyter = st.button("Create Jupyter Notebook") - with column_2: - username = st.text_input('Enter your username', "tchaton") - assert username - with column_3: - use_gpu = st.checkbox('Use GPU') - - # Step 2: If a user clicked the button, add an element to the list of configs - # Note: state.jupyter_config_requests = ... will sent the state update to the component. - if create_jupyter: - new_config = [{"use_gpu": use_gpu, "token": None, "username": username, "stop": False}] - state.jupyter_config_requests = state.jupyter_config_requests + new_config - - # Step 3: List of running notebooks. - for idx, config in enumerate(state.jupyter_config_requests): - column_1, column_2, column_3 = st.columns(3) - with column_1: - if not idx: - st.write(f"Idx") - st.write(f"{idx}") - with column_2: - if not idx: - st.write(f"Use GPU") - st.write(config['use_gpu']) - with column_3: - if not idx: - st.write(f"Stop") - if config["token"]: - should_stop = st.button("Stop this notebook") - - # Step 4: Change stop if the user clicked the button - if should_stop: - config["stop"] = should_stop - state.jupyter_config_requests = state.jupyter_config_requests diff --git a/docs/source-app/core_api/lightning_app/lightning_app.rst b/docs/source-app/core_api/lightning_app/lightning_app.rst index 497dde20d2..5415f0d9dc 100644 --- a/docs/source-app/core_api/lightning_app/lightning_app.rst +++ b/docs/source-app/core_api/lightning_app/lightning_app.rst @@ -7,6 +7,7 @@ LightningApp ############ +The :class:`~lightning_app.core.app.LightningApp` runs a tree of one or more components that interact to create end-to-end applications. There are two kinds of components: :class:`~lightning_app.core.flow.LightningFlow` and :class:`~lightning_app.core.work.LightningWork`. This modular design enables you to reuse components created by other users. + .. autoclass:: lightning_app.core.app.LightningApp - :exclude-members: _run, connect, get_component_by_name, maybe_apply_changes, set_state :noindex: diff --git a/docs/source-app/core_api/lightning_flow.rst b/docs/source-app/core_api/lightning_flow.rst index a6ee8b2eca..6776bfefcb 100644 --- a/docs/source-app/core_api/lightning_flow.rst +++ b/docs/source-app/core_api/lightning_flow.rst @@ -4,5 +4,8 @@ LightningFlow ############# +The :class:`~lightning_app.core.flow.LightningFlow` component coordinates long-running tasks :class:`~lightning_app.core.work.LightningWork` and runs its children :class:`~lightning_app.core.flow.LightningFlow` components. + + .. autoclass:: lightning_app.core.flow.LightningFlow - :exclude-members: _attach_backend, _exit, _is_state_attribute, set_state + :noindex: diff --git a/docs/source-app/core_api/lightning_work/compute.rst b/docs/source-app/core_api/lightning_work/compute.rst index 89313c4878..f41ca08380 100644 --- a/docs/source-app/core_api/lightning_work/compute.rst +++ b/docs/source-app/core_api/lightning_work/compute.rst @@ -8,8 +8,103 @@ Customize your Cloud Compute **Audience:** Users who want to select the hardware to run in the cloud. -**Level:** Intermediate +**Level:** Basic ---- -.. include:: compute_content.rst +*************************************** +How can I customize my Work resources ? +*************************************** + +In the cloud, you can simply configure which machine to run on by passing +a :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute` to your work ``__init__`` method: + +.. code-block:: python + + import lightning_app as la + + # Run on a free, shared CPU machine. This is the default for every LightningWork. + MyCustomWork(cloud_compute=lapp.CloudCompute()) + + # Run on a dedicated, medium-size CPU machine (see specs below) + MyCustomWork(cloud_compute=lapp.CloudCompute("cpu-medium")) + + # Run on cheap GPU machine with a single GPU (see specs below) + MyCustomWork(cloud_compute=lapp.CloudCompute("gpu")) + + # Run on a fast multi-GPU machine (see specs below) + MyCustomWork(cloud_compute=lapp.CloudCompute("gpu-fast-multi")) + + +Here is the full list of supported machine names: + +.. list-table:: Hardware by Accelerator Type + :widths: 25 25 25 25 + :header-rows: 1 + + * - Name + - # of CPUs + - GPUs + - Memory + * - default + - 2 + - 0 + - 3 GB + * - cpu-small + - 2 + - 0 + - 8 GB + * - cpu-medium + - 8 + - 0 + - 32 GB + * - gpu + - 4 + - 1 (T4, 16 GB) + - 16 GB + * - gpu-fast + - 8 + - 1 (V100, 16 GB) + - 61 GB + * - gpu-fast-multi + - 32 + - 4 (V100 16 GB) + - 244 GB + +The up-to-date prices for these instances can be found `here `_. + + +******************************************* +How can I run on spot/preemptible machine ? +******************************************* + +Most cloud provider offers ``preemptible`` (synonym of ``spot``) machine which are usually discounted up to 90 %. Those machines are cheaper but the cloud provider can retrieve them at any time. + +.. code-block:: python + + import lightning_app as la + + # Run on a single CPU + MyCustomWork(cloud_compute=lapp.CloudCompute("gpu", preemptible=True)) + + +*********************************** +How can I stop my work when idle ? +*********************************** + +By providing **idle_timeout=X Seconds**, the work is automatically stopped **X seconds** after doing nothing. + +.. code-block:: python + + import lightning_app as la + + # Run on a single CPU and turn down immediately when idle. + MyCustomWork(cloud_compute=lapp.CloudCompute("gpu", idle_timeout=0)) + + +############# +CloudCompute +############# + +.. autoclass:: lightning_app.utilities.packaging.cloud_compute.CloudCompute + :noindex: diff --git a/docs/source-app/core_api/lightning_work/compute_content.rst b/docs/source-app/core_api/lightning_work/compute_content.rst deleted file mode 100644 index 68853c949e..0000000000 --- a/docs/source-app/core_api/lightning_work/compute_content.rst +++ /dev/null @@ -1,100 +0,0 @@ - -*************************** -Customize my Work resources -*************************** - -In the cloud, you can simply configure which machine to run on by passing -a :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute` to your work ``__init__`` method: - -.. code-block:: python - - import lightning as L - - # Run on a free, shared CPU machine. This is the default for every LightningWork. - MyCustomWork(cloud_compute=L.CloudCompute()) - - # Run on a dedicated, medium-size CPU machine (see specs below) - MyCustomWork(cloud_compute=L.CloudCompute("cpu-medium")) - - # Run on cheap GPU machine with a single GPU (see specs below) - MyCustomWork(cloud_compute=L.CloudCompute("gpu")) - - # Run on a fast multi-GPU machine (see specs below) - MyCustomWork(cloud_compute=L.CloudCompute("gpu-fast-multi")) - - -Here is the full list of supported machine names: - -.. list-table:: Hardware by Accelerator Type - :widths: 25 25 25 25 - :header-rows: 1 - - * - Name - - # of CPUs - - GPUs - - Memory - * - default - - 2 - - 0 - - 3 GB - * - cpu-small - - 2 - - 0 - - 8 GB - * - cpu-medium - - 8 - - 0 - - 32 GB - * - gpu - - 4 - - 1 (T4, 16 GB) - - 16 GB - * - gpu-fast - - 8 - - 1 (V100, 16 GB) - - 61 GB - * - gpu-fast-multi - - 32 - - 4 (V100 16 GB) - - 244 GB - -The up-to-date prices for these instances can be found `here `_. - ----- - -******************************* -Run on spot/preemptible machine -******************************* - -Most cloud provider offers ``preemptible`` (synonym of ``spot``) machines that are usually discounted by up to 90 %. Those machines are cheaper but the cloud provider can retrieve them at any time and might take longer to be ready. - -.. code-block:: python - - import lightning as L - - # Run on a single CPU - MyCustomWork(cloud_compute=L.CloudCompute("gpu", preemptible=True)) - ----- - -********************** -Stop my work when idle -********************** - -By providing **idle_timeout=X Seconds**, the work is automatically stopped **X seconds** after doing nothing. - -.. code-block:: python - - import lightning as L - - # Run on a single CPU and turn down immediately when idle. - MyCustomWork(cloud_compute=L.CloudCompute("gpu", idle_timeout=0)) - ----- - -************ -CloudCompute -************ - -.. autoclass:: lightning_app.utilities.packaging.cloud_compute.CloudCompute - :noindex: diff --git a/docs/source-app/core_api/lightning_work/handling_app_exception.rst b/docs/source-app/core_api/lightning_work/handling_app_exception.rst index 20c9b618d9..60f432e817 100644 --- a/docs/source-app/core_api/lightning_work/handling_app_exception.rst +++ b/docs/source-app/core_api/lightning_work/handling_app_exception.rst @@ -1,13 +1,83 @@ :orphan: -############################### -Handle Lightning App exceptions -############################### +######################## +Handling App Exceptions +######################## -**Audience:** Users who want to make Lightning Apps more robust to potential issues. +**Audience:** Users who want to know how to implement app where errors are handled. **Level:** Advanced ---- -.. include:: handling_app_exception_content.rst +************************************************* +Why should I care about handling app exceptions ? +************************************************* + +Imagine you are creating an application where your team can launch model training by providing their own Github Repo any time they want. + +As the application admin, you don't want the application to go down if their code has a bug and breaks. + +Instead, you would like the work to capture the exception and surface this to the users on failures. + +**************************************** +How can I configure exception handling ? +**************************************** + + +The LightningWork accepts an argument **raise_exception** which is **True** by default. This aligns with Python default behaviors. + +However, for the user case stated above, we want to capture the work exceptions. This is done by providing ``raise_exception=False`` to the work ``__init__`` method. + +.. code-block:: python + + MyCustomWork(raise_exception=False) # <== HERE: The exception is captured. + + # Default behavior + MyCustomWork(raise_exception=True) # <== HERE: The exception is raised within the flow and terminates the app + + +And you can customize this behavior by overriding the ``on_exception`` hook to the Lightning Work. + +.. code-block:: python + + import lightning as L + + + class MyCustomWork(L.LightningWork): + def on_exception(self, exception: Exception): + # do something when an exception is triggered. + pass + + +******************* +Application Example +******************* + +This is the pseudo-code for the application described above. + +.. code-block:: python + + import lightning_app as lapp + + + class RootFlow(lapp.LightningFlow): + def __init__(self): + super().__init__() + self.user_jobs = lapp.structures.Dict() + self.requested_jobs = [] + + def run(self): + for request in self.requested_jobs: + job_id = request["id"] + if job_id not in self.user_jobs: + # Note: The `GithubRepoLauncher` doesn't exist yet. + self.user_jobs[job_id] = GithubRepoLauncher( + **request, + raise_exception=False, # <== HERE: The exception is captured. + ) + self.user_jobs[job_id].run() + + if self.user_jobs[job_id].status.stage == "failed" and "printed" not in request: + print(self.user_jobs[job_id].status) # <== HERE: Print the user exception. + request["printed"] = True diff --git a/docs/source-app/core_api/lightning_work/handling_app_exception_content.rst b/docs/source-app/core_api/lightning_work/handling_app_exception_content.rst deleted file mode 100644 index 4840cf5fdf..0000000000 --- a/docs/source-app/core_api/lightning_work/handling_app_exception_content.rst +++ /dev/null @@ -1,74 +0,0 @@ - -*************************************************** -What handling Lightning App exceptions does for you -*************************************************** - -Imagine you are creating a Lightning App (App) where your team can launch model training by providing their own Github Repo any time they want. - -As the App admin, you don't want the App to go down if their code has a bug and breaks. - -Instead, you would like the LightningWork (Work) to capture the exception and present the issue to users. - ----- - -**************************** -Configure exception handling -**************************** - -The LightningWork (Work) accepts an argument **raise_exception** which is **True** by default. This aligns with Python default behaviors. - -However, for the user case stated in the previous section, we want to capture the Work exceptions. This is done by providing ``raise_exception=False`` to the work ``__init__`` method. - -.. code-block:: python - - import lightning as L - - MyCustomWork(raise_exception=False) # <== HERE: The exception is captured. - - # Default behavior - MyCustomWork(raise_exception=True) # <== HERE: The exception is raised within the flow and terminates the app - - -And you can customize this behavior by overriding the ``on_exception`` hook to the Work. - -.. code-block:: python - - import lightning as L - - class MyCustomWork(L.LightningWork): - - def on_exception(self, exception: Exception): - # do something when an exception is triggered. - ----- - -************************** -Exception handling example -************************** - -This is the pseudo-code for the application described above. - -.. code-block:: python - - import lightning as L - - class RootFlow(L.LightningFlow): - def __init__(self): - super().__init__() - self.user_jobs = L.structures.Dict() - self.requested_jobs = [] - - def run(self): - for request in self.requested_jobs: - job_id = request["id"] - if job_id not in self.user_jobs: - # Note: The `GithubRepoLauncher` doesn't exist yet. - self.user_jobs[job_id] = GithubRepoLauncher( - **request, - raise_exception=False, # <== HERE: The exception is captured. - ) - self.user_jobs[job_id].run() - - if self.user_jobs[job_id].status.stage == "failed" and "printed" not in request: - print(self.user_jobs[job_id].status) # <== HERE: Print the user exception. - request["printed"] = True diff --git a/docs/source-app/core_api/lightning_work/lightning_work.rst b/docs/source-app/core_api/lightning_work/lightning_work.rst index f4f490dd1f..8a1d3593d6 100644 --- a/docs/source-app/core_api/lightning_work/lightning_work.rst +++ b/docs/source-app/core_api/lightning_work/lightning_work.rst @@ -6,6 +6,7 @@ LightningWork ############# +The :class:`~lightning_app.core.work.LightningWork` component is a building block optimized for long-running jobs or integrating third-party services. LightningWork can be used for training large models, downloading a dataset, or any long-lasting operation. + .. autoclass:: lightning_app.core.work.LightningWork - :exclude-members: _aggregate_status_timeout, _is_state_attribute, _is_state_attribute, set_state :noindex: diff --git a/docs/source-app/core_api/lightning_work/payload.rst b/docs/source-app/core_api/lightning_work/payload.rst index b23bf7ca32..5d6ab1f20d 100644 --- a/docs/source-app/core_api/lightning_work/payload.rst +++ b/docs/source-app/core_api/lightning_work/payload.rst @@ -1,15 +1,87 @@ :orphan: -###################################### -Sharing Objects between LightningWorks -###################################### +############################# +Sharing Objects between Works +############################# -**Audience:** Users who want to know how to transfer Python objects between their LightningWorks. +**Audience:** Users who want to know how to transfer python objects between their works. **Level:** Advanced -**Prerequisite**: Reach Level 16+, know about the `pandas DataFrames `_ and read and read the `Access app state guide <../../access_app_state.html>`_. +**Prerequisite**: Know about the pandas library and read the :ref:`access_app_state` guide. ---- -.. include:: payload_content.rst +************************************ +When do I need to transfer objects ? +************************************ + +Imagine your application is processing some data using `pandas DaFrame `_ and you want to pass those data to another work. This is when and what the **Payload API** is meant for. + + +************************************* +How can I use the Lightning Payload ? +************************************* + +The Payload enables non JSON-serializable attribute objects to be part of your work state and be communicated to other works. + +Here is an example how to use it: + +.. code-block:: python + + import lightning_app as la + import pandas as pd + + + class SourceWork(lapp.LightningWork): + def __init__(self): + super().__init__() + self.df = None + + def run(self): + # do some processing + + df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + + # The object you care about needs to be wrapped into a Payload object. + self.df = lapp.storage.Payload(df) + + # You can access the original object from the payload using its value property. + print("src", self.df.value) + # src col1 col2 + # 0 1 3 + # 1 2 4 + +Once the Payload object is attached to your work state, it can be passed to another work via the flow as follows: + +.. code-block:: python + + import lightning_app as la + import pandas as pd + + + class DestinationWork(lapp.LightningWork): + def run(self, df: lapp.storage.Payload): + # You can access the original object from the payload using its value property. + print("dst", df.value) + # dst col1 col2 + # 0 1 3 + # 1 2 4 + + + class Flow(lapp.LightningFlow): + def __init__(self): + super().__init__() + self.src = SourceWork() + self.dst = DestinationWork() + + def run(self): + self.src.run() + # The pandas DataFrame created by the ``SourceWork`` + # is passed to the ``DestinationWork``. + # Internally, Lightning pickles and un-pickle the python object, + # so you receive a copy of the original object. + self.dst.run(df=self.src.df) + + + app = lapp.LightningApp(Flow()) diff --git a/docs/source-app/core_api/lightning_work/payload_content.rst b/docs/source-app/core_api/lightning_work/payload_content.rst deleted file mode 100644 index 780f3985e3..0000000000 --- a/docs/source-app/core_api/lightning_work/payload_content.rst +++ /dev/null @@ -1,75 +0,0 @@ - -************************************** -What transferring objects does for you -************************************** - -Imagine your application is processing some data using `pandas DaFrame `_ and you want to pass that data to another LightningWork (Work). This is what the **Payload API** is meant for. - ----- - -************************* -Use the Lightning Payload -************************* - -The Payload enables non JSON-serializable attribute objects to be part of your Work's state and to be communicated to other Works. - -Here is an example: - -.. code-block:: python - - import lightning as L - import pandas as pd - - - class SourceWork(L.LightningWork): - def __init__(self): - super().__init__() - self.df = None - - def run(self): - # do some processing - - df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - - # The object you care about needs to be wrapped into a Payload object. - self.df = L.storage.Payload(df) - - # You can access the original object from the payload using its value property. - print("src", self.df.value) - # src col1 col2 - # 0 1 3 - # 1 2 4 - -Once the Payload object is attached to your Work's state, it can be passed to another work using the LightningFlow (Flow) as follows: - -.. code-block:: python - - import lightning as L - import pandas as pd - - - class DestinationWork(L.LightningWork): - def run(self, df: L.storage.Payload): - # You can access the original object from the payload using its value property. - print("dst", df.value) - # dst col1 col2 - # 0 1 3 - # 1 2 4 - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.src = SourceWork() - self.dst = DestinationWork() - - def run(self): - self.src.run() - # The pandas DataFrame created by the ``SourceWork`` - # is passed to the ``DestinationWork``. - # Internally, Lightning pickles and un-pickle the python object, - # so you receive a copy of the original object. - self.dst.run(df=self.src.df) - - - app = L.LightningApp(Flow()) diff --git a/docs/source-app/core_api/lightning_work/status.rst b/docs/source-app/core_api/lightning_work/status.rst index af3a27ac40..ff7322b6c9 100644 --- a/docs/source-app/core_api/lightning_work/status.rst +++ b/docs/source-app/core_api/lightning_work/status.rst @@ -1,8 +1,9 @@ :orphan: -#################### -LightningWork Status -#################### + +##################### +Lightning Work Status +##################### **Audience:** Users who want to understand ``LightningWork`` under the hood. @@ -10,4 +11,199 @@ LightningWork Status ---- -.. include:: status_content.rst +******************* +What are statuses ? +******************* + +Statuses indicates transition points in the life of a Lightning Work and contain metadata. + +The different stages are: + +.. code-block:: python + + class WorkStageStatus: + NOT_STARTED = "not_started" + STOPPED = "stopped" + PENDING = "pending" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + +And a single status is as follows: + +.. code-block:: python + + @dataclass + class WorkStatus: + stage: WorkStageStatus + timestamp: float + reason: Optional[str] = None + message: Optional[str] = None + count: int = 1 + + +On creation, the work's status flags all evaluate to ``False`` (in particular ``has_started``) and when calling ``work.run`` in your flow, +the work transition from ``is_pending`` to ``is_running`` and then to ``has_succeeded`` if everything when well or ``has_failed`` otherwise. + +.. code-block:: python + + from time import sleep + import lightning_app as la + + + class Work(lapp.LightningWork): + def run(self, value: int): + sleep(1) + if value == 0: + return + raise Exception(f"The provided value was {value}") + + + class Flow(lapp.LightningFlow): + def __init__(self): + super().__init__() + self.work = Work(raise_exception=False) + self.counter = 0 + + def run(self): + if not self.work.has_started: + print("NOT STARTED") + + elif self.work.is_pending: + print("PENDING") + + elif self.work.is_running: + print("RUNNING") + + elif self.work.has_succeeded: + print("SUCCESS") + + elif self.work.has_failed: + print("FAILED") + + elif self.work.has_stopped: + print("STOPPED") + self._exit() + + print(self.work.status) + self.work.run(self.counter) + self.counter += 1 + + + app = lapp.LightningApp(Flow()) + +Run this app as follows: + +.. code-block:: bash + + lightning run app test.py > app_log.txt + +And here is the expected output inside ``app_log.txt`` and as expected, +we are observing the following transition ``has_started``, ``is_pending``, ``is_running``, ``has_succeeded``, ``is_running`` and ``has_failed`` + +.. code-block:: console + + NOT STARTED + WorkStatus(stage='not_started', timestamp=1653498225.18468, reason=None, message=None, count=1) + PENDING + WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) + PENDING + WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) + PENDING + ... + PENDING + WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) + PENDING + WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) + RUNNING + WorkStatus(stage='running', timestamp=1653498228.825194, reason=None, message=None, count=1) + ... + SUCCESS + WorkStatus(stage='succeeded', timestamp=1653498229.831793, reason=None, message=None, count=1) + SUCCESS + WorkStatus(stage='succeeded', timestamp=1653498229.831793, reason=None, message=None, count=1) + SUCCESS + WorkStatus(stage='succeeded', timestamp=1653498229.831793, reason=None, message=None, count=1) + RUNNING + WorkStatus(stage='running', timestamp=1653498229.846451, reason=None, message=None, count=1) + RUNNING + ... + WorkStatus(stage='running', timestamp=1653498229.846451, reason=None, message=None, count=1) + RUNNING + WorkStatus(stage='running', timestamp=1653498229.846451, reason=None, message=None, count=1) + FAILED + WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) + FAILED + WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) + FAILED + WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) + FAILED + WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) + ... + +In order to access all statuses, simply do: + +.. code-block:: python + + from time import sleep + import lightning_app as la + + + class Work(lapp.LightningWork): + def run(self, value: int): + sleep(1) + if value == 0: + return + raise Exception(f"The provided value was {value}") + + + class Flow(lapp.LightningFlow): + def __init__(self): + super().__init__() + self.work = Work(raise_exception=False) + self.counter = 0 + + def run(self): + print(self.statuses) + self.work.run(self.counter) + self.counter += 1 + + + app = lapp.LightningApp(Flow()) + + +Run this app as follows: + +.. code-block:: bash + + lightning run app test.py > app_log.txt + +And here is the expected output inside ``app_log.txt``: + + +.. code-block:: console + + # First execution with value = 0 + + [] + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1)] + ... + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1)] + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1)] + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] + ... + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1), WorkStatus(stage='succeeded', timestamp=1653498627.191053, reason=None, message=None, count=1)] + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1), WorkStatus(stage='succeeded', timestamp=1653498627.191053, reason=None, message=None, count=1)] + [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1), WorkStatus(stage='succeeded', timestamp=1653498627.191053, reason=None, message=None, count=1)] + + # Second execution with value = 1 + + [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1)] + [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1)] + ... + [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1)] + [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='failed', timestamp=1653498628.210164, reason='user_exception', message='The provided value was 1', count=1)] + [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='failed', timestamp=1653498628.210164, reason='user_exception', message='The provided value was 1', count=1)] diff --git a/docs/source-app/core_api/lightning_work/status_content.rst b/docs/source-app/core_api/lightning_work/status_content.rst deleted file mode 100644 index 7ee8b65ac8..0000000000 --- a/docs/source-app/core_api/lightning_work/status_content.rst +++ /dev/null @@ -1,197 +0,0 @@ - -************************************* -Everything about LightningWork Status -************************************* - -Statuses indicate transition points in the life of a LightningWork (Work) and contain metadata. - -The different stages are: - -.. code-block:: python - - class WorkStageStatus: - NOT_STARTED = "not_started" - STOPPED = "stopped" - PENDING = "pending" - RUNNING = "running" - SUCCEEDED = "succeeded" - FAILED = "failed" - -And a single status is as follows: - -.. code-block:: python - - @dataclass - class WorkStatus: - stage: WorkStageStatus - timestamp: float - reason: Optional[str] = None - message: Optional[str] = None - count: int = 1 - - -On creation, the Work's status flags all evaluate to ``False`` (in particular ``has_started``) and when calling ``work.run`` in your Lightning Flow (Flow), -the Work transitions from ``is_pending`` to ``is_running`` and then to ``has_succeeded`` if everything went well or ``has_failed`` otherwise. - -.. code-block:: python - - from time import sleep - import lightning as L - - - class Work(L.LightningWork): - def run(self, value: int): - sleep(1) - if value == 0: - return - raise Exception(f"The provided value was {value}") - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.work = Work(raise_exception=False) - self.counter = 0 - - def run(self): - if not self.work.has_started: - print("NOT STARTED") - - elif self.work.is_pending: - print("PENDING") - - elif self.work.is_running: - print("RUNNING") - - elif self.work.has_succeeded: - print("SUCCESS") - - elif self.work.has_failed: - print("FAILED") - - elif self.work.has_stopped: - print("STOPPED") - self._exit() - - print(self.work.status) - self.work.run(self.counter) - self.counter += 1 - - - app = L.LightningApp(Flow()) - -Run this app as follows: - -.. code-block:: bash - - lightning run app test.py > app_log.txt - -And here is the expected output inside ``app_log.txt`` and as expected, -we are observing the following transition ``has_started``, ``is_pending``, ``is_running``, ``has_succeeded``, ``is_running`` and ``has_failed`` - -.. code-block:: console - - NOT STARTED - WorkStatus(stage='not_started', timestamp=1653498225.18468, reason=None, message=None, count=1) - PENDING - WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) - PENDING - WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) - PENDING - ... - PENDING - WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) - PENDING - WorkStatus(stage='pending', timestamp=1653498225.217413, reason=None, message=None, count=1) - RUNNING - WorkStatus(stage='running', timestamp=1653498228.825194, reason=None, message=None, count=1) - ... - SUCCESS - WorkStatus(stage='succeeded', timestamp=1653498229.831793, reason=None, message=None, count=1) - SUCCESS - WorkStatus(stage='succeeded', timestamp=1653498229.831793, reason=None, message=None, count=1) - SUCCESS - WorkStatus(stage='succeeded', timestamp=1653498229.831793, reason=None, message=None, count=1) - RUNNING - WorkStatus(stage='running', timestamp=1653498229.846451, reason=None, message=None, count=1) - RUNNING - ... - WorkStatus(stage='running', timestamp=1653498229.846451, reason=None, message=None, count=1) - RUNNING - WorkStatus(stage='running', timestamp=1653498229.846451, reason=None, message=None, count=1) - FAILED - WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) - FAILED - WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) - FAILED - WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) - FAILED - WorkStatus(stage='failed', timestamp=1653498230.852565, reason='user_exception', message='The provided value was 1', count=1) - ... - -In order to access all statuses: - -.. code-block:: python - - from time import sleep - import lightning as L - - - class Work(L.LightningWork): - def run(self, value: int): - sleep(1) - if value == 0: - return - raise Exception(f"The provided value was {value}") - - - class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - self.work = Work(raise_exception=False) - self.counter = 0 - - def run(self): - print(self.statuses) - self.work.run(self.counter) - self.counter += 1 - - - app = L.LightningApp(Flow()) - - -Run this app as follows: - -.. code-block:: bash - - lightning run app test.py > app_log.txt - -And here is the expected output inside ``app_log.txt``: - - -.. code-block:: console - - # First execution with value = 0 - - [] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1)] - ... - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] - ... - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1), WorkStatus(stage='succeeded', timestamp=1653498627.191053, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1), WorkStatus(stage='succeeded', timestamp=1653498627.191053, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498622.252016, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498626.185683, reason=None, message=None, count=1), WorkStatus(stage='succeeded', timestamp=1653498627.191053, reason=None, message=None, count=1)] - - # Second execution with value = 1 - - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1)] - ... - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1)] - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='failed', timestamp=1653498628.210164, reason='user_exception', message='The provided value was 1', count=1)] - [WorkStatus(stage='pending', timestamp=1653498627.204636, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='running', timestamp=1653498627.205509, reason=None, message=None, count=1), WorkStatus(stage='failed', timestamp=1653498628.210164, reason='user_exception', message='The provided value was 1', count=1)] diff --git a/docs/source-app/examples/dag/dag.rst b/docs/source-app/examples/dag/dag.rst index 8b1d71dd24..f9b304b736 100644 --- a/docs/source-app/examples/dag/dag.rst +++ b/docs/source-app/examples/dag/dag.rst @@ -2,8 +2,6 @@ Build a Directed Acyclic Graph (DAG) #################################### -.. _dag_example: - **Audience:** Users coming from MLOps to Lightning Apps, looking for more flexibility. A typical ML training workflow can be implemented with a simple DAG. @@ -12,10 +10,10 @@ Below is a pseudo-code using the lightning framework that uses a LightningFlow t .. code-block:: python - import lightning as L + import lightning_app as la - class DAGFlow(L.LightningFlow): + class DAGFlow(lapp.LightningFlow): def __init__(self): super().__init__() self.processor = DataProcessorWork(...) @@ -31,19 +29,21 @@ Below is a pseudo-code to run several works in parallel using a built-in :class: .. code-block:: python - import lightning as L + import lightning_app as la - class DAGFlow(L.LightningFlow): + class DAGFlow(lapp.LightningFlow): def __init__(self): super().__init__() ... - self.train_works = L.structures.Dict(**{ - "1": TrainingWork(..., parallel=True), - "2": TrainingWork(..., parallel=True), - "3": TrainingWork(..., parallel=True), - ... - }) + self.train_works = lapp.structures.Dict( + **{ + "1": TrainingWork(..., parallel=True), + "2": TrainingWork(..., parallel=True), + "3": TrainingWork(..., parallel=True), + # ... + } + ) ... def run(self): @@ -59,12 +59,13 @@ Below is a pseudo-code to run several works in parallel using a built-in :class: self.serve_work.run(...) ----- ********** -Next Steps +Next steps ********** +Depending on your use case, you might want to check one of these out next. + .. raw:: html
diff --git a/docs/source-app/examples/dag/dag_from_scratch.rst b/docs/source-app/examples/dag/dag_from_scratch.rst index cde4695332..7c68c8ea87 100644 --- a/docs/source-app/examples/dag/dag_from_scratch.rst +++ b/docs/source-app/examples/dag/dag_from_scratch.rst @@ -15,9 +15,7 @@ In this example, you will learn how to create a simple DAG which: and learn how to schedule this entire process. -Find the complete example `here `_. - ----- +Find the complete example `here `_. ************************** Step 1: Implement your DAG @@ -35,20 +33,19 @@ First, let's define the component we need: * Processing is responsible to execute a ``processing.py`` script. * A collection of model work to train all models in parallel. -.. literalinclude:: ../../../examples/app_dag/app.py +.. literalinclude:: ../../../../examples/dag/app.py :lines: 55-79 And its run method executes the steps described above. Additionally, ``work.stop`` is used to reduce cost when running in the cloud. -.. literalinclude:: ../../../examples/app_dag/app.py +.. literalinclude:: ../../../../examples/dag/app.py :lines: 81-108 ----- ***************************** Step 2: Define the scheduling ***************************** -.. literalinclude:: ../../../examples/app_dag/app.py +.. literalinclude:: ../../../../examples/dag/app.py :lines: 109-137 diff --git a/docs/source-app/examples/data_explore_app.rst b/docs/source-app/examples/data_explore_app.rst index cd7011a10e..f66b3c2cb1 100644 --- a/docs/source-app/examples/data_explore_app.rst +++ b/docs/source-app/examples/data_explore_app.rst @@ -1,5 +1,3 @@ -:orphan: - ########################## Build a Data Exploring App ########################## diff --git a/docs/source-app/examples/etl_app.rst b/docs/source-app/examples/etl_app.rst index 5b494e943e..4cb581e399 100644 --- a/docs/source-app/examples/etl_app.rst +++ b/docs/source-app/examples/etl_app.rst @@ -1,5 +1,3 @@ -:orphan: - ############### Build a ETL App ############### diff --git a/docs/source-app/examples/file_server/app.py b/docs/source-app/examples/file_server/app.py deleted file mode 100644 index 5de9f3720a..0000000000 --- a/docs/source-app/examples/file_server/app.py +++ /dev/null @@ -1,232 +0,0 @@ -import json -import os -import tarfile -import uuid -import zipfile -from dataclasses import dataclass -from pathlib import Path -from typing import List - -import lightning as L -from lightning.app.storage import Drive - - -class FileServer(L.LightningWork): - def __init__(self, drive: Drive, base_dir: str = "file_server", chunk_size=10240, **kwargs): - """This component uploads, downloads files to your application. - - Arguments: - drive: The drive can share data inside your application. - base_dir: The local directory where the data will be stored. - chunk_size: The quantity of bytes to download/upload at once. - """ - super().__init__( - cloud_build_config=L.BuildConfig(["flask, flask-cors"]), - parallel=True, - **kwargs, - ) - # 1: Attach the arguments to the state. - self.drive = drive - self.base_dir = base_dir - self.chunk_size = chunk_size - - # 2: Create a folder to store the data. - os.makedirs(self.base_dir, exist_ok=True) - - # 3: Keep a reference to the uploaded filenames. - self.uploaded_files = dict() - - def get_filepath(self, path: str) -> str: - """Returns file path stored on the file server.""" - return os.path.join(self.base_dir, path) - - def get_random_filename(self) -> str: - """Returns a random hash for the file name.""" - return uuid.uuid4().hex - - def upload_file(self, file): - """Upload a file while tracking its progress.""" - # 1: Track metadata about the file - filename = file.filename - uploaded_file = self.get_random_filename() - meta_file = uploaded_file + ".meta" - self.uploaded_files[filename] = {"progress": (0, None), "done": False} - - # 2: Create a stream and write bytes of - # the file to the disk under `uploaded_file` path. - with open(self.get_filepath(uploaded_file), "wb") as out_file: - content = file.read(self.chunk_size) - while content: - # 2.1 Write the file bytes - size = out_file.write(content) - - # 2.2 Update the progress metadata - self.uploaded_files[filename]["progress"] = ( - self.uploaded_files[filename]["progress"][0] + size, - None, - ) - # 4: Read next chunk of data - content = file.read(self.chunk_size) - - # 3: Update metadata that the file has been uploaded. - full_size = self.uploaded_files[filename]["progress"][0] - self.drive.put(self.get_filepath(uploaded_file)) - self.uploaded_files[filename] = { - "progress": (full_size, full_size), - "done": True, - "uploaded_file": uploaded_file, - } - - # 4: Write down the metadata about the file to the disk - meta = { - "original_path": filename, - "display_name": os.path.splitext(filename)[0], - "size": full_size, - "drive_path": uploaded_file, - } - with open(self.get_filepath(meta_file), "wt") as f: - json.dump(meta, f) - - # 5: Put the file to the drive. - # It means other components can access get or list them. - self.drive.put(self.get_filepath(meta_file)) - return meta - - def list_files(self, file_path: str): - # 1: Get the local file path of the file server. - file_path = self.get_filepath(file_path) - - # 2: If the file exists in the drive, transfer it locally. - if not os.path.exists(file_path): - self.drive.get(file_path) - - if os.path.isdir(file_path): - result = set() - for _, _, f in os.walk(file_path): - for file in f: - if not file.endswith(".meta"): - for filename, meta in self.uploaded_files.items(): - if meta["uploaded_file"] == file: - result.add(filename) - return {"asset_names": [v for v in result]} - - # 3: If the filepath is a tar or zip file, list their contents - if zipfile.is_zipfile(file_path): - with zipfile.ZipFile(file_path, "r") as zf: - result = zf.namelist() - elif tarfile.is_tarfile(file_path): - with tarfile.TarFile(file_path, "r") as tf: - result = tf.getnames() - else: - raise ValueError("Cannot open archive file!") - - # 4: Returns the matching files. - return {"asset_names": result} - - def run(self): - # 1: Imports flask requirements. - from flask import Flask, request - from flask_cors import CORS - - # 2: Create a flask app - flask_app = Flask(__name__) - CORS(flask_app) - - # 3: Define the upload file endpoint - @flask_app.post("/upload_file/") - def upload_file(): - """Upload a file directly as form data.""" - f = request.files["file"] - return self.upload_file(f) - - @flask_app.get("/") - def list_files(): - return self.list_files(str(Path(self.base_dir).resolve())) - - # 5: Start the flask app while providing the `host` and `port`. - flask_app.run(host=self.host, port=self.port, load_dotenv=False) - - def alive(self): - """Hack: Returns whether the server is alive.""" - return self.url != "" - - -import requests - -from lightning import LightningWork - - -class TestFileServer(LightningWork): - def __init__(self, drive: Drive): - super().__init__(cache_calls=True) - self.drive = drive - - def run(self, file_server_url: str, first=True): - if first: - with open("test.txt", "w") as f: - f.write("Some text.") - - response = requests.post(file_server_url + "/upload_file/", files={"file": open("test.txt", "rb")}) - assert response.status_code == 200 - else: - response = requests.get(file_server_url) - assert response.status_code == 200 - assert response.json() == {"asset_names": ["test.txt"]} - - -from lightning import LightningApp, LightningFlow - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - # 1: Create a drive to share data between works - self.drive = Drive("lit://file_server") - # 2: Create the filer server - self.file_server = FileServer(self.drive) - # 3: Create the file ser - self.test_file_server = TestFileServer(self.drive) - - def run(self): - # 1: Start the file server. - self.file_server.run() - - # 2: Trigger the test file server work when ready. - if self.file_server.alive(): - # 3 Execute the test file server work. - self.test_file_server.run(self.file_server.url) - self.test_file_server.run(self.file_server.url, first=False) - - # 4 When both execution are successful, exit the app. - if self.test_file_server.num_successes == 2: - self._exit() - - def configure_layout(self): - # Expose the file_server component - # in the UI using its `/` endpoint. - return {"name": "File Server", "content": self.file_server} - - -from lightning.app.runners import MultiProcessRuntime - - -def test_file_server(): - app = LightningApp(Flow()) - MultiProcessRuntime(app).dispatch() - - -from lightning.app.testing.testing import run_app_in_cloud - - -def test_file_server_in_cloud(): - # You need to provide the directory containing the app file. - app_dir = "docs/source-app/examples/file_server" - with run_app_in_cloud(app_dir) as (admin_page, view_page, get_logs_fn): - """# 1. `admin_page` and `view_page` are playwright Page Objects. - - # Check out https://playwright.dev/python/ doc to learn more. - # You can click the UI and trigger actions. - - # 2. By calling logs = get_logs_fn(), - # you get all the logs currently on the admin page. - """ diff --git a/docs/source-app/examples/file_server/file_server.rst b/docs/source-app/examples/file_server/file_server.rst deleted file mode 100644 index 430333a875..0000000000 --- a/docs/source-app/examples/file_server/file_server.rst +++ /dev/null @@ -1,12 +0,0 @@ - -.. _fileserver_example: - -################### -Build a File Server -################### - -**Prerequisite**: Reach :ref:`level 16+ ` and read the `Drive article `_. - ----- - -.. include:: file_server_content.rst diff --git a/docs/source-app/examples/file_server/file_server_content.rst b/docs/source-app/examples/file_server/file_server_content.rst deleted file mode 100644 index 26603e04f8..0000000000 --- a/docs/source-app/examples/file_server/file_server_content.rst +++ /dev/null @@ -1,82 +0,0 @@ -********* -Objective -********* - -Create a simple application where users can upload files and list the uploaded files. - ----- - -***************** -Final Application -***************** - -Here is a recording of the final application built in this example tested with pytest. - -.. raw:: html - - - ----- - -************* -System Design -************* - -In order to create such application, we need to build two components and an application: - -* A **File Server Component** that gives you the ability to download or list files shared with your application. This is particularly useful when you want to trigger an ML job but your users need to provide their own data or if the user wants to download the trained checkpoints. - -* A **Test File Server** Component to interact with the file server. - -* An application putting everything together and its associated pytest tests. - ----- - -******** -Tutorial -******** - -Let's dive in on how to create such application and component: - -.. raw:: html - -
-
- -.. displayitem:: - :header: 1. Implement the File Server general structure - :description: Put together the shape of the component - :col_css: col-md-4 - :button_link: file_server_step_1.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: 2. Implement the File Server upload and list files methods - :description: Add the core functionalities to the component - :col_css: col-md-4 - :button_link: file_server_step_2.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: 3. Implement a File Server Testing Component - :description: Create a component to test the file server - :col_css: col-md-4 - :button_link: file_server_step_3.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: 4. Implement tests for the File Server component with pytest - :description: Create an app to validate the upload and list files endpoints - :col_css: col-md-4 - :button_link: file_server_step_4.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/file_server/file_server_step_1.rst b/docs/source-app/examples/file_server/file_server_step_1.rst deleted file mode 100644 index 782e553e9f..0000000000 --- a/docs/source-app/examples/file_server/file_server_step_1.rst +++ /dev/null @@ -1,11 +0,0 @@ -:orphan: - -********************************************* -1. Implement the FileServer general structure -********************************************* - -Let's dive in on how to create such a component with the code below. - -.. literalinclude:: ./app.py - :lines: 1-44, 132-158 - :emphasize-lines: 16, 51- diff --git a/docs/source-app/examples/file_server/file_server_step_2.rst b/docs/source-app/examples/file_server/file_server_step_2.rst deleted file mode 100644 index 668b01b177..0000000000 --- a/docs/source-app/examples/file_server/file_server_step_2.rst +++ /dev/null @@ -1,37 +0,0 @@ -:orphan: - -********************************************************** -2. Implement the File Server upload and list_files methods -********************************************************** - -Let's dive in on how to implement such methods. - -*************************** -Implement the upload method -*************************** - -In this method, we are creating a stream between the uploaded file and the uploaded file stored on the file server disk. - -Once the file is uploaded, we are putting the file into the :class:`~lightning_app.storage.drive.Drive`, so it becomes persistent and accessible to all components. - -.. literalinclude:: ./app.py - :lines: 13, 52-100 - :emphasize-lines: 49 - -******************************* -Implement the fist_files method -******************************* - -First, in this method, we get the file in the file server filesystem, if available in the Drive. Once done, we list the the files under the provided paths and return the results. - -.. literalinclude:: ./app.py - :lines: 13, 101-131 - :emphasize-lines: 9 - - -******************* -Implement utilities -******************* - -.. literalinclude:: ./app.py - :lines: 13, 46-51 diff --git a/docs/source-app/examples/file_server/file_server_step_3.rst b/docs/source-app/examples/file_server/file_server_step_3.rst deleted file mode 100644 index 97b524a978..0000000000 --- a/docs/source-app/examples/file_server/file_server_step_3.rst +++ /dev/null @@ -1,16 +0,0 @@ -:orphan: - -******************************************** -3. Implement a File Server Testing Component -******************************************** - -Let's dive in on how to implement a testing component for a server. - -This component needs to test two things: - -* The **/upload_file/** endpoint by creating a file and sending its content to it. - -* The **/** endpoint listing files, by validating the that previously uploaded file is present in the response. - -.. literalinclude:: ./app.py - :lines: 161-183 diff --git a/docs/source-app/examples/file_server/file_server_step_4.rst b/docs/source-app/examples/file_server/file_server_step_4.rst deleted file mode 100644 index 06d9e051dc..0000000000 --- a/docs/source-app/examples/file_server/file_server_step_4.rst +++ /dev/null @@ -1,86 +0,0 @@ -:orphan: - -************************************************************ -4. Implement tests for the File Server component with pytest -************************************************************ - -Let's create a simple Lightning App (App) with our **File Server** and the **File Server Test** components. - -Once the File Server is up and running, we'll execute the **test_file_server** LightningWork and when both calls are successful, we exit the App using ``self._exit``. - -.. literalinclude:: ./app.py - :lines: 186-216 - - -Simply create a ``test.py`` file with the following code and run ``pytest tests.py`` - -.. literalinclude:: ./app.py - :lines: 218-222 - -To test the App in the cloud, create a ``cloud_test.py`` file with the following code and run ``pytest cloud_test.py``. Under the hood, we are using the end-to-end testing `playwright `_ library so you can interact with the UI. - -.. literalinclude:: ./app.py - :lines: 224- - ----- - -******************** -Test the application -******************** - -Clone the lightning repo and run the following command: - -.. code-block:: bash - - pytest docs/source-app/examples/file_server/app.py --capture=no -v - ----- - -****************** -Find more examples -****************** - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Build a DAG - :description: Create a dag pipeline - :col_css: col-md-4 - :button_link: ../dag/dag.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Build a Github Repo Script Runner - :description: Run any script on github in the cloud - :col_css: col-md-4 - :button_link: ../github_repo_runner/github_repo_runner.html - :height: 150 - :tag: Intermediate - - -.. displayitem:: - :header: Build a HPO Sweeper - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../hpo/hpo.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Build a Model Server - :description: Serve multiple models with different parameters - :col_css: col-md-4 - :button_link: ../model_server/model_server.html - :height: 150 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/.lightning b/docs/source-app/examples/github_repo_runner/.lightning deleted file mode 100644 index 4bae28430e..0000000000 --- a/docs/source-app/examples/github_repo_runner/.lightning +++ /dev/null @@ -1 +0,0 @@ -name: github_repo_runner diff --git a/docs/source-app/examples/github_repo_runner/app.py b/docs/source-app/examples/github_repo_runner/app.py deleted file mode 100644 index a020b69c02..0000000000 --- a/docs/source-app/examples/github_repo_runner/app.py +++ /dev/null @@ -1,299 +0,0 @@ -import io -import os -import subprocess -import sys -from copy import deepcopy -from functools import partial -from subprocess import Popen -from typing import Any, Dict, List, Optional - -from lightning import BuildConfig, CloudCompute, LightningApp, LightningFlow -from lightning.app import structures -from lightning.app.components.python import TracerPythonScript -from lightning.app.frontend import StreamlitFrontend -from lightning.app.storage.path import Path -from lightning.app.utilities.state import AppState - - -class GithubRepoRunner(TracerPythonScript): - def __init__( - self, - id: str, - github_repo: str, - script_path: str, - script_args: List[str], - requirements: List[str], - cloud_compute: Optional[CloudCompute] = None, - **kwargs: Any, - ): - """The GithubRepoRunner Component clones a repo, runs a specific script with provided arguments and collect - logs. - - Arguments: - id: Identified of the component. - github_repo: The Github Repo URL to clone. - script_path: The path to the script to execute. - script_args: The arguments to be provided to the script. - requirements: The python requirements tp run the script. - cloud_compute: The object to select the cloud instance. - """ - super().__init__( - script_path=script_path, - script_args=script_args, - cloud_compute=cloud_compute, - cloud_build_config=BuildConfig(requirements=requirements), - **kwargs, - ) - self.id = id - self.github_repo = github_repo - self.logs = [] - - def run(self, *args, **kwargs): - # 1. Hack: Patch stdout so we can capture the logs. - string_io = io.StringIO() - sys.stdout = string_io - - # 2: Use git command line to clone the repo. - repo_name = self.github_repo.split("/")[-1].replace(".git", "") - cwd = os.path.dirname(__file__) - subprocess.Popen(f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() - - # 3: Execute the parent run method of the TracerPythonScript class. - os.chdir(os.path.join(cwd, repo_name)) - super().run(*args, **kwargs) - - # 4: Get all the collected logs and add them to the state. - # This isn't optimal as heavy, but works for this demo purpose. - self.logs = string_io.getvalue() - string_io.close() - - def configure_layout(self): - return {"name": self.id, "content": self} - - -class PyTorchLightningGithubRepoRunner(GithubRepoRunner): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.best_model_path = None - self.best_model_score = None - - def configure_tracer(self): - from pytorch_lightning import Trainer - from pytorch_lightning.callbacks import Callback - - tracer = super().configure_tracer() - - class TensorboardServerLauncher(Callback): - def __init__(self, work): - # The provided `work` is the - # current ``PyTorchLightningScript`` work. - self.w = work - - def on_train_start(self, trainer, *_): - # Add `host` and `port` for tensorboard to work in the cloud. - cmd = f"tensorboard --logdir='{trainer.logger.log_dir}'" - server_args = f"--host {self.w.host} --port {self.w.port}" - Popen(cmd + " " + server_args, shell=True) - - def trainer_pre_fn(self, *args, work=None, **kwargs): - # Intercept Trainer __init__ call - # and inject a ``TensorboardServerLauncher`` component. - kwargs["callbacks"].append(TensorboardServerLauncher(work)) - return {}, args, kwargs - - # 5. Patch the `__init__` method of the Trainer - # to inject our callback with a reference to the work. - tracer.add_traced(Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) - return tracer - - def on_after_run(self, end_script_globals): - import torch - - # 1. Once the script has finished to execute, - # we can collect its globals and access any objects. - trainer = end_script_globals["cli"].trainer - checkpoint_callback = trainer.checkpoint_callback - lightning_module = trainer.lightning_module - - # 2. From the checkpoint_callback, - # we are accessing the best model weights - checkpoint = torch.load(checkpoint_callback.best_model_path) - - # 3. Load the best weights and torchscript the model. - lightning_module.load_state_dict(checkpoint["state_dict"]) - lightning_module.to_torchscript(f"{self.name}.pt") - - # 4. Use lightning.app.storage.Pathto create a reference to the - # torch scripted model. In the cloud with multiple machines, - # by simply passing this reference to another work, - # it triggers automatically a file transfer. - self.best_model_path = Path(f"{self.name}.pt") - - # 5. Keep track of the metrics. - self.best_model_score = float(checkpoint_callback.best_model_score) - - -class KerasGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement.""" - - -class TensorflowGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement.""" - - -GITHUB_REPO_RUNNERS = { - "PyTorch Lightning": PyTorchLightningGithubRepoRunner, - "Keras": KerasGithubRepoRunner, - "Tensorflow": TensorflowGithubRepoRunner, -} - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - # 1: Keep track of the requests within the state - self.requests = [] - # 2: Create a dictionary of components. - self.ws = structures.Dict() - - def run(self): - # Iterate continuously over all requests - for request_id, request in enumerate(self.requests): - self._handle_request(request_id, deepcopy(request)) - - def _handle_request(self, request_id: int, request: Dict): - # 1: Create a name and find selected framework - name = f"w_{request_id}" - ml_framework = request["train"].pop("ml_framework") - - # 2: If the component hasn't been created yet, create it. - if name not in self.ws: - work_cls = GITHUB_REPO_RUNNERS[ml_framework] - work = work_cls(id=request["id"], **request["train"]) - self.ws[name] = work - - # 3: Run the component - self.ws[name].run() - - # 4: Once the component has finished, - # add metadata to the original request for the UI. - if self.ws[name].best_model_path: - request = self.requests[request_id] - request["best_model_score"] = self.ws[name].best_model_score - request["best_model_path"] = self.ws[name].best_model_path - - def configure_layout(self): - # Create a StreamLit UI for the user to run his Github Repo. - return StreamlitFrontend(render_fn=render_fn) - - -def page_1__create_new_run(state): - import streamlit as st - - st.markdown("# Create a new Run 🎈") - - # 1: Collect arguments from the users - id = st.text_input("Name your run", value="my_first_run") - github_repo = st.text_input( - "Enter a Github Repo URL", value="https://github.com/Lightning-AI/lightning-quick-start.git" - ) - - default_script_args = "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 --trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" - default_requirements = "torchvision, pytorch_lightning, jsonargparse[signatures]" - - script_path = st.text_input("Enter your script to run", value="train_script.py") - script_args = st.text_input("Enter your base script arguments", value=default_script_args) - requirements = st.text_input("Enter your requirements", value=default_requirements) - ml_framework = st.radio("Select your ML Training Frameworks", options=["PyTorch Lightning", "Keras", "Tensorflow"]) - - if ml_framework not in ("PyTorch Lightning"): - st.write(f"{ml_framework} isn't supported yet.") - return - - clicked = st.button("Submit") - - # 2: If clicked, create a new request. - if clicked: - new_request = { - "id": id, - "train": { - "github_repo": github_repo, - "script_path": script_path, - "script_args": script_args.split(" "), - "requirements": requirements.split(" "), - "ml_framework": ml_framework, - }, - } - # 3: IMPORTANT: Add a new request to the state in-place. - # The flow receives the UI request and dynamically create - # and run the associated work from the request information. - state.requests = state.requests + [new_request] - - -def page_2__view_run_lists(state): - import streamlit as st - - st.markdown("# Run Lists 🎈") - # 1: Iterate through all the requests in the state. - for i, r in enumerate(state.requests): - i = str(i) - # 2: Display information such as request, logs, work state, model score. - work = state._state["structures"]["ws"]["works"][f"w_{i}"] - with st.expander(f"Expand to view Run {i}", expanded=False): - if st.checkbox(f"Expand to view your configuration", key=i): - st.json(r) - if st.checkbox(f"Expand to view logs", key=i): - st.code(body=work["vars"]["logs"]) - if st.checkbox(f"Expand to view your work state", key=i): - work["vars"].pop("logs") - st.json(work) - best_model_score = r.get("best_model_score", None) - if best_model_score: - if st.checkbox(f"Expand to view your run performance", key=i): - st.json({"best_model_score": best_model_score, "best_model_path": r.get("best_model_path")}) - - -def page_3__view_app_state(state): - import streamlit as st - - st.markdown("# App State 🎈") - st.write(state._state) - - -def render_fn(state: AppState): - import streamlit as st - - page_names_to_funcs = { - "Create a new Run": partial(page_1__create_new_run, state=state), - "View your Runs": partial(page_2__view_run_lists, state=state), - "View the App state": partial(page_3__view_app_state, state=state), - } - selected_page = st.sidebar.selectbox("Select a page", page_names_to_funcs.keys()) - page_names_to_funcs[selected_page]() - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - # Create the flow - self.flow = Flow() - - def run(self): - # Run the flow - self.flow.run() - - def configure_layout(self): - # 1: Add the main StreamLit UI - selection_tab = [ - { - "name": "Run your Github Repo", - "content": self.flow, - } - ] - # 2: Add a new tab whenever a new work is dynamically created - run_tabs = [e.configure_layout() for e in self.flow.ws.values()] - # 3: Returns the list of tabs. - return selection_tab + run_tabs - - -app = LightningApp(RootFlow()) diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner.rst deleted file mode 100644 index affb115b74..0000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _github_repo_script_runner_example: - -################################# -Build a Github Repo Script Runner -################################# - -**Audience:** Users that want to create interactive applications which runs Github Repo in the cloud at any scale for multiple users. - -**Prerequisite**: Reach :ref:`level 16+ ` and read the docstring of of :class:`~lightning_app.components.python.tracer.TracerPythonScript` component. - ----- - -.. include:: github_repo_runner_content.rst diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_content.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_content.rst deleted file mode 100644 index 335b21ce7f..0000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_content.rst +++ /dev/null @@ -1,98 +0,0 @@ - -********* -Objective -********* - -Create a simple application where users can enter information in a UI to run a given PyTorch Lightning Script from a given Github Repo with optionally some extra python requirements and arguments. - -Furthermore, the users should be able to monitor their training progress in real-time, view the logs, and get the best-monitored metric and associated checkpoint for their models. - ----- - -***************** -Final Application -***************** - -Here is a recording of the final application built in this example. The example is around 200 lines in total and should give you a great foundation to build your own Lightning App. - -.. raw:: html - - - ----- - -************* -System Design -************* - -In order to create such application, we need to build several components: - -* A GithubRepoRunner Component that clones a repo, runs a specific script with provided arguments and collect logs. - -* A PyTorch Lightning GithubRepoRunner Component that augments the GithubRepoRunner component to track PyTorch Lightning Trainer. - -* A UI for the users to provide to trigger dynamically a new execution. - -* A Flow to dynamically create GithubRepoRunner once a user submits information from the UI. - -Let's dive in on how to create such a component. - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 1. Implement the GithubRepoRunner Component - :description: Clone and execute script from a GitHub Repo. - :col_css: col-md-4 - :button_link: github_repo_runner_step_1.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: 2. Implement the PyTorch Lightning GithubRepoRunner Component - :description: Automate PyTorch Lightning execution - :col_css: col-md-4 - :button_link: github_repo_runner_step_2.html - :height: 180 - :tag: Advanced - -.. displayitem:: - :header: 3. Implement the Flow to manage user requests - :description: Dynamically create GithubRepoRunner - :col_css: col-md-4 - :button_link: github_repo_runner_step_3.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: 4. Implement the UI with StreamLit - :description: Several pages application - :col_css: col-md-4 - :button_link: github_repo_runner_step_4.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: 5. Putting everything together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_1.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_1.rst deleted file mode 100644 index 3a683501fa..0000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_1.rst +++ /dev/null @@ -1,62 +0,0 @@ -:orphan: - -******************************************* -1. Implement the GithubRepoRunner Component -******************************************* - -The GithubRepoRunner Component clones a repo, runs a specific script with provided arguments and collect logs. - -Let's dive in on how to create such a component with the code below. - -.. literalinclude:: ./app.py - :lines: -72 - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 2. Implement the PyTorch Lightning GithubRepoRunner Component - :description: Automate PyTorch Lightning execution - :col_css: col-md-4 - :button_link: github_repo_runner_step_2.html - :height: 180 - :tag: Advanced - -.. displayitem:: - :header: 3. Implement the Flow to manage user requests - :description: Dynamically create GithubRepoRunner - :col_css: col-md-4 - :button_link: github_repo_runner_step_3.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: 4. Implement the UI with StreamLit - :description: Several pages application - :col_css: col-md-4 - :button_link: github_repo_runner_step_4.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: 5. Putting everything together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_2.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_2.rst deleted file mode 100644 index c0825fa8ea..0000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_2.rst +++ /dev/null @@ -1,68 +0,0 @@ -:orphan: - -************************************************************* -2. Implement the PyTorch Lightning GithubRepoRunner Component -************************************************************* - -The PyTorch Lightning GithubRepoRunner Component subclasses the GithubRepoRunner but tailors the execution experience to PyTorch Lightning. - -As a matter of fact, this component adds two primary tailored features for PyTorch Lightning users: - -* It injects dynamically a custom callback ``TensorboardServerLauncher`` in the PyTorch Lightning Trainer to start a tensorboard server so it can be exposed in Lightning App UI. - -* Once the script has run, the ``on_after_run`` hook of the :class:`~lightning_app.components.python.tracer.TracerPythonScript` is invoked with the script globals, meaning we can collect anything we need. In particular, we are reloading the best model, torch scripting it, and storing its path in the state alongside the best metric score. - -Let's dive in on how to create such a component with the code below. - -.. literalinclude:: ./app.py - :lines: 75-136 - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 1. Implement the GithubRepoRunner Component - :description: Clone and execute script from a GitHub Repo. - :col_css: col-md-4 - :button_link: github_repo_runner_step_1.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: 3. Implement the Flow to manage user requests - :description: Dynamically create GithubRepoRunner - :col_css: col-md-4 - :button_link: github_repo_runner_step_3.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: 4. Implement the UI with StreamLit - :description: Several pages application - :col_css: col-md-4 - :button_link: github_repo_runner_step_4.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: 5. Putting everything together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_3.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_3.rst deleted file mode 100644 index fc1b3116be..0000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_3.rst +++ /dev/null @@ -1,62 +0,0 @@ -:orphan: - -********************************************* -3. Implement the Flow to manage user requests -********************************************* - -In step 1 and 2, we have implemented ``GithubRepoRunner`` and ``PyTorchLightningGithubRepoRunner`` components. - -Now, we are going to create a component to dynamically handle user requests. -Let's dive in on how to create such a component with the code below. - -.. literalinclude:: ./app.py - :lines: 138-187 - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 1. Implement the GithubRepoRunner Component - :description: Clone and execute script from a GitHub Repo. - :col_css: col-md-4 - :button_link: github_repo_runner_step_1.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: 2. Implement the PyTorch Lightning GithubRepoRunner Component - :description: Automate PyTorch Lightning execution - :col_css: col-md-4 - :button_link: github_repo_runner_step_2.html - :height: 180 - :tag: Advanced - -.. displayitem:: - :header: 4. Implement the UI with StreamLit - :description: Several pages application - :col_css: col-md-4 - :button_link: github_repo_runner_step_4.html - :height: 180 - :tag: Intermediate - - -.. displayitem:: - :header: 5. Putting everything together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_4.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_4.rst deleted file mode 100644 index 2716adbaf8..0000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_4.rst +++ /dev/null @@ -1,93 +0,0 @@ -:orphan: - -********************************** -4. Implement the UI with StreamLit -********************************** - -In step 3, we have implemented a flow that dynamically creates a Work when a new request is added to the requests list. - -From the UI, we create 3 pages with `StreamLit `_: - -* **Page 1**: Create a form to add a new request to the flow state **requests**. - -* **Page 2**: Iterate through all the requests and display associated information. - -* **Page 3**: Display the entire App State. - -**************** -Render All Pages -**************** - -.. literalinclude:: ./app.py - :lines: 263-274 - - -****** -Page 1 -****** - -.. literalinclude:: ./app.py - :lines: 189-231 - :emphasize-lines: 43 - -****** -Page 2 -****** - -.. literalinclude:: ./app.py - :lines: 233-255 - -****** -Page 3 -****** - -.. literalinclude:: ./app.py - :lines: 257-261 - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: 1. Implement the GithubRepoRunner Component - :description: Clone and execute script from a GitHub Repo. - :col_css: col-md-4 - :button_link: github_repo_runner_step_1.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: 2. Implement the PyTorch Lightning GithubRepoRunner Component - :description: Automate PyTorch Lightning execution - :col_css: col-md-4 - :button_link: github_repo_runner_step_2.html - :height: 180 - :tag: Advanced - -.. displayitem:: - :header: 3. Implement the Flow to manage user requests - :description: Dynamically create GithubRepoRunner - :col_css: col-md-4 - :button_link: github_repo_runner_step_3.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: 5. Putting everything together - :description: - :col_css: col-md-4 - :button_link: github_repo_runner_step_5.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_5.rst b/docs/source-app/examples/github_repo_runner/github_repo_runner_step_5.rst deleted file mode 100644 index bdad952332..0000000000 --- a/docs/source-app/examples/github_repo_runner/github_repo_runner_step_5.rst +++ /dev/null @@ -1,77 +0,0 @@ -:orphan: - -****************************** -5. Putting everything together -****************************** - -Let's dive in on how to create such a component with the code below. - -.. literalinclude:: ./app.py - :lines: 277- - - -******************* -Run the application -******************* - -Clone the lightning repo and run the following command: - -.. code-block:: bash - - lightning run app docs/source-app/examples/github_repo_runner/app.py - -Add **--cloud** to run this application in the cloud. - -.. code-block:: bash - - lightning run app docs/source-app/examples/github_repo_runner/app.py --cloud - ----- - -****************** -Find more examples -****************** - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Build a DAG - :description: Create a dag pipeline - :col_css: col-md-4 - :button_link: ../dag/dag.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Build a File Server - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../file_server/file_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Build a HPO Sweeper - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../hpo/hpo.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Build a Model Server - :description: Serve multiple models with different parameters - :col_css: col-md-4 - :button_link: ../model_server/model_server.html - :height: 150 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/hands_on_example.rst b/docs/source-app/examples/hands_on_example.rst deleted file mode 100644 index 57fa1e5ff1..0000000000 --- a/docs/source-app/examples/hands_on_example.rst +++ /dev/null @@ -1,50 +0,0 @@ -:orphan: - -################# -Hands-on Examples -################# - -.. raw:: html - -
-
- -.. displayitem:: - :header: Build a DAG - :description: Learn how to orchestrate workflows - :col_css: col-md-6 - :button_link: dag/dag.html - :height: 180 - -.. displayitem:: - :header: Build a File Server - :description: Learn how to upload and download files - :col_css: col-md-6 - :button_link: file_server/file_server.html - :height: 180 - -.. displayitem:: - :header: Build a Github Repo Script Runner - :description: Learn how to configure dynamic execution from the UI - :col_css: col-md-6 - :button_link: github_repo_runner/github_repo_runner.html - :height: 180 - -.. displayitem:: - :header: Build a HPO Sweeper - :description: Learn how to scale your training - :col_css: col-md-6 - :button_link: hpo/hpo.html - :height: 180 - -.. displayitem:: - :header: Build a Model Server - :description: Learn how to server your models - :col_css: col-md-6 - :button_link: model_server_app_content.html - :height: 180 - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/hpo/build_from_scratch.rst b/docs/source-app/examples/hpo/build_from_scratch.rst deleted file mode 100644 index cade8b7f6e..0000000000 --- a/docs/source-app/examples/hpo/build_from_scratch.rst +++ /dev/null @@ -1,41 +0,0 @@ -:orphan: - -####################################### -Implement an HPO component from scratch -####################################### - -**Audience:** Users who want to understand how to implement sweep training from scratch. - -**Prereqs:** Finish Intermediate Level. - ----- - -******** -Examples -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Step 1: Implement an HPO component with the Lightning Works. - :description: Learn how it works under the hood - :col_css: col-md-4 - :button_link: hpo_wo.html - :height: 180 - :tag: Intermediate - -.. displayitem:: - :header: Step 2: Add the flow to your HPO component - :description: Learn how it works under the hood - :col_css: col-md-4 - :button_link: hpo_wi.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/hpo/hpo.rst b/docs/source-app/examples/hpo/hpo.rst deleted file mode 100644 index 2849a62653..0000000000 --- a/docs/source-app/examples/hpo/hpo.rst +++ /dev/null @@ -1,80 +0,0 @@ -.. hpo: -.. _hpo_example: - - -####################################################### -Build a Lightning Hyperparameter Optimization (HPO) App -####################################################### - -******************* -A bit of background -******************* - -Traditionally, developing machine learning (ML) products requires choosing among a large space of -hyperparameters while creating and training the ML models. Hyperparameter optimization -(HPO) aims to find a well-performing hyperparameter configuration for a given ML model -on a dataset at hand, including the ML model, -its hyperparameters, and other data processing steps. - -HPOs free the human expert from a tedious and error-prone, manual hyperparameter tuning process. - -As an example, in the famous `scikit-learn `_ library, -hyperparameters are passed as arguments to the constructor of -the estimator classes such as ``C`` kernel for -`Support Vector Classifier `_, etc. - -It is possible and recommended to search the hyperparameter space for the best validation score. - -An HPO search consists of: - -* an objective method -* a defined parameter space -* a method for searching or sampling candidates - -A naive method for sampling candidates is grid search, which exhaustively considers all -hyperparameter combinations from a user-specified grid. - -Fortunately, HPO is an active area of research, and many methods have been developed to -optimize the time required to get strong candidates. - -In the following tutorial, you will learn how to use Lightning together with `Optuna `_. - -`Optuna `_ is an open source HPO framework to automate hyperparameter search. -Out-of-the-box, it provides efficient algorithms to search large spaces and prune unpromising trials for faster results. - -First, you will learn about the best practices on how to implement HPO without the Lightning Framework. -Secondly, we will dive into a working HPO application with Lightning, and finally create a neat -`HiPlot UI `_ -for our application. - ----- - -******** -Examples -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Re-use an existing HPO component - :description: Learn how to use Lightning HPO with your app. - :col_css: col-md-4 - :button_link: lightning_hpo.html - :height: 180 - :tag: Basic - -.. displayitem:: - :header: Implement an HPO component from scratch - :description: Learn how it works under the hood - :col_css: col-md-4 - :button_link: build_from_scratch.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/hpo/hpo_wi.rst b/docs/source-app/examples/hpo/hpo_wi.rst deleted file mode 100644 index 17dd971e9e..0000000000 --- a/docs/source-app/examples/hpo/hpo_wi.rst +++ /dev/null @@ -1,57 +0,0 @@ -:orphan: - -########################################## -Step 2: Add the flow to your HPO component -########################################## - -**Audience:** Users who want to understand how to implement HPO training from scratch with Lightning. - -**Prereqs:** Level 17+ - ----- - -Thanks to the simplified version, you should have a good grasp on how to implement HPO with Optuna. - -As the :class:`~lightning_app.core.app.LightningApp` handles the Infinite Loop, -it has been removed from within the run method of the HPORootFlow. - -However, the ``run`` method code is the same as the one defined above. - -.. literalinclude:: ../../../examples/app_hpo/app_wo_ui.py - :language: python - -The ``ObjectiveWork`` is sub-classing -the built-in :class:`~lightning_app.components.python.TracerPythonScript` -which enables launching scripts and more. - -.. literalinclude:: ../../../examples/app_hpo/objective.py - :language: python - -Finally, let's add the ``HiPlotFlow`` component to visualize our hyperparameter optimization. - -The metric and sampled parameters are added to the ``self.hi_plot.data`` list, enabling -updates to the dashboard in near-realtime. - -.. literalinclude:: ../../../examples/app_hpo/app_wi_ui.py - :diff: ../../../examples/app_hpo/app_wo_ui.py - -Here is the associated code with the ``HiPlotFlow`` component. - -In the ``render_fn`` method, the state of the ``HiPlotFlow`` is passed. -The ``state.data`` is accessed as it contains the metric and sampled parameters. - -.. literalinclude:: ../../../examples/app_hpo/hyperplot.py - -Run the HPO application with the following command: - -.. code-block:: console - - $ lightning run app examples/app_hpo/app_wi_ui.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - {0: ..., 1: ..., ..., 5: ...} - -Here is what the UI looks like when launched: - -.. image:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/hpo_ui_2.gif - :width: 100 % - :alt: Alternative text diff --git a/docs/source-app/examples/hpo/hpo_wo.rst b/docs/source-app/examples/hpo/hpo_wo.rst deleted file mode 100644 index 6a13ff253d..0000000000 --- a/docs/source-app/examples/hpo/hpo_wo.rst +++ /dev/null @@ -1,57 +0,0 @@ -:orphan: - -########################################################### -Step 1: Implement an HPO component with the Lightning Works -########################################################### - -**Audience:** Users who want to understand how to implement HPO training from scratch. - -**Prereqs:** Level 17+ - ----- - -In the example below, we are emulating the Lightning Infinite Loop. - -We are assuming we have already defined an ``ObjectiveWork`` component which is responsible to run the objective method and track the metric through its state. - -.. literalinclude:: ./hpo.py - :language: python - -We are running ``TOTAL_TRIALS`` trials by series of ``SIMULTANEOUS_TRIALS`` trials. -When starting, ``TOTAL_TRIALS`` ``ObjectiveWork`` are created. - -The entire code runs within an infinite loop as it would within Lightning. - -When iterating through the Works, if the current ``objective_work`` hasn't started, -some new parameters are sampled from the Optuna Study with our custom distributions -and then passed to run method of the ``objective_work``. - -The condition ``not objective_work.has_started`` will be ``False`` once ``objective_work.run()`` starts. - -Also, the second condition ``objective_work.has_told_study`` will be ``True`` when the metric -is defined within the state of the Work and has been shared with the study. - -Finally, once the current ``SIMULTANEOUS_TRIALS`` have both registered their -metric to the Optuna Study, simply increment ``NUM_TRIALS`` by ``SIMULTANEOUS_TRIALS`` to launch the next trials. - -Below, you can find the simplified version of the ``ObjectiveWork`` where the metric is randomly sampled using NumPy. - -In a realistic use case, the Work executes some user-defined code. - -.. literalinclude:: ./objective.py - :language: python - -Here are the logs produced when running the application above: - -.. code-block:: console - - $ python docs/source-app/tutorials/hpo/hpo.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - # After you have clicked `run` on the UI. - [I 2022-03-01 12:32:50,050] A new study created in memory with name: ... - {0: 13.994859806481264, 1: 59.866743330127825, ..., 5: 94.65919769609225} - -The following animation shows how this application works in the cloud: - -.. image:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/hpo.gif - :alt: Animation showing how to HPO works UI in a distributed manner. diff --git a/docs/source-app/examples/hpo/lightning_hpo.rst b/docs/source-app/examples/hpo/lightning_hpo.rst deleted file mode 100644 index b1e2f11d39..0000000000 --- a/docs/source-app/examples/hpo/lightning_hpo.rst +++ /dev/null @@ -1,99 +0,0 @@ -:orphan: - -################################ -Re-use an existing HPO component -################################ - -**Audience:** Users who want to easily get started with HPO training. - -**Prereqs:** Level 8+ - ----- - -********************* -Install Lightning HPO -********************* - -Lightning HPO provides a Pythonic implementation for Scalable Hyperparameter Tuning -and relies on Optuna for providing state-of-the-art sampling hyper-parameters algorithms and efficient trial pruning strategies. - -Find the `Lightning Sweeper App `_ on `lightning.ai `_ and its associated `Github repo `_. - -.. code-block:: bash - - lightning install app lightning/hpo - -********************* -Lightning HPO Example -********************* - -In this tutorial, we are going to convert `Optuna Efficient Optimization Algorithms `_ into a Lightning App. - -The Optuna example optimizes the value (example: learning-rate) of a ``SGDClassifier`` from ``sklearn`` trained over the `Iris Dataset `_. - -.. literalinclude:: ./optuna_reference.py - :language: python - - -As you can see, several trials were pruned (stopped) before they finished all of the iterations. - -.. code-block:: console - - A new study created in memory with name: no-name-4423c12c-22e1-4eaf-ba60-caf0020403c6 - Trial 0 finished with value: 0.07894736842105265 and parameters: {'alpha': 0.00020629773477269024}. Best is trial 0 with value: 0.07894736842105265. - Trial 1 finished with value: 0.368421052631579 and parameters: {'alpha': 0.0005250149151047217}. Best is trial 0 with value: 0.07894736842105265. - Trial 2 finished with value: 0.052631578947368474 and parameters: {'alpha': 5.9086862655635784e-05}. Best is trial 2 with value: 0.052631578947368474. - Trial 3 finished with value: 0.3421052631578947 and parameters: {'alpha': 0.07177263583415294}. Best is trial 2 with value: 0.052631578947368474. - Trial 4 finished with value: 0.23684210526315785 and parameters: {'alpha': 1.7451874636151302e-05}. Best is trial 2 with value: 0.052631578947368474. - Trial 5 pruned. - Trial 6 finished with value: 0.10526315789473684 and parameters: {'alpha': 1.4943994864178649e-05}. Best is trial 2 with value: 0.052631578947368474. - Trial 7 pruned. - Trial 8 pruned. - Trial 9 pruned. - Trial 10 pruned. - Trial 11 pruned. - Trial 12 pruned. - Trial 13 pruned. - Trial 14 pruned. - Trial 15 pruned. - Trial 16 finished with value: 0.07894736842105265 and parameters: {'alpha': 0.006166329613687364}. Best is trial 2 with value: 0.052631578947368474. - Trial 17 pruned. - Trial 18 pruned. - Trial 19 pruned. - -The example above has been re-organized in order to run as Lightning App. - -.. literalinclude:: ./lightning_hpo_target.py - :language: python - -Now, your code can run at scale in the cloud, if needed, and it has a simple neat UI. - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/lightning_hpo_optimizer.png - :alt: Lightning App UI - :width: 100 % - -As you can see, several trials were pruned (stopped) before they finished all of the iterations. Same as when using pure optuna. - -.. code-block:: console - - A new study created in memory with name: no-name-a93d848e-a225-4df3-a9c3-5f86680e295d - Trial 0 finished with value: 0.23684210526315785 and parameters: {'alpha': 0.006779437004523296}. Best is trial 0 with value: 0.23684210526315785. - Trial 1 finished with value: 0.07894736842105265 and parameters: {'alpha': 0.008936151407006062}. Best is trial 1 with value: 0.07894736842105265. - Trial 2 finished with value: 0.052631578947368474 and parameters: {'alpha': 0.0035836511240528008}. Best is trial 2 with value: 0.052631578947368474. - Trial 3 finished with value: 0.052631578947368474 and parameters: {'alpha': 0.0005393218926409795}. Best is trial 2 with value: 0.052631578947368474. - Trial 4 finished with value: 0.1578947368421053 and parameters: {'alpha': 6.572557493358585e-05}. Best is trial 2 with value: 0.052631578947368474. - Trial 5 finished with value: 0.02631578947368418 and parameters: {'alpha': 0.0013953760106345603}. Best is trial 5 with value: 0.02631578947368418. - Trail 6 pruned. - Trail 7 pruned. - Trail 8 pruned. - Trail 9 pruned. - Trial 10 finished with value: 0.07894736842105265 and parameters: {'alpha': 0.00555435554783454}. Best is trial 5 with value: 0.02631578947368418. - Trail 11 pruned. - Trial 12 finished with value: 0.052631578947368474 and parameters: {'alpha': 0.025624276147153992}. Best is trial 5 with value: 0.02631578947368418. - Trial 13 finished with value: 0.07894736842105265 and parameters: {'alpha': 0.014613957457075546}. Best is trial 5 with value: 0.02631578947368418. - Trail 14 pruned. - Trail 15 pruned. - Trail 16 pruned. - Trial 17 finished with value: 0.052631578947368474 and parameters: {'alpha': 0.01028208215647372}. Best is trial 5 with value: 0.02631578947368418. - Trail 18 pruned. - Trail 19 pruned. diff --git a/docs/source-app/examples/hpo/lightning_hpo_target.py b/docs/source-app/examples/hpo/lightning_hpo_target.py deleted file mode 100644 index 779f992554..0000000000 --- a/docs/source-app/examples/hpo/lightning_hpo_target.py +++ /dev/null @@ -1,53 +0,0 @@ -import optuna -from lightning_hpo import BaseObjective, Optimizer -from optuna.distributions import LogUniformDistribution -from sklearn import datasets -from sklearn.linear_model import SGDClassifier -from sklearn.model_selection import train_test_split - -from lightning import LightningApp, LightningFlow - - -class Objective(BaseObjective): - def run(self, params): - # WARNING: Don't forget to assign `params` to self, - # so they get tracked in the state. - self.params = params - - iris = datasets.load_iris() - classes = list(set(iris.target)) - train_x, valid_x, train_y, valid_y = train_test_split(iris.data, iris.target, test_size=0.25, random_state=0) - - clf = SGDClassifier(alpha=params["alpha"]) - - for step in range(100): - clf.partial_fit(train_x, train_y, classes=classes) - intermediate_value = 1.0 - clf.score(valid_x, valid_y) - - # WARNING: Assign to reports, - # so the state is instantly sent to the flow. - self.reports = self.reports + [[intermediate_value, step]] - - self.best_model_score = 1.0 - clf.score(valid_x, valid_y) - - def distributions(self): - return {"alpha": LogUniformDistribution(1e-5, 1e-1)} - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - self.optimizer = Optimizer( - objective_cls=Objective, - n_trials=20, - study=optuna.create_study(pruner=optuna.pruners.MedianPruner()), - ) - - def run(self): - self.optimizer.run() - - def configure_layout(self): - return {"name": "HyperPlot", "content": self.optimizer.hi_plot} - - -app = LightningApp(RootFlow()) diff --git a/docs/source-app/examples/hpo/optuna_reference.py b/docs/source-app/examples/hpo/optuna_reference.py deleted file mode 100644 index 46f76c8662..0000000000 --- a/docs/source-app/examples/hpo/optuna_reference.py +++ /dev/null @@ -1,36 +0,0 @@ -import logging -import sys - -import optuna -from sklearn import datasets -from sklearn.linear_model import SGDClassifier -from sklearn.model_selection import train_test_split - - -def objective(trial): - iris = datasets.load_iris() - classes = list(set(iris.target)) - train_x, valid_x, train_y, valid_y = train_test_split(iris.data, iris.target, test_size=0.25, random_state=0) - - alpha = trial.suggest_float("alpha", 1e-5, 1e-1, log=True) - clf = SGDClassifier(alpha=alpha) - - for step in range(100): - clf.partial_fit(train_x, train_y, classes=classes) - - # Report intermediate objective value. - intermediate_value = 1.0 - clf.score(valid_x, valid_y) - trial.report(intermediate_value, step) - - # Handle pruning based on the intermediate value. - if trial.should_prune(): - raise optuna.TrialPruned() - - return 1.0 - clf.score(valid_x, valid_y) - - -# Add stream handler of stdout to show the messages -logger = optuna.logging.get_logger("optuna") -logger.addHandler(logging.StreamHandler(sys.stdout)) -study = optuna.create_study(pruner=optuna.pruners.MedianPruner()) -study.optimize(objective, n_trials=20) diff --git a/docs/source-app/examples/model_deploy_app.rst b/docs/source-app/examples/model_deploy_app.rst new file mode 100644 index 0000000000..f4f564d773 --- /dev/null +++ b/docs/source-app/examples/model_deploy_app.rst @@ -0,0 +1,3 @@ +############################ +Build a Model Deployment App +############################ diff --git a/docs/source-app/examples/model_server_app/app.py b/docs/source-app/examples/model_server_app/app.py deleted file mode 100644 index 9985014b11..0000000000 --- a/docs/source-app/examples/model_server_app/app.py +++ /dev/null @@ -1,34 +0,0 @@ -from locust_component import Locust -from model_server import MLServer -from train import TrainModel - -from lightning import LightningApp, LightningFlow - - -class TrainAndServe(LightningFlow): - def __init__(self): - super().__init__() - self.train_model = TrainModel() - self.model_server = MLServer( - name="mnist-svm", - implementation="mlserver_sklearn.SKLearnModel", - workers=8, - ) - self.performance_tester = Locust(num_users=100) - - def run(self): - self.train_model.run() - self.model_server.run(self.train_model.best_model_path) - if self.model_server.alive(): - # The performance tester needs the model server to be up - # and running to be started, so the URL is added in the UI. - self.performance_tester.run(self.model_server.url) - - def configure_layout(self): - return [ - {"name": "Server", "content": self.model_server.url + "/docs"}, - {"name": "Server Testing", "content": self.performance_tester}, - ] - - -app = LightningApp(TrainAndServe()) diff --git a/docs/source-app/examples/model_server_app/load_testing.rst b/docs/source-app/examples/model_server_app/load_testing.rst deleted file mode 100644 index 97345dec4c..0000000000 --- a/docs/source-app/examples/model_server_app/load_testing.rst +++ /dev/null @@ -1,57 +0,0 @@ -:orphan: - -*********************************** -3. Build the Load Testing Component -*********************************** - -Now, we are going to create a component to test the performance of your model server. - -We are going to use a python performance testing tool called `Locust `_. - -.. literalinclude:: ./locust_component.py - - -Finally, once the component is done, we need to crate a ``locustfile.py`` file which defines the format of the request to send to your model server. - -The endpoint to hit has the following format: ``/v2/models/{MODEL_NAME}/versions/{VERSION}/infer``. - -.. literalinclude:: ./locustfile.py - - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1. Build a Train Component - :description: Train a model and store its checkpoints with SKlearn - :col_css: col-md-4 - :button_link: train.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 2. Build a Model Server Component - :description: Use MLServer to server your models - :col_css: col-md-4 - :button_link: model_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 4. Putting everything together. - :description: Ensemble the components together and run the app - :col_css: col-md-4 - :button_link: putting_everything_together.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/model_server_app/locust_component.py b/docs/source-app/examples/model_server_app/locust_component.py deleted file mode 100644 index 70b342facb..0000000000 --- a/docs/source-app/examples/model_server_app/locust_component.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import subprocess - -from lightning import LightningWork -from lightning.app.utilities.packaging.build_config import BuildConfig - - -class Locust(LightningWork): - def __init__(self, num_users: int = 100): - """This component checks the performance of a server. The server url is passed to its run method. - - Arguments: - num_users: Number of users emulated by Locust - """ - # Note: Using the default port 8089 of Locust. - super().__init__( - port=8089, - parallel=True, - cloud_build_config=BuildConfig(requirements=["locust"]), - ) - self.num_users = num_users - - def run(self, load_tested_url: str): - # 1: Create the locust command line. - cmd = " ".join( - [ - "locust", - "--master-host", - str(self.host), - "--master-port", - str(self.port), - "--host", - str(load_tested_url), - "-u", - str(self.num_users), - ] - ) - # 2: Create another process with locust - process = subprocess.Popen(cmd, cwd=os.path.dirname(__file__), shell=True) - - # 3: Wait for the process to finish. As locust is a server, - # this waits infinitely or if killed. - process.wait() diff --git a/docs/source-app/examples/model_server_app/locustfile.py b/docs/source-app/examples/model_server_app/locustfile.py deleted file mode 100644 index 198d6de6cb..0000000000 --- a/docs/source-app/examples/model_server_app/locustfile.py +++ /dev/null @@ -1,41 +0,0 @@ -from locust import FastHttpUser, task -from sklearn import datasets -from sklearn.model_selection import train_test_split - - -class HelloWorldUser(FastHttpUser): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._prepare_inference_request() - - @task - def predict(self): - self.client.post( - "/v2/models/mnist-svm/versions/v0.0.1/infer", - json=self.inference_request, - ) - - def _prepare_inference_request(self): - # The digits dataset - digits = datasets.load_digits() - - # To apply a classifier on this data, - # we need to flatten the image, to - # turn the data in a (samples, feature) matrix: - n_samples = len(digits.images) - data = digits.images.reshape((n_samples, -1)) - - # Split data into train and test subsets - _, X_test, _, _ = train_test_split(data, digits.target, test_size=0.5, shuffle=False) - - x_0 = X_test[0:1] - self.inference_request = { - "inputs": [ - { - "name": "predict", - "shape": x_0.shape, - "datatype": "FP32", - "data": x_0.tolist(), - } - ] - } diff --git a/docs/source-app/examples/model_server_app/model_server.py b/docs/source-app/examples/model_server_app/model_server.py deleted file mode 100644 index 1b914b0194..0000000000 --- a/docs/source-app/examples/model_server_app/model_server.py +++ /dev/null @@ -1,90 +0,0 @@ -import json -import subprocess - -from lightning import LightningWork -from lightning.app.storage import Path -from lightning.app.utilities.packaging.build_config import BuildConfig - -# ML_SERVER_URL = https://github.com/SeldonIO/MLServer - - -class MLServer(LightningWork): - - """This components uses SeldonIO MLServer library. - - The model endpoint: /v2/models/{MODEL_NAME}/versions/{VERSION}/infer. - - Arguments: - name: The name of the model for the endpoint. - implementation: The model loader class. - Example: "mlserver_sklearn.SKLearnModel". - Learn more here: $ML_SERVER_URL/tree/master/runtimes - workers: Number of server worker. - """ - - def __init__( - self, - name: str, - implementation: str, - workers: int = 1, - **kwargs, - ): - super().__init__( - parallel=True, - cloud_build_config=BuildConfig( - requirements=["mlserver", "mlserver-sklearn"], - ), - **kwargs, - ) - # 1: Collect the config's. - self.settings = { - "debug": True, - "parallel_workers": workers, - } - self.model_settings = { - "name": name, - "implementation": implementation, - } - # 2: Keep track of latest version - self.version = 1 - - def run(self, model_path: Path): - """The model is downloaded when the run method is invoked. - - Arguments: - model_path: The path to the trained model. - """ - # 1: Use the host and port at runtime so it works in the cloud. - # $ML_SERVER_URL/blob/master/mlserver/settings.py#L50 - if self.version == 1: - # TODO: Reload the next version model of the model. - - self.settings.update({"host": self.host, "http_port": self.port}) - - with open("settings.json", "w") as f: - json.dump(self.settings, f) - - # 2. Store the model-settings - # $ML_SERVER_URL/blob/master/mlserver/settings.py#L120 - self.model_settings["parameters"] = { - "version": f"v0.0.{self.version}", - "uri": str(model_path.absolute()), - } - with open("model-settings.json", "w") as f: - json.dump(self.model_settings, f) - - # 3. Launch the Model Server - subprocess.Popen("mlserver start .", shell=True) - - # 4. Increment the version for the next time run is called. - self.version += 1 - - else: - # TODO: Load the next model and unload the previous one. - pass - - def alive(self): - # Current hack, when the url is available, - # the server is up and running. - # This would be cleaned out and automated. - return self.url != "" diff --git a/docs/source-app/examples/model_server_app/model_server.rst b/docs/source-app/examples/model_server_app/model_server.rst deleted file mode 100644 index b1daa00d42..0000000000 --- a/docs/source-app/examples/model_server_app/model_server.rst +++ /dev/null @@ -1,48 +0,0 @@ -:orphan: - -*********************************** -2. Build the Model Server Component -*********************************** - -In the code below, we use `MLServer `_ which aims to provide an easy way to start serving your machine learning models through a REST and gRPC interface, -fully compliant with KFServing's V2 Dataplane spec. - -.. literalinclude:: ./model_server.py - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1. Build a Train Component - :description: Train a model and store its checkpoints with SKlearn - :col_css: col-md-4 - :button_link: train.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 3. Build a Load Testing Component - :description: Use Locust to test your model servers - :col_css: col-md-4 - :button_link: load_testing.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 4. Putting everything together. - :description: Ensemble the components together and run the app - :col_css: col-md-4 - :button_link: putting_everything_together.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/model_server_app/model_server_app.rst b/docs/source-app/examples/model_server_app/model_server_app.rst deleted file mode 100644 index 09d3619929..0000000000 --- a/docs/source-app/examples/model_server_app/model_server_app.rst +++ /dev/null @@ -1,15 +0,0 @@ -:orphan: - -.. _model_server_example: - -#################### -Build a Model Server -#################### - -**Audience:** Users who want to serve their trained models. - -**Prerequisite**: Reach :ref:`level 16+ `. - ----- - -.. include:: model_server_app_content.rst diff --git a/docs/source-app/examples/model_server_app/model_server_app_content.rst b/docs/source-app/examples/model_server_app/model_server_app_content.rst deleted file mode 100644 index 8675de939a..0000000000 --- a/docs/source-app/examples/model_server_app/model_server_app_content.rst +++ /dev/null @@ -1,84 +0,0 @@ - -********* -Objective -********* - -Create a simple application that trains and serves a `Sklearn `_ machine learning model with `MLServer from SeldonIO `_ - ----- - -***************** -Final Application -***************** - -Here is a gif of the final application built in this example. - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/ml_server_2.gif - ----- - -************* -System Design -************* - -In order to create such application, we need to build several components: - -* A Model Train Component that trains a model and provides its trained weights - -* A Model Server Component that serves as an API endpoint for the model generated by the **Model Train Component**. - -* A Load Testing Component that tests the model server works as expected. This could be used to CI/CD the performance of newly generated models (left to the users). - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/model_server_app_2.png - -Let's dive into the tutorial. - ----- - -******** -Tutorial -******** - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 1. Build a Train Component - :description: Train a model and store its checkpoints with SKlearn - :col_css: col-md-4 - :button_link: train.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 2. Build a Model Server Component - :description: Use MLServer to server your models - :col_css: col-md-4 - :button_link: model_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 3. Build a Load Testing Component - :description: Use Locust to test your model servers - :col_css: col-md-4 - :button_link: load_testing.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 4. Putting everything together. - :description: Ensemble the components together and run the app - :col_css: col-md-4 - :button_link: putting_everything_together.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/model_server_app/putting_everything_together.rst b/docs/source-app/examples/model_server_app/putting_everything_together.rst deleted file mode 100644 index c11a5289d3..0000000000 --- a/docs/source-app/examples/model_server_app/putting_everything_together.rst +++ /dev/null @@ -1,80 +0,0 @@ -:orphan: - -****************************** -4. Putting everything together -****************************** - -In the code below, we put together the **TrainWork**, the **MLServer** and the **Locust** components in an ``app.py`` file. - -.. literalinclude:: ./app.py - - -*********** -Run the App -*********** - -To run the app, simply open a terminal and execute this command: - -.. code-block:: bash - - lightning run app docs/source-app/examples/model_deploy_app/app.py - -Here is a gif of the UI. - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/ml_server_2.gif - -.. raw:: html - -
- -Congrats, you have finished the **Build a Model Server** example ! - ----- - -****************** -Find more examples -****************** - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Build a DAG - :description: Create a dag pipeline - :col_css: col-md-4 - :button_link: ../dag/dag.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Build a File Server - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../file_server/file_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Build a Github Repo Script Runner - :description: Run code from the internet in the cloud - :col_css: col-md-4 - :button_link: ../github_repo_runner/github_repo_runner.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Build a HPO Sweeper - :description: Train multiple models with different parameters - :col_css: col-md-4 - :button_link: ../hpo/hpo.html - :height: 150 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/model_server_app/train.py b/docs/source-app/examples/model_server_app/train.py deleted file mode 100644 index f3342b9f24..0000000000 --- a/docs/source-app/examples/model_server_app/train.py +++ /dev/null @@ -1,42 +0,0 @@ -import joblib -from sklearn import datasets, svm -from sklearn.model_selection import train_test_split - -from lightning import LightningWork -from lightning.app.storage import Path - - -class TrainModel(LightningWork): - - """This component trains a Sklearn SVC model on digits dataset.""" - - def __init__(self): - super().__init__() - # 1: Add element to the state. - self.best_model_path = None - - def run(self): - # 2: Load the Digits - digits = datasets.load_digits() - - # 3: To apply a classifier on this data, - # we need to flatten the image, to - # turn the data in a (samples, feature) matrix: - n_samples = len(digits.images) - data = digits.images.reshape((n_samples, -1)) - - # 4: Create a classifier: a support vector classifier - classifier = svm.SVC(gamma=0.001) - - # 5: Split data into train and test subsets - X_train, _, y_train, _ = train_test_split(data, digits.target, test_size=0.5, shuffle=False) - - # 6: We learn the digits on the first half of the digits - classifier.fit(X_train, y_train) - - # 7: Save the Sklearn model with `joblib`. - model_file_name = "mnist-svm.joblib" - joblib.dump(classifier, model_file_name) - - # 8: Keep a reference the the generated model. - self.best_model_path = Path("mnist-svm.joblib") diff --git a/docs/source-app/examples/model_server_app/train.rst b/docs/source-app/examples/model_server_app/train.rst deleted file mode 100644 index 4e828872f4..0000000000 --- a/docs/source-app/examples/model_server_app/train.rst +++ /dev/null @@ -1,49 +0,0 @@ -:orphan: - -**************************** -1. Build the Train Component -**************************** - -In the code below, we create a work which trains a simple `SVC `_ model on the digits dataset (classification). - -Once the model is trained, it is saved and a reference :class:`~lightning_app.storage.path.Path` with ``best_model_path`` state attribute. - -.. literalinclude:: ./train.py - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: 2. Build a Model Server Component - :description: Use MLServer to server your models - :col_css: col-md-4 - :button_link: model_server.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 3. Build a Load Testing Component - :description: Use Locust to test your model servers - :col_css: col-md-4 - :button_link: load_testing.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: 4. Putting everything together. - :description: Ensemble the components together and run the app - :col_css: col-md-4 - :button_link: putting_everything_together.html - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/examples/research_demo_app.rst b/docs/source-app/examples/research_demo_app.rst index 90276f9b95..cb74ebcf3d 100644 --- a/docs/source-app/examples/research_demo_app.rst +++ b/docs/source-app/examples/research_demo_app.rst @@ -1,5 +1,3 @@ -:orphan: - ######################### Build a Research Demo App ######################### diff --git a/docs/source-app/get_started/add_an_interactive_demo.rst b/docs/source-app/get_started/add_an_interactive_demo.rst deleted file mode 100644 index 98e6c3fd8a..0000000000 --- a/docs/source-app/get_started/add_an_interactive_demo.rst +++ /dev/null @@ -1,18 +0,0 @@ -:orphan: - -####################### -Add an Interactive Demo -####################### - -.. _add_an_interactive_Demo: - -**Required background:** Basic Python familiarity and complete the :ref:`install` guide. - -**Goal:** We'll walk you through the 4 key steps to run a Lightning App that trains and demos a model. - -.. join_slack:: - :align: left - ----- - -.. include:: go_beyond_training_content.rst diff --git a/docs/source-app/get_started/build_model.rst b/docs/source-app/get_started/build_model.rst deleted file mode 100644 index 34e7b26bc3..0000000000 --- a/docs/source-app/get_started/build_model.rst +++ /dev/null @@ -1,76 +0,0 @@ -:orphan: - -.. _build_model: - -####################### -Build and Train a Model -####################### - -**Required background:** Basic Python familiarity and complete the :ref:`install` guide. - -**Goal:** We'll walk you through the creation of a model using PyTorch Lightning. - -.. join_slack:: - :align: left - ----- - -********************************* -A simple PyTorch Lightning script -********************************* - -Let's assume you already have a folder with those two files. - -.. code-block:: bash - - pl_project/ - train.py # your own script to train your models - requirements.txt # your python requirements. - -If you don't, simply create a ``pl_project`` folder with those two files and add the following `PyTorch Lightning `_ code in the ``train.py`` file. This code trains a simple ``AutoEncoder`` on `MNIST Dataset `_. - -.. literalinclude:: ../code_samples/convert_pl_to_app/train.py - -Add the following to the ``requirements.txt`` file. - -.. literalinclude:: ../code_samples/convert_pl_to_app/requirements.py - -Simply run the following commands in your terminal to install the requirements and train the model. - -.. code-block:: bash - - pip install -r requirements.txt - python train.py - -Get through `PyTorch Lightning Introduction `_ to learn more. - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
-
- -.. displayitem:: - :header: Evolve a Model into an ML System - :description: Develop an App to train a model in the cloud - :col_css: col-md-6 - :button_link: training_with_apps.html - :height: 180 - -.. displayitem:: - :header: Start from a Template ML System - :description: Learn about Apps, from a template. - :col_css: col-md-6 - :button_link: go_beyond_training.html - :height: 180 - -.. raw:: html - -
-
diff --git a/docs/source-app/get_started/go_beyond_training.rst b/docs/source-app/get_started/go_beyond_training.rst deleted file mode 100644 index eade993e59..0000000000 --- a/docs/source-app/get_started/go_beyond_training.rst +++ /dev/null @@ -1,18 +0,0 @@ -:orphan: - -################################ -Start from an ML system template -################################ - -.. _go_beyond_training: - -**Required background:** Basic Python familiarity and complete the :ref:`install` guide. - -**Goal:** We'll walk you through the 4 key steps to run a Lightning App that trains and demos a model. - -.. join_slack:: - :align: left - ----- - -.. include:: go_beyond_training_content.rst diff --git a/docs/source-app/get_started/jumpstart_from_app_gallery.rst b/docs/source-app/get_started/jumpstart_from_app_gallery.rst deleted file mode 100644 index 998fce9a93..0000000000 --- a/docs/source-app/get_started/jumpstart_from_app_gallery.rst +++ /dev/null @@ -1,123 +0,0 @@ -:orphan: - -##################################### -Start from Ready-to-Run Template Apps -##################################### - -.. _jumpstart_from_app_gallery: - -Anyone can build Apps for their own use cases and promote them on the `App Gallery `_. - -In return, you can benefit from the work of others and get started faster by re-using a ready-to-run App close to your own use case. - -.. join_slack:: - :align: left - ----- - -************* -User Workflow -************* - -#. Visit the `App Gallery `_ and look for an App close to your own use case. - - .. raw:: html - -
- -#. If **Launch** is available, it means the App is live and ready to be used! Take it for a spin. - - .. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/launch_button.png - :alt: Launch Button on lightning.ai - :width: 100 % - -#. By clicking **Clone & Run**, a copy of the App is added to your account and an instance starts running. - - .. raw:: html - -
- - -#. If you found an App that matches what you need, move to **step 5**! Otherwise, go back to **step 1**. - - .. raw:: html - -
- -#. Copy the installation command (optionally from the clipboard on the right). - - .. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/install_command.png - :alt: Install command on lightning.ai - :width: 100 % - -#. Copy the command to your local terminal. - - .. code-block:: bash - - lightning install app lightning/hackernews-app - -#. Go through the installation steps. - - .. raw:: html - -
- - -#. Run the App locally. - - .. code-block:: bash - - cd LAI-Hackernews-App - lightning run app app.py - - .. raw:: html - -
- - -#. Open the code with your favorite IDE, modify it, and run it back in the cloud. - - .. raw:: html - -
- -
- ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Add Component made by others to your App - :description: Add more functionality to your projects - :col_css: col-md-6 - :button_link: jumpstart_from_component_gallery.html - :height: 180 - -.. displayitem:: - :header: Level-up your skills with Lightning Apps - :description: From Basic to Advanced Skills - :col_css: col-md-6 - :button_link: ../levels/basic/index.html - :height: 180 - -.. raw:: html - -
-
-
diff --git a/docs/source-app/get_started/jumpstart_from_component_gallery.rst b/docs/source-app/get_started/jumpstart_from_component_gallery.rst deleted file mode 100644 index 5d0a0e434a..0000000000 --- a/docs/source-app/get_started/jumpstart_from_component_gallery.rst +++ /dev/null @@ -1,155 +0,0 @@ -:orphan: - -######################################## -Add Component made by others to your App -######################################## - -.. _jumpstart_from_component_gallery: - -Anyone can build components for their own use case and promote them on the `Component Gallery `_. - -In return, you can benefit from the work of others and add new functionalities to your Apps with minimal effort. - -.. join_slack:: - :align: left - ----- - -************* -User Workflow -************* - -#. Visit the `Component Gallery `_ and look for a Component close to something you want to do. - - .. raw:: html - -
- -#. Check out the code for inspiration or simply install the component from PyPi and use it. - ----- - -************* -Success Story -************* - -The default `Train and Demo Application `_ trains a PyTorch Lightning -model and then starts a demo with `Gradio `_. - -.. code-block:: python - - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = PyTorchLightningScript( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - ) - - self.serve_work = ImageServeGradio(L.CloudCompute("cpu")) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - - app = L.LightningApp(TrainDeploy()) - -However, someone who wants to use this Aop (maybe you) found `Lightning HPO `_ -from browsing the `Component Gallery `_ and decided to give it a spin after checking the associated -`Github Repository `_. - -Once ``lightning_hpo`` installed, they improved the default App by easily adding HPO support to their project. - -Here is the resulting App. It is almost the same code, but it's way more powerful now! - -This is the power of `lightning.ai `_ ecosystem 🔥⚡🔥 - -.. code-block:: python - - import os.path as ops - import lightning as L - from quick_start.components import PyTorchLightningScript, ImageServeGradio - import optuna - from optuna.distributions import LogUniformDistribution - from lightning_hpo import Optimizer, BaseObjective - - - class HPOPyTorchLightningScript(PyTorchLightningScript, BaseObjective): - @staticmethod - def distributions(): - return {"model.lr": LogUniformDistribution(0.0001, 0.1)} - - - class TrainDeploy(L.LightningFlow): - def __init__(self): - super().__init__() - self.train_work = Optimizer( - script_path=ops.join(ops.dirname(__file__), "./train_script.py"), - script_args=["--trainer.max_epochs=5"], - objective_cls=HPOPyTorchLightningScript, - n_trials=4, - ) - - self.serve_work = ImageServeGradio(L.CloudCompute("cpu")) - - def run(self): - # 1. Run the python script that trains the model - self.train_work.run() - - # 2. when a checkpoint is available, deploy - if self.train_work.best_model_path: - self.serve_work.run(self.train_work.best_model_path) - - def configure_layout(self): - tab_1 = {"name": "Model training", "content": self.train_work.hi_plot} - tab_2 = {"name": "Interactive demo", "content": self.serve_work} - return [tab_1, tab_2] - - - app = L.LightningApp(TrainDeploy()) - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Start from Ready-to-Run Template Apps - :description: Jump-start your projects development - :col_css: col-md-6 - :button_link: jumpstart_from_app_gallery.html - :height: 180 - -.. displayitem:: - :header: Level-up your skills with Lightning Apps - :description: From Basic to Advanced Skills - :col_css: col-md-6 - :button_link: ../levels/basic/index.html - :height: 180 - -.. raw:: html - -
-
-
diff --git a/docs/source-app/get_started/lightning_apps_intro.rst b/docs/source-app/get_started/lightning_apps_intro.rst deleted file mode 100644 index bcbc9d8dbf..0000000000 --- a/docs/source-app/get_started/lightning_apps_intro.rst +++ /dev/null @@ -1,14 +0,0 @@ -############################ -Lightning Apps in 15 minutes -############################ - -**Required background:** Basic Python familiarity. - -**Goal:** Guide you to develop your first Lightning App or use an existing App from the `Apps Gallery `_. - -.. join_slack:: - :align: left - ----- - -.. include:: go_beyond_training_content.rst diff --git a/docs/source-app/get_started/training_with_apps.rst b/docs/source-app/get_started/training_with_apps.rst deleted file mode 100644 index a7061cae56..0000000000 --- a/docs/source-app/get_started/training_with_apps.rst +++ /dev/null @@ -1,136 +0,0 @@ -:orphan: - -################################ -Evolve a model into an ML system -################################ - -.. _convert_pl_to_app: - -**Required background:** Basic Python familiarity and complete the :ref:`build_model` guide. - -**Goal:** We'll walk you through the two key steps to build your first Lightning App from your existing Pytorch Lightning scripts. - -.. join_slack:: - :align: left - ----- - -******************* -Training and beyond -******************* - -With `PyTorch Lightning `_, we abstracted distributed training and hardware, by organizing PyTorch code. -With `Lightning Apps `_, we unified the local and cloud experience while abstracting infrastructure. - -By using `PyTorch Lightning `_ and `Lightning Apps `_ -together, a completely new world of possibilities emerges. - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/pl_to_app_4.png - :alt: From PyTorch Lightning to Lightning App - :width: 100 % - ----- - -****************************************** -1. Write an App to run the train.py script -****************************************** - -This article continues where the :ref:`build_model` guide finished. - -Create an additional file ``app.py`` in the ``pl_project`` folder as follows: - -.. code-block:: bash - - pl_project/ - app.py - train.py - requirements.txt - -Inside the ``app.py`` file, add the following code. - -.. literalinclude:: ../code_samples/convert_pl_to_app/app.py - -This App runs the Pytorch Lightning script contained in the ``train.py`` file using the powerful :class:`~lightning_app.components.python.tracer.TracerPythonScript` component. This is really worth checking out! - ----- - -************************************************ -2. Run the train.py file locally or in the cloud -************************************************ - -First, go to the ``pl_folder`` folder from the local terminal and install the requirements. - -.. code-block:: bash - - cd pl_folder - pip install -r requirements.txt - -To run your app, copy the following command to your local terminal: - -.. code-block:: bash - - lightning run app app.py - -Simply add ``--cloud`` to run this application in the cloud with a GPU machine 🤯 - -.. code-block:: bash - - lightning run app app.py --cloud - - -Congratulations! Now, you know how to run a `PyTorch Lightning `_ script with Lightning Apps. - -Lightning Apps can make your ML system way more powerful, keep reading to learn how. - ----- - -********** -Next Steps -********** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Level-up with Lightning Apps - :description: From Basics to Advanced Skills - :col_css: col-md-4 - :button_link: ../levels/basic/index.html - :height: 180 - -.. displayitem:: - :header: Add an Interactive Demo - :description: Add a Gradio Demo once the training is finished - :col_css: col-md-4 - :button_link: add_an_interactive_demo.html - :height: 180 - -.. displayitem:: - :header: Add Hyper Parameter Optimization - :description: Add a HPO to optimize your models - :col_css: col-md-4 - :button_link: ../examples/hpo/hpo.html - :height: 180 - -.. displayitem:: - :header: Add Model Serving - :description: Serve and load testing with MLServer and Locust - :col_css: col-md-4 - :button_link: ../examples/model_server_app/model_server_app.html - :height: 180 - -.. displayitem:: - :header: Add DAG Orchestration - :description: Organize your processing, training and metrics collection - :col_css: col-md-4 - :button_link: ../examples/dag/dag.html - :height: 180 - -.. displayitem:: - :header: Add Team Collaboration - :description: Create an app to run any PyTorch Lightning Script from Github - :col_css: col-md-4 - :button_link: ../examples/github_repo_runner/github_repo_runner.html - :height: 180 diff --git a/docs/source-app/get_started/what_app_can_do.rst b/docs/source-app/get_started/what_app_can_do.rst deleted file mode 100644 index a2c88a6f85..0000000000 --- a/docs/source-app/get_started/what_app_can_do.rst +++ /dev/null @@ -1,196 +0,0 @@ -:orphan: - -############################################ -Discover what Lightning Apps can do in 5 min -############################################ - -.. _what_app_can_do: - -Lightning Apps can be plenty things, and while a picture is worth a thousand words, videos showing you examples should be worth even more. - -.. join_slack:: - :align: left - ----- - -***************************** -Flashy - Auto ML App (Public) -***************************** - -Train a model on any image or text dataset without writing any code. Flashy uses `React.js `_ for its frontend. - -Find `Flashy `_ on the App Gallery and the `Flashy codebase. `_ on GitHub. - -.. raw:: html - - - -.. ---- - -.. *************************************** -.. NVIDIA Omniverse Sampling App (Private) -.. *************************************** - -.. Use `Nvidia Sampling Omniverse `_ to generate synthetic samples from 3D meshes and train an object detector on that data. - -.. .. raw:: html - -.. - ----- - -********************* -Research App (Public) -********************* - -Share your paper ``bundled`` with the arxiv link, poster, live jupyter notebook, interactive demo to try the model, and more! - -Find the `Research App `_ on the App Gallery and the `Research App codebase. `_ on GitHub. - -.. raw:: html - - - ----- - -************************************************ -ScratchPad - Notebook Manager for Team (Public) -************************************************ - -Run multiple Jupyter Notebooks on cloud CPUs or machines with multiple GPUs. - -Find the `ScratchPad App `_ on the App Gallery and the `ScratchPad App codebase `_ on GitHub. - -.. note:: ScratchPad is `tested end-to-end `_ on every Lightning App commit with `pytest `_. - -.. raw:: html - - - ----- - -****************************** -Lightning HPO Sweeper (Public) -****************************** - -Run a hyperparameter sweep over any model script across hundreds of cloud machines at once. This Lightning App uses Optuna to provide advanced tuning algorithms (from grid and random search to Hyperband). - -Find the `Lightning HPO Sweeper App `_ on the App Gallery and the `Lightning HPO Sweeper codebase. `_ on GitHub. - -.. raw:: html - - - ----- - -*********************** -InVideo Search (Public) -*********************** - -This App lets you find anything you're looking for inside a video. The engine is powered by `Open AI CLIP `_. - -Find the `InVideo Search App `_ on the App Gallery and the `InVideo Search App codebase. `_ in GitHub. - -.. raw:: html - - - ----- - -****************************** -AI-powered HackerNews (Public) -****************************** - -Save yourself time, and get Hacker News story recommendations, chosen for you specifically. This Lightning App was designed to illustrate a full end-to-end MLOPs workflow aimed at enterprise recommendation systems. - -Find the `AI-powered HackerNews App `_ on the App Gallery and the `AI-powered HackerNews App codebase. `_ on GitHub. - -.. raw:: html - - - ----- - -********************************************************************* -Lightning Apps can turn ML into scalable systems in days — not months -********************************************************************* - -Use the Lightning framework to develop any ML system: train and deploy a model, create an ETL pipeline, -or spin up a research demo — using the intuitive principles we pioneered with PyTorch Lightning. - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/apps_logos_2.png - :alt: Apps with Logos - :width: 100 % - -Anyone who knows Python can build a Lightning App, even without machine learning experience. - -Lightning Apps are: - -- cloud agnostic -- fault-tolerant, distributed, cost optimized -- production ready -- local and cloud debuggable -- highly reactive & interactive -- connect multiple UIs together -- built for team collaboration -- framework agnostic, use your own stack -- and much more - -.. raw:: html - -
- -
-
- -********** -Next Steps -********** - -.. raw:: html - -
-
-
- -.. displayitem:: - :header: Build & Train a Model - :description: Discover PyTorch Lightning and train your first Model. - :col_css: col-md-4 - :button_link: build_model.html - :height: 180 - -.. displayitem:: - :header: Evolve a Model into an ML System - :description: Develop an App to train a model in the cloud - :col_css: col-md-4 - :button_link: training_with_apps.html - :height: 180 - -.. displayitem:: - :header: Start from an ML system template - :description: Learn about Apps, from a template. - :col_css: col-md-4 - :button_link: go_beyond_training.html - :height: 180 - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/app_tree.rst b/docs/source-app/glossary/app_tree.rst index 9b16acda0a..85a0cee21e 100644 --- a/docs/source-app/glossary/app_tree.rst +++ b/docs/source-app/glossary/app_tree.rst @@ -1,5 +1,3 @@ -.. _app_component_tree: - ################### App Component Tree ################### @@ -8,11 +6,9 @@ App Component Tree **Level:** Basic ----- - -************************************** +**************************************** What is an Application Component Tree? -************************************** +**************************************** Components can be nested to form component trees where the LightningFlows are its branches and LightningWorks are its leaves. @@ -29,8 +25,6 @@ Here's a basic application with four flows and two works (associated tree struct A Lightning app runs all flows into a single process. Its flows coordinate the execution of the works each running in their own independent processes. ----- - *********************************************** How do I define my application component tree? *********************************************** @@ -41,25 +35,22 @@ You can attach your components in the **__init__** method of a flow. .. code-block:: python - import lightning as L + import lightning_app as la - class RootFlow(L.LightningFlow): + class RootFlow(lapp.LightningFlow): def __init__(self): super().__init__() - # The `Work` component is attached here. - self.work = Work() + self.work = Work() # The `Work` component is attached here. - # The `NestedFlow` component is attached here. - self.nested_flow = NestedFlow() + self.nested_flow = NestedFlow() # The `NestedFlow` component is attached here. Once done, simply add the root flow to a Lightning app as follows: .. code-block:: python - app = L.LightningApp(RootFlow()) + app = lapp.LightningApp(RootFlow()) ----- ****************************************** Is my application component tree static? @@ -71,20 +62,16 @@ You can simply attach your components in the **run** method of a flow using the .. code-block:: python - class RootFlow(L.LightningFlow): + class RootFlow(lapp.LightningFlow): def run(self): if not hasattr(self, "work"): - # The `Work` component is attached here. - setattr(self, "work", Work()) - # Run the `Work` component. - getattr(self, "work").run() + setattr(self, "work", Work()) # The `Work` component is attached here. + getattr(self, "work").run() # Run the `Work` component. if not hasattr(self, "nested_flow"): - # The `NestedFlow` component is attached here. - setattr(self, "nested_flow", NestedFlow()) - # Run the `NestedFlow` component. - getattr(self, "wonested_flowrk").run() + setattr(self, "nested_flow", NestedFlow()) # The `NestedFlow` component is attached here. + getattr(self, "wonested_flowrk").run() # Run the `NestedFlow` component. But it is usually more readable to use Lightning built-in :class:`~lightning_app.structures.Dict` or :class:`~lightning_app.structures.List` as follows: @@ -94,18 +81,16 @@ But it is usually more readable to use Lightning built-in :class:`~lightning_app from lightning_app.structures import Dict - class RootFlow(L.LightningFlow): + class RootFlow(lapp.LightningFlow): def __init__(self): super().__init__() self.dict = Dict() def run(self): if "work" not in self.dict: - # The `Work` component is attached here. - self.dict["work"] = Work() + self.dict["work"] = Work() # The `Work` component is attached here. self.dict["work"].run() if "nested_flow" not in self.dict: - # The `NestedFlow` component is attached here. - self.dict["nested_flow"] = NestedFlow() + self.dict["nested_flow"] = NestedFlow() # The `NestedFlow` component is attached here. self.dict["nested_flow"].run() diff --git a/docs/source-app/glossary/build_config/build_config_basic.rst b/docs/source-app/glossary/build_config/build_config_basic.rst index c3e3ae8c6f..7fa87b5e14 100644 --- a/docs/source-app/glossary/build_config/build_config_basic.rst +++ b/docs/source-app/glossary/build_config/build_config_basic.rst @@ -20,14 +20,14 @@ for more granular control. .. code-block:: bash ├── app.py - ├── requirements.txt # Global requirements for the entire app + ├── requirements.txt # Global requirements for the entire app └── works ├── serve - │ ├── requirements.txt # Requirements specific to the 'serve' work - │ └── serve.py # Source file for the LightningWork + │ ├── requirements.txt # Requirements specific to the 'serve' work + │ └── serve.py # Source file for the LightningWork └── train - ├── requirements.txt # Requirements specific to the 'train' work - └── train.py # Source file for the LightningWork + ├── requirements.txt # Requirements specific to the 'train' work + └── train.py # Source file for the LightningWork The requirements.txt file must be located in the same directry as the source file of the LightningWork. When the LightningWork starts up, it will pick up the requirements file if present and install all listed packages. @@ -46,7 +46,6 @@ Instead of listing the requirements in a file, you can also pass them to the Lig :class:`~lightning_app.utilities.packaging.build_config.BuildConfig`: .. code-block:: python - :emphasize-lines: 7 from lightning_app import LightningWork, BuildConfig diff --git a/docs/source-app/glossary/dag.rst b/docs/source-app/glossary/dag.rst index ef85d33cf3..6d843ffbd5 100644 --- a/docs/source-app/glossary/dag.rst +++ b/docs/source-app/glossary/dag.rst @@ -1,8 +1,18 @@ ###################### -Directed Acyclic Graph +Directed acyclic Graph ###################### **Audience:** Users coming from MLOps to Lightning Apps, looking for more flexibility. +.. warning:: documentation under development + +---- + +*************************************** +What is a Directed acyclic Graph (DAG)? +*************************************** + +.. note:: documentation under development + ---- ***************************** @@ -19,28 +29,4 @@ Can I Build a DAG with Lightning? ********************************* Yes! -DAGs are one of the easiest Lightning Apps to build. For example, here's a `full app that defines a DAG <../examples/dag/dag.html>`_. - ----- - -******** -Examples -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Build a DAG - :description: Learn how to create a DAG with Lightning - :col_css: col-md-4 - :button_link: ../examples/dag/dag.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
+DAGs are the (easiest) use-cases to build with Lightning Apps. For example, here's a full app that defines a DAG: diff --git a/docs/source-app/glossary/debug_app.rst b/docs/source-app/glossary/debug_app.rst index 2d5c0d1990..05c1091404 100644 --- a/docs/source-app/glossary/debug_app.rst +++ b/docs/source-app/glossary/debug_app.rst @@ -1,3 +1 @@ -:orphan: - .. include:: ../workflows/debug_locally.rst diff --git a/docs/source-app/glossary/distributed_fe.rst b/docs/source-app/glossary/distributed_fe.rst index 36d64b0143..c0483158df 100644 --- a/docs/source-app/glossary/distributed_fe.rst +++ b/docs/source-app/glossary/distributed_fe.rst @@ -1,5 +1,3 @@ -:orphan: - ##################### Distributed Front-End ##################### diff --git a/docs/source-app/glossary/distributed_hardware.rst b/docs/source-app/glossary/distributed_hardware.rst index 0a64f5f5c0..2dd51b3af2 100644 --- a/docs/source-app/glossary/distributed_hardware.rst +++ b/docs/source-app/glossary/distributed_hardware.rst @@ -1,5 +1,3 @@ -:orphan: - #################### Distributed Hardware #################### diff --git a/docs/source-app/glossary/event_loop.rst b/docs/source-app/glossary/event_loop.rst index 2b651267e6..eb09e4f34c 100644 --- a/docs/source-app/glossary/event_loop.rst +++ b/docs/source-app/glossary/event_loop.rst @@ -1,11 +1,11 @@ +.. _event_loop: + ########## Event loop ########## -Drawing inspiration from modern web frameworks like `React.js `_, the Lightning App runs all flows in an **event loop** (forever), which is triggered several times a second after collecting any works' state change. +Drawing inspiration from modern web frameworks like `React.js `_, the Lightning app runs all flows in an **event loop** (forever), which is triggered several times a second after collecting any works' state change. .. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/lightning_loop.gif -When running a Lightning App in the cloud, the ``LightningWork`` run on different machines. LightningWork communicates any state changes to the **event loop** which re-executes the flow with the newly-collected works' state. - -.. _app_event_loop: +When running an app in the cloud, the ``LightningWork`` run on different machines. LightningWork communicates any state changes to the **event loop** which re-executes the flow with the newly-collected works' state. diff --git a/docs/source-app/glossary/fault_tolerance.rst b/docs/source-app/glossary/fault_tolerance.rst index b0ee6dfd21..8f43962eb1 100644 --- a/docs/source-app/glossary/fault_tolerance.rst +++ b/docs/source-app/glossary/fault_tolerance.rst @@ -1,5 +1,3 @@ -:orphan: - ############### Fault tolerance ############### diff --git a/docs/source-app/glossary/index.rst b/docs/source-app/glossary/index.rst deleted file mode 100644 index 8d32358be7..0000000000 --- a/docs/source-app/glossary/index.rst +++ /dev/null @@ -1,80 +0,0 @@ -:orphan: - -######## -Glossary -######## - -.. raw:: html - -
-
- -.. displayitem:: - :header: App Components Tree - :description: Learn how components can be nested to form component trees where the LightningFlows are its branches and LightningWorks are its leaves. - :col_css: col-md-6 - :button_link: app_tree.html - :height: 180 - -.. displayitem:: - :header: Build Configuration - :description: Prepare your requirements, add custom build commands or use docker image - :col_css: col-md-6 - :button_link: build_config/build_config.html - :height: 180 - -.. displayitem:: - :header: DAG - :description: Learn about directed acyclic graph, their properties and usage - :col_css: col-md-6 - :button_link: dag.html - :height: 180 - -.. displayitem:: - :header: Event Loop - :description: Learn how the Infinite Event Loop enables high distributed reactivity by triggering after collecting state changes. - :col_css: col-md-6 - :button_link: event_loop.html - :height: 180 - -.. displayitem:: - :header: Environment Variables - :description: Add secrets such as API keys or access tokens - :col_css: col-md-6 - :button_link: environment_variables.html - :height: 180 - -.. displayitem:: - :header: Frontend - :description: Customize your App View with any framework you want - :col_css: col-md-6 - :button_link: ../workflows/add_web_ui/glossary_front_end.html - :height: 180 - -.. displayitem:: - :header: Sharing Components - :description: Let's create an ecosystem altogether - :col_css: col-md-6 - :button_link: sharing_components.html - :height: 180 - -.. displayitem:: - :header: Scheduling - :description: Orchestrate execution at specific times - :col_css: col-md-6 - :button_link: scheduling.html - :height: 180 - -.. displayitem:: - :header: Storage - :description: Easily share files even across multiple machines - :col_css: col-md-6 - :button_link: storage/storage.html - :height: 180 - -.. displayitem:: - :header: UI - :description: Combine multiple frameworks to create your own UI - :col_css: col-md-6 - :button_link: ../workflows/add_web_ui/glossary_ui.html - :height: 180 diff --git a/docs/source-app/glossary/lightning_app_overview/index.rst b/docs/source-app/glossary/lightning_app_overview/index.rst index 09de273aff..2dae01d9f0 100644 --- a/docs/source-app/glossary/lightning_app_overview/index.rst +++ b/docs/source-app/glossary/lightning_app_overview/index.rst @@ -3,7 +3,6 @@ ########################### Lightning Apps Key concepts ########################### - **Audience:** Users who want to know how the 🤯 magic works under the hood. ---- diff --git a/docs/source-app/glossary/scheduling.rst b/docs/source-app/glossary/scheduling.rst index 5f9bfdea7a..38f37a152b 100644 --- a/docs/source-app/glossary/scheduling.rst +++ b/docs/source-app/glossary/scheduling.rst @@ -22,19 +22,22 @@ The LightningFlow has a ``schedule`` method which can be used to schedule your c class MyFlow(LightningFlow): - def run(self): if self.schedule("hourly"): # run some code once every hour. + pass if self.schedule("daily"): # run some code once day. + pass if self.schedule("daily") and anything_else: # run some code once day if the anything else is also True. + pass if self.schedule("2 4 * * mon,fri"): # defined with cron syntax, run some code at 04:02 on every Monday and Friday. + pass Learn more about the cron syntax `here `_ @@ -53,6 +56,7 @@ In the example above, the line ``self.schedule("hourly")`` will return ``True`` from lightning_app import LightningFlow from lightning_app.core.structures import List + class ScheduledDAG(LightningFlow): def __init__(self): super().__init__() @@ -61,12 +65,10 @@ In the example above, the line ``self.schedule("hourly")`` will return ``True`` def run(self): if self.schedule("hourly"): # dynamically instantiate - # don't forget to always attach - # your components to the flow !!! + # don't forget to always attach your components to the flow !!! self.list.append(MyDAGFlow(...)) - # run all dags, but the completed ones - # are cached and don't re-execute. + # run all dags, but the completed ones are cached and don't re-execute. for dag in self.list: dag.run() @@ -78,6 +80,7 @@ In the example above, the line ``self.schedule("hourly")`` will return ``True`` from lightning_app import LightningFlow from time import time + class ScheduledDAG(LightningFlow): def __init__(self): super().__init__() @@ -96,6 +99,7 @@ In the example above, the line ``self.schedule("hourly")`` will return ``True`` from lightning_app import LightningFlow from time import time + class ScheduledDAG(LightningFlow): def __init__(self): super().__init__() @@ -108,18 +112,13 @@ In the example above, the line ``self.schedule("hourly")`` will return ``True`` if self.schedule("hourly"): self.should_execute = True - # Runs in 10 min - if self.should_execute: - # Runs in 5 min - self.data_processor.run(trigger_time=time()) + if self.should_execute: # Runs in 10 min + self.data_processor.run(trigger_time=time()) # Runs in 5 min if self.data_processor.has_succeeded: - # Runs in 5 min - self.training_work.run(self.data_processor.data) + self.training_work.run(self.data_processor.data) # Runs in 5 min if self.training_work.has_succeeded: self.should_execute = False ----- - *********** Limitations *********** @@ -134,6 +133,7 @@ Here is an example of something which **WON'T** work: from lightning_app import LightningFlow from time import time + class ScheduledDAG(LightningFlow): def __init__(self): super().__init__() @@ -143,11 +143,9 @@ Here is an example of something which **WON'T** work: def run(self): ... if self.schedule("hourly"): - # This finishes 5 min later - self.data_processor.run(trigger_time=time()) + self.data_processor.run(trigger_time=time()) # This executes and finishes 5 min later if self.data_processor.has_succeeded: - # This will never be reached as the - # data processor will keep processing forever... + # This will never be reached as the data processor will keep processing forever... self.training_work.run(self.data_processor.data) ---- @@ -156,30 +154,10 @@ Here is an example of something which **WON'T** work: Frequently Asked Questions ************************** -- **Q: Can I use multiple nested scheduler?** No, as they might cancel themselves out, but you can capture the event of one to trigger the next one. +- **Q: Can I use multiple nested schedule?** -- **Q: Can I use any arbitrary logic to schedule?** Yes, this design enables absolute flexibility, but you need to be careful to avoid bad practices. + Not really as they might cancel themselves out, but you can capture the event of one to trigger the next one. ----- +- **Q: Can I use any arbitrary logic to schedule?** -******** -Examples -******** - -.. raw:: html - -
-
- -.. displayitem:: - :header: Build a DAG - :description: Learn how to schedule a DAG execution - :col_css: col-md-4 - :button_link: ../examples/dag/dag.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
+ Yes, this design enables absolute flexibility, but you need to be careful to avoid bad practices. diff --git a/docs/source-app/glossary/sharing_components.rst b/docs/source-app/glossary/sharing_components.rst index 2699f5d3c1..c4df69f95c 100644 --- a/docs/source-app/glossary/sharing_components.rst +++ b/docs/source-app/glossary/sharing_components.rst @@ -6,8 +6,6 @@ Sharing my components **Level:** Basic ----- - ********************************************* Why should I consider sharing my components ? ********************************************* @@ -16,8 +14,6 @@ Lightning is community driven and its core objective is to make AI accessible to By creating components and sharing them with everyone else, the barrier to entry will be go down. ----- - ************************************* How should I organize my components ? ************************************* @@ -38,12 +34,10 @@ Here are the best practices steps before sharing the component: .. note:: As a Lightning user, it helps to implement your components thinking someone else is going use them. ----- - ****************************************** How should I proceed to share components ? ****************************************** Once your component is ready, create a PiPy package with your own library and then it can be re-used by anyone else. -Here is a `Component Template `_ from `William Falcon `_ to guide your component +Here is a `Component Template `_ from `William Falcon `_ to guide your component diff --git a/docs/source-app/glossary/storage/drive.rst b/docs/source-app/glossary/storage/drive.rst index e500e087ce..0888c80fe6 100644 --- a/docs/source-app/glossary/storage/drive.rst +++ b/docs/source-app/glossary/storage/drive.rst @@ -4,8 +4,205 @@ Drive Storage ############# -**Audience:** Users who want to put, list, and get files from a shared disk space. +**Audience:** Users who want to put, list and get files from a shared disk space. + + +The Lightning Storage system makes it easy to share files between LightningWork so you can run your app both locally and in the cloud without changing the code. ---- -.. include:: ../../glossary/storage/drive_content.rst +***************** +What is a Drive ? +***************** + +The Drive object provides a central place for your components to share data. + +The drive acts as an isolate folder and any component can access it by knowing its name. + +Your components can put, list, get, delete files from and to the Drive (except LightningFlow's). + +---- + +**************************** +Why should I use the Drive ? +**************************** + +Every instance of the Drive object acts as a Google Drive or Dropbox. + +By sharing the drive between components through the flow, +several components can have a shared place to read and write files from. + +---- + +************************* +How do I create a Drive ? +************************* + +In order to create a Drive, you simply need to pass its name with the prefix ``lit://`` as follows: + +.. code-block:: python + + from lightning_app.storage import Drive + + # The identifier of this Drive is ``drive_1`` + # Note: You need to add Lightning protocol ``lit://`` as a prefix. + + drive_1 = Drive("lit://drive_1") + + # The identifier of this Drive is ``drive_2`` + drive_2 = Drive("lit://drive_2") + +Any components can create a drive object. + +.. code-block:: python + + from lightning_app import LightningFlow, LightningWork + from lightning_app.storage import Drive + + + class Flow(LightningFlow): + def __init__(self): + super().__init__() + self.drive_1 = Drive("lit://drive_1") + + def run(self): + ... + + + class Work(LightningWork): + def __init__(self): + super().__init__() + self.drive_1 = Drive("lit://drive_1") + + def run(self): + ... + +---- + +************************************* +What actions does the drive support ? +************************************* + +A drive supports put, list, get, delete actions. + +.. code-block:: python + + from lightning_app.storage import Drive + + drive = Drive("lit://drive") + + drive.list(".") # Returns [] as empty + + # Created file. + with open("a.txt", "w") as f: + f.write("Hello World !") + + drive.put("a.txt") + + drive.list(".") # Returns ["a.txt"] as the file copied in the Drive during the put action. + + drive.get("a.txt") # Get the file into the current worker + + drive.delete("a.txt") + + drive.list(".") # Returns [] as empty + +---- + +****************************************** +How does component interacts with drives ? +****************************************** + +Here is an illustrated code example on how to create drives within works. + +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/drive_2.png + +.. code-block:: python + + from lightning_app import LightningFlow, LightningWork + from lightning_app.core.app import LightningApp + from lightning_app.storage.drive import Drive + + + class Work_A(LightningWork): + def __init__(self): + super().__init__() + # The identifier of the Drive is ``drive_1`` + # Note: You need to add Lightning protocol ``lit://`` as a prefix. + self.drive_1 = Drive("lit://drive_1") + + def run(self): + # 1. Create a file. + with open("a.txt", "w") as f: + f.write("Hello World !") + + # 2. Put the file into the drive. + self.drive_1.put("a.txt") + + + class Work_B(LightningWork): + def __init__(self): + super().__init__() + + # Note: Work B has access 2 drives. + + # The identifier of this Drive is ``drive_1`` + self.drive_1 = Drive("lit://drive_1") + # The identifier of this Drive is ``drive_2`` + self.drive_2 = Drive("lit://drive_2") + + def run(self): + # 1. Create a file. + with open("b.txt", "w") as f: + f.write("Hello World !") + + # 2. Put the file into both drives. + self.drive_1.put("b.txt") + self.drive_2.put("b.txt") + + + class Work_C(LightningWork): + def __init__(self): + super().__init__() + self.drive_2 = Drive("lit://drive_2") + + def run(self): + # 1. Create a file. + with open("c.txt", "w") as f: + f.write("Hello World !") + + # 2. Put the file into the drive. + self.drive_2.put("c.txt") + +---- + +****************************************** +How can I transfer files with the Drive ? +****************************************** + +In the example below, the Drive is created by the flow and passed to its LightningWork's. + +The ``Work_1`` put a file **a.txt** in the **Drive("lit://this_drive_id")** and the ``Work_2`` can list and get the **a.txt** file from it. + +.. literalinclude:: ../../../../examples/drive/app.py + + +---- + +.. raw:: html + +
+
+ +.. displayitem:: + :header: Learn about the Path Object. + :description: Transfer Files From One Component to Another by Reference. + :col_css: col-md-4 + :button_link: path.html + :height: 180 + :tag: Intermediate + +.. raw:: html + +
+
diff --git a/docs/source-app/glossary/storage/drive_content.rst b/docs/source-app/glossary/storage/drive_content.rst deleted file mode 100644 index c2c1357d0f..0000000000 --- a/docs/source-app/glossary/storage/drive_content.rst +++ /dev/null @@ -1,199 +0,0 @@ - - -************ -About Drives -************ - -Lightning Drive storage makes it easy to share files between LightningWorks so you can run your Lightning App both locally and in the cloud without changing the code. - -The Drive object provides a central place for your components to share data. - -The Drive acts as an isolate folder and any component can access it by knowing its name. - -Your components can put, list, get, and delete files from and to the Drive (except LightningFlows). - ----- - -*********************** -What Drive does for you -*********************** - -Think of every instance of the Drive object acting like a Google Drive or like Dropbox. - -By sharing the Drive between components through the LightningFlow, -several components can have a shared place to read and write files from. - ----- - -************** -Create a Drive -************** - -In order to create a Drive, you simply need to pass its name with the prefix ``lit://`` as follows: - -.. code-block:: python - - from lightning_app.storage import Drive - - # The identifier of this Drive is ``drive_1`` - # Note: You need to add Lightning protocol ``lit://`` as a prefix. - - drive_1 = Drive("lit://drive_1") - - # The identifier of this Drive is ``drive_2`` - drive_2 = Drive("lit://drive_2") - -Any components can create a drive object. - -.. code-block:: python - - from lightning_app import LightningFlow, LightningWork - from lightning_app.storage import Drive - - - class Flow(LightningFlow): - def __init__(self): - super().__init__() - self.drive_1 = Drive("lit://drive_1") - - def run(self): - ... - - - class Work(LightningWork): - def __init__(self): - super().__init__() - self.drive_1 = Drive("lit://drive_1") - - def run(self): - ... - ----- - -***************************** -Supported actions with Drives -***************************** - -A Drive supports put, list, get, and delete actions. - -.. code-block:: python - - from lightning_app.storage import Drive - - drive = Drive("lit://drive") - - drive.list(".") # Returns [] as empty - - # Created file. - with open("a.txt", "w") as f: - f.write("Hello World !") - - drive.put("a.txt") - - drive.list(".") # Returns ["a.txt"] as the file copied in the Drive during the put action. - - drive.get("a.txt") # Get the file into the current worker - - drive.delete("a.txt") - - drive.list(".") # Returns [] as empty - ----- - -********************************** -Component interactions with Drives -********************************** - -Here is an illustrated code example on how to create drives within works. - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/drive_2.png - -.. code-block:: python - - from lightning_app import LightningFlow, LightningWork - from lightning_app.core.app import LightningApp - from lightning_app.storage.drive import Drive - - - class Work_A(LightningWork): - def __init__(self): - super().__init__() - # The identifier of the Drive is ``drive_1`` - # Note: You need to add Lightning protocol ``lit://`` as a prefix. - self.drive_1 = Drive("lit://drive_1") - - def run(self): - # 1. Create a file. - with open("a.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into the drive. - self.drive_1.put("a.txt") - - - class Work_B(LightningWork): - def __init__(self): - super().__init__() - - # Note: Work B has access 2 drives. - - # The identifier of this Drive is ``drive_1`` - self.drive_1 = Drive("lit://drive_1") - # The identifier of this Drive is ``drive_2`` - self.drive_2 = Drive("lit://drive_2") - - def run(self): - # 1. Create a file. - with open("b.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into both drives. - self.drive_1.put("b.txt") - self.drive_2.put("b.txt") - - - class Work_C(LightningWork): - def __init__(self): - super().__init__() - self.drive_2 = Drive("lit://drive_2") - - def run(self): - # 1. Create a file. - with open("c.txt", "w") as f: - f.write("Hello World !") - - # 2. Put the file into the drive. - self.drive_2.put("c.txt") - ----- - -***************************** -Transfer files with Drive -***************************** - -In the example below, the Drive is created by the flow and passed to its LightningWork's. - -The ``Work_1`` put a file **a.txt** in the **Drive("lit://this_drive_id")** and the ``Work_2`` can list and get the **a.txt** file from it. - -.. literalinclude:: ../../../examples/app_drive/app.py - - ----- - -.. raw:: html - -
-
- -.. displayitem:: - :header: Learn about the Path Object. - :description: Transfer Files From One Component to Another by Reference. - :col_css: col-md-4 - :button_link: path.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/glossary/storage/path.rst b/docs/source-app/glossary/storage/path.rst index 6da6459cf1..b46cdbfb1d 100644 --- a/docs/source-app/glossary/storage/path.rst +++ b/docs/source-app/glossary/storage/path.rst @@ -81,7 +81,7 @@ Convert every filesystem path you want to share with other LightningWorks to by ... -Under the hood, we convert this string to a :class:`~lightning.app.storage.path.Path` object, which is a drop-in replacement for :class:`pathlib.Path` meaning it will work with :mod:`os`, :mod:`os.path` and :mod:`pathlib` filesystem operations out of the box! +Under the hood, we convert this string to a :class:`~lightning_app.storage.path.Path` object, which is a drop-in replacement for :class:`pathlib.Path` meaning it will work with :mod:`os`, :mod:`os.path` and :mod:`pathlib` filesystem operations out of the box! ---- @@ -268,7 +268,7 @@ Link both components via a parent component: self.deploy.run(checkpoint_dir=self.train.checkpoint_dir) - app = L.LightningApp(Flow()) + app = lapp.LightningApp(Flow()) ---- diff --git a/docs/source-app/glossary/storage/storage.rst b/docs/source-app/glossary/storage/storage.rst index 37bf992bf1..5bfa9689c0 100644 --- a/docs/source-app/glossary/storage/storage.rst +++ b/docs/source-app/glossary/storage/storage.rst @@ -48,30 +48,3 @@ Lightning storage provides two solutions :class:`~lightning_app.storage.drive.Dr
- - ----- - -******** -Examples -******** - - - -.. raw:: html - -
-
- -.. displayitem:: - :header: Build a File Server - :description: Learn how to use Drive to upload / download files to your app. - :col_css: col-md-4 - :button_link: ../../examples/file_server/file_server.html - :height: 180 - :tag: Intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/index.rst b/docs/source-app/index.rst index 68020dfab2..80e2873143 100644 --- a/docs/source-app/index.rst +++ b/docs/source-app/index.rst @@ -3,192 +3,97 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -############################ -Welcome to ⚡ Lightning Apps -############################ +Welcome to ⚡ Lightning AI +========================== .. twocolumns:: :left: - .. image:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/Lightning.gif + .. image:: https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/mov.gif :alt: Animation showing how to convert a standard training loop to a Lightning loop :right: - The `open-source Lightning framework `_ gives ML Researchers and Data Scientists, the fastest & most flexible - way to iterate on ML research ideas and deliver scalable ML systems with the performance enterprises requires at the same time. + PyTorch Lightning is the deep learning framework for professional AI researchers and machine learning engineers who need maximal flexibility without sacrificing performance at scale. + Lightning evolves with you as your projects go from idea to paper/production. + +.. raw:: html + +
+
+
+
.. join_slack:: :align: center :margin: 0 ----- +.. raw:: html + +
+
+ + +.. raw:: html + +
+ -***************** Install Lightning -***************** +----------------- -.. raw:: html +.. + .. raw:: html -
-
+
+
-Pip users +Make sure you use Python 3.8+ .. code-block:: bash - pip install lightning + python -m pip install -U lightning + +.. + .. raw:: html + +
+
+ + Conda users + + .. code-block:: bash + + Available after June 16th + + .. raw:: html + +
+
.. raw:: html -
-
+
-Conda users - -.. code-block:: bash - - conda install lightning -c conda-forge - -.. raw:: html - -
-
- -Or read the :ref:`advanced install ` guide. - ----- - -*********** Get Started -*********** +----------- .. raw:: html -
-
-
- -.. displayitem:: - :header: Discover what Lightning Apps can do in 5 min - :description: Browse through mind-blowing ML Systems - :col_css: col-md-6 - :button_link: get_started/what_app_can_do.html - :height: 180 - -.. displayitem:: - :header: Build and Train a Model - :description: Discover PyTorch Lightning and train your first Model. - :col_css: col-md-6 - :button_link: get_started/build_model.html - :height: 180 - -.. displayitem:: - :header: Evolve a Model into an ML System - :description: Develop an App to train a model in the cloud - :col_css: col-md-6 - :button_link: get_started/training_with_apps.html - :height: 180 - -.. displayitem:: - :header: Start from an ML system template - :description: Learn about Apps, from a template. - :col_css: col-md-6 - :button_link: get_started/go_beyond_training.html - :height: 180 - -.. raw:: html - -
-
- ----- - -*********************** -Current Lightning Users -*********************** - -.. raw:: html - -
- -Build with Template(s) from the App & Component Gallery -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. raw:: html - -
+
-.. displayitem:: - :header: Start from Ready-to-Run Template Apps - :description: Jump-start your project's development - :col_css: col-md-6 - :button_link: get_started/jumpstart_from_app_gallery.html - :height: 180 - -.. displayitem:: - :header: Add Component made by others to your App - :description: Add more functionalities to your projects - :col_css: col-md-6 - :button_link: get_started/jumpstart_from_component_gallery.html - :height: 180 +.. Add callout items below this line +.. customcalloutitem:: + :header: Build a Lightning App in 15 minutes + :description: Learn the 4 key steps to build a Lightning app. + :button_link: pages/lightning_apps_intro.html .. raw:: html -
-
-
+
+
- -Keep Learning -^^^^^^^^^^^^^ - -.. raw:: html - -
-
- -.. displayitem:: - :header: Level-up with PyTorch Lightning - :description: PyTorch Lightning Tutorials - :col_css: col-md-6 - :button_link: https://pytorch-lightning.readthedocs.io/en/latest/expertise_levels.html - :height: 180 - -.. displayitem:: - :header: Level-up with Lightning Apps - :description: From Basics to Advanced Skills - :col_css: col-md-6 - :button_link: levels/basic/index.html - :height: 180 - -.. displayitem:: - :header: API Reference - :description: Detailed description of each API package - :col_css: col-md-6 - :button_link: api_reference/api_references.html - :height: 180 - -.. displayitem:: - :header: Hands-on Examples - :description: Learn by building Apps and Components. - :col_css: col-md-6 - :button_link: examples/hands_on_example.html - :height: 180 - -.. displayitem:: - :header: Common Workflows - :description: Learn how to do ... - :col_css: col-md-6 - :button_link: workflows/index.html - :height: 180 - -.. displayitem:: - :header: Glossary - :description: Discover Lightning App Concepts - :col_css: col-md-6 - :button_link: glossary/index.html - :height: 180 +.. End of callout item section .. raw:: html @@ -198,69 +103,13 @@ Keep Learning
-.. toctree:: - :maxdepth: 1 - :caption: Home - - self - .. toctree:: :maxdepth: 1 :caption: Get Started - installation - get_started/lightning_apps_intro + pages/lightning_apps_intro + pages/installation -.. toctree:: - :maxdepth: 1 - :caption: App Building Skills - - Basic - Intermediate - Advanced - -.. toctree:: - :maxdepth: 1 - :caption: Practical Examples - - Build a DAG - Build a File Server - Build a Github Repo Script Runner - Build a HPO Sweeper - Build a Model Server - -.. - [Docs under construction] Build a data exploring app - [Docs under construction] Build a ETL app - [Docs under construction] Build a model deployment app - [Docs under construction] Build a research demo app - -.. toctree:: - :maxdepth: 1 - :caption: Common Workflows - - Add a web user interface (UI) - Add a web link - Arrange app tabs - Build a Lightning app - Build a Lightning component - Cache Work run calls - Customize your cloud compute - Extend an existing app - Publish a Lightning component - Run a server within a Lightning App - Run an app on the cloud - Run work in parallel - Share an app - Share files between components - -.. - [Docs under construction] Add a Lightning component - [Docs under construction] Debug a distributed cloud app locally - [Docs under construction] Enable fault tolerance - [Docs under construction] Run components on different hardware - [Docs under construction] Schedule app runs - [Docs under construction] Test an app .. toctree:: :maxdepth: 1 @@ -281,21 +130,57 @@ Keep Learning .. toctree:: :maxdepth: 1 - :caption: Glossary + :caption: App development workflows + + Access the app state + Add a web user interface (UI) + Add a web link + Arrange app tabs + Build a Lightning app + Build a Lightning component + Extend an existing app + Publish a Lightning component + Run a server within a Lightning App + Run an app on the cloud + Run work in parallel + Share an app + Share files between components + [Docs under construction] Add a Lightning component + [Docs under construction] Debug a distributed cloud app locally + [Docs under construction] Enable fault tolerance + [Docs under construction] Run components on different hardware + Cache Work run calls + [Docs under construction] Schedule app runs + [Docs under construction] Test an app + + +.. toctree:: + :maxdepth: 1 + :caption: Hands-on Examples [Docs under construction] + + Build a sweeps app + Build a data exploring app + Build a DAG + Build a ETL app + Build a model deployment app + Build a research demo app + +.. toctree:: + :maxdepth: 1 + :caption: Glossary [Docs under construction] + App Components Tree Build Configuration DAG - Event Loop + Debug an app + Distributed front-ends + Distributed hardware + Event loop Environment Variables - Frontend + Fault tolerance + Front-end Sharing Components Scheduling Storage UI - -.. - [Docs under construction] Debug an app - [Docs under construction] Distributed front-ends - [Docs under construction] Distributed hardware - [Docs under construction] Fault tolerance diff --git a/docs/source-app/installation.rst b/docs/source-app/installation.rst deleted file mode 100644 index 7f243e170e..0000000000 --- a/docs/source-app/installation.rst +++ /dev/null @@ -1,41 +0,0 @@ - -.. _install: - -############ -Installation -############ - -We strongly recommend to create a virtual environment first. -Don't know what this is? Follow our `beginner guide here `_. - -**Requirements** - -* Python 3.8.x or later (3.8.x, 3.9.x, 3.10.x) -* Pip (the latest versions of Python will already have this) -* Git -* PyTorch - https://pytorch.org/get-started/locally/ -* Setup an alias for Python: python=python3 -* Add the root folder of Lightning to the Environment Variables to PATH -* Install Z shell (zsh) (This is required for Windows to install the quickstart app) - ----- - -**************** -Install with pip -**************** - -0. Activate your virtual environment. - - .. raw:: html - -
- -1. Install the ``lightning`` package - - .. raw:: html - -
- - .. code:: bash - - python -m pip install -U lightning diff --git a/docs/source-app/levels/advanced/index.rst b/docs/source-app/levels/advanced/index.rst deleted file mode 100644 index 265a8ce961..0000000000 --- a/docs/source-app/levels/advanced/index.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. _advanced_level: - -.. toctree:: - :maxdepth: 1 - :hidden: - - level_16 - level_17 - level_18 - level_19 - level_20 - -############### -Advanced skills -############### - -Learn to build Lightning Apps for enterprise workloads or advanced research. - -.. join_slack:: - :align: left - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 16: Check work status - :description: Learn to use work status to coordinate complex apps. - :button_link: level_16.html - :col_css: col-md-6 - :height: 150 - :tag: advanced - -.. displayitem:: - :header: Level 17: Cache calls into run - :description: Learn about caching calls in work.run. - :button_link: level_17.html - :col_css: col-md-6 - :height: 150 - :tag: advanced - -.. displayitem:: - :header: Level 18: Share objects between works - :description: Learn to build things like DAGs where values return from Work. - :button_link: level_18.html - :col_css: col-md-6 - :height: 150 - :tag: advanced - -.. displayitem:: - :header: Level 19: Handle Lightning App exceptions - :description: Learn to handle Lightning App exceptions. - :button_link: level_19.html - :col_css: col-md-6 - :height: 150 - :tag: advanced - -.. displayitem:: - :header: Level 20: Enable dynamic Works - :description: Learn to enable dynamic works for complex systems. - :button_link: level_20.html - :col_css: col-md-6 - :height: 150 - :tag: advanced - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/advanced/level_16.rst b/docs/source-app/levels/advanced/level_16.rst deleted file mode 100644 index 58c1e1fc7e..0000000000 --- a/docs/source-app/levels/advanced/level_16.rst +++ /dev/null @@ -1,10 +0,0 @@ -########################### -Level 16: Check Work status -########################### -**Audience:** Users who want to stop/start Lightning Work based on a status. - -**Prereqs:** Level 16+ - ----- - -.. include:: ../../core_api/lightning_work/status_content.rst diff --git a/docs/source-app/levels/advanced/level_17.rst b/docs/source-app/levels/advanced/level_17.rst deleted file mode 100644 index 7bc8dda703..0000000000 --- a/docs/source-app/levels/advanced/level_17.rst +++ /dev/null @@ -1,10 +0,0 @@ -############################## -Level 17: Cache calls into run -############################## -**Audience:** Users who want Work.run() to activate multiple times in an app. - -**Prereqs:** Level 16+ and read the `Event Loop guide <../glossary/event_loop.html>`_. - ----- - -.. include:: ../../workflows/run_work_once_content.rst diff --git a/docs/source-app/levels/advanced/level_18.rst b/docs/source-app/levels/advanced/level_18.rst deleted file mode 100644 index c934d2fd08..0000000000 --- a/docs/source-app/levels/advanced/level_18.rst +++ /dev/null @@ -1,10 +0,0 @@ -############################################## -Level 18: Share objects between LightningWorks -############################################## -**Audience:** Users moving DataFrames or outputs, between Lightning Works (usually data engineers). - -**Prereqs:** Level 16+ and know about the Pandas library and read the `Access app state guide <../../access_app_state.html>`_. - ----- - -.. include:: ../../core_api/lightning_work/payload_content.rst diff --git a/docs/source-app/levels/advanced/level_19.rst b/docs/source-app/levels/advanced/level_19.rst deleted file mode 100644 index 99a859e1ad..0000000000 --- a/docs/source-app/levels/advanced/level_19.rst +++ /dev/null @@ -1,11 +0,0 @@ -######################################### -Level 19: Handle Lightning App exceptions -######################################### - -**Audience:** Users who want to make Lightning Apps more robust to potential issues. - -**Prereqs:** Level 16+ - ----- - -.. include:: ../../core_api/lightning_work/handling_app_exception_content.rst diff --git a/docs/source-app/levels/advanced/level_20.rst b/docs/source-app/levels/advanced/level_20.rst deleted file mode 100644 index 1d045e85fc..0000000000 --- a/docs/source-app/levels/advanced/level_20.rst +++ /dev/null @@ -1,11 +0,0 @@ -####################################### -Level 20: Enable dynamic LightningWorks -####################################### - -**Audience:** Users who want to create/run/stop multiple LightningWorks not defined at app instantiation. - -**Prereqs:** Level 16+ - ----- - -.. include:: ../../core_api/lightning_app/dynamic_work_content.rst diff --git a/docs/source-app/levels/basic/index.rst b/docs/source-app/levels/basic/index.rst deleted file mode 100644 index 24d34ae679..0000000000 --- a/docs/source-app/levels/basic/index.rst +++ /dev/null @@ -1,92 +0,0 @@ -.. _level_basic: - -.. toctree:: - :maxdepth: 1 - :hidden: - - level_1 - level_2 - level_3 - level_4 - level_5 - level_6 - level_7 - -############ -Basic skills -############ -Learn the basics of running Lightning Apps and modifying existing Lightning Apps. Researchers and machine learning engineers should start here. - -.. join_slack:: - :align: left - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 1: Clone and Run - :description: Learn how to get up and running in minutes - :button_link: level_1.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. displayitem:: - :header: Level 2: Run Lightning Apps locally - :description: Learn to run an Lightning Apps locally - :button_link: level_2.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. displayitem:: - :header: Level 3: Run Lightning Apps on the cloud - :description: Learn to run a Lightning Apps on the cloud - :button_link: level_3.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. displayitem:: - :header: Level 4: Modify existing Lightning Apps - :description: Modify a Lightning App from the Lightning App Gallery for your use case. - :button_link: level_4.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. displayitem:: - :header: Level 5: Share your Lightning App - :description: Learn how to share your Lightning App with colleagues - :button_link: level_5.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. displayitem:: - :header: Level 6: Publish your Lightning App - :description: Learn how to submit your Lightning App to the Lightning App Gallery - :button_link: level_6.html - :col_css: col-md-6 - :height: 150 - :tag: basic - - -.. displayitem:: - :header: Level 7: Run on your cloud account - :description: Learn how to run a Lightning App on your own cloud account. - :button_link: level_7.html - :col_css: col-md-6 - :height: 150 - :tag: basic - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/basic/level_1.rst b/docs/source-app/levels/basic/level_1.rst deleted file mode 100644 index cfb0045cfc..0000000000 --- a/docs/source-app/levels/basic/level_1.rst +++ /dev/null @@ -1,99 +0,0 @@ -###################### -Level 1: Clone and Run -###################### -**Audience:** New users to the framework. - -**Prereqs:** None. - ----- - -********************** -Choose a Lightning App -********************** -The simplest way to get started with Lightning Apps, is to find a Lightning App similar to something you are trying -to build. Simply visit `lightning.ai `_ and find a Lightning App that is relevant to your work. - -If you need inspiration, check out some of these examples: - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Lightning Sweeper - :description: Use this Lightning App to run hyperparameter sweeps, or build your own version. - :button_link: https://lightning.ai/app/8FOWcOVsdf-Lightning%20Sweeper - :col_css: col-md-4 - :height: 200 - :tag: research workflows - -.. displayitem:: - :header: InVideo search - :description: Use this Lightning App to see a more end-to-end production workflow which can be extended with deployment strategies and more. - :button_link: https://lightning.ai/app/7pmQNIDxAE-InVideo%20Search - :col_css: col-md-4 - :height: 200 - :tag: production workflows - -.. displayitem:: - :header: Unsplash Image Search - :description: Example Lightning App if you want to build research demos. - :button_link: https://lightning.ai/app/vIDPvY3YAA-Unsplash%20Image%20Search - :col_css: col-md-4 - :height: 200 - :tag: demo workflows - -.. raw:: html - -
-
- ----- - -************************ -Launch the Lightning App -************************ -The first thing you should try is to "launch" the Lightning App to see how a Lightning App works before you decide to dive into the code. -Some Lightning Apps cannot be launched because of the way they have been constructed, or for cost reasons for the author. - -For example, here's the page for the `Unsplash image search app `_. - -When you submit a Lightning App to the Lightning App Gallery, you can decide if a Lightning App can be launched. Otherwise, users -will have to clone & run the Lightning App to run on their accounts. - ----- - -*********** -Clone & Run -*********** -If you're happy with the Lightning App you found and want to try it out, the next action should be to **clone & run** the Lightning App. -To **clone & run** go to the details page of that Lightning App and press **Clone & Run**. - -For example, try the **Clone & Run** button `for this Lightning App `_. - -Clone and run starts the Lightning App on your account. If cloning and running is going to cost you money, you'll get -a prompt to verify that you want to do that. - ----- - -********************** -Open the Lightning App -********************** -Once the Lightning App has started, press "Launch" to see the Lightning App. You can send that link to any of your colleagues so -they can also use the Lightning App. - -For example, here's the page that appears for the `Unsplash Image Search Lightning App `_. - -.. Warning:: Remember, that if the Lightning App performs expensive actions, such as training a model, and you share with people, - they might accidentally incur costs on your account. Make sure this is what you are expecting. - ----- - -***************** -Download the code -***************** -Now that you verified that the Lightning App does indeed work end-to-end, go to the **Code** tab and click "Download zip", to get the full code -on your machine. diff --git a/docs/source-app/levels/basic/level_2.rst b/docs/source-app/levels/basic/level_2.rst deleted file mode 100644 index 0f75c33588..0000000000 --- a/docs/source-app/levels/basic/level_2.rst +++ /dev/null @@ -1,36 +0,0 @@ -################################### -Level 2: Run Lightning Apps locally -################################### -**Audience:** New users who want to run a Lightning App on their machines - -**Prereqs:** You already have the Lightning App code on your local machine. - ----- - -************ -Get the code -************ -If you followed the instructions for **Level 1: Clone and Run**, you already have the code for the Lightning App locally. Otherwise please go back to `Level 1 `_. - ----- - -*********** -Run locally -*********** -Run the Lightning App on your local machine by using the following command: - -.. code:: bash - - lightning run app app.py - -Now you'll see the Lightning App start up on your local machine. - -.. note:: At this time, you can only run one Lightning App locally at a time. **Submit a PR to unblock that!** - ----- - -************************* -Run on any remote machine -************************* -Remember you can always SSH into any of your cloud machines on your university or enterprise cluster and run -Lightning App from there. However, you will be responsible for the auto-scaling of those Lightning Apps. diff --git a/docs/source-app/levels/basic/level_3.rst b/docs/source-app/levels/basic/level_3.rst deleted file mode 100644 index b263250973..0000000000 --- a/docs/source-app/levels/basic/level_3.rst +++ /dev/null @@ -1,56 +0,0 @@ -############################# -Level 3: Run app on the cloud -############################# -**Audience:** Users who want to run an app on the Lightning Cloud. - -**Prereqs:** You have an app already running locally. - ----- - -**************************** -What is the Lightning Cloud? -**************************** -The Lightning Cloud is the platform that we've created to interface with the cloud providers. Today -the Lightning Cloud supports AWS. - -.. note:: Support for GCP and Azure is coming in the Fall of 2022! - -To use the Lightning Cloud, you buy credits that are used to pay the cloud providers. If you want to run -on your own AWS credentials, please contact us (support@lightning.ai) so we can get your clusters set up for you. - ----- - -**************** -Run on the cloud -**************** -To run the app on the cloud, simply add **--cloud** to the command - -.. code:: bash - - lightning run app app.py --cloud - -Lightning packages everything in that folder and uploads it to the cloud. Your code will be visible to everyone (just like Github). - -.. note:: To have private Lightning Apps, you'll need to upgrade your account. To upgrade, contact us ``_. - ----- - -************ -Ignore files -************ -Lightning sends everything in your Lightning App folder to the cloud. If you want to ignore certain files (such as datasets), -use the **.lightningignore** file which works just like the **.gitignore** - -.. code:: bash - - touch .lightningignore - ----- - -******************* -Manage requirements -******************* -A Lightning App is simply a Python file. However, when running on the cloud, it is encouraged that you add -a **requirements.txt** file so that the platform knows what requirements your Lightning App needs. - -If you require custom Docker images, each LightningWork has the ability to have a private Docker image. diff --git a/docs/source-app/levels/basic/level_4.rst b/docs/source-app/levels/basic/level_4.rst deleted file mode 100644 index c5ce24ff42..0000000000 --- a/docs/source-app/levels/basic/level_4.rst +++ /dev/null @@ -1,122 +0,0 @@ -############################ -Level 4: Modify existing app -############################ -**Audience:** Users who have already run a Lightning App locally or remote and want to modify it. - -**Prereqs:** You've ran a Lightning App locally and the cloud. - ----- - -*************** -Change the code -*************** -A Lightning App is simply organized Python code. To modify an existing Lightning App, simply change the code! -There's nothing more you need to do. - ----- - -*************** -Add a component -*************** -A major superpower you get with Lightning Apps is modular workflows. This allows you to use our gallery -of opensource components to power your work. Find a component in the `component gallery `_ and add it to your -Lightning App. - -If you need inspiration, here are some components you might want to use to extend your Lightning App: - - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Slack Messenger - :description: Send a Slack notification when anything happens in your Lightning App. - :button_link: https://lightning.ai/components/LJFITYeNBZ-Slack%20Messenger - :col_css: col-md-4 - :height: 200 - :tag: monitoring - -.. displayitem:: - :header: Lightning Serve - :description: Make your model serving infrastructure resilient with over 7+ strategies such as canary, recreate, ramped and more. - :button_link: https://lightning.ai/components/BA2slXIke9-Lightning%20Serve - :col_css: col-md-4 - :height: 200 - :tag: production - -.. displayitem:: - :header: Lightning HPO - :description: Add a hyperparameter sweep to your Lightning App. - :button_link: https://lightning.ai/components/BA2slXI093-Lightning%20HPO - :col_css: col-md-4 - :height: 200 - :tag: research - -.. displayitem:: - :header: Jupyter Notebook - :description: Add a Jupyter Notebook to your Lightning App. - :button_link: https://lightning.ai/components/cRH1UHnvBx-Jupyter%20Notebook - :col_css: col-md-4 - :height: 200 - :tag: data science - -.. displayitem:: - :header: Google BigQuery - :description: Connect a big BigQuery dataset to your Lightning App. - :button_link: https://lightning.ai/components/Mtt4fnRlUE-Google%20BigQuery - :col_css: col-md-4 - :height: 200 - :tag: production - -.. raw:: html - -
-
- ----- - -********************* -Install the component -********************* -To add a component, install the component first. - -We'll use the Slack messaging component as example: - -.. code:: bash - - lightning install component lightning/lit-slack-messenger - - -Now that the component is installed, make sure you add it to your requirements.txt - -.. code:: bash - - echo 'git+https://github.com/Lightning-AI/LAI-slack-messenger.git@4aa91554f51baf56fc14316365c67fcc67b61e7d' > requirements.txt - ----- - -***************** -Use the component -***************** -To use the component, simply import it and attach it to your Lightning App. - -.. code:: python - - import lightning as L - from lit_slack import SlackMessenger - - - class YourComponent(L.LightningFlow): - def __init__(self): - super().__init__() - self.slack_messenger = SlackMessenger(token="a-long-token", channel_id="A03CB4A6AK7") - - def run(self): - self.slack_messenger.send_message("hello from ⚡ lit slack ⚡") - - - app = L.LightningApp(YourComponent()) diff --git a/docs/source-app/levels/basic/level_5.rst b/docs/source-app/levels/basic/level_5.rst deleted file mode 100644 index c7ec29e784..0000000000 --- a/docs/source-app/levels/basic/level_5.rst +++ /dev/null @@ -1,31 +0,0 @@ -################################## -Level 5: Share your Lightning Apps -################################## -**Audience:** Users who work in teams. - -**Prereqs:** You've ran a Lightning App locally and the cloud. - ----- - -**************************************** -What sharing Lightning Apps does for you -**************************************** -When working on teams or to show your Lightning App to your users or collaborators, it's often -helpful to share a link to your Lightning App. - -Sharing a link to your Lightning App, allows your collaborators to see what you've built, instead of them having -to clone, install the repo, and very likely not get anything running. - ----- - -************************ -Share your Lightning App -************************ -The easiest way to share your Lightning Apps is to run them on Lightning Cloud: - -.. code:: bash - - lightning run app app.py --cloud - -Once your Lightning App starts running, press "Open", then copy the link from the browser and share the link with your -colleagues! diff --git a/docs/source-app/levels/basic/level_6.rst b/docs/source-app/levels/basic/level_6.rst deleted file mode 100644 index 8a228029ca..0000000000 --- a/docs/source-app/levels/basic/level_6.rst +++ /dev/null @@ -1,78 +0,0 @@ -################################### -Level 6: Publish your Lightning App -################################### -**Audience:** New users to the framework. - -**Prereqs:** None. - ----- - -******************************************* -What publishing Lightning Apps does for you -******************************************* -Publishing a Lightning App, allows you to share your work with the Lightning community so that others won't have -to repeat the work you've built. This is at the core of the opensource ethos, that allows us to build -on top of each other's work. - -If your Lightning App is for a company, or business you are trying to monetize, publishing your Lightning App allows your users -to visit a single place where they won't have to worry about installing your Lightning App or running it on their own -accounts. - ----- - -************************** -Publish your Lightning App -************************** -To publish a Lightning App, visit `lightning.ai `_, expand the "Apps & Components" menu, and hit "Submit yours". -This will take you to `this form `_. - ----- - -******************************************** -Guidelines for publishing your Lightning App -******************************************** -Only the very best apps get accepted to the Lightning Gallery. Here are some guidelines to help you with yours. - ----- - -Thumbnail -^^^^^^^^^ -Make sure you have a high-quality Thumbnail. The image should be 300x200 and smaller than 20kb. - ----- - -App name -^^^^^^^^ -App names can only be 1-3 words at most. Names should be unique/recognizable. Remember Uber, Snapchat, etc... - ----- - -Short Description -^^^^^^^^^^^^^^^^^ -The short description should be succinct and to the point. Avoid filler words like "this app allows you to". It should -describe what your users can do with the app NOT the tools that are being used in the app. - ----- - -Long Description -^^^^^^^^^^^^^^^^ -Use the long description to describe to the user what the app does in more detail, and why they should use it. -This is your time to shine! - -Don't use gimmiky sentences like "have you ever wanted to?", "wouldn't it be great if?". Be professional -and let the user know what value your app brings to them. - ----- - -Tags -^^^^ -Tags should allow users to know if the app is relevant for them. This is where it's helpful to let the -user know the domains (NLP, RL, CV) and even the key libraries used (PyTorch Lightning, Tensorboard, Timm, etc...). - ----- - -Code example -^^^^^^^^^^^^ -The code example should be minimal. Do not add anything the user doesn't need to know. Don't try to show -off all the features of your app or component, simply show the most minimal, critical piece of code that -can get your users started immediately. diff --git a/docs/source-app/levels/basic/level_7.rst b/docs/source-app/levels/basic/level_7.rst deleted file mode 100644 index 70f16e116c..0000000000 --- a/docs/source-app/levels/basic/level_7.rst +++ /dev/null @@ -1,36 +0,0 @@ -################################## -Level 7: Run on your cloud account -################################## -**Audience:** Users who want to run on their own cloud accounts - -**Prereqs:** Users ran a Lightning App locally and/or the cloud. - ----- - -**************************** -What is the Lightning Cloud? -**************************** -The Lightning Cloud is the platform that we've created to interface with the cloud providers. Today -the Lightning Cloud supports AWS. - -.. note:: Support for GCP and Azure is coming in the Fall of 2022! - -To use the Lightning Cloud, you buy credits that are used to pay the cloud providers. If you want to run -on your own AWS credentials, please contact us (support@lightning.ai) so we can get your clusters set up for you. - ----- - -**************************************** -Run Lightning Apps on your cloud account -**************************************** -To run Lightning Apps on your own account, you can simply SSH into the machines on your cloud account -and start the Lightning Apps there! This also works on any machine you have access to, such as your own -on-prem cluster, DGX machine, etc... However, you will have to manage your own auto-scaling -and distribition of work for complex Lightning Apps. - -If you want to automate all that complexity, we allow you to create as many clusters as you -want, on the cloud provider of your choice, using Lightning Cloud. Once you've configured your clusters, -you can run your Lightning Apps on those clusters. - -Free clusters on Lightning Cloud have limits of the number of machines you can run. To increase that limit, -please `contact us `_. diff --git a/docs/source-app/levels/intermediate/index.rst b/docs/source-app/levels/intermediate/index.rst deleted file mode 100644 index c55792e0a1..0000000000 --- a/docs/source-app/levels/intermediate/index.rst +++ /dev/null @@ -1,100 +0,0 @@ -.. _intermediate_level: - -.. toctree:: - :maxdepth: 1 - :hidden: - - level_8 - level_9 - level_10 - level_11 - level_12 - level_13 - level_14 - level_15 - -################### -Intermediate skills -################### -Learn to build your own Lightning Apps from scratch and the basics of the framework. - -.. join_slack:: - :align: left - ----- - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Level 8: Build a Lightning App - :description: Learn how to build a Lightning App from scratch. - :button_link: level_8.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level 9: Event loop - :description: Learn about the event loop. - :button_link: level_9.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level 10: Add web UIs - :description: Learn how to add web UIs to your Lightning App. - :button_link: level_10.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level 11: Customize cloud compute - :description: Learn to use different cloud machines. - :button_link: level_11.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level 12: LightningFlow vs LightningWork - :description: Learn the difference between LightningFlow and LightningWork. - :button_link: level_12.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level 13: Communicate between Lightning components - :description: Learn about how LightningFlows communicate with LightningWorks. - :button_link: level_13.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level 14: Share files between components - :description: Learn how Drives share files between components - :button_link: level_14.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. displayitem:: - :header: Level 15: Run LightningWorks in parallel - :description: Learn when to run LightningWorks in parallel - :button_link: level_15.html - :col_css: col-md-6 - :height: 150 - :tag: intermediate - -.. raw:: html - -
-
diff --git a/docs/source-app/levels/intermediate/level_10.rst b/docs/source-app/levels/intermediate/level_10.rst deleted file mode 100644 index 3e20771eab..0000000000 --- a/docs/source-app/levels/intermediate/level_10.rst +++ /dev/null @@ -1,14 +0,0 @@ -########### -Add web UIs -########### - -**Audience:** Users who want to add a web user interface (UI) to their Lightning App - -**Prereqs:** You must have finished the `Basic levels `_. - - -Every component in a Lightning App can have its own web user interface (UI). - ----- - -.. include:: ../../workflows/add_web_ui/index_content.rst diff --git a/docs/source-app/levels/intermediate/level_11.rst b/docs/source-app/levels/intermediate/level_11.rst deleted file mode 100644 index 9dc8804743..0000000000 --- a/docs/source-app/levels/intermediate/level_11.rst +++ /dev/null @@ -1,12 +0,0 @@ -################################# -Level 11: Customize cloud compute -################################# -**Audience:** Users who want to change their cloud compute - -**Prereqs:** - -**Level:** Intermediate - ----- - -.. include:: ../../core_api/lightning_work/compute_content.rst diff --git a/docs/source-app/levels/intermediate/level_12.rst b/docs/source-app/levels/intermediate/level_12.rst deleted file mode 100644 index 644ac59d98..0000000000 --- a/docs/source-app/levels/intermediate/level_12.rst +++ /dev/null @@ -1,10 +0,0 @@ -###################### -Level 12: Flow vs Work -###################### -**Audience:** Users who need to do non trivial workloads in their apps. - -**Prereqs:** Level 8+ - ----- - -.. include:: ../../workflows/build_lightning_component/from_scratch_component_content.rst diff --git a/docs/source-app/levels/intermediate/level_13.rst b/docs/source-app/levels/intermediate/level_13.rst deleted file mode 100644 index f5ce87eda3..0000000000 --- a/docs/source-app/levels/intermediate/level_13.rst +++ /dev/null @@ -1,10 +0,0 @@ -#################################################### -Level 13: Communication between Lightning Components -#################################################### -**Audience:** Users who have multiple LightningWorks communicating with LightningFlows. - -**Prereqs:** Level 8+ - ----- - -.. include:: ../../core_api/lightning_app/communication_content.rst diff --git a/docs/source-app/levels/intermediate/level_14.rst b/docs/source-app/levels/intermediate/level_14.rst deleted file mode 100644 index 5fbdb371c2..0000000000 --- a/docs/source-app/levels/intermediate/level_14.rst +++ /dev/null @@ -1,10 +0,0 @@ -######################################## -Level 14: Share files between components -######################################## -**Audience:** Users who are moving large files such as artifacts or datasets. - -**Prereqs:** Level 8+ - ----- - -.. include:: ../../glossary/storage/drive_content.rst diff --git a/docs/source-app/levels/intermediate/level_15.rst b/docs/source-app/levels/intermediate/level_15.rst deleted file mode 100644 index 255e573a71..0000000000 --- a/docs/source-app/levels/intermediate/level_15.rst +++ /dev/null @@ -1,10 +0,0 @@ -######################################## -Level 15: Run LightningWorks in parallel -######################################## -**Audience:** Users who want to run multiple LightningWorks at once. - -**Prereqs:** Level 8+ - ----- - -.. include:: ../../workflows/run_work_in_parallel_content.rst diff --git a/docs/source-app/levels/intermediate/level_8.rst b/docs/source-app/levels/intermediate/level_8.rst deleted file mode 100644 index a40fe75e82..0000000000 --- a/docs/source-app/levels/intermediate/level_8.rst +++ /dev/null @@ -1,10 +0,0 @@ -########################################### -Level 8: Build a Lightning App from Scratch -########################################### -**Audience:** Users who want to build an Lightning App from scratch - -**Prereqs:** You must have finished the `Basic levels `_. - ----- - -.. include:: ../../workflows/build_lightning_app/from_scratch_content.rst diff --git a/docs/source-app/levels/intermediate/level_9.rst b/docs/source-app/levels/intermediate/level_9.rst deleted file mode 100644 index 33a2e4f50c..0000000000 --- a/docs/source-app/levels/intermediate/level_9.rst +++ /dev/null @@ -1,10 +0,0 @@ -################### -Level 9: Event loop -################### -**Audience:** Users who want to build reactive Lightning Apps and move beyond DAGs. - -**Prereqs:** Level 8+ - ----- - -.. include:: ../../glossary/event_loop.rst diff --git a/docs/source-app/make.bat b/docs/source-app/make.bat deleted file mode 100644 index 9b565142ae..0000000000 --- a/docs/source-app/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=../build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/docs/source-app/basics.rst b/docs/source-app/pages/basics.rst similarity index 91% rename from docs/source-app/basics.rst rename to docs/source-app/pages/basics.rst index 57f04fd103..d0c6fa80f9 100644 --- a/docs/source-app/basics.rst +++ b/docs/source-app/pages/basics.rst @@ -8,29 +8,22 @@ Basics In this guide, we'll cover the basic terminology associated with the Lightning framework. ----- - -************** Lightning App -************** +============= The :class:`~lightning_app.core.app.LightningApp` runs a tree of one or more components that interact to create end-to-end applications. There are two kinds of components: :class:`~lightning_app.core.flow.LightningFlow` and :class:`~lightning_app.core.work.LightningWork`. This modular design enables you to reuse components created by other users. ----- Lightning Work ^^^^^^^^^^^^^^ The :class:`~lightning_app.core.work.LightningWork` component is a building block optimized for long-running jobs or integrating third-party services. LightningWork can be used for training large models, downloading a dataset, or any long-lasting operation. ----- - Lightning Flow ^^^^^^^^^^^^^^ The :class:`~lightning_app.core.flow.LightningFlow` component coordinates long-running tasks :class:`~lightning_app.core.work.LightningWork` and runs its children :class:`~lightning_app.core.flow.LightningFlow` components. ----- Lightning App Tree ^^^^^^^^^^^^^^^^^^ @@ -39,7 +32,7 @@ Components can be nested to form component trees where the LightningFlows are it Here's a basic application with four flows and two works: -.. literalinclude:: code_samples/quickstart/app_comp.py +.. literalinclude:: ../code_samples/quickstart/app_comp.py And here's its associated tree structure: @@ -49,8 +42,6 @@ And here's its associated tree structure: A Lightning App runs all flows into a single process. Its flows coordinate the execution of the works each running in their own independent processes. ----- - Lightning Distributed Event Loop ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -60,7 +51,6 @@ Drawing inspiration from modern web frameworks like `React.js + ---- + We will describe two choices here, pick one: 1. :ref:`Python virtualenv `. 2. :ref:`Conda virtual environment `. + ---- + .. _python-virtualenv: ******************** @@ -44,10 +50,9 @@ If you can't run the command above or it returns a version older than 3.8, `install the latest version of Python `_. After installing it, make sure you can run the above command without errors. ----- Creating a Virtual Environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +============================== When starting with a new Python project, you typically want to create a new Python virtual environment. Navigate to the location of your project and run the following command: @@ -59,10 +64,9 @@ Navigate to the location of your project and run the following command: The name of the environment here is *lightning* but you can choose any other name you like. By running the above command, Python will create a new folder *lightning* in the current working directory. ----- Activating the Virtual Environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +================================== Before you can install packages into the environment, you need to activate it: @@ -72,10 +76,12 @@ Before you can install packages into the environment, you need to activate it: You need to do this step every time you want to work on your project / open the terminal. With your virtual environment activated, you are now ready to -:doc:`install Lightning <../installation>` and get started with Apps! +:doc:`install Lightning ` and get started with Apps! + ---- + .. _conda: ******** @@ -91,10 +97,9 @@ To check that the installation was successful, open an new terminal and run: It should return a list of commands. ----- Creating a Conda Environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +============================ When starting with a new Python project, you typically want to create a new Conda virtual environment. Navigate to the location of your project and run the following command: @@ -106,10 +111,9 @@ Navigate to the location of your project and run the following command: The name of the environment here is *lightning* but you can choose any other name you like. Note how we can also specify the Python version here. ----- Activating the Conda Environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +================================ Before you can install packages into the environment, you need to activate it: @@ -119,4 +123,4 @@ Before you can install packages into the environment, you need to activate it: You need to do this step every time you want to work on your project / open the terminal. With your virtual environment activated, you are now ready to -:doc:`install Lightning <../installation>` and get started with Apps! +:doc:`install Lightning ` and get started with Apps! diff --git a/docs/source-app/pages/installation.rst b/docs/source-app/pages/installation.rst new file mode 100644 index 0000000000..6e704f2bda --- /dev/null +++ b/docs/source-app/pages/installation.rst @@ -0,0 +1,26 @@ + +.. _install: + + +############ +Installation +############ + +We strongly recommend to create a virtual environment first. +Don't know what this is? Follow our `beginner guide here `_. + +Python >= 3.8 is required. + +**************** +Install with pip +**************** + +0. Activate your virtual environment. + +1. Install the lightning package + + .. code:: bash + + python -m pip install -U lightning + + The ``--extra-index-url`` won't be required after June 16. diff --git a/docs/source-app/intro.rst b/docs/source-app/pages/introduction.rst similarity index 95% rename from docs/source-app/intro.rst rename to docs/source-app/pages/introduction.rst index 4b8f1317c1..5fff153dc8 100644 --- a/docs/source-app/intro.rst +++ b/docs/source-app/pages/introduction.rst @@ -2,9 +2,8 @@ .. _what: -################### What is Lightning? -################### +================== Lightning is a free, modular, distributed, and open-source framework for building AI applications where the components you want to use interact together. @@ -20,13 +19,10 @@ regardless of their level of engineering expertise. :alt: What is Lightning gif. :width: 100 % ----- - .. _why: -*************** Why Lightning? -*************** +============== Easy to learn @@ -35,24 +31,18 @@ Easy to learn Lightning was built for creating AI apps, not for dev-ops. It offers an intuitive, pythonic and highly composable interface that allows you to focus on solving the problems that are important to you. ----- - Quick to deliver ^^^^^^^^^^^^^^^^ Lightning speeds the development process by offering testable templates you can build from, accelerating the process of moving from idea to prototype and finally to market. ----- - Easy to scale ^^^^^^^^^^^^^ Lightning provides a mirrored experience locally and in the cloud. The `lightning.ai `_. cloud platform abstracts the infrastructure, so you can run your apps at any scale. ----- - Easy to collaborate ^^^^^^^^^^^^^^^^^^^ @@ -60,11 +50,8 @@ Lightning was built for collaboration. By following the best MLOps practices provided through our documentation and example use cases, you can deploy state-of-the-art ML applications that are ready to be used by teams of all sizes. ----- - -***************************** What's Novel With Lightning? -***************************** +============================ Cloud Infra Made Simple and Pythonic diff --git a/docs/source-app/get_started/go_beyond_training_content.rst b/docs/source-app/pages/lightning_apps_intro.rst similarity index 64% rename from docs/source-app/get_started/go_beyond_training_content.rst rename to docs/source-app/pages/lightning_apps_intro.rst index a471e91d85..ba3c962f16 100644 --- a/docs/source-app/get_started/go_beyond_training_content.rst +++ b/docs/source-app/pages/lightning_apps_intro.rst @@ -1,23 +1,63 @@ -************************************************ -The *Train & Demo PyTorch Lightning* Application -************************************************ +.. toctree:: + :maxdepth: 1 + :hidden: -Find the *Train & Demo PyTorch Lightning* application in the `Lightning.ai App Gallery `_. + ../workflows/extend_app + ../workflows/build_lightning_component/index + ../glossary/lightning_app_overview/index + ../workflows/run_on_private_cloud -Here is a recording of this App running locally and in the cloud with the same behavior. + +############################ +Lightning Apps in 15 minutes +############################ + +**Required background:** Basic Python familiarity. + +**Goal:** In this guide, we'll walk you through the 4 key steps to build your first Lightning app. + +---- + +The app we build in this guide trains and deploys a model. + +.. + (|qs_app|). + + .. |qs_app| raw:: html + + see the app live here + + +A Lightning app is **Organized Python**, it enables AI researchers and ML engineers to build complex AI workflows without any of the **cloud** boilerplate. + +With Lightning apps your favorite components can work together on any machine at any scale. Here's an illustration: .. raw:: html -
- -
-
+ -In the steps below, we are going to show you how to build this application. +| -Here are `the entire App's code `_ and `its commented components. `_ +Lightning Apps are: + +- cloud agnostic +- fault-tolerant +- production ready +- locally debuggable +- and much more + +.. join_slack:: + :align: left + :margin: 20 + + +---- + +******************* +Who can build apps? +******************* +Anyone who knows Python can build a Lightning app, even without ML experience. ---- @@ -25,21 +65,20 @@ Here are `the entire App's code `, don't forget to activate it before running commands. -You must do so in every new shell. +First, you'll need to install Lightning (make sure you use Python 3.8+). -.. tip:: We highly recommend using virtual environments. +.. code:: bash -.. code:: python + python -m pip install -U lightning - pip install lightning +(pip and conda install coming soon) ---- -**************************************** -Step 2: Install the *Train and Demo* App -**************************************** -The first Lightning App we'll explore is an App to train and demo a machine learning model. +******************************** +Step 2: Install Train Deploy App +******************************** +The first Lightning app we'll explore is an app to train and deploy a machine learning model. .. [|qs_code|], [|qs_live_app|]. @@ -50,65 +89,70 @@ The first Lightning App we'll explore is an App to train and demo a machine lear .. |qs_code| raw:: html - code + code -Install this App by typing: +Install this app by typing: .. code-block:: bash lightning install app lightning/quick-start -Verify the App was succesfully installed: +Verify the app was succesfully installed: .. code-block:: bash cd lightning-quick-start + ls ---- *************************** -Step 3: Run the App locally +Step 3: Run the app locally *************************** - -Run the app locally with the ``run`` command 🤯 +Run the app locally with the ``run`` command .. code:: bash lightning run app app.py +🤯 + ---- ******************************** -Step 4: Run the App in the cloud +Step 4: Run the app on the cloud ******************************** - -Add the ``--cloud`` argument to run on the `Lightning.AI cloud `_. 🤯🤯🤯 +Add the ``--cloud`` argument to run on the `Lightning.AI cloud `_. .. code:: bash lightning run app app.py --cloud +🤯🤯🤯 + .. Your app should look like this one (|qs_live_app|) + ---- ******************* Understand the code ******************* -The App that we just launched trained a PyTorch Lightning model (although any framework works), then added an interactive demo. +The app that we just launched trained a PyTorch Lightning model (although any framework works), then added an interactive demo. -This is the App's code: +This is the app code: .. code:: python # lightning-quick-start/app.py import os.path as ops - import lightning as L + import lightning_app as la from quick_start.components import PyTorchLightningScript, ImageServeGradio - class TrainDeploy(L.LightningFlow): + + class TrainDeploy(lapp.LightningFlow): def __init__(self): super().__init__() self.train_work = PyTorchLightningScript( @@ -116,7 +160,7 @@ This is the App's code: script_args=["--trainer.max_epochs=5"], ) - self.serve_work = ImageServeGradio(L.CloudCompute()) + self.serve_work = ImageServeGradio(lapp.CloudCompute()) def run(self): # 1. Run the python script that trains the model @@ -131,18 +175,18 @@ This is the App's code: tab_2 = {"name": "Interactive demo", "content": self.serve_work} return [tab_1, tab_2] - app = L.LightningApp(TrainDeploy()) -Let's break down the code section by section to understand what it is doing. + app = lapp.LightningApp(TrainDeploy()) + +Let's break down the app code by each section to understand what it is doing. ---- 1: Define root component ^^^^^^^^^^^^^^^^^^^^^^^^ +A Lightning app provides a cohesive product experience for a set of unrelated components. -A Lightning App provides a cohesive product experience for a set of unrelated components. - -The top-level component (Root) must subclass ``L.LightningFlow`` +The top-level component (Root) must subclass lapp.LightningFlow .. code:: python @@ -150,10 +194,11 @@ The top-level component (Root) must subclass ``L.LightningFlow`` # lightning-quick-start/app.py import os.path as ops - import lightning as L + import lightning_app as la from quick_start.components import PyTorchLightningScript, ImageServeGradio - class TrainDeploy(L.LightningFlow): + + class TrainDeploy(lapp.LightningFlow): def __init__(self): super().__init__() self.train_work = PyTorchLightningScript( @@ -161,7 +206,7 @@ The top-level component (Root) must subclass ``L.LightningFlow`` script_args=["--trainer.max_epochs=5"], ) - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) + self.serve_work = ImageServeGradio(lapp.CloudCompute("cpu-small")) def run(self): # 1. Run the python script that trains the model @@ -176,25 +221,27 @@ The top-level component (Root) must subclass ``L.LightningFlow`` tab_2 = {"name": "Interactive demo", "content": self.serve_work} return [tab_1, tab_2] - app = L.LightningApp(TrainDeploy()) + + app = lapp.LightningApp(TrainDeploy()) ---- 2: Define components ^^^^^^^^^^^^^^^^^^^^ -In the __init__ method, we define the components that make up the App. In this case, we have 2 components, -a component to execute any PyTorch Lightning script (model training) and a second component to -start a Gradio server for demo purposes. +In the __init__ method, we define the components that make up the app. In this case, we have 2 components, +a component to execute any pytorch lightning script (model training) and a second component to +start a gradio server for demo purposes. .. code:: python :emphasize-lines: 9, 14 # lightning-quick-start/app.py import os.path as ops - import lightning as L + import lightning_app as la from quick_start.components import PyTorchLightningScript, ImageServeGradio - class TrainDeploy(L.LightningFlow): + + class TrainDeploy(lapp.LightningFlow): def __init__(self): super().__init__() self.train_work = PyTorchLightningScript( @@ -202,7 +249,7 @@ start a Gradio server for demo purposes. script_args=["--trainer.max_epochs=5"], ) - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) + self.serve_work = ImageServeGradio(lapp.CloudCompute("cpu-small")) def run(self): # 1. Run the python script that trains the model @@ -217,13 +264,14 @@ start a Gradio server for demo purposes. tab_2 = {"name": "Interactive demo", "content": self.serve_work} return [tab_1, tab_2] - app = L.LightningApp(TrainDeploy()) + + app = lapp.LightningApp(TrainDeploy()) ---- -3: Define how components Flow +3: Define how components flow ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Every component has a ``run`` method. The run method defines the 🌊 Flow 🌊 of how components interact together. +Every component has a `run` method. The run method defines the 🌊 flow 🌊 of how components interact together. In this case, we train a model (until completion). When it's done AND there exists a checkpoint, we launch a demo server: @@ -233,10 +281,11 @@ demo server: # lightning-quick-start/app.py import os.path as ops - import lightning as L + import lightning_app as la from quick_start.components import PyTorchLightningScript, ImageServeGradio - class TrainDeploy(L.LightningFlow): + + class TrainDeploy(lapp.LightningFlow): def __init__(self): super().__init__() self.train_work = PyTorchLightningScript( @@ -244,7 +293,7 @@ demo server: script_args=["--trainer.max_epochs=5"], ) - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) + self.serve_work = ImageServeGradio(lapp.CloudCompute("cpu-small")) def run(self): # 1. Run the python script that trains the model @@ -259,9 +308,12 @@ demo server: tab_2 = {"name": "Interactive demo", "content": self.serve_work} return [tab_1, tab_2] - app = L.LightningApp(TrainDeploy()) -.. note:: If you've used other ML systems you'll be pleasantly surprised to not find decorators or YAML files. + app = lapp.LightningApp(TrainDeploy()) + +.. + If you've used other ML systems you'll be pleasantly surprised to not find decorators or YAML files. + Read here to understand the benefits more. ---- @@ -269,17 +321,18 @@ demo server: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ All our favorite tools normally have their own web user interfaces (UI). -Implement the ``configure_layout`` method to connect them together: +Implement the `configure_layout` method to connect them together: .. code:: python :emphasize-lines: 24-27 # lightning-quick-start/app.py import os.path as ops - import lightning as L + import lightning_app as la from quick_start.components import PyTorchLightningScript, ImageServeGradio - class TrainDeploy(L.LightningFlow): + + class TrainDeploy(lapp.LightningFlow): def __init__(self): super().__init__() self.train_work = PyTorchLightningScript( @@ -287,7 +340,7 @@ Implement the ``configure_layout`` method to connect them together: script_args=["--trainer.max_epochs=5"], ) - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) + self.serve_work = ImageServeGradio(lapp.CloudCompute("cpu-small")) def run(self): # 1. Run the python script that trains the model @@ -302,23 +355,25 @@ Implement the ``configure_layout`` method to connect them together: tab_2 = {"name": "Interactive demo", "content": self.serve_work} return [tab_1, tab_2] - app = L.LightningApp(TrainDeploy()) + + app = lapp.LightningApp(TrainDeploy()) ---- -5: Init the ``app`` object -^^^^^^^^^^^^^^^^^^^^^^^^^^ -Initalize an ``app`` object with the ``TrainDeploy`` component (this won't run the App yet): +5: Init the app object +^^^^^^^^^^^^^^^^^^^^^^ +Initalize an app object with the `TrainDeploy` component (this won't run the app yet): .. code:: python :emphasize-lines: 29 # lightning-quick-start/app.py import os.path as ops - import lightning as L + import lightning_app as la from quick_start.components import PyTorchLightningScript, ImageServeGradio - class TrainDeploy(L.LightningFlow): + + class TrainDeploy(lapp.LightningFlow): def __init__(self): super().__init__() self.train_work = PyTorchLightningScript( @@ -326,7 +381,7 @@ Initalize an ``app`` object with the ``TrainDeploy`` component (this won't run t script_args=["--trainer.max_epochs=5"], ) - self.serve_work = ImageServeGradio(L.CloudCompute("cpu-small")) + self.serve_work = ImageServeGradio(lapp.CloudCompute("cpu-small")) def run(self): # 1. Run the python script that trains the model @@ -341,7 +396,8 @@ Initalize an ``app`` object with the ``TrainDeploy`` component (this won't run t tab_2 = {"name": "Interactive demo", "content": self.serve_work} return [tab_1, tab_2] - app = L.LightningApp(TrainDeploy()) + + app = lapp.LightningApp(TrainDeploy()) ---- @@ -357,8 +413,9 @@ Any component can work with Lightning AI! ---- ********** -Next Steps +Next steps ********** +Depending on your use case, you might want to check one of these out next. .. raw:: html @@ -366,8 +423,8 @@ Next Steps
.. displayitem:: - :header: Add components to your App - :description: Expand your App by adding components. + :header: Add components to your app + :description: Expand your app by adding components. :col_css: col-md-4 :button_link: ../workflows/extend_app.html :height: 180 @@ -380,22 +437,22 @@ Next Steps :height: 180 .. displayitem:: - :header: Explore more Apps + :header: Explore more apps :description: Explore more apps for inspiration. :col_css: col-md-4 :button_link: https://lightning.ai/apps :height: 180 .. displayitem:: - :header: Under the hood + :header: How it works under the hood :description: Explore how it works under the hood. :col_css: col-md-4 - :button_link: ../core_api/lightning_app/index.html + :button_link: core_api/lightning_app/index.html :height: 180 -.. displayitem:: +.. displayitem::workflo :header: Run on your private cloud - :description: Run Lightning Apps on your private VPC or on-prem. + :description: Run lightning apps on your private VPC or on-prem. :button_link: ../workflows/run_on_private_cloud.html :col_css: col-md-4 :height: 180 diff --git a/docs/source-app/moving_to_the_cloud.rst b/docs/source-app/pages/moving_to_the_cloud.rst similarity index 85% rename from docs/source-app/moving_to_the_cloud.rst rename to docs/source-app/pages/moving_to_the_cloud.rst index 210fe32db8..27d01c11a1 100644 --- a/docs/source-app/moving_to_the_cloud.rst +++ b/docs/source-app/pages/moving_to_the_cloud.rst @@ -2,9 +2,9 @@ .. _moving_to_the_cloud: -#################### +******************* Moving to the Cloud -#################### +******************* .. warning:: This is in progress and not yet fully supported. @@ -14,12 +14,8 @@ that trains an image classifier and serve it once trained. In this tutorial, you'll learn how to extend that application so that it works seamlessly both locally and in the cloud. ----- - -******************************** Step 1: Distributed Application -******************************** - +=============================== Distributed Storage ^^^^^^^^^^^^^^^^^^^ @@ -40,17 +36,18 @@ Without doing this conscientiously for every single path, your application will In the example below, a file written by **SourceFileWork** is being transferred by the flow to the **DestinationFileAndServeWork** work. The Path object is the reference to the file. -.. literalinclude:: ../examples/app_boring/app.py +.. literalinclude:: ../../../examples/boring_app/app.py :emphasize-lines: 5, 22, 28, 48 -In the ``scripts/serve.py`` file, we are creating a **FastApi Service** running on port ``1111`` +In the ``../scripts/serve.py`` file, we are creating a **FastApi Service** running on port ``1111`` that returns the content of the file received from **SourceFileWork** when a post request is sent to ``/file``. -.. literalinclude:: ../examples/app_boring/scripts/serve.py +.. literalinclude:: ../../../examples/boring_app/scripts/serve.py :emphasize-lines: 21, 23-26 ----- + + Distributed Frontend ^^^^^^^^^^^^^^^^^^^^ @@ -64,14 +61,14 @@ In order to assemble them, you need to do two things: Here's how to expose the port: -.. literalinclude:: ../examples/app_boring/app.py +.. literalinclude:: ../../../examples/boring_app/app.py :emphasize-lines: 8 :lines: 33-44 And here's how to expose your services within the ``configure_layout`` flow hook: -.. literalinclude:: ../examples/app_boring/app.py +.. literalinclude:: ../../../examples/boring_app/app.py :emphasize-lines: 5 :lines: 53-57 @@ -79,7 +76,7 @@ In this example, we're appending ``/file`` to our **FastApi Service** url. This means that our ``Boring Tab`` triggers the ``get_file_content`` from the **FastAPI Service** and embeds its content as an `IFrame `_. -.. literalinclude:: ../examples/app_boring/scripts/serve.py +.. literalinclude:: ../../../examples/boring_app/scripts/serve.py :lines: 23-26 @@ -89,11 +86,9 @@ Here's a visualization of the application described above: :alt: Storage API Animation :width: 100 % ----- -***************************** Step 2: Scalable Application -***************************** +============================ The benefit of defining long-running code inside a :class:`~lightning_app.core.work.LightningWork` @@ -106,11 +101,8 @@ By adapting the :ref:`quick_start` example as follows, you can easily run your c Without doing much, you’re now running a script on its own cluster of machines! 🤯 ----- - -***************************** Step 3: Resilient Application -***************************** +============================= We designed Lightning with a strong emphasis on supporting failure cases. The framework shines when the developer embraces our fault-tolerance best practices, diff --git a/docs/source-app/quickstart.rst b/docs/source-app/pages/quickstart.rst similarity index 90% rename from docs/source-app/quickstart.rst rename to docs/source-app/pages/quickstart.rst index f14ba98e18..4ea358fa65 100644 --- a/docs/source-app/quickstart.rst +++ b/docs/source-app/pages/quickstart.rst @@ -2,23 +2,20 @@ .. _quick_start: -############ +*********** Quick Start -############ +*********** In this guide, we'll run an application that trains an image classification model with the `MNIST Dataset `_, and uses `FastAPI `_ to serve it. ----- - -********************** Step 1 - Installation -********************** +===================== First, you'll need to install Lightning from source. You can find the complete guide here: :ref:`install`. -Then, you'll need to install the `Lightning Quick Start package `_. +Then, you'll need to install the `Lightning Quick Start package `_. .. code-block:: bash @@ -30,11 +27,8 @@ And download the training script used by the App: curl https://gist.githubusercontent.com/tchaton/b81c8d8ba0f4dd39a47bfa607d81d6d5/raw/8d9d70573a006d95bdcda8492e798d0771d7e61b/train_script.py > train_script.py ----- - -********************** Step 2 - Run the app -********************** +==================== To run your app, copy the following command to your local terminal: @@ -83,10 +77,8 @@ The build command will launch the app admin panel UI. In your app admin, you can :alt: Quick Start UI :width: 100 % ----- - This app behind the scenes -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------- This application has one flow component which coordinates two works executing their own python script. Once the training is finished, the trained model weights are passed to the serve component. @@ -104,13 +96,11 @@ Here is the application timeline: :alt: Quick Start Timeline Application :width: 100 % ----- -************************************** Steps 3 - Build your app in the cloud -************************************** +===================================== -Simply add ``--cloud`` to run this application in the cloud 🤯 +Simply add **--cloud** to run this application in the cloud 🤯 .. code-block:: bash @@ -125,11 +115,9 @@ And with just one line of code, run on cloud GPUs! Congratulations! You've now run your first application with Lightning. ----- -*********** -Next Steps -*********** +Next steps +========== To learn how to build and modify apps, go to the :ref:`basics`. diff --git a/docs/source-app/testing.rst b/docs/source-app/pages/testing.rst similarity index 95% rename from docs/source-app/testing.rst rename to docs/source-app/pages/testing.rst index 51ec1841a4..8b38f21096 100644 --- a/docs/source-app/testing.rst +++ b/docs/source-app/pages/testing.rst @@ -10,11 +10,8 @@ TODO: Cleanup At the core of our system is an integration testing framework that will allow for a first-class experience creating integration tests for Lightning Apps. This document will explain how we can create a lightning app test, how we can execute it, and where to find more information. ----- - -*********** Philosophy -*********** +---------- Testing a Lightning app is unique. It is a superset of an application that converges machine learning, API development, and UI development. With that in mind, there are several philosophies (or "best practices") that you should adhere to: @@ -26,11 +23,8 @@ Testing a Lightning app is unique. It is a superset of an application that conve #. **Use your framework** - Testing apps should be framework agnostic. #. **Have fun!** - At the heart of testing is experimentation. Like any experiment, tests begin with a hypothesis of workability, but you can extend that to be more inclusive. Ask the question, write the test to answer your question, and make sure you have fun while doing it. ----- - -**************************************** Anatomy of a Lightning integration test -**************************************** +--------------------------------------- The following is a PyTest example of an integration test using the ``lightning_app.testing.testing`` module. @@ -54,7 +48,7 @@ The following is a PyTest example of an integration test using the ``lightning_a def test_v0_app_example(): command_line = [ - os.path.join(_PROJECT_ROOT, "examples/app_v0/app.py"), + os.path.join(_PROJECT_ROOT, "examples/v0_app/app.py"), "--blocking", "False", "--multiprocess", @@ -65,8 +59,6 @@ The following is a PyTest example of an integration test using the ``lightning_a assert "V0 App End" in str(result.stdout_bytes) assert result.exit_code == 0 ----- - Setting up the app ^^^^^^^^^^^^^^^^^^ @@ -80,10 +72,8 @@ To get started, you simply need to import the following: We will discuss ``application_testing`` in a bit, but first let's review the structure of ``TestLightningApp``. ----- - TestLightningApp -^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~ The :class:`lightning_app.testing.testing.TestingLightningApp` class is available to use for provisioning and setting up your testing needs. Note that you do not need this class to move forward with testing. Any application that inherits ``LightningApp`` should suffice as long as you override the correct methods. Reviewing the TestLightnigApp we see some overrides that are already there. Please revuew the class for more information. @@ -107,8 +97,6 @@ Besides ``run_once`` there are a few other overrides available: These methods will skew your tests, so use them when needed. ----- - The Test ^^^^^^^^ @@ -117,7 +105,7 @@ We provide ``application_testing`` as a helper funtion to get your application u .. code-block:: command_line = [ - os.path.join(_PROJECT_ROOT, "examples/app_v0/app.py"), + os.path.join(_PROJECT_ROOT, "examples/v0_app/app.py"), "--blocking", "False", "--multiprocess", @@ -149,10 +137,7 @@ As mentioned earlier, ``application_testing`` is a helper method that allows you Since we injected "V0 App End" to the end of our test flow. The state was changed to ``AppStatus.STOPPING`` which means the process is done. Finally, we check the result's exit code to make sure that we did not throw an error during execution. ----- - -************ End to End -************ +---------- TODO diff --git a/docs/source-app/ui_and_frontends.rst b/docs/source-app/pages/ui_and_frontends.rst similarity index 82% rename from docs/source-app/ui_and_frontends.rst rename to docs/source-app/pages/ui_and_frontends.rst index 53e50cd534..0db452fdfb 100644 --- a/docs/source-app/ui_and_frontends.rst +++ b/docs/source-app/pages/ui_and_frontends.rst @@ -14,8 +14,8 @@ You can easily embed other tools and services (like a GitHub repo, a `FastAPI Se To get started, you can use built-in templates for the following frameworks: -* `React.js `_ -* `StreamLit `_ +* `React.js `_ +* `StreamLit `_ diff --git a/docs/source-app/examples/hpo/hpo.py b/docs/source-app/tutorials/hpo/hpo.py similarity index 70% rename from docs/source-app/examples/hpo/hpo.py rename to docs/source-app/tutorials/hpo/hpo.py index fd05cc2e32..69bccf67ac 100644 --- a/docs/source-app/examples/hpo/hpo.py +++ b/docs/source-app/tutorials/hpo/hpo.py @@ -1,6 +1,5 @@ import optuna from objective import ObjectiveWork -from optuna.distributions import CategoricalDistribution, LogUniformDistribution TOTAL_TRIALS = 6 SIMULTANEOUS_TRIALS = 2 @@ -9,16 +8,14 @@ DONE = False STUDY = optuna.create_study() DISTRIBUTIONS = { - "backbone": CategoricalDistribution(["resnet18", "resnet34"]), - "learning_rate": LogUniformDistribution(0.0001, 0.1), + "backbone": optuna.distributions.CategoricalDistribution(["resnet18", "resnet34"]), + "learning_rate": optuna.distributions.LogUniformDistribution(0.0001, 0.1), } TRIALS = [ObjectiveWork() for _ in range(TOTAL_TRIALS)] -# Lightning Infinite Loop -while not DONE: +while not DONE: # Lightning Infinite Loop - # Finish the Hyperparameter Optimization - if NUM_TRIALS >= TOTAL_TRIALS: + if NUM_TRIALS > TOTAL_TRIALS: # Finish the Hyperparameter Optimization DONE = True continue @@ -31,12 +28,10 @@ while not DONE: # If a work has already started, it won't be started again. if not objective_work.has_started: - # Sample a new trial from the distributions - trial = STUDY.ask(DISTRIBUTIONS) - # Run the work - objective_work.run(trial_id=trial._trial_id, **trial.params) + trial = STUDY.ask(DISTRIBUTIONS) # Sample a new trial from the distributions + objective_work.run(trial_id=trial._trial_id, **trial.params) # Run the work - # With Lightning, the `objective_work` will run asynchronously + # With Lighting, the `objective_work` will run asynchronously # and the metric will be prodcued after X amount of time. # The Lightning Infinite Loop would have run a very large number of times by then. if objective_work.metric and not objective_work.has_told_study: diff --git a/docs/source-app/tutorials/hpo/hpo.rst b/docs/source-app/tutorials/hpo/hpo.rst new file mode 100644 index 0000000000..d6567194ae --- /dev/null +++ b/docs/source-app/tutorials/hpo/hpo.rst @@ -0,0 +1,146 @@ +.. hpo: + +****************** +Build a Sweeps App +****************** + +Introduction +============ + +Traditionally, developing ML Products requires to choose among a large space of +hyperparameters while creating and training the ML models. Hyperparameter Optimization +(HPO) aims at finding a well-performing hyperparameter configuration of a given machine +learning model on a dataset at hand, including the machine learning model, +its hyperparameters and other data processing steps. + +Thus, HPO frees the human expert from a tedious and error-prone hyperparameter tuning process. + +As an example, in the famous `scikit-learn `_, hyperparameter are passed as arguments to the constructor of +the estimator classes such as ``C`` kernel for +`Support Vector Classifier `_, etc. + +It is possible and recommended to search the hyper-parameter space for the best validation score. + +An HPO search consists of: + +* an objective method +* a parameter space defined +* a method for searching or sampling candidates + +A naive method for sampling candidates is Grid Search which exhaustively considers all parameter combinations. + +Hopefully, the field of Hyperparameter Optimization is very active and many methods have been developed to +optimize the time required to get strong candidates. + +In the following tutorial, you will learn how to use Lightning together with the `Optuna `_. + +`Optuna `_ is an open source hyperparameter optimization framework to automate hyperparameter search. +Out-of-the-box, it provides efficient algorithms to search large spaces and prune unpromising trials for faster results. + +First, you will learn about the best practices on how to implement HPO without the Lightning Framework. +Secondly, we will dive into a working HPO application with Lightning and finally create a neat +`HiPlot UI `_ +for our application. + +HPO Example Without Lightning +============================= + +In the example below, we are emulating the Lightning Infinite Loop. + +We are assuming have already defined an ``ObjectiveWork`` component which is responsible to run the objective method and track the metric through its state. + +We are running ``TOTAL_TRIALS`` trials by series of ``SIMULTANEOUS_TRIALS`` trials. +When starting, ``TOTAL_TRIALS`` ``ObjectiveWork`` are created. + +The entire code runs within an infinite loop as it would within Lightning. + +When iterating through the works, if the current ``objective_work`` hasn't started, +some new parameters are sampled from the Optuna Study with our custom distributions +and then passed to run method of the ``objective_work``. + +The condition ``not objective_work.has_started`` will be ``False`` once ``objective_work.run()`` starts. + +Also, the second condition will be ``True`` when the metric finally is defined within the state of the work, +which happens after many iterations of the Lightning Infinite Loop. + +Finally, once the current ``SIMULTANEOUS_TRIALS`` have both registered their +metric to the Optuna Study, simply increments ``NUM_TRIALS`` by ``SIMULTANEOUS_TRIALS`` to launch the next trials. + +.. literalinclude:: ./hpo.py + :language: python + :emphasize-lines: 14, 16, 30-32, 37-40, 43, 46-47 + +Below, you can find the simplified version of the ``ObjectiveWork`` where the metric is randomly sampled using numpy. + +In realistic use case, the work executes some user defined code. + +.. literalinclude:: ./objective.py + :language: python + :emphasize-lines: 8, 19-21 + +Here are the logs produced when running the application above: + +.. code-block:: console + + $ python docs/source/tutorials/hpo/hpo.py + INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view + # After you have clicked `run` on the UI. + [I 2022-03-01 12:32:50,050] A new study created in memory with name: ... + {0: 13.994859806481264, 1: 59.866743330127825, ..., 5: 94.65919769609225} + +In the cloud, here is an animation how this application works: + +.. image:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/hpo.gif + :alt: Animation showing how to HPO works UI in a distributed manner. + + +HPO Example With Lightning +========================== + +Thanks the simplified version, you should have a good grasp on how to implement HPO with Optuna + +As the :class:`~lightning_app.core.app.LightningApp` handles the Infinite Loop, +it has been removed from within the run method of the HPORootFlow. + +However, the run method code is the same as the one defined above. + +.. literalinclude:: ../../../../examples/hpo/app_wo_ui.py + :language: python + :emphasize-lines: 5, 17-23, 52-59 + +The ``ObjectiveWork`` is sub-classing +the built-in :class:`~lightning_app.components.python.TracerPythonScript` +which enables to launch scripts and more. + +.. literalinclude:: ../../../../examples/hpo/objective.py + :language: python + :emphasize-lines: 9, 11, 15, 21-30, 40-46 + +Finally, let's add ``HiPlotFlow`` component to visualize our hyperparameter optimization. + +The metric and sampled parameters are added to the ``self.hi_plot.data`` list, enabling +to get the dashboard updated in near-realtime. + +.. literalinclude:: ../../../../examples/hpo/app_wi_ui.py + :diff: ../../../../examples/hpo/app_wo_ui.py + +Here is the associated code with the ``HiPlotFlow`` component. + +In the ``render_fn`` method, the state of the ``HiPlotFlow`` is passed. +The ``state.data`` is accessed as it contains the metric and sampled parameters. + +.. literalinclude:: ../../../../examples/hpo/hyperplot.py + +Run the HPO application with the following command: + +.. code-block:: console + + $ lightning run app examples/hpo/app_wi_ui.py + INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view + {0: ..., 1: ..., ..., 5: ...} + +Here is how the UI looks like when launched: + +.. image:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/hpo_ui_2.gif + :width: 100 % + :alt: Alternative text diff --git a/docs/source-app/examples/hpo/objective.py b/docs/source-app/tutorials/hpo/objective.py similarity index 55% rename from docs/source-app/examples/hpo/objective.py rename to docs/source-app/tutorials/hpo/objective.py index d20232fb10..7e83551a14 100644 --- a/docs/source-app/examples/hpo/objective.py +++ b/docs/source-app/tutorials/hpo/objective.py @@ -1,11 +1,12 @@ import numpy as np -import lightning as L +from lightning_app import LightningWork -class ObjectiveWork(L.LightningWork): +class ObjectiveWork(LightningWork): def __init__(self): super().__init__(parallel=True) + self.has_started = False self.metric = None self.trial_id = None self.params = None @@ -13,9 +14,8 @@ class ObjectiveWork(L.LightningWork): def run(self, trial_id, **params): self.trial_id = trial_id - # Received suggested `backbone` and `learning_rate` - self.params = params - # Emulate metric computation would be - # computed once a script has been completed. + self.params = params # Received suggested `backbone` and `learning_rate` + self.has_started = True + # Emulate metric computation would be computed once a script has been completed. # In reality, this would excute a user defined script. self.metric = np.random.uniform(0, 100) diff --git a/docs/source-app/workflows/access_app_state.rst b/docs/source-app/workflows/access_app_state.rst new file mode 100644 index 0000000000..881d2c6840 --- /dev/null +++ b/docs/source-app/workflows/access_app_state.rst @@ -0,0 +1,61 @@ +.. _access_app_state: + +################ +Access App State +################ + +**Audience:** Users who want to know how the app state can be accessed. + +**Level:** Basic + +*********************** +What is the App State ? +*********************** + +In Lightning, each component is stateful and their state are composed of all attributes defined within their **__init__** method. + +The **App State** is the collection of all the components state forming the application. + +***************************************** +What is the special about the App State ? +***************************************** + +The **App State** is always up-to-date, even running an application in the cloud on multiple machines. + +This means that every time an attribute is modified in a work, that information is automatically broadcasted to the flow. + +With this mechanism, any component can **react** to any other component **state changes** through the flow and complex system can be easily implemented. + +Lightning requires a state based driven mindset when implementing the flow. + +************************************ +When do I need to access the state ? +************************************ + +As a user, you are interacting with your component attributes, so most likely, +you won't need to access the component state directly, but it can be helpful to +understand how the state works under the hood. + +For example, here we define a **Flow** component and **Work** component, where the work increments a counter indefinitely and the flow prints its state which contains the work. + +You can easily check the state of your entire app as follows: + +.. literalinclude:: ../code_samples/quickstart/app_01.py + +Run the app with: + +.. code-block:: bash + + lightning run app docs/quickstart/app_01.py + +And here's the output you get when running the above application using **Lightning CLI**: + +.. code-block:: console + + INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view + State: {'works': {'w': {'vars': {'counter': 1}}}} + State: {'works': {'w': {'vars': {'counter': 2}}}} + State: {'works': {'w': {'vars': {'counter': 3}}}} + State: {'works': {'w': {'vars': {'counter': 3}}}} + State: {'works': {'w': {'vars': {'counter': 4}}}} + ... diff --git a/docs/source-app/workflows/add_components/index.rst b/docs/source-app/workflows/add_components/index.rst index ca95e44e18..c0656a81a6 100644 --- a/docs/source-app/workflows/add_components/index.rst +++ b/docs/source-app/workflows/add_components/index.rst @@ -1,5 +1,3 @@ -:orphan: - ########################### Add a component to your app ########################### @@ -13,7 +11,7 @@ Install a component Any Lightning component can be installed with: -.. code:: python +.. code:: bash lightning install component org/the-component-name diff --git a/docs/source-app/workflows/add_server/any_server.rst b/docs/source-app/workflows/add_server/any_server.rst index 398951276c..bdf09c67ee 100644 --- a/docs/source-app/workflows/add_server/any_server.rst +++ b/docs/source-app/workflows/add_server/any_server.rst @@ -21,7 +21,6 @@ Add a server to a component Any server that listens on a port, can be enabled via a work. For example, here's a plain python server: .. code:: python - :emphasize-lines: 11-12 import socketserver from http import HTTPStatus, server @@ -31,8 +30,7 @@ Any server that listens on a port, can be enabled via a work. For example, here' def do_GET(self): self.send_response(HTTPStatus.OK) self.end_headers() - html = "

Hello lit world

" - self.wfile.write(html) + self.wfile.write(b"

Hello lit world

") httpd = socketserver.TCPServer(("localhost", "3000"), PlainServer) @@ -41,9 +39,9 @@ Any server that listens on a port, can be enabled via a work. For example, here' To enable the server inside the component, start the server in the run method and use the ``self.host`` and ``self.port`` properties: .. code:: python - :emphasize-lines: 14-15 + :emphasize-lines: 13, 14 - import lightning as L + import lightning_app as la import socketserver from http import HTTPStatus, server @@ -52,11 +50,10 @@ To enable the server inside the component, start the server in the run method an def do_GET(self): self.send_response(HTTPStatus.OK) self.end_headers() - html = "

Hello lit world

" - self.wfile.write(html) + self.wfile.write(b"

Hello lit world

") - class LitServer(L.LightningWork): + class LitServer(lapp.LightningWork): def run(self): httpd = socketserver.TCPServer((self.host, self.port), PlainServer) httpd.serve_forever() @@ -70,9 +67,9 @@ The final step, is to tell the Root component in which tab to render this compon In this case, we render the ``LitServer`` output in the ``home`` tab of the application. .. code:: python - :emphasize-lines: 20, 23, 28 + :emphasize-lines: 19, 25 - import lightning as L + import lightning_app as la import socketserver from http import HTTPStatus, server @@ -81,17 +78,16 @@ In this case, we render the ``LitServer`` output in the ``home`` tab of the appl def do_GET(self): self.send_response(HTTPStatus.OK) self.end_headers() - html = "

Hello lit world

" - self.wfile.write(html) + self.wfile.write(b"

Hello lit world

") - class LitServer(L.LightningWork): + class LitServer(lapp.LightningWork): def run(self): httpd = socketserver.TCPServer((self.host, self.port), PlainServer) httpd.serve_forever() - class Root(L.LightningFlow): + class Root(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_server = LitServer(parallel=True) @@ -104,7 +100,7 @@ In this case, we render the ``LitServer`` output in the ``home`` tab of the appl return tab1 - app = L.LightningApp(Root()) + app = lapp.LightningApp(Root()) We use the ``parallel=True`` argument of ``LightningWork`` to run the server in parallel while the rest of the Lightning App runs everything else. diff --git a/docs/source-app/workflows/add_server/flask_basic.rst b/docs/source-app/workflows/add_server/flask_basic.rst index 38ca282346..dff64e12ca 100644 --- a/docs/source-app/workflows/add_server/flask_basic.rst +++ b/docs/source-app/workflows/add_server/flask_basic.rst @@ -20,7 +20,6 @@ Add Flask to a component First, define your flask app as you normally would without Lightning: .. code:: python - :emphasize-lines: 9 from flask import Flask @@ -37,13 +36,13 @@ First, define your flask app as you normally would without Lightning: To enable the server inside the component, start the Flask server in the run method and use the ``self.host`` and ``self.port`` properties: .. code:: python - :emphasize-lines: 12 + :emphasize-lines: 11 - import lightning as L + import lightning_app as la from flask import Flask - class LitFlask(L.LightningWork): + class LitFlask(lapp.LightningWork): def run(self): flask_app = Flask(__name__) @@ -62,13 +61,13 @@ The final step, is to tell the Root component in which tab to render this compon In this case, we render the ``LitFlask`` output in the ``home`` tab of the application. .. code:: python - :emphasize-lines: 17, 23 + :emphasize-lines: 16, 22 - import lightning as L + import lightning_app as la from flask import Flask - class LitFlask(L.LightningWork): + class LitFlask(lapp.LightningWork): def run(self): flask_app = Flask(__name__) @@ -79,7 +78,7 @@ In this case, we render the ``LitFlask`` output in the ``home`` tab of the appli flask_app.run(host=self.host, port=self.port) - class Root(L.LightningFlow): + class Root(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_flask = LitFlask(parallel=True) @@ -92,7 +91,7 @@ In this case, we render the ``LitFlask`` output in the ``home`` tab of the appli return tab1 - app = L.LightningApp(Root()) + app = lapp.LightningApp(Root()) We use the ``parallel=True`` argument of ``LightningWork`` to run the server in the background while the rest of the Lightning App runs everything else. @@ -104,16 +103,31 @@ Run the app *********** Start the app to see your new UI! -.. code:: bash +.. code:: console lightning run app app.py To run the app on the cloud, use the ``--cloud`` argument. -.. code:: bash +.. code:: console lightning run app app.py --cloud +.. code:: python + + from flask import Flask + + + class LitFlask(lapp.LightningWork): + def run(self): + flask_app = Flask(__name__) + + @flask_app.route("/") + def hello(): + return "Hello, World!" + + flask_app.run(host=self.host, port=self.port) + ---- ******** diff --git a/docs/source-app/workflows/add_web_link.rst b/docs/source-app/workflows/add_web_link.rst index 01ffdf62ef..4236359ef4 100644 --- a/docs/source-app/workflows/add_web_link.rst +++ b/docs/source-app/workflows/add_web_link.rst @@ -18,23 +18,19 @@ and connect the UIs. Create a file named **app.py** with this code: the app running here .. code:: python - :emphasize-lines: 7,11 + :emphasize-lines: 5, 6, 7 - import lightning as L + import lightning_app as la - class LitApp(L.LightningFlow): + + class LitApp(lapp.LightningFlow): def configure_layout(self): - tab_1 = { - "name": "Logger", - "content": "https://bit.ly/tb-aasae" - } - tab_2 = { - "name": "Paper", - "content": "https://arxiv.org/pdf/2107.12329.pdf" - } + tab_1 = {"name": "TB logs", "content": "https://bit.ly/tb-aasae"} + tab_2 = {"name": "Paper", "content": "https://arxiv.org/pdf/2107.12329.pdf"} return tab_1, tab_2 - app = L.LightningApp(LitApp()) + + app = lapp.LightningApp(LitApp()) ---- @@ -43,12 +39,12 @@ Run the app *********** Run the app locally to see it! -.. code:: python +.. code:: bash lightning run app app.py Now run it on the cloud as well: -.. code:: python +.. code:: bash lightning run app app.py --cloud diff --git a/docs/source-app/workflows/add_web_ui/angular_js_intermediate.rst b/docs/source-app/workflows/add_web_ui/angular_js_intermediate.rst index 095dee362b..05b1731e4f 100644 --- a/docs/source-app/workflows/add_web_ui/angular_js_intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/angular_js_intermediate.rst @@ -1,5 +1,3 @@ -:orphan: - ########################################### Add a web UI with Angular.js (intermediate) ########################################### diff --git a/docs/source-app/workflows/add_web_ui/dash/basic.rst b/docs/source-app/workflows/add_web_ui/dash/basic.rst index 4316fc13a1..505a6535c1 100644 --- a/docs/source-app/workflows/add_web_ui/dash/basic.rst +++ b/docs/source-app/workflows/add_web_ui/dash/basic.rst @@ -39,26 +39,26 @@ First **create a file named app.py** with the app content: .. code:: bash - import lightning as L + import lightning_app as la import dash import plotly.express as px - class LitDash(L.LightningWork): + class LitDash(lapp.LightningWork): def run(self): - dash_app = dash.Dash(__name__) + app = dash.Dash(__name__) X = [1, 2, 3, 4, 5, 6] Y = [2, 4, 8, 16, 32, 64] fig = px.line(x=X, y=Y) - dash_app.layout = dash.html.Div(children=[ + app.layout = dash.html.Div(children=[ dash.html.H1(children='⚡ Hello Dash + Lightning⚡'), - dash.html.Div(children='The Dash framework running inside a ⚡ Lightning App'), + dash.html.Div(children='''The Dash framework running inside a ⚡ Lightning App'''), dash.dcc.Graph(id='example-graph', figure=fig) ]) - dash_app.run_server(host=self.host, port=self.port) + app.run_server(host=self.host, port=self.port) - class LitApp(L.LightningFlow): + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_dash = LitDash(parallel=True) @@ -70,7 +70,7 @@ First **create a file named app.py** with the app content: tab1 = {"name": "home", "content": self.lit_dash} return tab1 - app = L.LightningApp(LitApp()) + app = lapp.LightningApp(LitApp()) add 'dash' to a requirements.txt file: @@ -88,13 +88,13 @@ Run the app *********** Run the app locally to see it! -.. code:: python +.. code:: bash lightning run app app.py Now run it on the cloud as well: -.. code:: python +.. code:: console lightning run app app.py --cloud @@ -116,18 +116,20 @@ First, find the dash app you want to integrate. In this example, that app looks import dash import plotly.express as px - dash_app = dash.Dash(__name__) + app = dash.Dash(__name__) X = [1, 2, 3, 4, 5, 6] Y = [2, 4, 8, 16, 32, 64] fig = px.line(x=X, y=Y) - dash_app.layout = dash.html.Div(children=[ - dash.html.H1(children='⚡ Hello Dash + Lightning⚡'), - dash.html.Div(children='The Dash framework running inside a ⚡ Lightning App'), - dash.dcc.Graph(id='example-graph', figure=fig) - ]) + app.layout = dash.html.Div( + children=[ + dash.html.H1(children="⚡ Hello Dash + Lightning⚡"), + dash.html.Div(children="""The Dash framework running inside a ⚡ Lightning App"""), + dash.dcc.Graph(id="example-graph", figure=fig), + ] + ) - dash_app.run_server(host='0.0.0.0', port=80) + app.run_server(host="0.0.0.0", port=80) This dash app plots a simple line curve along with some HTMlapp. `Visit the Dash documentation for the full API `_. @@ -141,26 +143,30 @@ Add the dash app to the run method of a ``LightningWork`` component and run the .. code:: python :emphasize-lines: 6, 18 - import lightning as L + import lightning_app as la import dash import plotly.express as px - class LitDash(L.LightningWork): + + class LitDash(lapp.LightningWork): def run(self): - dash_app = dash.Dash(__name__) + app = dash.Dash(__name__) X = [1, 2, 3, 4, 5, 6] Y = [2, 4, 8, 16, 32, 64] fig = px.line(x=X, y=Y) - dash_app.layout = dash.html.Div(children=[ - dash.html.H1(children='⚡ Hello Dash + Lightning⚡'), - dash.html.Div(children='The Dash framework running inside a ⚡ Lightning App'), - dash.dcc.Graph(id='example-graph', figure=fig) - ]) + app.layout = dash.html.Div( + children=[ + dash.html.H1(children="⚡ Hello Dash + Lightning⚡"), + dash.html.Div(children="""The Dash framework running inside a ⚡ Lightning App"""), + dash.dcc.Graph(id="example-graph", figure=fig), + ] + ) - dash_app.run_server(host=self.host, port=self.port) + app.run_server(host=self.host, port=self.port) - class LitApp(L.LightningFlow): + + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_dash = LitDash(parallel=True) @@ -172,7 +178,8 @@ Add the dash app to the run method of a ``LightningWork`` component and run the tab1 = {"name": "home", "content": self.lit_dash} return tab1 - app = L.LightningApp(LitApp()) + + app = lapp.LightningApp(LitApp()) ---- @@ -184,26 +191,30 @@ In this case, we render the ``LitDash`` UI in the ``home`` tab of the applicatio .. code:: python :emphasize-lines: 23, 29 - import lightning as L + import lightning_app as la import dash import plotly.express as px - class LitDash(L.LightningWork): + + class LitDash(lapp.LightningWork): def run(self): - dash_app = dash.Dash(__name__) + app = dash.Dash(__name__) X = [1, 2, 3, 4, 5, 6] Y = [2, 4, 8, 16, 32, 64] fig = px.line(x=X, y=Y) - dash_app.layout = dash.html.Div(children=[ - dash.html.H1(children='⚡ Hello Dash + Lightning⚡'), - dash.html.Div(children='The Dash framework running inside a ⚡ Lightning App'), - dash.dcc.Graph(id='example-graph', figure=fig) - ]) + app.layout = dash.html.Div( + children=[ + dash.html.H1(children="⚡ Hello Dash + Lightning⚡"), + dash.html.Div(children="""The Dash framework running inside a ⚡ Lightning App"""), + dash.dcc.Graph(id="example-graph", figure=fig), + ] + ) - dash_app.run_server(host=self.host, port=self.port) + app.run_server(host=self.host, port=self.port) - class LitApp(L.LightningFlow): + + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_dash = LitDash(parallel=True) @@ -215,7 +226,8 @@ In this case, we render the ``LitDash`` UI in the ``home`` tab of the applicatio tab1 = {"name": "home", "content": self.lit_dash} return tab1 - app = L.LightningApp(LitApp()) + + app = lapp.LightningApp(LitApp()) We use the ``parallel=True`` argument of ``LightningWork`` to run the server in the background while the rest of the Lightning App runs everything else. diff --git a/docs/source-app/workflows/add_web_ui/dash/index.rst b/docs/source-app/workflows/add_web_ui/dash/index.rst index 5abb444c8e..1364172543 100644 --- a/docs/source-app/workflows/add_web_ui/dash/index.rst +++ b/docs/source-app/workflows/add_web_ui/dash/index.rst @@ -1,5 +1,3 @@ -:orphan: - .. toctree:: :maxdepth: 1 :hidden: diff --git a/docs/source-app/workflows/add_web_ui/dash/intermediate.rst b/docs/source-app/workflows/add_web_ui/dash/intermediate.rst index fac428238a..1b1abd4625 100644 --- a/docs/source-app/workflows/add_web_ui/dash/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/dash/intermediate.rst @@ -10,33 +10,11 @@ Add a web UI with Dash (intermediate) ******************************* Interact with the App from Dash ******************************* - -In the example below, every time you change the select year on the dashboard, this is directly communicated to the flow -and another work process the associated data frame with the provided year. - -.. literalinclude:: intermediate_plot.py - -Here is how the app looks like once running: - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/dash_plot.gif +TODO: is this a thing? ---- *********************************** Interact with Dash from a component *********************************** - -In the example below, when you click the toggle, the state of the work appears. - -Install the following libraries if you want to run the app. - -```bash -pip install dash_daq dash_renderjson -``` - -.. literalinclude:: intermediate_state.py - - -Here is how the app looks like once running: - -.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/dash_state.gif +TODO: is this a thing? diff --git a/docs/source-app/workflows/add_web_ui/dash/intermediate_plot.py b/docs/source-app/workflows/add_web_ui/dash/intermediate_plot.py deleted file mode 100644 index 52860fe90d..0000000000 --- a/docs/source-app/workflows/add_web_ui/dash/intermediate_plot.py +++ /dev/null @@ -1,88 +0,0 @@ -from typing import Optional - -import pandas as pd -import plotly.express as px -from dash import Dash, dcc, html, Input, Output - -import lightning as L -from lightning.app.storage.payload import Payload -from lightning_app.core.work import LightningWork - - -class LitDash(L.LightningWork): - def __init__(self): - super().__init__(parallel=True) - self.df = None - self.selected_year = None - - def run(self): - - df = pd.read_csv("https://raw.githubusercontent.com/plotly/datasets/master/gapminderDataFiveYear.csv") - self.df = Payload(df) - - dash_app = Dash(__name__) - - dash_app.layout = html.Div( - [ - dcc.Graph(id="graph-with-slider"), - dcc.Slider( - df["year"].min(), - df["year"].max(), - step=None, - value=df["year"].min(), - marks={str(year): str(year) for year in df["year"].unique()}, - id="year-slider", - ), - ] - ) - - @dash_app.callback(Output("graph-with-slider", "figure"), Input("year-slider", "value")) - def update_figure(selected_year): - self.selected_year = selected_year - filtered_df = df[df.year == selected_year] - - fig = px.scatter( - filtered_df, - x="gdpPercap", - y="lifeExp", - size="pop", - color="continent", - hover_name="country", - log_x=True, - size_max=55, - ) - - fig.update_layout(transition_duration=500) - - return fig - - dash_app.run_server(host=self.host, port=self.port) - - -class Processor(LightningWork): - def run(self, df: Payload, selected_year: Optional[str]): - if selected_year: - df = df.value - filtered_df = df[df.year == selected_year] - print(f"[PROCESSOR|selected_year={selected_year}]") - print(filtered_df) - - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_dash = LitDash() - self.processor = Processor(parallel=True) - - def run(self): - self.lit_dash.run() - - # Launch some processing based on the Dash Dashboard. - self.processor.run(self.lit_dash.df, self.lit_dash.selected_year) - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_dash} - return tab1 - - -app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/dash/intermediate_state.py b/docs/source-app/workflows/add_web_ui/dash/intermediate_state.py deleted file mode 100644 index d2e37a5a37..0000000000 --- a/docs/source-app/workflows/add_web_ui/dash/intermediate_state.py +++ /dev/null @@ -1,39 +0,0 @@ -import dash -import dash_daq as daq -import dash_renderjson -from dash import html, Input, Output - -import lightning as L -from lightning.app.utilities.state import AppState - - -class LitDash(L.LightningWork): - def run(self): - dash_app = dash.Dash(__name__) - - dash_app.layout = html.Div([daq.ToggleSwitch(id="my-toggle-switch", value=False), html.Div(id="output")]) - - @dash_app.callback(Output("output", "children"), [Input("my-toggle-switch", "value")]) - def display_output(value): - if value: - state = AppState() - state._request_state() - return dash_renderjson.DashRenderjson(id="input", data=state._state, max_depth=-1, invert_theme=True) - - dash_app.run_server(host=self.host, port=self.port) - - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_dash = LitDash(parallel=True) - - def run(self): - self.lit_dash.run() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_dash} - return tab1 - - -app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/gradio/basic.rst b/docs/source-app/workflows/add_web_ui/gradio/basic.rst index fbfdd55553..f278ccf22e 100644 --- a/docs/source-app/workflows/add_web_ui/gradio/basic.rst +++ b/docs/source-app/workflows/add_web_ui/gradio/basic.rst @@ -39,14 +39,15 @@ First **create a file named app.py** with the app content: .. code:: python - import lightning as L + import lightning_app as la from lightning_app.components.serve import ServeGradio import gradio as gr + class LitGradio(ServeGradio): - inputs = gr.inputs.Textbox(default='lightning', label='name input') - outputs = gr.outputs.Textbox(label='output') + inputs = gr.inputs.Textbox(default="lightning", label="name input") + outputs = gr.outputs.Textbox(label="output") examples = [["hello lightning"]] def predict(self, input_text): @@ -56,7 +57,8 @@ First **create a file named app.py** with the app content: fake_model = lambda x: f"hello {x}" return fake_model - class RootFlow(L.LightningFlow): + + class RootFlow(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_gradio = LitGradio() @@ -67,7 +69,8 @@ First **create a file named app.py** with the app content: def configure_layout(self): return [{"name": "home", "content": self.lit_gradio}] - app = L.LightningApp(RootFlow()) + + app = lapp.LightningApp(RootFlow()) add "gradio" to a requirements.txt file: @@ -84,13 +87,13 @@ Run the app *********** Run the app locally to see it! -.. code:: python +.. code:: bash lightning run app app.py Now run it on the cloud as well: -.. code:: python +.. code:: bash lightning run app app.py --cloud @@ -123,10 +126,11 @@ Here's an example: from lightning_app.components.serve import ServeGradio import gradio as gr + class LitGradio(ServeGradio): - inputs = gr.inputs.Textbox(default='lightning', label='name input') - outputs = gr.outputs.Textbox(label='output') + inputs = gr.inputs.Textbox(default="lightning", label="name input") + outputs = gr.outputs.Textbox(label="output") def predict(self, input_text): return self.model(input_text) @@ -147,14 +151,15 @@ In this case, we render the ``LitGradio`` UI in the ``home`` tab of the applicat .. code:: python :emphasize-lines: 21, 27 - import lightning as L + import lightning_app as la from lightning_app.components.serve import ServeGradio import gradio as gr + class LitGradio(ServeGradio): - inputs = gr.inputs.Textbox(default='lightning', label='name input') - outputs = gr.outputs.Textbox(label='output') + inputs = gr.inputs.Textbox(default="lightning", label="name input") + outputs = gr.outputs.Textbox(label="output") examples = [["hello lightning"]] def predict(self, input_text): @@ -164,7 +169,8 @@ In this case, we render the ``LitGradio`` UI in the ``home`` tab of the applicat fake_model = lambda x: f"hello {x}" return fake_model - class RootFlow(L.LightningFlow): + + class RootFlow(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_gradio = LitGradio() @@ -175,7 +181,8 @@ In this case, we render the ``LitGradio`` UI in the ``home`` tab of the applicat def configure_layout(self): return [{"name": "home", "content": self.lit_gradio}] - app = L.LightningApp(RootFlow()) + + app = lapp.LightningApp(RootFlow()) ---- @@ -186,14 +193,15 @@ Finally, don't forget to call run inside the Root Flow to serve the Gradio app. .. code:: python :emphasize-lines: 24 - import lightning as L + import lightning_app as la from lightning_app.components.serve import ServeGradio import gradio as gr + class LitGradio(ServeGradio): - inputs = gr.inputs.Textbox(default='lightning', label='name input') - outputs = gr.outputs.Textbox(label='output') + inputs = gr.inputs.Textbox(default="lightning", label="name input") + outputs = gr.outputs.Textbox(label="output") examples = [["hello lightning"]] def predict(self, input_text): @@ -203,7 +211,8 @@ Finally, don't forget to call run inside the Root Flow to serve the Gradio app. fake_model = lambda x: f"hello {x}" return fake_model - class RootFlow(L.LightningFlow): + + class RootFlow(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_gradio = LitGradio() @@ -214,4 +223,5 @@ Finally, don't forget to call run inside the Root Flow to serve the Gradio app. def configure_layout(self): return [{"name": "home", "content": self.lit_gradio}] - app = L.LightningApp(RootFlow()) + + app = lapp.LightningApp(RootFlow()) diff --git a/docs/source-app/workflows/add_web_ui/gradio/index.rst b/docs/source-app/workflows/add_web_ui/gradio/index.rst index 740ae93aae..a4d8d08eb9 100644 --- a/docs/source-app/workflows/add_web_ui/gradio/index.rst +++ b/docs/source-app/workflows/add_web_ui/gradio/index.rst @@ -1,5 +1,3 @@ -:orphan: - .. toctree:: :maxdepth: 1 :hidden: diff --git a/docs/source-app/workflows/add_web_ui/gradio/intermediate.rst b/docs/source-app/workflows/add_web_ui/gradio/intermediate.rst index bb20d56624..acbfa8e4a6 100644 --- a/docs/source-app/workflows/add_web_ui/gradio/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/gradio/intermediate.rst @@ -4,18 +4,19 @@ Add a web UI with Gradio (intermediate) .. note:: documentation coming soon. +.. + ************************************* + Interact with a component from the UI + ************************************* + .. warning:: is there such a thing for this with gradio? -************************************* -Interact with a component from the UI -************************************* +.. + ---- -.. warning:: is there such a thing for this with gradio? +.. + ************************************* + Interact with the UI from a component + ************************************* + .. warning:: is there such a thing for this with gradio? - ----- - -************************************* -Interact with the UI from a component -************************************* - -.. warning:: is there such a thing for this with gradio? + ---- diff --git a/docs/source-app/workflows/add_web_ui/html/basic.rst b/docs/source-app/workflows/add_web_ui/html/basic.rst index 83b6fb7d13..685a495aff 100644 --- a/docs/source-app/workflows/add_web_ui/html/basic.rst +++ b/docs/source-app/workflows/add_web_ui/html/basic.rst @@ -36,7 +36,7 @@ The first step is to create an HTML file named **index.html**: ---- ************************ -Create the HTML demo app +Create the html demo app ************************ .. @@ -52,13 +52,13 @@ First **create a file named app.py** with the app content (in the same folder as .. code:: bash # app.py - import lightning as L + import lightning_app as la - class HelloComponent(L.LightningFlow): + class HelloComponent(lapp.LightningFlow): def configure_layout(self): - return L.frontend.web.StaticWebFrontend(serve_dir='.') + return lapp.frontend.web.StaticWebFrontend(serve_dir='.') - class LitApp(L.LightningFlow): + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.hello_component = HelloComponent() @@ -67,7 +67,7 @@ First **create a file named app.py** with the app content (in the same folder as tab1 = {"name": "home", "content": self.hello_component} return tab1 - app = L.LightningApp(LitApp()) + app = lapp.LightningApp(LitApp()) ---- @@ -76,13 +76,13 @@ Run the app *********** Run the app locally to see it! -.. code:: python +.. code:: bash lightning run app app.py Now run it on the cloud as well: -.. code:: python +.. code:: bash lightning run app app.py --cloud @@ -103,13 +103,13 @@ Give the component an HTML UI, by returning a ``StaticWebFrontend`` object from :emphasize-lines: 5,6 # app.py - import lightning as L + import lightning_app as la - class HelloComponent(L.LightningFlow): + class HelloComponent(lapp.LightningFlow): def configure_layout(self): - return L.frontend.web.StaticWebFrontend(serve_dir='.') + return lapp.frontend.web.StaticWebFrontend(serve_dir='.') - class LitApp(L.LightningFlow): + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.hello_component = HelloComponent() @@ -118,7 +118,7 @@ Give the component an HTML UI, by returning a ``StaticWebFrontend`` object from tab1 = {"name": "home", "content": self.hello_component} return tab1 - app = L.LightningApp(LitApp()) + app = lapp.LightningApp(LitApp()) The folder path given in ``StaticWebFrontend(serve_dir=)`` must point to a folder with an ``index.html`` page. @@ -133,13 +133,13 @@ In this case, we render the ``HelloComponent`` UI in the ``home`` tab of the app :emphasize-lines: 14, 15 # app.py - import lightning as L + import lightning_app as la - class HelloComponent(L.LightningFlow): + class HelloComponent(lapp.LightningFlow): def configure_layout(self): - return L.frontend.web.StaticWebFrontend(serve_dir='.') + return lapp.frontend.web.StaticWebFrontend(serve_dir='.') - class LitApp(L.LightningFlow): + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.hello_component = HelloComponent() @@ -148,4 +148,4 @@ In this case, we render the ``HelloComponent`` UI in the ``home`` tab of the app tab1 = {"name": "home", "content": self.hello_component} return tab1 - app = L.LightningApp(LitApp()) + app = lapp.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/html/index.rst b/docs/source-app/workflows/add_web_ui/html/index.rst index 0eae930987..0e4830d3a9 100644 --- a/docs/source-app/workflows/add_web_ui/html/index.rst +++ b/docs/source-app/workflows/add_web_ui/html/index.rst @@ -1,5 +1,3 @@ -:orphan: - .. toctree:: :maxdepth: 1 :hidden: diff --git a/docs/source-app/workflows/add_web_ui/index.rst b/docs/source-app/workflows/add_web_ui/index.rst index 79c0f16d66..54180d4df5 100644 --- a/docs/source-app/workflows/add_web_ui/index.rst +++ b/docs/source-app/workflows/add_web_ui/index.rst @@ -1,9 +1,7 @@ - ############################# Add a web user interface (UI) ############################# - -**Audience:** Users who want to add a UI to their Lightning Apps +Every component in a Lightning App can have its own web user interface (UI). ---- diff --git a/docs/source-app/workflows/add_web_ui/index_content.rst b/docs/source-app/workflows/add_web_ui/index_content.rst index 9602537a53..6995d0e9b2 100644 --- a/docs/source-app/workflows/add_web_ui/index_content.rst +++ b/docs/source-app/workflows/add_web_ui/index_content.rst @@ -1,3 +1,21 @@ +.. toctree:: + :maxdepth: 1 + :hidden: + + dash/index + gradio/index + streamlit/index + +.. toctree:: + :maxdepth: 1 + :hidden: + + integrate_any_javascript_framework + angular_js_intermediate + html/index + react/index + vue_js_intermediate + ************************************* Web UIs for non Javascript Developers ************************************* diff --git a/docs/source-app/workflows/add_web_ui/integrate_any_javascript_framework.rst b/docs/source-app/workflows/add_web_ui/integrate_any_javascript_framework.rst index f1da660b09..7856d3713a 100644 --- a/docs/source-app/workflows/add_web_ui/integrate_any_javascript_framework.rst +++ b/docs/source-app/workflows/add_web_ui/integrate_any_javascript_framework.rst @@ -1,5 +1,3 @@ -:orphan: - ################################## Integrate any javascript framework ################################## diff --git a/docs/source-app/workflows/add_web_ui/jupyter_basic.rst b/docs/source-app/workflows/add_web_ui/jupyter_basic.rst index 61f58ab406..ce1916e527 100644 --- a/docs/source-app/workflows/add_web_ui/jupyter_basic.rst +++ b/docs/source-app/workflows/add_web_ui/jupyter_basic.rst @@ -17,8 +17,6 @@ What is a Jupyter Notebook? TODO ----- - ******************* Install Jupyter Lab ******************* diff --git a/docs/source-app/workflows/add_web_ui/react/communicate_between_react_and_lightning.rst b/docs/source-app/workflows/add_web_ui/react/communicate_between_react_and_lightning.rst index 524bdd5753..b3e1c7a9f6 100644 --- a/docs/source-app/workflows/add_web_ui/react/communicate_between_react_and_lightning.rst +++ b/docs/source-app/workflows/add_web_ui/react/communicate_between_react_and_lightning.rst @@ -15,11 +15,11 @@ Example code To illustrate how to communicate between a React app and a lightning App, we'll be using the `example_app.py` file which `lightning init react-ui `_ created: -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/example_app.py +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/example_app.py and the App.tsx file also created by `lightning init react-ui `_: -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/ui/src/App.tsx +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/ui/src/App.tsx ---- @@ -31,13 +31,13 @@ To change the Lightning app from the React app, use `updateLightningState`. In this example, when you press **Start printing** in the React UI, it toggles the `react_ui.vars.should_print`: -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/ui/src/App.tsx +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/ui/src/App.tsx :emphasize-lines: 20, 21, 23 By changing that variable in the Lightning app state, it sets **react_ui.should_print** to True, which enables the Lightning app to print: -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/example_app.py +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/example_app.py :emphasize-lines: 10, 22 ---- @@ -49,10 +49,10 @@ To change the React app from the Lightning app, use the values from the `lightni In this example, when the `react_ui.counter`` increaes in the Lightning app: -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/example_app.py +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/example_app.py :emphasize-lines: 18, 24 The React UI updates the text on the screen to reflect the count -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/ui/src/App.tsx +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/ui/src/App.tsx :emphasize-lines: 15 diff --git a/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst b/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst index c1c0c5e201..7b4b67583e 100644 --- a/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst +++ b/docs/source-app/workflows/add_web_ui/react/connect_react_and_lightning.rst @@ -15,11 +15,11 @@ Example code To illustrate how to connect a React app and a lightning App, we'll be using the `example_app.py` file which `lightning init react-ui `_ created: -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/example_app.py +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/example_app.py and the App.tsx file also created by `lightning init react-ui `_: -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/ui/src/App.tsx +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/ui/src/App.tsx ---- @@ -28,7 +28,7 @@ Connect the component to the react UI ************************************* The first step is to connect the dist folder of the react app using `StaticWebFrontend`: -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/example_app.py +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/example_app.py :emphasize-lines: 13 the dist folder must contain an index.html file which is generated by the compilating command `yarn build` which @@ -42,7 +42,7 @@ Connect component to the root flow Next, connect your component to the root flow. Display the react app on the tab of your choice using `configure_layout`: -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/example_app.py +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/example_app.py :emphasize-lines: 19, 27 ---- @@ -59,7 +59,7 @@ At this point, the React app will render in the Lightning app. Test it out! However, to make powerful React+Lightning apps, you must also connect the Lightning App state to the react app. These lines enable two-way communication between the react app and the Lightning app. -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/ui/src/App.tsx +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/ui/src/App.tsx :emphasize-lines: 10, 13 ---- @@ -69,7 +69,7 @@ Component vs App **************** Notice that in this guide, we connected a single react app to a single component. -.. literalinclude:: ../../../../../src/lightning_app/cli/react-ui-template/example_app.py +.. literalinclude:: ../../../../../lightning_app/cli/react-ui-template/example_app.py :emphasize-lines: 6-13 You can use this single react app for the FULL Lightning app, or you can specify a React app for EACH component. @@ -77,20 +77,20 @@ You can use this single react app for the FULL Lightning app, or you can specify .. code:: python :emphasize-lines: 5, 9, 18-20 - import lightning as L + import lightning_app as la - class ComponentA(L.LightningFlow): + class ComponentA(lapp.LightningFlow): def configure_layout(self): - return L.frontend.StaticWebFrontend(Path(__file__).parent / "react_app_1/dist") + return lapp.frontend.StaticWebFrontend(Path(__file__).parent / "react_app_1/dist") - class ComponentB(L.LightningFlow): + class ComponentB(lapp.LightningFlow): def configure_layout(self): - return L.frontend.StaticWebFrontend(Path(__file__).parent / "react_app_2/dist") + return lapp.frontend.StaticWebFrontend(Path(__file__).parent / "react_app_2/dist") - class HelloLitReact(L.LightningFlow): + class HelloLitReact(lapp.LightningFlow): def __init__(self): super().__init__() self.react_app_1 = ComponentA() @@ -102,6 +102,6 @@ You can use this single react app for the FULL Lightning app, or you can specify return tab_1, tab_2 - app = L.LightningApp(HelloLitReact()) + app = lapp.LightningApp(HelloLitReact()) This is a powerful idea that allows each Lightning component to have a self-contained web UI. diff --git a/docs/source-app/workflows/add_web_ui/react/index.rst b/docs/source-app/workflows/add_web_ui/react/index.rst index ba0f8d97d6..1a0463e105 100644 --- a/docs/source-app/workflows/add_web_ui/react/index.rst +++ b/docs/source-app/workflows/add_web_ui/react/index.rst @@ -1,5 +1,3 @@ -:orphan: - .. toctree:: :maxdepth: 1 :hidden: diff --git a/docs/source-app/workflows/add_web_ui/streamlit/basic.rst b/docs/source-app/workflows/add_web_ui/streamlit/basic.rst index 464b558032..920bd8d84a 100644 --- a/docs/source-app/workflows/add_web_ui/streamlit/basic.rst +++ b/docs/source-app/workflows/add_web_ui/streamlit/basic.rst @@ -38,17 +38,20 @@ First **create a file named app.py** with the app content: .. code:: python # app.py - import lightning as L + import lightning_app as la import streamlit as st + def your_streamlit_app(lightning_app_state): - st.write('hello world') + st.write("hello world") - class LitStreamlit(L.LightningFlow): + + class LitStreamlit(lapp.LightningFlow): def configure_layout(self): - return L.frontend.StreamlitFrontend(render_fn=your_streamlit_app) + return lapp.frontend.StreamlitFrontend(render_fn=your_streamlit_app) - class LitApp(L.LightningFlow): + + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_streamlit = LitStreamlit() @@ -57,7 +60,8 @@ First **create a file named app.py** with the app content: tab1 = {"name": "home", "content": self.lit_streamlit} return tab1 - app = L.LightningApp(LitApp()) + + app = lapp.LightningApp(LitApp()) add "streamlit" to a requirements.txt file: @@ -74,13 +78,13 @@ Run the app *********** Run the app locally to see it! -.. code:: python +.. code:: bash lightning run app app.py Now run it on the cloud as well: -.. code:: python +.. code:: bash lightning run app app.py --cloud @@ -101,8 +105,9 @@ First, find the streamlit app you want to integrate. In this example, that app l import streamlit as st + def your_streamlit_app(): - st.write('hello world') + st.write("hello world") Refer to the `Streamlit documentation `_ for more complex examples. @@ -117,17 +122,20 @@ the ``configure_layout`` method of the Lightning component you want to connect t :emphasize-lines: 8-10 # app.py - import lightning as L + import lightning_app as la import streamlit as st + def your_streamlit_app(lightning_app_state): - st.write('hello world') + st.write("hello world") - class LitStreamlit(L.LightningFlow): + + class LitStreamlit(lapp.LightningFlow): def configure_layout(self): - return L.frontend.StreamlitFrontend(render_fn=your_streamlit_app) + return lapp.frontend.StreamlitFrontend(render_fn=your_streamlit_app) - class LitApp(L.LightningFlow): + + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_streamlit = LitStreamlit() @@ -136,7 +144,8 @@ the ``configure_layout`` method of the Lightning component you want to connect t tab1 = {"name": "home", "content": self.lit_streamlit} return tab1 - app = L.LightningApp(LitApp()) + + app = lapp.LightningApp(LitApp()) The ``render_fn`` argument of the ``StreamlitFrontend`` class, points to a function that runs your Streamlit app. The first argument to the function is the lightning app state. Any changes to the app state update the app. @@ -152,17 +161,20 @@ In this case, we render the ``LitStreamlit`` UI in the ``home`` tab of the appli :emphasize-lines: 18 # app.py - import lightning as L + import lightning_app as la import streamlit as st + def your_streamlit_app(lightning_app_state): - st.write('hello world') + st.write("hello world") - class LitStreamlit(L.LightningFlow): + + class LitStreamlit(lapp.LightningFlow): def configure_layout(self): - return L.frontend.StreamlitFrontend(render_fn=your_streamlit_app) + return lapp.frontend.StreamlitFrontend(render_fn=your_streamlit_app) - class LitApp(L.LightningFlow): + + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_streamlit = LitStreamlit() @@ -171,4 +183,5 @@ In this case, we render the ``LitStreamlit`` UI in the ``home`` tab of the appli tab1 = {"name": "home", "content": self.lit_streamlit} return tab1 - app = L.LightningApp(LitApp()) + + app = lapp.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/streamlit/index.rst b/docs/source-app/workflows/add_web_ui/streamlit/index.rst index 2496729d45..68d9b2011f 100644 --- a/docs/source-app/workflows/add_web_ui/streamlit/index.rst +++ b/docs/source-app/workflows/add_web_ui/streamlit/index.rst @@ -1,5 +1,3 @@ -:orphan: - .. toctree:: :maxdepth: 1 :hidden: diff --git a/docs/source-app/workflows/add_web_ui/streamlit/intermediate.rst b/docs/source-app/workflows/add_web_ui/streamlit/intermediate.rst index ac289c2eb2..0926575ab1 100644 --- a/docs/source-app/workflows/add_web_ui/streamlit/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/streamlit/intermediate.rst @@ -15,30 +15,27 @@ To modify the variables of a Lightning component, access the ``lightning_app_sta For example, here we increase the count variable of the Lightning Component every time a user presses a button: -.. code:: python +.. code:: bash :emphasize-lines: 7, 13 # app.py - import lightning as L + import lightning_app as la import streamlit as st - def your_streamlit_app(lightning_app_state): - if st.button("press to increase count"): + if st.button('press to increase count'): lightning_app_state.count += 1 - st.write(f"current count: {lightning_app_state.count}") + st.write(f'current count: {lightning_app_state.count}') - - class LitStreamlit(L.LightningFlow): + class LitStreamlit(lapp.LightningFlow): def __init__(self): super().__init__() self.count = 0 def configure_layout(self): - return L.frontend.StreamlitFrontend(render_fn=your_streamlit_app) + return lapp.frontend.StreamlitFrontend(render_fn=your_streamlit_app) - - class LitApp(L.LightningFlow): + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_streamlit = LitStreamlit() @@ -47,8 +44,7 @@ For example, here we increase the count variable of the Lightning Component ever tab1 = {"name": "home", "content": self.lit_streamlit} return tab1 - - app = L.LightningApp(LitApp()) + app = lapp.LightningApp(LitApp()) ---- @@ -60,19 +56,17 @@ parent component. In this example we update the value of the counter from the component: -.. code:: python +.. code:: bash :emphasize-lines: 6, 14 # app.py - import lightning as L + import lightning_app as la import streamlit as st - def your_streamlit_app(lightning_app_state): - st.write(f"current count: {lightning_app_state.count}") + st.write(f'current count: {lightning_app_state.count}') - - class LitStreamlit(L.LightningFlow): + class LitStreamlit(lapp.LightningFlow): def __init__(self): super().__init__() self.count = 0 @@ -81,10 +75,9 @@ In this example we update the value of the counter from the component: self.count += 1 def configure_layout(self): - return L.frontend.StreamlitFrontend(render_fn=your_streamlit_app) + return lapp.frontend.StreamlitFrontend(render_fn=your_streamlit_app) - - class LitApp(L.LightningFlow): + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_streamlit = LitStreamlit() @@ -96,5 +89,4 @@ In this example we update the value of the counter from the component: tab1 = {"name": "home", "content": self.lit_streamlit} return tab1 - - app = L.LightningApp(LitApp()) + app = lapp.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/vue_js_intermediate.rst b/docs/source-app/workflows/add_web_ui/vue_js_intermediate.rst index e8d9f3e843..7caf13be94 100644 --- a/docs/source-app/workflows/add_web_ui/vue_js_intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/vue_js_intermediate.rst @@ -1,5 +1,3 @@ -:orphan: - ####################################### Add a web UI with Vue.js (intermediate) ####################################### diff --git a/docs/source-app/workflows/arrange_tabs/arrange_app_basic.rst b/docs/source-app/workflows/arrange_tabs/arrange_app_basic.rst index 91c0e53854..5780d8e7b0 100644 --- a/docs/source-app/workflows/arrange_tabs/arrange_app_basic.rst +++ b/docs/source-app/workflows/arrange_tabs/arrange_app_basic.rst @@ -5,32 +5,49 @@ Arrange app tabs (basic) ---- -***************************** -Enable a full-page single tab -***************************** - +******************* +Enable a single tab +******************* To enable a single tab on the app UI, return a single dictionary from the ``configure_layout`` method: .. code:: python :emphasize-lines: 9 - import lightning as L + import lightning_app as la - class DemoComponent(L.demo.dumb_component): + class DemoComponent(lapp.demo.dumb_component): def configure_layout(self): tab1 = {"name": "THE TAB NAME", "content": self.component_a} return tab1 - app = L.LightningApp(DemoComponent()) + app = lapp.LightningApp(DemoComponent()) - -The "name" key defines the visible name of the tab on the UI. It also shows up in the URL. +The "name" key defines the visible name of the tab on the UI. The **"content"** key defines the target component to render in that tab. -When returning a single tab element like shown above, the UI will display it in full-page mode. +---- + +***************************** +Enable a full-page single tab +***************************** + +.. code:: python + :emphasize-lines: 6 + + import lightning_app as la + + + class DemoComponent(lapp.demo.dumb_component): + def configure_layout(self): + tab1 = {"name": None, "content": self.component_a} + return tab1 + + + app = lapp.LightningApp(DemoComponent()) + ---- ******************** @@ -40,19 +57,19 @@ Enable multiple tabs .. code:: python :emphasize-lines: 7 - import lightning as L + import lightning_app as la - class DemoComponent(L.demo.dumb_component): + class DemoComponent(lapp.demo.dumb_component): def configure_layout(self): tab1 = {"name": "Tab A", "content": self.component_a} tab2 = {"name": "Tab B", "content": self.component_b} return tab1, tab2 - app = L.LightningApp(DemoComponent()) + app = lapp.LightningApp(DemoComponent()) -The order matters! Try any of the following configurations: +order matters! Try any of the following configurations: .. code:: python :emphasize-lines: 4, 9 diff --git a/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst b/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst index 798212bf7d..8136d907fb 100644 --- a/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst +++ b/docs/source-app/workflows/build_lightning_app/from_pytorch_lightning_script.rst @@ -75,10 +75,10 @@ The command above generates an app file like this: from your_app_name import ComponentA, ComponentB - import lightning as L + import lightning_app as la - class LitApp(L.LightningFlow): + class LitApp(lapp.LightningFlow): def __init__(self) -> None: super().__init__() self.component_a = ComponentA() @@ -89,7 +89,7 @@ The command above generates an app file like this: self.component_b.run() - app = L.LightningApp(LitApp()) + app = lapp.LightningApp(LitApp()) Now you can add your own components as you wish! diff --git a/docs/source-app/workflows/build_lightning_app/from_scratch.rst b/docs/source-app/workflows/build_lightning_app/from_scratch.rst index 38335cff9d..4f0a169d4b 100644 --- a/docs/source-app/workflows/build_lightning_app/from_scratch.rst +++ b/docs/source-app/workflows/build_lightning_app/from_scratch.rst @@ -1,10 +1,124 @@ -################################## -Build a Lightning App from Scratch -################################## -**Audience:** Users who want to build a Lightning App from scratch. - -**Prereqs:** You must have finished the `Basic levels `_. +################### +Build Lightning app +################### +**Audience:** Users who want to build a lightning app from scratch. ---- -.. include:: from_scratch_content.rst +************** +Fork and build +************** +Before you build a Lightning App from scratch, see if you can find an app that is similar to what you need +in the `Lightning App Gallery `_. + +Once you find the app you want, press "Get" to see it running on the cloud, then download the code +and change what you want! + +---- + +****************** +Build from scratch +****************** +If you didn't find an app similar to the one you need, simply create a file named **app.py** with these contents: + +.. code:: bash + + import lightning_app as la + + class WordComponent(lapp.LightningWork): + def __init__(self, word): + super().__init__() + self.word = word + def run(self): + print(self.word) + + + class LitApp(lapp.LightningFlow): + def __init__(self) -> None: + super().__init__() + self.hello = WordComponent('hello') + self.world = WordComponent('world') + + def run(self): + print('this is a simple Lightning app, make a better app!') + self.hello.run() + self.world.run() + + app = lapp.LightningApp(LitApp()) + +---- + +Run the app +^^^^^^^^^^^ +Run the app locally: + +.. code:: bash + + lightning run app app.py + +Run the app on the cloud: + +.. code:: bash + + lightning run app app.py --cloud + +---- + +********************* +Build from a template +********************* +If you didn't find an app similar to the one you need (in the `Lightning App gallery `_), another option is to start from a template. +The lightning CLI can generate a template with built-in testing that can be easily published to the +Lightning app gallery. + +Generate it with our template generator: + +.. code:: bash + + lightning init app your-app-name + +You'll see a print-out like this: + +.. code:: bash + + ➜ lightning init app your-app-name + + /Users/Your/Current/dir/your-app-name + INFO: laying out app template at /Users/Your/Current/dir/your-app-name + INFO: + Lightning app template created! + /Users/Your/Current/dir/your-app-name + + run your app with: + lightning run app your-app-name/your_app_name/app.py + + run it on the cloud to share with your collaborators: + lightning run app your-app-name/your_app_name/app.py --cloud + +---- + +Modify the template +^^^^^^^^^^^^^^^^^^^ +The command above generates an app file like this: + +.. code:: python + + from your_app_name import ComponentA, ComponentB + + import lightning_app as la + + + class LitApp(lapp.LightningFlow): + def __init__(self) -> None: + super().__init__() + self.component_a = ComponentA() + self.component_b = ComponentB() + + def run(self): + self.component_a.run() + self.component_b.run() + + + app = lapp.LightningApp(LitApp()) + +Now you can add your own components as you wish! diff --git a/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst b/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst deleted file mode 100644 index 91e7fea93e..0000000000 --- a/docs/source-app/workflows/build_lightning_app/from_scratch_content.rst +++ /dev/null @@ -1,121 +0,0 @@ - -************** -WAIT! -************** -Before you build a Lightning App from scratch, see if you can find an app that is similar to what you need -in the `Lightning App Gallery `_. - -Once you find the Lightning App you want, press "Clone & Run" to see it running on the cloud, then download the code -and change what you want! - ----- - -****************** -Build from scratch -****************** -If you didn't find a Lightning App similar to the one you need, simply create a file named **app.py** with these contents: - -.. code:: python - - import lightning as L - - - class WordComponent(L.LightningWork): - def __init__(self, word): - super().__init__() - self.word = word - - def run(self): - print(self.word) - - - class LitApp(L.LightningFlow): - def __init__(self) -> None: - super().__init__() - self.hello = WordComponent("hello") - self.world = WordComponent("world") - - def run(self): - print("This is a simple Lightning app, make a better app!") - self.hello.run() - self.world.run() - - - app = L.LightningApp(LitApp()) - ----- - -Run the Lightning App -^^^^^^^^^^^^^^^^^^^^^ -Run the Lightning App locally: - -.. code:: bash - - lightning run app app.py - -Run the Lightning App on the cloud: - -.. code:: bash - - lightning run app app.py --cloud - ----- - -************************************* -Build a Lightning App from a template -************************************* -If you didn't find an Lightning App similar to the one you need (in the `Lightning App gallery `_), another option is to start from a template. -The Lightning CLI can generate a template with built-in testing that can be easily published to the -Lightning App Gallery. - -Generate a Lightning App with our template generator: - -.. code:: bash - - lightning init app your-app-name - -You'll see a print-out like this: - -.. code:: bash - - ➜ lightning init app your-app-name - - /Users/Your/Current/dir/your-app-name - INFO: laying out app template at /Users/Your/Current/dir/your-app-name - INFO: - Lightning app template created! - /Users/Your/Current/dir/your-app-name - - run your app with: - lightning run app your-app-name/your_app_name/app.py - - run it on the cloud to share with your collaborators: - lightning run app your-app-name/your_app_name/app.py --cloud - ----- - -Modify the Lightning App template -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The command above generates a Lightning App file like this: - -.. code:: python - - from your_app_name import ComponentA, ComponentB - - import lightning as L - - - class LitApp(L.LightningFlow): - def __init__(self) -> None: - super().__init__() - self.component_a = ComponentA() - self.component_b = ComponentB() - - def run(self): - self.component_a.run() - self.component_b.run() - - - app = L.LightningApp(LitApp()) - -Now you can add your own components as you wish! diff --git a/docs/source-app/workflows/build_lightning_component/basic.rst b/docs/source-app/workflows/build_lightning_component/basic.rst index b72b9147e2..37099a3a6f 100644 --- a/docs/source-app/workflows/build_lightning_component/basic.rst +++ b/docs/source-app/workflows/build_lightning_component/basic.rst @@ -1,8 +1,205 @@ -########################### -Build a Lightning component -########################### +################################### +Build a Lightning component (basic) +################################### **Audience:** Users who want to build a Lightning component. ---- -.. include:: from_scratch_component_content.rst +******************************* +Why should I build a component? +******************************* +Lightning Components break up complex systems into modular components. The first obvious benefit is that components +can be reused across other apps. This means you can build once, test it and forget it. + +As a researcher it also means that your code can be taken to production without needing a team of engineers to help +productionize it. + +As a machine learning engineer, it means that your cloud system is: + +- fault tolerant +- cloud agnostic +- testable (unlike YAML/CI/CD code) +- version controlled +- enables cross-functional collaboration + +---- + +************** +Fork and build +************** +Before you build a Lightning component from scratch, see if you can find a component that is similar to what you need +in the `Lightning component Gallery `_. + +Once you find the component you want, download the code and change what you want! + +---- + +**************************** +Decide between Flow and Work +**************************** + +.. raw:: html + + Choosing between LightningFlow and LightningWork + +There are two types of components in Lightning, LightningFlow and LightningWork. + +Use a **LightningFlow** component for any programming logic that runs in less than 1 second. + +.. code:: python + + for i in range(10): + print(f"{i}: this kind of code belongs in a LightningFlow") + +Use a **LightningWork** component for any programming logic that takes more than 1 second or requires its own hardware. + +.. code:: python + + from time import sleep + + for i in range(100000): + sleep(2.0) + print(f"{i} LightningWork: work that is long running or may never end (like a server)") + +---- + +****************** +Build from scratch +****************** +The first option is if you want to build from scratch + +---- + +Option A: Build a LightningFlow +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +To implement a LightningFlow, simply subclass ``LightningFlow`` and define the run method: + +.. code:: python + :emphasize-lines: 5 + + # app.py + import lightning_app as la + + + class LitFlow(lapp.LightningFlow): + def run(self): + for i in range(10): + print(f"{i}: this kind of code belongs in a LightningFlow") + + + app = lapp.LightningApp(LitFlow()) + +run the app + +.. code:: bash + + lightning run app app.py + +---- + +Option B: build a LightningWork +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Only implement a LightningWork if this particular piece of code: + +- takes more than 1 second to execute +- or requires its own set of cloud resources +- or both + +To implement a LightningWork, simply subclass ``LightningWork`` and define the run method: + +.. code:: python + :emphasize-lines: 6 + + # app.py + from time import sleep + import lightning_app as la + + + class LitWork(lapp.LightningWork): + def run(self): + for i in range(100000): + sleep(2.0) + print(f"{i} LightningWork: work that is long running or may never end (like a server)") + +A LightningWork must always be attached to a LightningFlow and explicitely asked to ``run()``: + +.. code:: python + :emphasize-lines: 13, 16 + + from time import sleep + import lightning_app as la + + + class LitWork(lapp.LightningWork): + def run(self): + for i in range(100000): + sleep(2.0) + print(f"{i} LightningWork: work that is long running or may never end (like a server)") + + + class LitFlow(lapp.LightningFlow): + def __init__(self): + super().__init__() + self.lit_work = LitWork() + + def run(self): + self.lit_work.run() + + + app = lapp.LightningApp(LitFlow()) + +run the app + +.. code:: bash + + lightning run app app.py + +---- + +********************* +Build from a template +********************* +If you'd prefer a component template with built-in testing that can be easily published to the +Lightning component gallery, generate it with our template generator: + +.. code:: bash + + lightning init component your-component-name + +You'll see a print-out like this: + +.. code:: bash + + ➜ lightning init component your-component-name + INFO: laying out component template at /Users/williamfalcon/Developer/opensource/_/lightning/scratch/hello-world + INFO: + ⚡ Lightning component template created! ⚡ + /Users/williamfalcon/Developer/opensource/_/lightning/scratch/hello-world + + ... + +---- + +Modify the component template +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The command above generates a component file like this: + +.. code:: python + + import lightning_app as la + + + class TemplateComponent(lapp.LightningWork): + def __init__(self) -> None: + super().__init__() + self.value = 0 + + def run(self): + self.value += 1 + print("welcome to your work component") + print("this is running inside a work") + +Now you can modify the component as you wish! diff --git a/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst b/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst deleted file mode 100644 index 002816b074..0000000000 --- a/docs/source-app/workflows/build_lightning_component/from_scratch_component_content.rst +++ /dev/null @@ -1,200 +0,0 @@ -******************************* -LightningFlow vs. LightningWork -******************************* - -.. _flow_vs_work: - -.. raw:: html - - Choosing between LightningFlow and LightningWork - -There are two types of components in Lightning, **LightningFlow** and **LightningWork**. - -Use a **LightningFlow** component for any programming logic that runs in less than 1 second. - -.. code:: python - - for i in range(10): - print(f"{i}: this kind of code belongs in a LightningFlow") - -Use a **LightningWork** component for any programming logic that takes more than 1 second or requires its own hardware. - -.. code:: python - - from time import sleep - - for i in range(100000): - sleep(2.0) - print(f"{i} LightningWork: work that is long running or may never end (like a server)") - ----- - -************************************************ -What building a Lightning component does for you -************************************************ -Lightning Components break up complex systems into modular components. The first obvious benefit is that components -can be reused across other apps. This means you can build once, test it and forget it. - -As a researcher it also means that your code can be taken to production without needing a team of engineers to help -productionize it. - -As a machine learning engineer, it means that your cloud system is: - -- fault tolerant -- cloud agnostic -- testable (unlike YAML/CI/CD code) -- version controlled -- enables cross-functional collaboration - ----- - -************** -WAIT! -************** -Before you build a Lightning component from scratch, see if you can find a component that is similar to what you need -in the `Lightning component Gallery `_. - -Once you find the component you want, download the code and change what you want! - ----- - -***************************************** -Build a Lighitning component from scratch -***************************************** -If you didn't find a Lightning component similar to the one you need, you can build one from scratch. - ----- - -Build a LightningFlow -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To implement a LightningFlow, simply subclass ``LightningFlow`` and define the run method: - -.. code:: python - :emphasize-lines: 5 - - # app.py - import lightning as L - - - class LitFlow(L.LightningFlow): - def run(self): - for i in range(10): - print(f"{i}: this kind of code belongs in a LightningFlow") - - - app = L.LightningApp(LitFlow()) - -run the app - -.. code:: bash - - lightning run app app.py - ----- - -Build a LightningWork -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Only implement a LightningWork if this particular piece of code: - -- takes more than 1 second to execute -- requires its own set of cloud resources -- or both - -To implement a LightningWork, simply subclass ``LightningWork`` and define the run method: - -.. code:: python - :emphasize-lines: 6 - - # app.py - from time import sleep - import lightning as L - - - class LitWork(L.LightningWork): - def run(self): - for i in range(100000): - sleep(2.0) - print(f"{i} LightningWork: work that is long running or may never end (like a server)") - -A LightningWork must always be attached to a LightningFlow and explicitely asked to ``run()``: - -.. code:: python - :emphasize-lines: 13, 16 - - from time import sleep - import lightning as L - - - class LitWork(L.LightningWork): - def run(self): - for i in range(100000): - sleep(2.0) - print(f"{i} LightningWork: work that is long running or may never end (like a server)") - - - class LitFlow(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_work = LitWork() - - def run(self): - self.lit_work.run() - - - app = L.LightningApp(LitFlow()) - -run the app - -.. code:: bash - - lightning run app app.py - ----- - -******************************************* -Build a Lightning component from a template -******************************************* -If you'd prefer a component template with built-in testing that can be easily published to the -Lightning component gallery, generate it with our template generator: - -.. code:: bash - - lightning init component your-component-name - -You'll see a print-out like this: - -.. code:: bash - - ➜ lightning init component your-component-name - INFO: laying out component template at /Users/williamfalcon/Developer/opensource/_/lightning/scratch/hello-world - INFO: - ⚡ Lightning component template created! ⚡ - /Users/williamfalcon/Developer/opensource/_/lightning/scratch/hello-world - - ... - ----- - -Modify the component template -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The command above generates a component file like this: - -.. code:: python - - import lightning as L - - - class TemplateComponent(L.LightningWork): - def __init__(self) -> None: - super().__init__() - self.value = 0 - - def run(self): - self.value += 1 - print("welcome to your work component") - print("this is running inside a work") - -Now you can modify the component as you wish! diff --git a/docs/source-app/workflows/build_lightning_component/index_content.rst b/docs/source-app/workflows/build_lightning_component/index_content.rst index 00b4e99ad1..54dcad4d89 100644 --- a/docs/source-app/workflows/build_lightning_component/index_content.rst +++ b/docs/source-app/workflows/build_lightning_component/index_content.rst @@ -28,7 +28,7 @@ Basics
.. displayitem:: - :header: Build a Lightning component + :header: Build a component :description: Learn the basics of building a Lightning component :col_css: col-md-4 :button_link: basic.html @@ -36,8 +36,8 @@ Basics :tag: basic .. displayitem:: - :header: Explore community Lightning components - :description: Discover community-built Lightning components + :header: Explore community components + :description: Discover community-built components :col_css: col-md-4 :button_link: https://lightning.ai/components :height: 150 @@ -84,7 +84,7 @@ Intermediate :tag: intermediate .. displayitem:: - :header: Publish a Lightning component + :header: Publish a component :description: Learn the basics of publishing a Lightning component. :col_css: col-md-4 :button_link: publish_a_component.html diff --git a/docs/source-app/workflows/build_lightning_component/intermediate.rst b/docs/source-app/workflows/build_lightning_component/intermediate.rst index 070d3aa0ca..a1956b260d 100644 --- a/docs/source-app/workflows/build_lightning_component/intermediate.rst +++ b/docs/source-app/workflows/build_lightning_component/intermediate.rst @@ -29,11 +29,11 @@ To *connect* this user interface to the component, define the configure_layout m .. code:: python :emphasize-lines: 5, 6 - import lightning as L + import lightning_app as la from lightning_app.frontend.web import StaticWebFrontend - class LitHTMLComponent(L.LightningFlow): + class LitHTMLComponent(lapp.LightningFlow): def configure_layout(self): return StaticWebFrontend(serve_dir="path/to/folder/with/index.html/inside") @@ -43,15 +43,15 @@ Finally, route the component's UI through the root component's **configure_layou :emphasize-lines: 14 # app.py - import lightning as L + import lightning_app as la - class LitHTMLComponent(L.LightningFlow): + class LitHTMLComponent(lapp.LightningFlow): def configure_layout(self): - return L.frontend.web.StaticWebFrontend(serve_dir="path/to/folder/with/index.html/inside") + return lapp.frontend.web.StaticWebFrontend(serve_dir="path/to/folder/with/index.html/inside") - class LitApp(L.LightningFlow): + class LitApp(lapp.LightningFlow): def __init__(self): super().__init__() self.lit_html_component = LitHTMLComponent() @@ -61,7 +61,7 @@ Finally, route the component's UI through the root component's **configure_layou return tab1 - app = L.LightningApp(LitApp()) + app = lapp.LightningApp(LitApp()) Run your app and you'll see the UI on the Lightning App view: diff --git a/docs/source-app/workflows/build_lightning_component/publish_a_component.rst b/docs/source-app/workflows/build_lightning_component/publish_a_component.rst index 8364afd3d5..f4ed73b2bd 100644 --- a/docs/source-app/workflows/build_lightning_component/publish_a_component.rst +++ b/docs/source-app/workflows/build_lightning_component/publish_a_component.rst @@ -13,7 +13,7 @@ the default template. Generate your component template with this command: -.. code:: python +.. code:: bash lightning init component your-component-name @@ -36,18 +36,20 @@ Now import your component and use it in an app: # app.py from your_component import TemplateComponent - import lightning as L + import lightning_app as la - class LitApp(L.LightningFlow): + + class LitApp(lapp.LightningFlow): def __init__(self) -> None: super().__init__() self.your_component = TemplateComponent() def run(self): - print('this is a simple Lightning app to verify your component is working as expected') + print("this is a simple Lightning app to verify your component is working as expected") self.your_component.run() - app = L.LightningApp(LitApp()) + + app = lapp.LightningApp(LitApp()) and run the app: diff --git a/docs/source-app/workflows/debug_locally.rst b/docs/source-app/workflows/debug_locally.rst index cd5a5a80fd..77ea163b44 100644 --- a/docs/source-app/workflows/debug_locally.rst +++ b/docs/source-app/workflows/debug_locally.rst @@ -1,5 +1,3 @@ -:orphan: - ##################################### Debug a Distributed Cloud App Locally ##################################### diff --git a/docs/source-app/workflows/enable_fault_tolerance.rst b/docs/source-app/workflows/enable_fault_tolerance.rst index b1630d4d39..c7af476d34 100644 --- a/docs/source-app/workflows/enable_fault_tolerance.rst +++ b/docs/source-app/workflows/enable_fault_tolerance.rst @@ -1,5 +1,3 @@ -:orphan: - ###################### Enable Fault Tolerance ###################### diff --git a/docs/source-app/workflows/extend_app.rst b/docs/source-app/workflows/extend_app.rst index 2e17af481b..d0ac9feab3 100644 --- a/docs/source-app/workflows/extend_app.rst +++ b/docs/source-app/workflows/extend_app.rst @@ -1,7 +1,7 @@ ###################### Extend an Existing App ###################### -You can extend a Lightning App by using community components or building your own. +You can extend a Lightning app by using community components or building your own. ---- @@ -13,8 +13,8 @@ You can extend a Lightning App by using community components or building your ow .. Add callout items below this line .. displayitem:: - :header: Add more Components - :description: Extend an App by adding a prebuilt component. + :header: Add more components + :description: Extend an app by adding a prebuilt component. :col_css: col-md-4 :button_link: add_components/index.html :height: 150 @@ -22,23 +22,23 @@ You can extend a Lightning App by using community components or building your ow .. displayitem:: :header: Add a web user interface (UI) - :description: Extend an App by adding a web user interface (UI) + :description: Extend an app by adding a web user interface (UI) :col_css: col-md-4 :button_link: add_web_ui/index.html :height: 150 :tag: basic .. displayitem:: - :header: Add a URL link - :description: Extend an App by adding a web URL link + :header: Add a url link + :description: Extend an app by adding a web url link :col_css: col-md-4 :button_link: add_web_link.html :height: 150 :tag: basic .. displayitem:: - :header: Build a Component - :description: Extend an App by building a Component + :header: Build a component + :description: Extend an app by building a component :col_css: col-md-4 :button_link: build_lightning_component/index.html :height: 150 @@ -46,7 +46,7 @@ You can extend a Lightning App by using community components or building your ow .. displayitem:: :header: Add a server - :description: Extend an App by adding a server to a Component. + :description: Extend an app by adding a server to a component. :col_css: col-md-4 :button_link: add_server/index.html :height: 150 diff --git a/docs/source-app/workflows/index.rst b/docs/source-app/workflows/index.rst deleted file mode 100644 index 7f0642f661..0000000000 --- a/docs/source-app/workflows/index.rst +++ /dev/null @@ -1,114 +0,0 @@ -:orphan: - -################ -Common Workflows -################ - -.. raw:: html - -
-
- -.. displayitem:: - :header: Add a web user interface - :description: Learn how to add React, StreamLit, Dash to your App. - :col_css: col-md-4 - :button_link: add_web_ui/index.html - :height: 180 - -.. displayitem:: - :header: Add a web link - :description: Learn how to embed external websites - :col_css: col-md-4 - :button_link: add_web_link.html - :height: 180 - -.. displayitem:: - :header: Arrange App tabs - :description: Learn how to organize your UI - :col_css: col-md-4 - :button_link: arrange_tabs/index.html - :height: 180 - -.. displayitem:: - :header: Build a Lightning App - :description: Simple App to get started - :col_css: col-md-4 - :button_link: build_lightning_app/index.html - :height: 180 - -.. displayitem:: - :header: Build a Lightning Component - :description: Understand how to separated the glue from the actual work - :col_css: col-md-4 - :button_link: build_lightning_component/index.html - :height: 180 - -.. displayitem:: - :header: Cache Work run calls - :description: Understand how to trigger a work run method - :col_css: col-md-4 - :button_link: run_work_once.html - :height: 180 - -.. displayitem:: - :header: Customize your cloud compute - :description: Select machines to run on - :col_css: col-md-4 - :button_link: ../core_api/lightning_work/compute.html - :height: 180 - -.. displayitem:: - :header: Extend an existing App - :description: Learn where to go next with an App - :col_css: col-md-4 - :button_link: extend_app.html - :height: 180 - -.. displayitem:: - :header: Publish a Lightning Component - :description: Share your components with others - :col_css: col-md-4 - :button_link: build_lightning_component/publish_a_component.html - :height: 180 - -.. displayitem:: - :header: Run a server within a Lightning App - :description: Lightning Work can be infinite jobs - :col_css: col-md-4 - :button_link: add_server/index.html - :height: 180 - -.. displayitem:: - :header: Run an App on the cloud - :description: Learn how to get things done in the cloud with ease - :col_css: col-md-4 - :button_link: run_app_on_cloud/index.html - :height: 180 - -.. displayitem:: - :header: Run Works in parallel - :description: Learn how to make your Work non blocking - :col_css: col-md-4 - :button_link: run_work_in_parallel.html - :height: 180 - -.. displayitem:: - :header: Share an App - :description: Learn how to share your work with others - :col_css: col-md-4 - :button_link: share_app.html - :height: 180 - -.. displayitem:: - :header: Share files between components - :description: Learn how Lightning Storage emulates a single filesystem in a distributed setting - :col_css: col-md-4 - :button_link: share_files_between_components.html - :height: 180 - - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/run_app_on_cloud/cloud_files.rst b/docs/source-app/workflows/run_app_on_cloud/cloud_files.rst deleted file mode 100644 index 3130cd0f33..0000000000 --- a/docs/source-app/workflows/run_app_on_cloud/cloud_files.rst +++ /dev/null @@ -1,58 +0,0 @@ -.. _ignore: - -################################## -Configure Your Lightning Cloud App -################################## - -**Audience:** Users who want to control Lightning App files on the cloud. - ----- - -************************************** -Ignore file uploads to Lightning cloud -************************************** -Running Lightning Apps on the cloud will upload the source code of your app to the cloud. You can use ``.lightningignore`` file(s) to ignore files or directories while uploading. The `.lightningignore` file follows the same format as a `.gitignore` -file. - -For example, the source code directory below with the ``.lightningignore`` file will ignore the file named -``model.pt`` and directory named ``data_dir``. - -.. code:: bash - - . - ├── README.md - ├── app.py - ├── data_dir - │ ├── image1.png - │ ├── image2.png - │ └── ... - ├── .lightningignore - ├── requirements.txt - └── model.pt - - -.. code:: bash - - ~/project/home ❯ cat .lightningignore - model.pt - data_dir - -A sample ``.lightningignore`` file can be found `here `_. - - ----- - -******************* -Structure app files -******************* - -We recommend your app contain the following files: - -.. code:: bash - - . - ├── .lightning (auto-generated- conatins Lightning configuration) - ├── .lightningignore (contains files not to upload to the cloud) - ├── app.py - ├── README.md (optional- a markdown description of your app) - └── requirements.txt (optional- conatins all your app dependencies) diff --git a/docs/source-app/workflows/run_app_on_cloud/index.rst b/docs/source-app/workflows/run_app_on_cloud/index.rst deleted file mode 100644 index 55bc3b6807..0000000000 --- a/docs/source-app/workflows/run_app_on_cloud/index.rst +++ /dev/null @@ -1,5 +0,0 @@ -##################### -Run apps on the cloud -##################### - -.. include:: index_content.rst diff --git a/docs/source-app/workflows/run_app_on_cloud/index_content.rst b/docs/source-app/workflows/run_app_on_cloud/index_content.rst deleted file mode 100644 index b0f67570d7..0000000000 --- a/docs/source-app/workflows/run_app_on_cloud/index_content.rst +++ /dev/null @@ -1,115 +0,0 @@ -.. _run_app_in_cloud: - -.. toctree:: - :maxdepth: 1 - :hidden: - - cloud_files - lightning_cloud - on_prem - on_your_own_machine - -**Audience:** Users who want to share or scale Lightning Apps. - ----- - -***************************** -Run on Lightning Public Cloud -***************************** - -You can run Lightning Apps for free on the Public Lightning cloud with a single flag! - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Run on Lightning Cloud - :description: Learn how to run on the Lightning public cloud - :col_css: col-md-4 - :button_link: lightning_cloud.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Choose Hardware - :description: Configure you app cloud resources - :col_css: col-md-4 - :button_link: ../../core_api/lightning_work/compute.html - :height: 150 - :tag: Basic - -.. displayitem:: - :header: Set Environment Variables - :description: Manage your environment variables in the cloud - :col_css: col-md-4 - :button_link: ../../glossary/environment_variables.html - :height: 150 - :tag: Basic - -.. displayitem:: - :header: Configure Your Lightning Cloud App - :description: Customize your cloud apps files - :col_css: col-md-4 - :button_link: cloud_files.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Manage App Dependancies - :description: Configure your python requirements or use a custom docker image - :col_css: col-md-4 - :button_link: ../../glossary/build_config/build_config.html - :height: 150 - :tag: Intermediate - -.. displayitem:: - :header: Share Files Between Works - :description: Learn more about data transfering - :col_css: col-md-4 - :button_link: ../../glossary/storage/storage.html - :height: 150 - :tag: Intermediate - -.. raw:: html - -
-
- ----- - -************ -Other Clouds -************ - -.. raw:: html - -
-
- -.. Add callout items below this line - -.. displayitem:: - :header: Run On Your Own Machine - :description: Run Lightning Apps on any machine - :col_css: col-md-4 - :button_link: on_your_own_machine.html - :height: 150 - :tag: basic - -.. displayitem:: - :header: Run On Your Private Cloud - :description: Run Lightning Apps on your own cloud - :col_css: col-md-4 - :button_link: on_prem.html - :height: 150 - :tag: basic - - -.. raw:: html - -
-
diff --git a/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst b/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst deleted file mode 100644 index aaa9d7bb08..0000000000 --- a/docs/source-app/workflows/run_app_on_cloud/lightning_cloud.rst +++ /dev/null @@ -1,67 +0,0 @@ -####################### -Run an App on the Cloud -####################### - -**Audience:** Users who want to share their apps or run on specialized hardware (like GPUs). - ----- - -********************************* -Run on the public Lightning cloud -********************************* -To run any app on the public lightning cloud use the ``--cloud`` argument: - -.. code:: bash - - lightning run app app.py --cloud - - -.. note:: - By default, running your apps on the public Lightning cloud is free of charge using default CPUs, and any app uploaded to the Lightning cloud will be shared with the community (source code and app view will be public). If you would like to make your apps private please `contact us `_. - -If your app contains ``LightningWork`` components that require more compute resources, such as larger CPUs or **GPUs**, you'll need to add credits to your Lightning AI account. - - ----- - -************************** -Add dependencies to my app -************************** - - -Add all dependencies required to run your app to a `requirements.txt` file in your app's directory. Read :ref:`build_config` for more details. - - - ----- - - -******** -Name app -******** - -Simply use the ``--name`` flag when running your app, for example: - -.. code:: bash - - lightning run app app.py --cloud --name my-awesome-app - -Alternatively, you can change the name of the app in the ``.lightning`` file: - -.. code:: bash - - ~/project/home ❯ cat .lightning - name: my-awesome-app - -The ``.lightning`` file is a general configuration file. -To learn more about optional configuration file parameters, see :class:`~lightning.utilities.packaging.app_config.AppConfig`. - ------- - -******************** -Choose Cloud Compute -******************** - -You can configure the hardware your app is running on by setting a :class:`~lightning.utilities.packaging.cloud_compute.CloudCompute` object onto the ``cloud_compute`` property of your work's. - -Learn more with the :ref:`cloud_compute` guide diff --git a/docs/source-app/workflows/run_app_on_cloud/on_prem.rst b/docs/source-app/workflows/run_app_on_cloud/on_prem.rst deleted file mode 100644 index be0a954f29..0000000000 --- a/docs/source-app/workflows/run_app_on_cloud/on_prem.rst +++ /dev/null @@ -1,6 +0,0 @@ -########################### -Run an App on Private Cloud -########################### - - -To run Lightning apps on a private or on-prem cluster, `contact us `_. diff --git a/docs/source-app/workflows/run_app_on_cloud/on_your_own_machine.rst b/docs/source-app/workflows/run_app_on_cloud/on_your_own_machine.rst deleted file mode 100644 index 795205f552..0000000000 --- a/docs/source-app/workflows/run_app_on_cloud/on_your_own_machine.rst +++ /dev/null @@ -1,23 +0,0 @@ -####################### -Run on your own machine -####################### - -**Audience:** Users who want to run Lightning App on a remote machine. - ----- - -*********** -Run via ssh -*********** -To run a Lightning App on any machine, simply ssh to the machine and run the app directly - -.. code:: bash - - # log into your cloud machine - ssh your_name@your_cloud_machine - - # get your code on the machine and install deps - ... - - # start the app - lightning run app app.py diff --git a/docs/source-app/workflows/run_components_on_different_hardware.rst b/docs/source-app/workflows/run_components_on_different_hardware.rst index 9685c3461e..ef3aac100d 100644 --- a/docs/source-app/workflows/run_components_on_different_hardware.rst +++ b/docs/source-app/workflows/run_components_on_different_hardware.rst @@ -1,5 +1,3 @@ -:orphan: - #################################### Run components on different hardware #################################### diff --git a/docs/source-app/workflows/run_work_in_parallel.rst b/docs/source-app/workflows/run_work_in_parallel.rst index 58089b6ed7..542b36b3b3 100644 --- a/docs/source-app/workflows/run_work_in_parallel.rst +++ b/docs/source-app/workflows/run_work_in_parallel.rst @@ -1,11 +1,56 @@ -############################## -Run LightningWorks in parallel -############################## +#################### +Run work in parallel +#################### -**Audience:** Users who want to run multiple LightningWorks at once. - -**Prereqs:** Level 8+ +When there is a long-running workload such as a model training, or a deployment server, it may be desirable to run that work in parallel +while the rest of the app continues to execute. ---- -.. include:: run_work_in_parallel_content.rst +**************************** +Why do I need parallel work? +**************************** +The default behavior of the ``LightningWork`` is to wait for the ``run`` method to complete: + +.. code:: python + + import lightning_app as la + + + class Root(lapp.LightningFlow): + def __init__(self): + self.work_component_a = lapp.demo.InfinteWorkComponent() + + def run(self): + self.work_component_a.run() + print("this will never print") + +Since this Work component we created loops forever, the print statement will never execute. In practice +``LightningWork`` workloads are finite and don't run forever. + +When a ``LightningWork`` performs a heavy operation (longer than 1 second), or requires its own hardware, +work that is *not* done in parallel will slow down your app. + +---- + +******************** +Enable parallel work +******************** +To run work in parallel while the rest of the app executes without delays, enable ``parallel=True``: + +.. code:: python + :emphasize-lines: 5 + + import lightning_app as la + + + class Root(lapp.LightningFlow): + def __init__(self): + self.work_component_a = lapp.demo.InfinteWorkComponent(parallel=True) + + def run(self): + self.work_component_a.run() + print("repeats while the infinite work runs ONCE (and forever) in parallel") + +Any work that will take more than **1 second** should be run in parallel +unless the rest of your app depends on the output of this work (for example, downloading a dataset). diff --git a/docs/source-app/workflows/run_work_in_parallel_content.rst b/docs/source-app/workflows/run_work_in_parallel_content.rst deleted file mode 100644 index ecb87c5cea..0000000000 --- a/docs/source-app/workflows/run_work_in_parallel_content.rst +++ /dev/null @@ -1,53 +0,0 @@ - - -**************************************************** -What running LightningWorks in parallel does for you -**************************************************** - -When there is a long-running workload such as a model training, or a deployment server, you might want to run that LightningWork in parallel -while the rest of the Lightning App continues to execute. - -The default behavior of the ``LightningWork`` is to wait for the ``run`` method to complete: - -.. code:: python - - import lightning as L - - - class Root(L.LightningFlow): - def __init__(self): - self.work_component_a = L.demo.InfinteWorkComponent() - - def run(self): - self.work_component_a.run() - print("this will never print") - -Since this LightningWork component we created loops forever, the print statement will never execute. In practice -``LightningWork`` workloads are finite and don't run forever. - -When a ``LightningWork`` performs a heavy operation (longer than 1 second), or requires its own hardware, -LightningWork that is *not* done in parallel will slow down your app. - ----- - -****************************** -Enable parallel LightningWorks -****************************** -To run LightningWorks in parallel, while the rest of the app executes without delays, enable ``parallel=True``: - -.. code:: python - :emphasize-lines: 5 - - import lightning as L - - - class Root(L.LightningFlow): - def __init__(self): - self.work_component_a = L.demo.InfinteWorkComponent(parallel=True) - - def run(self): - self.work_component_a.run() - print("repeats while the infinite work runs ONCE (and forever) in parallel") - -Any LightningWorks that will take more than **1 second** should be run in parallel -unless the rest of your Lightning App depends on the output of this work (for example, downloading a dataset). diff --git a/docs/source-app/workflows/run_work_once.rst b/docs/source-app/workflows/run_work_once.rst index 240cde3d08..b98df496bc 100644 --- a/docs/source-app/workflows/run_work_once.rst +++ b/docs/source-app/workflows/run_work_once.rst @@ -1,13 +1,126 @@ -########################## -Cache LightningWork Runs -########################## +.. _cache_calls: + +################# +Caching Work Runs +################# **Audience:** Users who want to know how ``LightningWork`` works. -**Level:** Advanced +**Level:** Basic -**Prereqs**: Level 16+ and read the `Event Loop guide <../glossary/event_loop.html>`_. +**Prerequisite**: Read the :ref:`event_loop` guide. ---- -.. include:: run_work_once_content.rst +********************************************************** +What does it mean to cache the calls of Work's run method? +********************************************************** + +By default, the run method in a LightningWork "remembers" (caches) the input arguments it is getting called with and does not execute again if called with the same arguments again. +In other words, the run method only executes when the input arguments have never been seen before. + +This behavior can be toggled on or off: + +.. code-block:: python + + # Run only when the input arguments change (default) + work = MyWork(cache_calls=True) + + # Run everytime regardless of whether input arguments change or not + work = MyWork(cache_calls=False) + + +To better understand this, imagine you want every day to sequentially download and process some data and then train a model on those data. +As explained in the pre-requisite, the Lightning App runs within an infinite while loop, so the pseudo-code of your application might looks like this: + +.. code-block:: python + + from datetime import datetime + + # Lightning code + while True: # This is the Lightning Event Loop + + # Your code + today = datetime.now().strftime("%D") # '05/25/22' + data_processor.run(today) + train_model.run(data_processor.data) + +In this scenario, you want your components to run ``once`` a day, no more! But your code is running within an infinite loop, how can this even work? +This is where the work's internal caching mechanism comes in. By default, Lightning caches a hash of the input provided to its run method and won't re-execute the method if the same input is provided again. +In the example above, the **data_processor** component run method receives the string **"05/25/22"**. It runs one time and any further execution during the day is skipped until tomorrow is reached and the work run method receives **06/25/22**. This logic applies everyday. +This caching mechanism is inspired from how `React.js Components and Props `_ renders website. Only changes to the inputs re-trigger execution. + +******************************* +How can I verify this behavior? +******************************* + +Here's an example of this behavior with LightningWork: + +.. literalinclude:: ../code_samples/basics/0.py + :language: python + :emphasize-lines: 10, 19 + +And you should see the following by running the code above: + +.. code-block:: console + + $ python example.py + INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view + # After you have clicked `run` on the UI. + I received the following props: args: () kwargs: {'value': 1} + I received the following props: args: () kwargs: {'value': 10} + +As you can see, the intermediate run didn't execute, as we would expected when ``cache_calls=True``. + +************************************************ +What are the implications of turnin caching off? +************************************************ + +By setting ``cache_calls=False``, Lightning won't cache the return value and re-execute the run method on every call. + +.. literalinclude:: ../code_samples/basics/1.py + :diff: ../code_samples/basics/0.py + +.. code-block:: console + + $ python example.py + INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view + # After you have clicked `run` on the UI. + I received the following props: args: () kwargs: {'value': 1} + I received the following props: args: () kwargs: {'value': 1} + I received the following props: args: () kwargs: {'value': 1} + I received the following props: args: () kwargs: {'value': 1} + I received the following props: args: () kwargs: {'value': 1} + I received the following props: args: () kwargs: {'value': 10} + + +Be aware than when setting both ``cache_calls=False`` and ``parallel=False`` to a work, the code after the ``self.work.run()`` is unreachable +as the work continuously execute in a blocking way. + +.. code-block:: python + + from lightning_app import LightningApp, LightningFlow, LightningWork + + + class Flow(LightningFlow): + def __init__(self): + super().__init__() + + self.work = Work(cache_calls=False, parallel=False) + + def run(self): + print("HERE BEFORE") + self.work.run() + print("HERE AFTER") + + + app = LightningApp(Flow()) + +.. code-block:: console + + $ lightning run app app.py + INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view + print("HERE BEFORE") + print("HERE BEFORE") + print("HERE BEFORE") + ... diff --git a/docs/source-app/workflows/run_work_once_content.rst b/docs/source-app/workflows/run_work_once_content.rst deleted file mode 100644 index dbef378135..0000000000 --- a/docs/source-app/workflows/run_work_once_content.rst +++ /dev/null @@ -1,151 +0,0 @@ - -******************************************************** -What caching the calls of Work's run method does for you -******************************************************** - -By default, the run method in a LightningWork (Work) "remembers" (caches) the input arguments it is getting called with and does not execute again if called with the same arguments again. -In other words, the run method only executes when the input arguments have never been seen before. - -You can turn caching on or off: - -.. code-block:: python - - # Run only when the input arguments change (default) - work = MyWork(cache_calls=True) - - # Run everytime regardless of whether input arguments change or not - work = MyWork(cache_calls=False) - -To better understand this, imagine that every day you want to sequentially download and process some data and then train a model on that data. -As explained in the `Event Loop guide <../glossary/event_loop.html>`_, the Lightning App runs within an infinite while loop, so the pseudo-code of your application might looks like this: - -.. code-block:: python - - from datetime import datetime - - # Lightning code - while True: # This is the Lightning Event Loop - - # Your code - today = datetime.now().strftime("%D") # '05/25/22' - data_processor.run(today) - train_model.run(data_processor.data) - -In this scenario, you want your components to run ``once`` a day, and no more than that! But your code is running within an infinite loop, how can this even work? -This is where the Work's internal caching mechanism comes in. By default, Lightning caches a hash of the input provided to its run method and won't re-execute the method if the same input is provided again. -In the example above, the **data_processor** component run method receives the string **"05/25/22"**. It runs one time and any further execution during the day is skipped until tomorrow is reached and the work run method receives **06/25/22**. This logic applies everyday. -This caching mechanism is inspired from how `React.js Components and Props `_ renders websites. Only changes to the inputs re-trigger execution. - -*************** -Caching Example -*************** - -Here's an example of this behavior with LightningWork: - -.. code:: python - :emphasize-lines: 11, 17 - - import lightning as L - - - class ExampleWork(L.LightningWork): - def run(self, *args, **kwargs): - print(f"I received the following props: args: {args} kwargs: {kwargs}") - - - work = ExampleWork() - work.run(value=1) - - # Providing the same value. This won't run as already cached. - work.run(value=1) - work.run(value=1) - work.run(value=1) - work.run(value=1) - - # Changing the provided value. This isn't cached and will run again. - work.run(value=10) - -And you should see the following by running the code above: - -.. code-block:: console - - $ python example.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - # After you have clicked `run` on the UI. - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 10} - -As you can see, the intermediate run didn't execute, as we would expected when ``cache_calls=True``. - -*********************************** -Implications of turning caching off -*********************************** - -By setting ``cache_calls=False``, Lightning won't cache the return value and re-execute the run method on every call. - -.. code:: python - :emphasize-lines: 7 - - from lightning_app import LightningWork - - - class ExampleWork(LightningWork): - def run(self, *args, **kwargs): - print(f"I received the following props: args: {args} kwargs: {kwargs}") - - - work = ExampleWork(cache_calls=False) - work.run(value=1) - - # Providing the same value. This won't run as already cached. - work.run(value=1) - work.run(value=1) - work.run(value=1) - work.run(value=1) - - # Changing the provided value. This isn't cached and will run again. - work.run(value=10) - -.. code-block:: console - - $ python example.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - # After you have clicked `run` on the UI. - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 1} - I received the following props: args: () kwargs: {'value': 10} - -Be aware than when setting both ``cache_calls=False`` and ``parallel=False`` to a work, the code after the ``self.work.run()`` is unreachable -as the work continuously execute in a blocking way. - -.. code-block:: python - :emphasize-lines: 9-10 - - from lightning_app import LightningApp, LightningFlow, LightningWork - - - class Flow(LightningFlow): - def __init__(self): - super().__init__() - - self.work = Work(cache_calls=False, parallel=False) - - def run(self): - print("HERE BEFORE") - self.work.run() - print("HERE AFTER") - - - app = LightningApp(Flow()) - -.. code-block:: console - - $ lightning run app app.py - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - print("HERE BEFORE") - print("HERE BEFORE") - print("HERE BEFORE") - ... diff --git a/docs/source-app/workflows/schedule_apps.rst b/docs/source-app/workflows/schedule_apps.rst index 7b596cd08b..e64c64872e 100644 --- a/docs/source-app/workflows/schedule_apps.rst +++ b/docs/source-app/workflows/schedule_apps.rst @@ -1,5 +1,3 @@ -:orphan: - ################# Schedule App Runs ################# diff --git a/docs/source-app/workflows/share_app.rst b/docs/source-app/workflows/share_app.rst index 87045bde8d..7dd6be7c37 100644 --- a/docs/source-app/workflows/share_app.rst +++ b/docs/source-app/workflows/share_app.rst @@ -30,4 +30,4 @@ Run local: lightning run app app.py -And then, use one of the many guides to `expose a tunnel `_. +then use one of the many guides to `expose a tunnel `. diff --git a/docs/source-app/workflows/share_files_between_components.rst b/docs/source-app/workflows/share_files_between_components.rst index 15108515b3..7292236a18 100644 --- a/docs/source-app/workflows/share_files_between_components.rst +++ b/docs/source-app/workflows/share_files_between_components.rst @@ -62,7 +62,7 @@ To use a file, pass the reference to the file: ****************************** Use a directory - coming soon - ****************************** + *************** TODO ---- @@ -73,32 +73,57 @@ Example: Share a model checkpoint A common workflow in ML is to use a checkpoint created by another component. First, define a component that saves a checkpoint: -.. literalinclude:: ./share_files_between_components/app.py - :lines: -19 +.. code:: python + + import lightning_app as lalit + from lightning_app.storage.path import Path + import torch + import os + + + class ModelTraining(lit.LightningWork): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.model_checkpoints_path = Path("/checkpoints") + + def run(self): + # make fake checkpoints + checkpoint_1 = torch.tensor([0, 1, 2, 3, 4]) + checkpoint_2 = torch.tensor([0, 1, 2, 3, 4]) + torch.save(checkpoint_1, self.model_checkpoints_path + "checkpoint_1.ckpt") + torch.save(checkpoint_2, self.model_checkpoints_path + "checkpoint_2.ckpt") + Next, define a component that needs the checkpoints: -.. literalinclude:: ./share_files_between_components/app.py - :lines: 20-31 +.. code:: python + + class ModelDeploy(lit.LightningWork): + def __init__(self, ckpt_path, *args, **kwargs): + super().__init__() + self.ckpt_path = ckpt_path + + def run(self): + ckpts = os.list_dir(self.ckpt_path) + checkpoint_1 = torch.load(ckpts[0]) + checkpoint_2 = torch.load(ckpts[1]) Link both components via a parent component: -.. literalinclude:: ./share_files_between_components/app.py - :lines: 32- +.. code:: python + + class Root(lit.LightningFlow): + def __init__(self): + super().__init__() + self.train = ModelTraining() + self.deploy = ModelDeploy(ckpt_path=self.train.model_checkpoints_path) + + def run(self): + self.train.run() + self.deploy.run() -Run the app above with the following command: - -.. code-block:: bash - - lightning run app docs/source-app/workflows/share_files_between_components/app.py - -.. code-block:: console - - Your Lightning App is starting. This won't take long. - INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view - Loaded checkpoint_1: tensor([0, 1, 2, 3, 4]) - Loaded checkpoint_2: tensor([0, 1, 2, 3, 4]) + app = lit.LightningApp(Root()) For example, here we save a file on one component and use it in another component: @@ -111,7 +136,7 @@ For example, here we save a file on one component and use it in another componen class ComponentA(LightningWork): def __init__(self): super().__init__() - self.boring_path = None + self.boring_path = Path("boring_file.txt") def run(self): # This should be used as a REFERENCE to the file. diff --git a/docs/source-app/workflows/share_files_between_components/app.py b/docs/source-app/workflows/share_files_between_components/app.py deleted file mode 100644 index c087fc8117..0000000000 --- a/docs/source-app/workflows/share_files_between_components/app.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -import torch - -import lightning as L -from lightning.app.storage.path import Path - - -class ModelTraining(L.LightningWork): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.checkpoints_path = Path("./checkpoints") - - def run(self): - # make fake checkpoints - checkpoint_1 = torch.tensor([0, 1, 2, 3, 4]) - checkpoint_2 = torch.tensor([0, 1, 2, 3, 4]) - os.makedirs(self.checkpoints_path, exist_ok=True) - checkpoint_path = str(self.checkpoints_path / "checkpoint_{}.ckpt") - torch.save(checkpoint_1, str(checkpoint_path).format("1")) - torch.save(checkpoint_2, str(checkpoint_path).format("2")) - - -class ModelDeploy(L.LightningWork): - def __init__(self, ckpt_path, *args, **kwargs): - super().__init__() - self.ckpt_path = ckpt_path - - def run(self): - ckpts = os.listdir(self.ckpt_path) - checkpoint_1 = torch.load(os.path.join(self.ckpt_path, ckpts[0])) - checkpoint_2 = torch.load(os.path.join(self.ckpt_path, ckpts[1])) - print(f"Loaded checkpoint_1: {checkpoint_1}") - print(f"Loaded checkpoint_2: {checkpoint_2}") - - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.train = ModelTraining() - self.deploy = ModelDeploy(ckpt_path=self.train.checkpoints_path) - - def run(self): - self.train.run() - self.deploy.run() - - -app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/test_an_app.rst b/docs/source-app/workflows/test_an_app.rst index c51ae3aa8f..d8e48e2b58 100644 --- a/docs/source-app/workflows/test_an_app.rst +++ b/docs/source-app/workflows/test_an_app.rst @@ -1,5 +1,3 @@ -:orphan: - ########### Test an App ########### diff --git a/docs/source-pytorch/conf.py b/docs/source-pytorch/conf.py index e75c2ed863..6ebb32ce87 100644 --- a/docs/source-pytorch/conf.py +++ b/docs/source-pytorch/conf.py @@ -188,7 +188,7 @@ html_theme_path = [pt_lightning_sphinx_theme.get_html_theme_path()] html_theme_options = { "pytorch_project": "https://pytorchlightning.ai", - "canonical_url": pytorch_lightning.__about__.__docs_url__, + "canonical_url": pytorch_lightning.__docs_url__, "collapse_navigation": False, "display_version": True, "logo_only": False, diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md deleted file mode 100644 index d1edc3553a..0000000000 --- a/src/lightning_app/CHANGELOG.md +++ /dev/null @@ -1,17 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - -## \[0.5.2\] - 2022-MM-DD - -### Added - -- Update the Lightning App docs ([#13537](https://github.com/PyTorchLightning/pytorch-lightning/pull/13537)) - -### Changed - -### Deprecated - -### Fixed