move lightning_app >> lightning/app (#16553)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.actions/assistant.py

@@ -256,33 +256,48 @@ def _retrieve_files(directory: str, *ext: str) -> List[str]:
     return all_files
 
 
-def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]]) -> List[str]:
+def _replace_imports(lines: List[str], mapping: List[Tuple[str, str]], lightning_by: str = "") -> List[str]:
     """Replace imports of standalone package to lightning.
 
     >>> lns = [
+    ...     '"lightning_app"',
     ...     "lightning_app",
+    ...     "lightning_app/",
     ...     "delete_cloud_lightning_apps",
     ...     "from lightning_app import",
     ...     "lightning_apps = []",
     ...     "lightning_app and pytorch_lightning are ours",
     ...     "def _lightning_app():",
-    ...     ":class:`~lightning_app.core.flow.LightningFlow`"
+    ...     ":class:`~lightning_app.core.flow.LightningFlow`",
+    ...     "from lightning import __version__",
+    ...     "@lightning.ai"
     ... ]
     >>> mapping = [("lightning_app", "lightning.app"), ("pytorch_lightning", "lightning.pytorch")]
-    >>> _replace_imports(lns, mapping)  # doctest: +NORMALIZE_WHITESPACE
-    ['lightning.app', 'delete_cloud_lightning_apps', 'from lightning.app import', 'lightning_apps = []',\
-    'lightning.app and lightning.pytorch are ours', 'def _lightning_app():',\
-    ':class:`~lightning.app.core.flow.LightningFlow`']
+    >>> _replace_imports(lns, mapping, lightning_by="lightning_fabric")  # doctest: +NORMALIZE_WHITESPACE
+    ['"lightning.app"', 'lightning.app', 'lightning_app/', 'delete_cloud_lightning_apps', 'from lightning.app import', \
+     'lightning_apps = []', 'lightning.app and lightning.pytorch are ours', 'def _lightning_app():', \
+     ':class:`~lightning.app.core.flow.LightningFlow`', 'from lightning_fabric import __version__', '@lightning.ai']
     """
     out = lines[:]
     for source_import, target_import in mapping:
         for i, ln in enumerate(out):
-            out[i] = re.sub(rf"([^_]|^){source_import}([^_\w]|$)", rf"\1{target_import}\2", ln)
+            out[i] = re.sub(
+                rf"([^_/@]|^){source_import}([^_\w/]|$)",
+                rf"\1{target_import}\2",
+                ln,
+            )
+            if lightning_by:  # in addition, replace base package
+                out[i] = out[i].replace("from lightning import ", f"from {lightning_by} import ")
+                out[i] = out[i].replace("import lightning ", f"import {lightning_by} ")
     return out
 
 
 def copy_replace_imports(
-    source_dir: str, source_imports: List[str], target_imports: List[str], target_dir: Optional[str] = None
+    source_dir: str,
+    source_imports: Sequence[str],
+    target_imports: Sequence[str],
+    target_dir: Optional[str] = None,
+    lightning_by: str = "",
 ) -> None:
     """Copy package content with import adjustments."""
     print(f"Replacing imports: {locals()}")
@@ -312,23 +327,37 @@ def copy_replace_imports(
                 # a binary file, skip
                 print(f"Skipped replacing imports for {fp}")
                 continue
-        lines = _replace_imports(lines, list(zip(source_imports, target_imports)))
+        lines = _replace_imports(lines, list(zip(source_imports, target_imports)), lightning_by=lightning_by)
         os.makedirs(os.path.dirname(fp_new), exist_ok=True)
         with open(fp_new, "w", encoding="utf-8") as fo:
             fo.writelines(lines)
 
 
-def create_mirror_package(source_dir: str, package_mapping: Dict[str, str]) -> None:
+def create_mirror_package(source_dir: str, package_mapping: Dict[str, str], reverse: Sequence[str]) -> None:
     # replace imports and copy the code
     mapping = package_mapping.copy()
     mapping.pop("lightning", None)  # pop this key to avoid replacing `lightning` to `lightning.lightning`
-    for new, previous in mapping.items():
+
+    mapping = {f"lightning.{sp}": sl for sp, sl in mapping.items()}
+    source_imports = mapping.values()
+    target_imports = mapping.keys()
+
+    for pkg_to, pkg_from in mapping.items():
+        if pkg_to.split(".")[-1] in reverse:
+            source_imports_, target_imports_ = target_imports, source_imports
+            pkg_to, pkg_from = pkg_from, pkg_to
+            lightning_by = pkg_from
+        else:
+            source_imports_, target_imports_ = source_imports, target_imports
+            lightning_by = ""
+
         copy_replace_imports(
-            source_dir=os.path.join(source_dir, previous),
+            source_dir=os.path.join(source_dir, pkg_from.replace(".", os.sep)),
             # pytorch_lightning uses lightning_fabric, so we need to replace all imports for all directories
-            source_imports=list(mapping.values()),
-            target_imports=[f"lightning.{new}" for new in mapping],
-            target_dir=os.path.join(source_dir, "lightning", new),
+            source_imports=source_imports_,
+            target_imports=target_imports_,
+            target_dir=os.path.join(source_dir, pkg_to.replace(".", os.sep)),
+            lightning_by=lightning_by,
         )
 
 
@@ -373,12 +402,18 @@ class AssistantCLI:
 
     @staticmethod
     def copy_replace_imports(
-        source_dir: str, source_import: str, target_import: str, target_dir: Optional[str] = None
+        source_dir: str,
+        source_import: str,
+        target_import: str,
+        target_dir: Optional[str] = None,
+        lightning_by: str = "",
     ) -> None:
         """Copy package content with import adjustments."""
         source_imports = source_import.strip().split(",")
         target_imports = target_import.strip().split(",")
-        copy_replace_imports(source_dir, source_imports, target_imports, target_dir=target_dir)
+        copy_replace_imports(
+            source_dir, source_imports, target_imports, target_dir=target_dir, lightning_by=lightning_by
+        )
 
 
 if __name__ == "__main__":

.azure/app-cloud-e2e.yml

@@ -29,7 +29,8 @@ pr:
     include:
       - ".actions/**"
       - ".azure/app-cloud-e2e.yml"
-      - "src/lightning_app/**"
+      - "src/lightning/app/**"
+      - "src/lightning_app/*"
       - "requirements/app/**"
       - "tests/integrations_app/**"
       - "setup.py"

.github/checkgroup.yml

@@ -221,7 +221,8 @@ subprojects:
     paths:
       - ".actions/**"
       - ".github/workflows/ci-tests-app.yml"
-      - "src/lightning_app/**"
+      - "src/lightning/app/**"
+      - "src/lightning_app/*"
       - "tests/tests_app/**"
       - "requirements/app/**"
       - "setup.py"
@@ -243,7 +244,8 @@ subprojects:
     paths:
       - ".actions/**"
       - ".github/workflows/ci-examples-app.yml"
-      - "src/lightning_app/**"
+      - "src/lightning/app/**"
+      - "src/lightning_app/*"
       - "tests/integrations_app/**"
       - "!tests/integrations_app/flagship/**"
       - "examples/app_*/**"
@@ -285,7 +287,8 @@ subprojects:
     paths:
       - ".actions/**"
       - ".azure/app-cloud-e2e.yml"
-      - "src/lightning_app/**"
+      - "src/lightning/app/**"
+      - "src/lightning_app/*"
       - "requirements/app/**"
       - "tests/integrations_app/**"
       - "!tests/integrations_app/flagship/**"
@@ -299,7 +302,8 @@ subprojects:
   - id: "lightning_app: Docs"
     paths:
       - ".actions/**"
-      - "src/lightning_app/**"
+      - "src/lightning/app/**"
+      - "src/lightning_app/*"
       - "docs/source-app/**"
       - ".github/workflows/docs-checks.yml"
       - "requirements/docs.txt"

.github/labeler.yml

@@ -1,5 +1,6 @@
 app:
-  - 'src/lightning_app/**'
+  - 'src/lightning/app/**'
+  - "src/lightning_app/*"
   - 'tests/tests_app/**'
   - 'tests/integrations_app/**'
   - 'tests/integrations_app_examples/**'

.github/workflows/ci-examples-app.yml

@@ -10,7 +10,8 @@ on:
     paths:
       - ".actions/**"
       - ".github/workflows/ci-examples-app.yml"
-      - "src/lightning_app/**"
+      - "src/lightning/app/**"
+      - "src/lightning_app/*"
       - "tests/integrations_app/**"
       - "!tests/integrations_app/flagship/**"
       - "examples/app_*/**"
@@ -90,15 +91,19 @@ jobs:
       - name: Install Yarn
         run: npm install -g yarn
 
-      - name: Adjust tests
+      - name: Adjust imports -> Lit
         if: ${{ matrix.pkg-name == 'lightning' }}
         run: |
           python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
             --source_import="lightning_app" --target_import="lightning.app"
 
-      - name: Adjust examples
+      - name: Adjust imports -> App
         if: ${{ matrix.pkg-name != 'lightning' }}
         run: |
+          python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
+            --source_import="lightning.app" \
+            --target_import="lightning_app" \
+            --lightning_by="lightning_app"
           python .actions/assistant.py copy_replace_imports --source_dir="./examples" \
             --source_import="lightning.app,lightning" \
             --target_import="lightning_app,lightning_app"

.github/workflows/ci-tests-app.yml

@@ -10,7 +10,8 @@ on:
     paths:
       - ".actions/**"
       - ".github/workflows/ci-tests-app.yml"
-      - "src/lightning_app/**"
+      - "src/lightning/app/**"
+      - "src/lightning_app/*"
       - "tests/tests_app/**"
       - "requirements/app/**"
       - "setup.py"
@@ -90,19 +91,24 @@ jobs:
       - name: Install Yarn
         run: npm install -g yarn
 
-      - name: Adjust tests
+      - name: Adjust imports -> Lit
         if: ${{ matrix.pkg-name == 'lightning' }}
         run: |
           python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
             --source_import="lightning_app,lightning_fabric,pytorch_lightning" \
             --target_import="lightning.app,lightning.fabric,lightning.pytorch"
 
-      - name: Adjust examples
+      - name: Adjust imports -> App
         if: ${{ matrix.pkg-name != 'lightning' }}
         run: |
+          python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
+            --source_import="lightning.app" \
+            --target_import="lightning_app" \
+            --lightning_by="lightning_app"
           python .actions/assistant.py copy_replace_imports --source_dir="./examples" \
-            --source_import="lightning.app,lightning" \
-            --target_import="lightning_app,lightning_app"
+            --source_import="lightning.app" \
+            --target_import="lightning_app" \
+            --lightning_by="lightning_app"
 
       - name: Switch coverage scope
         run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV

.gitignore

@@ -49,7 +49,10 @@ wheels/
 .installed.cfg
 *.egg
 src/*/version.info
-src/lightning/app/
+src/lightning_app/
+!src/lightning_app/__*__.py
+!src/lightning_app/MANIFEST.in
+!src/lightning_app/README.md
 src/lightning/fabric/
 src/lightning/pytorch/
 

.pre-commit-config.yaml

@@ -98,8 +98,8 @@ repos:
           - mdformat_frontmatter
         exclude: |
           (?x)^(
+              src/lightning/app/CHANGELOG.md|
               src/pytorch_lightning/CHANGELOG.md|
-              src/lightning_app/CHANGELOG.md|
               src/lightning_fabric/CHANGELOG.md
           )$
 

pyproject.toml

@@ -68,16 +68,17 @@ max-complexity = 10
 
 [tool.mypy]
 files = [
+    "src/lightning",
     "src/pytorch_lightning",
     "src/lightning_fabric",
     "src/lightning_app",
 ]
 # This section is for folders with "-" as they are not valid python modules
 exclude = [
-    "src/lightning_app/cli/app-template",
-    "src/lightning_app/cli/component-template",
-    "src/lightning_app/cli/pl-app-template",
-    "src/lightning_app/cli/react-ui-template",
+    "src/lightning/app/cli/app-template",
+    "src/lightning/app/cli/component-template",
+    "src/lightning/app/cli/pl-app-template",
+    "src/lightning/app/cli/react-ui-template",
 ]
 install_types = "True"
 non_interactive = "True"
@@ -99,94 +100,98 @@ warn_no_return = "False"
 # the list can be generated with:
 # mypy --no-error-summary 2>&1 | tr ':' ' ' | awk '{print $1}' | sort | uniq | sed 's/\.py//g; s|src/||g;  s|\/|\.|g' | xargs -I {} echo '"{}",'
 module = [
-    "lightning_app.components.multi_node.fabric",
-    "lightning_app.components.multi_node.base",
-    "lightning_app.components.multi_node.pytorch_spawn",
-    "lightning_app.components.multi_node.trainer",
-    "lightning_app.api.http_methods",
-    "lightning_app.api.request_types",
-    "lightning_app.cli.commands.app_commands",
-    "lightning_app.cli.commands.connection",
-    "lightning_app.cli.commands.lightning_cli",
-    "lightning_app.cli.commands.cmd_install",
-    "lightning_app.cli.cmd_install",
-    "lightning_app.components.database.client",
-    "lightning_app.components.database.server",
-    "lightning_app.components.database.utilities",
-    "lightning_app.components.python.popen",
-    "lightning_app.components.python.tracer",
-    "lightning_app.components.serve.gradio_server",
-    "lightning_app.components.serve.serve",
-    "lightning_app.components.serve.streamlit",
-    "lightning_app.components.serve.types.image",
-    "lightning_app.components.serve.types.type",
-    "lightning_app.components.serve.python_server",
-    "lightning_app.components.serve.auto_scaler",
-    "lightning_app.components.training",
-    "lightning_app.core.api",
-    "lightning_app.core.app",
-    "lightning_app.core.flow",
-    "lightning_app.core.queues",
-    "lightning_app.core.work",
-    "lightning_app.frontend.panel.app_state_comm",
-    "lightning_app.frontend.panel.app_state_watcher",
-    "lightning_app.frontend.panel.panel_frontend",
-    "lightning_app.frontend.panel.panel_serve_render_fn",
-    "lightning_app.frontend.stream_lit",
-    "lightning_app.frontend.streamlit_base",
-    "lightning_app.frontend.utils",
-    "lightning_app.frontend.web",
-    "lightning_app.perf.pdb",
-    "lightning_app.runners.backends.__init__",
-    "lightning_app.runners.backends.backend",
-    "lightning_app.runners.backends.cloud",
-    "lightning_app.runners.backends.docker",
-    "lightning_app.runners.backends.mp_process",
-    "lightning_app.runners.cloud",
-    "lightning_app.runners.multiprocess",
-    "lightning_app.runners.runtime",
-    "lightning_app.source_code.copytree",
-    "lightning_app.source_code.hashing",
-    "lightning_app.source_code.local",
-    "lightning_app.source_code.tar",
-    "lightning_app.source_code.uploader",
-    "lightning_app.storage.copier",
-    "lightning_app.storage.drive",
-    "lightning_app.storage.orchestrator",
-    "lightning_app.storage.path",
-    "lightning_app.storage.payload",
-    "lightning_app.structures.dict",
-    "lightning_app.structures.list",
-    "lightning_app.testing.helpers",
-    "lightning_app.testing.testing",
-    "lightning_app.utilities.app_helpers",
-    "lightning_app.utilities.app_logs",
-    "lightning_app.utilities.cli_helpers",
-    "lightning_app.utilities.cloud",
-    "lightning_app.utilities.cluster_logs",
-    "lightning_app.utilities.commands.base",
-    "lightning_app.utilities.component",
-    "lightning_app.utilities.enum",
-    "lightning_app.utilities.exceptions",
-    "lightning_app.utilities.git",
-    "lightning_app.utilities.imports",
-    "lightning_app.utilities.introspection",
-    "lightning_app.utilities.layout",
-    "lightning_app.utilities.load_app",
-    "lightning_app.utilities.log_helpers",
-    "lightning_app.utilities.login",
-    "lightning_app.utilities.name_generator",
-    "lightning_app.utilities.network",
-    "lightning_app.utilities.openapi",
-    "lightning_app.utilities.packaging.cloud_compute",
-    "lightning_app.utilities.packaging.docker",
-    "lightning_app.utilities.packaging.lightning_utils",
-    "lightning_app.utilities.port",
-    "lightning_app.utilities.proxies",
-    "lightning_app.utilities.scheduler",
-    "lightning_app.utilities.state",
-    "lightning_app.utilities.tracer",
-    "lightning_app.utilities.tree",
+    "lightning.app.components.multi_node.fabric",
+    "lightning.app.components.multi_node.base",
+    "lightning.app.components.multi_node.pytorch_spawn",
+    "lightning.app.components.multi_node.trainer",
+    "lightning.app.api.http_methods",
+    "lightning.app.api.request_types",
+    "lightning.app.cli.commands.app_commands",
+    "lightning.app.cli.commands.connection",
+    "lightning.app.cli.commands.lightning_cli",
+    "lightning.app.cli.commands.cmd_install",
+    "lightning.app.cli.cmd_install",
+    "lightning.app.components.database.client",
+    "lightning.app.components.database.server",
+    "lightning.app.components.database.utilities",
+    "lightning.app.components.python.popen",
+    "lightning.app.components.python.tracer",
+    "lightning.app.components.serve.gradio_server",
+    "lightning.app.components.serve.serve",
+    "lightning.app.components.serve.streamlit",
+    "lightning.app.components.serve.types.image",
+    "lightning.app.components.serve.types.type",
+    "lightning.app.components.serve.python_server",
+    "lightning.app.components.serve.auto_scaler",
+    "lightning.app.components.training",
+    "lightning.app.core.api",
+    "lightning.app.core.app",
+    "lightning.app.core.flow",
+    "lightning.app.core.queues",
+    "lightning.app.core.work",
+    "lightning.app.frontend.panel.app_state_comm",
+    "lightning.app.frontend.panel.app_state_watcher",
+    "lightning.app.frontend.panel.panel_frontend",
+    "lightning.app.frontend.panel.panel_serve_render_fn",
+    "lightning.app.frontend.stream_lit",
+    "lightning.app.frontend.streamlit_base",
+    "lightning.app.frontend.utils",
+    "lightning.app.frontend.web",
+    "lightning.app.perf.pdb",
+    "lightning.app.runners.backends.__init__",
+    "lightning.app.runners.backends.backend",
+    "lightning.app.runners.backends.cloud",
+    "lightning.app.runners.backends.docker",
+    "lightning.app.runners.backends.mp_process",
+    "lightning.app.runners.cloud",
+    "lightning.app.runners.multiprocess",
+    "lightning.app.runners.runtime",
+    "lightning.app.source_code.copytree",
+    "lightning.app.source_code.hashing",
+    "lightning.app.source_code.local",
+    "lightning.app.source_code.tar",
+    "lightning.app.source_code.uploader",
+    "lightning.app.storage.copier",
+    "lightning.app.storage.drive",
+    "lightning.app.storage.orchestrator",
+    "lightning.app.storage.path",
+    "lightning.app.storage.payload",
+    "lightning.app.structures.dict",
+    "lightning.app.structures.list",
+    "lightning.app.testing.helpers",
+    "lightning.app.testing.testing",
+    "lightning.app.utilities.app_helpers",
+    "lightning.app.utilities.app_logs",
+    "lightning.app.utilities.cli_helpers",
+    "lightning.app.utilities.cloud",
+    "lightning.app.utilities.cluster_logs",
+    "lightning.app.utilities.commands.base",
+    "lightning.app.utilities.component",
+    "lightning.app.utilities.enum",
+    "lightning.app.utilities.exceptions",
+    "lightning.app.utilities.git",
+    "lightning.app.utilities.imports",
+    "lightning.app.utilities.introspection",
+    "lightning.app.utilities.layout",
+    "lightning.app.utilities.load_app",
+    "lightning.app.utilities.log_helpers",
+    "lightning.app.utilities.login",
+    "lightning.app.utilities.name_generator",
+    "lightning.app.utilities.network",
+    "lightning.app.utilities.openapi",
+    "lightning.app.utilities.packaging.cloud_compute",
+    "lightning.app.utilities.packaging.docker",
+    "lightning.app.utilities.packaging.lightning_utils",
+    "lightning.app.utilities.port",
+    "lightning.app.utilities.proxies",
+    "lightning.app.utilities.scheduler",
+    "lightning.app.utilities.state",
+    "lightning.app.utilities.tracer",
+    "lightning.app.utilities.tree",
+    "lightning.store.save",
+    "lightning.store.utils",
+    "lightning.store.authentication",
+    "lightning.store.cloud_api",
 ]
 ignore_errors = "True"
 

setup.py

@@ -45,7 +45,7 @@ import os
 import tempfile
 from importlib.util import module_from_spec, spec_from_file_location
 from types import ModuleType
-from typing import Generator, Optional
+from typing import Generator, Mapping, Optional
 
 import setuptools
 import setuptools.command.egg_info
@@ -57,6 +57,8 @@ _PACKAGE_MAPPING = {
     "app": "lightning_app",
     "fabric": "lightning_fabric",
 }
+# TODO: drop this reverse list when all packages are moved
+_MIRROR_PACKAGE_REVERSED = ("app",)
 # https://packaging.python.org/guides/single-sourcing-package-version/
 # http://blog.ionelmc.ro/2014/05/25/python-packaging/
 _PATH_ROOT = os.path.dirname(__file__)
@@ -83,11 +85,10 @@ def _named_temporary_file(directory: Optional[str] = None) -> str:
 
 
 @contextlib.contextmanager
-def _set_manifest_path(manifest_dir: str, aggregate: bool = False) -> Generator:
+def _set_manifest_path(manifest_dir: str, aggregate: bool = False, mapping: Mapping = _PACKAGE_MAPPING) -> Generator:
     if aggregate:
         # aggregate all MANIFEST.in contents into a single temporary file
         manifest_path = _named_temporary_file(manifest_dir)
-        mapping = _PACKAGE_MAPPING.copy()
         lines = ["include src/lightning/version.info\n", "include requirements/base.txt\n"]
         # load manifest and aggregated all manifests
         for pkg in mapping.values():
@@ -139,11 +140,12 @@ if __name__ == "__main__":
                 f"Unexpected package name: {_PACKAGE_NAME}. Possible choices are: {list(_PACKAGE_MAPPING)}"
             )
         package_to_install = _PACKAGE_MAPPING.get(_PACKAGE_NAME, "lightning")
-        if package_to_install == "lightning":  # install everything
+        if package_to_install == "lightning":
             # merge all requirements files
             assistant._load_aggregate_requirements(_PATH_REQUIRE, _FREEZE_REQUIREMENTS)
+        if package_to_install == "lightning" or _PACKAGE_NAME in _MIRROR_PACKAGE_REVERSED:  # install everything
             # replace imports and copy the code
-            assistant.create_mirror_package(_PATH_SRC, _PACKAGE_MAPPING)
+            assistant.create_mirror_package(_PATH_SRC, _PACKAGE_MAPPING, reverse=_MIRROR_PACKAGE_REVERSED)
     else:
         assert len(local_pkgs) > 0
         # PL as a package is distributed together with Fabric, so in such case there are more than one candidate

src/lightning/app/CHANGELOG.md

@@ -16,8 +16,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - Changed the default `LightningClient(retry=False)` to `retry=True` ([#16382](https://github.com/Lightning-AI/lightning/pull/16382))
 
+
 - Add support for async predict method in PythonServer and remove torch context ([#16453](https://github.com/Lightning-AI/lightning/pull/16453))
 
+
 - Renamed `lightning.app.components.LiteMultiNode` to `lightning.app.components.FabricMultiNode` ([#16505](https://github.com/Lightning-AI/lightning/pull/16505))
 
 
@@ -42,13 +44,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - Added a possibility to set up basic authentication for Lightning apps ([#16105](https://github.com/Lightning-AI/lightning/pull/16105))
 
-
 ### Changed
 
 - The LoadBalancer now uses internal ip + port instead of URL exposed ([#16119](https://github.com/Lightning-AI/lightning/pull/16119))
-- Added support for logging in different trainer stages with  `DeviceStatsMonitor`
-([#16002](https://github.com/Lightning-AI/lightning/pull/16002))
-- Changed `lightning_app.components.serve.gradio` to  `lightning_app.components.serve.gradio_server` ([#16201](https://github.com/Lightning-AI/lightning/pull/16201))
+- Added support for logging in different trainer stages with  `DeviceStatsMonitor` ([#16002](https://github.com/Lightning-AI/lightning/pull/16002))
+- Changed `lightning.app.components.serve.gradio` to  `lightning.app.components.serve.gradio_server` ([#16201](https://github.com/Lightning-AI/lightning/pull/16201))
 - Made cluster creation/deletion async by default ([#16185](https://github.com/Lightning-AI/lightning/pull/16185))
 - Expose `LightningFlow.stop` method to stop the flow similar to works ([##16378](https://github.com/Lightning-AI/lightning/pull/16378))
 

src/lightning/app/__init__.py

@@ -0,0 +1,49 @@
+"""Root package info."""
+import logging
+import os
+
+from lightning_utilities.core.imports import module_available
+
+_root_logger = logging.getLogger()
+_logger = logging.getLogger(__name__)
+_logger.setLevel(logging.INFO)
+
+_console = logging.StreamHandler()
+_console.setLevel(logging.INFO)
+
+formatter = logging.Formatter("%(levelname)s: %(message)s")
+_console.setFormatter(formatter)
+
+# if root logger has handlers, propagate messages up and let root logger process them,
+# otherwise use our own handler
+if not _root_logger.hasHandlers():
+    _logger.addHandler(_console)
+    _logger.propagate = False
+
+
+from lightning.app import components  # noqa: E402, F401
+
+if os.path.isfile(os.path.join(os.path.dirname(__file__), "__about__.py")):
+    from lightning.app.__about__ import *  # noqa: F401, F403
+if "__version__" not in locals():
+    if os.path.isfile(os.path.join(os.path.dirname(__file__), "__version__.py")):
+        from lightning.app.__version__ import version as __version__
+    elif module_available("lightning"):
+        from lightning import __version__  # noqa: F401
+
+from lightning.app.core.app import LightningApp  # noqa: E402
+from lightning.app.core.flow import LightningFlow  # noqa: E402
+from lightning.app.core.work import LightningWork  # noqa: E402
+from lightning.app.perf import pdb  # noqa: E402
+from lightning.app.utilities.packaging.build_config import BuildConfig  # noqa: E402
+from lightning.app.utilities.packaging.cloud_compute import CloudCompute  # noqa: E402
+
+if module_available("lightning.app.components.demo"):
+    from lightning.app.components import demo  # noqa: F401
+
+__package_name__ = "lightning.app".split(".")[0]
+
+_PACKAGE_ROOT = os.path.dirname(__file__)
+_PROJECT_ROOT = os.path.dirname(os.path.dirname(_PACKAGE_ROOT))
+
+__all__ = ["LightningApp", "LightningFlow", "LightningWork", "BuildConfig", "CloudCompute", "pdb"]

src/lightning/app/api/__init__.py

@@ -1,4 +1,4 @@
-from lightning_app.api.http_methods import Delete, Get, Post, Put
+from lightning.app.api.http_methods import Delete, Get, Post, Put
 
 __all__ = [
     "Delete",

src/lightning/app/api/http_methods.py

@@ -25,8 +25,8 @@ from uuid import uuid4
 from fastapi import FastAPI, HTTPException, Request, status
 from lightning_utilities.core.apply_func import apply_to_collection
 
-from lightning_app.api.request_types import _APIRequest, _CommandRequest, _RequestResponse
-from lightning_app.utilities.app_helpers import Logger
+from lightning.app.api.request_types import _APIRequest, _CommandRequest, _RequestResponse
+from lightning.app.utilities.app_helpers import Logger
 
 logger = Logger(__name__)
 

src/lightning/app/cli/app-template/.gitignore

@@ -130,7 +130,7 @@ infra
 data
 coverage.*
 # Frontend build artifacts
-*lightning_app/ui*
+*lightning/app/ui*
 gradio_cached_examples
 /docs/source-app/api_reference/generated/*
 examples/my_own_leaderboard/submissions/*
@@ -146,11 +146,11 @@ examples/quick_start
 examples/template_react_ui/*
 examples/template_react_ui
 # Ignore external components
-lightning_app/components/*
-!lightning_app/components/python
-!lightning_app/components/serve
-!lightning_app/components/__init__.py
-!lightning_app/components/README.md
+lightning/app/components/*
+!lightning/app/components/python
+!lightning/app/components/serve
+!lightning/app/components/__init__.py
+!lightning/app/components/README.md
 train_script.py
 *return_values*
 scratch

src/lightning/app/cli/app-template/tests/test_placeholdername_app.py

@@ -10,7 +10,7 @@ import io
 import os
 from contextlib import redirect_stdout
 
-from lightning_app.testing.testing import application_testing, LightningTestApp
+from lightning.app.testing.testing import application_testing, LightningTestApp
 
 
 class LightningAppTestInt(LightningTestApp):

src/lightning/app/cli/cmd_apps.py

@@ -27,9 +27,9 @@ from rich.console import Console
 from rich.table import Table
 from rich.text import Text
 
-from lightning_app.cli.core import Formatable
-from lightning_app.utilities.cloud import _get_project
-from lightning_app.utilities.network import LightningClient
+from lightning.app.cli.core import Formatable
+from lightning.app.utilities.cloud import _get_project
+from lightning.app.utilities.network import LightningClient
 
 
 class _AppManager:

src/lightning/app/cli/cmd_clusters.py

@@ -42,10 +42,10 @@ from rich.console import Console
 from rich.table import Table
 from rich.text import Text
 
-from lightning_app.cli.core import Formatable
-from lightning_app.utilities.cloud import _get_project
-from lightning_app.utilities.network import LightningClient
-from lightning_app.utilities.openapi import create_openapi_object, string2dict
+from lightning.app.cli.core import Formatable
+from lightning.app.utilities.cloud import _get_project
+from lightning.app.utilities.network import LightningClient
+from lightning.app.utilities.openapi import create_openapi_object, string2dict
 
 MAX_CLUSTER_WAIT_TIME = 5400
 

src/lightning/app/cli/cmd_init.py

@@ -17,7 +17,7 @@ import re
 import shutil
 from typing import List, Tuple
 
-from lightning_app.utilities.app_helpers import Logger
+from lightning.app.utilities.app_helpers import Logger
 
 logger = Logger(__name__)
 
@@ -139,7 +139,7 @@ def component(component_name: str) -> None:
     ⚡ Use the component inside an app: ⚡
 
     from {name_for_files} import TemplateComponent
-    import lightning_app as la
+    import lightning.app as la
 
     class LitApp(la.LightningFlow):
         def __init__(self) -> None:

src/lightning/app/cli/cmd_install.py

@@ -23,8 +23,8 @@ import click
 import requests
 from packaging.version import Version
 
-from lightning_app.core.constants import LIGHTNING_APPS_PUBLIC_REGISTRY, LIGHTNING_COMPONENT_PUBLIC_REGISTRY
-from lightning_app.utilities.app_helpers import Logger
+from lightning.app.core.constants import LIGHTNING_APPS_PUBLIC_REGISTRY, LIGHTNING_COMPONENT_PUBLIC_REGISTRY
+from lightning.app.utilities.app_helpers import Logger
 
 logger = Logger(__name__)
 

src/lightning/app/cli/cmd_pl_init.py

@@ -31,7 +31,7 @@ from rich.status import Status
 from rich.text import Text
 from rich.tree import Tree
 
-import lightning_app
+import lightning.app
 
 _REPORT_HELP_TEXTS = {
     "core": "Important files for the app such as various components",
@@ -89,7 +89,7 @@ def pl_app(source_dir: str, script_path: str, name: str, overwrite: bool) -> Non
         else:
             shutil.rmtree(destination)
 
-    template_dir = Path(lightning_app.cli.__file__).parent / "pl-app-template"
+    template_dir = Path(lightning.app.cli.__file__).parent / "pl-app-template"
 
     with Status("[bold green]Copying app files"):
         shutil.copytree(template_dir, destination, ignore=shutil.ignore_patterns("node_modules", "build"))

src/lightning/app/cli/cmd_react_ui_init.py

@@ -17,7 +17,7 @@ import re
 import shutil
 import subprocess
 
-from lightning_app.utilities.app_helpers import Logger
+from lightning.app.utilities.app_helpers import Logger
 
 logger = Logger(__name__)
 

src/lightning/app/cli/cmd_ssh_keys.py

@@ -21,8 +21,8 @@ from lightning_cloud.openapi import V1CreateSSHPublicKeyRequest, V1SSHPublicKey
 from rich.console import Console
 from rich.table import Table
 
-from lightning_app.cli.core import Formatable
-from lightning_app.utilities.network import LightningClient
+from lightning.app.cli.core import Formatable
+from lightning.app.utilities.network import LightningClient
 
 
 class _SSHKeyList(Formatable):

src/lightning/app/cli/commands/app_commands.py

@@ -4,14 +4,14 @@ from typing import Dict, Optional
 
 import requests
 
-from lightning_app.cli.commands.connection import (
+from lightning.app.cli.commands.connection import (
     _clean_lightning_connection,
     _install_missing_requirements,
     _resolve_command_path,
 )
-from lightning_app.utilities.cli_helpers import _LightningAppOpenAPIRetriever
-from lightning_app.utilities.commands.base import _download_command
-from lightning_app.utilities.enum import OpenAPITags
+from lightning.app.utilities.cli_helpers import _LightningAppOpenAPIRetriever
+from lightning.app.utilities.commands.base import _download_command
+from lightning.app.utilities.enum import OpenAPITags
 
 
 def _is_running_help(argv) -> bool:

src/lightning/app/cli/commands/connection.py

@@ -10,11 +10,11 @@ import psutil
 from lightning_utilities.core.imports import package_available
 from rich.progress import Progress
 
-from lightning_app.utilities.cli_helpers import _LightningAppOpenAPIRetriever
-from lightning_app.utilities.cloud import _get_project
-from lightning_app.utilities.enum import OpenAPITags
-from lightning_app.utilities.log import get_logfile
-from lightning_app.utilities.network import LightningClient
+from lightning.app.utilities.cli_helpers import _LightningAppOpenAPIRetriever
+from lightning.app.utilities.cloud import _get_project
+from lightning.app.utilities.enum import OpenAPITags
+from lightning.app.utilities.log import get_logfile
+from lightning.app.utilities.network import LightningClient
 
 _HOME = os.path.expanduser("~")
 _PPID = os.getenv("LIGHTNING_CONNECT_PPID", str(psutil.Process(os.getpid()).ppid()))
@@ -44,7 +44,7 @@ def connect(app_name_or_id: str):
     # once done, disconnect and go back to the standard lightning CLI commands
     lightning disconnect
     """
-    from lightning_app.utilities.commands.base import _download_command
+    from lightning.app.utilities.commands.base import _download_command
 
     _clean_lightning_connection()
 

src/lightning/app/cli/commands/logs.py

@@ -4,11 +4,11 @@ import click
 import rich
 from rich.color import ANSI_COLOR_NAMES
 
-from lightning_app.utilities.app_helpers import Logger
-from lightning_app.utilities.app_logs import _app_logs_reader
-from lightning_app.utilities.cloud import _get_project
-from lightning_app.utilities.logs_socket_api import _LightningLogsSocketAPI
-from lightning_app.utilities.network import LightningClient
+from lightning.app.utilities.app_helpers import Logger
+from lightning.app.utilities.app_logs import _app_logs_reader
+from lightning.app.utilities.cloud import _get_project
+from lightning.app.utilities.logs_socket_api import _LightningLogsSocketAPI
+from lightning.app.utilities.network import LightningClient
 
 logger = Logger(__name__)
 

src/lightning/app/cli/component-template/.gitignore

@@ -130,7 +130,7 @@ infra
 data
 coverage.*
 # Frontend build artifacts
-*lightning_app/ui*
+*lightning/app/ui*
 gradio_cached_examples
 /docs/source-app/api_reference/generated/*
 examples/my_own_leaderboard/submissions/*
@@ -146,11 +146,11 @@ examples/quick_start
 examples/template_react_ui/*
 examples/template_react_ui
 # Ignore external components
-lightning_app/components/*
-!lightning_app/components/python
-!lightning_app/components/serve
-!lightning_app/components/__init__.py
-!lightning_app/components/README.md
+lightning/app/components/*
+!lightning/app/components/python
+!lightning/app/components/serve
+!lightning/app/components/__init__.py
+!lightning/app/components/README.md
 train_script.py
 *return_values*
 scratch

src/lightning/app/cli/lightning_cli.py

@@ -27,40 +27,40 @@ from lightning_cloud.openapi.rest import ApiException
 from lightning_utilities.core.imports import RequirementCache
 from requests.exceptions import ConnectionError
 
-import lightning_app.core.constants as constants
-from lightning_app import __version__ as ver
-from lightning_app.cli import cmd_init, cmd_install, cmd_pl_init, cmd_react_ui_init
-from lightning_app.cli.cmd_apps import _AppManager
-from lightning_app.cli.cmd_clusters import AWSClusterManager
-from lightning_app.cli.commands.app_commands import _run_app_command
-from lightning_app.cli.commands.connection import (
+import lightning.app.core.constants as constants
+from lightning.app import __version__ as ver
+from lightning.app.cli import cmd_init, cmd_install, cmd_pl_init, cmd_react_ui_init
+from lightning.app.cli.cmd_apps import _AppManager
+from lightning.app.cli.cmd_clusters import AWSClusterManager
+from lightning.app.cli.commands.app_commands import _run_app_command
+from lightning.app.cli.commands.connection import (
     _list_app_commands,
     _retrieve_connection_to_an_app,
     connect,
     disconnect,
 )
-from lightning_app.cli.commands.logs import logs
-from lightning_app.cli.lightning_cli_create import create
-from lightning_app.cli.lightning_cli_delete import delete
-from lightning_app.cli.lightning_cli_list import get_list
-from lightning_app.core.constants import DEBUG, ENABLE_APP_COMMENT_COMMAND_EXECUTION, get_lightning_cloud_url
-from lightning_app.runners.cloud import CloudRuntime
-from lightning_app.runners.runtime import dispatch
-from lightning_app.runners.runtime_type import RuntimeType
-from lightning_app.utilities.app_commands import run_app_commands
-from lightning_app.utilities.app_helpers import Logger
-from lightning_app.utilities.cli_helpers import (
+from lightning.app.cli.commands.logs import logs
+from lightning.app.cli.lightning_cli_create import create
+from lightning.app.cli.lightning_cli_delete import delete
+from lightning.app.cli.lightning_cli_list import get_list
+from lightning.app.core.constants import DEBUG, ENABLE_APP_COMMENT_COMMAND_EXECUTION, get_lightning_cloud_url
+from lightning.app.runners.cloud import CloudRuntime
+from lightning.app.runners.runtime import dispatch
+from lightning.app.runners.runtime_type import RuntimeType
+from lightning.app.utilities.app_commands import run_app_commands
+from lightning.app.utilities.app_helpers import Logger
+from lightning.app.utilities.cli_helpers import (
     _arrow_time_callback,
     _check_environment_and_redirect,
     _check_version_and_upgrade,
     _format_input_env_variables,
 )
-from lightning_app.utilities.cluster_logs import _cluster_logs_reader
-from lightning_app.utilities.exceptions import _ApiExceptionHandler, LogLinesLimitExceeded
-from lightning_app.utilities.login import Auth
-from lightning_app.utilities.logs_socket_api import _ClusterLogsSocketAPI
-from lightning_app.utilities.network import LightningClient
-from lightning_app.utilities.port import _find_lit_app_port
+from lightning.app.utilities.cluster_logs import _cluster_logs_reader
+from lightning.app.utilities.exceptions import _ApiExceptionHandler, LogLinesLimitExceeded
+from lightning.app.utilities.login import Auth
+from lightning.app.utilities.logs_socket_api import _ClusterLogsSocketAPI
+from lightning.app.utilities.network import LightningClient
+from lightning.app.utilities.port import _find_lit_app_port
 
 logger = Logger(__name__)
 
@@ -391,7 +391,7 @@ def run_app(
 
 if RequirementCache("lightning-fabric>=1.9.0") or RequirementCache("lightning>=1.9.0"):
     # note it is automatically replaced to `from lightning.fabric.cli` when building monolithic/mirror package
-    from lightning_fabric.cli import _run_model
+    from lightning.fabric.cli import _run_model
 
     run.add_command(_run_model)
 

src/lightning/app/cli/lightning_cli_create.py

@@ -19,8 +19,8 @@ from typing import Any, Optional, Union
 import click
 from lightning_cloud.openapi.rest import ApiException
 
-from lightning_app.cli.cmd_clusters import _check_cluster_id_is_valid, AWSClusterManager
-from lightning_app.cli.cmd_ssh_keys import _SSHKeyManager
+from lightning.app.cli.cmd_clusters import _check_cluster_id_is_valid, AWSClusterManager
+from lightning.app.cli.cmd_ssh_keys import _SSHKeyManager
 
 
 @click.group("create")

src/lightning/app/cli/lightning_cli_delete.py

@@ -20,9 +20,9 @@ from inquirer.themes import GreenPassion
 from lightning_cloud.openapi import V1ClusterType
 from rich.console import Console
 
-from lightning_app.cli.cmd_apps import _AppManager
-from lightning_app.cli.cmd_clusters import AWSClusterManager
-from lightning_app.cli.cmd_ssh_keys import _SSHKeyManager
+from lightning.app.cli.cmd_apps import _AppManager
+from lightning.app.cli.cmd_clusters import AWSClusterManager
+from lightning.app.cli.cmd_ssh_keys import _SSHKeyManager
 
 
 @click.group("delete")

src/lightning/app/cli/lightning_cli_list.py

@@ -16,9 +16,9 @@ from typing import Any
 
 import click
 
-from lightning_app.cli.cmd_apps import _AppManager
-from lightning_app.cli.cmd_clusters import AWSClusterManager
-from lightning_app.cli.cmd_ssh_keys import _SSHKeyManager
+from lightning.app.cli.cmd_apps import _AppManager
+from lightning.app.cli.cmd_clusters import AWSClusterManager
+from lightning.app.cli.cmd_ssh_keys import _SSHKeyManager
 
 
 @click.group(name="list")

src/lightning/app/cli/pl-app-template/app.py

@@ -4,10 +4,10 @@ from typing import Dict, List, Optional, Union
 from core.components import TensorBoard, WeightsAndBiases
 from core.components.script_runner import ScriptRunner
 
-from lightning_app import LightningApp, LightningFlow
-from lightning_app.frontend import StaticWebFrontend
-from lightning_app.storage import Path
-from lightning_app.utilities.packaging.cloud_compute import CloudCompute
+from lightning.app import LightningApp, LightningFlow
+from lightning.app.frontend import StaticWebFrontend
+from lightning.app.storage import Path
+from lightning.app.utilities.packaging.cloud_compute import CloudCompute
 
 
 class ReactUI(LightningFlow):

src/lightning/app/cli/pl-app-template/core/callbacks.py

@@ -3,13 +3,13 @@ from typing import Any, Dict, TYPE_CHECKING, Union
 
 from core.state import ProgressBarState, TrainerState
 
-import pytorch_lightning as pl
-from lightning_app.storage import Path
-from lightning_app.utilities.app_helpers import Logger
-from pytorch_lightning import Callback
-from pytorch_lightning.callbacks.progress.base import get_standard_metrics
-from pytorch_lightning.loggers import TensorBoardLogger, WandbLogger
-from pytorch_lightning.utilities.parsing import collect_init_args
+import lightning.pytorch as pl
+from lightning.app.storage import Path
+from lightning.app.utilities.app_helpers import Logger
+from lightning.pytorch import Callback
+from lightning.pytorch.callbacks.progress.base import get_standard_metrics
+from lightning.pytorch.loggers import TensorBoardLogger, WandbLogger
+from lightning.pytorch.utilities.parsing import collect_init_args
 
 if TYPE_CHECKING:
     from core.components.script_runner import ScriptRunner

src/lightning/app/cli/pl-app-template/core/components/logger/tensorboard.py

@@ -2,8 +2,8 @@ import subprocess
 import time
 from typing import Dict, List
 
-from lightning_app import BuildConfig, LightningFlow, LightningWork
-from lightning_app.storage import Path
+from lightning.app import BuildConfig, LightningFlow, LightningWork
+from lightning.app.storage import Path
 
 
 class TensorBoard(LightningFlow):

src/lightning/app/cli/pl-app-template/core/components/logger/weights_and_biases.py

@@ -1,7 +1,7 @@
 import os
 from typing import Dict, List, Optional, TYPE_CHECKING
 
-from lightning_app import LightningFlow
+from lightning.app import LightningFlow
 
 if TYPE_CHECKING:
     import wandb

src/lightning/app/cli/pl-app-template/core/components/script_runner/script_runner.py

@@ -2,10 +2,10 @@ import sys
 import traceback
 from typing import Any, Dict, List, Optional, Tuple
 
-from lightning_app.components.python import TracerPythonScript
-from lightning_app.storage import Path
-from lightning_app.utilities.packaging.build_config import BuildConfig, load_requirements
-from lightning_app.utilities.tracer import Tracer
+from lightning.app.components.python import TracerPythonScript
+from lightning.app.storage import Path
+from lightning.app.utilities.packaging.build_config import BuildConfig, load_requirements
+from lightning.app.utilities.tracer import Tracer
 
 
 class ScriptRunner(TracerPythonScript):
@@ -27,7 +27,7 @@ class ScriptRunner(TracerPythonScript):
     def configure_tracer(self) -> Tracer:
         from core.callbacks import PLAppArtifactsTracker, PLAppProgressTracker, PLAppSummary, PLAppTrainerStateTracker
 
-        from pytorch_lightning import Trainer
+        from lightning.pytorch import Trainer
 
         tracer = Tracer()
         trainer_artifacts_tracker = PLAppArtifactsTracker(work=self)

src/lightning/app/cli/pl-app-template/tests/core/test_callbacks.py

@@ -5,9 +5,9 @@ import pytest
 from core.callbacks import PLAppArtifactsTracker, PLAppProgressTracker, PLAppSummary
 from core.components.script_runner import ScriptRunner
 
-from lightning_app.storage import Path
-from pytorch_lightning import LightningModule, Trainer
-from pytorch_lightning.loggers import TensorBoardLogger
+from lightning.app.storage import Path
+from lightning.pytorch import LightningModule, Trainer
+from lightning.pytorch.loggers import TensorBoardLogger
 
 
 @pytest.mark.parametrize("rank", (0, 1))