Add CI for python lightning app Python unit tests (#13491)
* Update lightning_app src * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update lightning app tests * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add CI * update tests * requirements * fix version tests * todo * fix tests * fix tests * fix tests * fix tests * fix formatting Co-authored-by: mansy <mansy@lightning.ai> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: awaelchli <aedu.waelchli@gmail.com>
This commit is contained in:
Mansy
GitHub
parent
3daa244458
commit
dc70b6511c
|
|
@ -19,18 +19,10 @@ jobs:
|
|||
echo "$file"
|
||||
done
|
||||
|
||||
- name: Block edits in src/lightning_app
|
||||
if: contains(steps.changed-files.outputs.all_changed_and_modified_files, 'src/lightning_app')
|
||||
run: exit 1
|
||||
|
||||
- name: Block edits in docs/source-app
|
||||
if: contains(steps.changed-files.outputs.all_changed_and_modified_files, 'docs/source-app')
|
||||
run: exit 1
|
||||
|
||||
- name: Block edits in tests/tests_app
|
||||
if: contains(steps.changed-files.outputs.all_changed_and_modified_files, 'tests/tests_app')
|
||||
run: exit 1
|
||||
|
||||
- name: Block edits in examples/app
|
||||
if: contains(steps.changed-files.outputs.all_changed_and_modified_files, 'examples/app_')
|
||||
run: exit 1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
name: CI App Tests
|
||||
|
||||
# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
|
||||
on: # Trigger the workflow on push or pull request, but only for the master branch
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
pull_request:
|
||||
paths:
|
||||
- "src/lightning_app/**"
|
||||
- "tests/tests_app/**"
|
||||
- "requirements/app/**"
|
||||
- "setup.py"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
|
||||
cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
|
||||
|
||||
jobs:
|
||||
pytest:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-20.04, macOS-10.15, windows-2019]
|
||||
python-version: [3.8]
|
||||
requires: ["oldest", "latest"]
|
||||
|
||||
# Timeout: https://stackoverflow.com/a/59076067/4521646
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
# TODO: use replace_oldest_ver() instead
|
||||
- name: Set min. dependencies
|
||||
if: matrix.requires == 'oldest'
|
||||
run: |
|
||||
for fpath in ('requirements/app/base.txt', 'requirements/app/test.txt'):
|
||||
req = open(fpath).read().replace('>=', '==')
|
||||
open(fpath, 'w').write(req)
|
||||
shell: python
|
||||
|
||||
- run: echo "::set-output name=period::$(python -c 'import time ; days = time.time() / 60 / 60 / 24 ; print(int(days / 7))' 2>&1)"
|
||||
if: matrix.requires == 'latest'
|
||||
id: times
|
||||
|
||||
# Note: This uses an internal pip API and may not always work
|
||||
# https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
|
||||
- name: Get pip cache
|
||||
id: pip-cache
|
||||
run: |
|
||||
python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
|
||||
|
||||
- name: Cache pip
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ matrix.requires }}-td${{ steps.times.outputs.period }}-${{ hashFiles('requirements/app/base.txt') }}
|
||||
restore-keys: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ matrix.requires }}-td${{ steps.times.outputs.period }}-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip --version
|
||||
pip install -r requirements/app/devel.txt --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
|
||||
pip list
|
||||
shell: bash
|
||||
|
||||
# - name: Start Redis
|
||||
# if: runner.os == 'Linux'
|
||||
# uses: supercharge/redis-github-action@1.4.0
|
||||
# with:
|
||||
# redis-version: 6
|
||||
# redis-port: 6379
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '16'
|
||||
|
||||
- name: Install Yarn
|
||||
run: npm install -g yarn
|
||||
|
||||
- name: Install Lightning as top-level
|
||||
run: pip install -e . -r requirements/app/base.txt
|
||||
shell: bash
|
||||
|
||||
- name: Tests
|
||||
working-directory: ./tests
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
PYTEST_ARTIFACT: results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
|
||||
run: |
|
||||
coverage run --source lightning_app -m pytest -m "not cloud" tests_app --timeout=300 -vvvv --junitxml=$PYTEST_ARTIFACT --durations=0
|
||||
|
||||
- name: Upload pytest test results
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: unittest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}
|
||||
path: tests/results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
|
||||
if: failure()
|
||||
|
||||
- name: Statistics
|
||||
if: success()
|
||||
working-directory: ./tests
|
||||
run: |
|
||||
coverage xml -i
|
||||
coverage report -i
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v1
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
file: tests/coverage.xml
|
||||
flags: unittests
|
||||
env_vars: OS,PYTHON
|
||||
name: codecov-umbrella
|
||||
fail_ci_if_error: false
|
||||
|
||||
# TODO: figure out why we clone and install quick-start
|
||||
# - name: Clone Quick Start Example Repo
|
||||
# uses: actions/checkout@v3
|
||||
# # TODO: this needs to be git submodule
|
||||
# if: matrix.os == 'windows-2019' # because the install doesn't work on windows
|
||||
# with:
|
||||
# repository: Lightning-AI/lightning-quick-start
|
||||
# ref: 'main'
|
||||
# path: lightning-quick-start
|
||||
#
|
||||
# - name: Lightning Install quick-start
|
||||
# shell: bash
|
||||
# if: matrix.os != 'windows-2019' # because the install doesn't work on windows
|
||||
# run: |
|
||||
# python -m lightning install app lightning/quick-start -y
|
||||
|
|
@ -37,6 +37,6 @@ if _module_available("lightning_app.components.demo"):
|
|||
from lightning_app.components import demo # noqa: F401
|
||||
|
||||
_PACKAGE_ROOT = os.path.dirname(__file__)
|
||||
_PROJECT_ROOT = os.path.dirname(_PACKAGE_ROOT)
|
||||
_PROJECT_ROOT = os.path.dirname(os.path.dirname(_PACKAGE_ROOT))
|
||||
|
||||
__all__ = ["LightningApp", "LightningFlow", "LightningWork", "BuildConfig", "CloudCompute"]
|
||||
|
|
|
|||
|
|
@ -58,8 +58,8 @@ coverage.xml
|
|||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
docs/source/api/
|
||||
docs/source/*.md
|
||||
docs/source-app/api/
|
||||
docs/source-app/*.md
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
|
@ -132,9 +132,9 @@ coverage.*
|
|||
# Frontend build artifacts
|
||||
*lightning_app/ui*
|
||||
gradio_cached_examples
|
||||
/docs/source/api_reference/generated/*
|
||||
/docs/source-app/api_reference/generated/*
|
||||
examples/my_own_leaderboard/submissions/*
|
||||
docs/source/api_reference/generated/*
|
||||
docs/source-app/api_reference/generated/*
|
||||
*.ckpt
|
||||
redis-stable
|
||||
node_modules
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from placeholdername import ComponentA, ComponentB
|
||||
|
||||
import lightning_app as la
|
||||
import lightning as L
|
||||
|
||||
|
||||
class LitApp(la.LightningFlow):
|
||||
class LitApp(L.LightningFlow):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.component_a = ComponentA()
|
||||
|
|
@ -14,4 +14,4 @@ class LitApp(la.LightningFlow):
|
|||
self.component_b.run()
|
||||
|
||||
|
||||
app = la.LightningApp(LitApp())
|
||||
app = L.LightningApp(LitApp())
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import lightning_app as la
|
||||
import lightning as L
|
||||
|
||||
|
||||
class ComponentA(la.LightningFlow):
|
||||
class ComponentA(L.LightningFlow):
|
||||
def run(self):
|
||||
print("hello from component A")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import lightning_app as la
|
||||
import lightning as L
|
||||
|
||||
|
||||
class ComponentB(la.LightningFlow):
|
||||
class ComponentB(L.LightningFlow):
|
||||
def run(self):
|
||||
print("hello from component B")
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import io
|
|||
import os
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
from lightning_app.testing.testing import application_testing, LightningTestApp
|
||||
from lightning.app.testing.testing import application_testing, LightningTestApp
|
||||
|
||||
|
||||
class LightningAppTestInt(LightningTestApp):
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ def component(component_name):
|
|||
⚡ Use the component inside an app: ⚡
|
||||
|
||||
from {name_for_files} import TemplateComponent
|
||||
import lightning_app as la
|
||||
import lightning.app as la
|
||||
|
||||
class LitApp(la.LightningFlow):
|
||||
def __init__(self) -> None:
|
||||
|
|
|
|||
|
|
@ -49,10 +49,12 @@ def gallery_app(name, yes_arg, version_arg, cwd=None, overwrite=False):
|
|||
app_entry = _resolve_resource(registry_url, name=name, version_arg=version_arg, resource_type="app")
|
||||
|
||||
# give the user the chance to do a manual install
|
||||
source_url, git_url, folder_name = _show_install_app_prompt(app_entry, app, org, yes_arg, resource_type="app")
|
||||
source_url, git_url, folder_name, git_sha = _show_install_app_prompt(
|
||||
app_entry, app, org, yes_arg, resource_type="app"
|
||||
)
|
||||
|
||||
# run installation if requested
|
||||
_install_app(source_url, git_url, folder_name, cwd=cwd, overwrite=overwrite)
|
||||
_install_app(source_url, git_url, folder_name, cwd=cwd, overwrite=overwrite, git_sha=git_sha)
|
||||
|
||||
|
||||
def non_gallery_app(gh_url, yes_arg, cwd=None, overwrite=False):
|
||||
|
|
@ -161,14 +163,16 @@ def _show_non_gallery_install_component_prompt(gh_url, yes_arg):
|
|||
def _show_install_app_prompt(entry, app, org, yes_arg, resource_type):
|
||||
source_url = entry["sourceUrl"] # This URL is used only to display the repo and extract folder name
|
||||
full_git_url = entry["gitUrl"] # Used to clone the repo (can include tokens for private repos)
|
||||
git_url = full_git_url.split("#ref=")[0]
|
||||
git_url_parts = full_git_url.split("#ref=")
|
||||
git_url = git_url_parts[0]
|
||||
git_sha = git_url_parts[1] if len(git_url_parts) == 2 else None
|
||||
|
||||
folder_name = source_url.split("/")[-1]
|
||||
|
||||
# yes arg does not prompt the user for permission to install anything
|
||||
# automatically creates env and sets up the project
|
||||
if yes_arg:
|
||||
return source_url, git_url, folder_name
|
||||
return source_url, git_url, folder_name, git_sha
|
||||
|
||||
prompt = f"""
|
||||
⚡ Installing Lightning {resource_type} ⚡
|
||||
|
|
@ -192,7 +196,7 @@ def _show_install_app_prompt(entry, app, org, yes_arg, resource_type):
|
|||
if not should_install:
|
||||
raise KeyboardInterrupt()
|
||||
|
||||
return source_url, git_url, folder_name
|
||||
return source_url, git_url, folder_name, git_sha
|
||||
except KeyboardInterrupt:
|
||||
repo = entry["sourceUrl"]
|
||||
m = f"""
|
||||
|
|
@ -367,7 +371,9 @@ def _install_with_env(repo_url, folder_name, cwd=None):
|
|||
logger.info(m)
|
||||
|
||||
|
||||
def _install_app(source_url: str, git_url: str, folder_name: str, cwd=None, overwrite: bool = False):
|
||||
def _install_app(
|
||||
source_url: str, git_url: str, folder_name: str, cwd=None, overwrite: bool = False, git_sha: str = None
|
||||
):
|
||||
"""Installing lighting app from the `git_url`
|
||||
|
||||
Args:
|
||||
|
|
@ -381,6 +387,8 @@ def _install_app(source_url: str, git_url: str, folder_name: str, cwd=None, over
|
|||
Working director. If not specified, current working directory is used.
|
||||
overwrite:
|
||||
If true, overwrite the app directory without asking if it already exists
|
||||
git_sha:
|
||||
The git_sha for checking out the git repo of the app.
|
||||
"""
|
||||
|
||||
if not cwd:
|
||||
|
|
@ -412,6 +420,15 @@ def _install_app(source_url: str, git_url: str, folder_name: str, cwd=None, over
|
|||
os.chdir(f"{folder_name}")
|
||||
cwd = os.getcwd()
|
||||
|
||||
try:
|
||||
if git_sha:
|
||||
subprocess.check_output(["git", "checkout", git_sha], stderr=subprocess.STDOUT)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if "did not match any" in str(e.output):
|
||||
raise SystemExit("Looks like the git SHA is not valid or doesn't exist in app repo.")
|
||||
else:
|
||||
raise Exception(e)
|
||||
|
||||
# activate and install reqs
|
||||
# TODO: remove shell=True... but need to run command in venv
|
||||
logger.info("⚡ RUN: install requirements (pip install -r requirements.txt)")
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
- name: Clone Template React UI Repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
repository: PyTorchLightning/lightning
|
||||
repository: Lightning-AI/lightning
|
||||
token: ${{ secrets.PAT_GHOST }}
|
||||
ref: 'master'
|
||||
path: lightning
|
||||
|
|
|
|||
|
|
@ -58,8 +58,8 @@ coverage.xml
|
|||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
docs/source/api/
|
||||
docs/source/*.md
|
||||
docs/source-app/api/
|
||||
docs/source-app/*.md
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
|
@ -132,9 +132,9 @@ coverage.*
|
|||
# Frontend build artifacts
|
||||
*lightning_app/ui*
|
||||
gradio_cached_examples
|
||||
/docs/source/api_reference/generated/*
|
||||
/docs/source-app/api_reference/generated/*
|
||||
examples/my_own_leaderboard/submissions/*
|
||||
docs/source/api_reference/generated/*
|
||||
docs/source-app/api_reference/generated/*
|
||||
*.ckpt
|
||||
redis-stable
|
||||
node_modules
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ lightning init component placeholdername
|
|||
|
||||
## To run placeholdername
|
||||
|
||||
First, install placeholdername (warning: this app has not been officially approved on the lightning gallery):
|
||||
First, install placeholdername (warning: this component has not been officially approved on the lightning gallery):
|
||||
|
||||
```bash
|
||||
lightning install component https://github.com/theUser/placeholdername
|
||||
|
|
@ -18,10 +18,10 @@ Once the app is installed, use it in an app:
|
|||
|
||||
```python
|
||||
from placeholdername import TemplateComponent
|
||||
import lightning_app as la
|
||||
import lightning as L
|
||||
|
||||
|
||||
class LitApp(lapp.LightningFlow):
|
||||
class LitApp(L.LightningFlow):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.placeholdername = TemplateComponent()
|
||||
|
|
@ -31,5 +31,5 @@ class LitApp(lapp.LightningFlow):
|
|||
self.placeholdername.run()
|
||||
|
||||
|
||||
app = lapp.LightningApp(LitApp())
|
||||
app = L.LightningApp(LitApp())
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from placeholdername import TemplateComponent
|
||||
|
||||
import lightning_app as la
|
||||
import lightning as L
|
||||
|
||||
|
||||
class LitApp(la.LightningFlow):
|
||||
class LitApp(L.LightningFlow):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.placeholdername = TemplateComponent()
|
||||
|
|
@ -13,4 +13,4 @@ class LitApp(la.LightningFlow):
|
|||
self.placeholdername.run()
|
||||
|
||||
|
||||
app = la.LightningApp(LitApp())
|
||||
app = L.LightningApp(LitApp())
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import lightning_app as la
|
||||
import lightning as L
|
||||
|
||||
|
||||
class TemplateComponent(la.LightningWork):
|
||||
class TemplateComponent(L.LightningWork):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.value = 0
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ def _run_app(
|
|||
)
|
||||
|
||||
env_vars = _format_input_env_variables(env)
|
||||
os.environ.update(env_vars)
|
||||
|
||||
def on_before_run(*args):
|
||||
if open_ui and not without_server:
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@ from typing import Dict, List, Optional, Union
|
|||
from core.components import TensorBoard, WeightsAndBiases
|
||||
from core.components.script_runner import ScriptRunner
|
||||
|
||||
from lightning_app import LightningApp, LightningFlow
|
||||
from lightning_app.frontend import StaticWebFrontend
|
||||
from lightning_app.storage import Path
|
||||
from lightning_app.utilities.packaging.cloud_compute import CloudCompute
|
||||
from lightning.app import LightningApp, LightningFlow
|
||||
from lightning.app.frontend import StaticWebFrontend
|
||||
from lightning.app.storage import Path
|
||||
from lightning.app.utilities.packaging.cloud_compute import CloudCompute
|
||||
|
||||
|
||||
class ReactUI(LightningFlow):
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from typing import Any, Dict, Optional, TYPE_CHECKING, Union
|
|||
from core.state import ProgressBarState, TrainerState
|
||||
|
||||
import pytorch_lightning as pl
|
||||
from lightning_app.storage import Path
|
||||
from lightning.app.storage import Path
|
||||
from pytorch_lightning import Callback
|
||||
from pytorch_lightning.callbacks.progress.base import get_standard_metrics
|
||||
from pytorch_lightning.loggers import TensorBoardLogger, WandbLogger
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ import subprocess
|
|||
import time
|
||||
from typing import Dict, List
|
||||
|
||||
from lightning_app import BuildConfig, LightningFlow, LightningWork
|
||||
from lightning_app.storage import Path
|
||||
from lightning.app import BuildConfig, LightningFlow, LightningWork
|
||||
from lightning.app.storage import Path
|
||||
|
||||
|
||||
class TensorBoard(LightningFlow):
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import os
|
||||
from typing import Dict, List, Optional, TYPE_CHECKING
|
||||
|
||||
from lightning_app import LightningFlow
|
||||
from lightning.app import LightningFlow
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import wandb
|
||||
|
|
|
|||
|
|
@ -1,14 +1,12 @@
|
|||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from pkg_resources import parse_requirements
|
||||
|
||||
from lightning_app.components.python import TracerPythonScript
|
||||
from lightning_app.storage import Path
|
||||
from lightning_app.utilities.packaging.build_config import BuildConfig
|
||||
from lightning_app.utilities.tracer import Tracer
|
||||
from lightning.app.components.python import TracerPythonScript
|
||||
from lightning.app.storage import Path
|
||||
from lightning.app.utilities.packaging.build_config import BuildConfig
|
||||
from lightning.app.utilities.tracer import Tracer
|
||||
from lightning_app.utilities.packaging.build_config import load_requirements
|
||||
|
||||
|
||||
class ScriptRunner(TracerPythonScript):
|
||||
|
|
@ -76,7 +74,6 @@ class ScriptRunner(TracerPythonScript):
|
|||
]
|
||||
if Path(root_path, "requirements.txt").exists():
|
||||
# Requirements from the user's code folder
|
||||
path_req = os.path.join(root_path, "requirements.txt")
|
||||
requirements.extend(list(map(str, parse_requirements(open(path_req).readlines()))))
|
||||
requirements.extend(load_requirements(root_path, file_name="requirements.txt"))
|
||||
|
||||
return BuildConfig(requirements=requirements)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import pytest
|
|||
from core.callbacks import PLAppArtifactsTracker, PLAppProgressTracker, PLAppSummary
|
||||
from core.components.script_runner import ScriptRunner
|
||||
|
||||
from lightning_app.storage import Path
|
||||
from lightning.app.storage import Path
|
||||
from pytorch_lightning import LightningModule, Trainer
|
||||
from pytorch_lightning.loggers import TensorBoardLogger
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ This is a full react template ready to use in a component
|
|||
|
||||
This UI was automatically generated with:
|
||||
|
||||
```bash
|
||||
```commandline
|
||||
lightning init react-ui
|
||||
```
|
||||
|
||||
|
|
@ -34,12 +34,12 @@ lightning run app react-ui/example_app.py
|
|||
To connect the react UI to your component, simply point the `StaticWebFrontend` to the `dist/` folder generated by yarn after building your react website.
|
||||
|
||||
```python
|
||||
import lightning_app as la
|
||||
import lightning as L
|
||||
|
||||
|
||||
class YourComponent(lapp.LightningFlow):
|
||||
class YourComponent(L.LightningFlow):
|
||||
def configure_layout(self):
|
||||
return lapp.frontend.StaticWebFrontend(Path(__file__).parent / "react-ui/src/dist")
|
||||
return Lapp.frontend.StaticWebFrontend(Path(__file__).parent / "react-ui/src/dist")
|
||||
```
|
||||
|
||||
### Set up interactions between React and the component
|
||||
|
|
|
|||
|
|
@ -2,20 +2,21 @@
|
|||
|
||||
from pathlib import Path
|
||||
|
||||
import lightning_app as la
|
||||
import lightning as L
|
||||
from lightning.app import frontend
|
||||
|
||||
|
||||
class YourComponent(la.LightningFlow):
|
||||
class YourComponent(L.LightningFlow):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.message_to_print = "Hello World!"
|
||||
self.should_print = False
|
||||
|
||||
def configure_layout(self):
|
||||
return la.frontend.StaticWebFrontend(Path(__file__).parent / "ui/dist")
|
||||
return frontend.StaticWebFrontend(Path(__file__).parent / "ui/dist")
|
||||
|
||||
|
||||
class HelloLitReact(la.LightningFlow):
|
||||
class HelloLitReact(L.LightningFlow):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.counter = 0
|
||||
|
|
@ -30,4 +31,4 @@ class HelloLitReact(la.LightningFlow):
|
|||
return [{"name": "React UI", "content": self.react_ui}]
|
||||
|
||||
|
||||
app = la.LightningApp(HelloLitReact())
|
||||
app = L.LightningApp(HelloLitReact())
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
"dependencies": {
|
||||
"@emotion/react": "^11.8.2",
|
||||
"@emotion/styled": "^11.8.1",
|
||||
"@mui/material": "^5.5.0",
|
||||
"@mui/material": "5.8.5",
|
||||
"axios": "^0.26.1",
|
||||
"lodash": "^4.17.21",
|
||||
"nanoid": "^3.3.1",
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -31,7 +31,7 @@ class PopenPythonScript(LightningWork):
|
|||
env: Optional[Dict] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""The PopenPythonScript Class enables to easily run a Python Script.
|
||||
"""The PopenPythonScript component enables to easily run a python script within a subprocess.
|
||||
|
||||
Arguments:
|
||||
script_path: Path of the python script to run.
|
||||
|
|
@ -56,7 +56,7 @@ class PopenPythonScript(LightningWork):
|
|||
|
||||
In this example, the script will be launch with the :class:`~subprocess.Popen`.
|
||||
|
||||
.. literalinclude:: ../../../../examples/components/python/component_popen.py
|
||||
.. literalinclude:: ../../../../examples/app_components/python/component_popen.py
|
||||
:language: python
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
|
|
|
|||
|
|
@ -33,9 +33,12 @@ class TracerPythonScript(LightningWork):
|
|||
env: Optional[Dict] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""The TracerPythonScript Class enables to easily run a Python Script with Lightning
|
||||
:class:`~lightning_app.utilities.tracer.Tracer`. Simply overrides the
|
||||
:meth:`~lightning_app.components.python.tracer.TracerPythonScript.configure_tracer` method.
|
||||
"""The TracerPythonScript class enables to easily run a python script.
|
||||
|
||||
When subclassing this class, you can configure your own :class:`~lightning_app.utilities.tracer.Tracer`
|
||||
by :meth:`~lightning_app.components.python.tracer.TracerPythonScript.configure_tracer` method
|
||||
|
||||
The tracer is quite a magical class. It enables you to inject core into a script execution without changing it.
|
||||
|
||||
Arguments:
|
||||
script_path: Path of the python script to run.
|
||||
|
|
@ -47,6 +50,13 @@ class TracerPythonScript(LightningWork):
|
|||
Raises:
|
||||
FileNotFoundError: If the provided `script_path` doesn't exists.
|
||||
|
||||
**How does it works ?**
|
||||
|
||||
It works by executing the python script with python built-in `runpy
|
||||
<https://docs.python.org/3/library/runpy.html>`_ run_path method.
|
||||
This method takes any python globals before executing the script,
|
||||
e.g you can modify classes or function from the script.
|
||||
|
||||
.. doctest::
|
||||
|
||||
>>> from lightning_app.components.python import TracerPythonScript
|
||||
|
|
@ -59,17 +69,27 @@ class TracerPythonScript(LightningWork):
|
|||
Hello World !
|
||||
>>> os.remove("a.py")
|
||||
|
||||
In this example, you would be able to implement your own :class:`~lightning_app.utilities.tracer.Tracer`
|
||||
and intercept / modify elements while the script is being executed.
|
||||
In the example below, we subclass the :class:`~lightning_app.components.python.TracerPythonScript`
|
||||
component and override its configure_tracer method.
|
||||
|
||||
.. literalinclude:: ../../../../examples/components/python/component_tracer.py
|
||||
Using the Tracer, we are patching the ``__init__`` method of the PyTorch Lightning Trainer.
|
||||
Once the script starts running and if a Trainer is instantiated, the provided ``pre_fn`` is
|
||||
called and we inject a Lightning callback.
|
||||
|
||||
This callback has a reference to the work and on every batch end, we are capturing the
|
||||
trainer ``global_step`` and ``best_model_path``.
|
||||
|
||||
Even more interesting, this component works for ANY Pytorch Lightning script and
|
||||
its state can be used in real time in a UI.
|
||||
|
||||
.. literalinclude:: ../../../../examples/app_components/python/component_tracer.py
|
||||
:language: python
|
||||
|
||||
|
||||
Once implemented, this component can easily be integrated within a larger app
|
||||
to execute a specific python script.
|
||||
|
||||
.. literalinclude:: ../../../../examples/components/python/app.py
|
||||
.. literalinclude:: ../../../../examples/app_components/python/app.py
|
||||
:language: python
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class ServeGradio(LightningWork, abc.ABC):
|
|||
|
||||
In the example below, the ``ServeGradio`` is subclassed to deploy ``AnimeGANv2``.
|
||||
|
||||
.. literalinclude:: ../../../../examples/components/serve/gradio/app.py
|
||||
.. literalinclude:: ../../../../examples/app_components/serve/gradio/app.py
|
||||
:language: python
|
||||
|
||||
The result would be the following:
|
||||
|
|
|
|||
|
|
@ -37,26 +37,33 @@ class LightningApp:
|
|||
root: "lightning_app.LightningFlow",
|
||||
debug: bool = False,
|
||||
):
|
||||
"""LightningApp, or App in short, alternatively run its root
|
||||
:class:`~lightning_app.core.flow.LightningFlow` component and collects state changes from external
|
||||
sources to maintain the application state up-to-date or performs checkpointing. All those operations
|
||||
are executed within an infinite loop.
|
||||
"""The Lightning App, or App in short runs a tree of one or more components that interact to create end-to-end
|
||||
applications. There are two kinds of components: :class:`~lightning_app.core.flow.LightningFlow` and
|
||||
:class:`~lightning_app.core.work.LightningWork`. This modular design enables you to reuse components
|
||||
created by other users.
|
||||
|
||||
The Lightning App alternatively run an event loop triggered by delta changes sent from
|
||||
either :class:`~lightning.app.core.work.LightningWork` or from the Lightning UI.
|
||||
Once deltas are received, the Lightning App runs
|
||||
the :class:`~lightning.app.core.flow.LightningFlow` provided.
|
||||
|
||||
Arguments:
|
||||
root: The root LightningFlow component, that defined all the app's nested components, running infinitely.
|
||||
debug: Whether to run the application in debug model.
|
||||
root: The root LightningFlow component, that defined all
|
||||
the app's nested components, running infinitely.
|
||||
debug: Whether to activate the Lightning Logger debug mode.
|
||||
This can be helpful when reporting bugs on Lightning repo.
|
||||
|
||||
.. doctest::
|
||||
|
||||
>>> from lightning_app import LightningFlow, LightningApp
|
||||
>>> from lightning_app.runners import SingleProcessRuntime
|
||||
>>> from lightning import LightningFlow, LightningApp
|
||||
>>> from lightning.app.runners import MultiProcessRuntime
|
||||
>>> class RootFlow(LightningFlow):
|
||||
... def run(self):
|
||||
... print("Hello World!")
|
||||
... self._exit()
|
||||
...
|
||||
>>> app = LightningApp(RootFlow()) # application can be dispatched using the `runners`.
|
||||
>>> SingleProcessRuntime(app).dispatch()
|
||||
>>> MultiProcessRuntime(app).dispatch()
|
||||
Hello World!
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -25,10 +25,10 @@ class LightningFlow:
|
|||
}
|
||||
|
||||
def __init__(self):
|
||||
"""The LightningFlow is a building block to coordinate and manage long running-tasks contained within
|
||||
:class:`~lightning_app.core.work.LightningWork` or nested LightningFlow.
|
||||
"""The LightningFlow is used by the :class:`~lightning_app.core.app.LightningApp` to coordinate and manage
|
||||
long- running jobs contained, the :class:`~lightning_app.core.work.LightningWork`.
|
||||
|
||||
At a minimum, a LightningFlow is characterized by:
|
||||
A LightningFlow is characterized by:
|
||||
|
||||
* A set of state variables.
|
||||
* Long-running jobs (:class:`~lightning_app.core.work.LightningWork`).
|
||||
|
|
@ -41,11 +41,6 @@ class LightningFlow:
|
|||
|
||||
They also may not reach into global variables unless they are constant.
|
||||
|
||||
.. note ::
|
||||
The limitation to primitive types will be lifted in time for
|
||||
certain aggregate types, and will be made extensible so that component
|
||||
developers will be able to add custom state-compatible types.
|
||||
|
||||
The attributes need to be all defined in `__init__` method,
|
||||
and eventually assigned to different values throughout the lifetime of the object.
|
||||
However, defining new attributes outside of `__init__` is not allowed.
|
||||
|
|
@ -83,7 +78,7 @@ class LightningFlow:
|
|||
|
||||
.. doctest::
|
||||
|
||||
>>> from lightning_app import LightningFlow
|
||||
>>> from lightning import LightningFlow
|
||||
>>> class RootFlow(LightningFlow):
|
||||
... def __init__(self):
|
||||
... super().__init__()
|
||||
|
|
@ -345,6 +340,7 @@ class LightningFlow:
|
|||
return name in LightningFlow._INTERNAL_STATE_VARS or not name.startswith("_")
|
||||
|
||||
def run(self, *args, **kwargs) -> None:
|
||||
"""Override with your own logic."""
|
||||
pass
|
||||
|
||||
def schedule(
|
||||
|
|
@ -352,15 +348,16 @@ class LightningFlow:
|
|||
) -> bool:
|
||||
"""The schedule method is used to run a part of the flow logic on timely manner.
|
||||
|
||||
.. code-block::
|
||||
.. code-block:: python
|
||||
|
||||
from lightning_app import LightningFlow
|
||||
|
||||
class Flow(LightningFlow):
|
||||
|
||||
class Flow(LightningFlow):
|
||||
def run(self):
|
||||
if self.schedule("hourly"):
|
||||
# run some code once every hour.
|
||||
print("run this every hour")
|
||||
|
||||
Arguments:
|
||||
cron_pattern: The cron pattern to provide. Learn more at https://crontab.guru/.
|
||||
|
|
@ -370,7 +367,7 @@ class LightningFlow:
|
|||
A best practice is to avoid running a dynamic flow or work under the self.schedule method.
|
||||
Instead, instantiate them within the condition, but run them outside.
|
||||
|
||||
.. code-block:: python
|
||||
.. code-block:: python
|
||||
|
||||
from lightning_app import LightningFlow
|
||||
from lightning_app.structures import List
|
||||
|
|
@ -382,11 +379,40 @@ class LightningFlow:
|
|||
self.dags = List()
|
||||
|
||||
def run(self):
|
||||
if self.schedule("@hourly"):
|
||||
if self.schedule("hourly"):
|
||||
self.dags.append(DAG(...))
|
||||
|
||||
for dag in self.dags:
|
||||
payload = dag.run()
|
||||
|
||||
**Learn more about Scheduling**
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div class="display-card-container">
|
||||
<div class="row">
|
||||
|
||||
.. displayitem::
|
||||
:header: Schedule your components
|
||||
:description: Learn more scheduling.
|
||||
:col_css: col-md-4
|
||||
:button_link: ../../../glossary/scheduling.html
|
||||
:height: 180
|
||||
:tag: Basic
|
||||
|
||||
.. displayitem::
|
||||
:header: Build your own DAG
|
||||
:description: Learn more DAG scheduling with examples.
|
||||
:col_css: col-md-4
|
||||
:button_link: ../../../examples/app_dag/dag.html
|
||||
:height: 180
|
||||
:tag: Basic
|
||||
|
||||
.. raw:: html
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<br />
|
||||
"""
|
||||
if not user_key:
|
||||
frame = cast(FrameType, inspect.currentframe()).f_back
|
||||
|
|
@ -454,40 +480,48 @@ class LightningFlow:
|
|||
|
||||
**Example:** Serve a static directory (with at least a file index.html inside).
|
||||
|
||||
.. code-block::
|
||||
.. code-block:: python
|
||||
|
||||
from lightning_app.frontend import StaticWebFrontend
|
||||
|
||||
|
||||
class Flow(LightningFlow):
|
||||
...
|
||||
|
||||
def configure_layout(self):
|
||||
return StaticWebFrontend("path/to/folder/to/serve")
|
||||
|
||||
**Example:** Serve a streamlit UI (needs the streamlit package to be installed).
|
||||
|
||||
.. code-block::
|
||||
.. code-block:: python
|
||||
|
||||
from lightning_app.frontend import StaticWebFrontend
|
||||
|
||||
|
||||
class Flow(LightningFlow):
|
||||
...
|
||||
|
||||
def configure_layout(self):
|
||||
return StreamlitFrontend(render_fn=my_streamlit_ui)
|
||||
|
||||
|
||||
def my_streamlit_ui(state):
|
||||
# add your streamlit code here!
|
||||
import streamlit as st
|
||||
|
||||
st.button("Hello!")
|
||||
|
||||
**Example:** Arrange the UI of my children in tabs (default UI by Lightning).
|
||||
|
||||
.. code-block::
|
||||
.. code-block:: python
|
||||
|
||||
class Flow(LightningFlow):
|
||||
...
|
||||
|
||||
def configure_layout(self):
|
||||
return [
|
||||
dict(name="First Tab", content=self.child0),
|
||||
dict(name="Second Tab", content=self.child1),
|
||||
...
|
||||
# You can include direct URLs too
|
||||
dict(name="Lightning", content="https://lightning.ai"),
|
||||
]
|
||||
|
|
@ -500,6 +534,27 @@ class LightningFlow:
|
|||
returned layout configuration can depend on the state. The only exception are the flows that return a
|
||||
:class:`~lightning_app.frontend.frontend.Frontend`. These need to be provided at the time of app creation
|
||||
in order for the runtime to start the server.
|
||||
|
||||
**Learn more about adding UI**
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div class="display-card-container">
|
||||
<div class="row">
|
||||
|
||||
.. displayitem::
|
||||
:header: Add a web user interface (UI)
|
||||
:description: Learn more how to integrate several UIs.
|
||||
:col_css: col-md-4
|
||||
:button_link: ../../../workflows/add_web_ui/index.html
|
||||
:height: 180
|
||||
:tag: Basic
|
||||
|
||||
.. raw:: html
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<br />
|
||||
"""
|
||||
return [dict(name=name, content=component) for (name, component) in self.flows.items()]
|
||||
|
||||
|
|
|
|||
|
|
@ -63,7 +63,31 @@ class LightningWork(abc.ABC):
|
|||
with the same arguments in subsequent calls.
|
||||
raise_exception: Whether to re-raise an exception in the flow when raised from within the work run method.
|
||||
host: Bind socket to this host
|
||||
port: Bind socket to this port
|
||||
port: Bind socket to this port. Be default, this is None and should be called within your run method.
|
||||
local_build_config: The local BuildConfig isn't used until Lightning supports DockerRuntime.
|
||||
cloud_build_config: The cloud BuildConfig enables user to easily configure machine before running this work.
|
||||
run_once: Deprecated in favor of cache_calls. This will be removed soon.
|
||||
|
||||
**Learn More About Lightning Work Inner Workings**
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div class="display-card-container">
|
||||
<div class="row">
|
||||
|
||||
.. displayitem::
|
||||
:header: The Lightning Work inner workings.
|
||||
:description: Learn more Lightning Work.
|
||||
:col_css: col-md-4
|
||||
:button_link: ../../../core_api/lightning_work/index.html
|
||||
:height: 180
|
||||
:tag: Basic
|
||||
|
||||
.. raw:: html
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<br />
|
||||
"""
|
||||
from lightning_app.runners.backends.backend import Backend
|
||||
|
||||
|
|
@ -98,6 +122,7 @@ class LightningWork(abc.ABC):
|
|||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
"""Returns the current url of the work."""
|
||||
return self._url
|
||||
|
||||
@url.setter
|
||||
|
|
@ -106,6 +131,7 @@ class LightningWork(abc.ABC):
|
|||
|
||||
@property
|
||||
def host(self) -> str:
|
||||
"""Returns the current host of the work."""
|
||||
return self._host
|
||||
|
||||
@property
|
||||
|
|
@ -166,6 +192,7 @@ class LightningWork(abc.ABC):
|
|||
|
||||
@property
|
||||
def cloud_build_config(self) -> BuildConfig:
|
||||
"""Returns the cloud build config used to prepare the selected cloud hardware."""
|
||||
return self._cloud_build_config
|
||||
|
||||
@cloud_build_config.setter
|
||||
|
|
@ -179,6 +206,7 @@ class LightningWork(abc.ABC):
|
|||
|
||||
@cloud_compute.setter
|
||||
def cloud_compute(self, cloud_compute) -> None:
|
||||
"""Returns the cloud compute used to select the cloud hardware."""
|
||||
self._cloud_compute = cloud_compute
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ def call_script(
|
|||
if args is None:
|
||||
args = []
|
||||
args = [str(a) for a in args]
|
||||
command = [sys.executable, "-m", "coverage", "run", filepath] + args
|
||||
command = [sys.executable, filepath] + args # todo: add back coverage
|
||||
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
try:
|
||||
stdout, stderr = p.communicate(timeout=timeout)
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ def application_testing(lightning_app_cls: Type[LightningTestApp], command_line:
|
|||
|
||||
from click.testing import CliRunner
|
||||
|
||||
with mock.patch("lightning_app.LightningApp", lightning_app_cls):
|
||||
with mock.patch("lightning.LightningApp", lightning_app_cls):
|
||||
runner = CliRunner()
|
||||
return runner.invoke(run_app, command_line, catch_exceptions=False)
|
||||
|
||||
|
|
@ -208,22 +208,14 @@ def run_app_in_cloud(app_folder: str, app_name: str = "app.py") -> Generator:
|
|||
""",
|
||||
[LIGHTNING_CLOUD_PROJECT_ID],
|
||||
)
|
||||
admin_page.reload()
|
||||
admin_page.goto(f"{Config.url}/{Config.username}/apps")
|
||||
try:
|
||||
# Closing the Create Project modal
|
||||
button = admin_page.locator('button:has-text("Cancel")')
|
||||
button.wait_for(timeout=1 * 1000)
|
||||
button.wait_for(timeout=3 * 1000)
|
||||
button.click()
|
||||
except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError):
|
||||
pass
|
||||
try:
|
||||
# Skipping the Hubspot form
|
||||
button = admin_page.locator('button:has-text("Skip for now")')
|
||||
button.wait_for(timeout=1 * 1000)
|
||||
button.click()
|
||||
except (playwright._impl._api_types.Error, playwright._impl._api_types.TimeoutError):
|
||||
pass
|
||||
admin_page.goto(f"{Config.url}/{Config.username}/apps")
|
||||
admin_page.locator(f"text={name}").click()
|
||||
admin_page.evaluate(
|
||||
"""data => {
|
||||
|
|
|
|||
|
|
@ -232,7 +232,7 @@ class StreamLitStatePlugin(BaseStatePlugin):
|
|||
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/Lightning-AI/lightning/blob/master/pytorch_lightning/utilities/model_helpers.py#L21
|
||||
# https://github.com/Lightning-AI/pytorch-lightning/blob/master/pytorch_lightning/utilities/model_helpers.py#L21
|
||||
def is_overridden(method_name: str, instance: Optional[object] = None, parent: Optional[Type[object]] = None) -> bool:
|
||||
if instance is None:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ def _configure_session() -> Session:
|
|||
return http
|
||||
|
||||
|
||||
def _check_service_url_is_ready(url: str, timeout: float = 0.1) -> bool:
|
||||
def _check_service_url_is_ready(url: str, timeout: float = 0.5) -> bool:
|
||||
try:
|
||||
response = requests.get(url, timeout=timeout)
|
||||
return response.status_code in (200, 404)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from dataclasses import asdict, dataclass
|
||||
from types import FrameType
|
||||
from typing import cast, List, Optional, TYPE_CHECKING, Union
|
||||
|
||||
from pkg_resources import parse_requirements
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lightning_app import LightningWork
|
||||
from lightning_app.utilities.packaging.cloud_compute import CloudCompute
|
||||
|
|
@ -15,6 +14,37 @@ if TYPE_CHECKING:
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_requirements(
|
||||
path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: bool = True
|
||||
) -> List[str]:
|
||||
"""Load requirements from a file.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
path_req = os.path.join(_PROJECT_ROOT, "requirements")
|
||||
requirements = load_requirements(path_req)
|
||||
print(requirements) # ['numpy...', 'torch...', ...]
|
||||
"""
|
||||
with open(os.path.join(path_dir, file_name)) as file:
|
||||
lines = [ln.strip() for ln in file.readlines()]
|
||||
reqs = []
|
||||
for ln in lines:
|
||||
# filer all comments
|
||||
comment = ""
|
||||
if comment_char in ln:
|
||||
comment = ln[ln.index(comment_char) :]
|
||||
ln = ln[: ln.index(comment_char)]
|
||||
req = ln.strip()
|
||||
# skip directly installed dependencies
|
||||
if not req or req.startswith("http") or "@http" in req:
|
||||
continue
|
||||
# remove version restrictions unless they are strict
|
||||
if unfreeze and "<" in req and "strict" not in comment:
|
||||
req = re.sub(r",? *<=? *[\d\.\*]+", "", req).strip()
|
||||
reqs.append(req)
|
||||
return reqs
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildConfig:
|
||||
"""The Build Configuration describes how the environment a LightningWork runs in should be set up.
|
||||
|
|
@ -61,7 +91,7 @@ class BuildConfig:
|
|||
class MyOwnBuildConfig(BuildConfig):
|
||||
|
||||
def build_commands(self):
|
||||
return ["sudo apt-get install libsparsehash-dev"]
|
||||
return ["apt-get install libsparsehash-dev"]
|
||||
|
||||
BuildConfig(requirements=["git+https://github.com/mit-han-lab/torchsparse.git@v1.4.0"])
|
||||
"""
|
||||
|
|
@ -84,8 +114,9 @@ class BuildConfig:
|
|||
requirement_files = [os.path.join(dirname, f) for f in os.listdir(dirname) if f == "requirements.txt"]
|
||||
if not requirement_files:
|
||||
return []
|
||||
dirname, basename = os.path.dirname(requirement_files[0]), os.path.basename(requirement_files[0])
|
||||
try:
|
||||
requirements = list(map(str, parse_requirements(open(requirement_files[0]).readlines())))
|
||||
requirements = load_requirements(dirname, basename)
|
||||
except NotADirectoryError:
|
||||
requirements = []
|
||||
return [r for r in requirements if r != "lightning"]
|
||||
|
|
@ -116,7 +147,9 @@ class BuildConfig:
|
|||
path = os.path.join(self._call_dir, req)
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
requirements.extend(list(map(str, parse_requirements(open(path).readlines()))))
|
||||
requirements.extend(
|
||||
load_requirements(os.path.dirname(path), os.path.basename(path)),
|
||||
)
|
||||
except NotADirectoryError:
|
||||
pass
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -32,8 +32,8 @@ class CloudCompute:
|
|||
This timeout starts whenever your run() method succeeds (or fails).
|
||||
If the timeout is reached, the instance pauses until the next run() call happens.
|
||||
|
||||
shm_size: Shared memory size in MiB, backed by RAM. min 512, max 4096, it will auto update in steps of 512.
|
||||
For example 1100 will become 1024. If set to zero (the default) will get the default 65MB inside docker.
|
||||
shm_size: Shared memory size in MiB, backed by RAM. min 512, max 8192, it will auto update in steps of 512.
|
||||
For example 1100 will become 1024. If set to zero (the default) will get the default 64MiB inside docker.
|
||||
"""
|
||||
|
||||
name: str = "default"
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from typing import Any, Callable, Optional
|
|||
from packaging.version import Version
|
||||
|
||||
from lightning_app import _logger, _PROJECT_ROOT, _root_logger
|
||||
from lightning_app.__about__ import __version__
|
||||
from lightning_app.__version__ import version
|
||||
from lightning_app.core.constants import PREPARE_LIGHTING
|
||||
from lightning_app.utilities.git import check_github_repository, get_dir_name
|
||||
|
||||
|
|
@ -29,7 +29,7 @@ def download_frontend(root):
|
|||
"""Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct
|
||||
directory."""
|
||||
build_dir = "build"
|
||||
frontend_dir = pathlib.Path(root, "lightning_app", "ui")
|
||||
frontend_dir = pathlib.Path(root, "src", "lightning_app", "ui")
|
||||
download_dir = tempfile.mkdtemp()
|
||||
|
||||
shutil.rmtree(frontend_dir, ignore_errors=True)
|
||||
|
|
@ -43,41 +43,51 @@ def download_frontend(root):
|
|||
print("The Lightning UI has successfully been downloaded!")
|
||||
|
||||
|
||||
def _cleanup(tar_file: str):
|
||||
shutil.rmtree(os.path.join(_PROJECT_ROOT, "dist"), ignore_errors=True)
|
||||
os.remove(tar_file)
|
||||
def _cleanup(*tar_files: str):
|
||||
for tar_file in tar_files:
|
||||
shutil.rmtree(os.path.join(_PROJECT_ROOT, "dist"), ignore_errors=True)
|
||||
os.remove(tar_file)
|
||||
|
||||
|
||||
def _prepare_lightning_wheels():
|
||||
def _prepare_wheel(path):
|
||||
with open("log.txt", "w") as logfile:
|
||||
with subprocess.Popen(
|
||||
["rm", "-r", "dist"], stdout=logfile, stderr=logfile, bufsize=0, close_fds=True, cwd=_PROJECT_ROOT
|
||||
["rm", "-r", "dist"], stdout=logfile, stderr=logfile, bufsize=0, close_fds=True, cwd=path
|
||||
) as proc:
|
||||
proc.wait()
|
||||
|
||||
with subprocess.Popen(
|
||||
["python", "setup.py", "sdist"],
|
||||
stdout=logfile,
|
||||
stderr=logfile,
|
||||
bufsize=0,
|
||||
close_fds=True,
|
||||
cwd=_PROJECT_ROOT,
|
||||
cwd=path,
|
||||
) as proc:
|
||||
proc.wait()
|
||||
|
||||
os.remove("log.txt")
|
||||
|
||||
|
||||
def _copy_lightning_tar(root: Path) -> str:
|
||||
dist_dir = os.path.join(_PROJECT_ROOT, "dist")
|
||||
def _copy_tar(project_root, dest: Path) -> str:
|
||||
dist_dir = os.path.join(project_root, "dist")
|
||||
tar_files = os.listdir(dist_dir)
|
||||
assert len(tar_files) == 1
|
||||
tar_name = tar_files[0]
|
||||
tar_path = os.path.join(dist_dir, tar_name)
|
||||
shutil.copy(tar_path, root)
|
||||
shutil.copy(tar_path, dest)
|
||||
return tar_name
|
||||
|
||||
|
||||
def get_dist_path_if_editable_install(project_name) -> str:
|
||||
"""Is distribution an editable install - modified version from pip that
|
||||
fetches egg-info instead of egg-link"""
|
||||
for path_item in sys.path:
|
||||
egg_info = os.path.join(path_item, project_name + ".egg-info")
|
||||
if os.path.isdir(egg_info):
|
||||
return path_item
|
||||
return ""
|
||||
|
||||
|
||||
def _prepare_lightning_wheels_and_requirements(root: Path) -> Optional[Callable]:
|
||||
|
||||
if "site-packages" in _PROJECT_ROOT:
|
||||
|
|
@ -88,20 +98,37 @@ def _prepare_lightning_wheels_and_requirements(root: Path) -> Optional[Callable]
|
|||
|
||||
if not PREPARE_LIGHTING and (not git_dir_name or (git_dir_name and not git_dir_name.startswith("lightning"))):
|
||||
return
|
||||
|
||||
if not bool(int(os.getenv("SKIP_LIGHTING_WHEELS_BUILD", "0"))):
|
||||
download_frontend(_PROJECT_ROOT)
|
||||
_prepare_lightning_wheels()
|
||||
_prepare_wheel(_PROJECT_ROOT)
|
||||
|
||||
logger.info("Packaged Lightning with your application.")
|
||||
|
||||
tar_name = _copy_lightning_tar(root)
|
||||
tar_name = _copy_tar(_PROJECT_ROOT, root)
|
||||
|
||||
return functools.partial(_cleanup, tar_file=os.path.join(root, tar_name))
|
||||
tar_files = [os.path.join(root, tar_name)]
|
||||
|
||||
# skipping this by default
|
||||
if not bool(int(os.getenv("SKIP_LIGHTING_UTILITY_WHEELS_BUILD", "1"))):
|
||||
# building and copying launcher wheel if installed in editable mode
|
||||
launcher_project_path = get_dist_path_if_editable_install("lightning_launcher")
|
||||
if launcher_project_path:
|
||||
_prepare_wheel(launcher_project_path)
|
||||
tar_name = _copy_tar(launcher_project_path, root)
|
||||
tar_files.append(os.path.join(root, tar_name))
|
||||
|
||||
# building and copying lightning-cloud wheel if installed in editable mode
|
||||
lightning_cloud_project_path = get_dist_path_if_editable_install("lightning_cloud")
|
||||
if lightning_cloud_project_path:
|
||||
_prepare_wheel(lightning_cloud_project_path)
|
||||
tar_name = _copy_tar(lightning_cloud_project_path, root)
|
||||
tar_files.append(os.path.join(root, tar_name))
|
||||
|
||||
return functools.partial(_cleanup, *tar_files)
|
||||
|
||||
|
||||
def _enable_debugging():
|
||||
tar_file = os.path.join(os.getcwd(), f"lightning-{__version__}.tar.gz")
|
||||
tar_file = os.path.join(os.getcwd(), f"lightning-{version}.tar.gz")
|
||||
|
||||
if not os.path.exists(tar_file):
|
||||
return
|
||||
|
|
@ -138,7 +165,7 @@ def _fetch_latest_version(package_name: str) -> str:
|
|||
if proc.stdout:
|
||||
logs = " ".join([line.decode("utf-8") for line in iter(proc.stdout.readline, b"")])
|
||||
return logs.split(")\n")[0].split(",")[-1].replace(" ", "")
|
||||
return __version__
|
||||
return version
|
||||
|
||||
|
||||
def _verify_lightning_version():
|
||||
|
|
@ -149,7 +176,7 @@ def _verify_lightning_version():
|
|||
|
||||
lightning_latest_version = _fetch_latest_version("lightning")
|
||||
|
||||
if Version(lightning_latest_version) > Version(__version__):
|
||||
if Version(lightning_latest_version) > Version(version):
|
||||
raise Exception(
|
||||
f"You need to use the latest version of Lightning ({lightning_latest_version}) to run in the cloud. "
|
||||
"Please, run `pip install -U lightning`"
|
||||
|
|
|
|||
|
|
@ -56,4 +56,4 @@ def test_copy_and_setup_react_ui(tmpdir):
|
|||
def test_correct_num_react_template_files():
|
||||
template_dir = os.path.join(la.__path__[0], "cli/react-ui-template")
|
||||
files = cmd_init._ls_recursively(template_dir)
|
||||
assert len(files) == 15, "react-ui template files must be minimal... do not add nice to haves"
|
||||
assert len(files) == 16, "react-ui template files must be minimal... do not add nice to haves"
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import os
|
|||
|
||||
import pytest
|
||||
|
||||
from lightning import __about__
|
||||
from lightning.__version__ import version
|
||||
from lightning_app.testing.helpers import RunIf
|
||||
from lightning_app.utilities.packaging import lightning_utils
|
||||
from lightning_app.utilities.packaging.lightning_utils import (
|
||||
|
|
@ -15,7 +15,7 @@ def test_prepare_lightning_wheels_and_requirement(tmpdir):
|
|||
"""This test ensures the lightning source gets packaged inside the lightning repo."""
|
||||
|
||||
cleanup_handle = _prepare_lightning_wheels_and_requirements(tmpdir)
|
||||
tar_name = f"lightning-{__about__.__version__}.tar.gz"
|
||||
tar_name = f"lightning-{version}.tar.gz"
|
||||
assert sorted(os.listdir(tmpdir)) == [tar_name]
|
||||
cleanup_handle()
|
||||
assert os.listdir(tmpdir) == []
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ def test_execute_git_command():
|
|||
res = execute_git_command(["pull"])
|
||||
assert res
|
||||
|
||||
assert get_dir_name() == "lightning-app"
|
||||
assert get_dir_name() == "lightning"
|
||||
|
||||
assert check_github_repository()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue