LAI: creating mirror package (#15105)
* placeholder * mirror + prune * makedir * setup * ci * ci * name * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * ci clean * empty * py * parallel * doctest * flake8 * ci * typo * replace * clean * Apply suggestions from code review * re.sub * fix UI path * full replace * ui path? * replace * updates * regex * ci * fix * ci * path * ci * replace * Update .actions/setup_tools.py Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com> * also convert lightning_lite tests for PL tests to adapt mocking paths * fix app example test * update logger propagation for PL tests * update logger propagation for PL tests * Apply suggestions from code review * Revert "update logger propagation for PL tests" This reverts commitc1a5e119c7. * playwright * py * update import in tests * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * try edit import in overwrite * debug code * rev playwright * Revert "try edit import in overwrite" This reverts commitc02f766521. * ci: adjust examples * adjust examples cloud * mock lightning_app * Install assistant dependencies * lightning * setup * Apply suggestions from code review * Apply suggestions from code review Co-authored-by: Carlos Mocholí <carlossmocholi@gmail.com> * Apply suggestions from code review * disable cache * move doctest to install * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * ) * echo ./ * ci * lru * revert disabling cache, prints * ci * prune ci jobs * prune ci jobs * training loop standalone tests * add sys modules cleanup fixture * make use of fixture * revert standalone * ci e2e * fix imports in lightning * fix imports of lightning in tests * Revert "make use of fixture" This reverts commitc15efdd205. * Revert other commits for fixtures * revert use of fixture * py3.9 * fix mocking * fix paths * hack mocking * docs * Apply suggestions from code review * rev suggestion * Minor changes to the parametrizations * Update checkgroup with the new and changed jobs * include frontend dir * cli * fix imports and entry point * Revert standalone * rc1 * e2e on staging * Revert "Revert standalone" This reverts commit9df96685b8. * groups * to * ci: pt ver * docker * Apply suggestions from code review * Copy over changes from previous commit to other groups * Add back changes from bad merge * Uppercase step name everywhere * update * ci * ci: lai oldest Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Justus Schock <12886177+justusschock@users.noreply.github.com> Co-authored-by: Justus Schock <justus.schock@posteo.de> Co-authored-by: manskx <ahmed.mansy156@gmail.com> Co-authored-by: Carlos Mocholí <carlossmocholi@gmail.com> Co-authored-by: thomas chaton <thomas@grid.ai> Co-authored-by: Luca Antiga <luca.antiga@gmail.com>
This commit is contained in:
Jirka Borovec
GitHub
parent
6426d43235
commit
95ae393ca8
|
|
@ -12,7 +12,6 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
|
|
@ -23,9 +22,9 @@ import urllib.request
|
|||
from datetime import datetime
|
||||
from distutils.version import LooseVersion
|
||||
from importlib.util import module_from_spec, spec_from_file_location
|
||||
from itertools import chain, groupby
|
||||
from itertools import chain
|
||||
from types import ModuleType
|
||||
from typing import List, Sequence
|
||||
from typing import Dict, List, Sequence
|
||||
|
||||
from pkg_resources import parse_requirements
|
||||
|
||||
|
|
@ -114,8 +113,8 @@ def load_requirements(
|
|||
"""Loading requirements from a file.
|
||||
|
||||
>>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
|
||||
>>> load_requirements(path_req, unfreeze="major") # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
|
||||
['pytorch_lightning...', 'lightning_app...']
|
||||
>>> load_requirements(path_req, "docs.txt", unfreeze="major") # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
|
||||
['sphinx>=4.0, <6.0 # strict', ...]
|
||||
"""
|
||||
with open(os.path.join(path_dir, file_name)) as file:
|
||||
lines = [ln.strip() for ln in file.readlines()]
|
||||
|
|
@ -165,221 +164,6 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
|
|||
return text
|
||||
|
||||
|
||||
def replace_block_with_imports(lines: List[str], import_path: str, kword: str = "class") -> List[str]:
|
||||
"""Parse a file and replace implementtaions bodies of function or class.
|
||||
|
||||
>>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "loggers", "logger.py")
|
||||
>>> import_path = ".".join(["pytorch_lightning", "loggers", "logger"])
|
||||
>>> with open(py_file, encoding="utf-8") as fp:
|
||||
... lines = [ln.rstrip() for ln in fp.readlines()]
|
||||
>>> lines = replace_block_with_imports(lines, import_path, "class")
|
||||
>>> lines = replace_block_with_imports(lines, import_path, "def")
|
||||
"""
|
||||
body, tracking, skip_offset = [], False, 0
|
||||
for i, ln in enumerate(lines):
|
||||
# support for defining a class with this condition
|
||||
conditional_class_definitions = ("if TYPE_CHECKING", "if typing.TYPE_CHECKING", "if torch.", "if _TORCH_")
|
||||
if (
|
||||
any(ln.startswith(pattern) for pattern in conditional_class_definitions)
|
||||
# avoid bug in CI for the <1.7 meta code
|
||||
and "pytorch_lightning.utilities.meta" not in import_path
|
||||
):
|
||||
# dedent the next line
|
||||
lines[i + 1] = lines[i + 1].lstrip()
|
||||
continue
|
||||
|
||||
offset = len(ln) - len(ln.lstrip())
|
||||
# in case of mating the class args are multi-line
|
||||
if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]"):
|
||||
tracking = False
|
||||
if ln.lstrip().startswith(f"{kword} ") and not tracking:
|
||||
name = ln.replace(f"{kword} ", "").strip()
|
||||
idxs = [name.index(c) for c in ":(" if c in name]
|
||||
name = name[: min(idxs)]
|
||||
# skip private, TODO: consider skip even protected
|
||||
if not name.startswith("__"):
|
||||
body.append(f"{' ' * offset}from {import_path} import {name} # noqa: F401")
|
||||
tracking, skip_offset = True, offset
|
||||
continue
|
||||
if not tracking:
|
||||
body.append(ln)
|
||||
return body
|
||||
|
||||
|
||||
def replace_vars_with_imports(lines: List[str], import_path: str) -> List[str]:
|
||||
"""Parse a file and replace variable filling with import.
|
||||
|
||||
>>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "utilities", "imports.py")
|
||||
>>> import_path = ".".join(["pytorch_lightning", "utilities", "imports"])
|
||||
>>> with open(py_file, encoding="utf-8") as fp:
|
||||
... lines = [ln.rstrip() for ln in fp.readlines()]
|
||||
>>> lines = replace_vars_with_imports(lines, import_path)
|
||||
"""
|
||||
copied = []
|
||||
body, tracking, skip_offset = [], False, 0
|
||||
for ln in lines:
|
||||
offset = len(ln) - len(ln.lstrip())
|
||||
# in case of mating the class args are multi-line
|
||||
if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]}"):
|
||||
tracking = False
|
||||
var = re.match(r"^([\w_\d]+)[: [\w\., \[\]]*]? = ", ln.lstrip())
|
||||
if var:
|
||||
name = var.groups()[0]
|
||||
# skip private or apply white-list for allowed vars
|
||||
if name not in copied and (not name.startswith("__") or name in ("__all__",)):
|
||||
body.append(f"{' ' * offset}from {import_path} import {name} # noqa: F401")
|
||||
copied.append(name)
|
||||
tracking, skip_offset = True, offset
|
||||
continue
|
||||
if not tracking:
|
||||
body.append(ln)
|
||||
return body
|
||||
|
||||
|
||||
def prune_imports_callables(lines: List[str]) -> List[str]:
|
||||
"""Prune imports and calling functions from a file, even multi-line.
|
||||
|
||||
>>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "utilities", "cli.py")
|
||||
>>> import_path = ".".join(["pytorch_lightning", "utilities", "cli"])
|
||||
>>> with open(py_file, encoding="utf-8") as fp:
|
||||
... lines = [ln.rstrip() for ln in fp.readlines()]
|
||||
>>> lines = prune_imports_callables(lines)
|
||||
"""
|
||||
body, tracking, skip_offset = [], False, 0
|
||||
for ln in lines:
|
||||
if ln.lstrip().startswith("import "):
|
||||
continue
|
||||
offset = len(ln) - len(ln.lstrip())
|
||||
# in case of mating the class args are multi-line
|
||||
if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]}"):
|
||||
tracking = False
|
||||
# catching callable
|
||||
call = re.match(r"^[\w_\d\.]+\(", ln.lstrip())
|
||||
if (ln.lstrip().startswith("from ") and " import " in ln) or call:
|
||||
tracking, skip_offset = True, offset
|
||||
continue
|
||||
if not tracking:
|
||||
body.append(ln)
|
||||
return body
|
||||
|
||||
|
||||
def prune_func_calls(lines: List[str]) -> List[str]:
|
||||
"""Prune calling functions from a file, even multi-line.
|
||||
|
||||
>>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "loggers", "__init__.py")
|
||||
>>> import_path = ".".join(["pytorch_lightning", "loggers"])
|
||||
>>> with open(py_file, encoding="utf-8") as fp:
|
||||
... lines = [ln.rstrip() for ln in fp.readlines()]
|
||||
>>> lines = prune_func_calls(lines)
|
||||
"""
|
||||
body, tracking, score = [], False, 0
|
||||
for ln in lines:
|
||||
# catching callable
|
||||
calling = re.match(r"^@?[\w_\d\.]+ *\(", ln.lstrip())
|
||||
if calling and " import " not in ln:
|
||||
tracking = True
|
||||
score = 0
|
||||
if tracking:
|
||||
score += ln.count("(") - ln.count(")")
|
||||
if score == 0:
|
||||
tracking = False
|
||||
else:
|
||||
body.append(ln)
|
||||
return body
|
||||
|
||||
|
||||
def prune_empty_statements(lines: List[str]) -> List[str]:
|
||||
"""Prune emprty if/else and try/except.
|
||||
|
||||
>>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "utilities", "cli.py")
|
||||
>>> import_path = ".".join(["pytorch_lightning", "utilities", "cli"])
|
||||
>>> with open(py_file, encoding="utf-8") as fp:
|
||||
... lines = [ln.rstrip() for ln in fp.readlines()]
|
||||
>>> lines = prune_imports_callables(lines)
|
||||
>>> lines = prune_empty_statements(lines)
|
||||
"""
|
||||
kwords_pairs = ("with", "if ", "elif ", "else", "try", "except")
|
||||
body, tracking, skip_offset, last_count = [], False, 0, 0
|
||||
# todo: consider some more complex logic as for example only some leaves of if/else tree are empty
|
||||
for i, ln in enumerate(lines):
|
||||
offset = len(ln) - len(ln.lstrip())
|
||||
# skipp all decorators
|
||||
if ln.lstrip().startswith("@"):
|
||||
# consider also multi-line decorators
|
||||
if "(" in ln and ")" not in ln:
|
||||
tracking, skip_offset = True, offset
|
||||
continue
|
||||
# in case of mating the class args are multi-line
|
||||
if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]}"):
|
||||
tracking = False
|
||||
starts = [k for k in kwords_pairs if ln.lstrip().startswith(k)]
|
||||
if starts:
|
||||
start, count = starts[0], -1
|
||||
# look forward if this statement has a body
|
||||
for ln_ in lines[i:]:
|
||||
offset_ = len(ln_) - len(ln_.lstrip())
|
||||
if count == -1 and ln_.rstrip().endswith(":"):
|
||||
count = 0
|
||||
elif ln_ and offset_ <= offset:
|
||||
break
|
||||
# skipp all til end of statement
|
||||
elif ln_.lstrip():
|
||||
# count non-zero body lines
|
||||
count += 1
|
||||
# cache the last key body as the supplement canot be without
|
||||
if start in ("if", "elif", "try"):
|
||||
last_count = count
|
||||
if count <= 0 or (start in ("else", "except") and last_count <= 0):
|
||||
tracking, skip_offset = True, offset
|
||||
if not tracking:
|
||||
body.append(ln)
|
||||
return body
|
||||
|
||||
|
||||
def prune_comments_docstrings(lines: List[str]) -> List[str]:
|
||||
"""Prune all doctsrings with triple " notation.
|
||||
|
||||
>>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "loggers", "csv_logs.py")
|
||||
>>> import_path = ".".join(["pytorch_lightning", "loggers", "csv_logs"])
|
||||
>>> with open(py_file, encoding="utf-8") as fp:
|
||||
... lines = [ln.rstrip() for ln in fp.readlines()]
|
||||
>>> lines = prune_comments_docstrings(lines)
|
||||
"""
|
||||
body, tracking = [], False
|
||||
for ln in lines:
|
||||
if "#" in ln and "noqa:" not in ln:
|
||||
ln = ln[: ln.index("#")]
|
||||
if not tracking and any(ln.lstrip().startswith(s) for s in ['"""', 'r"""']):
|
||||
# oneliners skip directly
|
||||
if len(ln.strip()) >= 6 and ln.rstrip().endswith('"""'):
|
||||
continue
|
||||
tracking = True
|
||||
elif ln.rstrip().endswith('"""'):
|
||||
tracking = False
|
||||
continue
|
||||
if not tracking:
|
||||
body.append(ln.rstrip())
|
||||
return body
|
||||
|
||||
|
||||
def wrap_try_except(body: List[str], pkg: str, ver: str) -> List[str]:
|
||||
"""Wrap the file with try/except for better traceability of import misalignment."""
|
||||
not_empty = sum(1 for ln in body if ln)
|
||||
if not_empty == 0:
|
||||
return body
|
||||
body = ["try:"] + [f" {ln}" if ln else "" for ln in body]
|
||||
body += [
|
||||
"",
|
||||
"except ImportError as err:",
|
||||
"",
|
||||
" from os import linesep",
|
||||
f" from {pkg} import __version__",
|
||||
f" msg = f'Your `lightning` package was built for `{pkg}=={ver}`," + " but you are running {__version__}'",
|
||||
" raise type(err)(str(err) + linesep + msg)",
|
||||
]
|
||||
return body
|
||||
|
||||
|
||||
def parse_version_from_file(pkg_root: str) -> str:
|
||||
"""Loading the package version from file."""
|
||||
file_ver = os.path.join(pkg_root, "__version__.py")
|
||||
|
|
@ -393,74 +177,64 @@ def parse_version_from_file(pkg_root: str) -> str:
|
|||
return ver
|
||||
|
||||
|
||||
def prune_duplicate_lines(body):
|
||||
body_ = []
|
||||
# drop duplicated lines
|
||||
for ln in body:
|
||||
if ln.lstrip() not in body_ or ln.lstrip() in (")", ""):
|
||||
body_.append(ln)
|
||||
return body_
|
||||
def _replace_imports_in_file(lines: List[str], pkg_lut: Dict[str, str]) -> List[str]:
|
||||
"""Replace imports of standalone package to lightning.
|
||||
|
||||
|
||||
def create_meta_package(src_folder: str, pkg_name: str = "pytorch_lightning", lit_name: str = "pytorch"):
|
||||
"""Parse the real python package and for each module create a mirroe version with repalcing all function and
|
||||
class implementations by cross-imports to the true package.
|
||||
|
||||
As validation run in termnal: `flake8 src/lightning/ --ignore E402,F401,E501`
|
||||
|
||||
>>> create_meta_package(os.path.join(_PROJECT_ROOT, "src"))
|
||||
>>> lns = ["lightning_app",
|
||||
... "delete_cloud_lightning_apps",
|
||||
... "from lightning_app import",
|
||||
... "lightning_apps = []",
|
||||
... "lightning_app is ours",
|
||||
... "def _lightning_app():",
|
||||
... ":class:`~lightning_app.core.flow.LightningFlow`"]
|
||||
>>> from pprint import pprint
|
||||
>>> pprint(_replace_imports_in_file(lns, {"app": "lightning_app"}))
|
||||
['lightning.app',
|
||||
'delete_cloud_lightning_apps',
|
||||
'from lightning.app import',
|
||||
'lightning_apps = []',
|
||||
'lightning.app is ours',
|
||||
'def _lightning_app():',
|
||||
':class:`~lightning.app.core.flow.LightningFlow`']
|
||||
"""
|
||||
for n2, n1 in pkg_lut.items():
|
||||
for i, ln in enumerate(lines):
|
||||
lines[i] = re.sub(rf"([^_]|^){n1}([^_\w]|$)", rf"\1lightning.{n2}\2", ln)
|
||||
return lines
|
||||
|
||||
|
||||
# TODO: unify usage with assistant function, such that import this function in there
|
||||
def copy_adjusted_modules(src_folder: str, pkg_name: str, lit_name: str, pkg_lut: dict) -> None:
|
||||
"""Recursively replace imports in given folder."""
|
||||
package_dir = os.path.join(src_folder, pkg_name)
|
||||
pkg_ver = parse_version_from_file(package_dir)
|
||||
# shutil.rmtree(os.path.join(src_folder, "lightning", lit_name))
|
||||
py_files = glob.glob(os.path.join(src_folder, pkg_name, "**", "*.py"), recursive=True)
|
||||
for py_file in py_files:
|
||||
local_path = py_file.replace(package_dir + os.path.sep, "")
|
||||
fname = os.path.basename(py_file)
|
||||
if "-" in local_path:
|
||||
continue
|
||||
with open(py_file, encoding="utf-8") as fp:
|
||||
lines = [ln.rstrip() for ln in fp.readlines()]
|
||||
import_path = pkg_name + "." + local_path.replace(".py", "").replace(os.path.sep, ".")
|
||||
import_path = import_path.replace(".__init__", "")
|
||||
|
||||
if fname in ("__about__.py", "__version__.py"):
|
||||
body = lines
|
||||
else:
|
||||
if fname.startswith("_") and fname not in ("__init__.py", "__main__.py"):
|
||||
logging.warning(f"unsupported file: {local_path}")
|
||||
continue
|
||||
# ToDO: perform some smarter parsing - preserve Constants, lambdas, etc
|
||||
body = prune_comments_docstrings([ln.rstrip() for ln in lines])
|
||||
if fname not in ("__init__.py", "__main__.py"):
|
||||
body = prune_imports_callables(body)
|
||||
for key_word in ("class", "def", "async def"):
|
||||
body = replace_block_with_imports(body, import_path, key_word)
|
||||
# TODO: fix reimporting which is artefact after replacing var assignment with import;
|
||||
# after fixing , update CI by remove F811 from CI/check pkg
|
||||
body = replace_vars_with_imports(body, import_path)
|
||||
if fname not in ("__main__.py",):
|
||||
body = prune_func_calls(body)
|
||||
body_len = -1
|
||||
# in case of several in-depth statements
|
||||
while body_len != len(body):
|
||||
body_len = len(body)
|
||||
body = prune_duplicate_lines(body)
|
||||
body = prune_empty_statements(body)
|
||||
# add try/catch wrapper for whole body,
|
||||
# so when import fails it tells you what is the package version this meta package was generated for...
|
||||
body = wrap_try_except(body, pkg_name, pkg_ver)
|
||||
|
||||
# todo: apply pre-commit formatting
|
||||
# clean to many empty lines
|
||||
body = [ln for ln, _group in groupby(body)]
|
||||
# drop duplicated lines
|
||||
body = prune_duplicate_lines(body)
|
||||
# compose the target file name
|
||||
all_files = glob.glob(os.path.join(package_dir, "**", "*.*"), recursive=True)
|
||||
for fname in all_files:
|
||||
local_path = fname.replace(package_dir + os.path.sep, "")
|
||||
new_file = os.path.join(src_folder, "lightning", lit_name, local_path)
|
||||
if not fname.endswith(".py"):
|
||||
if not fname.endswith(".pyc"):
|
||||
os.makedirs(os.path.dirname(new_file), exist_ok=True)
|
||||
shutil.copy2(fname, new_file)
|
||||
continue
|
||||
|
||||
with open(fname, encoding="utf-8") as fo:
|
||||
py = fo.readlines()
|
||||
py = _replace_imports_in_file(py, pkg_lut)
|
||||
os.makedirs(os.path.dirname(new_file), exist_ok=True)
|
||||
with open(new_file, "w", encoding="utf-8") as fp:
|
||||
fp.writelines([ln + os.linesep for ln in body])
|
||||
with open(new_file, "w", encoding="utf-8") as fo:
|
||||
fo.writelines(py)
|
||||
|
||||
|
||||
def create_mirror_package(src_folder: str, lit_pkg_mapping: dict) -> None:
|
||||
"""Recursively replace imports in given folder.
|
||||
|
||||
>>> create_mirror_package(
|
||||
... os.path.join(_PROJECT_ROOT, "src"),
|
||||
... {"pytorch": "pytorch_lightning", "app": "lightning_app", "lite": "lightning_lite"}
|
||||
... )
|
||||
"""
|
||||
for lit_name, pkg_name in lit_pkg_mapping.items():
|
||||
copy_adjusted_modules(src_folder, pkg_name, lit_name, lit_pkg_mapping)
|
||||
|
||||
|
||||
def set_version_today(fpath: str) -> None:
|
||||
|
|
@ -477,12 +251,12 @@ def set_version_today(fpath: str) -> None:
|
|||
fp.writelines(lines)
|
||||
|
||||
|
||||
def _download_frontend(root: str = _PROJECT_ROOT):
|
||||
def _download_frontend(pkg_path: str):
|
||||
"""Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct
|
||||
directory."""
|
||||
|
||||
try:
|
||||
frontend_dir = pathlib.Path(root, "src", "lightning_app", "ui")
|
||||
frontend_dir = pathlib.Path(pkg_path, "ui")
|
||||
download_dir = tempfile.mkdtemp()
|
||||
|
||||
shutil.rmtree(frontend_dir, ignore_errors=True)
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ jobs:
|
|||
path: $(pip_cache_dir)
|
||||
displayName: Cache pip
|
||||
|
||||
# TODO: we are testing it as `lightning`, so add also version for `lightning_app`
|
||||
- bash: git restore . && python -m pip install -e . -r requirements/app/test.txt -r requirements/app/ui.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
|
||||
displayName: 'Install Lightning & dependencies'
|
||||
|
||||
|
|
@ -125,6 +126,16 @@ jobs:
|
|||
condition: eq(variables['name'], 'quick_start')
|
||||
displayName: 'Install Quick Start'
|
||||
|
||||
- bash: python -m pip install -q -r .actions/requirements.txt
|
||||
displayName: 'Install assistant dependencies'
|
||||
|
||||
# Fix imports to use `lightning` instead of `lightning_app` since we install lightning only ATM
|
||||
# TODO: fixme when installing `lightning_app` as well
|
||||
- bash: |
|
||||
python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning_app" --target_import="lightning.app"
|
||||
python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
|
||||
displayName: 'Adjust examples'
|
||||
|
||||
- bash: |
|
||||
pip --version
|
||||
pip list
|
||||
|
|
@ -144,12 +155,12 @@ jobs:
|
|||
TEST_APP_NAME: $(name)
|
||||
HAR_LOCATION: './artifacts/hars'
|
||||
SLOW_MO: '50'
|
||||
# LAI_USER: $(LAI_USER)
|
||||
# LAI_PASS: $(LAI_PASS)
|
||||
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
|
||||
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
|
||||
LAI_USER: $(LAI_USER) # for STAGING
|
||||
LAI_PASS: $(LAI_PASS) # for STAGING
|
||||
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_STAGING)
|
||||
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_STAGING)
|
||||
LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
|
||||
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
|
||||
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_STAGING)
|
||||
LIGHTNING_DEBUG: '1'
|
||||
displayName: 'Run the tests'
|
||||
|
||||
|
|
@ -162,12 +173,12 @@ jobs:
|
|||
time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
|
||||
condition: always()
|
||||
env:
|
||||
# LAI_USER: $(LAI_USER)
|
||||
# LAI_PASS: $(LAI_PASS)
|
||||
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
|
||||
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
|
||||
LAI_USER: $(LAI_USER) # for STAGING
|
||||
LAI_PASS: $(LAI_PASS) # for STAGING
|
||||
LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_STAGING)
|
||||
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_STAGING)
|
||||
LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
|
||||
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
|
||||
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_STAGING)
|
||||
PR_NUMBER: $(local_id)
|
||||
TEST_APP_NAME: $(name)
|
||||
# GRID_USER_ID: $(LIGHTNING_USER_ID) # TODO: clarify the meaning
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ inputs:
|
|||
pkg-name:
|
||||
description: package name inside lightning.*
|
||||
required: true
|
||||
default: ""
|
||||
nb-dirs:
|
||||
description: nb of packages in the wrap/distribution
|
||||
required: false
|
||||
|
|
@ -21,10 +22,7 @@ runs:
|
|||
- name: Source check
|
||||
env:
|
||||
PACKAGE_NAME: ${{ inputs.pkg-name }}
|
||||
run: |
|
||||
python setup.py check --metadata --strict
|
||||
# TODO: fix reimporting (F811) which is aftefact after rplacing var assigne with import in meta package
|
||||
flake8 src/lightning/ --ignore E402,F401,E501,W391,E303,F811
|
||||
run: python setup.py check --metadata --strict
|
||||
shell: bash
|
||||
|
||||
- name: Create package
|
||||
|
|
@ -40,10 +38,6 @@ runs:
|
|||
# python setup.py clean
|
||||
shell: bash
|
||||
|
||||
- name: copy/export pkg
|
||||
run: cp dist/* pypi/
|
||||
shell: bash
|
||||
|
||||
- name: Unzip packages
|
||||
if: ${{ inputs.pkg-name != '' }}
|
||||
working-directory: dist
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ runs:
|
|||
run: echo "PKG_NAME=${{ inputs.pkg-name }}" >> $GITHUB_ENV
|
||||
shell: bash
|
||||
|
||||
- name: Install | Uninstall package - archive
|
||||
- name: Install package - archive
|
||||
working-directory: ./dist
|
||||
run: |
|
||||
pip install *.tar.gz ${{ inputs.pip-flags }}
|
||||
|
|
@ -33,7 +33,7 @@ runs:
|
|||
python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)"
|
||||
shell: bash
|
||||
|
||||
- name: Install | Uninstall package - wheel
|
||||
- name: Install package - wheel
|
||||
working-directory: ./dist
|
||||
run: |
|
||||
pip install *.whl ${{ inputs.pip-flags }}
|
||||
|
|
|
|||
|
|
@ -30,16 +30,23 @@ subprojects:
|
|||
- "pl-cpu (macOS-11, pytorch, 3.10, 1.12)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.11)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.8, 1.9)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.9, 1.10)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.10, 1.11)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.10, 1.12)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)"
|
||||
- "pl-cpu (macOS-11, lightning, 3.10, 1.12)"
|
||||
#- "pl-cpu (macOS-11, lightning, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)"
|
||||
#- "pl-cpu (ubuntu-20.04, lightning, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (windows-2022, lightning, 3.10, 1.12)"
|
||||
#- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)"
|
||||
- "make-doctest (pytorch)"
|
||||
- "make-html (pytorch)"
|
||||
- "mypy"
|
||||
|
|
@ -56,21 +63,28 @@ subprojects:
|
|||
paths:
|
||||
- ".github/workflows/ci-pytorch-tests.yml"
|
||||
checks:
|
||||
- "pl-cpu (macOS-11, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.8, 1.10)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.10, 1.12)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.11)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.8, 1.9)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.9, 1.10)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.10, 1.11)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.10, 1.12)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)"
|
||||
- "pl-cpu (macOS-11, lightning, 3.10, 1.12)"
|
||||
#- "pl-cpu (macOS-11, lightning, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)"
|
||||
#- "pl-cpu (ubuntu-20.04, lightning, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (windows-2022, lightning, 3.10, 1.12)"
|
||||
#- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)"
|
||||
|
||||
- id: "pytorch_lightning: Slow"
|
||||
paths:
|
||||
|
|
@ -165,24 +179,34 @@ subprojects:
|
|||
- "lite-cpu (windows-2022, lite, 3.10, 1.12)"
|
||||
- "lite-cpu (windows-2022, lite, 3.7, 1.9, oldest)"
|
||||
- "lite-cpu (windows-2022, lite, 3.8, 1.13, pre)"
|
||||
- "lite-cpu (macOS-11, lightning, 3.8, 1.12)"
|
||||
- "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)"
|
||||
- "lite-cpu (windows-2022, lightning, 3.8, 1.12)"
|
||||
- "lightning-lite (GPUs)"
|
||||
- "mypy"
|
||||
# Lite also requires PL checks as it depends on Lite
|
||||
- "pl-cpu (macOS-11, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.8, 1.10)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.10, 1.12)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.11)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.8, 1.9)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.9, 1.10)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.9, 1.11)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.10, 1.11)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.10, 1.12)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)"
|
||||
- "pl-cpu (macOS-11, lightning, 3.10, 1.12)"
|
||||
#- "pl-cpu (macOS-11, lightning, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)"
|
||||
#- "pl-cpu (ubuntu-20.04, lightning, 3.7, 1.9, oldest)"
|
||||
- "pl-cpu (windows-2022, lightning, 3.10, 1.12)"
|
||||
#- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)"
|
||||
- "make-doctest (pytorch)"
|
||||
- "make-html (pytorch)"
|
||||
- "pytorch-lightning (GPUs)"
|
||||
|
|
@ -210,6 +234,9 @@ subprojects:
|
|||
- "lite-cpu (windows-2022, lite, 3.10, 1.12)"
|
||||
- "lite-cpu (windows-2022, lite, 3.7, 1.9, oldest)"
|
||||
- "lite-cpu (windows-2022, lite, 3.8, 1.13, pre)"
|
||||
- "lite-cpu (macOS-11, lightning, 3.8, 1.12)"
|
||||
- "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)"
|
||||
- "lite-cpu (windows-2022, lightning, 3.8, 1.12)"
|
||||
- "lightning-lite (GPUs)"
|
||||
|
||||
- id: "lightning_lite: Azure GPU"
|
||||
|
|
@ -235,18 +262,24 @@ subprojects:
|
|||
- "App.cloud-e2e"
|
||||
- "make-doctest (app)"
|
||||
- "make-html (app)"
|
||||
- "app-examples (macOS-11, app, 3.8, latest)"
|
||||
- "app-examples (macOS-11, app, 3.8, oldest)"
|
||||
- "app-examples (ubuntu-20.04, app, 3.8, latest)"
|
||||
- "app-examples (ubuntu-20.04, app, 3.8, oldest)"
|
||||
- "app-examples (windows-2022, app, 3.8, latest)"
|
||||
- "app-examples (windows-2022, app, 3.8, oldest)"
|
||||
- "app-examples (macOS-11, app, 3.9, latest)"
|
||||
- "app-examples (macOS-11, app, 3.9, oldest)"
|
||||
- "app-examples (macOS-11, lightning, 3.9, latest)"
|
||||
- "app-examples (ubuntu-20.04, app, 3.9, latest)"
|
||||
- "app-examples (ubuntu-20.04, app, 3.9, oldest)"
|
||||
- "app-examples (ubuntu-20.04, lightning, 3.9, latest)"
|
||||
- "app-examples (windows-2022, app, 3.9, latest)"
|
||||
- "app-examples (windows-2022, app, 3.9, oldest)"
|
||||
- "app-examples (windows-2022, lightning, 3.9, latest)"
|
||||
- "app-pytest (macOS-11, app, 3.8, latest)"
|
||||
- "app-pytest (macOS-11, app, 3.8, oldest)"
|
||||
- "app-pytest (macOS-11, lightning, 3.9, latest)"
|
||||
- "app-pytest (ubuntu-20.04, app, 3.8, latest)"
|
||||
- "app-pytest (ubuntu-20.04, app, 3.8, oldest)"
|
||||
- "app-pytest (ubuntu-20.04, lightning, 3.9, latest)"
|
||||
- "app-pytest (windows-2022, app, 3.8, latest)"
|
||||
- "app-pytest (windows-2022, app, 3.8, oldest)"
|
||||
- "app-pytest (windows-2022, lightning, 3.9, latest)"
|
||||
|
||||
- id: "lightning_app: Azure"
|
||||
paths:
|
||||
|
|
@ -287,15 +320,21 @@ subprojects:
|
|||
- "install-pkg (ubuntu-22.04, lite, 3.10)"
|
||||
- "install-pkg (ubuntu-22.04, pytorch, 3.7)"
|
||||
- "install-pkg (ubuntu-22.04, pytorch, 3.10)"
|
||||
- "install-pkg (ubuntu-22.04, 3.7)"
|
||||
- "install-pkg (ubuntu-22.04, 3.10)"
|
||||
- "install-pkg (macOS-12, app, 3.7)"
|
||||
- "install-pkg (macOS-12, app, 3.10)"
|
||||
- "install-pkg (macOS-12, lite, 3.7)"
|
||||
- "install-pkg (macOS-12, lite, 3.10)"
|
||||
- "install-pkg (macOS-12, pytorch, 3.7)"
|
||||
- "install-pkg (macOS-12, pytorch, 3.10)"
|
||||
- "install-pkg (macOS-12, 3.7)"
|
||||
- "install-pkg (macOS-12, 3.10)"
|
||||
- "install-pkg (windows-2022, app, 3.7)"
|
||||
- "install-pkg (windows-2022, app, 3.10)"
|
||||
- "install-pkg (windows-2022, lite, 3.7)"
|
||||
- "install-pkg (windows-2022, lite, 3.10)"
|
||||
- "install-pkg (windows-2022, pytorch, 3.7)"
|
||||
- "install-pkg (windows-2022, pytorch, 3.10)"
|
||||
- "install-pkg (windows-2022, 3.7)"
|
||||
- "install-pkg (windows-2022, 3.10)"
|
||||
|
|
|
|||
|
|
@ -32,12 +32,17 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-20.04, macOS-11, windows-2022]
|
||||
pkg-name: ["app"] # TODO: add "LAI" # "LAI" installs the monolithic package
|
||||
python-version: [3.8]
|
||||
pkg-name: ["app"]
|
||||
python-version: ["3.9"]
|
||||
requires: ["oldest", "latest"]
|
||||
include:
|
||||
# "lightning" installs the monolithic package
|
||||
- {os: "macOS-11", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
|
||||
- {os: "windows-2022", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
|
||||
|
||||
# Timeout: https://stackoverflow.com/a/59076067/4521646
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
|
@ -89,11 +94,17 @@ jobs:
|
|||
run: pip install -e .
|
||||
|
||||
- name: Adjust tests
|
||||
if: ${{ matrix.pkg-name == 'LAI' }}
|
||||
if: ${{ matrix.pkg-name == 'lightning' }}
|
||||
run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
|
||||
|
||||
- name: switch coverage scope
|
||||
run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_app'))" >> $GITHUB_ENV
|
||||
- name: Adjust examples
|
||||
if: ${{ matrix.pkg-name != 'lightning' }}
|
||||
run: |
|
||||
python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app"
|
||||
python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app"
|
||||
|
||||
- name: Switch coverage scope
|
||||
run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV
|
||||
|
||||
- name: Tests
|
||||
working-directory: ./tests
|
||||
|
|
|
|||
|
|
@ -33,12 +33,17 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-20.04, macOS-11, windows-2022]
|
||||
pkg-name: ["app"] # TODO: add "LAI" # "LAI" installs the monolithic package
|
||||
python-version: [3.8]
|
||||
pkg-name: ["app"]
|
||||
python-version: ["3.8"]
|
||||
requires: ["oldest", "latest"]
|
||||
include:
|
||||
# "lightning" installs the monolithic package
|
||||
- {os: "macOS-11", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
|
||||
- {os: "windows-2022", pkg-name: "lightning", python-version: "3.9", requires: "latest"}
|
||||
|
||||
# Timeout: https://stackoverflow.com/a/59076067/4521646
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
|
@ -54,21 +59,6 @@ jobs:
|
|||
if: ${{ matrix.requires == 'oldest' }}
|
||||
run: python .actions/assistant.py replace_oldest_ver
|
||||
|
||||
- name: switch PyTorch URL
|
||||
run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
|
||||
|
||||
- name: Install package
|
||||
env:
|
||||
PACKAGE_NAME: ${{ matrix.pkg-name }}
|
||||
run: |
|
||||
pip install -e . pytest --upgrade --find-links ${TORCH_URL}
|
||||
pip list
|
||||
|
||||
- name: DocTests App
|
||||
if: ${{ matrix.pkg-name != 'LAI' }}
|
||||
working-directory: src
|
||||
run: python -m pytest lightning_app --ignore-glob="**/cli/*-template/**"
|
||||
|
||||
- name: Get pip cache
|
||||
id: pip-cache
|
||||
run: |
|
||||
|
|
@ -81,6 +71,15 @@ jobs:
|
|||
key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.requires }}-${{ hashFiles('requirements/app/base.txt') }}
|
||||
restore-keys: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.requires }}-
|
||||
|
||||
- name: Switch PyTorch URL
|
||||
run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
|
||||
|
||||
- name: Install package
|
||||
env:
|
||||
PACKAGE_NAME: ${{ matrix.pkg-name }}
|
||||
run: |
|
||||
pip install -e . pytest --upgrade --find-links ${TORCH_URL}
|
||||
pip list
|
||||
|
||||
- name: Install dependencies
|
||||
env: # TODO: drop this when we will be using regular releases for testing
|
||||
|
|
@ -98,17 +97,17 @@ jobs:
|
|||
run: npm install -g yarn
|
||||
|
||||
- name: Adjust tests
|
||||
if: ${{ matrix.pkg-name == 'LAI' }}
|
||||
if: ${{ matrix.pkg-name == 'lightning' }}
|
||||
run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app"
|
||||
|
||||
- name: Adjust examples
|
||||
if: ${{ matrix.pkg-name != 'LAI' }}
|
||||
if: ${{ matrix.pkg-name != 'lightning' }}
|
||||
run: |
|
||||
python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app"
|
||||
python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app"
|
||||
|
||||
- name: switch coverage scope
|
||||
run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_app'))" >> $GITHUB_ENV
|
||||
- name: Switch coverage scope
|
||||
run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV
|
||||
|
||||
- name: Tests
|
||||
working-directory: ./tests
|
||||
|
|
|
|||
|
|
@ -54,8 +54,12 @@ jobs:
|
|||
- {os: "macOS-11", pkg-name: "lite", python-version: "3.10", pytorch-version: "1.13", release: "pre"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "lite", python-version: "3.9", pytorch-version: "1.13", release: "pre"}
|
||||
- {os: "windows-2022", pkg-name: "lite", python-version: "3.8", pytorch-version: "1.13", release: "pre"}
|
||||
# "lightning" installs the monolithic package
|
||||
- {os: "macOS-11", pkg-name: "lightning", python-version: "3.8", pytorch-version: "1.12"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.8", pytorch-version: "1.12"}
|
||||
- {os: "windows-2022", pkg-name: "lightning", python-version: "3.8", pytorch-version: "1.12"}
|
||||
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
|
@ -108,9 +112,10 @@ jobs:
|
|||
PACKAGE_NAME: ${{ matrix.pkg-name }}
|
||||
run: |
|
||||
pip install -e . "pytest-timeout" -r requirements/lite/devel.txt --upgrade --find-links ${TORCH_URL}
|
||||
pip list
|
||||
|
||||
- name: Adjust tests
|
||||
if: ${{ matrix.pkg-name == 'LAI' }}
|
||||
if: ${{ matrix.pkg-name == 'lightning' }}
|
||||
run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_lite" --target_import="lightning.lite"
|
||||
|
||||
- name: Testing Warnings
|
||||
|
|
@ -121,7 +126,7 @@ jobs:
|
|||
run: python utilities/test_warnings.py
|
||||
|
||||
- name: Switch coverage scope
|
||||
run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_lite'))" >> $GITHUB_ENV
|
||||
run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_lite'))" >> $GITHUB_ENV
|
||||
|
||||
- name: Testing Lite
|
||||
working-directory: tests/tests_lite
|
||||
|
|
|
|||
|
|
@ -23,24 +23,13 @@ defaults:
|
|||
|
||||
jobs:
|
||||
|
||||
init-temp:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- run: mkdir pypi && touch pypi/.placeholder
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ci-packages-${{ github.sha }}
|
||||
path: pypi
|
||||
|
||||
install-pkg:
|
||||
needs: init-temp
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: true
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-22.04, macOS-12, windows-2022]
|
||||
pkg: ["app", "lite", "pytorch"]
|
||||
pkg: ["app", "lite", "pytorch", ""]
|
||||
python-version: ["3.7" , "3.10"]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
|
@ -51,112 +40,24 @@ jobs:
|
|||
- name: DocTests actions
|
||||
working-directory: .actions/
|
||||
run: |
|
||||
grep 'pytest=' ../requirements/pytorch/test.txt | xargs -0 pip install
|
||||
pip install pytest -q
|
||||
python -m pytest setup_tools.py
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: ci-packages-${{ github.sha }}
|
||||
path: pypi
|
||||
- run: ls -lh pypi/
|
||||
|
||||
- run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg }}' == 'pytorch' else 1))" >> $GITHUB_ENV
|
||||
- uses: ./.github/actions/pkg-check
|
||||
with:
|
||||
pkg-name: ${{ matrix.pkg }}
|
||||
nb-dirs: ${{ env.NB_DIRS }}
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ci-packages-${{ github.sha }}
|
||||
path: pypi
|
||||
|
||||
- uses: ./.github/actions/pkg-install
|
||||
|
||||
install-meta-src:
|
||||
needs: install-pkg
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# max-parallel: 1
|
||||
matrix:
|
||||
os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022]
|
||||
pkg: ["", "lightning"]
|
||||
python-version: [3.8] # , 3.9
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- run: mkdir -p pypi
|
||||
- uses: actions/download-artifact@v3
|
||||
if: matrix.pkg != ''
|
||||
with:
|
||||
name: ci-packages-${{ github.sha }}
|
||||
path: pypi
|
||||
- run: ls -lh pypi/
|
||||
|
||||
- uses: ./.github/actions/pkg-check
|
||||
with:
|
||||
pkg-name: ${{ matrix.pkg }}
|
||||
|
||||
- uses: ./.github/actions/pkg-install
|
||||
with:
|
||||
pkg-name: "lightning"
|
||||
pip-flags: "-U --pre --find-links ../pypi/"
|
||||
|
||||
- name: Run CLI
|
||||
# todo: add testing for `lightning_app`
|
||||
if: ${{ matrix.pkg == '' }}
|
||||
run: python -m lightning --version
|
||||
|
||||
install-meta-pypi:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# max-parallel: 1
|
||||
matrix:
|
||||
os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022]
|
||||
python-version: [3.8] # , 3.9
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Dowload package
|
||||
# todo: download also lite after it is fist published
|
||||
- name: DocTest package
|
||||
run: |
|
||||
pip install -q -r .actions/requirements.txt
|
||||
for pkg in 'app' 'pytorch' ; do
|
||||
python .actions/assistant.py download_package --package "$pkg" --folder pypi
|
||||
done
|
||||
ls -lh pypi/
|
||||
|
||||
- name: Unzip packages
|
||||
working-directory: pypi
|
||||
run: for file in `ls *.gz`; do tar -xzf $file; done
|
||||
- name: Show upacked pkgs
|
||||
if: runner.os == 'linux'
|
||||
run: |
|
||||
sudo apt install -y tree
|
||||
tree pypi/ -L 3
|
||||
|
||||
- name: Miror source
|
||||
run: |
|
||||
pip install -q -r .actions/requirements.txt
|
||||
python .actions/assistant.py mirror_pkg2source --pypi_folder pypi --src_folder src
|
||||
ls -R src/
|
||||
|
||||
- uses: ./.github/actions/pkg-check
|
||||
with:
|
||||
pkg-name: "lightning"
|
||||
|
||||
- uses: ./.github/actions/pkg-install
|
||||
with:
|
||||
pkg-name: "lightning"
|
||||
pip-flags: "-U --pre --find-links ../pypi/"
|
||||
|
||||
- name: Run CLI
|
||||
run: python -m lightning --version
|
||||
pip list
|
||||
scope=$(python -c "lut = {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning'} ; print(lut.get('${{matrix.pkg}}', 'lightning'))")
|
||||
python -m pytest src/${scope} --ignore-glob="**/cli/*-template/**"
|
||||
|
|
|
|||
|
|
@ -42,18 +42,26 @@ jobs:
|
|||
# note: there's no distribution of Torch==1.9 for Python>=3.9 or torch==1.10 for Python>=3.10
|
||||
- {os: "macOS-11", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.11"}
|
||||
- {os: "macOS-11", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.10"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.10"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.11"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.11"}
|
||||
- {os: "windows-2022", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.10"}
|
||||
- {os: "windows-2022", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.11"}
|
||||
- {os: "windows-2022", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.11"}
|
||||
- {os: "windows-2022", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.10"}
|
||||
- {os: "windows-2022", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.9"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.11"}
|
||||
# only run PyTorch latest with Python latest
|
||||
- {os: "macOS-11", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.12"}
|
||||
- {os: "macOS-11", pkg-name: "lightning", python-version: "3.10", pytorch-version: "1.12"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.12"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.10", pytorch-version: "1.12"}
|
||||
- {os: "windows-2022", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.12"}
|
||||
- {os: "windows-2022", pkg-name: "lightning", python-version: "3.10", pytorch-version: "1.12"}
|
||||
# "oldest" versions tests, only on minimum Python
|
||||
- {os: "macOS-11", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"}
|
||||
- {os: "macOS-11", pkg-name: "lightning", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"}
|
||||
- {os: "windows-2022", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"}
|
||||
- {os: "windows-2022", pkg-name: "lightning", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"}
|
||||
# release-candidate tests, mixed Python versions
|
||||
- {os: "macOS-11", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.13", release: "pre"}
|
||||
- {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.13", release: "pre"}
|
||||
|
|
@ -99,22 +107,6 @@ jobs:
|
|||
python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${{ matrix.pytorch-version }}
|
||||
cat requirements/pytorch/base.txt
|
||||
|
||||
- name: switch PyTorch URL
|
||||
run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
|
||||
|
||||
- name: Install package
|
||||
env:
|
||||
PACKAGE_NAME: ${{ matrix.pkg-name }}
|
||||
run: |
|
||||
pip install -e . pytest --upgrade --find-links ${TORCH_URL}
|
||||
pip install pytest-timeout
|
||||
pip list
|
||||
|
||||
- name: DocTests PL
|
||||
if: ${{ matrix.pkg-name != 'LAI' }}
|
||||
working-directory: src
|
||||
run: python -m pytest pytorch_lightning
|
||||
|
||||
- name: Get pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
|
|
@ -127,9 +119,14 @@ jobs:
|
|||
restore-keys: |
|
||||
${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.release }}-${{ matrix.requires }}-
|
||||
|
||||
- name: Install dependencies
|
||||
- name: Switch PyTorch URL
|
||||
run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV
|
||||
|
||||
- name: Install package & dependencies
|
||||
env:
|
||||
PACKAGE_NAME: ${{ matrix.pkg-name }}
|
||||
run: |
|
||||
pip install -r requirements/pytorch/devel.txt --upgrade --find-links ${TORCH_URL}
|
||||
pip install -e . "pytest-timeout" -r requirements/pytorch/devel.txt --upgrade --find-links ${TORCH_URL}
|
||||
pip list
|
||||
|
||||
- name: Reinstall Horovod if necessary
|
||||
|
|
@ -161,8 +158,8 @@ jobs:
|
|||
python requirements/pytorch/check-avail-extras.py
|
||||
|
||||
- name: Adjust tests
|
||||
if: ${{ matrix.pkg-name == 'LAI' }}
|
||||
run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="pytorch_lightning" --target_import="lightning.pytorch"
|
||||
if: ${{ matrix.pkg-name == 'lightning' }}
|
||||
run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="pytorch_lightning,lightning_lite" --target_import="lightning.pytorch,lightning.lite"
|
||||
|
||||
- name: Testing Warnings
|
||||
# the stacklevel can only be set on >=3.7
|
||||
|
|
@ -171,8 +168,8 @@ jobs:
|
|||
# needs to run outside of `pytest`
|
||||
run: python utilities/test_warnings.py
|
||||
|
||||
- name: switch coverage scope
|
||||
run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'pytorch_lightning'))" >> $GITHUB_ENV
|
||||
- name: Switch coverage scope
|
||||
run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'pytorch_lightning'))" >> $GITHUB_ENV
|
||||
|
||||
- name: Testing PyTorch
|
||||
working-directory: tests/tests_pytorch
|
||||
|
|
|
|||
|
|
@ -50,9 +50,6 @@ RUN \
|
|||
|
||||
RUN pip install jupyterlab[all] -U
|
||||
|
||||
RUN pip install lightning-grid -U && \
|
||||
pip install "py>=1.10" "protobuf>=3.15.6" --upgrade-strategy only-if-needed
|
||||
|
||||
ENV PYTHONPATH="/workspace"
|
||||
|
||||
RUN \
|
||||
|
|
@ -63,6 +60,6 @@ RUN \
|
|||
pip --version && \
|
||||
pip list | grep torch && \
|
||||
python -c "import torch; assert torch.__version__.startswith('$TORCH_VERSION'), torch.__version__" && \
|
||||
python -c "import pytorch_lightning as pl; print(pl.__version__)"
|
||||
python -c "import lightning as L; print(L.__version__)"
|
||||
|
||||
CMD ["jupyter", "notebook", "--port=8888", "--no-browser", "--ip=0.0.0.0", "--allow-root"]
|
||||
|
|
|
|||
|
|
@ -398,20 +398,20 @@ import lightning as L
|
|||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import pytorch_lightning as pl
|
||||
import lightning.pytorch as pl
|
||||
from torch import nn
|
||||
from torch.utils.data import IterableDataset, DataLoader, Dataset
|
||||
from pytorch_lightning import LightningDataModule, LightningModule, Trainer, seed_everything
|
||||
from pytorch_lightning.callbacks import Callback
|
||||
from pytorch_lightning.cli import _JSONARGPARSE_SIGNATURES_AVAILABLE as _JSONARGPARSE_AVAILABLE
|
||||
from pytorch_lightning.utilities import (
|
||||
from lightning.pytorch import LightningDataModule, LightningModule, Trainer, seed_everything
|
||||
from lightning.pytorch.callbacks import Callback
|
||||
from lightning.pytorch.cli import _JSONARGPARSE_SIGNATURES_AVAILABLE as _JSONARGPARSE_AVAILABLE
|
||||
from lightning.pytorch.utilities import (
|
||||
_APEX_AVAILABLE,
|
||||
_TORCHVISION_AVAILABLE,
|
||||
_TORCH_GREATER_EQUAL_1_10,
|
||||
)
|
||||
from pytorch_lightning.loggers.neptune import _NEPTUNE_AVAILABLE
|
||||
from pytorch_lightning.loggers.comet import _COMET_AVAILABLE
|
||||
from pytorch_lightning.loggers.mlflow import _MLFLOW_AVAILABLE
|
||||
from pytorch_lightning.loggers.wandb import _WANDB_AVAILABLE
|
||||
from lightning.pytorch.loggers.neptune import _NEPTUNE_AVAILABLE
|
||||
from lightning.pytorch.loggers.comet import _COMET_AVAILABLE
|
||||
from lightning.pytorch.loggers.mlflow import _MLFLOW_AVAILABLE
|
||||
from lightning.pytorch.loggers.wandb import _WANDB_AVAILABLE
|
||||
"""
|
||||
coverage_skip_undoc_in_source = True
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
lightning-cloud>=0.5.10
|
||||
packaging
|
||||
typing-extensions>=4.0.0, <4.3.1
|
||||
deepdiff>=5.7.0, <=5.8.1
|
||||
starsessions>=1.2.1, <2.0 # strict
|
||||
fsspec>=2022.5.0, <=2022.7.1
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
pytorch_lightning>=1.6.5
|
||||
lightning_app>=0.5.2
|
||||
14
setup.py
14
setup.py
|
|
@ -25,12 +25,13 @@ There are considered three main scenarios for installing this project:
|
|||
In case you want to install just one package you need to export env. variable before calling `pip`
|
||||
|
||||
- for `pytorch-lightning` use `export PACKAGE_NAME=pytorch ; pip install .`
|
||||
- for `lightning-lite` use `export PACKAGE_NAME=lite ; pip install .`
|
||||
- for `lightning-app` use `export PACKAGE_NAME=app ; pip install .`
|
||||
|
||||
3. Building packages as sdist or binary wheel and installing or publish to PyPI afterwords you use command
|
||||
`python setup.py sdist` or `python setup.py bdist_wheel` accordingly.
|
||||
In case you want to build just a particular package you would use exporting env. variable as above:
|
||||
`export PACKAGE_NAME=pytorch|app ; python setup.py sdist bdist_wheel`
|
||||
`export PACKAGE_NAME=pytorch|app|lite ; python setup.py sdist bdist_wheel`
|
||||
|
||||
4. Automated releasing with GitHub action is natural extension of 3) is composed of three consecutive steps:
|
||||
a) determine which packages shall be released based on version increment in `__version__.py` and eventually
|
||||
|
|
@ -94,17 +95,10 @@ def _load_py_module(name: str, location: str) -> ModuleType:
|
|||
if __name__ == "__main__":
|
||||
_SETUP_TOOLS = _load_py_module(name="setup_tools", location=os.path.join(".actions", "setup_tools.py"))
|
||||
|
||||
if _PACKAGE_NAME == "lightning": # install just the meta package
|
||||
_SETUP_TOOLS._relax_require_versions(_PATH_SRC, _PATH_REQUIRE)
|
||||
elif _PACKAGE_NAME not in _PACKAGE_MAPPING: # install everything
|
||||
if _PACKAGE_NAME not in _PACKAGE_MAPPING: # install everything
|
||||
_SETUP_TOOLS._load_aggregate_requirements(_PATH_REQUIRE, _FREEZE_REQUIREMENTS)
|
||||
|
||||
if _PACKAGE_NAME not in _PACKAGE_MAPPING:
|
||||
_SETUP_TOOLS.set_version_today(os.path.join(_PATH_SRC, "lightning", "__version__.py"))
|
||||
|
||||
for lit_name, pkg_name in _PACKAGE_MAPPING.items():
|
||||
# fixme: if we run creation of meta pkg against stable we shall pull the source
|
||||
_SETUP_TOOLS.create_meta_package(os.path.join(_PATH_ROOT, "src"), pkg_name, lit_name)
|
||||
_SETUP_TOOLS.create_mirror_package(os.path.join(_PATH_ROOT, "src"), _PACKAGE_MAPPING)
|
||||
|
||||
_SETUP_MODULE = _load_py_module(name="pkg_setup", location=_PATH_SETUP)
|
||||
_SETUP_MODULE._adjust_manifest(pkg_name=_REAL_PKG_NAME)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"""Root package info."""
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
_DETAIL = 15 # between logging.INFO and logging.DEBUG, used for logging in production use cases
|
||||
|
|
@ -39,6 +40,10 @@ from lightning.pytorch.core import LightningDataModule, LightningModule # noqa:
|
|||
from lightning.pytorch.trainer import Trainer # noqa: E402
|
||||
from lightning.pytorch.utilities.seed import seed_everything # noqa: E402
|
||||
|
||||
import lightning.app # isort: skip # noqa: E402
|
||||
|
||||
lightning.app._PROJECT_ROOT = os.path.dirname(lightning.app._PROJECT_ROOT)
|
||||
|
||||
__all__ = [
|
||||
"LightningApp",
|
||||
"LightningFlow",
|
||||
|
|
|
|||
|
|
@ -30,21 +30,16 @@ def _adjust_manifest(**kwargs: Any) -> None:
|
|||
assert os.path.isfile(manifest_path)
|
||||
with open(manifest_path) as fp:
|
||||
lines = [ln.rstrip() for ln in fp.readlines()]
|
||||
if kwargs["pkg_name"] == "lightning":
|
||||
lines += [
|
||||
"recursive-include src/lightning *.md",
|
||||
"include requirements/base.txt",
|
||||
# fixme: this is strange, this shall work with setup find package - include
|
||||
"prune src/lightning_app",
|
||||
"prune src/pytorch_lightning",
|
||||
]
|
||||
else:
|
||||
lines += [
|
||||
"recursive-include src *.md",
|
||||
"recursive-include requirements *.txt",
|
||||
"recursive-include src/lightning_app/ui *",
|
||||
"recursive-include src/lightning_app/cli/*-template *", # Add templates as build-in
|
||||
]
|
||||
lines += [
|
||||
"recursive-include src/lightning *.md",
|
||||
"recursive-include requirements *.txt",
|
||||
"recursive-include src/lightning/app/ui *",
|
||||
"recursive-include src/lightning/cli/*-template *", # Add templates as build-in
|
||||
# fixme: this is strange, this shall work with setup find package - include
|
||||
"prune src/lightning_app",
|
||||
"prune src/lightning_lite",
|
||||
"prune src/pytorch_lightning",
|
||||
]
|
||||
with open(manifest_path, "w") as fp:
|
||||
fp.writelines([ln + os.linesep for ln in lines])
|
||||
|
||||
|
|
@ -55,12 +50,12 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]:
|
|||
_long_description = _SETUP_TOOLS.load_readme_description(
|
||||
_PROJECT_ROOT, homepage=_about.__homepage__, version=_version.version
|
||||
)
|
||||
_include_pkgs = ["lightning", "lightning.*"] if kwargs["pkg_name"] == "lightning" else ["*"]
|
||||
_include_pkgs = ["lightning", "lightning.*"]
|
||||
|
||||
# TODO: consider invaliding some additional arguments from packages, for example if include data or safe to zip
|
||||
|
||||
# TODO: remove this once lightning-ui package is ready as a dependency
|
||||
_SETUP_TOOLS._download_frontend(_PROJECT_ROOT)
|
||||
_SETUP_TOOLS._download_frontend(os.path.join(_SOURCE_ROOT, "lightning", "app"))
|
||||
|
||||
return dict(
|
||||
name="lightning",
|
||||
|
|
@ -81,7 +76,7 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]:
|
|||
python_requires=">=3.7", # todo: take the lowes based on all packages
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
"lightning = lightning_app.cli.lightning_cli:main",
|
||||
"lightning = lightning.app.cli.lightning_cli:main",
|
||||
],
|
||||
},
|
||||
setup_requires=[],
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
version = "YYYY.-M.-D"
|
||||
version = "1.8.0rc1"
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ def _setup_args(**__: Any) -> Dict[str, Any]:
|
|||
)
|
||||
|
||||
# TODO: remove this once lightning-ui package is ready as a dependency
|
||||
_setup_tools._download_frontend(_PROJECT_ROOT)
|
||||
_setup_tools._download_frontend(_PACKAGE_ROOT)
|
||||
|
||||
return dict(
|
||||
name="lightning-app",
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
version = "1.8.0rc0"
|
||||
version = "1.8.0rc1"
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ def pydantic_column_type(pydantic_type: Any) -> Any:
|
|||
return PydanticJSONType
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
@functools.lru_cache(maxsize=128) # compatibility for py3.7
|
||||
def get_primary_key(model_type: Type["SQLModel"]) -> str:
|
||||
primary_keys = sqlalchemy_inspect(model_type).primary_key
|
||||
|
||||
|
|
|
|||
|
|
@ -168,6 +168,9 @@ class BasePayload(ABC):
|
|||
_logger.debug(f"Attempting to copy {str(response.path)} -> {str(local_path)}")
|
||||
fs.get(str(response.path), str(local_path), recursive=False)
|
||||
|
||||
# Ensure the file is properly written
|
||||
sleep(0.5)
|
||||
|
||||
self._value = self.load(local_path)
|
||||
return self._value
|
||||
|
||||
|
|
|
|||
|
|
@ -136,7 +136,9 @@ def application_testing(
|
|||
|
||||
from click.testing import CliRunner
|
||||
|
||||
with mock.patch("lightning.LightningApp", lightning_app_cls):
|
||||
with mock.patch("lightning.LightningApp", lightning_app_cls), mock.patch(
|
||||
"lightning_app.LightningApp", lightning_app_cls
|
||||
):
|
||||
original = sys.argv
|
||||
sys.argv = command_line
|
||||
runner = CliRunner()
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from packaging.version import Version
|
|||
|
||||
from lightning_app import _logger, _PROJECT_ROOT, _root_logger
|
||||
from lightning_app.__version__ import version
|
||||
from lightning_app.core.constants import PACKAGE_LIGHTNING
|
||||
from lightning_app.core.constants import FRONTEND_DIR, PACKAGE_LIGHTNING
|
||||
from lightning_app.utilities.app_helpers import Logger
|
||||
from lightning_app.utilities.git import check_github_repository, get_dir_name
|
||||
|
||||
|
|
@ -30,7 +30,7 @@ def download_frontend(root: str = _PROJECT_ROOT):
|
|||
"""Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct
|
||||
directory."""
|
||||
build_dir = "build"
|
||||
frontend_dir = pathlib.Path(root, "src", "lightning_app", "ui")
|
||||
frontend_dir = pathlib.Path(FRONTEND_DIR)
|
||||
download_dir = tempfile.mkdtemp()
|
||||
|
||||
shutil.rmtree(frontend_dir, ignore_errors=True)
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
version = "1.8.0rc0"
|
||||
version = "1.8.0rc1"
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
version = "1.8.0rc0"
|
||||
version = "1.8.0rc1"
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from unittest.mock import MagicMock
|
|||
import click
|
||||
import pytest
|
||||
|
||||
from lightning_app import _PACKAGE_ROOT
|
||||
from lightning_app import _PROJECT_ROOT
|
||||
from lightning_app.cli.commands.connection import (
|
||||
_list_app_commands,
|
||||
_PPID,
|
||||
|
|
@ -30,7 +30,7 @@ def test_connect_disconnect_local(monkeypatch):
|
|||
data = json.load(f)
|
||||
|
||||
data["paths"]["/command/command_with_client"]["post"]["cls_path"] = os.path.join(
|
||||
os.path.dirname(os.path.dirname(_PACKAGE_ROOT)),
|
||||
_PROJECT_ROOT,
|
||||
data["paths"]["/command/command_with_client"]["post"]["cls_path"],
|
||||
)
|
||||
|
||||
|
|
@ -92,7 +92,7 @@ def test_connect_disconnect_cloud(monkeypatch):
|
|||
data = json.load(f)
|
||||
|
||||
data["paths"]["/command/command_with_client"]["post"]["cls_path"] = os.path.join(
|
||||
os.path.dirname(os.path.dirname(_PACKAGE_ROOT)),
|
||||
_PROJECT_ROOT,
|
||||
data["paths"]["/command/command_with_client"]["post"]["cls_path"],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
from tests_app import _PROJECT_ROOT
|
||||
from tests_app import _TESTS_ROOT
|
||||
|
||||
from lightning_app.testing import application_testing, LightningTestApp
|
||||
from lightning_app.utilities.packaging.build_config import BuildConfig
|
||||
|
|
@ -17,7 +17,7 @@ class NoRequirementsLightningTestApp(LightningTestApp):
|
|||
|
||||
|
||||
def test_build_config_no_requirements():
|
||||
command_line = [os.path.join(_PROJECT_ROOT, "tests/utilities/packaging/projects/no_req/app.py")]
|
||||
command_line = [os.path.join(_TESTS_ROOT, "utilities/packaging/projects/no_req/app.py")]
|
||||
application_testing(NoRequirementsLightningTestApp, command_line + EXTRAS_ARGS)
|
||||
sys.path = sys.path[:-1]
|
||||
|
||||
|
|
@ -60,7 +60,7 @@ class DockerfileLightningTestApp(LightningTestApp):
|
|||
|
||||
|
||||
def test_build_config_dockerfile():
|
||||
command_line = [os.path.join(_PROJECT_ROOT, "tests/utilities/packaging/projects/dockerfile/app.py")]
|
||||
command_line = [os.path.join(_TESTS_ROOT, "utilities/packaging/projects/dockerfile/app.py")]
|
||||
application_testing(DockerfileLightningTestApp, command_line + EXTRAS_ARGS)
|
||||
sys.path = sys.path[:-1]
|
||||
|
||||
|
|
@ -76,6 +76,6 @@ class RequirementsLightningTestApp(LightningTestApp):
|
|||
|
||||
|
||||
def test_build_config_requirements():
|
||||
command_line = [os.path.join(_PROJECT_ROOT, "tests/utilities/packaging/projects/req/app.py")]
|
||||
command_line = [os.path.join(_TESTS_ROOT, "utilities/packaging/projects/req/app.py")]
|
||||
application_testing(RequirementsLightningTestApp, command_line + EXTRAS_ARGS)
|
||||
sys.path = sys.path[:-1]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
from lightning_app import _PACKAGE_ROOT
|
||||
from lightning_app import _PROJECT_ROOT
|
||||
from lightning_app.testing.testing import application_testing
|
||||
from lightning_app.utilities.load_app import _patch_sys_argv
|
||||
|
||||
|
|
@ -10,7 +10,7 @@ def test_app_argparse_example():
|
|||
original_argv = sys.argv
|
||||
|
||||
command_line = [
|
||||
os.path.join(os.path.dirname(os.path.dirname(_PACKAGE_ROOT)), "examples/app_argparse/app.py"),
|
||||
os.path.join(_PROJECT_ROOT, "examples/app_argparse/app.py"),
|
||||
"--app_args",
|
||||
"--use_gpu",
|
||||
"--without-server",
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ import os
|
|||
from time import sleep
|
||||
|
||||
import pytest
|
||||
from tests_app import _PROJECT_ROOT
|
||||
|
||||
from lightning_app import _PROJECT_ROOT
|
||||
from lightning_app.testing.testing import run_app_in_cloud
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -191,11 +191,23 @@ def caplog(caplog):
|
|||
"""
|
||||
import logging
|
||||
|
||||
lightning_logger = logging.getLogger("pytorch_lightning")
|
||||
propagate = lightning_logger.propagate
|
||||
lightning_logger.propagate = True
|
||||
root_logger = logging.getLogger()
|
||||
root_propagate = root_logger.propagate
|
||||
root_logger.propagate = True
|
||||
|
||||
propagation_dict = {
|
||||
name: logging.getLogger(name).propagate
|
||||
for name in logging.root.manager.loggerDict
|
||||
if name.startswith("pytorch_lightning")
|
||||
}
|
||||
for name in propagation_dict.keys():
|
||||
logging.getLogger(name).propagate = True
|
||||
|
||||
yield caplog
|
||||
lightning_logger.propagate = propagate
|
||||
|
||||
root_logger.propagate = root_propagate
|
||||
for name, propagate in propagation_dict.items():
|
||||
logging.getLogger(name).propagate = propagate
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import torch
|
|||
from pytorch_lightning import seed_everything, Trainer
|
||||
from pytorch_lightning.demos.boring_classes import BoringModel
|
||||
from pytorch_lightning.loops import FitLoop
|
||||
from tests_pytorch.helpers.runif import RunIf
|
||||
|
||||
|
||||
def test_outputs_format(tmpdir):
|
||||
|
|
@ -140,6 +141,7 @@ def test_should_stop_mid_epoch(tmpdir):
|
|||
assert model.validation_called_at == (0, 5)
|
||||
|
||||
|
||||
@RunIf(standalone=True)
|
||||
def test_fit_loop_done_log_messages(caplog):
|
||||
fit_loop = FitLoop(max_epochs=1)
|
||||
trainer = Mock(spec=Trainer)
|
||||
|
|
@ -210,6 +212,7 @@ def test_warning_valid_train_step_end(tmpdir):
|
|||
(4, 10, 4, True, True, True),
|
||||
],
|
||||
)
|
||||
@RunIf(standalone=True)
|
||||
def test_should_stop_early_stopping_conditions_met(
|
||||
caplog, min_epochs, min_steps, current_epoch, early_stop, fit_loop_done, raise_debug_msg
|
||||
):
|
||||
|
|
|
|||
Loading…
Reference in New Issue