diff --git a/.actions/setup_tools.py b/.actions/setup_tools.py index 45478600b5..cb079cd71e 100644 --- a/.actions/setup_tools.py +++ b/.actions/setup_tools.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import glob -import logging import os import pathlib import re @@ -23,9 +22,9 @@ import urllib.request from datetime import datetime from distutils.version import LooseVersion from importlib.util import module_from_spec, spec_from_file_location -from itertools import chain, groupby +from itertools import chain from types import ModuleType -from typing import List, Sequence +from typing import Dict, List, Sequence from pkg_resources import parse_requirements @@ -114,8 +113,8 @@ def load_requirements( """Loading requirements from a file. >>> path_req = os.path.join(_PROJECT_ROOT, "requirements") - >>> load_requirements(path_req, unfreeze="major") # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE - ['pytorch_lightning...', 'lightning_app...'] + >>> load_requirements(path_req, "docs.txt", unfreeze="major") # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + ['sphinx>=4.0, <6.0 # strict', ...] """ with open(os.path.join(path_dir, file_name)) as file: lines = [ln.strip() for ln in file.readlines()] @@ -165,221 +164,6 @@ def load_readme_description(path_dir: str, homepage: str, version: str) -> str: return text -def replace_block_with_imports(lines: List[str], import_path: str, kword: str = "class") -> List[str]: - """Parse a file and replace implementtaions bodies of function or class. - - >>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "loggers", "logger.py") - >>> import_path = ".".join(["pytorch_lightning", "loggers", "logger"]) - >>> with open(py_file, encoding="utf-8") as fp: - ... lines = [ln.rstrip() for ln in fp.readlines()] - >>> lines = replace_block_with_imports(lines, import_path, "class") - >>> lines = replace_block_with_imports(lines, import_path, "def") - """ - body, tracking, skip_offset = [], False, 0 - for i, ln in enumerate(lines): - # support for defining a class with this condition - conditional_class_definitions = ("if TYPE_CHECKING", "if typing.TYPE_CHECKING", "if torch.", "if _TORCH_") - if ( - any(ln.startswith(pattern) for pattern in conditional_class_definitions) - # avoid bug in CI for the <1.7 meta code - and "pytorch_lightning.utilities.meta" not in import_path - ): - # dedent the next line - lines[i + 1] = lines[i + 1].lstrip() - continue - - offset = len(ln) - len(ln.lstrip()) - # in case of mating the class args are multi-line - if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]"): - tracking = False - if ln.lstrip().startswith(f"{kword} ") and not tracking: - name = ln.replace(f"{kword} ", "").strip() - idxs = [name.index(c) for c in ":(" if c in name] - name = name[: min(idxs)] - # skip private, TODO: consider skip even protected - if not name.startswith("__"): - body.append(f"{' ' * offset}from {import_path} import {name} # noqa: F401") - tracking, skip_offset = True, offset - continue - if not tracking: - body.append(ln) - return body - - -def replace_vars_with_imports(lines: List[str], import_path: str) -> List[str]: - """Parse a file and replace variable filling with import. - - >>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "utilities", "imports.py") - >>> import_path = ".".join(["pytorch_lightning", "utilities", "imports"]) - >>> with open(py_file, encoding="utf-8") as fp: - ... lines = [ln.rstrip() for ln in fp.readlines()] - >>> lines = replace_vars_with_imports(lines, import_path) - """ - copied = [] - body, tracking, skip_offset = [], False, 0 - for ln in lines: - offset = len(ln) - len(ln.lstrip()) - # in case of mating the class args are multi-line - if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]}"): - tracking = False - var = re.match(r"^([\w_\d]+)[: [\w\., \[\]]*]? = ", ln.lstrip()) - if var: - name = var.groups()[0] - # skip private or apply white-list for allowed vars - if name not in copied and (not name.startswith("__") or name in ("__all__",)): - body.append(f"{' ' * offset}from {import_path} import {name} # noqa: F401") - copied.append(name) - tracking, skip_offset = True, offset - continue - if not tracking: - body.append(ln) - return body - - -def prune_imports_callables(lines: List[str]) -> List[str]: - """Prune imports and calling functions from a file, even multi-line. - - >>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "utilities", "cli.py") - >>> import_path = ".".join(["pytorch_lightning", "utilities", "cli"]) - >>> with open(py_file, encoding="utf-8") as fp: - ... lines = [ln.rstrip() for ln in fp.readlines()] - >>> lines = prune_imports_callables(lines) - """ - body, tracking, skip_offset = [], False, 0 - for ln in lines: - if ln.lstrip().startswith("import "): - continue - offset = len(ln) - len(ln.lstrip()) - # in case of mating the class args are multi-line - if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]}"): - tracking = False - # catching callable - call = re.match(r"^[\w_\d\.]+\(", ln.lstrip()) - if (ln.lstrip().startswith("from ") and " import " in ln) or call: - tracking, skip_offset = True, offset - continue - if not tracking: - body.append(ln) - return body - - -def prune_func_calls(lines: List[str]) -> List[str]: - """Prune calling functions from a file, even multi-line. - - >>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "loggers", "__init__.py") - >>> import_path = ".".join(["pytorch_lightning", "loggers"]) - >>> with open(py_file, encoding="utf-8") as fp: - ... lines = [ln.rstrip() for ln in fp.readlines()] - >>> lines = prune_func_calls(lines) - """ - body, tracking, score = [], False, 0 - for ln in lines: - # catching callable - calling = re.match(r"^@?[\w_\d\.]+ *\(", ln.lstrip()) - if calling and " import " not in ln: - tracking = True - score = 0 - if tracking: - score += ln.count("(") - ln.count(")") - if score == 0: - tracking = False - else: - body.append(ln) - return body - - -def prune_empty_statements(lines: List[str]) -> List[str]: - """Prune emprty if/else and try/except. - - >>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "utilities", "cli.py") - >>> import_path = ".".join(["pytorch_lightning", "utilities", "cli"]) - >>> with open(py_file, encoding="utf-8") as fp: - ... lines = [ln.rstrip() for ln in fp.readlines()] - >>> lines = prune_imports_callables(lines) - >>> lines = prune_empty_statements(lines) - """ - kwords_pairs = ("with", "if ", "elif ", "else", "try", "except") - body, tracking, skip_offset, last_count = [], False, 0, 0 - # todo: consider some more complex logic as for example only some leaves of if/else tree are empty - for i, ln in enumerate(lines): - offset = len(ln) - len(ln.lstrip()) - # skipp all decorators - if ln.lstrip().startswith("@"): - # consider also multi-line decorators - if "(" in ln and ")" not in ln: - tracking, skip_offset = True, offset - continue - # in case of mating the class args are multi-line - if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]}"): - tracking = False - starts = [k for k in kwords_pairs if ln.lstrip().startswith(k)] - if starts: - start, count = starts[0], -1 - # look forward if this statement has a body - for ln_ in lines[i:]: - offset_ = len(ln_) - len(ln_.lstrip()) - if count == -1 and ln_.rstrip().endswith(":"): - count = 0 - elif ln_ and offset_ <= offset: - break - # skipp all til end of statement - elif ln_.lstrip(): - # count non-zero body lines - count += 1 - # cache the last key body as the supplement canot be without - if start in ("if", "elif", "try"): - last_count = count - if count <= 0 or (start in ("else", "except") and last_count <= 0): - tracking, skip_offset = True, offset - if not tracking: - body.append(ln) - return body - - -def prune_comments_docstrings(lines: List[str]) -> List[str]: - """Prune all doctsrings with triple " notation. - - >>> py_file = os.path.join(_PROJECT_ROOT, "src", "pytorch_lightning", "loggers", "csv_logs.py") - >>> import_path = ".".join(["pytorch_lightning", "loggers", "csv_logs"]) - >>> with open(py_file, encoding="utf-8") as fp: - ... lines = [ln.rstrip() for ln in fp.readlines()] - >>> lines = prune_comments_docstrings(lines) - """ - body, tracking = [], False - for ln in lines: - if "#" in ln and "noqa:" not in ln: - ln = ln[: ln.index("#")] - if not tracking and any(ln.lstrip().startswith(s) for s in ['"""', 'r"""']): - # oneliners skip directly - if len(ln.strip()) >= 6 and ln.rstrip().endswith('"""'): - continue - tracking = True - elif ln.rstrip().endswith('"""'): - tracking = False - continue - if not tracking: - body.append(ln.rstrip()) - return body - - -def wrap_try_except(body: List[str], pkg: str, ver: str) -> List[str]: - """Wrap the file with try/except for better traceability of import misalignment.""" - not_empty = sum(1 for ln in body if ln) - if not_empty == 0: - return body - body = ["try:"] + [f" {ln}" if ln else "" for ln in body] - body += [ - "", - "except ImportError as err:", - "", - " from os import linesep", - f" from {pkg} import __version__", - f" msg = f'Your `lightning` package was built for `{pkg}=={ver}`," + " but you are running {__version__}'", - " raise type(err)(str(err) + linesep + msg)", - ] - return body - - def parse_version_from_file(pkg_root: str) -> str: """Loading the package version from file.""" file_ver = os.path.join(pkg_root, "__version__.py") @@ -393,74 +177,64 @@ def parse_version_from_file(pkg_root: str) -> str: return ver -def prune_duplicate_lines(body): - body_ = [] - # drop duplicated lines - for ln in body: - if ln.lstrip() not in body_ or ln.lstrip() in (")", ""): - body_.append(ln) - return body_ +def _replace_imports_in_file(lines: List[str], pkg_lut: Dict[str, str]) -> List[str]: + """Replace imports of standalone package to lightning. - -def create_meta_package(src_folder: str, pkg_name: str = "pytorch_lightning", lit_name: str = "pytorch"): - """Parse the real python package and for each module create a mirroe version with repalcing all function and - class implementations by cross-imports to the true package. - - As validation run in termnal: `flake8 src/lightning/ --ignore E402,F401,E501` - - >>> create_meta_package(os.path.join(_PROJECT_ROOT, "src")) + >>> lns = ["lightning_app", + ... "delete_cloud_lightning_apps", + ... "from lightning_app import", + ... "lightning_apps = []", + ... "lightning_app is ours", + ... "def _lightning_app():", + ... ":class:`~lightning_app.core.flow.LightningFlow`"] + >>> from pprint import pprint + >>> pprint(_replace_imports_in_file(lns, {"app": "lightning_app"})) + ['lightning.app', + 'delete_cloud_lightning_apps', + 'from lightning.app import', + 'lightning_apps = []', + 'lightning.app is ours', + 'def _lightning_app():', + ':class:`~lightning.app.core.flow.LightningFlow`'] """ + for n2, n1 in pkg_lut.items(): + for i, ln in enumerate(lines): + lines[i] = re.sub(rf"([^_]|^){n1}([^_\w]|$)", rf"\1lightning.{n2}\2", ln) + return lines + + +# TODO: unify usage with assistant function, such that import this function in there +def copy_adjusted_modules(src_folder: str, pkg_name: str, lit_name: str, pkg_lut: dict) -> None: + """Recursively replace imports in given folder.""" package_dir = os.path.join(src_folder, pkg_name) - pkg_ver = parse_version_from_file(package_dir) - # shutil.rmtree(os.path.join(src_folder, "lightning", lit_name)) - py_files = glob.glob(os.path.join(src_folder, pkg_name, "**", "*.py"), recursive=True) - for py_file in py_files: - local_path = py_file.replace(package_dir + os.path.sep, "") - fname = os.path.basename(py_file) - if "-" in local_path: - continue - with open(py_file, encoding="utf-8") as fp: - lines = [ln.rstrip() for ln in fp.readlines()] - import_path = pkg_name + "." + local_path.replace(".py", "").replace(os.path.sep, ".") - import_path = import_path.replace(".__init__", "") - - if fname in ("__about__.py", "__version__.py"): - body = lines - else: - if fname.startswith("_") and fname not in ("__init__.py", "__main__.py"): - logging.warning(f"unsupported file: {local_path}") - continue - # ToDO: perform some smarter parsing - preserve Constants, lambdas, etc - body = prune_comments_docstrings([ln.rstrip() for ln in lines]) - if fname not in ("__init__.py", "__main__.py"): - body = prune_imports_callables(body) - for key_word in ("class", "def", "async def"): - body = replace_block_with_imports(body, import_path, key_word) - # TODO: fix reimporting which is artefact after replacing var assignment with import; - # after fixing , update CI by remove F811 from CI/check pkg - body = replace_vars_with_imports(body, import_path) - if fname not in ("__main__.py",): - body = prune_func_calls(body) - body_len = -1 - # in case of several in-depth statements - while body_len != len(body): - body_len = len(body) - body = prune_duplicate_lines(body) - body = prune_empty_statements(body) - # add try/catch wrapper for whole body, - # so when import fails it tells you what is the package version this meta package was generated for... - body = wrap_try_except(body, pkg_name, pkg_ver) - - # todo: apply pre-commit formatting - # clean to many empty lines - body = [ln for ln, _group in groupby(body)] - # drop duplicated lines - body = prune_duplicate_lines(body) - # compose the target file name + all_files = glob.glob(os.path.join(package_dir, "**", "*.*"), recursive=True) + for fname in all_files: + local_path = fname.replace(package_dir + os.path.sep, "") new_file = os.path.join(src_folder, "lightning", lit_name, local_path) + if not fname.endswith(".py"): + if not fname.endswith(".pyc"): + os.makedirs(os.path.dirname(new_file), exist_ok=True) + shutil.copy2(fname, new_file) + continue + + with open(fname, encoding="utf-8") as fo: + py = fo.readlines() + py = _replace_imports_in_file(py, pkg_lut) os.makedirs(os.path.dirname(new_file), exist_ok=True) - with open(new_file, "w", encoding="utf-8") as fp: - fp.writelines([ln + os.linesep for ln in body]) + with open(new_file, "w", encoding="utf-8") as fo: + fo.writelines(py) + + +def create_mirror_package(src_folder: str, lit_pkg_mapping: dict) -> None: + """Recursively replace imports in given folder. + + >>> create_mirror_package( + ... os.path.join(_PROJECT_ROOT, "src"), + ... {"pytorch": "pytorch_lightning", "app": "lightning_app", "lite": "lightning_lite"} + ... ) + """ + for lit_name, pkg_name in lit_pkg_mapping.items(): + copy_adjusted_modules(src_folder, pkg_name, lit_name, lit_pkg_mapping) def set_version_today(fpath: str) -> None: @@ -477,12 +251,12 @@ def set_version_today(fpath: str) -> None: fp.writelines(lines) -def _download_frontend(root: str = _PROJECT_ROOT): +def _download_frontend(pkg_path: str): """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct directory.""" try: - frontend_dir = pathlib.Path(root, "src", "lightning_app", "ui") + frontend_dir = pathlib.Path(pkg_path, "ui") download_dir = tempfile.mkdtemp() shutil.rmtree(frontend_dir, ignore_errors=True) diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml index 01776baed3..909b8d29a1 100644 --- a/.azure/app-cloud-e2e.yml +++ b/.azure/app-cloud-e2e.yml @@ -100,6 +100,7 @@ jobs: path: $(pip_cache_dir) displayName: Cache pip + # TODO: we are testing it as `lightning`, so add also version for `lightning_app` - bash: git restore . && python -m pip install -e . -r requirements/app/test.txt -r requirements/app/ui.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html displayName: 'Install Lightning & dependencies' @@ -125,6 +126,16 @@ jobs: condition: eq(variables['name'], 'quick_start') displayName: 'Install Quick Start' + - bash: python -m pip install -q -r .actions/requirements.txt + displayName: 'Install assistant dependencies' + + # Fix imports to use `lightning` instead of `lightning_app` since we install lightning only ATM + # TODO: fixme when installing `lightning_app` as well + - bash: | + python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning_app" --target_import="lightning.app" + python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app" + displayName: 'Adjust examples' + - bash: | pip --version pip list @@ -144,12 +155,12 @@ jobs: TEST_APP_NAME: $(name) HAR_LOCATION: './artifacts/hars' SLOW_MO: '50' - # LAI_USER: $(LAI_USER) - # LAI_PASS: $(LAI_PASS) - LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD) - LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD) + LAI_USER: $(LAI_USER) # for STAGING + LAI_PASS: $(LAI_PASS) # for STAGING + LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_STAGING) + LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_STAGING) LIGHTNING_USERNAME: $(LIGHTNING_USERNAME) - LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) + LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_STAGING) LIGHTNING_DEBUG: '1' displayName: 'Run the tests' @@ -162,12 +173,12 @@ jobs: time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()" condition: always() env: - # LAI_USER: $(LAI_USER) - # LAI_PASS: $(LAI_PASS) - LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD) - LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD) + LAI_USER: $(LAI_USER) # for STAGING + LAI_PASS: $(LAI_PASS) # for STAGING + LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_STAGING) + LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_STAGING) LIGHTNING_USERNAME: $(LIGHTNING_USERNAME) - LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD) + LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_STAGING) PR_NUMBER: $(local_id) TEST_APP_NAME: $(name) # GRID_USER_ID: $(LIGHTNING_USER_ID) # TODO: clarify the meaning diff --git a/.github/actions/pkg-check/action.yml b/.github/actions/pkg-check/action.yml index bfd1602e69..9f86319774 100644 --- a/.github/actions/pkg-check/action.yml +++ b/.github/actions/pkg-check/action.yml @@ -5,6 +5,7 @@ inputs: pkg-name: description: package name inside lightning.* required: true + default: "" nb-dirs: description: nb of packages in the wrap/distribution required: false @@ -21,10 +22,7 @@ runs: - name: Source check env: PACKAGE_NAME: ${{ inputs.pkg-name }} - run: | - python setup.py check --metadata --strict - # TODO: fix reimporting (F811) which is aftefact after rplacing var assigne with import in meta package - flake8 src/lightning/ --ignore E402,F401,E501,W391,E303,F811 + run: python setup.py check --metadata --strict shell: bash - name: Create package @@ -40,10 +38,6 @@ runs: # python setup.py clean shell: bash - - name: copy/export pkg - run: cp dist/* pypi/ - shell: bash - - name: Unzip packages if: ${{ inputs.pkg-name != '' }} working-directory: dist diff --git a/.github/actions/pkg-install/action.yml b/.github/actions/pkg-install/action.yml index c39d11eca4..155c4edf99 100644 --- a/.github/actions/pkg-install/action.yml +++ b/.github/actions/pkg-install/action.yml @@ -25,7 +25,7 @@ runs: run: echo "PKG_NAME=${{ inputs.pkg-name }}" >> $GITHUB_ENV shell: bash - - name: Install | Uninstall package - archive + - name: Install package - archive working-directory: ./dist run: | pip install *.tar.gz ${{ inputs.pip-flags }} @@ -33,7 +33,7 @@ runs: python -c "import ${PKG_NAME} ; print(${PKG_NAME}.__version__)" shell: bash - - name: Install | Uninstall package - wheel + - name: Install package - wheel working-directory: ./dist run: | pip install *.whl ${{ inputs.pip-flags }} diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index e0f4a3060a..4e26e87005 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -30,16 +30,23 @@ subprojects: - "pl-cpu (macOS-11, pytorch, 3.10, 1.12)" - "pl-cpu (macOS-11, pytorch, 3.7, 1.9, oldest)" - "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.11)" + - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)" + - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)" + - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)" - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)" - "pl-cpu (ubuntu-20.04, pytorch, 3.7, 1.9, oldest)" - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)" - - "pl-cpu (windows-2022, pytorch, 3.8, 1.9)" - - "pl-cpu (windows-2022, pytorch, 3.9, 1.10)" + - "pl-cpu (windows-2022, pytorch, 3.9, 1.11)" - "pl-cpu (windows-2022, pytorch, 3.10, 1.11)" - "pl-cpu (windows-2022, pytorch, 3.10, 1.12)" - "pl-cpu (windows-2022, pytorch, 3.7, 1.9, oldest)" - "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)" + - "pl-cpu (macOS-11, lightning, 3.10, 1.12)" + #- "pl-cpu (macOS-11, lightning, 3.7, 1.9, oldest)" + - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)" + #- "pl-cpu (ubuntu-20.04, lightning, 3.7, 1.9, oldest)" + - "pl-cpu (windows-2022, lightning, 3.10, 1.12)" + #- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)" - "make-doctest (pytorch)" - "make-html (pytorch)" - "mypy" @@ -56,21 +63,28 @@ subprojects: paths: - ".github/workflows/ci-pytorch-tests.yml" checks: - - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)" - "pl-cpu (macOS-11, pytorch, 3.8, 1.10)" + - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)" - "pl-cpu (macOS-11, pytorch, 3.10, 1.12)" - "pl-cpu (macOS-11, pytorch, 3.7, 1.9, oldest)" - "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.11)" + - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)" + - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)" + - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)" - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)" - "pl-cpu (ubuntu-20.04, pytorch, 3.7, 1.9, oldest)" - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)" - - "pl-cpu (windows-2022, pytorch, 3.8, 1.9)" - - "pl-cpu (windows-2022, pytorch, 3.9, 1.10)" + - "pl-cpu (windows-2022, pytorch, 3.9, 1.11)" - "pl-cpu (windows-2022, pytorch, 3.10, 1.11)" - "pl-cpu (windows-2022, pytorch, 3.10, 1.12)" - "pl-cpu (windows-2022, pytorch, 3.7, 1.9, oldest)" - "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)" + - "pl-cpu (macOS-11, lightning, 3.10, 1.12)" + #- "pl-cpu (macOS-11, lightning, 3.7, 1.9, oldest)" + - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)" + #- "pl-cpu (ubuntu-20.04, lightning, 3.7, 1.9, oldest)" + - "pl-cpu (windows-2022, lightning, 3.10, 1.12)" + #- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)" - id: "pytorch_lightning: Slow" paths: @@ -165,24 +179,34 @@ subprojects: - "lite-cpu (windows-2022, lite, 3.10, 1.12)" - "lite-cpu (windows-2022, lite, 3.7, 1.9, oldest)" - "lite-cpu (windows-2022, lite, 3.8, 1.13, pre)" + - "lite-cpu (macOS-11, lightning, 3.8, 1.12)" + - "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)" + - "lite-cpu (windows-2022, lightning, 3.8, 1.12)" - "lightning-lite (GPUs)" - "mypy" # Lite also requires PL checks as it depends on Lite - - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)" - "pl-cpu (macOS-11, pytorch, 3.8, 1.10)" + - "pl-cpu (macOS-11, pytorch, 3.9, 1.11)" - "pl-cpu (macOS-11, pytorch, 3.10, 1.12)" - "pl-cpu (macOS-11, pytorch, 3.7, 1.9, oldest)" - "pl-cpu (macOS-11, pytorch, 3.10, 1.13, pre)" - - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.11)" + - "pl-cpu (ubuntu-20.04, pytorch, 3.8, 1.10)" + - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.11)" + - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.11)" - "pl-cpu (ubuntu-20.04, pytorch, 3.10, 1.12)" - "pl-cpu (ubuntu-20.04, pytorch, 3.7, 1.9, oldest)" - "pl-cpu (ubuntu-20.04, pytorch, 3.9, 1.13, pre)" - - "pl-cpu (windows-2022, pytorch, 3.8, 1.9)" - - "pl-cpu (windows-2022, pytorch, 3.9, 1.10)" + - "pl-cpu (windows-2022, pytorch, 3.9, 1.11)" - "pl-cpu (windows-2022, pytorch, 3.10, 1.11)" - "pl-cpu (windows-2022, pytorch, 3.10, 1.12)" - "pl-cpu (windows-2022, pytorch, 3.7, 1.9, oldest)" - "pl-cpu (windows-2022, pytorch, 3.8, 1.13, pre)" + - "pl-cpu (macOS-11, lightning, 3.10, 1.12)" + #- "pl-cpu (macOS-11, lightning, 3.7, 1.9, oldest)" + - "pl-cpu (ubuntu-20.04, lightning, 3.10, 1.12)" + #- "pl-cpu (ubuntu-20.04, lightning, 3.7, 1.9, oldest)" + - "pl-cpu (windows-2022, lightning, 3.10, 1.12)" + #- "pl-cpu (windows-2022, lightning, 3.7, 1.9, oldest)" - "make-doctest (pytorch)" - "make-html (pytorch)" - "pytorch-lightning (GPUs)" @@ -210,6 +234,9 @@ subprojects: - "lite-cpu (windows-2022, lite, 3.10, 1.12)" - "lite-cpu (windows-2022, lite, 3.7, 1.9, oldest)" - "lite-cpu (windows-2022, lite, 3.8, 1.13, pre)" + - "lite-cpu (macOS-11, lightning, 3.8, 1.12)" + - "lite-cpu (ubuntu-20.04, lightning, 3.8, 1.12)" + - "lite-cpu (windows-2022, lightning, 3.8, 1.12)" - "lightning-lite (GPUs)" - id: "lightning_lite: Azure GPU" @@ -235,18 +262,24 @@ subprojects: - "App.cloud-e2e" - "make-doctest (app)" - "make-html (app)" - - "app-examples (macOS-11, app, 3.8, latest)" - - "app-examples (macOS-11, app, 3.8, oldest)" - - "app-examples (ubuntu-20.04, app, 3.8, latest)" - - "app-examples (ubuntu-20.04, app, 3.8, oldest)" - - "app-examples (windows-2022, app, 3.8, latest)" - - "app-examples (windows-2022, app, 3.8, oldest)" + - "app-examples (macOS-11, app, 3.9, latest)" + - "app-examples (macOS-11, app, 3.9, oldest)" + - "app-examples (macOS-11, lightning, 3.9, latest)" + - "app-examples (ubuntu-20.04, app, 3.9, latest)" + - "app-examples (ubuntu-20.04, app, 3.9, oldest)" + - "app-examples (ubuntu-20.04, lightning, 3.9, latest)" + - "app-examples (windows-2022, app, 3.9, latest)" + - "app-examples (windows-2022, app, 3.9, oldest)" + - "app-examples (windows-2022, lightning, 3.9, latest)" - "app-pytest (macOS-11, app, 3.8, latest)" - "app-pytest (macOS-11, app, 3.8, oldest)" + - "app-pytest (macOS-11, lightning, 3.9, latest)" - "app-pytest (ubuntu-20.04, app, 3.8, latest)" - "app-pytest (ubuntu-20.04, app, 3.8, oldest)" + - "app-pytest (ubuntu-20.04, lightning, 3.9, latest)" - "app-pytest (windows-2022, app, 3.8, latest)" - "app-pytest (windows-2022, app, 3.8, oldest)" + - "app-pytest (windows-2022, lightning, 3.9, latest)" - id: "lightning_app: Azure" paths: @@ -287,15 +320,21 @@ subprojects: - "install-pkg (ubuntu-22.04, lite, 3.10)" - "install-pkg (ubuntu-22.04, pytorch, 3.7)" - "install-pkg (ubuntu-22.04, pytorch, 3.10)" + - "install-pkg (ubuntu-22.04, 3.7)" + - "install-pkg (ubuntu-22.04, 3.10)" - "install-pkg (macOS-12, app, 3.7)" - "install-pkg (macOS-12, app, 3.10)" - "install-pkg (macOS-12, lite, 3.7)" - "install-pkg (macOS-12, lite, 3.10)" - "install-pkg (macOS-12, pytorch, 3.7)" - "install-pkg (macOS-12, pytorch, 3.10)" + - "install-pkg (macOS-12, 3.7)" + - "install-pkg (macOS-12, 3.10)" - "install-pkg (windows-2022, app, 3.7)" - "install-pkg (windows-2022, app, 3.10)" - "install-pkg (windows-2022, lite, 3.7)" - "install-pkg (windows-2022, lite, 3.10)" - "install-pkg (windows-2022, pytorch, 3.7)" - "install-pkg (windows-2022, pytorch, 3.10)" + - "install-pkg (windows-2022, 3.7)" + - "install-pkg (windows-2022, 3.10)" diff --git a/.github/workflows/ci-app-examples.yml b/.github/workflows/ci-app-examples.yml index 684c096879..35c748a4da 100644 --- a/.github/workflows/ci-app-examples.yml +++ b/.github/workflows/ci-app-examples.yml @@ -32,12 +32,17 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, macOS-11, windows-2022] - pkg-name: ["app"] # TODO: add "LAI" # "LAI" installs the monolithic package - python-version: [3.8] + pkg-name: ["app"] + python-version: ["3.9"] requires: ["oldest", "latest"] + include: + # "lightning" installs the monolithic package + - {os: "macOS-11", pkg-name: "lightning", python-version: "3.9", requires: "latest"} + - {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.9", requires: "latest"} + - {os: "windows-2022", pkg-name: "lightning", python-version: "3.9", requires: "latest"} # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 10 + timeout-minutes: 15 steps: - uses: actions/checkout@v3 @@ -89,11 +94,17 @@ jobs: run: pip install -e . - name: Adjust tests - if: ${{ matrix.pkg-name == 'LAI' }} + if: ${{ matrix.pkg-name == 'lightning' }} run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app" - - name: switch coverage scope - run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_app'))" >> $GITHUB_ENV + - name: Adjust examples + if: ${{ matrix.pkg-name != 'lightning' }} + run: | + python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app" + python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app" + + - name: Switch coverage scope + run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV - name: Tests working-directory: ./tests diff --git a/.github/workflows/ci-app-tests.yml b/.github/workflows/ci-app-tests.yml index 4ad14c2829..2e09c9e8c2 100644 --- a/.github/workflows/ci-app-tests.yml +++ b/.github/workflows/ci-app-tests.yml @@ -33,12 +33,17 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, macOS-11, windows-2022] - pkg-name: ["app"] # TODO: add "LAI" # "LAI" installs the monolithic package - python-version: [3.8] + pkg-name: ["app"] + python-version: ["3.8"] requires: ["oldest", "latest"] + include: + # "lightning" installs the monolithic package + - {os: "macOS-11", pkg-name: "lightning", python-version: "3.9", requires: "latest"} + - {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.9", requires: "latest"} + - {os: "windows-2022", pkg-name: "lightning", python-version: "3.9", requires: "latest"} # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 25 + timeout-minutes: 30 steps: - uses: actions/checkout@v3 @@ -54,21 +59,6 @@ jobs: if: ${{ matrix.requires == 'oldest' }} run: python .actions/assistant.py replace_oldest_ver - - name: switch PyTorch URL - run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV - - - name: Install package - env: - PACKAGE_NAME: ${{ matrix.pkg-name }} - run: | - pip install -e . pytest --upgrade --find-links ${TORCH_URL} - pip list - - - name: DocTests App - if: ${{ matrix.pkg-name != 'LAI' }} - working-directory: src - run: python -m pytest lightning_app --ignore-glob="**/cli/*-template/**" - - name: Get pip cache id: pip-cache run: | @@ -81,6 +71,15 @@ jobs: key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.requires }}-${{ hashFiles('requirements/app/base.txt') }} restore-keys: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.requires }}- + - name: Switch PyTorch URL + run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV + + - name: Install package + env: + PACKAGE_NAME: ${{ matrix.pkg-name }} + run: | + pip install -e . pytest --upgrade --find-links ${TORCH_URL} + pip list - name: Install dependencies env: # TODO: drop this when we will be using regular releases for testing @@ -98,17 +97,17 @@ jobs: run: npm install -g yarn - name: Adjust tests - if: ${{ matrix.pkg-name == 'LAI' }} + if: ${{ matrix.pkg-name == 'lightning' }} run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_app" --target_import="lightning.app" - name: Adjust examples - if: ${{ matrix.pkg-name != 'LAI' }} + if: ${{ matrix.pkg-name != 'lightning' }} run: | python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning.app" --target_import="lightning_app" python .actions/assistant.py copy_replace_imports --source_dir="./examples" --source_import="lightning" --target_import="lightning_app" - - name: switch coverage scope - run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_app'))" >> $GITHUB_ENV + - name: Switch coverage scope + run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_app'))" >> $GITHUB_ENV - name: Tests working-directory: ./tests diff --git a/.github/workflows/ci-lite-tests.yml b/.github/workflows/ci-lite-tests.yml index 2d94a34b0e..2a49ce8836 100644 --- a/.github/workflows/ci-lite-tests.yml +++ b/.github/workflows/ci-lite-tests.yml @@ -54,8 +54,12 @@ jobs: - {os: "macOS-11", pkg-name: "lite", python-version: "3.10", pytorch-version: "1.13", release: "pre"} - {os: "ubuntu-20.04", pkg-name: "lite", python-version: "3.9", pytorch-version: "1.13", release: "pre"} - {os: "windows-2022", pkg-name: "lite", python-version: "3.8", pytorch-version: "1.13", release: "pre"} + # "lightning" installs the monolithic package + - {os: "macOS-11", pkg-name: "lightning", python-version: "3.8", pytorch-version: "1.12"} + - {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.8", pytorch-version: "1.12"} + - {os: "windows-2022", pkg-name: "lightning", python-version: "3.8", pytorch-version: "1.12"} - timeout-minutes: 10 + timeout-minutes: 15 steps: - uses: actions/checkout@v3 @@ -108,9 +112,10 @@ jobs: PACKAGE_NAME: ${{ matrix.pkg-name }} run: | pip install -e . "pytest-timeout" -r requirements/lite/devel.txt --upgrade --find-links ${TORCH_URL} + pip list - name: Adjust tests - if: ${{ matrix.pkg-name == 'LAI' }} + if: ${{ matrix.pkg-name == 'lightning' }} run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="lightning_lite" --target_import="lightning.lite" - name: Testing Warnings @@ -121,7 +126,7 @@ jobs: run: python utilities/test_warnings.py - name: Switch coverage scope - run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_lite'))" >> $GITHUB_ENV + run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_lite'))" >> $GITHUB_ENV - name: Testing Lite working-directory: tests/tests_lite diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml index 4940fd232b..942d6eb5a9 100644 --- a/.github/workflows/ci-pkg-install.yml +++ b/.github/workflows/ci-pkg-install.yml @@ -23,24 +23,13 @@ defaults: jobs: - init-temp: - runs-on: ubuntu-20.04 - steps: - - run: mkdir pypi && touch pypi/.placeholder - - uses: actions/upload-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: pypi - install-pkg: - needs: init-temp runs-on: ${{ matrix.os }} strategy: - fail-fast: true - max-parallel: 1 + fail-fast: false matrix: os: [ubuntu-22.04, macOS-12, windows-2022] - pkg: ["app", "lite", "pytorch"] + pkg: ["app", "lite", "pytorch", ""] python-version: ["3.7" , "3.10"] steps: - uses: actions/checkout@v3 @@ -51,112 +40,24 @@ jobs: - name: DocTests actions working-directory: .actions/ run: | - grep 'pytest=' ../requirements/pytorch/test.txt | xargs -0 pip install + pip install pytest -q python -m pytest setup_tools.py - - uses: actions/download-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: pypi - - run: ls -lh pypi/ - - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg }}' == 'pytorch' else 1))" >> $GITHUB_ENV - uses: ./.github/actions/pkg-check with: pkg-name: ${{ matrix.pkg }} nb-dirs: ${{ env.NB_DIRS }} - - uses: actions/upload-artifact@v3 - with: - name: ci-packages-${{ github.sha }} - path: pypi - - uses: ./.github/actions/pkg-install - install-meta-src: - needs: install-pkg - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - # max-parallel: 1 - matrix: - os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022] - pkg: ["", "lightning"] - python-version: [3.8] # , 3.9 - - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - run: mkdir -p pypi - - uses: actions/download-artifact@v3 - if: matrix.pkg != '' - with: - name: ci-packages-${{ github.sha }} - path: pypi - - run: ls -lh pypi/ - - - uses: ./.github/actions/pkg-check - with: - pkg-name: ${{ matrix.pkg }} - - - uses: ./.github/actions/pkg-install - with: - pkg-name: "lightning" - pip-flags: "-U --pre --find-links ../pypi/" - - name: Run CLI + # todo: add testing for `lightning_app` + if: ${{ matrix.pkg == '' }} run: python -m lightning --version - install-meta-pypi: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - # max-parallel: 1 - matrix: - os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022] - python-version: [3.8] # , 3.9 - - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Dowload package - # todo: download also lite after it is fist published + - name: DocTest package run: | - pip install -q -r .actions/requirements.txt - for pkg in 'app' 'pytorch' ; do - python .actions/assistant.py download_package --package "$pkg" --folder pypi - done - ls -lh pypi/ - - - name: Unzip packages - working-directory: pypi - run: for file in `ls *.gz`; do tar -xzf $file; done - - name: Show upacked pkgs - if: runner.os == 'linux' - run: | - sudo apt install -y tree - tree pypi/ -L 3 - - - name: Miror source - run: | - pip install -q -r .actions/requirements.txt - python .actions/assistant.py mirror_pkg2source --pypi_folder pypi --src_folder src - ls -R src/ - - - uses: ./.github/actions/pkg-check - with: - pkg-name: "lightning" - - - uses: ./.github/actions/pkg-install - with: - pkg-name: "lightning" - pip-flags: "-U --pre --find-links ../pypi/" - - - name: Run CLI - run: python -m lightning --version + pip list + scope=$(python -c "lut = {'app': 'lightning_app', 'lite': 'lightning_lite', 'pytorch': 'pytorch_lightning'} ; print(lut.get('${{matrix.pkg}}', 'lightning'))") + python -m pytest src/${scope} --ignore-glob="**/cli/*-template/**" diff --git a/.github/workflows/ci-pytorch-tests.yml b/.github/workflows/ci-pytorch-tests.yml index 1d2087c430..4595bb65ce 100644 --- a/.github/workflows/ci-pytorch-tests.yml +++ b/.github/workflows/ci-pytorch-tests.yml @@ -42,18 +42,26 @@ jobs: # note: there's no distribution of Torch==1.9 for Python>=3.9 or torch==1.10 for Python>=3.10 - {os: "macOS-11", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.11"} - {os: "macOS-11", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.10"} + - {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.10"} + - {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.11"} + - {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.11"} + - {os: "windows-2022", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.10"} + - {os: "windows-2022", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.11"} - {os: "windows-2022", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.11"} - - {os: "windows-2022", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.10"} - - {os: "windows-2022", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.9"} - - {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.8", pytorch-version: "1.11"} # only run PyTorch latest with Python latest - {os: "macOS-11", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.12"} + - {os: "macOS-11", pkg-name: "lightning", python-version: "3.10", pytorch-version: "1.12"} - {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.12"} + - {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.10", pytorch-version: "1.12"} - {os: "windows-2022", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.12"} + - {os: "windows-2022", pkg-name: "lightning", python-version: "3.10", pytorch-version: "1.12"} # "oldest" versions tests, only on minimum Python - {os: "macOS-11", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"} + - {os: "macOS-11", pkg-name: "lightning", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"} - {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"} + - {os: "ubuntu-20.04", pkg-name: "lightning", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"} - {os: "windows-2022", pkg-name: "pytorch", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"} + - {os: "windows-2022", pkg-name: "lightning", python-version: "3.7", pytorch-version: "1.9", requires: "oldest"} # release-candidate tests, mixed Python versions - {os: "macOS-11", pkg-name: "pytorch", python-version: "3.10", pytorch-version: "1.13", release: "pre"} - {os: "ubuntu-20.04", pkg-name: "pytorch", python-version: "3.9", pytorch-version: "1.13", release: "pre"} @@ -99,22 +107,6 @@ jobs: python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${{ matrix.pytorch-version }} cat requirements/pytorch/base.txt - - name: switch PyTorch URL - run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV - - - name: Install package - env: - PACKAGE_NAME: ${{ matrix.pkg-name }} - run: | - pip install -e . pytest --upgrade --find-links ${TORCH_URL} - pip install pytest-timeout - pip list - - - name: DocTests PL - if: ${{ matrix.pkg-name != 'LAI' }} - working-directory: src - run: python -m pytest pytorch_lightning - - name: Get pip cache dir id: pip-cache run: echo "::set-output name=dir::$(pip cache dir)" @@ -127,9 +119,14 @@ jobs: restore-keys: | ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.release }}-${{ matrix.requires }}- - - name: Install dependencies + - name: Switch PyTorch URL + run: python -c "print('TORCH_URL=https://download.pytorch.org/whl/' + str('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html'))" >> $GITHUB_ENV + + - name: Install package & dependencies + env: + PACKAGE_NAME: ${{ matrix.pkg-name }} run: | - pip install -r requirements/pytorch/devel.txt --upgrade --find-links ${TORCH_URL} + pip install -e . "pytest-timeout" -r requirements/pytorch/devel.txt --upgrade --find-links ${TORCH_URL} pip list - name: Reinstall Horovod if necessary @@ -161,8 +158,8 @@ jobs: python requirements/pytorch/check-avail-extras.py - name: Adjust tests - if: ${{ matrix.pkg-name == 'LAI' }} - run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="pytorch_lightning" --target_import="lightning.pytorch" + if: ${{ matrix.pkg-name == 'lightning' }} + run: python .actions/assistant.py copy_replace_imports --source_dir="./tests" --source_import="pytorch_lightning,lightning_lite" --target_import="lightning.pytorch,lightning.lite" - name: Testing Warnings # the stacklevel can only be set on >=3.7 @@ -171,8 +168,8 @@ jobs: # needs to run outside of `pytest` run: python utilities/test_warnings.py - - name: switch coverage scope - run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'LAI' else 'pytorch_lightning'))" >> $GITHUB_ENV + - name: Switch coverage scope + run: python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'pytorch_lightning'))" >> $GITHUB_ENV - name: Testing PyTorch working-directory: tests/tests_pytorch diff --git a/dockers/nvidia/Dockerfile b/dockers/nvidia/Dockerfile index 5328d6dea8..5e67d2044f 100644 --- a/dockers/nvidia/Dockerfile +++ b/dockers/nvidia/Dockerfile @@ -50,9 +50,6 @@ RUN \ RUN pip install jupyterlab[all] -U -RUN pip install lightning-grid -U && \ - pip install "py>=1.10" "protobuf>=3.15.6" --upgrade-strategy only-if-needed - ENV PYTHONPATH="/workspace" RUN \ @@ -63,6 +60,6 @@ RUN \ pip --version && \ pip list | grep torch && \ python -c "import torch; assert torch.__version__.startswith('$TORCH_VERSION'), torch.__version__" && \ - python -c "import pytorch_lightning as pl; print(pl.__version__)" + python -c "import lightning as L; print(L.__version__)" CMD ["jupyter", "notebook", "--port=8888", "--no-browser", "--ip=0.0.0.0", "--allow-root"] diff --git a/docs/source-lit/conf.py b/docs/source-lit/conf.py index ecf45c56ab..d5dd5e242e 100644 --- a/docs/source-lit/conf.py +++ b/docs/source-lit/conf.py @@ -398,20 +398,20 @@ import lightning as L from typing import Optional import torch -import pytorch_lightning as pl +import lightning.pytorch as pl from torch import nn from torch.utils.data import IterableDataset, DataLoader, Dataset -from pytorch_lightning import LightningDataModule, LightningModule, Trainer, seed_everything -from pytorch_lightning.callbacks import Callback -from pytorch_lightning.cli import _JSONARGPARSE_SIGNATURES_AVAILABLE as _JSONARGPARSE_AVAILABLE -from pytorch_lightning.utilities import ( +from lightning.pytorch import LightningDataModule, LightningModule, Trainer, seed_everything +from lightning.pytorch.callbacks import Callback +from lightning.pytorch.cli import _JSONARGPARSE_SIGNATURES_AVAILABLE as _JSONARGPARSE_AVAILABLE +from lightning.pytorch.utilities import ( _APEX_AVAILABLE, _TORCHVISION_AVAILABLE, _TORCH_GREATER_EQUAL_1_10, ) -from pytorch_lightning.loggers.neptune import _NEPTUNE_AVAILABLE -from pytorch_lightning.loggers.comet import _COMET_AVAILABLE -from pytorch_lightning.loggers.mlflow import _MLFLOW_AVAILABLE -from pytorch_lightning.loggers.wandb import _WANDB_AVAILABLE +from lightning.pytorch.loggers.neptune import _NEPTUNE_AVAILABLE +from lightning.pytorch.loggers.comet import _COMET_AVAILABLE +from lightning.pytorch.loggers.mlflow import _MLFLOW_AVAILABLE +from lightning.pytorch.loggers.wandb import _WANDB_AVAILABLE """ coverage_skip_undoc_in_source = True diff --git a/requirements/app/base.txt b/requirements/app/base.txt index 99a8e70aea..b987369650 100644 --- a/requirements/app/base.txt +++ b/requirements/app/base.txt @@ -1,5 +1,5 @@ lightning-cloud>=0.5.10 -packaging +typing-extensions>=4.0.0, <4.3.1 deepdiff>=5.7.0, <=5.8.1 starsessions>=1.2.1, <2.0 # strict fsspec>=2022.5.0, <=2022.7.1 diff --git a/requirements/base.txt b/requirements/base.txt index fea4ee10f4..e69de29bb2 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,2 +0,0 @@ -pytorch_lightning>=1.6.5 -lightning_app>=0.5.2 diff --git a/setup.py b/setup.py index d558a24e0b..b400133b14 100755 --- a/setup.py +++ b/setup.py @@ -25,12 +25,13 @@ There are considered three main scenarios for installing this project: In case you want to install just one package you need to export env. variable before calling `pip` - for `pytorch-lightning` use `export PACKAGE_NAME=pytorch ; pip install .` + - for `lightning-lite` use `export PACKAGE_NAME=lite ; pip install .` - for `lightning-app` use `export PACKAGE_NAME=app ; pip install .` 3. Building packages as sdist or binary wheel and installing or publish to PyPI afterwords you use command `python setup.py sdist` or `python setup.py bdist_wheel` accordingly. In case you want to build just a particular package you would use exporting env. variable as above: - `export PACKAGE_NAME=pytorch|app ; python setup.py sdist bdist_wheel` + `export PACKAGE_NAME=pytorch|app|lite ; python setup.py sdist bdist_wheel` 4. Automated releasing with GitHub action is natural extension of 3) is composed of three consecutive steps: a) determine which packages shall be released based on version increment in `__version__.py` and eventually @@ -94,17 +95,10 @@ def _load_py_module(name: str, location: str) -> ModuleType: if __name__ == "__main__": _SETUP_TOOLS = _load_py_module(name="setup_tools", location=os.path.join(".actions", "setup_tools.py")) - if _PACKAGE_NAME == "lightning": # install just the meta package - _SETUP_TOOLS._relax_require_versions(_PATH_SRC, _PATH_REQUIRE) - elif _PACKAGE_NAME not in _PACKAGE_MAPPING: # install everything + if _PACKAGE_NAME not in _PACKAGE_MAPPING: # install everything _SETUP_TOOLS._load_aggregate_requirements(_PATH_REQUIRE, _FREEZE_REQUIREMENTS) - if _PACKAGE_NAME not in _PACKAGE_MAPPING: - _SETUP_TOOLS.set_version_today(os.path.join(_PATH_SRC, "lightning", "__version__.py")) - - for lit_name, pkg_name in _PACKAGE_MAPPING.items(): - # fixme: if we run creation of meta pkg against stable we shall pull the source - _SETUP_TOOLS.create_meta_package(os.path.join(_PATH_ROOT, "src"), pkg_name, lit_name) + _SETUP_TOOLS.create_mirror_package(os.path.join(_PATH_ROOT, "src"), _PACKAGE_MAPPING) _SETUP_MODULE = _load_py_module(name="pkg_setup", location=_PATH_SETUP) _SETUP_MODULE._adjust_manifest(pkg_name=_REAL_PKG_NAME) diff --git a/src/lightning/__init__.py b/src/lightning/__init__.py index cf8873d3d8..62275b02b0 100644 --- a/src/lightning/__init__.py +++ b/src/lightning/__init__.py @@ -1,5 +1,6 @@ """Root package info.""" import logging +import os from typing import Any _DETAIL = 15 # between logging.INFO and logging.DEBUG, used for logging in production use cases @@ -39,6 +40,10 @@ from lightning.pytorch.core import LightningDataModule, LightningModule # noqa: from lightning.pytorch.trainer import Trainer # noqa: E402 from lightning.pytorch.utilities.seed import seed_everything # noqa: E402 +import lightning.app # isort: skip # noqa: E402 + +lightning.app._PROJECT_ROOT = os.path.dirname(lightning.app._PROJECT_ROOT) + __all__ = [ "LightningApp", "LightningFlow", diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index cc3a4f8d11..58f0ee4260 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -30,21 +30,16 @@ def _adjust_manifest(**kwargs: Any) -> None: assert os.path.isfile(manifest_path) with open(manifest_path) as fp: lines = [ln.rstrip() for ln in fp.readlines()] - if kwargs["pkg_name"] == "lightning": - lines += [ - "recursive-include src/lightning *.md", - "include requirements/base.txt", - # fixme: this is strange, this shall work with setup find package - include - "prune src/lightning_app", - "prune src/pytorch_lightning", - ] - else: - lines += [ - "recursive-include src *.md", - "recursive-include requirements *.txt", - "recursive-include src/lightning_app/ui *", - "recursive-include src/lightning_app/cli/*-template *", # Add templates as build-in - ] + lines += [ + "recursive-include src/lightning *.md", + "recursive-include requirements *.txt", + "recursive-include src/lightning/app/ui *", + "recursive-include src/lightning/cli/*-template *", # Add templates as build-in + # fixme: this is strange, this shall work with setup find package - include + "prune src/lightning_app", + "prune src/lightning_lite", + "prune src/pytorch_lightning", + ] with open(manifest_path, "w") as fp: fp.writelines([ln + os.linesep for ln in lines]) @@ -55,12 +50,12 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]: _long_description = _SETUP_TOOLS.load_readme_description( _PROJECT_ROOT, homepage=_about.__homepage__, version=_version.version ) - _include_pkgs = ["lightning", "lightning.*"] if kwargs["pkg_name"] == "lightning" else ["*"] + _include_pkgs = ["lightning", "lightning.*"] # TODO: consider invaliding some additional arguments from packages, for example if include data or safe to zip # TODO: remove this once lightning-ui package is ready as a dependency - _SETUP_TOOLS._download_frontend(_PROJECT_ROOT) + _SETUP_TOOLS._download_frontend(os.path.join(_SOURCE_ROOT, "lightning", "app")) return dict( name="lightning", @@ -81,7 +76,7 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]: python_requires=">=3.7", # todo: take the lowes based on all packages entry_points={ "console_scripts": [ - "lightning = lightning_app.cli.lightning_cli:main", + "lightning = lightning.app.cli.lightning_cli:main", ], }, setup_requires=[], diff --git a/src/lightning/__version__.py b/src/lightning/__version__.py index 1a929693c0..037ac1fafe 100644 --- a/src/lightning/__version__.py +++ b/src/lightning/__version__.py @@ -1 +1 @@ -version = "YYYY.-M.-D" +version = "1.8.0rc1" diff --git a/src/lightning_app/__setup__.py b/src/lightning_app/__setup__.py index 87998c6914..55ba6d09fa 100644 --- a/src/lightning_app/__setup__.py +++ b/src/lightning_app/__setup__.py @@ -70,7 +70,7 @@ def _setup_args(**__: Any) -> Dict[str, Any]: ) # TODO: remove this once lightning-ui package is ready as a dependency - _setup_tools._download_frontend(_PROJECT_ROOT) + _setup_tools._download_frontend(_PACKAGE_ROOT) return dict( name="lightning-app", diff --git a/src/lightning_app/__version__.py b/src/lightning_app/__version__.py index ea312c65c4..037ac1fafe 100644 --- a/src/lightning_app/__version__.py +++ b/src/lightning_app/__version__.py @@ -1 +1 @@ -version = "1.8.0rc0" +version = "1.8.0rc1" diff --git a/src/lightning_app/components/database/utilities.py b/src/lightning_app/components/database/utilities.py index 5edcb321dd..8e34828afc 100644 --- a/src/lightning_app/components/database/utilities.py +++ b/src/lightning_app/components/database/utilities.py @@ -111,7 +111,7 @@ def pydantic_column_type(pydantic_type: Any) -> Any: return PydanticJSONType -@functools.lru_cache +@functools.lru_cache(maxsize=128) # compatibility for py3.7 def get_primary_key(model_type: Type["SQLModel"]) -> str: primary_keys = sqlalchemy_inspect(model_type).primary_key diff --git a/src/lightning_app/storage/payload.py b/src/lightning_app/storage/payload.py index 88a6099b1f..c7dbf5e0fc 100644 --- a/src/lightning_app/storage/payload.py +++ b/src/lightning_app/storage/payload.py @@ -168,6 +168,9 @@ class BasePayload(ABC): _logger.debug(f"Attempting to copy {str(response.path)} -> {str(local_path)}") fs.get(str(response.path), str(local_path), recursive=False) + # Ensure the file is properly written + sleep(0.5) + self._value = self.load(local_path) return self._value diff --git a/src/lightning_app/testing/testing.py b/src/lightning_app/testing/testing.py index 2794b5231e..7cb06602ba 100644 --- a/src/lightning_app/testing/testing.py +++ b/src/lightning_app/testing/testing.py @@ -136,7 +136,9 @@ def application_testing( from click.testing import CliRunner - with mock.patch("lightning.LightningApp", lightning_app_cls): + with mock.patch("lightning.LightningApp", lightning_app_cls), mock.patch( + "lightning_app.LightningApp", lightning_app_cls + ): original = sys.argv sys.argv = command_line runner = CliRunner() diff --git a/src/lightning_app/utilities/packaging/lightning_utils.py b/src/lightning_app/utilities/packaging/lightning_utils.py index 224b8e6614..4fc6fba69b 100644 --- a/src/lightning_app/utilities/packaging/lightning_utils.py +++ b/src/lightning_app/utilities/packaging/lightning_utils.py @@ -15,7 +15,7 @@ from packaging.version import Version from lightning_app import _logger, _PROJECT_ROOT, _root_logger from lightning_app.__version__ import version -from lightning_app.core.constants import PACKAGE_LIGHTNING +from lightning_app.core.constants import FRONTEND_DIR, PACKAGE_LIGHTNING from lightning_app.utilities.app_helpers import Logger from lightning_app.utilities.git import check_github_repository, get_dir_name @@ -30,7 +30,7 @@ def download_frontend(root: str = _PROJECT_ROOT): """Downloads an archive file for a specific release of the Lightning frontend and extracts it to the correct directory.""" build_dir = "build" - frontend_dir = pathlib.Path(root, "src", "lightning_app", "ui") + frontend_dir = pathlib.Path(FRONTEND_DIR) download_dir = tempfile.mkdtemp() shutil.rmtree(frontend_dir, ignore_errors=True) diff --git a/src/lightning_lite/__version__.py b/src/lightning_lite/__version__.py index ea312c65c4..037ac1fafe 100644 --- a/src/lightning_lite/__version__.py +++ b/src/lightning_lite/__version__.py @@ -1 +1 @@ -version = "1.8.0rc0" +version = "1.8.0rc1" diff --git a/src/pytorch_lightning/__version__.py b/src/pytorch_lightning/__version__.py index ea312c65c4..037ac1fafe 100644 --- a/src/pytorch_lightning/__version__.py +++ b/src/pytorch_lightning/__version__.py @@ -1 +1 @@ -version = "1.8.0rc0" +version = "1.8.0rc1" diff --git a/tests/tests_app/cli/test_connect.py b/tests/tests_app/cli/test_connect.py index ba7616bdf6..605e1f97af 100644 --- a/tests/tests_app/cli/test_connect.py +++ b/tests/tests_app/cli/test_connect.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock import click import pytest -from lightning_app import _PACKAGE_ROOT +from lightning_app import _PROJECT_ROOT from lightning_app.cli.commands.connection import ( _list_app_commands, _PPID, @@ -30,7 +30,7 @@ def test_connect_disconnect_local(monkeypatch): data = json.load(f) data["paths"]["/command/command_with_client"]["post"]["cls_path"] = os.path.join( - os.path.dirname(os.path.dirname(_PACKAGE_ROOT)), + _PROJECT_ROOT, data["paths"]["/command/command_with_client"]["post"]["cls_path"], ) @@ -92,7 +92,7 @@ def test_connect_disconnect_cloud(monkeypatch): data = json.load(f) data["paths"]["/command/command_with_client"]["post"]["cls_path"] = os.path.join( - os.path.dirname(os.path.dirname(_PACKAGE_ROOT)), + _PROJECT_ROOT, data["paths"]["/command/command_with_client"]["post"]["cls_path"], ) diff --git a/tests/tests_app/utilities/packaging/test_build_spec.py b/tests/tests_app/utilities/packaging/test_build_spec.py index 4d3855d6f9..e1b9466dc0 100644 --- a/tests/tests_app/utilities/packaging/test_build_spec.py +++ b/tests/tests_app/utilities/packaging/test_build_spec.py @@ -1,7 +1,7 @@ import os import sys -from tests_app import _PROJECT_ROOT +from tests_app import _TESTS_ROOT from lightning_app.testing import application_testing, LightningTestApp from lightning_app.utilities.packaging.build_config import BuildConfig @@ -17,7 +17,7 @@ class NoRequirementsLightningTestApp(LightningTestApp): def test_build_config_no_requirements(): - command_line = [os.path.join(_PROJECT_ROOT, "tests/utilities/packaging/projects/no_req/app.py")] + command_line = [os.path.join(_TESTS_ROOT, "utilities/packaging/projects/no_req/app.py")] application_testing(NoRequirementsLightningTestApp, command_line + EXTRAS_ARGS) sys.path = sys.path[:-1] @@ -60,7 +60,7 @@ class DockerfileLightningTestApp(LightningTestApp): def test_build_config_dockerfile(): - command_line = [os.path.join(_PROJECT_ROOT, "tests/utilities/packaging/projects/dockerfile/app.py")] + command_line = [os.path.join(_TESTS_ROOT, "utilities/packaging/projects/dockerfile/app.py")] application_testing(DockerfileLightningTestApp, command_line + EXTRAS_ARGS) sys.path = sys.path[:-1] @@ -76,6 +76,6 @@ class RequirementsLightningTestApp(LightningTestApp): def test_build_config_requirements(): - command_line = [os.path.join(_PROJECT_ROOT, "tests/utilities/packaging/projects/req/app.py")] + command_line = [os.path.join(_TESTS_ROOT, "utilities/packaging/projects/req/app.py")] application_testing(RequirementsLightningTestApp, command_line + EXTRAS_ARGS) sys.path = sys.path[:-1] diff --git a/tests/tests_app_examples/test_argparse.py b/tests/tests_app_examples/test_argparse.py index 0c1e55b0d4..7d34b7d0c0 100644 --- a/tests/tests_app_examples/test_argparse.py +++ b/tests/tests_app_examples/test_argparse.py @@ -1,7 +1,7 @@ import os import sys -from lightning_app import _PACKAGE_ROOT +from lightning_app import _PROJECT_ROOT from lightning_app.testing.testing import application_testing from lightning_app.utilities.load_app import _patch_sys_argv @@ -10,7 +10,7 @@ def test_app_argparse_example(): original_argv = sys.argv command_line = [ - os.path.join(os.path.dirname(os.path.dirname(_PACKAGE_ROOT)), "examples/app_argparse/app.py"), + os.path.join(_PROJECT_ROOT, "examples/app_argparse/app.py"), "--app_args", "--use_gpu", "--without-server", diff --git a/tests/tests_app_examples/test_collect_failures.py b/tests/tests_app_examples/test_collect_failures.py index c149211e10..6bda6bf4bd 100644 --- a/tests/tests_app_examples/test_collect_failures.py +++ b/tests/tests_app_examples/test_collect_failures.py @@ -2,8 +2,8 @@ import os from time import sleep import pytest -from tests_app import _PROJECT_ROOT +from lightning_app import _PROJECT_ROOT from lightning_app.testing.testing import run_app_in_cloud diff --git a/tests/tests_pytorch/conftest.py b/tests/tests_pytorch/conftest.py index b6f4942dc6..5d8616ad00 100644 --- a/tests/tests_pytorch/conftest.py +++ b/tests/tests_pytorch/conftest.py @@ -191,11 +191,23 @@ def caplog(caplog): """ import logging - lightning_logger = logging.getLogger("pytorch_lightning") - propagate = lightning_logger.propagate - lightning_logger.propagate = True + root_logger = logging.getLogger() + root_propagate = root_logger.propagate + root_logger.propagate = True + + propagation_dict = { + name: logging.getLogger(name).propagate + for name in logging.root.manager.loggerDict + if name.startswith("pytorch_lightning") + } + for name in propagation_dict.keys(): + logging.getLogger(name).propagate = True + yield caplog - lightning_logger.propagate = propagate + + root_logger.propagate = root_propagate + for name, propagate in propagation_dict.items(): + logging.getLogger(name).propagate = propagate @pytest.fixture diff --git a/tests/tests_pytorch/loops/test_training_loop.py b/tests/tests_pytorch/loops/test_training_loop.py index dc52a08f06..203bcf38c6 100644 --- a/tests/tests_pytorch/loops/test_training_loop.py +++ b/tests/tests_pytorch/loops/test_training_loop.py @@ -20,6 +20,7 @@ import torch from pytorch_lightning import seed_everything, Trainer from pytorch_lightning.demos.boring_classes import BoringModel from pytorch_lightning.loops import FitLoop +from tests_pytorch.helpers.runif import RunIf def test_outputs_format(tmpdir): @@ -140,6 +141,7 @@ def test_should_stop_mid_epoch(tmpdir): assert model.validation_called_at == (0, 5) +@RunIf(standalone=True) def test_fit_loop_done_log_messages(caplog): fit_loop = FitLoop(max_epochs=1) trainer = Mock(spec=Trainer) @@ -210,6 +212,7 @@ def test_warning_valid_train_step_end(tmpdir): (4, 10, 4, True, True, True), ], ) +@RunIf(standalone=True) def test_should_stop_early_stopping_conditions_met( caplog, min_epochs, min_steps, current_epoch, early_stop, fit_loop_done, raise_debug_msg ):