From 291dc1b61553813bc8057995d91c963fc77bc1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 2 Sep 2022 00:13:12 +0200 Subject: [PATCH] Standalone Lite CI setup (#14451) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jirka Co-authored-by: Carlos Mocholí Co-authored-by: Jirka Borovec --- .azure/gpu-tests-lite.yml | 101 ++++++++++++++++ .azure/gpu-tests.yml | 1 + .github/checkgroup.yml | 16 +++ .github/workflows/ci-lite-test-full.yml | 120 ++++++++++++++++++++ .github/workflows/ci-pkg-install.yml | 3 +- .pre-commit-config.yaml | 6 +- dockers/tpu-tests/tpu_test_cases.jsonnet | 1 + pyproject.toml | 1 + requirements/lite/base.txt | 7 ++ requirements/lite/devel.txt | 2 + requirements/lite/strategies.txt | 5 + setup.py | 2 +- src/lightning_lite/CHANGELOG.md | 28 +++++ src/lightning_lite/README.md | 1 + src/lightning_lite/__about__.py | 23 ++++ src/lightning_lite/__init__.py | 4 + src/lightning_lite/__setup__.py | 98 ++++++++++++++++ src/lightning_lite/__version__.py | 1 + src/lightning_lite/lite.py | 3 + tests/tests_lite/__init__.py | 0 tests/tests_lite/helpers/__init__.py | 0 tests/tests_lite/helpers/runif.py | 75 ++++++++++++ tests/tests_lite/run_standalone_tests.sh | 92 +++++++++++++++ tests/tests_lite/test_lite.py | 12 ++ tests/tests_pytorch/run_standalone_tests.sh | 5 +- 25 files changed, 601 insertions(+), 6 deletions(-) create mode 100644 .azure/gpu-tests-lite.yml create mode 100644 .github/workflows/ci-lite-test-full.yml create mode 100644 requirements/lite/base.txt create mode 100644 requirements/lite/devel.txt create mode 100644 requirements/lite/strategies.txt create mode 100644 src/lightning_lite/CHANGELOG.md create mode 100644 src/lightning_lite/README.md create mode 100644 src/lightning_lite/__about__.py create mode 100644 src/lightning_lite/__init__.py create mode 100644 src/lightning_lite/__setup__.py create mode 100644 src/lightning_lite/__version__.py create mode 100644 src/lightning_lite/lite.py create mode 100644 tests/tests_lite/__init__.py create mode 100644 tests/tests_lite/helpers/__init__.py create mode 100644 tests/tests_lite/helpers/runif.py create mode 100644 tests/tests_lite/run_standalone_tests.sh create mode 100644 tests/tests_lite/test_lite.py diff --git a/.azure/gpu-tests-lite.yml b/.azure/gpu-tests-lite.yml new file mode 100644 index 0000000000..3260fdd702 --- /dev/null +++ b/.azure/gpu-tests-lite.yml @@ -0,0 +1,101 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: + tags: + include: + - '*' + branches: + include: + - "master" + - "release/*" + - "refs/tags/*" + paths: + include: + - ".azure/gpu-tests-lite.yml" + - "requirements/lite/**" + - "src/lightning_lite/**" + - "tests/tests_lite/**" + +pr: + - "master" + - "release/*" + +jobs: + - job: testing + # how long to run the job before automatically cancelling + timeoutInMinutes: "20" + # how much time to give 'run always even if cancelled tasks' before stopping them + cancelTimeoutInMinutes: "2" + pool: azure-jirka-spot + container: + image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.6.1" + # default shm size is 64m. Increase it to avoid: + # 'Error while creating shared memory: unhandled system error, NCCL version 2.7.8' + options: "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all --shm-size=512m" + workspace: + clean: all + + steps: + - bash: | + lspci | egrep 'VGA|3D' + whereis nvidia + nvidia-smi + which python && which pip + python --version + pip --version + pip list + displayName: 'Image info & NVIDIA' + + - bash: | + set -e + TORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])") + CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))") + python ./requirements/pytorch/adjust-versions.py requirements/lite/base.txt ${PYTORCH_VERSION} + pip install -e .[strategies] --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html + pip install --requirement requirements/pytorch/devel.txt --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html + pip list + env: + PACKAGE_NAME: pytorch + FREEZE_REQUIREMENTS: 1 + displayName: 'Install dependencies' + + - bash: | + set -e + python requirements/collect_env_details.py + python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'" + displayName: 'Env details' + + - bash: python -m coverage run --source lightning_lite -m pytest --ignore benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50 + env: + PL_RUN_CUDA_TESTS: "1" + workingDirectory: tests/tests_lite + displayName: 'Testing: Lite standard' + timeoutInMinutes: "10" + + - bash: bash run_standalone_tests.sh + workingDirectory: tests/tests_lite + env: + PL_USE_MOCKED_MNIST: "1" + PL_RUN_CUDA_TESTS: "1" + PL_STANDALONE_TESTS_SOURCE: "lightning_lite" + displayName: 'Testing: Lite standalone tests' + timeoutInMinutes: "10" + + - bash: | + python -m coverage report + python -m coverage xml + python -m coverage html + python -m codecov --token=$(CODECOV_TOKEN) --commit=$(Build.SourceVersion) --flags=gpu,pytest --name="GPU-coverage" --env=linux,azure + ls -l + workingDirectory: tests/tests_lite + displayName: 'Statistics' + + - task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testResultsFiles: '$(Build.StagingDirectory)/test-results.xml' + testRunTitle: '$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)' + condition: succeededOrFailed() diff --git a/.azure/gpu-tests.yml b/.azure/gpu-tests.yml index 3ed4601d1b..e53d8f0756 100644 --- a/.azure/gpu-tests.yml +++ b/.azure/gpu-tests.yml @@ -119,6 +119,7 @@ jobs: env: PL_USE_MOCKED_MNIST: "1" PL_RUN_CUDA_TESTS: "1" + PL_STANDALONE_TESTS_SOURCE: "pytorch_lightning" displayName: 'Testing: PyTorch standalone tests' timeoutInMinutes: "35" condition: eq(variables['continue'], '1') diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 531df1ebea..e8892926f6 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -41,6 +41,15 @@ subprojects: - "pl-cpu (windows-2022, 3.10, latest, stable)" - "pl-cpu (windows-2022, 3.7, latest, stable)" - "pl-cpu (windows-2022, 3.7, oldest, stable)" + - "lite-cpu (macOS-11, 3.10, latest, stable)" + - "lite-cpu (macOS-11, 3.7, latest, stable)" + - "lite-cpu (macOS-11, 3.7, oldest, stable)" + - "lite-cpu (ubuntu-20.04, 3.10, latest, stable)" + - "lite-cpu (ubuntu-20.04, 3.7, latest, stable)" + - "lite-cpu (ubuntu-20.04, 3.7, oldest, stable)" + - "lite-cpu (windows-2022, 3.10, latest, stable)" + - "lite-cpu (windows-2022, 3.7, latest, stable)" + - "lite-cpu (windows-2022, 3.7, oldest, stable)" - "make-doctest (pytorch)" - "make-html (pytorch)" - "mypy" @@ -60,6 +69,13 @@ subprojects: checks: - "pytorch-lightning (GPUs)" + - id: "lightning_lite: Azure GPU" + paths: + - ".azure/gpu-tests-lite.yml" + - "tests/tests_lite/run_standalone_*.sh" + checks: + - "lightning-lite (GPUs)" + - id: "pytorch_lightning: Azure HPU" paths: - ".azure/hpu-tests.yml" diff --git a/.github/workflows/ci-lite-test-full.yml b/.github/workflows/ci-lite-test-full.yml new file mode 100644 index 0000000000..69c8a12069 --- /dev/null +++ b/.github/workflows/ci-lite-test-full.yml @@ -0,0 +1,120 @@ +name: Test Lite full + +# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows +on: # Trigger the workflow on push or pull request, but only for the master branch + push: + branches: [master, "release/*"] + pull_request: + branches: [master, "release/*"] + types: [opened, reopened, ready_for_review, synchronize] + paths: + - "requirements/lite/**" + - "src/lightning_lite/**" + - "tests/tests_lite/**" + - "setup.cfg" # includes pytest config + - ".github/workflows/ci-lite-test-full.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} + cancel-in-progress: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }} + +jobs: + + lite-cpu: + runs-on: ${{ matrix.os }} + if: github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, windows-2022, macOS-11] + python-version: ["3.7", "3.10"] # minimum, maximum + requires: ["oldest", "latest"] + release: ["stable"] + exclude: + # There's no distribution of the oldest PyTorch 1.9 for Python 3.10. + # TODO: Remove the exclusion when dropping PyTorch 1.9 support. + - {python-version: "3.10", requires: "oldest"} + + timeout-minutes: 40 + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Reset caching + run: python -c "import time; days = time.time() / 60 / 60 / 24; print(f'TIME_PERIOD=d{int(days / 2) * 2}')" >> $GITHUB_ENV + + - name: basic setup + run: | + pip --version + pip install -q fire + + - name: Setup Windows + if: runner.os == 'windows' + run: | + python .actions/assistant.py requirements_prune_pkgs horovod + + - name: Set min. dependencies + if: matrix.requires == 'oldest' + run: | + python .actions/assistant.py replace_oldest_ver + + # Note: This uses an internal pip API and may not always work + # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow + - name: Get pip cache dir + id: pip-cache + run: echo "::set-output name=dir::$(pip cache dir)" + + - name: pip cache + uses: actions/cache@v3 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-${{ hashFiles('requirements/lite/*.txt') }} + restore-keys: | + ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}- + + - name: Install dependencies + env: + PACKAGE_NAME: pytorch # TODO(lite) does this need to say lite? + FREEZE_REQUIREMENTS: 1 + run: | + flag=$(python -c "print('--pre' if '${{matrix.release}}' == 'pre' else '')" 2>&1) + url=$(python -c "print('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html')" 2>&1) + pip install -e .[test] --upgrade $flag --find-links "https://download.pytorch.org/whl/${url}" + pip list + shell: bash + + - name: Testing Lite + working-directory: tests/tests_lite + # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003 + run: coverage run --source lightning_lite -m pytest -v --durations=50 --junitxml=results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml + + - name: Upload pytest results + if: failure() + uses: actions/upload-artifact@v3 + with: + name: unittest-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }} + path: tests/tests_lite/results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml + + - name: Statistics + if: success() + working-directory: tests/tests_lite + run: | + coverage report + coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + if: always() + # see: https://github.com/actions/toolkit/issues/399 + continue-on-error: true + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: tests/tests_lite/coverage.xml + flags: cpu,pytest,python${{ matrix.python-version }} + name: CPU-coverage + fail_ci_if_error: false diff --git a/.github/workflows/ci-pkg-install.yml b/.github/workflows/ci-pkg-install.yml index a9fdd36693..6eedad2539 100644 --- a/.github/workflows/ci-pkg-install.yml +++ b/.github/workflows/ci-pkg-install.yml @@ -34,7 +34,7 @@ jobs: max-parallel: 1 matrix: os: [ubuntu-20.04, macOS-11, windows-2022] - pkg: ["app", "pytorch"] + pkg: ["app", "lite", "pytorch"] python-version: [3.8] # , 3.9 steps: @@ -110,6 +110,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Dowload package + # todo: download also lite after it is fist published run: | pip install -q fire requests for pkg in 'app' 'pytorch' ; do diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b8ec52396..1930f99cd1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,8 @@ repos: docs/source-pytorch/_static/images/general/pl_overview_flat.jpg| docs/source-pytorch/_static/images/general/pl_overview.gif| src/lightning_app/cli/pl-app-template/ui/yarn.lock| - src/pytorch_lightning/CHANGELOG.md + src/pytorch_lightning/CHANGELOG.md| + src/lightning_lite/CHANGELOG.md )$ - id: detect-private-key @@ -98,7 +99,8 @@ repos: exclude: | (?x)^( src/pytorch_lightning/CHANGELOG.md| - src/lightning_app/CHANGELOG.md + src/lightning_app/CHANGELOG.md| + src/lightning_lite/CHANGELOG.md )$ - repo: https://github.com/PyCQA/flake8 diff --git a/dockers/tpu-tests/tpu_test_cases.jsonnet b/dockers/tpu-tests/tpu_test_cases.jsonnet index f2c106f220..1f6bf4c41b 100644 --- a/dockers/tpu-tests/tpu_test_cases.jsonnet +++ b/dockers/tpu-tests/tpu_test_cases.jsonnet @@ -39,6 +39,7 @@ local tputests = base.BaseTest { cd tests/tests_pytorch coverage run --source=pytorch_lightning -m pytest -vv --durations=0 ./ echo "\n||| Running standalone tests |||\n" + export PL_STANDALONE_TESTS_SOURCE=pytorch_lightning export PL_STANDALONE_TESTS_BATCH_SIZE=1 bash run_standalone_tests.sh echo "\n||| END PYTEST LOGS |||\n" diff --git a/pyproject.toml b/pyproject.toml index 19702524bc..df89de3d09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ requires = [ known_first_party = [ "pl_examples", "pytorch_lightning", + "lightning_lite", "tests_pytorch", ] profile = "black" diff --git a/requirements/lite/base.txt b/requirements/lite/base.txt new file mode 100644 index 0000000000..89061745d4 --- /dev/null +++ b/requirements/lite/base.txt @@ -0,0 +1,7 @@ +# NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package +# in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment + +torch>=1.9.*, <1.13.0 +fsspec[http]>=2021.05.0, !=2021.06.0, <2022.6.0 +packaging>=17.0, <=21.3 +typing-extensions>=4.0.0, <4.3.1 diff --git a/requirements/lite/devel.txt b/requirements/lite/devel.txt new file mode 100644 index 0000000000..a7d1aa9843 --- /dev/null +++ b/requirements/lite/devel.txt @@ -0,0 +1,2 @@ +# install all mandatory dependencies +-r ./base.txt diff --git a/requirements/lite/strategies.txt b/requirements/lite/strategies.txt new file mode 100644 index 0000000000..c06fac969b --- /dev/null +++ b/requirements/lite/strategies.txt @@ -0,0 +1,5 @@ +# NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package +# in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment + +fairscale>=0.4.5, <=0.4.6 +deepspeed>=0.6.0, <=0.7.0 diff --git a/setup.py b/setup.py index 82a4a969ec..3048f8a1ae 100755 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ from types import ModuleType from setuptools import setup _PACKAGE_NAME = os.environ.get("PACKAGE_NAME", "") -_PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"} +_PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app", "lite": "lightning_lite"} _REAL_PKG_NAME = _PACKAGE_MAPPING.get(_PACKAGE_NAME, _PACKAGE_NAME) # https://packaging.python.org/guides/single-sourcing-package-version/ # http://blog.ionelmc.ro/2014/05/25/python-packaging/ diff --git a/src/lightning_lite/CHANGELOG.md b/src/lightning_lite/CHANGELOG.md new file mode 100644 index 0000000000..280a7e2ac0 --- /dev/null +++ b/src/lightning_lite/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). + +## [0.0.x] - 2022-MM-DD + + +### Added + +- + +### Changed + +- + +### Deprecated + +- + +### Removed + +- + +### Fixed + +- diff --git a/src/lightning_lite/README.md b/src/lightning_lite/README.md new file mode 100644 index 0000000000..464090415c --- /dev/null +++ b/src/lightning_lite/README.md @@ -0,0 +1 @@ +# TODO diff --git a/src/lightning_lite/__about__.py b/src/lightning_lite/__about__.py new file mode 100644 index 0000000000..deb737cdf4 --- /dev/null +++ b/src/lightning_lite/__about__.py @@ -0,0 +1,23 @@ +import time + +__author__ = "Lightning AI et al." +__author_email__ = "pytorch@lightning.ai" +__license__ = "Apache-2.0" +__copyright__ = f"Copyright (c) 2022-{time.strftime('%Y')}, {__author__}." +__homepage__ = "https://github.com/Lightning-AI/lightning" +__docs_url__ = "https://pytorch-lightning.readthedocs.io/en/stable/" +# TODO +__docs__ = "" +__long_docs__ = """ + +""" + +__all__ = [ + "__author__", + "__author_email__", + "__copyright__", + "__docs__", + "__docs_url__", + "__homepage__", + "__license__", +] diff --git a/src/lightning_lite/__init__.py b/src/lightning_lite/__init__.py new file mode 100644 index 0000000000..5e0d0ad5cb --- /dev/null +++ b/src/lightning_lite/__init__.py @@ -0,0 +1,4 @@ +"""Root package info.""" + +from lightning_lite.__about__ import * # noqa: F401, F403 +from lightning_lite.__version__ import version as __version__ # noqa: F401 diff --git a/src/lightning_lite/__setup__.py b/src/lightning_lite/__setup__.py new file mode 100644 index 0000000000..5c0c8ae660 --- /dev/null +++ b/src/lightning_lite/__setup__.py @@ -0,0 +1,98 @@ +import os +from importlib.util import module_from_spec, spec_from_file_location +from types import ModuleType +from typing import Any, Dict + +from setuptools import find_packages + +_PROJECT_ROOT = "." +_SOURCE_ROOT = os.path.join(_PROJECT_ROOT, "src") +_PACKAGE_ROOT = os.path.join(_SOURCE_ROOT, "lightning_lite") +_PATH_REQUIREMENTS = os.path.join("requirements", "lite") +_FREEZE_REQUIREMENTS = bool(int(os.environ.get("FREEZE_REQUIREMENTS", 0))) + + +def _load_py_module(name: str, location: str) -> ModuleType: + spec = spec_from_file_location(name, location) + assert spec, f"Failed to load module {name} from {location}" + py = module_from_spec(spec) + assert spec.loader, f"ModuleSpec.loader is None for {name} from {location}" + spec.loader.exec_module(py) + return py + + +def _adjust_manifest(**__: Any) -> None: + manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in") + assert os.path.isfile(manifest_path) + with open(manifest_path) as fp: + lines = fp.readlines() + lines += [ + "recursive-exclude src *.md" + os.linesep, + "recursive-exclude requirements *.txt" + os.linesep, + "recursive-include requirements/lite *.txt" + os.linesep, + "recursive-include src/lightning_lite *.md" + os.linesep, + ] + + # TODO: remove this once lightning-ui package is ready as a dependency + lines += ["recursive-include src/lightning_app/ui *" + os.linesep] + + with open(manifest_path, "w") as fp: + fp.writelines(lines) + + +def _setup_args(**__: Any) -> Dict[str, Any]: + _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py") + _setup_tools = _load_py_module("setup_tools", _path_setup_tools) + _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py")) + _version = _load_py_module("version", os.path.join(_PACKAGE_ROOT, "__version__.py")) + _long_description = _setup_tools.load_readme_description( + _PACKAGE_ROOT, homepage=_about.__homepage__, version=_version.version + ) + + return dict( + name="lightning-lite", + version=_version.version, # todo: consider using date version + branch for installation from source + description=_about.__docs__, + author=_about.__author__, + author_email=_about.__author_email__, + url=_about.__homepage__, + download_url="https://github.com/Lightning-AI/lightning", + license=_about.__license__, + packages=find_packages(where="src", include=["lightning_lite", "lightning_lite.*"]), + package_dir={"": "src"}, + long_description=_long_description, + long_description_content_type="text/markdown", + include_package_data=True, + zip_safe=False, + keywords=["deep learning", "pytorch", "AI"], + python_requires=">=3.7", + setup_requires=["wheel"], + install_requires=_setup_tools.load_requirements(_PATH_REQUIREMENTS, unfreeze=not _FREEZE_REQUIREMENTS), + # extras_require=_prepare_extras(), # todo + project_urls={ + "Bug Tracker": "https://github.com/Lightning-AI/lightning/issues", + "Documentation": "https://pytorch-lightning.rtfd.io/en/latest/", + "Source Code": "https://github.com/Lightning-AI/lightning", + }, + classifiers=[ + "Environment :: Console", + "Natural Language :: English", + # How mature is this project? Common values are + # 3 - Alpha, 4 - Beta, 5 - Production/Stable + "Development Status :: 4 - Beta", + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Information Analysis", + # Pick your license as you wish + # 'License :: OSI Approved :: BSD License', + "Operating System :: OS Independent", + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + ], + ) diff --git a/src/lightning_lite/__version__.py b/src/lightning_lite/__version__.py new file mode 100644 index 0000000000..61767daea3 --- /dev/null +++ b/src/lightning_lite/__version__.py @@ -0,0 +1 @@ +version = "0.0.0dev" diff --git a/src/lightning_lite/lite.py b/src/lightning_lite/lite.py new file mode 100644 index 0000000000..65fee1bf09 --- /dev/null +++ b/src/lightning_lite/lite.py @@ -0,0 +1,3 @@ +class LightningLite: + # Placeholder for real implementation + pass diff --git a/tests/tests_lite/__init__.py b/tests/tests_lite/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/tests_lite/helpers/__init__.py b/tests/tests_lite/helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/tests_lite/helpers/runif.py b/tests/tests_lite/helpers/runif.py new file mode 100644 index 0000000000..280af6a96f --- /dev/null +++ b/tests/tests_lite/helpers/runif.py @@ -0,0 +1,75 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pytest +import torch + + +# TODO(lite): Add all RunIf conditions once the relevant utilities have moved to lite source dir +class RunIf: + """RunIf wrapper for simple marking specific cases, fully compatible with pytest.mark:: + + @RunIf(min_torch="0.0") + @pytest.mark.parametrize("arg1", [1, 2.0]) + def test_wrapper(arg1): + assert arg1 > 0.0 + """ + + def __new__( + self, + *args, + min_cuda_gpus: int = 0, + standalone: bool = False, + **kwargs, + ): + """ + Args: + *args: Any :class:`pytest.mark.skipif` arguments. + min_cuda_gpus: Require this number of gpus and that the ``PL_RUN_CUDA_TESTS=1`` environment variable is set. + standalone: Mark the test as standalone, our CI will run it in a separate process. + This requires that the ``PL_RUN_STANDALONE_TESTS=1`` environment variable is set. + **kwargs: Any :class:`pytest.mark.skipif` keyword arguments. + """ + conditions = [] + reasons = [] + + if min_cuda_gpus: + conditions.append(torch.cuda.device_count() < min_cuda_gpus) + reasons.append(f"GPUs>={min_cuda_gpus}") + # used in conftest.py::pytest_collection_modifyitems + kwargs["min_cuda_gpus"] = True + + if standalone: + env_flag = os.getenv("PL_RUN_STANDALONE_TESTS", "0") + conditions.append(env_flag != "1") + reasons.append("Standalone execution") + # used in conftest.py::pytest_collection_modifyitems + kwargs["standalone"] = True + + reasons = [rs for cond, rs in zip(conditions, reasons) if cond] + return pytest.mark.skipif( + *args, condition=any(conditions), reason=f"Requires: [{' + '.join(reasons)}]", **kwargs + ) + + +@RunIf(min_torch="99") +def test_always_skip(): + exit(1) + + +@pytest.mark.parametrize("arg1", [0.5, 1.0, 2.0]) +@RunIf(min_torch="0.0") +def test_wrapper(arg1: float): + assert arg1 > 0.0 diff --git a/tests/tests_lite/run_standalone_tests.sh b/tests/tests_lite/run_standalone_tests.sh new file mode 100644 index 0000000000..9f91f4bc3f --- /dev/null +++ b/tests/tests_lite/run_standalone_tests.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +# THIS FILE ASSUMES IT IS RUN INSIDE THE tests/tests_ DIRECTORY + +# Batch size for testing: Determines how many standalone test invocations run in parallel +# It can be set through the env variable PL_STANDALONE_TESTS_BATCH_SIZE and defaults to 6 if not set +test_batch_size="${PL_STANDALONE_TESTS_BATCH_SIZE:-6}" +source="${PL_STANDALONE_TESTS_SOURCE}" + +# this environment variable allows special tests to run +export PL_RUN_STANDALONE_TESTS=1 +# python arguments +defaults="-m coverage run --source $source --append -m pytest --no-header" + +# find tests marked as `@RunIf(standalone=True)`. done manually instead of with pytest because it is faster +grep_output=$(grep --recursive --word-regexp . --regexp 'standalone=True' --include '*.py') + +# file paths, remove duplicates +files=$(echo "$grep_output" | cut -f1 -d: | sort | uniq) + +# get the list of parametrizations. we need to call them separately. the last two lines are removed. +# note: if there's a syntax error, this will fail with some garbled output +if [[ "$OSTYPE" == "darwin"* ]]; then + parametrizations=$(python -m pytest $files --collect-only --quiet "$@" | tail -r | sed -e '1,3d' | tail -r) +else + parametrizations=$(python -m pytest $files --collect-only --quiet "$@" | head -n -2) +fi +# remove the "tests/tests_lite" path suffixes +parametrizations=${parametrizations//"tests/tests_lite/"/} +parametrizations_arr=($parametrizations) + +# tests to skip - space separated +blocklist='utilities/test_warnings.py' +report='' + +rm -f standalone_test_output.txt # in case it exists, remove it +function show_batched_output { + if [ -f standalone_test_output.txt ]; then # if exists + cat standalone_test_output.txt + rm standalone_test_output.txt + fi +} +trap show_batched_output EXIT # show the output on exit + +for i in "${!parametrizations_arr[@]}"; do + parametrization=${parametrizations_arr[$i]} + + # check blocklist + if echo $blocklist | grep -F "${parametrization}"; then + report+="Skipped\t$parametrization\n" + # do not continue the loop because we might need to wait for batched jobs + else + echo "Running $parametrization" + # execute the test in the background + # redirect to a log file that buffers test output. since the tests will run in the background, we cannot let them + # output to std{out,err} because the outputs would be garbled together + python ${defaults} "$parametrization" &>> standalone_test_output.txt & + # save the PID in an array + pids[${i}]=$! + # add row to the final report + report+="Ran\t$parametrization\n" + fi + + if ((($i + 1) % $test_batch_size == 0)); then + # wait for running tests + for pid in ${pids[*]}; do wait $pid; done + unset pids # empty the array + show_batched_output + fi +done +# wait for leftover tests +for pid in ${pids[*]}; do wait $pid; done +show_batched_output + +# echo test report +printf '=%.s' {1..80} +printf "\n$report" +printf '=%.s' {1..80} +printf '\n' diff --git a/tests/tests_lite/test_lite.py b/tests/tests_lite/test_lite.py new file mode 100644 index 0000000000..a7df3089cb --- /dev/null +++ b/tests/tests_lite/test_lite.py @@ -0,0 +1,12 @@ +from tests_lite.helpers.runif import RunIf + +from lightning_lite.lite import LightningLite # noqa: F401 + + +def test_placeholder(tmpdir): + assert True + + +@RunIf(min_cuda_gpus=2, standalone=True) +def test_placeholder_standalone(tmpdir): + assert True diff --git a/tests/tests_pytorch/run_standalone_tests.sh b/tests/tests_pytorch/run_standalone_tests.sh index 7e9292f445..1443c6885c 100644 --- a/tests/tests_pytorch/run_standalone_tests.sh +++ b/tests/tests_pytorch/run_standalone_tests.sh @@ -13,16 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. set -e -# THIS FILE ASSUMES IT IS RUN INSIDE THE tests/tests_pytorch DIRECTORY +# THIS FILE ASSUMES IT IS RUN INSIDE THE tests/tests_ DIRECTORY # Batch size for testing: Determines how many standalone test invocations run in parallel # It can be set through the env variable PL_STANDALONE_TESTS_BATCH_SIZE and defaults to 6 if not set test_batch_size="${PL_STANDALONE_TESTS_BATCH_SIZE:-6}" +source="${PL_STANDALONE_TESTS_SOURCE}" # this environment variable allows special tests to run export PL_RUN_STANDALONE_TESTS=1 # python arguments -defaults='-m coverage run --source pytorch_lightning --append -m pytest --no-header' +defaults="-m coverage run --source $source --append -m pytest --no-header" # find tests marked as `@RunIf(standalone=True)`. done manually instead of with pytest because it is faster grep_output=$(grep --recursive --word-regexp . --regexp 'standalone=True' --include '*.py')