From 783ec43a85271d4fce6d215471d1aa27ffb7dbe6 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 9 May 2022 22:25:53 +0900 Subject: [PATCH] parse strategies as own extras (#12975) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * parse strategies as own extras * prune devel * Update Makefile Co-authored-by: Carlos Mocholí * revert parse_requirements Co-authored-by: Carlos Mocholí --- .azure-pipelines/gpu-tests.yml | 5 +++-- .azure-pipelines/ipu-tests.yml | 2 +- .github/workflows/ci_test-conda.yml | 1 + .github/workflows/docs-checks.yml | 2 +- Makefile | 5 ++--- dockers/base-cuda/Dockerfile | 2 +- dockers/base-xla/Dockerfile | 2 +- dockers/tpu-tests/Dockerfile | 2 +- requirements/devel-base.txt | 14 -------------- requirements/devel.txt | 3 --- setup.py | 14 +++++--------- 11 files changed, 16 insertions(+), 36 deletions(-) delete mode 100644 requirements/devel-base.txt diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml index ee7e952e69..c45613dbb7 100644 --- a/.azure-pipelines/gpu-tests.yml +++ b/.azure-pipelines/gpu-tests.yml @@ -52,10 +52,11 @@ jobs: - bash: | python -c "fname = 'requirements/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)" CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))") - pip install "bagua-cuda$CUDA_VERSION_MM>=0.9.0" - pip install . --requirement requirements/devel.txt # TODO: Prepare a docker image with 1.8.2 (LTS) installed and remove manual installation. pip install torch==1.8.2+cu102 torchvision==0.9.2+cu102 torchtext==0.9.2 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html + pip install "bagua-cuda$CUDA_VERSION_MM>=0.9.0" + pip install . --requirement requirements/devel.txt + pip install . --requirement requirements/strategies.txt pip list displayName: 'Install dependencies' diff --git a/.azure-pipelines/ipu-tests.yml b/.azure-pipelines/ipu-tests.yml index 0a60cf7867..80d49f9da3 100644 --- a/.azure-pipelines/ipu-tests.yml +++ b/.azure-pipelines/ipu-tests.yml @@ -55,7 +55,7 @@ jobs: export GIT_TERMINAL_PROMPT=1 python ./requirements/adjust-versions.py requirements/extra.txt python ./requirements/adjust-versions.py requirements/examples.txt - pip install . --requirement ./requirements/devel-base.txt + pip install . --requirement ./requirements/devel.txt pip list displayName: 'Install dependencies' diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index 31bd11429a..c907e13b4a 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -48,6 +48,7 @@ jobs: python ./requirements/adjust-versions.py requirements/extra.txt python ./requirements/adjust-versions.py requirements/examples.txt pip install -r requirements/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html + pip install -r requirements/strategies.txt # set a per-test timeout of 2.5 minutes to fail sooner; this aids with hanging tests pip install pytest-timeout pip list diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 7c9069e53d..dd7ac9314c 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -43,7 +43,7 @@ jobs: pip install -q fire # python -m pip install --upgrade --user pip pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet - pip install --requirement requirements/devel-base.txt + pip install --requirement requirements/devel.txt pip install --requirement requirements/docs.txt pip list shell: bash diff --git a/Makefile b/Makefile index e6cc00207e..efd2a7f3c5 100644 --- a/Makefile +++ b/Makefile @@ -20,10 +20,9 @@ clean: rm -rf ./docs/source/api test: clean - # Review the CONTRIBUTING docmentation for other ways to test. + # Review the CONTRIBUTING documentation for other ways to test. pip install -r requirements/devel.txt - # install APEX, see https://github.com/NVIDIA/apex#linux - + pip install -r requirements/strategies.txt # run tests with coverage python -m coverage run --source pytorch_lightning -m pytest pytorch_lightning tests pl_examples -v python -m coverage report diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile index c39670c1b8..2938a570aa 100644 --- a/dockers/base-cuda/Dockerfile +++ b/dockers/base-cuda/Dockerfile @@ -90,7 +90,7 @@ RUN \ python ./requirements/adjust-versions.py requirements/extra.txt ${PYTORCH_VERSION} && \ python ./requirements/adjust-versions.py requirements/examples.txt ${PYTORCH_VERSION} && \ # Install all requirements \ - pip install -r requirements/devel-base.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \ + pip install -r requirements/devel.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \ rm -rf requirements.* && \ rm assistant.py diff --git a/dockers/base-xla/Dockerfile b/dockers/base-xla/Dockerfile index 2fb5b99905..d933aac67d 100644 --- a/dockers/base-xla/Dockerfile +++ b/dockers/base-xla/Dockerfile @@ -97,7 +97,7 @@ RUN \ # drop unnecessary packages python ./requirements/adjust-versions.py ./requirements/extra.txt && \ # install PL dependencies - pip install --requirement ./requirements/devel-base.txt --no-cache-dir && \ + pip install --requirement ./requirements/devel.txt --no-cache-dir && \ cd .. && \ rm -rf pytorch-lightning && \ rm -rf /root/.cache diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile index 9a7528cc22..602ef1684b 100644 --- a/dockers/tpu-tests/Dockerfile +++ b/dockers/tpu-tests/Dockerfile @@ -30,7 +30,7 @@ RUN cd pytorch-lightning && \ RUN \ pip install -q fire && \ # drop unnecessary packages - pip install -r pytorch-lightning/requirements/devel-base.txt --no-cache-dir + pip install -r pytorch-lightning/requirements/devel.txt --no-cache-dir COPY ./dockers/tpu-tests/docker-entrypoint.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/docker-entrypoint.sh diff --git a/requirements/devel-base.txt b/requirements/devel-base.txt deleted file mode 100644 index 3252ebf39f..0000000000 --- a/requirements/devel-base.txt +++ /dev/null @@ -1,14 +0,0 @@ -# install all mandatory dependencies --r ../requirements.txt - -# install all extra dependencies for full package testing --r ./extra.txt - -# install all loggers for full package testing --r ./loggers.txt - -# extended list of dependencies for development and run lint and tests --r ./test.txt - -# install all extra dependencies for running examples --r ./examples.txt diff --git a/requirements/devel.txt b/requirements/devel.txt index 09c2699ccd..3252ebf39f 100644 --- a/requirements/devel.txt +++ b/requirements/devel.txt @@ -7,9 +7,6 @@ # install all loggers for full package testing -r ./loggers.txt -# install all strategies for full package testing --r ./strategies.txt - # extended list of dependencies for development and run lint and tests -r ./test.txt diff --git a/setup.py b/setup.py index e5be7cd37d..d3cc4a0602 100755 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ import os from importlib.util import module_from_spec, spec_from_file_location +from pkg_resources import parse_requirements from setuptools import find_packages, setup # https://packaging.python.org/guides/single-sourcing-package-version/ @@ -46,16 +47,11 @@ extras = { "strategies": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="strategies.txt"), "test": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="test.txt"), } -extras["dev"] = extras["extra"] + extras["loggers"] + extras["strategies"] + extras["test"] -extras["all"] = extras["dev"] + extras["examples"] # + extras['docs'] -# These packages shall be installed only on GPU machines -PACKAGES_GPU_ONLY = ["horovod"] -# create a version for CPU machines -for ex in ("cpu", "cpu-extra"): - kw = ex.split("-")[1] if "-" in ex else "all" - # filter cpu only packages - extras[ex] = [pkg for pkg in extras[kw] if not any(pgpu.lower() in pkg.lower() for pgpu in PACKAGES_GPU_ONLY)] +for req in parse_requirements(extras["strategies"]): + extras[req.key] = [str(req)] +extras["dev"] = extras["extra"] + extras["loggers"] + extras["test"] +extras["all"] = extras["dev"] + extras["examples"] + extras["strategies"] # + extras['docs'] long_description = setup_tools._load_readme_description( _PATH_ROOT, homepage=about.__homepage__, version=about.__version__