parse strategies as own extras (#12975)

* parse strategies as own extras

* prune devel

* Update Makefile

Co-authored-by: Carlos Mocholí <carlossmocholi@gmail.com>

* revert parse_requirements

Co-authored-by: Carlos Mocholí <carlossmocholi@gmail.com>

.azure-pipelines/gpu-tests.yml

@@ -52,10 +52,11 @@ jobs:
     - bash: |
         python -c "fname = 'requirements/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)"
         CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
-        pip install "bagua-cuda$CUDA_VERSION_MM>=0.9.0"
-        pip install . --requirement requirements/devel.txt
         # TODO: Prepare a docker image with 1.8.2 (LTS) installed and remove manual installation.
         pip install torch==1.8.2+cu102 torchvision==0.9.2+cu102 torchtext==0.9.2 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+        pip install "bagua-cuda$CUDA_VERSION_MM>=0.9.0"
+        pip install . --requirement requirements/devel.txt
+        pip install . --requirement requirements/strategies.txt
         pip list
       displayName: 'Install dependencies'
 

.azure-pipelines/ipu-tests.yml

@@ -55,7 +55,7 @@ jobs:
         export GIT_TERMINAL_PROMPT=1
         python ./requirements/adjust-versions.py requirements/extra.txt
         python ./requirements/adjust-versions.py requirements/examples.txt
-        pip install . --requirement ./requirements/devel-base.txt
+        pip install . --requirement ./requirements/devel.txt
         pip list
       displayName: 'Install dependencies'
 

.github/workflows/ci_test-conda.yml

@@ -48,6 +48,7 @@ jobs:
         python ./requirements/adjust-versions.py requirements/extra.txt
         python ./requirements/adjust-versions.py requirements/examples.txt
         pip install -r requirements/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+        pip install -r requirements/strategies.txt
         # set a per-test timeout of 2.5 minutes to fail sooner; this aids with hanging tests
         pip install pytest-timeout
         pip list

.github/workflows/docs-checks.yml

@@ -43,7 +43,7 @@ jobs:
           pip install -q fire
           # python -m pip install --upgrade --user pip
           pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet
-          pip install --requirement requirements/devel-base.txt
+          pip install --requirement requirements/devel.txt
           pip install --requirement requirements/docs.txt
           pip list
         shell: bash

Makefile

@@ -20,10 +20,9 @@ clean:
 	rm -rf ./docs/source/api
 
 test: clean
-	# Review the CONTRIBUTING docmentation for other ways to test.
+	# Review the CONTRIBUTING documentation for other ways to test.
 	pip install -r requirements/devel.txt
-	# install APEX, see https://github.com/NVIDIA/apex#linux
-
+	pip install -r requirements/strategies.txt
 	# run tests with coverage
 	python -m coverage run --source pytorch_lightning -m pytest pytorch_lightning tests pl_examples -v
 	python -m coverage report

dockers/base-cuda/Dockerfile

@@ -90,7 +90,7 @@ RUN \
     python ./requirements/adjust-versions.py requirements/extra.txt ${PYTORCH_VERSION} && \
     python ./requirements/adjust-versions.py requirements/examples.txt ${PYTORCH_VERSION} && \
     # Install all requirements \
-    pip install -r requirements/devel-base.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \
+    pip install -r requirements/devel.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \
     rm -rf requirements.* && \
     rm assistant.py
 

dockers/base-xla/Dockerfile

@@ -97,7 +97,7 @@ RUN \
     # drop unnecessary packages
     python ./requirements/adjust-versions.py ./requirements/extra.txt && \
     # install PL dependencies
-    pip install --requirement ./requirements/devel-base.txt --no-cache-dir && \
+    pip install --requirement ./requirements/devel.txt --no-cache-dir && \
     cd .. && \
     rm -rf pytorch-lightning && \
     rm -rf /root/.cache

dockers/tpu-tests/Dockerfile

@@ -30,7 +30,7 @@ RUN cd pytorch-lightning && \
 RUN \
     pip install -q fire && \
     # drop unnecessary packages
-    pip install -r pytorch-lightning/requirements/devel-base.txt --no-cache-dir
+    pip install -r pytorch-lightning/requirements/devel.txt --no-cache-dir
 
 COPY ./dockers/tpu-tests/docker-entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh

requirements/devel-base.txt

@@ -1,14 +0,0 @@
-# install all mandatory dependencies
--r ../requirements.txt
-
-# install all extra dependencies for full package testing
--r ./extra.txt
-
-# install all loggers for full package testing
--r ./loggers.txt
-
-# extended list of dependencies for development and run lint and tests
--r ./test.txt
-
-# install all extra dependencies for running examples
--r ./examples.txt

requirements/devel.txt

@@ -7,9 +7,6 @@
 # install all loggers for full package testing
 -r ./loggers.txt
 
-# install all strategies for full package testing
--r ./strategies.txt
-
 # extended list of dependencies for development and run lint and tests
 -r ./test.txt
 

setup.py

@@ -16,6 +16,7 @@
 import os
 from importlib.util import module_from_spec, spec_from_file_location
 
+from pkg_resources import parse_requirements
 from setuptools import find_packages, setup
 
 # https://packaging.python.org/guides/single-sourcing-package-version/
@@ -46,16 +47,11 @@ extras = {
     "strategies": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="strategies.txt"),
     "test": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="test.txt"),
 }
-extras["dev"] = extras["extra"] + extras["loggers"] + extras["strategies"] + extras["test"]
-extras["all"] = extras["dev"] + extras["examples"]  # + extras['docs']
 
-# These packages shall be installed only on GPU machines
-PACKAGES_GPU_ONLY = ["horovod"]
-# create a version for CPU machines
-for ex in ("cpu", "cpu-extra"):
-    kw = ex.split("-")[1] if "-" in ex else "all"
-    # filter cpu only packages
-    extras[ex] = [pkg for pkg in extras[kw] if not any(pgpu.lower() in pkg.lower() for pgpu in PACKAGES_GPU_ONLY)]
+for req in parse_requirements(extras["strategies"]):
+    extras[req.key] = [str(req)]
+extras["dev"] = extras["extra"] + extras["loggers"] + extras["test"]
+extras["all"] = extras["dev"] + extras["examples"] + extras["strategies"]  # + extras['docs']
 
 long_description = setup_tools._load_readme_description(
     _PATH_ROOT, homepage=about.__homepage__, version=about.__version__