clean requirements (#2128)

* clean requirements

* missing

* missing

* req

* min

* default >> base

* base.txt

.circleci/config.yml

@@ -12,9 +12,9 @@ references:
       command: |
         sudo apt-get update && sudo apt-get install -y cmake
         pip install "$TORCH_VERSION"
-        pip install -r requirements.txt -q
+        pip install -r requirements/base.txt -q
         sudo pip install pytest -q
-        pip install -r ./tests/requirements-devel.txt -q
+        pip install -r requirements/devel.txt -q
 
   tests: &tests
    run:
@@ -28,7 +28,7 @@ references:
    run:
      name: PL Examples
      command: |
-       pip install -r ./pl_examples/requirements.txt --user
+       pip install -r ./requirements/examples.txt --user
        python --version ; pip --version ; pip list
        py.test pl_examples -v --junitxml=test-reports/pytest_junit.xml
      no_output_timeout: 20m
@@ -69,7 +69,7 @@ references:
        # we need to use py3.7 ot higher becase of an issue with metaclass inheritence
        pyenv global 3.7.3
        python --version
-       pip install -r docs/requirements.txt
+       pip install -r requirements/docs.txt
        cd docs; make clean; make html --debug --jobs 2 SPHINXOPTS="-W"
 
   test_docs: &test_docs
@@ -78,7 +78,7 @@ references:
      command: |
        # Second run examples in docs
        sudo apt-get update && sudo apt-get install -y cmake
-       sudo pip install -r docs/requirements.txt
+       sudo pip install -r requirements/docs.txt
        cd docs; make doctest; make coverage
 
 jobs:

.drone.yml

@@ -33,9 +33,9 @@ steps:
     - nvidia-smi
     #- bash ./tests/install_AMP.sh
     - apt-get update && apt-get install -y cmake
-    - pip install -r requirements.txt --user -q
-    - pip install -r ./tests/requirements-devel.txt --user -q
-    #- pip install -r ./docs/requirements.txt --user -q
+    - pip install -r ./requirements/base.txt --user -q
+    - pip install -r ./requirements/devel.txt --user -q
+    #- pip install -r ./requirements/docs.txt --user -q
     - pip list
     - python -c "import torch ; print(' & '.join([torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]) if torch.cuda.is_available() else 'only CPU')"
     - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests benchmarks -v  --durations=25 # --flake8

.github/CONTRIBUTING.md

@@ -60,7 +60,7 @@ Have a favorite feature from other libraries like fast.ai or transformers? Those
 
 We are currently looking for help implementing new features or adding bug fixes.
 
-A lot of good work has already been done in project mechanics (requirements.txt, setup.py, pep8, badges, ci, etc...) so we're in a good state there thanks to all the early contributors (even pre-beta release)!
+A lot of good work has already been done in project mechanics (requirements/base.txt, setup.py, pep8, badges, ci, etc...) so we're in a good state there thanks to all the early contributors (even pre-beta release)!
 
 ### Bug Fixes:
 
@@ -137,8 +137,8 @@ formatting errors. In certain cases, a missing blank line or a wrong indent can
 Run these commands
 
 ```bash
+pip install -r requirements/docs.txt
 cd docs
-pip install -r requirements.txt
 make html
 ```
 
@@ -153,9 +153,8 @@ Testing your work locally will help you speed up the process since it allows you
 To setup a local development environment, install both local and test dependencies:
 
 ```bash
-python -m pip install -r requirements.txt
-python -m pip install -r tests/requirements-devel.txt
-python -m pip install -r pl_examples/requirements.txt
+python -m pip install -r requirements/devel.txt
+python -m pip install -r requirements/examples.txt
 python -m pip pre-commit install
 ```
 

.github/workflows/ci-testing.yml

@@ -51,26 +51,26 @@ jobs:
     - name: Setup Windows
       if: runner.os == 'windows'
       run: |
-        python -c "lines = [line for line in open('requirements-extra.txt').readlines() if not line.startswith('horovod')] ; open('requirements-extra.txt', 'w').writelines(lines)"
+        python -c "lines = [line for line in open('requirements/extra.txt').readlines() if not line.startswith('horovod')] ; open('requirements/extra.txt', 'w').writelines(lines)"
 
     # TODO: remove after https://github.com/pytorch/pytorch/issues/32186 is resolved
     - name: Setup Windows on Latest
       if: runner.os == 'windows' && matrix.requires == 'latest'
       run: |
-        python -c "req = open('requirements.txt').read().replace('torch>=1.3', 'torch<1.5') ; open('requirements.txt', 'w').write(req)"
+        python -c "req = open('requirements/base.txt').read().replace('torch>=1.3', 'torch<1.5') ; open('requirements/base.txt', 'w').write(req)"
 
     # versions <= 1.3 may have issues on mac with some BLAS ops due to missing mkl (https://github.com/pytorch/pytorch/issues/18996)
     - name: Setup MacOS Minimal
       if: runner.os == 'macOS' && matrix.requires == 'minimal'
       run : |
-        python -c "req = open('requirements.txt').read().replace('torch>=1.3', 'torch>=1.4') ; open('requirements.txt', 'w').write(req)"
+        python -c "req = open('requirements/base.txt').read().replace('torch>=1.3', 'torch>=1.4') ; open('requirements/base.txt', 'w').write(req)"
 
     - name: Set min. dependencies
       if: matrix.requires == 'minimal'
       run: |
-        python -c "req = open('requirements.txt').read().replace('>=', '==') ; open('requirements.txt', 'w').write(req)"
-        python -c "req = open('requirements-extra.txt').read().replace('>=', '==') ; open('requirements-extra.txt', 'w').write(req)"
-        python -c "req = open('tests/requirements-devel.txt').read().replace('>=', '==') ; open('tests/requirements-devel.txt', 'w').write(req)"
+        python -c "req = open('requirements/base.txt').read().replace('>=', '==') ; open('requirements/base.txt', 'w').write(req)"
+        python -c "req = open('requirements/extra.txt').read().replace('>=', '==') ; open('requirements/extra.txt', 'w').write(req)"
+        python -c "req = open('requirements/devel.txt').read().replace('>=', '==') ; open('requirements/devel.txt', 'w').write(req)"
 
     # Note: This uses an internal pip API and may not always work
     # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
@@ -83,15 +83,15 @@ jobs:
       uses: actions/cache@v1
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-extra.txt') }}
+        key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-${{ hashFiles('requirements/base.txt') }}-${{ hashFiles('requirements/extra.txt') }}
         restore-keys: |
           ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-
 
     - name: Install dependencies
       run: |
         # python -m pip install --upgrade --user pip
-        pip install -r requirements.txt -U -f https://download.pytorch.org/whl/torch_stable.html -q
-        HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install -r ./tests/requirements-devel.txt -q
+        pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html -q
+        HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install -r ./requirements/devel.txt -q
         # pip install tox coverage
         python --version
         pip --version
@@ -104,7 +104,7 @@ jobs:
         HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')")
         if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
           pip uninstall -y horovod
-          HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install --no-cache-dir $(grep "horovod" requirements-extra.txt)
+          HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install --no-cache-dir $(grep "horovod" requirements/extra.txt)
         fi
         horovodrun --check-build
       shell: bash

.github/workflows/docs-check.yml

@@ -12,6 +12,7 @@ jobs:
     - uses: ammaraskar/sphinx-action@master
       with:
         # git is required to clone the docs theme
-        pre-build-command: "apt-get update -y && apt-get install -y git"
+        # before custom requirement are resolved https://github.com/ammaraskar/sphinx-action/issues/16
+        pre-build-command: "apt-get update -y && apt-get install -y git && pip install -r requirements/docs.txt"
         docs-folder: "docs/"
         repo-token: "${{ secrets.GITHUB_TOKEN }}"

.readthedocs.yml

@@ -24,5 +24,5 @@ formats:
 python:
   version: 3.7
   install:
-    - requirements: docs/requirements.txt
-    #- requirements: requirements.txt
+    - requirements: requirements/docs.txt
+    #- requirements: requirements/base.txt

MANIFEST.in

@@ -29,8 +29,7 @@ recursive-include docs/source/_images/logos/ *
 recursive-include docs/source/_images/general/ pl_overview* tf_* tutorial_*
 
 # Include the Requirements
-include requirements.txt
-include requirements-extra.txt
+recursive-include requirements *.txt
 
 # Exclude build configs
 exclude *.yml

docker/Dockerfile

@@ -34,7 +34,7 @@ ENV PATH /home/containeruser/conda/bin:$PATH
 RUN pip install torch==$PYTORCH_VERSION
 RUN git clone https://github.com/PyTorchLightning/pytorch-lightning.git --single-branch --branch $LIGHTNING_VERSION && \
     pip install ./pytorch-lightning && \
-    pip install -r pytorch-lightning/requirements-extra.txt && \
+    pip install -r pytorch-lightning/requirements/extra.txt && \
     rm -rf pytorch-lightning
 
 RUN python -c "import pytorch_lightning as pl; print(pl.__version__)"

docs/source/conf.py

@@ -324,8 +324,8 @@ def package_list_from_file(file):
 MOCK_PACKAGES = []
 if SPHINX_MOCK_REQUIREMENTS:
     # mock also base packages when we are on RTD since we don't install them there
-    MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements.txt'))
-    MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements-extra.txt'))
+    MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements/base.txt'))
+    MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements/extra.txt'))
 
 MOCK_MANUAL_PACKAGES = [
     'torchvision',

requirements/devel.txt

@@ -1,7 +1,7 @@
 # install all extra dependencies for full package testing
--r ../requirements-extra.txt
+-r ./extra.txt
 
 # extended list of dependencies dor development and run lint and tests
--r ./requirements.txt
+-r ./test.txt
 
 cloudpickle>=1.2

setup.py

@@ -20,7 +20,7 @@ import pytorch_lightning  # noqa: E402
 
 
 def load_requirements(path_dir=PATH_ROOT, comment_char='#'):
-    with open(os.path.join(path_dir, 'requirements.txt'), 'r') as file:
+    with open(os.path.join(path_dir, 'requirements', 'base.txt'), 'r') as file:
         lines = [ln.strip() for ln in file.readlines()]
     reqs = []
     for ln in lines:

tests/Dockerfile

@@ -14,9 +14,9 @@ ENV PATH: "$PATH:/root/.local/bin"
 ENV MAKEFLAGS: "-j$(nproc)"
 
 COPY ./tests/install_AMP.sh install_AMP.sh
-COPY ./requirements.txt requirements.txt
-COPY ./requirements-extra.txt requirements-extra.txt
-COPY ./tests/requirements.txt requirements-tests.txt
+COPY ./requirements/base.txt requirements.txt
+COPY ./requirements/extra.txt requirements-extra.txt
+COPY ./requirements/test.txt requirements-tests.txt
 
 # Install AMP
 RUN apt-get update && apt-get install -y cmake && \

tests/README.md

@@ -20,7 +20,7 @@ cd pytorch-lightning
 bash tests/install_AMP.sh
 
 # install dev deps
-pip install -r tests/requirements-devel.txt
+pip install -r requirements/devel.txt
 
 # run tests
 py.test -v
@@ -39,7 +39,7 @@ Make sure to run coverage on a GPU machine with at least 2 GPUs and NVIDIA apex
 ```bash
 cd pytorch-lightning
 
-# generate coverage (coverage is also installed as part of dev dependencies under tests/requirements-devel.txt)
+# generate coverage (coverage is also installed as part of dev dependencies under requirements/devel.txt)
 coverage run --source pytorch_lightning -m py.test pytorch_lightning tests examples -v
 
 # print coverage stats