add freeze for development and full range for install (#12994)

* freeze versions

* unfreeze

* dependabot

* Apply suggestions from code review

Co-authored-by: Carlos Mocholí <carlossmocholi@gmail.com>

* fix all req

* ...

* use base

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix refs

* Apply suggestions from code review

Co-authored-by: Akihiro Nitta <nitta@akihironitta.com>

* Apply suggestions from code review

* dockers

Co-authored-by: Carlos Mocholí <carlossmocholi@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Akihiro Nitta <nitta@akihironitta.com>

.actions/assistant.py

@@ -7,7 +7,7 @@ from typing import Sequence
 import fire
 
 REQUIREMENT_FILES = (
-    "requirements.txt",
+    "requirements/base.txt",
     "requirements/extra.txt",
     "requirements/loggers.txt",
     "requirements/strategies.txt",

.github/dependabot.yml

@@ -0,0 +1,24 @@
+# Basic dependabot.yml file with
+# minimum configuration for two package managers
+
+version: 2
+updates:
+  # Enable version updates for python
+  - package-ecosystem: "pip"
+    # Look for a `requirements` in the `root` directory
+    directory: "/requirements"
+    # Check for updates once a week
+    schedule:
+      interval: "weekly"
+    # Labels on pull requests for version updates only
+    labels:
+      - "ci"
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      # for example, `dependabot-npm_and_yarn-next_js-acorn-6.4.1`
+      separator: "-"
+    # Allow up to 10 open pull requests for pip dependencies
+    open-pull-requests-limit: 10
+    reviewers:
+      - "carmocca"
+      - "PyTorchLightning/teams/core-lightning"

.github/workflows/ci_dockers.yml

@@ -11,8 +11,8 @@ on: # Trigger the workflow on push or pull request, but only for the master bran
       - "dockers/**"
       - "!dockers/README.md"
       - "requirements/*"
-      - "environment.yml"
       - "requirements.txt"
+      - "environment.yml"
       - ".github/workflows/*docker*.yml"
       - ".github/workflows/events-nightly.yml"
       - "setup.py"

.github/workflows/ci_test-base.yml

@@ -45,7 +45,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements.txt') }}
+        key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements/base.txt') }}
         restore-keys: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.requires }}-
 
     - name: Install dependencies
@@ -53,8 +53,8 @@ jobs:
         python --version
         python -m pip install --upgrade --user pip
         pip --version
-        pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
-        pip install --requirement requirements/test.txt
+        pip install -r requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
+        pip install -r requirements/test.txt
         pip list
       shell: bash
 

.github/workflows/ci_test-full.yml

@@ -72,7 +72,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements/extra.txt') }}
+        key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-${{ hashFiles('requirements/*.txt') }}
         restore-keys: |
           ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-
 
@@ -89,11 +89,11 @@ jobs:
       run: |
         flag=$(python -c "print('--pre' if '${{matrix.release}}' == 'pre' else '')" 2>&1)
         url=$(python -c "print('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html')" 2>&1)
-        pip install --requirement requirements.txt --upgrade $flag --find-links "https://download.pytorch.org/whl/${url}"
+        pip install -r requirements.txt --upgrade $flag --find-links "https://download.pytorch.org/whl/${url}"
         # adjust versions according installed Torch version
         python ./requirements/adjust-versions.py requirements/examples.txt
-        pip install --requirement requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
-        pip install --requirement requirements/test.txt --upgrade
+        pip install -r requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
+        pip install -r requirements/test.txt --upgrade
         pip list
       shell: bash
 

.github/workflows/ci_test-slow.yml

@@ -41,7 +41,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
-        key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+        key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ hashFiles('requirements/base.txt') }}
         restore-keys: |
           ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-
 
@@ -49,8 +49,8 @@ jobs:
       run: |
         # adjust versions according installed Torch version
         python ./requirements/adjust-versions.py requirements.txt ${{ matrix.pytorch-version }}
-        pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
-        pip install --requirement requirements/test.txt
+        pip install -r requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
+        pip install -r requirements/test.txt
         pip list
       shell: bash
 

.github/workflows/code-checks.yml

@@ -25,7 +25,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.txt') }}
         restore-keys: |
           ${{ runner.os }}-pip-
 

.github/workflows/docs-checks.yml

@@ -31,7 +31,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements/base.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-
 
@@ -42,9 +42,9 @@ jobs:
           pip --version
           pip install -q fire
           # python -m pip install --upgrade --user pip
-          pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet
-          pip install --requirement requirements/devel.txt
-          pip install --requirement requirements/docs.txt
+          pip install -r requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet
+          pip install -r requirements/devel.txt
+          pip install -r requirements/docs.txt
           pip list
         shell: bash
 
@@ -74,7 +74,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
 

.github/workflows/release-pypi.yml

@@ -102,7 +102,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements.txt') }}
+        key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements/base.txt') }}
         restore-keys: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-
 
     - name: Install dependencies

dockers/base-conda/Dockerfile

@@ -88,25 +88,19 @@ ENV \
     PATH=/root/miniconda3/envs/${CONDA_ENV}/bin:$PATH \
     LD_LIBRARY_PATH="/root/miniconda3/envs/${CONDA_ENV}/lib:$LD_LIBRARY_PATH"
 
-COPY ./requirements.txt requirements.txt
-COPY ./requirements/extra.txt requirements-extra.txt
-COPY ./requirements/examples.txt requirements-examples.txt
-COPY ./requirements/strategies.txt requirements-strategies.txt
-COPY ./requirements/adjust-versions.py requirements_adjust_versions.py
+COPY ./requirements/ ./requirements/
 COPY ./.actions/assistant.py assistant.py
 
 RUN \
     pip list | grep torch && \
     python -c "import torch; print(torch.__version__)" && \
-    python requirements_adjust_versions.py requirements-extra.txt && \
-    python -c "print(' '.join([ln for ln in open('requirements-extra.txt').readlines() if 'horovod' in ln]))" > requirements_horovod.txt && \
     pip install -q fire && \
-    python assistant.py requirements_prune_pkgs "horovod" --req_files requirements-extra.txt && \
-    python requirements_adjust_versions.py requirements-examples.txt && \
+    python requirements/adjust-versions.py requirements/extra.txt && \
+    python requirements/adjust-versions.py requirements/examples.txt && \
     # Install remaining requirements
-    pip install -r requirements.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \
-    pip install -r requirements-extra.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \
-    pip install -r requirements-examples.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \
+    pip install -r requirements/base.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \
+    pip install -r requirements/extra.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \
+    pip install -r requirements/examples.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \
     rm assistant.py
 
 ENV \
@@ -121,11 +115,9 @@ ENV \
     HOROVOD_WITH_MPI=1
 
 RUN \
-   HOROVOD_BUILD_CUDA_CC_LIST=${TORCH_CUDA_ARCH_LIST//";"/","} && \
-   export HOROVOD_BUILD_CUDA_CC_LIST=${HOROVOD_BUILD_CUDA_CC_LIST//"."/""} && \
-   cat requirements_horovod.txt && \
-   pip install --no-cache-dir -r requirements-strategies.txt && \
-   rm requirements*
+    HOROVOD_BUILD_CUDA_CC_LIST=${TORCH_CUDA_ARCH_LIST//";"/","} && \
+    export HOROVOD_BUILD_CUDA_CC_LIST=${HOROVOD_BUILD_CUDA_CC_LIST//"."/""} && \
+    pip install --no-cache-dir -r requirements/strategies.txt
 
 RUN \
     CUDA_VERSION_MAJOR=$(python -c "import torch ; print(torch.version.cuda.split('.')[0])") && \
@@ -149,9 +141,6 @@ RUN \
     python -c "import bagua_core; bagua_core.install_deps()" && \
     python -c "import bagua; print(bagua.__version__)"
 
-COPY requirements/check-avail-extras.py check-avail-extras.py
-COPY requirements/check-avail-strategies.py check-avail-strategies.py
-
 RUN \
     # Show what we have
     pip --version && \
@@ -159,6 +148,6 @@ RUN \
     pip list && \
     python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \
     python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__" && \
-    python check-avail-extras.py && \
-    python check-avail-strategies.py && \
-    rm check-avail-*.py
+    python requirements/check-avail-extras.py && \
+    python requirements/check-avail-strategies.py && \
+    rm -rf requirements/

dockers/base-cuda/Dockerfile

@@ -70,7 +70,6 @@ RUN \
     rm -rf /root/.cache && \
     rm -rf /var/lib/apt/lists/*
 
-COPY ./requirements.txt requirements.txt
 COPY ./requirements/ ./requirements/
 COPY ./.actions/assistant.py assistant.py
 
@@ -86,12 +85,11 @@ RUN \
     CUDA_VERSION_MM=$(python -c "print(''.join('$CUDA_VERSION'.split('.')[:2]))") && \
     pip config set global.cache-dir false && \
     # set particular PyTorch version
-    python ./requirements/adjust-versions.py requirements.txt ${PYTORCH_VERSION} && \
+    python ./requirements/adjust-versions.py requirements/base.txt ${PYTORCH_VERSION} && \
     python ./requirements/adjust-versions.py requirements/extra.txt ${PYTORCH_VERSION} && \
     python ./requirements/adjust-versions.py requirements/examples.txt ${PYTORCH_VERSION} && \
     # Install all requirements \
     pip install -r requirements/devel.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \
-    rm -rf requirements.* && \
     rm assistant.py
 
 RUN \
@@ -121,8 +119,7 @@ RUN \
     echo $HOROVOD_BUILD_CUDA_CC_LIST && \
     cmake --version && \
     pip install --no-cache-dir -r ./requirements/strategies.txt && \
-    horovodrun --check-build && \
-    rm -rf requirements/
+    horovodrun --check-build
 
 RUN \
     CUDA_VERSION_MAJOR=$(python -c "import torch; print(torch.version.cuda.split('.')[0])") && \
@@ -157,6 +154,6 @@ RUN \
     pip list && \
     python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \
     python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__" && \
-    python check-avail-extras.py && \
-    python check-avail-strategies.py && \
-    rm check-avail-*.py
+    python requirements/check-avail-extras.py && \
+    python requirements/check-avail-strategies.py && \
+    rm -rf requirements/

docs/source/conf.py

@@ -345,11 +345,12 @@ PACKAGE_MAPPING = {
 MOCK_PACKAGES = []
 if SPHINX_MOCK_REQUIREMENTS:
     MOCK_PACKAGES += ["fairscale"]
+    _path_require = lambda fname: os.path.join(PATH_ROOT, "requirements", fname)
     # mock also base packages when we are on RTD since we don't install them there
-    MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, "requirements.txt"))
-    MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, "requirements", "extra.txt"))
-    MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, "requirements", "strategies.txt"))
-    MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, "requirements", "loggers.txt"))
+    MOCK_PACKAGES += package_list_from_file(_path_require("base.txt"))
+    MOCK_PACKAGES += package_list_from_file(_path_require("extra.txt"))
+    MOCK_PACKAGES += package_list_from_file(_path_require("loggers.txt"))
+    MOCK_PACKAGES += package_list_from_file(_path_require("strategies.txt"))
 MOCK_PACKAGES = [PACKAGE_MAPPING.get(pkg, pkg) for pkg in MOCK_PACKAGES]
 
 autodoc_mock_imports = MOCK_PACKAGES

pytorch_lightning/setup_tools.py

@@ -19,10 +19,12 @@ from typing import List
 _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
 
 
-def _load_requirements(path_dir: str, file_name: str = "requirements.txt", comment_char: str = "#") -> List[str]:
+def _load_requirements(
+    path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: bool = True
+) -> List[str]:
     """Load requirements from a file.
 
-    >>> _load_requirements(_PROJECT_ROOT)  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    >>> _load_requirements(os.path.join(_PROJECT_ROOT, "requirements"))  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
     ['numpy...', 'torch...', ...]
     """
     with open(os.path.join(path_dir, file_name)) as file:
@@ -31,12 +33,16 @@ def _load_requirements(path_dir: str, file_name: str = "requirements.txt", comme
     for ln in lines:
         # filer all comments
         if comment_char in ln:
-            ln = ln[: ln.index(comment_char)].strip()
+            ln = ln[: ln.index(comment_char)]
+        comment = ln[ln.index(comment_char) :] if comment_char in ln else ""
+        req = ln.strip()
         # skip directly installed dependencies
-        if ln.startswith("http") or "@http" in ln:
+        if not req or req.startswith("http") or "@http" in req:
             continue
-        if ln:  # if requirement is not empty
-            reqs.append(ln)
+        # remove version restrictions unless they are strict
+        if unfreeze and "<=" in req and "strict" not in comment:
+            req = re.sub(r",? *<= *[\d\.\*]+", "", req).strip()
+        reqs.append(req)
     return reqs
 
 

requirements.txt

@@ -1,12 +1,2 @@
 # the default package dependencies
-
-numpy>=1.17.2
-torch>=1.8.*
-tqdm>=4.57.0
-PyYAML>=5.4
-fsspec[http]>=2021.05.0, !=2021.06.0
-tensorboard>=2.2.0
-torchmetrics>=0.4.1
-pyDeprecate>=0.3.1, <0.4.0
-packaging>=17.0
-typing-extensions>=4.0.0
+-r ./requirements/base.txt

requirements/base.txt

@@ -0,0 +1,10 @@
+numpy>=1.17.2, <=1.22.3
+torch>=1.8.*, <=1.11.0
+tqdm>=4.57.0, <=4.63.0
+PyYAML>=5.4, <=6.0
+fsspec[http]>=2021.05.0, !=2021.06.0, <=2022.2.0
+tensorboard>=2.2.0, <=2.8.0
+torchmetrics>=0.4.1, <=0.7.2
+pyDeprecate>=0.3.1, <=0.3.2
+packaging>=17.0, <=21.3
+typing-extensions>=4.0.0, <=4.1.1

requirements/devel.txt

@@ -1,5 +1,5 @@
 # install all mandatory dependencies
--r ../requirements.txt
+-r ./base.txt
 
 # install all extra dependencies for full package testing
 -r ./extra.txt

requirements/examples.txt

@@ -1,3 +1,3 @@
-torchvision>=0.9.*
-gym[classic_control]>=0.17.0
-ipython[all]
+torchvision>=0.9.*, <=0.12.0
+gym[classic_control]>=0.17.0, <=0.23.1
+ipython[all] <=8.1.1

requirements/extra.txt

@@ -1,9 +1,9 @@
 # extended list of package dependencies to reach full functionality
 
-matplotlib>3.1
-torchtext>=0.9.*
-omegaconf>=2.0.5
-hydra-core>=1.0.5
-jsonargparse[signatures]>=4.7.1
-gcsfs>=2021.5.0
-rich>=10.2.2,!=10.15.*
+matplotlib>3.1, <=3.5.1
+torchtext>=0.9.*, <=0.12.0
+omegaconf>=2.0.5, <=2.1.*
+hydra-core>=1.0.5, <=1.1.*
+jsonargparse[signatures]>=4.7.1, <=4.7.1
+gcsfs>=2021.5.0, <=2022.2.0
+rich>=10.2.2,!=10.15.*, <=12.0.0

requirements/loggers.txt

@@ -1,5 +1,7 @@
 # all supported loggers
-neptune-client>=0.10.0
-comet-ml>=3.1.12
-mlflow>=1.0.0
-wandb>=0.8.21
+
+neptune-client>=0.10.0, <=0.15.2
+comet-ml>=3.1.12, <=3.28.2
+mlflow>=1.0.0, <=1.24.0
+test_tube>=0.7.5, <=0.7.5
+wandb>=0.8.21, <=0.12.11

requirements/strategies.txt

@@ -1,4 +1,5 @@
-fairscale>=0.4.5
+fairscale>=0.4.5, <=0.4.6
 deepspeed<0.6.0
-horovod>=0.21.2,!=0.24.0  # no need to install with [pytorch] as pytorch is already installed
-hivemind>=1.0.1; sys_platform == 'linux'
+# no need to install with [pytorch] as pytorch is already installed
+horovod>=0.21.2,!=0.24.0, <=0.24.3
+hivemind>=1.0.1, <=1.0.1; sys_platform == 'linux'

setup.py

@@ -79,7 +79,7 @@ setup(
     keywords=["deep learning", "pytorch", "AI"],
     python_requires=">=3.7",
     setup_requires=[],
-    install_requires=setup_tools._load_requirements(_PATH_ROOT),
+    install_requires=setup_tools._load_requirements(_PATH_REQUIRE),
     extras_require=extras,
     project_urls={
         "Bug Tracker": "https://github.com/PyTorchLightning/pytorch-lightning/issues",