From bfaabd7b7fa3ddc7efbd50d06e22c0978b95cb39 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sat, 13 Jun 2020 16:15:22 +0200 Subject: [PATCH] clean requirements (#2128) * clean requirements * missing * missing * req * min * default >> base * base.txt --- .circleci/config.yml | 10 +++++----- .drone.yml | 6 +++--- .github/CONTRIBUTING.md | 9 ++++----- .github/workflows/ci-testing.yml | 20 +++++++++---------- .github/workflows/docs-check.yml | 3 ++- .readthedocs.yml | 4 ++-- MANIFEST.in | 3 +-- docker/Dockerfile | 2 +- docs/source/conf.py | 4 ++-- requirements.txt => requirements/base.txt | 0 .../devel.txt | 4 ++-- .../requirements.txt => requirements/docs.txt | 0 .../examples.txt | 0 .../extra.txt | 0 .../requirements.txt => requirements/test.txt | 0 setup.py | 2 +- tests/Dockerfile | 6 +++--- tests/README.md | 4 ++-- 18 files changed, 38 insertions(+), 39 deletions(-) rename requirements.txt => requirements/base.txt (100%) rename tests/requirements-devel.txt => requirements/devel.txt (66%) rename docs/requirements.txt => requirements/docs.txt (100%) rename pl_examples/requirements.txt => requirements/examples.txt (100%) rename requirements-extra.txt => requirements/extra.txt (100%) rename tests/requirements.txt => requirements/test.txt (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b7f2ad578..66217133e6 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,9 +12,9 @@ references: command: | sudo apt-get update && sudo apt-get install -y cmake pip install "$TORCH_VERSION" - pip install -r requirements.txt -q + pip install -r requirements/base.txt -q sudo pip install pytest -q - pip install -r ./tests/requirements-devel.txt -q + pip install -r requirements/devel.txt -q tests: &tests run: @@ -28,7 +28,7 @@ references: run: name: PL Examples command: | - pip install -r ./pl_examples/requirements.txt --user + pip install -r ./requirements/examples.txt --user python --version ; pip --version ; pip list py.test pl_examples -v --junitxml=test-reports/pytest_junit.xml no_output_timeout: 20m @@ -69,7 +69,7 @@ references: # we need to use py3.7 ot higher becase of an issue with metaclass inheritence pyenv global 3.7.3 python --version - pip install -r docs/requirements.txt + pip install -r requirements/docs.txt cd docs; make clean; make html --debug --jobs 2 SPHINXOPTS="-W" test_docs: &test_docs @@ -78,7 +78,7 @@ references: command: | # Second run examples in docs sudo apt-get update && sudo apt-get install -y cmake - sudo pip install -r docs/requirements.txt + sudo pip install -r requirements/docs.txt cd docs; make doctest; make coverage jobs: diff --git a/.drone.yml b/.drone.yml index e2d051ee53..2ae15a0d33 100644 --- a/.drone.yml +++ b/.drone.yml @@ -33,9 +33,9 @@ steps: - nvidia-smi #- bash ./tests/install_AMP.sh - apt-get update && apt-get install -y cmake - - pip install -r requirements.txt --user -q - - pip install -r ./tests/requirements-devel.txt --user -q - #- pip install -r ./docs/requirements.txt --user -q + - pip install -r ./requirements/base.txt --user -q + - pip install -r ./requirements/devel.txt --user -q + #- pip install -r ./requirements/docs.txt --user -q - pip list - python -c "import torch ; print(' & '.join([torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]) if torch.cuda.is_available() else 'only CPU')" - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests benchmarks -v --durations=25 # --flake8 diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index ebdcb998ca..e059ccc4c1 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -60,7 +60,7 @@ Have a favorite feature from other libraries like fast.ai or transformers? Those We are currently looking for help implementing new features or adding bug fixes. -A lot of good work has already been done in project mechanics (requirements.txt, setup.py, pep8, badges, ci, etc...) so we're in a good state there thanks to all the early contributors (even pre-beta release)! +A lot of good work has already been done in project mechanics (requirements/base.txt, setup.py, pep8, badges, ci, etc...) so we're in a good state there thanks to all the early contributors (even pre-beta release)! ### Bug Fixes: @@ -137,8 +137,8 @@ formatting errors. In certain cases, a missing blank line or a wrong indent can Run these commands ```bash +pip install -r requirements/docs.txt cd docs -pip install -r requirements.txt make html ``` @@ -153,9 +153,8 @@ Testing your work locally will help you speed up the process since it allows you To setup a local development environment, install both local and test dependencies: ```bash -python -m pip install -r requirements.txt -python -m pip install -r tests/requirements-devel.txt -python -m pip install -r pl_examples/requirements.txt +python -m pip install -r requirements/devel.txt +python -m pip install -r requirements/examples.txt python -m pip pre-commit install ``` diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index d905df63de..5929209f26 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -51,26 +51,26 @@ jobs: - name: Setup Windows if: runner.os == 'windows' run: | - python -c "lines = [line for line in open('requirements-extra.txt').readlines() if not line.startswith('horovod')] ; open('requirements-extra.txt', 'w').writelines(lines)" + python -c "lines = [line for line in open('requirements/extra.txt').readlines() if not line.startswith('horovod')] ; open('requirements/extra.txt', 'w').writelines(lines)" # TODO: remove after https://github.com/pytorch/pytorch/issues/32186 is resolved - name: Setup Windows on Latest if: runner.os == 'windows' && matrix.requires == 'latest' run: | - python -c "req = open('requirements.txt').read().replace('torch>=1.3', 'torch<1.5') ; open('requirements.txt', 'w').write(req)" + python -c "req = open('requirements/base.txt').read().replace('torch>=1.3', 'torch<1.5') ; open('requirements/base.txt', 'w').write(req)" # versions <= 1.3 may have issues on mac with some BLAS ops due to missing mkl (https://github.com/pytorch/pytorch/issues/18996) - name: Setup MacOS Minimal if: runner.os == 'macOS' && matrix.requires == 'minimal' run : | - python -c "req = open('requirements.txt').read().replace('torch>=1.3', 'torch>=1.4') ; open('requirements.txt', 'w').write(req)" + python -c "req = open('requirements/base.txt').read().replace('torch>=1.3', 'torch>=1.4') ; open('requirements/base.txt', 'w').write(req)" - name: Set min. dependencies if: matrix.requires == 'minimal' run: | - python -c "req = open('requirements.txt').read().replace('>=', '==') ; open('requirements.txt', 'w').write(req)" - python -c "req = open('requirements-extra.txt').read().replace('>=', '==') ; open('requirements-extra.txt', 'w').write(req)" - python -c "req = open('tests/requirements-devel.txt').read().replace('>=', '==') ; open('tests/requirements-devel.txt', 'w').write(req)" + python -c "req = open('requirements/base.txt').read().replace('>=', '==') ; open('requirements/base.txt', 'w').write(req)" + python -c "req = open('requirements/extra.txt').read().replace('>=', '==') ; open('requirements/extra.txt', 'w').write(req)" + python -c "req = open('requirements/devel.txt').read().replace('>=', '==') ; open('requirements/devel.txt', 'w').write(req)" # Note: This uses an internal pip API and may not always work # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow @@ -83,15 +83,15 @@ jobs: uses: actions/cache@v1 with: path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-extra.txt') }} + key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip-${{ hashFiles('requirements/base.txt') }}-${{ hashFiles('requirements/extra.txt') }} restore-keys: | ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-pip- - name: Install dependencies run: | # python -m pip install --upgrade --user pip - pip install -r requirements.txt -U -f https://download.pytorch.org/whl/torch_stable.html -q - HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install -r ./tests/requirements-devel.txt -q + pip install -r requirements/base.txt -U -f https://download.pytorch.org/whl/torch_stable.html -q + HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install -r ./requirements/devel.txt -q # pip install tox coverage python --version pip --version @@ -104,7 +104,7 @@ jobs: HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')") if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then pip uninstall -y horovod - HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install --no-cache-dir $(grep "horovod" requirements-extra.txt) + HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install --no-cache-dir $(grep "horovod" requirements/extra.txt) fi horovodrun --check-build shell: bash diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index bc8717a2f0..dca048272b 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -12,6 +12,7 @@ jobs: - uses: ammaraskar/sphinx-action@master with: # git is required to clone the docs theme - pre-build-command: "apt-get update -y && apt-get install -y git" + # before custom requirement are resolved https://github.com/ammaraskar/sphinx-action/issues/16 + pre-build-command: "apt-get update -y && apt-get install -y git && pip install -r requirements/docs.txt" docs-folder: "docs/" repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.readthedocs.yml b/.readthedocs.yml index 93e2feca15..600b27ab72 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -24,5 +24,5 @@ formats: python: version: 3.7 install: - - requirements: docs/requirements.txt - #- requirements: requirements.txt + - requirements: requirements/docs.txt + #- requirements: requirements/base.txt diff --git a/MANIFEST.in b/MANIFEST.in index b7190cf03f..fc2610caea 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -29,8 +29,7 @@ recursive-include docs/source/_images/logos/ * recursive-include docs/source/_images/general/ pl_overview* tf_* tutorial_* # Include the Requirements -include requirements.txt -include requirements-extra.txt +recursive-include requirements *.txt # Exclude build configs exclude *.yml diff --git a/docker/Dockerfile b/docker/Dockerfile index f6720e1f8e..2ef0bb3a9e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,7 +34,7 @@ ENV PATH /home/containeruser/conda/bin:$PATH RUN pip install torch==$PYTORCH_VERSION RUN git clone https://github.com/PyTorchLightning/pytorch-lightning.git --single-branch --branch $LIGHTNING_VERSION && \ pip install ./pytorch-lightning && \ - pip install -r pytorch-lightning/requirements-extra.txt && \ + pip install -r pytorch-lightning/requirements/extra.txt && \ rm -rf pytorch-lightning RUN python -c "import pytorch_lightning as pl; print(pl.__version__)" diff --git a/docs/source/conf.py b/docs/source/conf.py index 301ce61595..20ac3f1e81 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -324,8 +324,8 @@ def package_list_from_file(file): MOCK_PACKAGES = [] if SPHINX_MOCK_REQUIREMENTS: # mock also base packages when we are on RTD since we don't install them there - MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements.txt')) - MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements-extra.txt')) + MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements/base.txt')) + MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements/extra.txt')) MOCK_MANUAL_PACKAGES = [ 'torchvision', diff --git a/requirements.txt b/requirements/base.txt similarity index 100% rename from requirements.txt rename to requirements/base.txt diff --git a/tests/requirements-devel.txt b/requirements/devel.txt similarity index 66% rename from tests/requirements-devel.txt rename to requirements/devel.txt index 4261c72dbb..a66f197446 100644 --- a/tests/requirements-devel.txt +++ b/requirements/devel.txt @@ -1,7 +1,7 @@ # install all extra dependencies for full package testing --r ../requirements-extra.txt +-r ./extra.txt # extended list of dependencies dor development and run lint and tests --r ./requirements.txt +-r ./test.txt cloudpickle>=1.2 \ No newline at end of file diff --git a/docs/requirements.txt b/requirements/docs.txt similarity index 100% rename from docs/requirements.txt rename to requirements/docs.txt diff --git a/pl_examples/requirements.txt b/requirements/examples.txt similarity index 100% rename from pl_examples/requirements.txt rename to requirements/examples.txt diff --git a/requirements-extra.txt b/requirements/extra.txt similarity index 100% rename from requirements-extra.txt rename to requirements/extra.txt diff --git a/tests/requirements.txt b/requirements/test.txt similarity index 100% rename from tests/requirements.txt rename to requirements/test.txt diff --git a/setup.py b/setup.py index 0cd3605b15..0246b3cd5d 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ import pytorch_lightning # noqa: E402 def load_requirements(path_dir=PATH_ROOT, comment_char='#'): - with open(os.path.join(path_dir, 'requirements.txt'), 'r') as file: + with open(os.path.join(path_dir, 'requirements', 'base.txt'), 'r') as file: lines = [ln.strip() for ln in file.readlines()] reqs = [] for ln in lines: diff --git a/tests/Dockerfile b/tests/Dockerfile index 876d6fbf54..65c75c1ba3 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -14,9 +14,9 @@ ENV PATH: "$PATH:/root/.local/bin" ENV MAKEFLAGS: "-j$(nproc)" COPY ./tests/install_AMP.sh install_AMP.sh -COPY ./requirements.txt requirements.txt -COPY ./requirements-extra.txt requirements-extra.txt -COPY ./tests/requirements.txt requirements-tests.txt +COPY ./requirements/base.txt requirements.txt +COPY ./requirements/extra.txt requirements-extra.txt +COPY ./requirements/test.txt requirements-tests.txt # Install AMP RUN apt-get update && apt-get install -y cmake && \ diff --git a/tests/README.md b/tests/README.md index 4f35a29603..4857da1724 100644 --- a/tests/README.md +++ b/tests/README.md @@ -20,7 +20,7 @@ cd pytorch-lightning bash tests/install_AMP.sh # install dev deps -pip install -r tests/requirements-devel.txt +pip install -r requirements/devel.txt # run tests py.test -v @@ -39,7 +39,7 @@ Make sure to run coverage on a GPU machine with at least 2 GPUs and NVIDIA apex ```bash cd pytorch-lightning -# generate coverage (coverage is also installed as part of dev dependencies under tests/requirements-devel.txt) +# generate coverage (coverage is also installed as part of dev dependencies under requirements/devel.txt) coverage run --source pytorch_lightning -m py.test pytorch_lightning tests examples -v # print coverage stats