From d3567c33a6aab5ef6ae21637247e3476df0a966d Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sun, 18 Oct 2020 20:40:18 +0200 Subject: [PATCH] move base req. to root (#4219) * move base req. to root * check-manifest * check-manifest * manifest * req --- .github/CONTRIBUTING.md | 2 +- .github/workflows/ci_pkg-install.yml | 8 ++++++-- .github/workflows/ci_test-base.yml | 4 ++-- .github/workflows/ci_test-full.yml | 8 ++++---- .github/workflows/code-formatting.yml | 4 ++-- .github/workflows/docs-checks.yml | 8 ++++---- .readthedocs.yml | 2 +- MANIFEST.in | 4 +++- dockers/base-xla/Dockerfile | 2 +- dockers/tpu-tests/Dockerfile | 2 +- docs/source/conf.py | 2 +- requirements/base.txt => requirements.txt | 0 requirements/devel.txt | 2 +- setup.py | 12 ++++++------ 14 files changed, 33 insertions(+), 27 deletions(-) rename requirements/base.txt => requirements.txt (100%) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index effdee498e..ca521d9b57 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -60,7 +60,7 @@ Have a favorite feature from other libraries like fast.ai or transformers? Those We are always looking for help implementing new features or fixing bugs. -A lot of good work has already been done in project mechanics (requirements/base.txt, setup.py, pep8, badges, ci, etc...) so we're in a good state there thanks to all the early contributors (even pre-beta release)! +A lot of good work has already been done in project mechanics (requirements.txt, setup.py, pep8, badges, ci, etc...) so we're in a good state there thanks to all the early contributors (even pre-beta release)! ### Bug Fixes: diff --git a/.github/workflows/ci_pkg-install.yml b/.github/workflows/ci_pkg-install.yml index e945aaff02..3004e2456e 100644 --- a/.github/workflows/ci_pkg-install.yml +++ b/.github/workflows/ci_pkg-install.yml @@ -25,14 +25,18 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Prepare env + run: | + pip install check-manifest "twine==1.13.0" + - name: Create package run: | + check-manifest # python setup.py check --metadata --strict python setup.py sdist - name: Check package run: | - pip install twine==1.13.0 twine check dist/* python setup.py clean @@ -40,7 +44,7 @@ jobs: if: runner.os == 'windows' run: | # this is just a hotfix because of Win cannot install it directly - pip install -r requirements/base.txt --find-links https://download.pytorch.org/whl/torch_stable.html + pip install -r requirements.txt --find-links https://download.pytorch.org/whl/torch_stable.html - name: Install package run: | diff --git a/.github/workflows/ci_test-base.yml b/.github/workflows/ci_test-base.yml index b718596044..808abd9b45 100644 --- a/.github/workflows/ci_test-base.yml +++ b/.github/workflows/ci_test-base.yml @@ -44,14 +44,14 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements/base.txt') }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements.txt') }} restore-keys: | ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}- - name: Install dependencies run: | python -m pip install --upgrade --user pip - pip install --requirement ./requirements/base.txt --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade + pip install --requirement ./requirements.txt --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade pip install --requirement ./requirements/test.txt --quiet --upgrade-strategy only-if-needed # pip install tox coverage python --version diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 2ff6ebe444..79a26a0dce 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -55,12 +55,12 @@ jobs: - name: Setup MacOS Minimal if: runner.os == 'macOS' && matrix.requires == 'minimal' run : | - python -c "fname = 'requirements/base.txt' ; req = open(fname).read().replace('torch>=1.3', 'torch>=1.4') ; open(fname, 'w').write(req)" + python -c "fname = 'requirements.txt' ; req = open(fname).read().replace('torch>=1.3', 'torch>=1.4') ; open(fname, 'w').write(req)" - name: Set min. dependencies if: matrix.requires == 'minimal' run: | - python -c "fname = 'requirements/base.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)" + python -c "fname = 'requirements.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)" python -c "fname = 'requirements/extra.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)" python -c "fname = 'requirements/test.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)" python -c "fname = 'requirements/examples.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)" @@ -76,7 +76,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements/base.txt') }}-${{ hashFiles('requirements/extra.txt') }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements/extra.txt') }} restore-keys: | ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}- @@ -88,7 +88,7 @@ jobs: HOROVOD_WITHOUT_TENSORFLOW: 1 run: | # python -m pip install --upgrade --user pip - pip install --requirement requirements/base.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet --upgrade + pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet --upgrade pip install --requirement ./requirements/devel.txt --quiet --upgrade python --version pip --version diff --git a/.github/workflows/code-formatting.yml b/.github/workflows/code-formatting.yml index a36898be04..54bdbe23e0 100644 --- a/.github/workflows/code-formatting.yml +++ b/.github/workflows/code-formatting.yml @@ -42,7 +42,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-extras-${{ hashFiles('requirements/base.txt') }}-${{ hashFiles('requirements/extra.txt') }} + key: ${{ runner.os }}-pip-extras-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements/extra.txt') }} restore-keys: | ${{ runner.os }}-pip-extras- @@ -53,7 +53,7 @@ jobs: HOROVOD_WITHOUT_TENSORFLOW: 1 run: | # python -m pip install --upgrade --user pip - pip install --requirement requirements/base.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet + pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet pip install --requirement ./requirements/devel.txt --quiet # pip install tox coverage python --version ; pip --version ; pip list diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index ecd1f9a7eb..fa7580e0f9 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -35,7 +35,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- @@ -46,7 +46,7 @@ jobs: HOROVOD_WITHOUT_TENSORFLOW: 1 run: | # python -m pip install --upgrade --user pip - pip install --requirement requirements/base.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet + pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet pip install --requirement requirements/extra.txt pip install --requirement requirements/docs.txt python --version @@ -79,13 +79,13 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install --requirement requirements/base.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet + pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet pip install --requirement requirements/docs.txt # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux sudo apt-get update && sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures diff --git a/.readthedocs.yml b/.readthedocs.yml index d8c2b39d0e..32a5a16248 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -39,4 +39,4 @@ python: version: 3.7 install: - requirements: requirements/docs.txt - #- requirements: requirements/base.txt + #- requirements: requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index 4c5b022439..ac8ba7cc62 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -47,6 +47,8 @@ recursive-include docs/source/_images/general/ pl_overview* tf_* tutorial_* PTL1 # Include the Requirements recursive-include requirements *.txt +recursive-exclude requirements *.sh +include requirements.txt # Exclude build configs exclude *.yml @@ -62,5 +64,5 @@ prune notebook* prune temp* prune test* prune benchmark* -prune docker +prune dockers diff --git a/dockers/base-xla/Dockerfile b/dockers/base-xla/Dockerfile index 6297d6acc2..69666115a7 100644 --- a/dockers/base-xla/Dockerfile +++ b/dockers/base-xla/Dockerfile @@ -94,7 +94,7 @@ RUN \ # Install PL dependencies cd pytorch-lightning && \ # drop Torch as it was installed with XLA - python -c "fname = 'requirements/base.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('torch')] ; open(fname, 'w').writelines(lines)" && \ + python -c "fname = 'requirements.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('torch')] ; open(fname, 'w').writelines(lines)" && \ # drop Horovod as it is not needed python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" && \ # drop TorchVision as it was installed with XLA diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile index 79852b14a8..d0f7321d5f 100644 --- a/dockers/tpu-tests/Dockerfile +++ b/dockers/tpu-tests/Dockerfile @@ -24,7 +24,7 @@ COPY ./ ./pytorch-lightning/ # If using this image for tests, intall more dependencies and don"t delete the source code where the tests live. RUN \ # Install pytorch-lightning at the current PR, plus dependencies. - #pip install -r pytorch-lightning/requirements/base.txt --no-cache-dir && \ + #pip install -r pytorch-lightning/requirements.txt --no-cache-dir && \ # drop Horovod #python -c "fname = 'pytorch-lightning/requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" && \ pip install -r pytorch-lightning/requirements/devel.txt --no-cache-dir --upgrade-strategy only-if-needed diff --git a/docs/source/conf.py b/docs/source/conf.py index e6df4c4fcf..eb45eecc57 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -305,7 +305,7 @@ def package_list_from_file(file): MOCK_PACKAGES = [] if SPHINX_MOCK_REQUIREMENTS: # mock also base packages when we are on RTD since we don't install them there - MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements/base.txt')) + MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements.txt')) MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements/extra.txt')) MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements/loggers.txt')) diff --git a/requirements/base.txt b/requirements.txt similarity index 100% rename from requirements/base.txt rename to requirements.txt diff --git a/requirements/devel.txt b/requirements/devel.txt index 38f3fdf9d8..5d0262ec17 100644 --- a/requirements/devel.txt +++ b/requirements/devel.txt @@ -1,5 +1,5 @@ # install all mandatory dependencies --r ./base.txt +-r ../requirements.txt # install all extra dependencies for full package testing -r ./extra.txt diff --git a/setup.py b/setup.py index 0580772ca3..d25918417e 100755 --- a/setup.py +++ b/setup.py @@ -31,8 +31,8 @@ builtins.__LIGHTNING_SETUP__ = True import pytorch_lightning # noqa: E402 -def load_requirements(path_dir=PATH_ROOT, file_name='base.txt', comment_char='#'): - with open(os.path.join(path_dir, 'requirements', file_name), 'r') as file: +def load_requirements(path_dir=PATH_ROOT, file_name='requirements.txt', comment_char='#'): + with open(os.path.join(path_dir, file_name), 'r') as file: lines = [ln.strip() for ln in file.readlines()] reqs = [] for ln in lines: @@ -64,10 +64,10 @@ def load_long_description(): # From local copy of repo, use like `pip install ".[dev, docs]"` extras = { # 'docs': load_requirements(file_name='docs.txt'), - 'examples': load_requirements(file_name='examples.txt'), - 'loggers': load_requirements(file_name='loggers.txt'), - 'extra': load_requirements(file_name='extra.txt'), - 'test': load_requirements(file_name='test.txt') + 'examples': load_requirements(path_dir=os.path.join(PATH_ROOT, 'requirements'), file_name='examples.txt'), + 'loggers': load_requirements(path_dir=os.path.join(PATH_ROOT, 'requirements'), file_name='loggers.txt'), + 'extra': load_requirements(path_dir=os.path.join(PATH_ROOT, 'requirements'), file_name='extra.txt'), + 'test': load_requirements(path_dir=os.path.join(PATH_ROOT, 'requirements'), file_name='test.txt') } extras['dev'] = extras['extra'] + extras['loggers'] + extras['test'] extras['all'] = extras['dev'] + extras['examples'] # + extras['docs']