CI: update prune_pkgs (#12382)

This commit is contained in:
Jirka Borovec 2022-03-21 13:50:50 +01:00 committed by GitHub
parent 1eff3b53c1
commit fe940e195d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 19 additions and 11 deletions

View File

@ -35,7 +35,15 @@ class AssistantCLI:
fp.write(init)
@staticmethod
def requirements_prune_pkgs(req_file: str, packages: Sequence[str]) -> None:
def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES) -> None:
"""Remove some packages from given requirement files."""
if isinstance(req_files, str):
req_files = [req_files]
for req in req_files:
AssistantCLI._prune_packages(req, packages)
@staticmethod
def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
"""Remove some packages from given requirement files."""
with open(req_file) as fp:
lines = fp.readlines()

View File

@ -28,5 +28,6 @@ jobs:
steps:
- bash: |
apt-get install -y hwinfo
hwinfo --short
displayName: 'Instance HW info'

View File

@ -57,7 +57,7 @@ jobs:
- name: Setup Windows
if: runner.os == 'windows'
run: |
python .actions/assistant.py requirements_prune_pkgs requirements/extra.txt --packages horovod
python .actions/assistant.py requirements_prune_pkgs horovod
- name: Set min. dependencies
if: matrix.requires == 'oldest'
run: |

View File

@ -43,7 +43,7 @@ jobs:
pip --version
pip install -q fire
# remove Horovod from requirements
python .actions/assistant.py requirements_prune_pkgs requirements/extra.txt --packages horovod
python .actions/assistant.py requirements_prune_pkgs horovod
# python -m pip install --upgrade --user pip
pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet
pip install --requirement requirements/extra.txt

View File

@ -96,7 +96,7 @@ RUN \
python requirements_adjust_versions.py requirements-extra.txt && \
python -c "print(' '.join([ln for ln in open('requirements-extra.txt').readlines() if 'horovod' in ln]))" > requirements_horovod.txt && \
pip install -q fire && \
python assistant.py requirements_prune_pkgs requirements-extra.txt "horovod" && \
python assistant.py requirements_prune_pkgs "horovod" --req_files requirements-extra.txt && \
python requirements_adjust_versions.py requirements-examples.txt && \
# Install remaining requirements
pip install -r requirements.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \

View File

@ -83,7 +83,7 @@ RUN \
python ./requirements/adjust-versions.py requirements/extra.txt ${PYTORCH_VERSION} && \
python ./requirements/adjust-versions.py requirements/examples.txt ${PYTORCH_VERSION} && \
python -c "print(' '.join([ln for ln in open('requirements/extra.txt').readlines() if 'horovod' in ln]))" > ./requirements/horovod.txt && \
python assistant.py requirements_prune_pkgs requirements/extra.txt "horovod" && \
python assistant.py requirements_prune_pkgs "horovod" && \
# Install all requirements \
pip install -r requirements/devel.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \
rm -rf requirements.* && \

View File

@ -83,7 +83,7 @@ RUN \
python -c "import torch; print(torch.__version__)" && \
python adjust_versions.py requirements-extra.txt && \
pip install -q fire && \
python assistant.py requirements_prune_pkgs requirements-extra.txt --packages fairscale,horovod && \
python assistant.py requirements_prune_pkgs fairscale,horovod --req_files requirements-extra.txt && \
# Install remaining requirements
pip install -r requirements-extra.txt --no-cache-dir && \
pip install -r requirements-test.txt --no-cache-dir && \

View File

@ -93,10 +93,9 @@ RUN \
cd pytorch-lightning && \
pip install -q fire && \
# drop packages installed with XLA
python .actions/assistant.py requirements_prune_pkgs requirements.txt --packages torch && \
python .actions/assistant.py requirements_prune_pkgs requirements/examples.txt --packages torchvision && \
python .actions/assistant.py requirements_prune_pkgs torch,torchvision && \
# drop unnecessary packages
python .actions/assistant.py requirements_prune_pkgs requirements/extra.txt --packages fairscale,horovod && \
python .actions/assistant.py requirements_prune_pkgs fairscale,horovod && \
python ./requirements/adjust-versions.py ./requirements/extra.txt && \
# install PL dependencies
pip install --requirement ./requirements/devel.txt --no-cache-dir && \

View File

@ -42,7 +42,7 @@ RUN \
# Installations \
pip install -q fire && \
python ./pytorch-lightning/.actions/assistant.py requirements_prune_pkgs ./pytorch-lightning/requirements/extra.txt --packages horovod && \
python ./pytorch-lightning/.actions/assistant.py requirements_prune_pkgs horovod --req_files ./pytorch-lightning/requirements/extra.txt && \
pip install "Pillow>=8.2, !=8.3.0" "cryptography>=3.4" "py>=1.10" --no-cache-dir --upgrade-strategy only-if-needed && \
pip install -r ./pytorch-lightning/requirements/extra.txt --no-cache-dir --upgrade-strategy only-if-needed && \
pip install -r ./pytorch-lightning/requirements/examples.txt --no-cache-dir --upgrade-strategy only-if-needed && \

View File

@ -30,7 +30,7 @@ RUN cd pytorch-lightning && \
RUN \
pip install -q fire && \
# drop unnecessary packages
python .actions/assistant.py requirements_prune_pkgs ./pytorch-lightning/requirements/extra.txt --packages fairscale,horovod && \
python .actions/assistant.py requirements_prune_pkgs fairscale,horovod --req_files ./pytorch-lightning/requirements/extra.txt && \
pip install -r pytorch-lightning/requirements/devel.txt --no-cache-dir
COPY ./dockers/tpu-tests/docker-entrypoint.sh /usr/local/bin/