CI: update prune_pkgs (#12382)
This commit is contained in:
parent
1eff3b53c1
commit
fe940e195d
|
@ -35,7 +35,15 @@ class AssistantCLI:
|
|||
fp.write(init)
|
||||
|
||||
@staticmethod
|
||||
def requirements_prune_pkgs(req_file: str, packages: Sequence[str]) -> None:
|
||||
def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES) -> None:
|
||||
"""Remove some packages from given requirement files."""
|
||||
if isinstance(req_files, str):
|
||||
req_files = [req_files]
|
||||
for req in req_files:
|
||||
AssistantCLI._prune_packages(req, packages)
|
||||
|
||||
@staticmethod
|
||||
def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
|
||||
"""Remove some packages from given requirement files."""
|
||||
with open(req_file) as fp:
|
||||
lines = fp.readlines()
|
||||
|
|
|
@ -28,5 +28,6 @@ jobs:
|
|||
|
||||
steps:
|
||||
- bash: |
|
||||
apt-get install -y hwinfo
|
||||
hwinfo --short
|
||||
displayName: 'Instance HW info'
|
||||
|
|
|
@ -57,7 +57,7 @@ jobs:
|
|||
- name: Setup Windows
|
||||
if: runner.os == 'windows'
|
||||
run: |
|
||||
python .actions/assistant.py requirements_prune_pkgs requirements/extra.txt --packages horovod
|
||||
python .actions/assistant.py requirements_prune_pkgs horovod
|
||||
- name: Set min. dependencies
|
||||
if: matrix.requires == 'oldest'
|
||||
run: |
|
||||
|
|
|
@ -43,7 +43,7 @@ jobs:
|
|||
pip --version
|
||||
pip install -q fire
|
||||
# remove Horovod from requirements
|
||||
python .actions/assistant.py requirements_prune_pkgs requirements/extra.txt --packages horovod
|
||||
python .actions/assistant.py requirements_prune_pkgs horovod
|
||||
# python -m pip install --upgrade --user pip
|
||||
pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet
|
||||
pip install --requirement requirements/extra.txt
|
||||
|
|
|
@ -96,7 +96,7 @@ RUN \
|
|||
python requirements_adjust_versions.py requirements-extra.txt && \
|
||||
python -c "print(' '.join([ln for ln in open('requirements-extra.txt').readlines() if 'horovod' in ln]))" > requirements_horovod.txt && \
|
||||
pip install -q fire && \
|
||||
python assistant.py requirements_prune_pkgs requirements-extra.txt "horovod" && \
|
||||
python assistant.py requirements_prune_pkgs "horovod" --req_files requirements-extra.txt && \
|
||||
python requirements_adjust_versions.py requirements-examples.txt && \
|
||||
# Install remaining requirements
|
||||
pip install -r requirements.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \
|
||||
|
|
|
@ -83,7 +83,7 @@ RUN \
|
|||
python ./requirements/adjust-versions.py requirements/extra.txt ${PYTORCH_VERSION} && \
|
||||
python ./requirements/adjust-versions.py requirements/examples.txt ${PYTORCH_VERSION} && \
|
||||
python -c "print(' '.join([ln for ln in open('requirements/extra.txt').readlines() if 'horovod' in ln]))" > ./requirements/horovod.txt && \
|
||||
python assistant.py requirements_prune_pkgs requirements/extra.txt "horovod" && \
|
||||
python assistant.py requirements_prune_pkgs "horovod" && \
|
||||
# Install all requirements \
|
||||
pip install -r requirements/devel.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \
|
||||
rm -rf requirements.* && \
|
||||
|
|
|
@ -83,7 +83,7 @@ RUN \
|
|||
python -c "import torch; print(torch.__version__)" && \
|
||||
python adjust_versions.py requirements-extra.txt && \
|
||||
pip install -q fire && \
|
||||
python assistant.py requirements_prune_pkgs requirements-extra.txt --packages fairscale,horovod && \
|
||||
python assistant.py requirements_prune_pkgs fairscale,horovod --req_files requirements-extra.txt && \
|
||||
# Install remaining requirements
|
||||
pip install -r requirements-extra.txt --no-cache-dir && \
|
||||
pip install -r requirements-test.txt --no-cache-dir && \
|
||||
|
|
|
@ -93,10 +93,9 @@ RUN \
|
|||
cd pytorch-lightning && \
|
||||
pip install -q fire && \
|
||||
# drop packages installed with XLA
|
||||
python .actions/assistant.py requirements_prune_pkgs requirements.txt --packages torch && \
|
||||
python .actions/assistant.py requirements_prune_pkgs requirements/examples.txt --packages torchvision && \
|
||||
python .actions/assistant.py requirements_prune_pkgs torch,torchvision && \
|
||||
# drop unnecessary packages
|
||||
python .actions/assistant.py requirements_prune_pkgs requirements/extra.txt --packages fairscale,horovod && \
|
||||
python .actions/assistant.py requirements_prune_pkgs fairscale,horovod && \
|
||||
python ./requirements/adjust-versions.py ./requirements/extra.txt && \
|
||||
# install PL dependencies
|
||||
pip install --requirement ./requirements/devel.txt --no-cache-dir && \
|
||||
|
|
|
@ -42,7 +42,7 @@ RUN \
|
|||
|
||||
# Installations \
|
||||
pip install -q fire && \
|
||||
python ./pytorch-lightning/.actions/assistant.py requirements_prune_pkgs ./pytorch-lightning/requirements/extra.txt --packages horovod && \
|
||||
python ./pytorch-lightning/.actions/assistant.py requirements_prune_pkgs horovod --req_files ./pytorch-lightning/requirements/extra.txt && \
|
||||
pip install "Pillow>=8.2, !=8.3.0" "cryptography>=3.4" "py>=1.10" --no-cache-dir --upgrade-strategy only-if-needed && \
|
||||
pip install -r ./pytorch-lightning/requirements/extra.txt --no-cache-dir --upgrade-strategy only-if-needed && \
|
||||
pip install -r ./pytorch-lightning/requirements/examples.txt --no-cache-dir --upgrade-strategy only-if-needed && \
|
||||
|
|
|
@ -30,7 +30,7 @@ RUN cd pytorch-lightning && \
|
|||
RUN \
|
||||
pip install -q fire && \
|
||||
# drop unnecessary packages
|
||||
python .actions/assistant.py requirements_prune_pkgs ./pytorch-lightning/requirements/extra.txt --packages fairscale,horovod && \
|
||||
python .actions/assistant.py requirements_prune_pkgs fairscale,horovod --req_files ./pytorch-lightning/requirements/extra.txt && \
|
||||
pip install -r pytorch-lightning/requirements/devel.txt --no-cache-dir
|
||||
|
||||
COPY ./dockers/tpu-tests/docker-entrypoint.sh /usr/local/bin/
|
||||
|
|
Loading…
Reference in New Issue