From fe940e195dceb18eb9f3bd512cea56ae3405d464 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 21 Mar 2022 13:50:50 +0100 Subject: [PATCH] CI: update prune_pkgs (#12382) --- .actions/assistant.py | 10 +++++++++- .azure-pipelines/hpu-tests.yml | 1 + .github/workflows/ci_test-full.yml | 2 +- .github/workflows/docs-checks.yml | 2 +- dockers/base-conda/Dockerfile | 2 +- dockers/base-cuda/Dockerfile | 2 +- dockers/base-ipu/Dockerfile | 2 +- dockers/base-xla/Dockerfile | 5 ++--- dockers/nvidia/Dockerfile | 2 +- dockers/tpu-tests/Dockerfile | 2 +- 10 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 3ac0209bd6..4e68b548e7 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -35,7 +35,15 @@ class AssistantCLI: fp.write(init) @staticmethod - def requirements_prune_pkgs(req_file: str, packages: Sequence[str]) -> None: + def requirements_prune_pkgs(packages: Sequence[str], req_files: Sequence[str] = REQUIREMENT_FILES) -> None: + """Remove some packages from given requirement files.""" + if isinstance(req_files, str): + req_files = [req_files] + for req in req_files: + AssistantCLI._prune_packages(req, packages) + + @staticmethod + def _prune_packages(req_file: str, packages: Sequence[str]) -> None: """Remove some packages from given requirement files.""" with open(req_file) as fp: lines = fp.readlines() diff --git a/.azure-pipelines/hpu-tests.yml b/.azure-pipelines/hpu-tests.yml index aaec3caabd..1be1677045 100644 --- a/.azure-pipelines/hpu-tests.yml +++ b/.azure-pipelines/hpu-tests.yml @@ -28,5 +28,6 @@ jobs: steps: - bash: | + apt-get install -y hwinfo hwinfo --short displayName: 'Instance HW info' diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 48dabe4980..f0710a5342 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -57,7 +57,7 @@ jobs: - name: Setup Windows if: runner.os == 'windows' run: | - python .actions/assistant.py requirements_prune_pkgs requirements/extra.txt --packages horovod + python .actions/assistant.py requirements_prune_pkgs horovod - name: Set min. dependencies if: matrix.requires == 'oldest' run: | diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 90e0341a52..bb5482f469 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -43,7 +43,7 @@ jobs: pip --version pip install -q fire # remove Horovod from requirements - python .actions/assistant.py requirements_prune_pkgs requirements/extra.txt --packages horovod + python .actions/assistant.py requirements_prune_pkgs horovod # python -m pip install --upgrade --user pip pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet pip install --requirement requirements/extra.txt diff --git a/dockers/base-conda/Dockerfile b/dockers/base-conda/Dockerfile index 280ab0687f..5499c613d5 100644 --- a/dockers/base-conda/Dockerfile +++ b/dockers/base-conda/Dockerfile @@ -96,7 +96,7 @@ RUN \ python requirements_adjust_versions.py requirements-extra.txt && \ python -c "print(' '.join([ln for ln in open('requirements-extra.txt').readlines() if 'horovod' in ln]))" > requirements_horovod.txt && \ pip install -q fire && \ - python assistant.py requirements_prune_pkgs requirements-extra.txt "horovod" && \ + python assistant.py requirements_prune_pkgs "horovod" --req_files requirements-extra.txt && \ python requirements_adjust_versions.py requirements-examples.txt && \ # Install remaining requirements pip install -r requirements.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \ diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile index e5f3c8d132..9c5b0f681c 100644 --- a/dockers/base-cuda/Dockerfile +++ b/dockers/base-cuda/Dockerfile @@ -83,7 +83,7 @@ RUN \ python ./requirements/adjust-versions.py requirements/extra.txt ${PYTORCH_VERSION} && \ python ./requirements/adjust-versions.py requirements/examples.txt ${PYTORCH_VERSION} && \ python -c "print(' '.join([ln for ln in open('requirements/extra.txt').readlines() if 'horovod' in ln]))" > ./requirements/horovod.txt && \ - python assistant.py requirements_prune_pkgs requirements/extra.txt "horovod" && \ + python assistant.py requirements_prune_pkgs "horovod" && \ # Install all requirements \ pip install -r requirements/devel.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \ rm -rf requirements.* && \ diff --git a/dockers/base-ipu/Dockerfile b/dockers/base-ipu/Dockerfile index fff685eb6e..727bb29203 100644 --- a/dockers/base-ipu/Dockerfile +++ b/dockers/base-ipu/Dockerfile @@ -83,7 +83,7 @@ RUN \ python -c "import torch; print(torch.__version__)" && \ python adjust_versions.py requirements-extra.txt && \ pip install -q fire && \ - python assistant.py requirements_prune_pkgs requirements-extra.txt --packages fairscale,horovod && \ + python assistant.py requirements_prune_pkgs fairscale,horovod --req_files requirements-extra.txt && \ # Install remaining requirements pip install -r requirements-extra.txt --no-cache-dir && \ pip install -r requirements-test.txt --no-cache-dir && \ diff --git a/dockers/base-xla/Dockerfile b/dockers/base-xla/Dockerfile index 05be54179b..57a2bd3e95 100644 --- a/dockers/base-xla/Dockerfile +++ b/dockers/base-xla/Dockerfile @@ -93,10 +93,9 @@ RUN \ cd pytorch-lightning && \ pip install -q fire && \ # drop packages installed with XLA - python .actions/assistant.py requirements_prune_pkgs requirements.txt --packages torch && \ - python .actions/assistant.py requirements_prune_pkgs requirements/examples.txt --packages torchvision && \ + python .actions/assistant.py requirements_prune_pkgs torch,torchvision && \ # drop unnecessary packages - python .actions/assistant.py requirements_prune_pkgs requirements/extra.txt --packages fairscale,horovod && \ + python .actions/assistant.py requirements_prune_pkgs fairscale,horovod && \ python ./requirements/adjust-versions.py ./requirements/extra.txt && \ # install PL dependencies pip install --requirement ./requirements/devel.txt --no-cache-dir && \ diff --git a/dockers/nvidia/Dockerfile b/dockers/nvidia/Dockerfile index 2984c5b93e..792835c574 100644 --- a/dockers/nvidia/Dockerfile +++ b/dockers/nvidia/Dockerfile @@ -42,7 +42,7 @@ RUN \ # Installations \ pip install -q fire && \ - python ./pytorch-lightning/.actions/assistant.py requirements_prune_pkgs ./pytorch-lightning/requirements/extra.txt --packages horovod && \ + python ./pytorch-lightning/.actions/assistant.py requirements_prune_pkgs horovod --req_files ./pytorch-lightning/requirements/extra.txt && \ pip install "Pillow>=8.2, !=8.3.0" "cryptography>=3.4" "py>=1.10" --no-cache-dir --upgrade-strategy only-if-needed && \ pip install -r ./pytorch-lightning/requirements/extra.txt --no-cache-dir --upgrade-strategy only-if-needed && \ pip install -r ./pytorch-lightning/requirements/examples.txt --no-cache-dir --upgrade-strategy only-if-needed && \ diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile index 4016b30e9b..cd0c3c779d 100644 --- a/dockers/tpu-tests/Dockerfile +++ b/dockers/tpu-tests/Dockerfile @@ -30,7 +30,7 @@ RUN cd pytorch-lightning && \ RUN \ pip install -q fire && \ # drop unnecessary packages - python .actions/assistant.py requirements_prune_pkgs ./pytorch-lightning/requirements/extra.txt --packages fairscale,horovod && \ + python .actions/assistant.py requirements_prune_pkgs fairscale,horovod --req_files ./pytorch-lightning/requirements/extra.txt && \ pip install -r pytorch-lightning/requirements/devel.txt --no-cache-dir COPY ./dockers/tpu-tests/docker-entrypoint.sh /usr/local/bin/