From 939a861853059d4a632843dbbf83e9ab802ad424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Thu, 4 Nov 2021 18:26:24 +0100 Subject: [PATCH] Update Python testing (#10269) --- .azure-pipelines/gpu-benchmark.yml | 4 ++-- .github/workflows/ci_dockers.yml | 4 ++-- .github/workflows/ci_test-conda.yml | 2 +- .github/workflows/ci_test-full.yml | 26 +++++++++++++++----------- .github/workflows/events-nightly.yml | 4 ++-- dockers/base-cuda/Dockerfile | 2 +- dockers/base-ipu/Dockerfile | 5 ++--- dockers/base-xla/Dockerfile | 6 +++--- dockers/ipu-ci-runner/Dockerfile | 2 +- dockers/release/Dockerfile | 2 +- dockers/tpu-tests/Dockerfile | 2 +- 11 files changed, 31 insertions(+), 28 deletions(-) diff --git a/.azure-pipelines/gpu-benchmark.yml b/.azure-pipelines/gpu-benchmark.yml index a63c9e8640..f8b9593d72 100644 --- a/.azure-pipelines/gpu-benchmark.yml +++ b/.azure-pipelines/gpu-benchmark.yml @@ -28,8 +28,8 @@ jobs: cancelTimeoutInMinutes: "2" pool: gridai-spot-pool container: - # base ML image: mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04 - image: "pytorchlightning/pytorch_lightning:base-cuda-py3.8-torch1.8" + # should match the one in '.azure-pipelines/gpu-benchmark.yml' + image: "pytorchlightning/pytorch_lightning:base-cuda-py3.7-torch1.8" options: "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all --shm-size=32g" workspace: clean: all diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml index 5b5a140e77..701223c795 100644 --- a/.github/workflows/ci_dockers.yml +++ b/.github/workflows/ci_dockers.yml @@ -92,7 +92,7 @@ jobs: fail-fast: false matrix: # the config used in '.github/workflows/ci_test-conda.yml' - python_version: ["3.7"] + python_version: ["3.8"] pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"] steps: - name: Checkout @@ -119,7 +119,7 @@ jobs: fail-fast: false matrix: # the config used in 'dockers/ipu-ci-runner/Dockerfile' - python_version: ["3.8"] # latest + python_version: ["3.9"] # latest # TODO: upgrade - PopTorch 2.2 uses torch 1.9, see: # https://docs.graphcore.ai/projects/poptorch-user-guide/en/latest/installation.html#version-compatibility pytorch_version: ["1.7"] diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index cdd18d13e9..edae03db79 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7"] + python-version: ["3.8"] # previous to last Python version as that one is already used in test-full pytorch-version: ["1.6", "1.7", "1.8", "1.9", "1.10"] # Timeout: https://stackoverflow.com/a/59076067/4521646 diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 5cc4827d84..8be8fd1146 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -18,17 +18,21 @@ jobs: fail-fast: false matrix: os: [ubuntu-18.04, windows-2019, macOS-10.15] - python-version: [3.6, 3.8, 3.9] - requires: ['minimal', 'latest'] - release: ['stable'] - exclude: - - python-version: 3.9 - requires: 'minimal' + python-version: ["3.7", "3.9"] # minimum, maximum + requires: ["oldest", "latest"] + release: ["stable"] include: - - os: ubuntu-20.04 - python-version: 3.9 - requires: 'latest' - release: 'pre' + # test 3.6 only on oldest until EOL: https://github.com/PyTorchLightning/pytorch-lightning/issues/9981 + - {os: ubuntu-18.04, python-version: "3.6", requires: "oldest", release: "stable"} + - {os: windows-2019, python-version: "3.6", requires: "oldest", release: "stable"} + - {os: macOS-10.15, python-version: "3.6", requires: "oldest", release: "stable"} + # nightly: add when there's a release candidate + #- {os: ubuntu-20.04, python-version: "3.10", requires: "latest", release: "pre"} + exclude: + # PyTorch 1.6 is not available with Python 3.9: https://github.com/pytorch/pytorch/issues/46205 + - {os: ubuntu-18.04, python-version: "3.9", requires: "oldest", release: "stable"} + - {os: windows-2019, python-version: "3.9", requires: "oldest", release: "stable"} + - {os: macOS-10.15, python-version: "3.9", requires: "oldest", release: "stable"} # Timeout: https://stackoverflow.com/a/59076067/4521646 # TODO: the macOS is taking too long, probably caching did not work... @@ -64,7 +68,7 @@ jobs: python .github/prune-packages.py requirements/extra.txt "horovod" - name: Set min. dependencies - if: matrix.requires == 'minimal' + if: matrix.requires == 'oldest' run: | python .github/set-min-requirements.py diff --git a/.github/workflows/events-nightly.yml b/.github/workflows/events-nightly.yml index 4afcec0496..ce2072e5f4 100644 --- a/.github/workflows/events-nightly.yml +++ b/.github/workflows/events-nightly.yml @@ -122,7 +122,7 @@ jobs: fail-fast: false matrix: # the config used in '.github/workflows/ci_test-conda.yml' - python_version: ["3.7"] + python_version: ["3.8"] pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"] steps: @@ -163,7 +163,7 @@ jobs: matrix: # the config used in 'dockers/ipu-ci-runner/Dockerfile' include: - - python_version: "3.8" + - python_version: "3.9" pytorch_version: "1.7" steps: diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile index 978f8bcdf1..ab26af6c7a 100644 --- a/dockers/base-cuda/Dockerfile +++ b/dockers/base-cuda/Dockerfile @@ -16,7 +16,7 @@ ARG CUDA_VERSION=10.2 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 -ARG PYTHON_VERSION=3.8 +ARG PYTHON_VERSION=3.9 ARG PYTORCH_VERSION=1.6 SHELL ["/bin/bash", "-c"] diff --git a/dockers/base-ipu/Dockerfile b/dockers/base-ipu/Dockerfile index 01b5920d88..e91a0dc4a0 100644 --- a/dockers/base-ipu/Dockerfile +++ b/dockers/base-ipu/Dockerfile @@ -16,8 +16,7 @@ FROM ubuntu:20.04 LABEL maintainer="PyTorchLightning " -ARG PYTHON_VERSION=3.8 -ARG PYTORCH_VERSION=1.7 +ARG PYTHON_VERSION=3.9 ARG CONDA_VERSION=4.9.2 SHELL ["/bin/bash", "-c"] @@ -41,7 +40,7 @@ RUN apt-get update -qq && \ && \ # Install conda and python. # NOTE new Conda does not forward the exit status... https://github.com/conda/conda/issues/8385 - curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_${CONDA_VERSION}-Linux-x86_64.sh && \ + curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py39_${CONDA_VERSION}-Linux-x86_64.sh && \ chmod +x ~/miniconda.sh && \ ~/miniconda.sh -b && \ rm ~/miniconda.sh && \ diff --git a/dockers/base-xla/Dockerfile b/dockers/base-xla/Dockerfile index 1a2554b6b9..5c86da2147 100644 --- a/dockers/base-xla/Dockerfile +++ b/dockers/base-xla/Dockerfile @@ -16,8 +16,8 @@ FROM google/cloud-sdk:slim LABEL maintainer="PyTorchLightning " -# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile . --build-arg PYTHON_VERSION=3.6 -ARG PYTHON_VERSION=3.7 +# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile . --build-arg PYTHON_VERSION=3.8 +ARG PYTHON_VERSION=3.9 ARG CONDA_VERSION=4.9.2 ARG XLA_VERSION=1.6 @@ -42,7 +42,7 @@ RUN apt-get update -qq && \ && \ # Install conda and python. # NOTE new Conda does not forward the exit status... https://github.com/conda/conda/issues/8385 - curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_${CONDA_VERSION}-Linux-x86_64.sh && \ + curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py39_${CONDA_VERSION}-Linux-x86_64.sh && \ chmod +x ~/miniconda.sh && \ ~/miniconda.sh -b && \ rm ~/miniconda.sh && \ diff --git a/dockers/ipu-ci-runner/Dockerfile b/dockers/ipu-ci-runner/Dockerfile index aa8672a34a..98f769f78f 100644 --- a/dockers/ipu-ci-runner/Dockerfile +++ b/dockers/ipu-ci-runner/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG PYTHON_VERSION=3.8 +ARG PYTHON_VERSION=3.9 ARG PYTORCH_VERSION=1.7 FROM pytorchlightning/pytorch_lightning:base-ipu-py${PYTHON_VERSION}-torch${PYTORCH_VERSION} diff --git a/dockers/release/Dockerfile b/dockers/release/Dockerfile index bea977899c..5296800597 100644 --- a/dockers/release/Dockerfile +++ b/dockers/release/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG PYTHON_VERSION=3.7 +ARG PYTHON_VERSION=3.9 ARG PYTORCH_VERSION=1.6 FROM pytorchlightning/pytorch_lightning:base-cuda-py${PYTHON_VERSION}-torch${PYTORCH_VERSION} diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile index 3fc703edb2..086bd349bc 100644 --- a/dockers/tpu-tests/Dockerfile +++ b/dockers/tpu-tests/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG PYTHON_VERSION=3.7 +ARG PYTHON_VERSION=3.9 ARG PYTORCH_VERSION=1.6 FROM pytorchlightning/pytorch_lightning:base-xla-py${PYTHON_VERSION}-torch${PYTORCH_VERSION}