From 116027088262ab6cb2f094d61e63d991dd391d05 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 2 Oct 2020 11:26:21 +0200 Subject: [PATCH] fix path in CI for release & python version in all dockers & duplicated badges (#3765) * typo * path * check * trigger * fix conda * pip ver * fix cuda * fix XLA * fix xla * ci * docker * BIULD * unBIULD * update * py 3.8 * apex * apex --- .drone.yml | 2 +- .github/workflows/ci_dockers.yml | 10 ++--- .github/workflows/nightly.yml | 33 ++------------ .github/workflows/pypi-release.yml | 4 +- .pyrightconfig.json | 1 + README.md | 2 - dockers/base-cuda/Dockerfile | 34 +++++++++++---- dockers/base-xla/Dockerfile | 59 +++++++++++++++++++------- dockers/conda/Dockerfile | 18 +++++++- dockers/tpu-tests/Dockerfile | 14 ++++++ environment.yml | 10 +++-- {tests => requirements}/install_AMP.sh | 0 tests/README.md | 2 +- 13 files changed, 121 insertions(+), 68 deletions(-) rename {tests => requirements}/install_AMP.sh (100%) diff --git a/.drone.yml b/.drone.yml index 40a3746bd8..8760b9b458 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,7 +46,7 @@ steps: - pip install pip -U - pip --version - nvidia-smi - #- bash ./tests/install_AMP.sh + #- bash ./requirements/install_AMP.sh - apt-get update && apt-get install -y cmake - pip install -r ./requirements/base.txt --user -q --upgrade-strategy only-if-needed - pip install -r ./requirements/devel.txt --user -q --upgrade-strategy only-if-needed diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml index aca7a835ae..ce25c75869 100644 --- a/.github/workflows/ci_dockers.yml +++ b/.github/workflows/ci_dockers.yml @@ -14,11 +14,8 @@ jobs: strategy: fail-fast: false matrix: - python_version: [3.6, 3.8] - pytorch_version: [1.3, 1.5, 1.7] - exclude: - - python_version: 3.8 - pytorch_version: 1.3 + python_version: [3.6] + pytorch_version: [1.3, 1.7] steps: - name: Checkout uses: actions/checkout@v2 @@ -62,6 +59,9 @@ jobs: pytorch_version: 1.7 pytorch_channel: pytorch-nightly - python_version: 3.8 + pytorch_version: 1.6 + pytorch_channel: pytorch + - python_version: 3.6 pytorch_version: 1.5 pytorch_channel: pytorch steps: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bbb5c71bc1..549b844876 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -23,7 +23,7 @@ jobs: python -m pip install --user --upgrade setuptools wheel - name: Build run: | - python prepare-nightly_version.py + python .github/prepare-nightly_version.py python setup.py sdist bdist_wheel ls -lh dist/ @@ -47,7 +47,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Publish Master to Docker + - name: Publish XLA to Docker Hub # publish master uses: docker/build-push-action@v1.1.0 with: @@ -83,7 +83,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Publish Master to Docker + - name: Publish CUDA to Docker Hub # publish master uses: docker/build-push-action@v1.1.0 with: @@ -95,32 +95,7 @@ jobs: tags: "base-cuda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}" timeout-minutes: 55 - - docker-Conda: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - python_version: [ 3.6, 3.7, 3.8 ] - pytorch_version: [ 1.3, 1.4, 1.5, 1.6, 1.7 ] - pytorch_channel: [ "pytorch", "pytorch-nightly" ] - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-including-new-combinations - exclude: - - pytorch_version: 1.7 - pytorch_channel: pytorch - - pytorch_version: 1.3 - pytorch_channel: pytorch-nightly - - pytorch_version: 1.4 - pytorch_channel: pytorch-nightly - - pytorch_version: 1.5 - pytorch_channel: pytorch-nightly - - pytorch_version: 1.6 - pytorch_channel: pytorch-nightly - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Publish Master to Docker + - name: Publish Conda to Docker Hub # publish master uses: docker/build-push-action@v1.1.0 with: diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 1cc76e7792..7579fc0b19 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -7,10 +7,10 @@ on: # Trigger the workflow on push or pull request, but only for the master bra release: types: [created] -# based on https://github.com/pypa/gh-action-pypi-publish jobs: - build: + # based on https://github.com/pypa/gh-action-pypi-publish + build-publish: runs-on: ubuntu-20.04 steps: diff --git a/.pyrightconfig.json b/.pyrightconfig.json index ade1db6558..cb14993e2c 100644 --- a/.pyrightconfig.json +++ b/.pyrightconfig.json @@ -35,6 +35,7 @@ "pytorch_lightning/trainer/connectors/checkpoint_connector.py", "pytorch_lightning/trainer/connectors/data_connector.py", "pytorch_lightning/trainer/connectors/logger_connector.py", + "pytorch_lightning/distributed/dist.py", "pytorch_lightning/tuner", "pytorch_lightning/plugins" ], diff --git a/README.md b/README.md index 4075b6ebb0..5c1a31545e 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,6 @@ Get started with our [3 steps guide](https://pytorch-lightning.readthedocs.io/en | System / PyTorch ver. | 1.3 (min. req.)* | 1.4 | 1.5 | 1.6 (latest) | 1.7 (nightly) | | :---: | :---: | :---: | :---: | :---: | :---: | | Conda py3.7 [linux] | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | -| Linux py3.7 [GPUs**] | - | - | - | [![Build Status](http://35.192.60.23/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://35.192.60.23/PyTorchLightning/pytorch-lightning) | - | -| Linux py3.6 / py3.7 [TPUs***] | - | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | - | | Linux py3.7 [GPUs**] | - | - |[![Build Status](http://104.154.220.231/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://104.154.220.231/PyTorchLightning/pytorch-lightning) | - | - | | Linux py3.7 [TPUs***] | - | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | - | | Linux py3.6 / py3.7 / py3.8 | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile index 2408fb5570..2090320f3c 100644 --- a/dockers/base-cuda/Dockerfile +++ b/dockers/base-cuda/Dockerfile @@ -1,3 +1,17 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Existing images: # --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.7 --build-arg PYTORCH_CHANNEL=pytorch-nightly --build-arg CUDA_VERSION=10.1 # --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.6 --build-arg PYTORCH_CHANNEL=pytorch --build-arg CUDA_VERSION=10.1 @@ -56,15 +70,17 @@ ENV CONDA_ENV=lightning COPY environment.yml environment.yml # conda init -RUN conda create -y --name $CONDA_ENV "python=$PYTHON_VERSION" "pytorch=$PYTORCH_VERSION" torchvision "cudatoolkit=$CUDA_VERSION" --channel=$PYTORCH_CHANNEL && \ +RUN conda create -y --name $CONDA_ENV "cudatoolkit=$CUDA_VERSION" && \ conda init bash && \ # NOTE: this requires that the channel is presented in the yaml before packages - # replace channel to nigtly if neede, fix PT version and remove Horovod as it will be installe later - python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'torch>=[\d\.]+', 'torch=${PYTORCH_VERSION}', open(fname).read().replace('pytorch', '${PYTORCH_CHANNEL}', 1)) ; open(fname, 'w').writelines([l for l in req if 'horovod' not in l])" && \ + # replace channel to nigtly if needed, fix PT version and remove Horovod as it will be installe later + python -c "fname = 'environment.yml' ; req = open(fname).read().replace('pytorch', '${PYTORCH_CHANNEL}', 1) ; open(fname, 'w').write(req)" && \ + python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python[>=]+[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \ + python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'torch[>=]+[\d\.]+', 'torch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \ + python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if 'horovod' not in ln])" && \ cat environment.yml && \ conda env update --file environment.yml && \ conda clean -ya && \ - # conda config --set auto_activate_base true && \ rm environment.yml ENV PATH /root/miniconda3/envs/${CONDA_ENV}/bin:$PATH @@ -74,7 +90,6 @@ ENV CONDA_DEFAULT_ENV=${CONDA_ENV} COPY ./requirements/extra.txt requirements-extra.txt COPY ./requirements/test.txt requirements-tests.txt -COPY ./requirements/examples.txt requirements-examples.txt RUN \ # Disable cache @@ -82,18 +97,21 @@ RUN \ #echo ". ${WORKDIR}/miniconda/etc/profile.d/conda.sh" >> ~/.bashrc && \ #echo "conda activate ${CONDA_ENV}" >> ~/.bashrc && \ #source ~/.bashrc && \ + # install NVIDIA AMP + git clone https://github.com/NVIDIA/apex && \ + pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./apex && \ + rm -rf apex && \ # filter only Horovod python -c "fname = 'requirements-extra.txt' ; req = open(fname).readlines() ; open(fname, 'w').writelines([l for l in req if 'horovod' in l])" && \ # Install all requirements MAKEFLAGS="-j$(nproc)" ; pip install -r requirements-extra.txt && \ pip install -r requirements-tests.txt --upgrade-strategy only-if-needed && \ - pip install -r requirements-examples.txt --upgrade-strategy only-if-needed && \ rm requirements* RUN \ # Show what we have pip --version && \ conda info && \ - conda list && \ pip list && \ - python -c "import torch; print(torch.__version__)" \ No newline at end of file + python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \ + python -c "import torch; assert torch.__version__[:3] == '$PYTORCH_VERSION', torch.__version__" \ No newline at end of file diff --git a/dockers/base-xla/Dockerfile b/dockers/base-xla/Dockerfile index 9e659338ba..6297d6acc2 100644 --- a/dockers/base-xla/Dockerfile +++ b/dockers/base-xla/Dockerfile @@ -1,15 +1,29 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM google/cloud-sdk:slim -# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile . +# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile . --build-arg PYTHON_VERSION=3.6 # This Dockerfile installs pytorch/xla 3.7 wheels. There are also 3.6 wheels available; see below. ARG PYTHON_VERSION=3.7 -ARG XLA_VERSION="1.6" +ARG XLA_VERSION=1.6 SHELL ["/bin/bash", "-c"] # for skipping configurations ENV DEBIAN_FRONTEND=noninteractive -ENV CONDA_ENV=pytorch-xla +ENV CONDA_ENV=lightning # show system inforation RUN lsb_release -a && cat /etc/*-release @@ -40,15 +54,28 @@ RUN apt-get update && \ ENV PATH="/root/miniconda3/bin:$PATH" ENV LD_LIBRARY_PATH="/root/miniconda3/lib:$LD_LIBRARY_PATH" +COPY environment.yml environment.yml -RUN conda create -y --name $CONDA_ENV "python=$PYTHON_VERSION" && \ +RUN conda create -y --name $CONDA_ENV && \ conda init bash && \ - conda install -y mkl && \ + # replace channel to nigtly if neede, fix PT version and remove Horovod as it will be installe later + python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python>=[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \ + python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if not any(n in ln for n in ['pytorch>', 'horovod'])])" && \ + cat environment.yml && \ + conda env update --file environment.yml && \ + conda clean -ya && \ + rm environment.yml + +ENV PATH /root/miniconda3/envs/${CONDA_ENV}/bin:$PATH +ENV LD_LIBRARY_PATH="/root/miniconda3/envs/${CONDA_ENV}/lib:$LD_LIBRARY_PATH" +# if you want this environment to be the default one, uncomment the following line: +ENV CONDA_DEFAULT_ENV=${CONDA_ENV} # Disable cache +RUN pip --version && \ pip config set global.cache-dir false && \ - pip install "pip>20.1" -U && \ + conda remove pytorch torchvision && \ # Install Pytorch XLA py_version=${PYTHON_VERSION/./} && \ # Python 3.7 wheels are available. Replace cp36-cp36m with cp37-cp37m @@ -58,27 +85,29 @@ RUN conda create -y --name $CONDA_ENV "python=$PYTHON_VERSION" && \ pip install *.whl && \ rm *.whl -ENV LD_LIBRARY_PATH="/root/miniconda3/envs/$CONDA_ENV/lib:$LD_LIBRARY_PATH" -# if you want this environment to be the default one, uncomment the following line: -ENV CONDA_DEFAULT_ENV=${CONDA_ENV} - # Get package COPY ./ ./pytorch-lightning/ # Install pytorch-lightning dependencies. RUN \ + python --version && \ # Install PL dependencies cd pytorch-lightning && \ # drop Torch as it was installed with XLA - python -c "fname = \"./requirements/base.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"torch\")] ; open(fname, \"w\").writelines(lines)" && \ + python -c "fname = 'requirements/base.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('torch')] ; open(fname, 'w').writelines(lines)" && \ # drop Horovod as it is not needed - python -c "fname = \"./requirements/extra.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"horovod\")] ; open(fname, \"w\").writelines(lines)" && \ + python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" && \ # drop TorchVision as it was installed with XLA - python -c "fname = \"./requirements/examples.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"torchvision\")] ; open(fname, \"w\").writelines(lines)" && \ + python -c "fname = 'requirements/examples.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('torchvision')] ; open(fname, 'w').writelines(lines)" && \ pip install --requirement ./requirements/devel.txt --upgrade-strategy only-if-needed && \ cd .. && \ rm -rf pytorch-lightning && \ rm -rf /root/.cache -RUN pip --version && \ - python -c "import torch; print(torch.__version__)" +RUN \ + # Show what we have + pip --version && \ + conda info && \ + pip list && \ + python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \ + python -c "import torch; ver = '$XLA_VERSION' ; ver = dict(nightly='1.7').get(ver, ver) ; assert torch.__version__[:3] == ver, torch.__version__" diff --git a/dockers/conda/Dockerfile b/dockers/conda/Dockerfile index 036e13a275..17ad4f9c7e 100644 --- a/dockers/conda/Dockerfile +++ b/dockers/conda/Dockerfile @@ -1,3 +1,17 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ARG PYTHON_VERSION=3.7 ARG PYTORCH_VERSION=1.5 @@ -5,7 +19,7 @@ FROM pytorchlightning/pytorch_lightning:base-cuda-py${PYTHON_VERSION}-torch${PYT ARG LIGHTNING_VERSION="" -COPY --chown=flash ./ ./pytorch-lightning/ +COPY ./ ./pytorch-lightning/ # install dependencies RUN \ @@ -17,7 +31,7 @@ RUN \ rm -rf pytorch-lightning ; \ else \ rm -rf pytorch-lightning ; \ - pip install git+https://github.com/PyTorchLightning/pytorch-lightning.git@${LIGHTNING_VERSION} --upgrade-strategy only-if-needed ; \ + pip install https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --upgrade-strategy only-if-needed ; \ fi RUN python --version && \ diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile index e4f8ef49df..79852b14a8 100644 --- a/dockers/tpu-tests/Dockerfile +++ b/dockers/tpu-tests/Dockerfile @@ -1,3 +1,17 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ARG PYTHON_VERSION=3.7 ARG PYTORCH_VERSION=1.6 diff --git a/environment.yml b/environment.yml index aa86ce0224..09eb997b78 100644 --- a/environment.yml +++ b/environment.yml @@ -24,22 +24,25 @@ channels: dependencies: - python>=3.6 - - pip + - pip>20.1 - numpy>=1.16.4 - pytorch>=1.3 - future>=0.17.1 - PyYAML>=5.1 - tqdm>=4.41.0 - fsspec>=0.8.0 - #- tensorboard>=2.2.0 + #- tensorboard>=2.2.0 # not needed, already included in pytorch # Optional - - nvidia-apex + #- nvidia-apex # missing for py3.8 - scikit-learn>=0.20.0 - matplotlib>=3.1.1 - omegaconf>=2.0.0 - torchtext>=0.3.1 + # Examples + - torchvision>=0.4.1 + - pip: - test-tube>=0.7.5 - mlflow>=1.0.0 @@ -48,3 +51,4 @@ dependencies: - neptune-client>=0.4.109 - horovod>=0.19.2 - onnxruntime>=1.3.0 + - gym>=0.17.0 diff --git a/tests/install_AMP.sh b/requirements/install_AMP.sh similarity index 100% rename from tests/install_AMP.sh rename to requirements/install_AMP.sh diff --git a/tests/README.md b/tests/README.md index ccd62301aa..7fd3c90c02 100644 --- a/tests/README.md +++ b/tests/README.md @@ -17,7 +17,7 @@ git clone https://github.com/PyTorchLightning/pytorch-lightning cd pytorch-lightning # install AMP support -bash tests/install_AMP.sh +bash requirements/install_AMP.sh # install dev deps pip install -r requirements/devel.txt