fix path in CI for release & python version in all dockers & duplicated badges (#3765)

* typo * path * check * trigger * fix conda * pip ver * fix cuda * fix XLA * fix xla * ci * docker * BIULD * unBIULD * update * py 3.8 * apex * apex
2020-10-02 11:26:21 +02:00 · 2020-10-02 11:26:21 +02:00 · 1160270882
parent ebc1b23fa3
commit 1160270882
13 changed files with 121 additions and 68 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -46,7 +46,7 @@ steps:
    - pip install pip -U
    - pip --version
    - nvidia-smi
-    #- bash ./tests/install_AMP.sh
+    #- bash ./requirements/install_AMP.sh
    - apt-get update && apt-get install -y cmake
    - pip install -r ./requirements/base.txt --user -q --upgrade-strategy only-if-needed
    - pip install -r ./requirements/devel.txt --user -q --upgrade-strategy only-if-needed
--- a/.github/workflows/ci_dockers.yml
+++ b/.github/workflows/ci_dockers.yml
@ -14,11 +14,8 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        python_version: [3.6, 3.8]
-        pytorch_version: [1.3, 1.5, 1.7]
-        exclude:
-          - python_version: 3.8
-            pytorch_version: 1.3
+        python_version: [3.6]
+        pytorch_version: [1.3, 1.7]
    steps:
      - name: Checkout
        uses: actions/checkout@v2
@ -62,6 +59,9 @@ jobs:
            pytorch_version: 1.7
            pytorch_channel: pytorch-nightly
          - python_version: 3.8
+            pytorch_version: 1.6
+            pytorch_channel: pytorch
+          - python_version: 3.6
            pytorch_version: 1.5
            pytorch_channel: pytorch
    steps:
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@ -23,7 +23,7 @@ jobs:
        python -m pip install --user --upgrade setuptools wheel
    - name: Build
      run: |
-        python prepare-nightly_version.py
+        python .github/prepare-nightly_version.py
        python setup.py sdist bdist_wheel
        ls -lh dist/

@ -47,7 +47,7 @@ jobs:
      - name: Checkout
        uses: actions/checkout@v2

-      - name: Publish Master to Docker
+      - name: Publish XLA to Docker Hub
        # publish master
        uses: docker/build-push-action@v1.1.0
        with:
@ -83,7 +83,7 @@ jobs:
      - name: Checkout
        uses: actions/checkout@v2

-      - name: Publish Master to Docker
+      - name: Publish CUDA to Docker Hub
        # publish master
        uses: docker/build-push-action@v1.1.0
        with:
@ -95,32 +95,7 @@ jobs:
          tags: "base-cuda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}"
        timeout-minutes: 55

-
-  docker-Conda:
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        python_version: [ 3.6, 3.7, 3.8 ]
-        pytorch_version: [ 1.3, 1.4, 1.5, 1.6, 1.7 ]
-        pytorch_channel: [ "pytorch", "pytorch-nightly" ]
-        # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-including-new-combinations
-        exclude:
-          - pytorch_version: 1.7
-            pytorch_channel: pytorch
-          - pytorch_version: 1.3
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.4
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.5
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.6
-            pytorch_channel: pytorch-nightly
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Publish Master to Docker
+      - name: Publish Conda to Docker Hub
        # publish master
        uses: docker/build-push-action@v1.1.0
        with:
--- a/.github/workflows/pypi-release.yml
+++ b/.github/workflows/pypi-release.yml
@ -7,10 +7,10 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
  release:
    types: [created]

-# based on https://github.com/pypa/gh-action-pypi-publish

 jobs:
-  build:
+  # based on https://github.com/pypa/gh-action-pypi-publish
+  build-publish:
    runs-on: ubuntu-20.04

    steps:
--- a/.pyrightconfig.json
+++ b/.pyrightconfig.json
@ -35,6 +35,7 @@
    "pytorch_lightning/trainer/connectors/checkpoint_connector.py",
    "pytorch_lightning/trainer/connectors/data_connector.py",
    "pytorch_lightning/trainer/connectors/logger_connector.py",
+    "pytorch_lightning/distributed/dist.py",
    "pytorch_lightning/tuner",
    "pytorch_lightning/plugins"
  ],
--- a/README.md
+++ b/README.md
@ -82,8 +82,6 @@ Get started with our [3 steps guide](https://pytorch-lightning.readthedocs.io/en
 | System / PyTorch ver. | 1.3 (min. req.)* | 1.4 | 1.5 | 1.6 (latest) | 1.7 (nightly) |
 | :---: | :---: | :---: | :---: | :---: | :---: |
 | Conda py3.7 [linux] | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) |
-| Linux py3.7 [GPUs**] | - | - | - | [![Build Status](http://35.192.60.23/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://35.192.60.23/PyTorchLightning/pytorch-lightning) | - |
-| Linux py3.6 / py3.7 [TPUs***] | - | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | - |
 | Linux py3.7 [GPUs**] | - | - |[![Build Status](http://104.154.220.231/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://104.154.220.231/PyTorchLightning/pytorch-lightning) | - | - |
 | Linux py3.7 [TPUs***] | - | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | - |
 | Linux py3.6 / py3.7 / py3.8 | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |
--- a/dockers/base-cuda/Dockerfile
+++ b/dockers/base-cuda/Dockerfile
@ -1,3 +1,17 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Existing images:
 # --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.7 --build-arg PYTORCH_CHANNEL=pytorch-nightly --build-arg CUDA_VERSION=10.1
 # --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.6 --build-arg PYTORCH_CHANNEL=pytorch --build-arg CUDA_VERSION=10.1
@ -56,15 +70,17 @@ ENV CONDA_ENV=lightning
 COPY environment.yml environment.yml

 # conda init
-RUN conda create -y --name $CONDA_ENV "python=$PYTHON_VERSION" "pytorch=$PYTORCH_VERSION" torchvision "cudatoolkit=$CUDA_VERSION" --channel=$PYTORCH_CHANNEL && \
+RUN conda create -y --name $CONDA_ENV "cudatoolkit=$CUDA_VERSION" && \
    conda init bash && \
    # NOTE: this requires that the channel is presented in the yaml before packages
-    # replace channel to nigtly if neede, fix PT version and remove Horovod as it will be installe later
-    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'torch>=[\d\.]+', 'torch=${PYTORCH_VERSION}', open(fname).read().replace('pytorch', '${PYTORCH_CHANNEL}', 1)) ; open(fname, 'w').writelines([l for l in req if 'horovod' not in l])" && \
+    # replace channel to nigtly if needed, fix PT version and remove Horovod as it will be installe later
+    python -c "fname = 'environment.yml' ; req = open(fname).read().replace('pytorch', '${PYTORCH_CHANNEL}', 1) ; open(fname, 'w').write(req)" && \
+    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python[>=]+[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
+    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'torch[>=]+[\d\.]+', 'torch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
+    python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if 'horovod' not in ln])" && \
    cat environment.yml && \
    conda env update --file environment.yml && \
    conda clean -ya && \
-    # conda config --set auto_activate_base true && \
    rm environment.yml

 ENV PATH /root/miniconda3/envs/${CONDA_ENV}/bin:$PATH
@ -74,7 +90,6 @@ ENV CONDA_DEFAULT_ENV=${CONDA_ENV}

 COPY ./requirements/extra.txt requirements-extra.txt
 COPY ./requirements/test.txt requirements-tests.txt
-COPY ./requirements/examples.txt requirements-examples.txt

 RUN \
    # Disable cache
@ -82,18 +97,21 @@ RUN \
    #echo ". ${WORKDIR}/miniconda/etc/profile.d/conda.sh" >> ~/.bashrc && \
    #echo "conda activate ${CONDA_ENV}" >> ~/.bashrc && \
    #source ~/.bashrc && \
+    # install NVIDIA AMP
+    git clone https://github.com/NVIDIA/apex && \
+    pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./apex && \
+    rm -rf apex && \
    # filter only Horovod
    python -c "fname = 'requirements-extra.txt' ; req = open(fname).readlines() ; open(fname, 'w').writelines([l for l in req if 'horovod' in l])" && \
    # Install all requirements
    MAKEFLAGS="-j$(nproc)" ; pip install -r requirements-extra.txt && \
    pip install -r requirements-tests.txt --upgrade-strategy only-if-needed && \
-    pip install -r requirements-examples.txt --upgrade-strategy only-if-needed && \
    rm requirements*

 RUN \
    # Show what we have
    pip --version && \
    conda info && \
-    conda list && \
    pip list && \
-    python -c "import torch; print(torch.__version__)"
+    python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \
+    python -c "import torch; assert torch.__version__[:3] == '$PYTORCH_VERSION', torch.__version__"
--- a/dockers/base-xla/Dockerfile
+++ b/dockers/base-xla/Dockerfile
@ -1,15 +1,29 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM google/cloud-sdk:slim

-# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile .
+# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile . --build-arg PYTHON_VERSION=3.6
 # This Dockerfile installs pytorch/xla 3.7 wheels. There are also 3.6 wheels available; see below.
 ARG PYTHON_VERSION=3.7
-ARG XLA_VERSION="1.6"
+ARG XLA_VERSION=1.6

 SHELL ["/bin/bash", "-c"]

 # for skipping configurations
 ENV DEBIAN_FRONTEND=noninteractive
-ENV CONDA_ENV=pytorch-xla
+ENV CONDA_ENV=lightning

 # show system inforation
 RUN lsb_release -a && cat /etc/*-release
@ -40,15 +54,28 @@ RUN apt-get update && \

 ENV PATH="/root/miniconda3/bin:$PATH"
 ENV LD_LIBRARY_PATH="/root/miniconda3/lib:$LD_LIBRARY_PATH"
+COPY environment.yml environment.yml

-RUN conda create -y --name $CONDA_ENV "python=$PYTHON_VERSION" && \
+RUN conda create -y --name $CONDA_ENV && \
    conda init bash && \
-    conda install -y mkl && \
+    # replace channel to nigtly if neede, fix PT version and remove Horovod as it will be installe later
+    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python>=[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
+    python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if not any(n in ln for n in ['pytorch>', 'horovod'])])" && \
+    cat environment.yml && \
+    conda env update --file environment.yml && \
+    conda clean -ya && \
+    rm environment.yml
+
+ENV PATH /root/miniconda3/envs/${CONDA_ENV}/bin:$PATH
+ENV LD_LIBRARY_PATH="/root/miniconda3/envs/${CONDA_ENV}/lib:$LD_LIBRARY_PATH"
+# if you want this environment to be the default one, uncomment the following line:
+ENV CONDA_DEFAULT_ENV=${CONDA_ENV}

 # Disable cache
+RUN pip --version && \
    pip config set global.cache-dir false && \
-    pip install "pip>20.1" -U  && \

+    conda remove pytorch torchvision && \
 # Install Pytorch XLA
    py_version=${PYTHON_VERSION/./} && \
    # Python 3.7 wheels are available. Replace cp36-cp36m with cp37-cp37m
@ -58,27 +85,29 @@ RUN conda create -y --name $CONDA_ENV "python=$PYTHON_VERSION" && \
    pip install *.whl && \
    rm *.whl

-ENV LD_LIBRARY_PATH="/root/miniconda3/envs/$CONDA_ENV/lib:$LD_LIBRARY_PATH"
-# if you want this environment to be the default one, uncomment the following line:
-ENV CONDA_DEFAULT_ENV=${CONDA_ENV}
-
 # Get package
 COPY ./ ./pytorch-lightning/

 # Install pytorch-lightning dependencies.
 RUN \
+    python --version && \
 # Install PL dependencies
    cd pytorch-lightning && \
    # drop Torch as it was installed with XLA
-    python -c "fname = \"./requirements/base.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"torch\")] ; open(fname, \"w\").writelines(lines)" && \
+    python -c "fname = 'requirements/base.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('torch')] ; open(fname, 'w').writelines(lines)" && \
    # drop Horovod as it is not needed
-    python -c "fname = \"./requirements/extra.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"horovod\")] ; open(fname, \"w\").writelines(lines)" && \
+    python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" && \
    # drop TorchVision as it was installed with XLA
-    python -c "fname = \"./requirements/examples.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"torchvision\")] ; open(fname, \"w\").writelines(lines)" && \
+    python -c "fname = 'requirements/examples.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('torchvision')] ; open(fname, 'w').writelines(lines)" && \
    pip install --requirement ./requirements/devel.txt --upgrade-strategy only-if-needed && \
    cd .. && \
    rm -rf pytorch-lightning && \
    rm -rf /root/.cache

-RUN pip --version && \
-    python -c "import torch; print(torch.__version__)"
+RUN \
+    # Show what we have
+    pip --version && \
+    conda info && \
+    pip list && \
+    python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \
+    python -c "import torch; ver = '$XLA_VERSION' ; ver = dict(nightly='1.7').get(ver, ver) ; assert torch.__version__[:3] == ver, torch.__version__"
--- a/dockers/conda/Dockerfile
+++ b/dockers/conda/Dockerfile
@ -1,3 +1,17 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ARG PYTHON_VERSION=3.7
 ARG PYTORCH_VERSION=1.5

@ -5,7 +19,7 @@ FROM pytorchlightning/pytorch_lightning:base-cuda-py${PYTHON_VERSION}-torch${PYT

 ARG LIGHTNING_VERSION=""

-COPY --chown=flash ./ ./pytorch-lightning/
+COPY ./ ./pytorch-lightning/

 # install dependencies
 RUN \
@ -17,7 +31,7 @@ RUN \
        rm -rf pytorch-lightning ; \
    else \
        rm -rf pytorch-lightning ; \
-        pip install git+https://github.com/PyTorchLightning/pytorch-lightning.git@${LIGHTNING_VERSION} --upgrade-strategy only-if-needed ; \
+        pip install https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --upgrade-strategy only-if-needed ; \
    fi

 RUN python --version && \
--- a/dockers/tpu-tests/Dockerfile
+++ b/dockers/tpu-tests/Dockerfile
@ -1,3 +1,17 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ARG PYTHON_VERSION=3.7
 ARG PYTORCH_VERSION=1.6

--- a/environment.yml
+++ b/environment.yml
@ -24,22 +24,25 @@ channels:

 dependencies:
    - python>=3.6
-    - pip
+    - pip>20.1
    - numpy>=1.16.4
    - pytorch>=1.3
    - future>=0.17.1
    - PyYAML>=5.1
    - tqdm>=4.41.0
    - fsspec>=0.8.0
-    #- tensorboard>=2.2.0
+    #- tensorboard>=2.2.0  # not needed, already included in pytorch

    # Optional
-    - nvidia-apex
+    #- nvidia-apex  # missing for py3.8
    - scikit-learn>=0.20.0
    - matplotlib>=3.1.1
    - omegaconf>=2.0.0
    - torchtext>=0.3.1

+    # Examples
+    - torchvision>=0.4.1
+
    - pip:
        - test-tube>=0.7.5
        - mlflow>=1.0.0
@ -48,3 +51,4 @@ dependencies:
        - neptune-client>=0.4.109
        - horovod>=0.19.2
        - onnxruntime>=1.3.0
+        - gym>=0.17.0
--- a/requirements/install_AMP.sh
+++ b/requirements/install_AMP.sh
--- a/tests/README.md
+++ b/tests/README.md
@ -17,7 +17,7 @@ git clone https://github.com/PyTorchLightning/pytorch-lightning
 cd pytorch-lightning

 # install AMP support
-bash tests/install_AMP.sh
+bash requirements/install_AMP.sh

 # install dev deps
 pip install -r requirements/devel.txt