add CI for building dockers (#3383)

* rename * fix badges * add docker build * mergify * update * env * ci * times * CI * name * comment
2020-09-11 00:38:29 +02:00 · 2020-09-11 00:38:29 +02:00 · cbc4f6f8a4
parent a1ea681c47
commit cbc4f6f8a4
17 changed files with 138 additions and 90 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -55,7 +55,7 @@ steps:
    - pip list
    - python -c "import torch ; print(' & '.join([torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]) if torch.cuda.is_available() else 'only CPU')"
    - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --durations=25 # --flake8
-    - python -m py.test benchmarks pl_examples -v --maxfail=2  --durations=0 # --flake8
+    - python -m py.test benchmarks pl_examples -v --maxfail=2 --durations=0 # --flake8
    #- cd docs; make doctest; make coverage
    - coverage report
    # see: https://docs.codecov.io/docs/merging-reports
--- a/.github/workflows/ci_dockers.yml
+++ b/.github/workflows/ci_dockers.yml
@ -0,0 +1,74 @@
+name: CI build Docker
+# https://www.docker.com/blog/first-docker-github-action-is-here
+# https://github.com/docker/build-push-action
+# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
+on:
+  # Trigger the workflow on push or pull request, but only for the master branch
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build-Conda:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: [3.7]
+        pytorch_version: [1.6]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build Docker
+        # publish master
+        uses: docker/build-push-action@v1.1.0
+        with:
+          dockerfile: dockers/conda/Dockerfile
+          build_args: PYTHON_VERSION=${{ matrix.python_version }},PYTORCH_VERSION=${{ matrix.pytorch_version }}
+          push: false
+        timeout-minutes: 40
+
+  build-XLA:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: [3.7]
+        xla_version: ["nightly"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build Docker
+        # publish master
+        uses: docker/build-push-action@v1.1.0
+        with:
+          dockerfile: dockers/base-xla/Dockerfile
+          build_args: PYTHON_VERSION=${{ matrix.python_version }},XLA_VERSION=${{ matrix.xla_version }}
+          push: false
+        timeout-minutes: 40
+
+# TODO: uncomment this with fixing CUDA docker, no need to increase mergify count
+#  build-cuda:
+#    runs-on: ubuntu-20.04
+#    strategy:
+#      fail-fast: false
+#      matrix:
+#        python_version: [3.7]
+#        pytorch_version: [1.5]
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v2
+#
+#      - name: Publish Master to Docker
+#        # publish master
+#        uses: docker/build-push-action@v1.1.0
+#        with:
+#          dockerfile: dockers/base-cuda/Dockerfile
+#          build_args: PYTHON_VERSION=${{ matrix.python_version }},PYTORCH_VERSION=${{ matrix.pytorch_version }}
+#          push: false
+#        timeout-minutes: 40
--- a/.github/workflows/ci_pkg-install.yml
+++ b/.github/workflows/ci_pkg-install.yml
@ -1,9 +1,7 @@
 name: Install pkg

 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
-on:
-  # Trigger the workflow on push or pull request,
-  # but only for the master branch
+on:  # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches:
      - master
--- a/.github/workflows/ci_pt-conda.yml
+++ b/.github/workflows/ci_pt-conda.yml
@ -1,9 +1,7 @@
 name: PyTorch & Conda

 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
-on:
-  # Trigger the workflow on push or pull request,
-  # but only for the master branch
+on:  # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches:
      - master
@ -65,7 +63,7 @@ jobs:
          conda-version: 4.7.12
          python-version: ${{ matrix.python-version }}
          environment-file: environment.yml
-          activate-environment: pl-env
+          activate-environment: lightning
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!

    - name: Reinstall Horovod if necessary
@ -90,8 +88,10 @@ jobs:
      run: |
        conda info
        conda list
+        pip install --requirement requirements/test.txt
+        pip list
        # NOTE: run coverage on tests does not propagare faler status for Win, https://github.com/nedbat/coveragepy/issues/1003
-        python -m pytest pytorch_lightning tests -v --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
+        python -m pytest pytorch_lightning tests -v --durations=0 --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
        # coverage report
      shell: bash -l {0}

--- a/.github/workflows/ci_test-base.yml
+++ b/.github/workflows/ci_test-base.yml
@ -1,9 +1,7 @@
 name: CI base testing

 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
-on:
-  # Trigger the workflow on push or pull request,
-  # but only for the master branch
+on:  # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches:
      - master
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@ -1,9 +1,7 @@
 name: CI complete testing

 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
-on:
-  # Trigger the workflow on push or pull request,
-  # but only for the master branch
+on:  # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches:
      - master
@ -68,7 +66,7 @@ jobs:
      run: |
        python -c "fname = 'requirements/base.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)"
        python -c "fname = 'requirements/extra.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)"
-        python -c "fname = 'requirements/devel.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)"
+        python -c "fname = 'requirements/test.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)"

    # Note: This uses an internal pip API and may not always work
    # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
@ -116,7 +114,7 @@ jobs:
    - name: Tests
      run: |
        # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
+        coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --durations=0 --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml

    - name: Upload pytest test results
      uses: actions/upload-artifact@master
--- a/.github/workflows/ci_test-tpu.yml
+++ b/.github/workflows/ci_test-tpu.yml
@ -4,7 +4,7 @@ on:
  push:
    branches:
      - master
-  # TODO: temporal disable TPU testing until we find way how to pass credentials to forked PRs
+# TODO: temporal disable TPU testing until we find way how to pass credentials to forked PRs
 #  pull_request:
 #    branches:
 #      - master
--- a/.github/workflows/code-formatting.yml
+++ b/.github/workflows/code-formatting.yml
@ -1,7 +1,6 @@
 name: "Check Code Format"
-on:
-  # Trigger the workflow on push or pull request,
-  # but only for the master branch
+
+on:  # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches:
      - master
--- a/.github/workflows/docker-builds.yml
+++ b/.github/workflows/docker-builds.yml
@ -24,9 +24,6 @@ jobs:
    steps:
      - name: Checkout
        uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7

      - name: Publish Master to Docker
        # publish master
@ -69,14 +66,10 @@ jobs:
    steps:
      - name: Checkout
        uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7

      - name: Publish Master to Docker
        # publish master
        uses: docker/build-push-action@v1.1.0
-        if: github.event_name == 'push'
        with:
          repository: pytorchlightning/pytorch_lightning
          username: ${{ secrets.DOCKER_USERNAME }}
@ -96,14 +89,10 @@ jobs:
    steps:
      - name: Checkout
        uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7

      - name: Publish Master to Docker
        # publish master
        uses: docker/build-push-action@v1.1.0
-        if: github.event_name == 'push'
        with:
          repository: pytorchlightning/pytorch_lightning
          username: ${{ secrets.DOCKER_USERNAME }}
--- a/.github/workflows/pypi-release.yml
+++ b/.github/workflows/pypi-release.yml
@ -1,9 +1,7 @@
 name: PyPI Release

 # https://help.github.com/en/actions/reference/events-that-trigger-workflows
-on:
-  # Trigger the workflow on push or pull request,
-  # but only for the master branch
+on:  # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches:
      - master
--- a/.mergify.yml
+++ b/.mergify.yml
@ -24,7 +24,7 @@ pull_request_rules:
      # no requested chnages from any reviewer
      - "#changes-requested-reviews-by=0"
      # this serves as ALL check has to pass as we have actually around 40 tests in total
-      - "#status-success>=41"
+      - "#status-success>=44"
      # this is just in case since we rely on GPU tests (note: redundand to the above)
      - status-success=continuous-integration/drone/pr
      - "status-success=ci/circleci: TPU-tests"
--- a/README.md
+++ b/README.md
@ -74,9 +74,9 @@ Get started with our [3 steps guide](https://pytorch-lightning.readthedocs.io/en
 | Conda py3.7 [linux] | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | 
 | Linux py3.7 [GPUs**] | - | - | - | [![Build Status](http://35.192.60.23/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://35.192.60.23/PyTorchLightning/pytorch-lightning) |
 | Linux py3.7 [TPUs***] | - | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) |
-| Linux py3.6 / py3.7 / py3.8 | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) |
-| OSX py3.6 / py3.7 | - | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) |
-| Windows py3.6 / py3.7 / py3.8 | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) 
+| Linux py3.6 / py3.7 / py3.8 | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) |
+| OSX py3.6 / py3.7 | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) |
+| Windows py3.6 / py3.7 / py3.8 | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) 

 - _\* `torch>=1.4` is the minimal pytorch version for Python 3.8_
 - _\** tests run on two NVIDIA K80_
--- a/dockers/base-xla/Dockerfile
+++ b/dockers/base-xla/Dockerfile
@ -61,26 +61,24 @@ ENV LD_LIBRARY_PATH="/root/miniconda3/envs/$CONDA_ENV/lib:$LD_LIBRARY_PATH"
 # if you want this environment to be the default one, uncomment the following line:
 ENV CONDA_DEFAULT_ENV=${CONDA_ENV}

+# Get package
+COPY ./ ./pytorch-lightning/
+
 # Install pytorch-lightning dependencies.
 RUN \
-# Get package
-    wget https://github.com/PyTorchLightning/pytorch-lightning/archive/master.zip --progress=bar:force:noscroll && \
-    unzip -q master.zip && \
-    rm master.zip && \
-
 # Install PL dependencies
-    cd pytorch-lightning-master && \
+    cd pytorch-lightning && \
    # drop Torch
    python -c "fname = \"./requirements/base.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"torch\")] ; open(fname, \"w\").writelines(lines)" && \
-    pip install --requirement ./requirements/base.txt && \
+    pip install --requirement ./requirements/base.txt --upgrade-strategy only-if-needed && \
    # drop Horovod
    python -c "fname = \"./requirements/extra.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"horovod\")] ; open(fname, \"w\").writelines(lines)" && \
-    pip install --requirement ./requirements/extra.txt && \
+    pip install --requirement ./requirements/extra.txt --upgrade-strategy only-if-needed && \
    # drop TorchVision
    python -c "fname = \"./requirements/examples.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"torchvision\")] ; open(fname, \"w\").writelines(lines)" && \
-    pip install --requirement ./requirements/examples.txt && \
+    pip install --requirement ./requirements/examples.txt --upgrade-strategy only-if-needed && \
    cd .. && \
-    rm -rf pytorch-lightning-master && \
+    rm -rf pytorch-lightning && \
    rm -rf /root/.cache

 RUN pip --version && \
--- a/dockers/conda/Dockerfile
+++ b/dockers/conda/Dockerfile
@ -1,10 +1,10 @@
 ARG CUDA_VERSION=10.1
-FROM nvidia/cuda:${CUDA_VERSION}-base
+FROM nvidia/cuda:${CUDA_VERSION}-devel

 # install versions
 ARG PYTHON_VERSION=3.7
 ARG PYTORCH_VERSION=1.4
-ARG LIGHTNING_VERSION=master
+ARG LIGHTNING_VERSION=""
 # NOTE new Conda does not forward the exit status... https://github.com/conda/conda/issues/8385
 ARG CONDA_VERSION=4.7.12

@ -23,41 +23,54 @@ RUN apt-get update && \
    rm -rf /root/.cache
         
 # add non-root user
-RUN useradd --create-home --shell /bin/bash containeruser
+RUN useradd --create-home --shell /bin/bash flash

-USER containeruser
-WORKDIR /home/containeruser
+USER flash
 ENV CONDA_ENV=lightning
+ENV WORKDIR=/home/flash
+WORKDIR $WORKDIR

 # install conda and python
 RUN curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh  && \
    chmod +x ~/miniconda.sh && \
-    ~/miniconda.sh -b -p /home/containeruser/miniconda && \
+    ~/miniconda.sh -b -p ${WORKDIR}/miniconda && \
    rm ~/miniconda.sh

 # add conda to path
-ENV PATH="/home/containeruser/miniconda/bin:$PATH"
-ENV LD_LIBRARY_PATH="/home/containeruser/miniconda/lib:$LD_LIBRARY_PATH"
+ENV PATH="${WORKDIR}/miniconda/bin:$PATH"
+ENV LD_LIBRARY_PATH="${WORKDIR}/miniconda/lib:$LD_LIBRARY_PATH"
+ENV CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda"
+
+COPY --chown=flash environment.yml environment.yml

 # conda init
 RUN conda create -y --name $CONDA_ENV python=$PYTHON_VERSION && \
    conda init bash && \
-    conda install -y python=$PYTHON_VERSION && \
+    # conda install -y python=$PYTHON_VERSION && \
+    conda install pytorch=$PYTORCH_VERSION cudatoolkit=$CUDA_VERSION --channel=pytorch && \
+    conda env update --file environment.yml && \
+    rm environment.yml && \

 # Disable cache
-    pip config set global.cache-dir false && \
-    pip install "pip>20.1" -U
+    conda install "pip>20.1" && \
+    pip config set global.cache-dir false

-ENV LD_LIBRARY_PATH="/home/containeruser/miniconda/envs/$CONDA_ENV/lib:$LD_LIBRARY_PATH"
+ENV LD_LIBRARY_PATH="${WORKDIR}/miniconda/envs/${CONDA_ENV}/lib:$LD_LIBRARY_PATH"
 # if you want this environment to be the default one, uncomment the following line:
 ENV CONDA_DEFAULT_ENV=${CONDA_ENV}

+# Get package
+COPY --chown=flash ./ ./pytorch-lightning/
+
 # install dependencies
-RUN pip install torch==$PYTORCH_VERSION  -f https://download.pytorch.org/whl/torch_stable.html && \
-    git clone https://github.com/PyTorchLightning/pytorch-lightning.git --single-branch --branch $LIGHTNING_VERSION && \
-    pip install ./pytorch-lightning && \
-    pip install -r pytorch-lightning/requirements/extra.txt && \
-    rm -rf pytorch-lightning
+RUN \
+    if [ -z $LIGHTNING_VERSION ] ; then \
+        pip install ./pytorch-lightning --upgrade-strategy only-if-needed ; \
+        rm -rf pytorch-lightning ; \
+    else \
+        rm -rf pytorch-lightning ; \
+        pip install git+https://github.com/PyTorchLightning/pytorch-lightning.git@${LIGHTNING_VERSION} --upgrade-strategy only-if-needed ; \
+    fi

 RUN python --version && \
    pip --version && \
--- a/environment.yml
+++ b/environment.yml
@ -16,7 +16,7 @@
 # Usage: `conda env update -f environment.yml`

 name:
-    pl-env
+    lightning

 channels:
    - conda-forge
@ -34,23 +34,6 @@ dependencies:
    - fsspec>=0.8.0
    - nvidia-apex

-    # For dev and testing
-    - black==19.10b0
-    - check-manifest
-    - cloudpickle>=1.2
-    - coverage
-    - codecov>=2.1
-    - pytest>=3.0.5
-    - pytest-cov
-    - pytest-flake8
-    - flake8
-    - flake8-black
-    - autopep8
-    - twine==1.13.0
-    - pillow<7.0.0
-    - scikit-image
-    - nltk>=3.3
-
    # Optional
    - scikit-learn>=0.20.0
    - matplotlib>=3.1.1
@ -63,5 +46,5 @@ dependencies:
        - comet_ml>=3.1.12
        - wandb>=0.8.21
        - neptune-client>=0.4.109
-        - horovod>=0.19.1
+        - horovod>=0.19.2
        - onnxruntime>=1.3.0
--- a/requirements/test.txt
+++ b/requirements/test.txt
@ -1,9 +1,9 @@
-coverage
+coverage>=5.0
 codecov>=2.1
-pytest>=3.0.5
+pytest>=5.0
 pytest-cov
 pytest-flake8
-flake8
+flake8>=3.6
 flake8-black
 check-manifest
 twine==1.13.0
--- a/setup.py
+++ b/setup.py
@ -63,13 +63,13 @@ def load_long_description():
 # From remote, use like `pip install pytorch-lightning[dev, docs]`
 # From local copy of repo, use like `pip install ".[dev, docs]"`
 extras = {
-    'docs': load_requirements(file_name='docs.txt'),
+    # 'docs': load_requirements(file_name='docs.txt'),
    'examples': load_requirements(file_name='examples.txt'),
    'extra': load_requirements(file_name='extra.txt'),
    'test': load_requirements(file_name='test.txt')
 }
 extras['dev'] = extras['extra'] + extras['test']
-extras['all'] = extras['dev'] + extras['examples'] + extras['docs']
+extras['all'] = extras['dev'] + extras['examples']  # + extras['docs']

 # https://packaging.python.org/discussions/install-requires-vs-requirements /
 # keep the meta-data here for simplicity in reading this file... it's not obvious