try fix: Docker with Conda & PT 1.8 (#5842)

* ci * ver * list * pt * nk * ch * 4.9
2021-02-09 09:22:35 +01:00 · 2021-02-09 09:22:35 +01:00 · 937f11c05b
parent 774e9bd0d5
commit 937f11c05b
6 changed files with 26 additions and 20 deletions
--- a/.github/workflows/ci_dockers.yml
+++ b/.github/workflows/ci_dockers.yml
@ -13,9 +13,8 @@ on: # Trigger the workflow on push or pull request, but only for the master bran
      - "requirements/*.txt"
      - "environment.yml"
      - "requirements.txt"
-      - ".github/workflows/ci_dockers.yml"
+      - ".github/workflows/*docker*.yml"
      - ".github/workflows/events-nightly.yml"
-      - ".github/workflows/release-docker.yml"
      - "setup.py"

 jobs:
@ -131,8 +130,6 @@ jobs:
      - run: |
          cuda=$(python -c "print(10.2 if float(${{matrix.pytorch_version}}) > 1.4 else 10.1)" 2>&1)
          echo "::set-output name=CUDA::$cuda"
-          channel=$(python -c "print('pytorch-nightly' if float(${{matrix.pytorch_version}}) > 1.7 else 'pytorch')" 2>&1)
-          echo "::set-output name=CHANNEL::$channel"
        id: extend

      # https://github.com/docker/setup-buildx-action
@ -145,7 +142,6 @@ jobs:
          build-args: |
            PYTHON_VERSION=${{ matrix.python_version }}
            PYTORCH_VERSION=${{ matrix.pytorch_version }}
-            PYTORCH_CHANNEL=${{ steps.extend.outputs.CHANNEL }}
            CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
          file: dockers/base-conda/Dockerfile
          push: false
--- a/.github/workflows/events-nightly.yml
+++ b/.github/workflows/events-nightly.yml
@ -101,8 +101,6 @@ jobs:
      - run: |
          cuda=$(python -c "print(10.2 if float(${{matrix.pytorch_version}}) > 1.4 else 10.1)" 2>&1)
          echo "::set-output name=CUDA::$cuda"
-          channel=$(python -c "print('pytorch-nightly' if float(${{matrix.pytorch_version}}) > 1.7 else 'pytorch')" 2>&1)
-          echo "::set-output name=CHANNEL::$channel"
        id: extend

      - name: Publish CUDA to Docker Hub
@ -127,7 +125,6 @@ jobs:
          build-args: |
            PYTHON_VERSION=${{ matrix.python_version }}
            PYTORCH_VERSION=${{ matrix.pytorch_version }}
-            PYTORCH_CHANNEL=${{ steps.extend.outputs.CHANNEL }}
            CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
          file: dockers/base-conda/Dockerfile
          push: true
--- a/dockers/README.md
+++ b/dockers/README.md
@ -20,6 +20,16 @@ docker image build \
    --build-arg PYTORCH_VERSION=1.6 \
    .
 ```
+or nightly version from Coda
+```bash
+git clone <git-repository>
+docker image build \
+    -t pytorch-lightning:py3.7-pt1.8 \
+    -f dockers/base-conda/Dockerfile \
+    --build-arg PYTHON_VERSION=3.7 \
+    --build-arg PYTORCH_VERSION=1.8 \
+    .
+```

 To run your docker use

--- a/dockers/base-conda/Dockerfile
+++ b/dockers/base-conda/Dockerfile
@ -13,10 +13,10 @@
 # limitations under the License.

 # Existing images:
-# --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.8 --build-arg PYTORCH_CHANNEL=pytorch-nightly
-# --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.6 --build-arg PYTORCH_CHANNEL=pytorch
-# --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.5 --build-arg PYTORCH_CHANNEL=pytorch
-# --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.4 --build-arg PYTORCH_CHANNEL=pytorch
+# --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.8
+# --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.6
+# --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.5
+# --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.4

 ARG CUDNN_VERSION=8
 ARG CUDA_VERSION=10.2
@ -27,8 +27,7 @@ FROM nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu18.04

 ARG PYTHON_VERSION=3.7
 ARG PYTORCH_VERSION=1.6
-ARG PYTORCH_CHANNEL=pytorch
-ARG CONDA_VERSION=4.7.12
+ARG CONDA_VERSION=4.9.2

 SHELL ["/bin/bash", "-c"]

@ -47,7 +46,7 @@ RUN apt-get update -qq && \

 # Install conda and python.
 # NOTE new Conda does not forward the exit status... https://github.com/conda/conda/issues/8385
-    curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh  && \
+    curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_${CONDA_VERSION}-Linux-x86_64.sh && \
    chmod +x ~/miniconda.sh && \
    ~/miniconda.sh -b && \
    rm ~/miniconda.sh && \
@ -76,13 +75,13 @@ ENV \
 COPY environment.yml environment.yml

 # conda init
-RUN conda create -y --name $CONDA_ENV python=${PYTHON_VERSION} pytorch=${PYTORCH_VERSION} cudatoolkit=${CUDA_VERSION} -c ${PYTORCH_CHANNEL} && \
+RUN conda create -y --name $CONDA_ENV python=${PYTHON_VERSION} pytorch=${PYTORCH_VERSION} cudatoolkit=${CUDA_VERSION} -c pytorch -c pytorch-test -c pytorch-nightly && \
    conda init bash && \
    # NOTE: this requires that the channel is presented in the yaml before packages
    # replace channel to nigtly if needed, fix PT version and remove Horovod as it will be installed later
-    python -c "fname = 'environment.yml' ; req = open(fname).read().replace('pytorch', '${PYTORCH_CHANNEL}', 1) ; open(fname, 'w').write(req)" && \
    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- python[>=]+[\d\.]+', '# - python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- pytorch[>=]+[\d\.]+', '# - pytorch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
+    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- horovod[>=]+[\d\.]+', '# - horovod', open(fname).read()) ; open(fname, 'w').write(req)" && \
    python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if 'horovod' not in ln])" && \
    cat environment.yml && \
    conda env update --name $CONDA_ENV --file environment.yml && \
@ -99,6 +98,8 @@ COPY ./requirements/extra.txt requirements-extra.txt
 COPY ./requirements/test.txt requirements-test.txt

 RUN \
+    pip list | grep torch && \
+    python -c "import torch; print(torch.__version__)" && \
    # Install remaining requirements
    pip install -r requirements-extra.txt --no-cache-dir && \
    pip install -r requirements-test.txt --no-cache-dir && \
--- a/dockers/base-xla/Dockerfile
+++ b/dockers/base-xla/Dockerfile
@ -23,7 +23,7 @@ ARG XLA_VERSION=1.6

 SHELL ["/bin/bash", "-c"]

-ARG CONDA_VERSION=4.7.12
+ARG CONDA_VERSION=4.9.2
 # for skipping configurations
 ENV \
    DEBIAN_FRONTEND=noninteractive \
@ -44,7 +44,7 @@ RUN apt-get update -qq && \
    && \
 # Install conda and python.
 # NOTE new Conda does not forward the exit status... https://github.com/conda/conda/issues/8385
-    curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh  && \
+    curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_${CONDA_VERSION}-Linux-x86_64.sh && \
    chmod +x ~/miniconda.sh && \
    ~/miniconda.sh -b && \
    rm ~/miniconda.sh && \
--- a/environment.yml
+++ b/environment.yml
@ -21,6 +21,8 @@ name:
 channels:
    - conda-forge
    - pytorch
+    - pytorch-test
+    - pytorch-nightly

 dependencies:
    - python>=3.6
@ -49,6 +51,6 @@ dependencies:
        - comet_ml>=3.1.12
        - wandb>=0.8.21
        - neptune-client>=0.4.109
-        - horovod>=0.19.2
+        - horovod>=0.20.2
        - onnxruntime>=1.3.0
        - gym>=0.17.0