diff --git a/.drone.yml b/.drone.yml
index 40a3746bd8..8760b9b458 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -46,7 +46,7 @@ steps:
     - pip install pip -U
     - pip --version
     - nvidia-smi
-    #- bash ./tests/install_AMP.sh
+    #- bash ./requirements/install_AMP.sh
     - apt-get update && apt-get install -y cmake
     - pip install -r ./requirements/base.txt --user -q --upgrade-strategy only-if-needed
     - pip install -r ./requirements/devel.txt --user -q --upgrade-strategy only-if-needed
diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml
index aca7a835ae..ce25c75869 100644
--- a/.github/workflows/ci_dockers.yml
+++ b/.github/workflows/ci_dockers.yml
@@ -14,11 +14,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: [3.6, 3.8]
-        pytorch_version: [1.3, 1.5, 1.7]
-        exclude:
-          - python_version: 3.8
-            pytorch_version: 1.3
+        python_version: [3.6]
+        pytorch_version: [1.3, 1.7]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -62,6 +59,9 @@ jobs:
             pytorch_version: 1.7
             pytorch_channel: pytorch-nightly
           - python_version: 3.8
+            pytorch_version: 1.6
+            pytorch_channel: pytorch
+          - python_version: 3.6
             pytorch_version: 1.5
             pytorch_channel: pytorch
     steps:
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index bbb5c71bc1..549b844876 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -23,7 +23,7 @@ jobs:
         python -m pip install --user --upgrade setuptools wheel
     - name: Build
       run: |
-        python prepare-nightly_version.py
+        python .github/prepare-nightly_version.py
         python setup.py sdist bdist_wheel
         ls -lh dist/
 
@@ -47,7 +47,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Publish Master to Docker
+      - name: Publish XLA to Docker Hub
         # publish master
         uses: docker/build-push-action@v1.1.0
         with:
@@ -83,7 +83,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Publish Master to Docker
+      - name: Publish CUDA to Docker Hub
         # publish master
         uses: docker/build-push-action@v1.1.0
         with:
@@ -95,32 +95,7 @@ jobs:
           tags: "base-cuda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}"
         timeout-minutes: 55
 
-
-  docker-Conda:
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        python_version: [ 3.6, 3.7, 3.8 ]
-        pytorch_version: [ 1.3, 1.4, 1.5, 1.6, 1.7 ]
-        pytorch_channel: [ "pytorch", "pytorch-nightly" ]
-        # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-including-new-combinations
-        exclude:
-          - pytorch_version: 1.7
-            pytorch_channel: pytorch
-          - pytorch_version: 1.3
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.4
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.5
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.6
-            pytorch_channel: pytorch-nightly
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Publish Master to Docker
+      - name: Publish Conda to Docker Hub
         # publish master
         uses: docker/build-push-action@v1.1.0
         with:
diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml
index 1cc76e7792..7579fc0b19 100644
--- a/.github/workflows/pypi-release.yml
+++ b/.github/workflows/pypi-release.yml
@@ -7,10 +7,10 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   release:
     types: [created]
 
-# based on https://github.com/pypa/gh-action-pypi-publish
 
 jobs:
-  build:
+  # based on https://github.com/pypa/gh-action-pypi-publish
+  build-publish:
     runs-on: ubuntu-20.04
 
     steps:
diff --git a/.pyrightconfig.json b/.pyrightconfig.json
index ade1db6558..cb14993e2c 100644
--- a/.pyrightconfig.json
+++ b/.pyrightconfig.json
@@ -35,6 +35,7 @@
     "pytorch_lightning/trainer/connectors/checkpoint_connector.py",
     "pytorch_lightning/trainer/connectors/data_connector.py",
     "pytorch_lightning/trainer/connectors/logger_connector.py",
+    "pytorch_lightning/distributed/dist.py",
     "pytorch_lightning/tuner",
     "pytorch_lightning/plugins"
   ],
diff --git a/README.md b/README.md
index 4075b6ebb0..5c1a31545e 100644
--- a/README.md
+++ b/README.md
@@ -82,8 +82,6 @@ Get started with our [3 steps guide](https://pytorch-lightning.readthedocs.io/en
 | System / PyTorch ver. | 1.3 (min. req.)* | 1.4 | 1.5 | 1.6 (latest) | 1.7 (nightly) |
 | :---: | :---: | :---: | :---: | :---: | :---: |
 | Conda py3.7 [linux] | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) |
-| Linux py3.7 [GPUs**] | - | - | - | [![Build Status](http://35.192.60.23/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://35.192.60.23/PyTorchLightning/pytorch-lightning) | - |
-| Linux py3.6 / py3.7 [TPUs***] | - | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | - |
 | Linux py3.7 [GPUs**] | - | - |[![Build Status](http://104.154.220.231/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://104.154.220.231/PyTorchLightning/pytorch-lightning) | - | - |
 | Linux py3.7 [TPUs***] | - | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | - |
 | Linux py3.6 / py3.7 / py3.8 | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |
diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile
index 2408fb5570..2090320f3c 100644
--- a/dockers/base-cuda/Dockerfile
+++ b/dockers/base-cuda/Dockerfile
@@ -1,3 +1,17 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Existing images:
 # --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.7 --build-arg PYTORCH_CHANNEL=pytorch-nightly --build-arg CUDA_VERSION=10.1
 # --build-arg PYTHON_VERSION=3.7 --build-arg PYTORCH_VERSION=1.6 --build-arg PYTORCH_CHANNEL=pytorch --build-arg CUDA_VERSION=10.1
@@ -56,15 +70,17 @@ ENV CONDA_ENV=lightning
 COPY environment.yml environment.yml
 
 # conda init
-RUN conda create -y --name $CONDA_ENV "python=$PYTHON_VERSION" "pytorch=$PYTORCH_VERSION" torchvision "cudatoolkit=$CUDA_VERSION" --channel=$PYTORCH_CHANNEL && \
+RUN conda create -y --name $CONDA_ENV "cudatoolkit=$CUDA_VERSION" && \
     conda init bash && \
     # NOTE: this requires that the channel is presented in the yaml before packages
-    # replace channel to nigtly if neede, fix PT version and remove Horovod as it will be installe later
-    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'torch>=[\d\.]+', 'torch=${PYTORCH_VERSION}', open(fname).read().replace('pytorch', '${PYTORCH_CHANNEL}', 1)) ; open(fname, 'w').writelines([l for l in req if 'horovod' not in l])" && \
+    # replace channel to nigtly if needed, fix PT version and remove Horovod as it will be installe later
+    python -c "fname = 'environment.yml' ; req = open(fname).read().replace('pytorch', '${PYTORCH_CHANNEL}', 1) ; open(fname, 'w').write(req)" && \
+    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python[>=]+[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
+    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'torch[>=]+[\d\.]+', 'torch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
+    python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if 'horovod' not in ln])" && \
     cat environment.yml && \
     conda env update --file environment.yml && \
     conda clean -ya && \
-    # conda config --set auto_activate_base true && \
     rm environment.yml
 
 ENV PATH /root/miniconda3/envs/${CONDA_ENV}/bin:$PATH
@@ -74,7 +90,6 @@ ENV CONDA_DEFAULT_ENV=${CONDA_ENV}
 
 COPY ./requirements/extra.txt requirements-extra.txt
 COPY ./requirements/test.txt requirements-tests.txt
-COPY ./requirements/examples.txt requirements-examples.txt
 
 RUN \
     # Disable cache
@@ -82,18 +97,21 @@ RUN \
     #echo ". ${WORKDIR}/miniconda/etc/profile.d/conda.sh" >> ~/.bashrc && \
     #echo "conda activate ${CONDA_ENV}" >> ~/.bashrc && \
     #source ~/.bashrc && \
+    # install NVIDIA AMP
+    git clone https://github.com/NVIDIA/apex && \
+    pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./apex && \
+    rm -rf apex && \
     # filter only Horovod
     python -c "fname = 'requirements-extra.txt' ; req = open(fname).readlines() ; open(fname, 'w').writelines([l for l in req if 'horovod' in l])" && \
     # Install all requirements
     MAKEFLAGS="-j$(nproc)" ; pip install -r requirements-extra.txt && \
     pip install -r requirements-tests.txt --upgrade-strategy only-if-needed && \
-    pip install -r requirements-examples.txt --upgrade-strategy only-if-needed && \
     rm requirements*
 
 RUN \
     # Show what we have
     pip --version && \
     conda info && \
-    conda list && \
     pip list && \
-    python -c "import torch; print(torch.__version__)"
\ No newline at end of file
+    python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \
+    python -c "import torch; assert torch.__version__[:3] == '$PYTORCH_VERSION', torch.__version__"
\ No newline at end of file
diff --git a/dockers/base-xla/Dockerfile b/dockers/base-xla/Dockerfile
index 9e659338ba..6297d6acc2 100644
--- a/dockers/base-xla/Dockerfile
+++ b/dockers/base-xla/Dockerfile
@@ -1,15 +1,29 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM google/cloud-sdk:slim
 
-# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile .
+# CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile . --build-arg PYTHON_VERSION=3.6
 # This Dockerfile installs pytorch/xla 3.7 wheels. There are also 3.6 wheels available; see below.
 ARG PYTHON_VERSION=3.7
-ARG XLA_VERSION="1.6"
+ARG XLA_VERSION=1.6
 
 SHELL ["/bin/bash", "-c"]
 
 # for skipping configurations
 ENV DEBIAN_FRONTEND=noninteractive
-ENV CONDA_ENV=pytorch-xla
+ENV CONDA_ENV=lightning
 
 # show system inforation
 RUN lsb_release -a && cat /etc/*-release
@@ -40,15 +54,28 @@ RUN apt-get update && \
 
 ENV PATH="/root/miniconda3/bin:$PATH"
 ENV LD_LIBRARY_PATH="/root/miniconda3/lib:$LD_LIBRARY_PATH"
+COPY environment.yml environment.yml
 
-RUN conda create -y --name $CONDA_ENV "python=$PYTHON_VERSION" && \
+RUN conda create -y --name $CONDA_ENV && \
     conda init bash && \
-    conda install -y mkl && \
+    # replace channel to nigtly if neede, fix PT version and remove Horovod as it will be installe later
+    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python>=[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
+    python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if not any(n in ln for n in ['pytorch>', 'horovod'])])" && \
+    cat environment.yml && \
+    conda env update --file environment.yml && \
+    conda clean -ya && \
+    rm environment.yml
+
+ENV PATH /root/miniconda3/envs/${CONDA_ENV}/bin:$PATH
+ENV LD_LIBRARY_PATH="/root/miniconda3/envs/${CONDA_ENV}/lib:$LD_LIBRARY_PATH"
+# if you want this environment to be the default one, uncomment the following line:
+ENV CONDA_DEFAULT_ENV=${CONDA_ENV}
 
 # Disable cache
+RUN pip --version && \
     pip config set global.cache-dir false && \
-    pip install "pip>20.1" -U  && \
 
+    conda remove pytorch torchvision && \
 # Install Pytorch XLA
     py_version=${PYTHON_VERSION/./} && \
     # Python 3.7 wheels are available. Replace cp36-cp36m with cp37-cp37m
@@ -58,27 +85,29 @@ RUN conda create -y --name $CONDA_ENV "python=$PYTHON_VERSION" && \
     pip install *.whl && \
     rm *.whl
 
-ENV LD_LIBRARY_PATH="/root/miniconda3/envs/$CONDA_ENV/lib:$LD_LIBRARY_PATH"
-# if you want this environment to be the default one, uncomment the following line:
-ENV CONDA_DEFAULT_ENV=${CONDA_ENV}
-
 # Get package
 COPY ./ ./pytorch-lightning/
 
 # Install pytorch-lightning dependencies.
 RUN \
+    python --version && \
 # Install PL dependencies
     cd pytorch-lightning && \
     # drop Torch as it was installed with XLA
-    python -c "fname = \"./requirements/base.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"torch\")] ; open(fname, \"w\").writelines(lines)" && \
+    python -c "fname = 'requirements/base.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('torch')] ; open(fname, 'w').writelines(lines)" && \
     # drop Horovod as it is not needed
-    python -c "fname = \"./requirements/extra.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"horovod\")] ; open(fname, \"w\").writelines(lines)" && \
+    python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" && \
     # drop TorchVision as it was installed with XLA
-    python -c "fname = \"./requirements/examples.txt\" ; lines = [line for line in open(fname).readlines() if not line.startswith(\"torchvision\")] ; open(fname, \"w\").writelines(lines)" && \
+    python -c "fname = 'requirements/examples.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('torchvision')] ; open(fname, 'w').writelines(lines)" && \
     pip install --requirement ./requirements/devel.txt --upgrade-strategy only-if-needed && \
     cd .. && \
     rm -rf pytorch-lightning && \
     rm -rf /root/.cache
 
-RUN pip --version && \
-    python -c "import torch; print(torch.__version__)"
+RUN \
+    # Show what we have
+    pip --version && \
+    conda info && \
+    pip list && \
+    python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \
+    python -c "import torch; ver = '$XLA_VERSION' ; ver = dict(nightly='1.7').get(ver, ver) ; assert torch.__version__[:3] == ver, torch.__version__"
diff --git a/dockers/conda/Dockerfile b/dockers/conda/Dockerfile
index 036e13a275..17ad4f9c7e 100644
--- a/dockers/conda/Dockerfile
+++ b/dockers/conda/Dockerfile
@@ -1,3 +1,17 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ARG PYTHON_VERSION=3.7
 ARG PYTORCH_VERSION=1.5
 
@@ -5,7 +19,7 @@ FROM pytorchlightning/pytorch_lightning:base-cuda-py${PYTHON_VERSION}-torch${PYT
 
 ARG LIGHTNING_VERSION=""
 
-COPY --chown=flash ./ ./pytorch-lightning/
+COPY ./ ./pytorch-lightning/
 
 # install dependencies
 RUN \
@@ -17,7 +31,7 @@ RUN \
         rm -rf pytorch-lightning ; \
     else \
         rm -rf pytorch-lightning ; \
-        pip install git+https://github.com/PyTorchLightning/pytorch-lightning.git@${LIGHTNING_VERSION} --upgrade-strategy only-if-needed ; \
+        pip install https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --upgrade-strategy only-if-needed ; \
     fi
 
 RUN python --version && \
diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile
index e4f8ef49df..79852b14a8 100644
--- a/dockers/tpu-tests/Dockerfile
+++ b/dockers/tpu-tests/Dockerfile
@@ -1,3 +1,17 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ARG PYTHON_VERSION=3.7
 ARG PYTORCH_VERSION=1.6
 
diff --git a/environment.yml b/environment.yml
index aa86ce0224..09eb997b78 100644
--- a/environment.yml
+++ b/environment.yml
@@ -24,22 +24,25 @@ channels:
 
 dependencies:
     - python>=3.6
-    - pip
+    - pip>20.1
     - numpy>=1.16.4
     - pytorch>=1.3
     - future>=0.17.1
     - PyYAML>=5.1
     - tqdm>=4.41.0
     - fsspec>=0.8.0
-    #- tensorboard>=2.2.0
+    #- tensorboard>=2.2.0  # not needed, already included in pytorch
 
     # Optional
-    - nvidia-apex
+    #- nvidia-apex  # missing for py3.8
     - scikit-learn>=0.20.0
     - matplotlib>=3.1.1
     - omegaconf>=2.0.0
     - torchtext>=0.3.1
 
+    # Examples
+    - torchvision>=0.4.1
+
     - pip:
         - test-tube>=0.7.5
         - mlflow>=1.0.0
@@ -48,3 +51,4 @@ dependencies:
         - neptune-client>=0.4.109
         - horovod>=0.19.2
         - onnxruntime>=1.3.0
+        - gym>=0.17.0
diff --git a/tests/install_AMP.sh b/requirements/install_AMP.sh
similarity index 100%
rename from tests/install_AMP.sh
rename to requirements/install_AMP.sh
diff --git a/tests/README.md b/tests/README.md
index ccd62301aa..7fd3c90c02 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -17,7 +17,7 @@ git clone https://github.com/PyTorchLightning/pytorch-lightning
 cd pytorch-lightning
 
 # install AMP support
-bash tests/install_AMP.sh
+bash requirements/install_AMP.sh
 
 # install dev deps
 pip install -r requirements/devel.txt