diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml
index f08e54d136..a3b9f42ef9 100644
--- a/.github/workflows/ci-testing.yml
+++ b/.github/workflows/ci-testing.yml
@@ -101,9 +101,9 @@ jobs:
       shell: bash
 
     - name: Reinstall Horovod if necessary
-      if: runner.os != 'windows' && matrix.python-version != '3.8'
+      if: runner.os != 'windows'
       run: |
-        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')")
+        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
         if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
           pip uninstall -y horovod
           HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install --no-cache-dir $(grep "horovod" requirements/extra.txt)
diff --git a/.github/workflows/pt-conda.yml b/.github/workflows/pt-conda.yml
index d4864d4704..98c9a038ed 100644
--- a/.github/workflows/pt-conda.yml
+++ b/.github/workflows/pt-conda.yml
@@ -59,9 +59,9 @@ jobs:
           use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
 
     - name: Reinstall Horovod if necessary
-      if: runner.os != 'windows' && matrix.python-version != '3.8'
+      if: runner.os != 'windows'
       run: |
-        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')")
+        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
         if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
           pip uninstall -y horovod
           HOROVOD_BUILD_ARCH_FLAGS="-mfma" pip install --no-cache-dir $(grep "horovod" requirements/extra.txt)
diff --git a/requirements/extra.txt b/requirements/extra.txt
index 113c35b466..191d24125a 100644
--- a/requirements/extra.txt
+++ b/requirements/extra.txt
@@ -7,7 +7,7 @@ test_tube>=0.7.5
 wandb>=0.8.21
 matplotlib>=3.1.1
 # no need to install with [pytorch] as pytorch is already installed and torchvision is required only for Horovod examples
-horovod>=0.19.1
+horovod>=0.19.2
 omegaconf>=2.0.0
 # scipy>=0.13.3
 scikit-learn>=0.20.0