diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml
index 8f49b33460..3085aec7ad 100644
--- a/.azure-pipelines/gpu-tests.yml
+++ b/.azure-pipelines/gpu-tests.yml
@@ -50,7 +50,7 @@ jobs:
 
     - bash: |
         python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)"
-        pip install fairscale>=0.3.4
+        pip install fairscale==0.4.0
         pip install deepspeed==0.5.4
         pip install . --requirement requirements/devel.txt
         pip list
diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile
index c1de7babf6..1e2618eb00 100644
--- a/dockers/base-cuda/Dockerfile
+++ b/dockers/base-cuda/Dockerfile
@@ -108,11 +108,11 @@ RUN \
 
 RUN \
     # install FairScale
-    pip install fairscale>=0.3.4
+    pip install fairscale==0.4.0
 
 RUN \
     # install DeepSpeed
-    pip install deepspeed==0.4.0
+    pip install deepspeed==0.5.4
 
 RUN \
     # Show what we have