diff --git a/requirements/extra.txt b/requirements/extra.txt
index 2fd1e94a91..3d51d044e4 100644
--- a/requirements/extra.txt
+++ b/requirements/extra.txt
@@ -3,8 +3,6 @@
 matplotlib>=3.1.1
 horovod>=0.20.2  # no need to install with [pytorch] as pytorch is already installed
 omegaconf>=2.0.1
-# scipy>=0.13.3
-scikit-learn>=0.22.2
 torchtext>=0.3.1, <0.7  # TODO: temporary fix fix for compatibility
 onnx>=1.7.0
 onnxruntime>=1.3.0
diff --git a/requirements/test.txt b/requirements/test.txt
index d553bb9b78..74c62f82c7 100644
--- a/requirements/test.txt
+++ b/requirements/test.txt
@@ -7,6 +7,8 @@ flake8>=3.6
 flake8-black
 check-manifest
 twine==1.13.0
+# scipy>=0.13.3
+scikit-learn>=0.22.2
 scikit-image>=0.17.2
 black>=20.8b1
 isort>=5.6.4