diff --git a/spacy/cli/validate.py b/spacy/cli/validate.py
index 2d8cba891..d675c4b95 100644
--- a/spacy/cli/validate.py
+++ b/spacy/cli/validate.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals, print_function
 import requests
 import pkg_resources
 from pathlib import Path
+import sys
 
 from ..compat import path2str, locale_escape
 from ..util import prints, get_data_path, read_json
@@ -62,6 +63,9 @@ def validate(cmd):
                "them from the data directory. Data path: {}"
                .format(path2str(get_data_path())))
 
+    if incompat_models or incompat_links:
+        sys.exit(1)
+
 
 def get_model_links(compat):
     links = {}