diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py
index 8161ddf45..5d044e617 100644
--- a/spacy/cli/debug_data.py
+++ b/spacy/cli/debug_data.py
@@ -121,6 +121,8 @@ def debug_data(
     msg.text("{} training docs".format(len(train_docs)))
     msg.text("{} evaluation docs".format(len(dev_docs)))
 
+    if not len(dev_docs):
+        msg.fail("No evaluation docs")
     overlap = len(train_texts.intersection(dev_texts))
     if overlap:
         msg.warn("{} training examples also in evaluation data".format(overlap))