diff --git a/spacy/pipeline/pipes.pyx b/spacy/pipeline/pipes.pyx
index 536c2a8a5..b3fa77732 100644
--- a/spacy/pipeline/pipes.pyx
+++ b/spacy/pipeline/pipes.pyx
@@ -394,12 +394,11 @@ class Tagger(Pipe):
                     new_tag_map[tag] = orig_tag_map[tag]
                 else:
                     new_tag_map[tag] = {POS: X}
-        # TODO: do we still need this?
-        if "_SP" in orig_tag_map:
-            new_tag_map["_SP"] = orig_tag_map["_SP"]
 
         cdef Vocab vocab = self.vocab
         if new_tag_map:
+            if "_SP" in orig_tag_map:
+                new_tag_map["_SP"] = orig_tag_map["_SP"]
             vocab.morphology = Morphology(vocab.strings, new_tag_map,
                                           vocab.morphology.lemmatizer,
                                           exc=vocab.morphology.exc)