diff --git a/spacy/tests/tokenizer/test_explain.py b/spacy/tests/tokenizer/test_explain.py index 5b4eeca16..78932f653 100644 --- a/spacy/tests/tokenizer/test_explain.py +++ b/spacy/tests/tokenizer/test_explain.py @@ -85,6 +85,18 @@ def test_tokenizer_explain_special_matcher(en_vocab): assert tokens == explain_tokens +def test_tokenizer_explain_special_matcher_whitespace(en_vocab): + rules = {":]": [{"ORTH": ":]"}]} + tokenizer = Tokenizer( + en_vocab, + rules=rules, + ) + text = ": ]" + tokens = [t.text for t in tokenizer(text)] + explain_tokens = [t[1] for t in tokenizer.explain(text)] + assert tokens == explain_tokens + + @hypothesis.strategies.composite def sentence_strategy(draw: hypothesis.strategies.DrawFn, max_n_words: int = 4) -> str: """ @@ -123,6 +135,9 @@ def test_tokenizer_explain_fuzzy(lang: str, sentence: str) -> None: """ tokenizer: Tokenizer = spacy.blank(lang).tokenizer - tokens = [t.text for t in tokenizer(sentence) if not t.is_space] + # Tokenizer.explain is not intended to handle whitespace or control + # characters in the same way as Tokenizer + sentence = re.sub(r"\s+", " ", sentence).strip() + tokens = [t.text for t in tokenizer(sentence)] debug_tokens = [t[1] for t in tokenizer.explain(sentence)] assert tokens == debug_tokens, f"{tokens}, {debug_tokens}, {sentence}" diff --git a/spacy/tokenizer.pyx b/spacy/tokenizer.pyx index a239eaf45..6f2b10734 100644 --- a/spacy/tokenizer.pyx +++ b/spacy/tokenizer.pyx @@ -730,9 +730,16 @@ cdef class Tokenizer: if i in spans_by_start: span = spans_by_start[i] exc = [d[ORTH] for d in special_cases[span.label_]] - for j, orth in enumerate(exc): - final_tokens.append((f"SPECIAL-{j + 1}", self.vocab.strings[orth])) - i += len(span) + # The phrase matcher can overmatch for tokens separated by + # spaces in the text but not in the underlying rule, so skip + # cases where the texts aren't identical + if span.text != "".join([self.vocab.strings[orth] for orth in exc]): + final_tokens.append(tokens[i]) + i += 1 + else: + for j, orth in enumerate(exc): + final_tokens.append((f"SPECIAL-{j + 1}", self.vocab.strings[orth])) + i += len(span) else: final_tokens.append(tokens[i]) i += 1