Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add xfailing test for orth mismatch in retokenizer.split

This commit is contained in:
Ines Montani 2019-02-15 13:55:04 +01:00
parent 819768483f
commit 1aa57690dc
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
spacy/tests/doc/test_doc_spilt.py
View File

@ -112,3 +112,17 @@ def test_spans_sentence_update_after_merge(en_vocab):
sent1, sent2 = list(doc.sents) sent1, sent2 = list(doc.sents)
assert len(sent1) == init_len + 1 assert len(sent1) == init_len + 1
assert len(sent2) == init_len2 + 1 assert len(sent2) == init_len2 + 1
@pytest.mark.xfail
def test_split_orths_mismatch(en_vocab):
"""Test that the regular retokenizer.split raises an error if the orths
don't match the original token text. There might still be a method that
allows this, but for the default use cases, merging and splitting should
always conform with spaCy's non-destructive tokenization policy. Otherwise,
it can lead to very confusing and unexpected results.
"""
doc = Doc(en_vocab, words=["LosAngeles", "start", "."])
with pytest.raises(ValueError):
with doc.retokenize() as retokenizer:
retokenizer.split(doc[0], ["L", "A"], [0, -1])