Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

* Add test for Issue #154: Additional whitespace introduced when string ends with a whitespace token.

This commit is contained in:
Matthew Honnibal 2016-01-16 16:41:26 +01:00
parent 3dc398b727
commit 223d2b3484
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spacy/tests/tokenizer/test_tokenizer.py
View File

@ -142,6 +142,12 @@ def test_ie(en_tokenizer):
assert len(tokens) == 6 assert len(tokens) == 6
assert tokens[3].orth_ == "i.e." assert tokens[3].orth_ == "i.e."
def test_two_whitespace(en_tokenizer):
orig_str = u'there are 2 spaces after this '
tokens = en_tokenizer(orig_str)
assert repr(tokens.text_with_ws) == repr(orig_str)
#def test_cnts7(): #def test_cnts7():
# text = 'But then the 6,000-year ice age came...' # text = 'But then the 6,000-year ice age came...'
# tokens = EN.tokenize(text) # tokens = EN.tokenize(text)