mirror of https://github.com/explosion/spaCy.git
* Upd tokenizer with i.e. tests
This commit is contained in:
parent
ba1d3ddd7f
commit
47a4371fea
|
@ -119,6 +119,13 @@ def test_bracket_period(EN):
|
|||
tokens = EN(text)
|
||||
assert tokens[len(tokens) - 1].orth_ == u'.'
|
||||
|
||||
|
||||
def test_ie(EN):
|
||||
text = u"It's mediocre i.e. bad."
|
||||
tokens = EN(text)
|
||||
assert len(tokens) == 6
|
||||
assert tokens[3].orth_ == "i.e."
|
||||
|
||||
#def test_cnts7():
|
||||
# text = 'But then the 6,000-year ice age came...'
|
||||
# tokens = EN.tokenize(text)
|
||||
|
|
Loading…
Reference in New Issue