From 47a4371fea4427ecf362409acbc4b6b7f8565dfd Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 18 Feb 2015 06:37:04 -0500 Subject: [PATCH] * Upd tokenizer with i.e. tests --- tests/test_tokenizer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index 259719b71..6c3ce3271 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -119,6 +119,13 @@ def test_bracket_period(EN): tokens = EN(text) assert tokens[len(tokens) - 1].orth_ == u'.' + +def test_ie(EN): + text = u"It's mediocre i.e. bad." + tokens = EN(text) + assert len(tokens) == 6 + assert tokens[3].orth_ == "i.e." + #def test_cnts7(): # text = 'But then the 6,000-year ice age came...' # tokens = EN.tokenize(text)