From 93d4bd6c2ee054b71752fa2fa978611100bc128b Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 22 Jan 2015 22:25:18 +1100 Subject: [PATCH] * Add test for ). in tokenizer --- tests/test_tokenizer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index c5b50041f..786b9fb7a 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -110,6 +110,11 @@ def test_cnts6(EN): words = [t.string for t in tokens] assert len(words) == 6 +def test_bracket_period(EN): + text = u'(And a 6a.m. run through Washington Park).' + tokens = EN(text) + assert tokens[len(tokens) - 1].string == u'.' + #def test_cnts7(): # text = 'But then the 6,000-year ice age came...' # tokens = EN.tokenize(text)