* Test hyphenation etc

2014-10-14 20:26:16 +11:00 · 2014-10-14 20:26:16 +11:00 · 31aad7c08a
parent 99f5e59286
commit 31aad7c08a
1 changed files with 17 additions and 0 deletions
--- a/tests/test_infix.py
+++ b/tests/test_infix.py
@ -0,0 +1,17 @@
+from __future__ import unicode_literals
+
+import pytest
+
+from spacy.en import EN
+
+
+def test_hyphen():
+    tokens = EN.tokenize('best-known')
+    assert len(tokens) == 3
+
+
+def test_period():
+    tokens = EN.tokenize('best.Known')
+    assert len(tokens) == 3
+    tokens = EN.tokenize('zombo.com')
+    assert len(tokens) == 1