* Test hyphenation etc

This commit is contained in:
Matthew Honnibal 2014-10-14 20:26:16 +11:00
parent 99f5e59286
commit 31aad7c08a
1 changed files with 17 additions and 0 deletions

17
tests/test_infix.py Normal file
View File

@ -0,0 +1,17 @@
from __future__ import unicode_literals
import pytest
from spacy.en import EN
def test_hyphen():
tokens = EN.tokenize('best-known')
assert len(tokens) == 3
def test_period():
tokens = EN.tokenize('best.Known')
assert len(tokens) == 3
tokens = EN.tokenize('zombo.com')
assert len(tokens) == 1