From 7d2964f6739d0f97869e163e0f34c3ff23909b38 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 9 Jul 2015 13:31:40 +0200 Subject: [PATCH] * Test that whitespace is not assigned a tag --- tests/tagger/test_spaces.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/tagger/test_spaces.py diff --git a/tests/tagger/test_spaces.py b/tests/tagger/test_spaces.py new file mode 100644 index 000000000..c3052160e --- /dev/null +++ b/tests/tagger/test_spaces.py @@ -0,0 +1,26 @@ +"""Ensure spaces are assigned the POS tag SPACE""" + + +from __future__ import unicode_literals +from spacy.parts_of_speech import SPACE + +import pytest + + + +@pytest.fixture +def tagged(EN): + string = u'Some\nspaces are\tnecessary.' + tokens = EN(string, tag=True, parse=False) + return tokens + +def test_spaces(tagged): + assert tagged[0].pos != SPACE + assert tagged[0].pos_ != 'SPACE' + assert tagged[1].pos == SPACE + assert tagged[1].pos_ == 'SPACE' + assert tagged[1].tag_ == 'SP' + assert tagged[2].pos != SPACE + assert tagged[3].pos != SPACE + assert tagged[4].pos == SPACE +