2021-04-22 08:14:57 +00:00
|
|
|
import re
|
2021-12-04 19:34:48 +00:00
|
|
|
|
|
|
|
import numpy
|
|
|
|
import pytest
|
|
|
|
|
|
|
|
from spacy.lang.de import German
|
2023-06-14 15:48:41 +00:00
|
|
|
from spacy.lang.en import English
|
|
|
|
from spacy.symbols import ORTH
|
2021-12-04 19:34:48 +00:00
|
|
|
from spacy.tokenizer import Tokenizer
|
|
|
|
from spacy.tokens import Doc
|
|
|
|
from spacy.training import Example
|
2023-06-14 15:48:41 +00:00
|
|
|
from spacy.util import (
|
|
|
|
compile_infix_regex,
|
|
|
|
compile_prefix_regex,
|
|
|
|
compile_suffix_regex,
|
|
|
|
ensure_path,
|
|
|
|
)
|
2021-12-04 19:34:48 +00:00
|
|
|
from spacy.vocab import Vocab
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(743)
|
|
|
|
def test_issue743():
|
|
|
|
doc = Doc(Vocab(), ["hello", "world"])
|
|
|
|
token = doc[0]
|
|
|
|
s = set([token])
|
|
|
|
items = list(s)
|
|
|
|
assert items[0] is token
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(801)
|
|
|
|
@pytest.mark.skip(
|
|
|
|
reason="Can not be fixed unless with variable-width lookbehinds, cf. PR #3218"
|
|
|
|
)
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"text,tokens",
|
|
|
|
[
|
|
|
|
('"deserve,"--and', ['"', "deserve", ',"--', "and"]),
|
|
|
|
("exception;--exclusive", ["exception", ";--", "exclusive"]),
|
|
|
|
("day.--Is", ["day", ".--", "Is"]),
|
|
|
|
("refinement:--just", ["refinement", ":--", "just"]),
|
|
|
|
("memories?--To", ["memories", "?--", "To"]),
|
|
|
|
("Useful.=--Therefore", ["Useful", ".=--", "Therefore"]),
|
|
|
|
("=Hope.=--Pandora", ["=", "Hope", ".=--", "Pandora"]),
|
|
|
|
],
|
|
|
|
)
|
|
|
|
def test_issue801(en_tokenizer, text, tokens):
|
|
|
|
"""Test that special characters + hyphens are split correctly."""
|
|
|
|
doc = en_tokenizer(text)
|
|
|
|
assert len(doc) == len(tokens)
|
|
|
|
assert [t.text for t in doc] == tokens
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(1061)
|
|
|
|
def test_issue1061():
|
|
|
|
"""Test special-case works after tokenizing. Was caching problem."""
|
|
|
|
text = "I like _MATH_ even _MATH_ when _MATH_, except when _MATH_ is _MATH_! but not _MATH_."
|
|
|
|
tokenizer = English().tokenizer
|
|
|
|
doc = tokenizer(text)
|
|
|
|
assert "MATH" in [w.text for w in doc]
|
|
|
|
assert "_MATH_" not in [w.text for w in doc]
|
|
|
|
|
|
|
|
tokenizer.add_special_case("_MATH_", [{ORTH: "_MATH_"}])
|
|
|
|
doc = tokenizer(text)
|
|
|
|
assert "_MATH_" in [w.text for w in doc]
|
|
|
|
assert "MATH" not in [w.text for w in doc]
|
|
|
|
|
|
|
|
# For sanity, check it works when pipeline is clean.
|
|
|
|
tokenizer = English().tokenizer
|
|
|
|
tokenizer.add_special_case("_MATH_", [{ORTH: "_MATH_"}])
|
|
|
|
doc = tokenizer(text)
|
|
|
|
assert "_MATH_" in [w.text for w in doc]
|
|
|
|
assert "MATH" not in [w.text for w in doc]
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(1963)
|
|
|
|
def test_issue1963(en_tokenizer):
|
|
|
|
"""Test that doc.merge() resizes doc.tensor"""
|
|
|
|
doc = en_tokenizer("a b c d")
|
|
|
|
doc.tensor = numpy.ones((len(doc), 128), dtype="f")
|
|
|
|
with doc.retokenize() as retokenizer:
|
|
|
|
retokenizer.merge(doc[0:2])
|
|
|
|
assert len(doc) == 3
|
|
|
|
assert doc.tensor.shape == (3, 128)
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.skip(
|
|
|
|
reason="Can not be fixed without variable-width look-behind (which we don't want)"
|
|
|
|
)
|
|
|
|
@pytest.mark.issue(1235)
|
|
|
|
def test_issue1235():
|
|
|
|
"""Test that g is not split of if preceded by a number and a letter"""
|
|
|
|
nlp = English()
|
|
|
|
testwords = "e2g 2g 52g"
|
|
|
|
doc = nlp(testwords)
|
|
|
|
assert len(doc) == 5
|
|
|
|
assert doc[0].text == "e2g"
|
|
|
|
assert doc[1].text == "2"
|
|
|
|
assert doc[2].text == "g"
|
|
|
|
assert doc[3].text == "52"
|
|
|
|
assert doc[4].text == "g"
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(1242)
|
|
|
|
def test_issue1242():
|
|
|
|
nlp = English()
|
|
|
|
doc = nlp("")
|
|
|
|
assert len(doc) == 0
|
|
|
|
docs = list(nlp.pipe(["", "hello"]))
|
|
|
|
assert len(docs[0]) == 0
|
|
|
|
assert len(docs[1]) == 1
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(1257)
|
|
|
|
def test_issue1257():
|
|
|
|
"""Test that tokens compare correctly."""
|
|
|
|
doc1 = Doc(Vocab(), words=["a", "b", "c"])
|
|
|
|
doc2 = Doc(Vocab(), words=["a", "c", "e"])
|
|
|
|
assert doc1[0] != doc2[0]
|
|
|
|
assert not doc1[0] == doc2[0]
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(1375)
|
|
|
|
def test_issue1375():
|
|
|
|
"""Test that token.nbor() raises IndexError for out-of-bounds access."""
|
|
|
|
doc = Doc(Vocab(), words=["0", "1", "2"])
|
|
|
|
with pytest.raises(IndexError):
|
|
|
|
assert doc[0].nbor(-1)
|
|
|
|
assert doc[1].nbor(-1).text == "0"
|
|
|
|
with pytest.raises(IndexError):
|
|
|
|
assert doc[2].nbor(1)
|
|
|
|
assert doc[1].nbor(1).text == "2"
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(1488)
|
|
|
|
def test_issue1488():
|
|
|
|
"""Test that tokenizer can parse DOT inside non-whitespace separators"""
|
|
|
|
prefix_re = re.compile(r"""[\[\("']""")
|
|
|
|
suffix_re = re.compile(r"""[\]\)"']""")
|
|
|
|
infix_re = re.compile(r"""[-~\.]""")
|
|
|
|
simple_url_re = re.compile(r"""^https?://""")
|
|
|
|
|
|
|
|
def my_tokenizer(nlp):
|
|
|
|
return Tokenizer(
|
|
|
|
nlp.vocab,
|
|
|
|
{},
|
|
|
|
prefix_search=prefix_re.search,
|
|
|
|
suffix_search=suffix_re.search,
|
|
|
|
infix_finditer=infix_re.finditer,
|
|
|
|
token_match=simple_url_re.match,
|
|
|
|
)
|
|
|
|
|
|
|
|
nlp = English()
|
|
|
|
nlp.tokenizer = my_tokenizer(nlp)
|
|
|
|
doc = nlp("This is a test.")
|
|
|
|
for token in doc:
|
|
|
|
assert token.text
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(1494)
|
|
|
|
def test_issue1494():
|
|
|
|
"""Test if infix_finditer works correctly"""
|
|
|
|
infix_re = re.compile(r"""[^a-z]""")
|
|
|
|
test_cases = [
|
|
|
|
("token 123test", ["token", "1", "2", "3", "test"]),
|
|
|
|
("token 1test", ["token", "1test"]),
|
|
|
|
("hello...test", ["hello", ".", ".", ".", "test"]),
|
|
|
|
]
|
|
|
|
|
|
|
|
def new_tokenizer(nlp):
|
|
|
|
return Tokenizer(nlp.vocab, {}, infix_finditer=infix_re.finditer)
|
|
|
|
|
|
|
|
nlp = English()
|
|
|
|
nlp.tokenizer = new_tokenizer(nlp)
|
|
|
|
for text, expected in test_cases:
|
|
|
|
assert [token.text for token in nlp(text)] == expected
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.skip(
|
|
|
|
reason="Can not be fixed without iterative looping between prefix/suffix and infix"
|
|
|
|
)
|
|
|
|
@pytest.mark.issue(2070)
|
|
|
|
def test_issue2070():
|
|
|
|
"""Test that checks that a dot followed by a quote is handled
|
|
|
|
appropriately.
|
|
|
|
"""
|
|
|
|
# Problem: The dot is now properly split off, but the prefix/suffix rules
|
|
|
|
# are not applied again afterwards. This means that the quote will still be
|
|
|
|
# attached to the remaining token.
|
|
|
|
nlp = English()
|
|
|
|
doc = nlp('First sentence."A quoted sentence" he said ...')
|
|
|
|
assert len(doc) == 11
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(2926)
|
|
|
|
def test_issue2926(fr_tokenizer):
|
|
|
|
"""Test that the tokenizer correctly splits tokens separated by a slash (/)
|
|
|
|
ending in a digit.
|
|
|
|
"""
|
|
|
|
doc = fr_tokenizer("Learn html5/css3/javascript/jquery")
|
|
|
|
assert len(doc) == 8
|
|
|
|
assert doc[0].text == "Learn"
|
|
|
|
assert doc[1].text == "html5"
|
|
|
|
assert doc[2].text == "/"
|
|
|
|
assert doc[3].text == "css3"
|
|
|
|
assert doc[4].text == "/"
|
|
|
|
assert doc[5].text == "javascript"
|
|
|
|
assert doc[6].text == "/"
|
|
|
|
assert doc[7].text == "jquery"
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"text",
|
|
|
|
[
|
|
|
|
"ABLEItemColumn IAcceptance Limits of ErrorIn-Service Limits of ErrorColumn IIColumn IIIColumn IVColumn VComputed VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeCubic FeetCubic FeetCubic FeetCubic FeetCubic Feet1Up to 10.0100.0050.0100.005220.0200.0100.0200.010350.0360.0180.0360.0184100.0500.0250.0500.0255Over 100.5% of computed volume0.25% of computed volume0.5% of computed volume0.25% of computed volume TABLE ItemColumn IAcceptance Limits of ErrorIn-Service Limits of ErrorColumn IIColumn IIIColumn IVColumn VComputed VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeCubic FeetCubic FeetCubic FeetCubic FeetCubic Feet1Up to 10.0100.0050.0100.005220.0200.0100.0200.010350.0360.0180.0360.0184100.0500.0250.0500.0255Over 100.5% of computed volume0.25% of computed volume0.5% of computed volume0.25% of computed volume ItemColumn IAcceptance Limits of ErrorIn-Service Limits of ErrorColumn IIColumn IIIColumn IVColumn VComputed VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeCubic FeetCubic FeetCubic FeetCubic FeetCubic Feet1Up to 10.0100.0050.0100.005220.0200.0100.0200.010350.0360.0180.0360.0184100.0500.0250.0500.0255Over 100.5% of computed volume0.25% of computed volume0.5% of computed volume0.25% of computed volume",
|
|
|
|
"oow.jspsearch.eventoracleopenworldsearch.technologyoraclesolarissearch.technologystoragesearch.technologylinuxsearch.technologyserverssearch.technologyvirtualizationsearch.technologyengineeredsystemspcodewwmkmppscem:",
|
|
|
|
],
|
|
|
|
)
|
|
|
|
@pytest.mark.issue(2626)
|
|
|
|
def test_issue2626_2835(en_tokenizer, text):
|
|
|
|
"""Check that sentence doesn't cause an infinite loop in the tokenizer."""
|
|
|
|
doc = en_tokenizer(text)
|
|
|
|
assert doc
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(2656)
|
|
|
|
def test_issue2656(en_tokenizer):
|
|
|
|
"""Test that tokenizer correctly splits off punctuation after numbers with
|
|
|
|
decimal points.
|
|
|
|
"""
|
|
|
|
doc = en_tokenizer("I went for 40.3, and got home by 10.0.")
|
|
|
|
assert len(doc) == 11
|
|
|
|
assert doc[0].text == "I"
|
|
|
|
assert doc[1].text == "went"
|
|
|
|
assert doc[2].text == "for"
|
|
|
|
assert doc[3].text == "40.3"
|
|
|
|
assert doc[4].text == ","
|
|
|
|
assert doc[5].text == "and"
|
|
|
|
assert doc[6].text == "got"
|
|
|
|
assert doc[7].text == "home"
|
|
|
|
assert doc[8].text == "by"
|
|
|
|
assert doc[9].text == "10.0"
|
|
|
|
assert doc[10].text == "."
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(2754)
|
|
|
|
def test_issue2754(en_tokenizer):
|
|
|
|
"""Test that words like 'a' and 'a.m.' don't get exceptional norm values."""
|
|
|
|
a = en_tokenizer("a")
|
|
|
|
assert a[0].norm_ == "a"
|
|
|
|
am = en_tokenizer("am")
|
|
|
|
assert am[0].norm_ == "am"
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(3002)
|
|
|
|
def test_issue3002():
|
|
|
|
"""Test that the tokenizer doesn't hang on a long list of dots"""
|
|
|
|
nlp = German()
|
|
|
|
doc = nlp(
|
|
|
|
"880.794.982.218.444.893.023.439.794.626.120.190.780.624.990.275.671 ist eine lange Zahl"
|
|
|
|
)
|
|
|
|
assert len(doc) == 5
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.skip(reason="default suffix rules avoid one upper-case letter before dot")
|
|
|
|
@pytest.mark.issue(3449)
|
|
|
|
def test_issue3449():
|
|
|
|
nlp = English()
|
|
|
|
nlp.add_pipe("sentencizer")
|
|
|
|
text1 = "He gave the ball to I. Do you want to go to the movies with I?"
|
|
|
|
text2 = "He gave the ball to I. Do you want to go to the movies with I?"
|
|
|
|
text3 = "He gave the ball to I.\nDo you want to go to the movies with I?"
|
|
|
|
t1 = nlp(text1)
|
|
|
|
t2 = nlp(text2)
|
|
|
|
t3 = nlp(text3)
|
|
|
|
assert t1[5].text == "I"
|
|
|
|
assert t2[5].text == "I"
|
|
|
|
assert t3[5].text == "I"
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"text,words", [("A'B C", ["A", "'", "B", "C"]), ("A-B", ["A-B"])]
|
|
|
|
)
|
|
|
|
def test_gold_misaligned(en_tokenizer, text, words):
|
|
|
|
doc = en_tokenizer(text)
|
|
|
|
Example.from_dict(doc, {"words": words})
|
2017-01-13 00:34:14 +00:00
|
|
|
|
2017-01-05 15:25:38 +00:00
|
|
|
|
|
|
|
def test_tokenizer_handles_no_word(tokenizer):
|
|
|
|
tokens = tokenizer("")
|
|
|
|
assert len(tokens) == 0
|
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize("text", ["lorem"])
|
2017-01-05 15:25:38 +00:00
|
|
|
def test_tokenizer_handles_single_word(tokenizer, text):
|
|
|
|
tokens = tokenizer(text)
|
|
|
|
assert tokens[0].text == text
|
|
|
|
|
|
|
|
|
|
|
|
def test_tokenizer_handles_punct(tokenizer):
|
|
|
|
text = "Lorem, ipsum."
|
|
|
|
tokens = tokenizer(text)
|
|
|
|
assert len(tokens) == 4
|
|
|
|
assert tokens[0].text == "Lorem"
|
|
|
|
assert tokens[1].text == ","
|
|
|
|
assert tokens[2].text == "ipsum"
|
|
|
|
assert tokens[1].text != "Lorem"
|
|
|
|
|
|
|
|
|
2019-02-01 07:05:22 +00:00
|
|
|
def test_tokenizer_handles_punct_braces(tokenizer):
|
|
|
|
text = "Lorem, (ipsum)."
|
|
|
|
tokens = tokenizer(text)
|
|
|
|
assert len(tokens) == 6
|
|
|
|
|
|
|
|
|
2017-01-05 15:25:38 +00:00
|
|
|
def test_tokenizer_handles_digits(tokenizer):
|
2017-03-05 01:11:26 +00:00
|
|
|
exceptions = ["hu", "bn"]
|
2017-01-05 15:25:38 +00:00
|
|
|
text = "Lorem ipsum: 1984."
|
|
|
|
tokens = tokenizer(text)
|
|
|
|
|
|
|
|
if tokens[0].lang_ not in exceptions:
|
|
|
|
assert len(tokens) == 5
|
|
|
|
assert tokens[0].text == "Lorem"
|
|
|
|
assert tokens[3].text == "1984"
|
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"text",
|
|
|
|
["google.com", "python.org", "spacy.io", "explosion.ai", "http://www.google.com"],
|
|
|
|
)
|
2017-01-05 17:09:29 +00:00
|
|
|
def test_tokenizer_keep_urls(tokenizer, text):
|
|
|
|
tokens = tokenizer(text)
|
|
|
|
assert len(tokens) == 1
|
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize("text", ["NASDAQ:GOOG"])
|
2017-03-04 22:13:11 +00:00
|
|
|
def test_tokenizer_colons(tokenizer, text):
|
|
|
|
tokens = tokenizer(text)
|
|
|
|
assert len(tokens) == 3
|
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"text", ["hello123@example.com", "hi+there@gmail.it", "matt@explosion.ai"]
|
|
|
|
)
|
2017-01-05 17:09:29 +00:00
|
|
|
def test_tokenizer_keeps_email(tokenizer, text):
|
|
|
|
tokens = tokenizer(text)
|
|
|
|
assert len(tokens) == 1
|
|
|
|
|
|
|
|
|
2017-01-05 15:25:38 +00:00
|
|
|
def test_tokenizer_handles_long_text(tokenizer):
|
|
|
|
text = """Lorem ipsum dolor sit amet, consectetur adipiscing elit
|
|
|
|
|
|
|
|
Cras egestas orci non porttitor maximus.
|
|
|
|
Maecenas quis odio id dolor rhoncus dignissim. Curabitur sed velit at orci ultrices sagittis. Nulla commodo euismod arcu eget vulputate.
|
|
|
|
|
|
|
|
Phasellus tincidunt, augue quis porta finibus, massa sapien consectetur augue, non lacinia enim nibh eget ipsum. Vestibulum in bibendum mauris.
|
|
|
|
|
|
|
|
"Nullam porta fringilla enim, a dictum orci consequat in." Mauris nec malesuada justo."""
|
|
|
|
|
|
|
|
tokens = tokenizer(text)
|
|
|
|
assert len(tokens) > 5
|
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize("file_name", ["sun.txt"])
|
2017-01-05 17:09:29 +00:00
|
|
|
def test_tokenizer_handle_text_from_file(tokenizer, file_name):
|
2018-07-24 21:38:44 +00:00
|
|
|
loc = ensure_path(__file__).parent / file_name
|
2021-05-31 09:04:29 +00:00
|
|
|
with loc.open("r", encoding="utf8") as infile:
|
|
|
|
text = infile.read()
|
2017-01-05 17:09:29 +00:00
|
|
|
assert len(text) != 0
|
|
|
|
tokens = tokenizer(text)
|
|
|
|
assert len(tokens) > 100
|
|
|
|
|
|
|
|
|
2017-01-05 15:25:38 +00:00
|
|
|
def test_tokenizer_suspected_freeing_strings(tokenizer):
|
|
|
|
text1 = "Lorem dolor sit amet, consectetur adipiscing elit."
|
|
|
|
text2 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."
|
|
|
|
tokens1 = tokenizer(text1)
|
|
|
|
tokens2 = tokenizer(text2)
|
|
|
|
assert tokens1[0].text == "Lorem"
|
|
|
|
assert tokens2[0].text == "Lorem"
|
2017-01-13 00:34:14 +00:00
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize("text,tokens", [("lorem", [{"orth": "lo"}, {"orth": "rem"}])])
|
2017-01-13 00:34:14 +00:00
|
|
|
def test_tokenizer_add_special_case(tokenizer, text, tokens):
|
|
|
|
tokenizer.add_special_case(text, tokens)
|
|
|
|
doc = tokenizer(text)
|
2018-11-27 00:09:36 +00:00
|
|
|
assert doc[0].text == tokens[0]["orth"]
|
|
|
|
assert doc[1].text == tokens[1]["orth"]
|
2017-01-13 00:34:14 +00:00
|
|
|
|
|
|
|
|
2020-08-31 07:42:06 +00:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"text,tokens",
|
|
|
|
[
|
|
|
|
("lorem", [{"orth": "lo"}, {"orth": "re"}]),
|
|
|
|
("lorem", [{"orth": "lo", "tag": "A"}, {"orth": "rem"}]),
|
|
|
|
],
|
|
|
|
)
|
Generalize handling of tokenizer special cases (#4259)
* Generalize handling of tokenizer special cases
Handle tokenizer special cases more generally by using the Matcher
internally to match special cases after the affix/token_match
tokenization is complete.
Instead of only matching special cases while processing balanced or
nearly balanced prefixes and suffixes, this recognizes special cases in
a wider range of contexts:
* Allows arbitrary numbers of prefixes/affixes around special cases
* Allows special cases separated by infixes
Existing tests/settings that couldn't be preserved as before:
* The emoticon '")' is no longer a supported special case
* The emoticon ':)' in "example:)" is a false positive again
When merged with #4258 (or the relevant cache bugfix), the affix and
token_match properties should be modified to flush and reload all
special cases to use the updated internal tokenization with the Matcher.
* Remove accidentally added test case
* Really remove accidentally added test
* Reload special cases when necessary
Reload special cases when affixes or token_match are modified. Skip
reloading during initialization.
* Update error code number
* Fix offset and whitespace in Matcher special cases
* Fix offset bugs when merging and splitting tokens
* Set final whitespace on final token in inserted special case
* Improve cache flushing in tokenizer
* Separate cache and specials memory (temporarily)
* Flush cache when adding special cases
* Repeated `self._cache = PreshMap()` and `self._specials = PreshMap()`
are necessary due to this bug:
https://github.com/explosion/preshed/issues/21
* Remove reinitialized PreshMaps on cache flush
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Use special Matcher only for cases with affixes
* Reinsert specials cache checks during normal tokenization for special
cases as much as possible
* Additionally include specials cache checks while splitting on infixes
* Since the special Matcher needs consistent affix-only tokenization
for the special cases themselves, introduce the argument
`with_special_cases` in order to do tokenization with or without
specials cache checks
* After normal tokenization, postprocess with special cases Matcher for
special cases containing affixes
* Replace PhraseMatcher with Aho-Corasick
Replace PhraseMatcher with the Aho-Corasick algorithm over numpy arrays
of the hash values for the relevant attribute. The implementation is
based on FlashText.
The speed should be similar to the previous PhraseMatcher. It is now
possible to easily remove match IDs and matches don't go missing with
large keyword lists / vocabularies.
Fixes #4308.
* Restore support for pickling
* Fix internal keyword add/remove for numpy arrays
* Add test for #4248, clean up test
* Improve efficiency of special cases handling
* Use PhraseMatcher instead of Matcher
* Improve efficiency of merging/splitting special cases in document
* Process merge/splits in one pass without repeated token shifting
* Merge in place if no splits
* Update error message number
* Remove UD script modifications
Only used for timing/testing, should be a separate PR
* Remove final traces of UD script modifications
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Add missing loop for match ID set in search loop
* Remove cruft in matching loop for partial matches
There was a bit of unnecessary code left over from FlashText in the
matching loop to handle partial token matches, which we don't have with
PhraseMatcher.
* Replace dict trie with MapStruct trie
* Fix how match ID hash is stored/added
* Update fix for match ID vocab
* Switch from map_get_unless_missing to map_get
* Switch from numpy array to Token.get_struct_attr
Access token attributes directly in Doc instead of making a copy of the
relevant values in a numpy array.
Add unsatisfactory warning for hash collision with reserved terminal
hash key. (Ideally it would change the reserved terminal hash and redo
the whole trie, but for now, I'm hoping there won't be collisions.)
* Restructure imports to export find_matches
* Implement full remove()
Remove unnecessary trie paths and free unused maps.
Parallel to Matcher, raise KeyError when attempting to remove a match ID
that has not been added.
* Switch to PhraseMatcher.find_matches
* Switch to local cdef functions for span filtering
* Switch special case reload threshold to variable
Refer to variable instead of hard-coded threshold
* Move more of special case retokenize to cdef nogil
Move as much of the special case retokenization to nogil as possible.
* Rewrap sort as stdsort for OS X
* Rewrap stdsort with specific types
* Switch to qsort
* Fix merge
* Improve cmp functions
* Fix realloc
* Fix realloc again
* Initialize span struct while retokenizing
* Temporarily skip retokenizing
* Revert "Move more of special case retokenize to cdef nogil"
This reverts commit 0b7e52c797cd8ff1548f214bd4186ebb3a7ce8b1.
* Revert "Switch to qsort"
This reverts commit a98d71a942fc9bca531cf5eb05cf89fa88153b60.
* Fix specials check while caching
* Modify URL test with emoticons
The multiple suffix tests result in the emoticon `:>`, which is now
retokenized into one token as a special case after the suffixes are
split off.
* Refactor _apply_special_cases()
* Use cdef ints for span info used in multiple spots
* Modify _filter_special_spans() to prefer earlier
Parallel to #4414, modify _filter_special_spans() so that the earlier
span is preferred for overlapping spans of the same length.
* Replace MatchStruct with Entity
Replace MatchStruct with Entity since the existing Entity struct is
nearly identical.
* Replace Entity with more general SpanC
* Replace MatchStruct with SpanC
* Add error in debug-data if no dev docs are available (see #4575)
* Update azure-pipelines.yml
* Revert "Update azure-pipelines.yml"
This reverts commit ed1060cf59e5895b5fe92ad5b894fd1078ec4c49.
* Use latest wasabi
* Reorganise install_requires
* add dframcy to universe.json (#4580)
* Update universe.json [ci skip]
* Fix multiprocessing for as_tuples=True (#4582)
* Fix conllu script (#4579)
* force extensions to avoid clash between example scripts
* fix arg order and default file encoding
* add example config for conllu script
* newline
* move extension definitions to main function
* few more encodings fixes
* Add load_from_docbin example [ci skip]
TODO: upload the file somewhere
* Update README.md
* Add warnings about 3.8 (resolves #4593) [ci skip]
* Fixed typo: Added space between "recognize" and "various" (#4600)
* Fix DocBin.merge() example (#4599)
* Replace function registries with catalogue (#4584)
* Replace functions registries with catalogue
* Update __init__.py
* Fix test
* Revert unrelated flag [ci skip]
* Bugfix/dep matcher issue 4590 (#4601)
* add contributor agreement for prilopes
* add test for issue #4590
* fix on_match params for DependencyMacther (#4590)
* Minor updates to language example sentences (#4608)
* Add punctuation to Spanish example sentences
* Combine multilanguage examples for lang xx
* Add punctuation to nb examples
* Always realloc to a larger size
Avoid potential (unlikely) edge case and cymem error seen in #4604.
* Add error in debug-data if no dev docs are available (see #4575)
* Update debug-data for GoldCorpus / Example
* Ignore None label in misaligned NER data
2019-11-13 20:24:35 +00:00
|
|
|
def test_tokenizer_validate_special_case(tokenizer, text, tokens):
|
|
|
|
with pytest.raises(ValueError):
|
|
|
|
tokenizer.add_special_case(text, tokens)
|
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize(
|
2020-08-07 13:27:13 +00:00
|
|
|
"text,tokens", [("lorem", [{"orth": "lo", "norm": "LO"}, {"orth": "rem"}])]
|
2018-11-27 00:09:36 +00:00
|
|
|
)
|
2017-01-13 00:34:14 +00:00
|
|
|
def test_tokenizer_add_special_case_tag(text, tokens):
|
2020-08-07 13:27:13 +00:00
|
|
|
vocab = Vocab()
|
2017-01-13 00:34:14 +00:00
|
|
|
tokenizer = Tokenizer(vocab, {}, None, None, None)
|
|
|
|
tokenizer.add_special_case(text, tokens)
|
|
|
|
doc = tokenizer(text)
|
2018-11-27 00:09:36 +00:00
|
|
|
assert doc[0].text == tokens[0]["orth"]
|
2020-08-07 13:27:13 +00:00
|
|
|
assert doc[0].norm_ == tokens[0]["norm"]
|
2018-11-27 00:09:36 +00:00
|
|
|
assert doc[1].text == tokens[1]["orth"]
|
Generalize handling of tokenizer special cases (#4259)
* Generalize handling of tokenizer special cases
Handle tokenizer special cases more generally by using the Matcher
internally to match special cases after the affix/token_match
tokenization is complete.
Instead of only matching special cases while processing balanced or
nearly balanced prefixes and suffixes, this recognizes special cases in
a wider range of contexts:
* Allows arbitrary numbers of prefixes/affixes around special cases
* Allows special cases separated by infixes
Existing tests/settings that couldn't be preserved as before:
* The emoticon '")' is no longer a supported special case
* The emoticon ':)' in "example:)" is a false positive again
When merged with #4258 (or the relevant cache bugfix), the affix and
token_match properties should be modified to flush and reload all
special cases to use the updated internal tokenization with the Matcher.
* Remove accidentally added test case
* Really remove accidentally added test
* Reload special cases when necessary
Reload special cases when affixes or token_match are modified. Skip
reloading during initialization.
* Update error code number
* Fix offset and whitespace in Matcher special cases
* Fix offset bugs when merging and splitting tokens
* Set final whitespace on final token in inserted special case
* Improve cache flushing in tokenizer
* Separate cache and specials memory (temporarily)
* Flush cache when adding special cases
* Repeated `self._cache = PreshMap()` and `self._specials = PreshMap()`
are necessary due to this bug:
https://github.com/explosion/preshed/issues/21
* Remove reinitialized PreshMaps on cache flush
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Use special Matcher only for cases with affixes
* Reinsert specials cache checks during normal tokenization for special
cases as much as possible
* Additionally include specials cache checks while splitting on infixes
* Since the special Matcher needs consistent affix-only tokenization
for the special cases themselves, introduce the argument
`with_special_cases` in order to do tokenization with or without
specials cache checks
* After normal tokenization, postprocess with special cases Matcher for
special cases containing affixes
* Replace PhraseMatcher with Aho-Corasick
Replace PhraseMatcher with the Aho-Corasick algorithm over numpy arrays
of the hash values for the relevant attribute. The implementation is
based on FlashText.
The speed should be similar to the previous PhraseMatcher. It is now
possible to easily remove match IDs and matches don't go missing with
large keyword lists / vocabularies.
Fixes #4308.
* Restore support for pickling
* Fix internal keyword add/remove for numpy arrays
* Add test for #4248, clean up test
* Improve efficiency of special cases handling
* Use PhraseMatcher instead of Matcher
* Improve efficiency of merging/splitting special cases in document
* Process merge/splits in one pass without repeated token shifting
* Merge in place if no splits
* Update error message number
* Remove UD script modifications
Only used for timing/testing, should be a separate PR
* Remove final traces of UD script modifications
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Add missing loop for match ID set in search loop
* Remove cruft in matching loop for partial matches
There was a bit of unnecessary code left over from FlashText in the
matching loop to handle partial token matches, which we don't have with
PhraseMatcher.
* Replace dict trie with MapStruct trie
* Fix how match ID hash is stored/added
* Update fix for match ID vocab
* Switch from map_get_unless_missing to map_get
* Switch from numpy array to Token.get_struct_attr
Access token attributes directly in Doc instead of making a copy of the
relevant values in a numpy array.
Add unsatisfactory warning for hash collision with reserved terminal
hash key. (Ideally it would change the reserved terminal hash and redo
the whole trie, but for now, I'm hoping there won't be collisions.)
* Restructure imports to export find_matches
* Implement full remove()
Remove unnecessary trie paths and free unused maps.
Parallel to Matcher, raise KeyError when attempting to remove a match ID
that has not been added.
* Switch to PhraseMatcher.find_matches
* Switch to local cdef functions for span filtering
* Switch special case reload threshold to variable
Refer to variable instead of hard-coded threshold
* Move more of special case retokenize to cdef nogil
Move as much of the special case retokenization to nogil as possible.
* Rewrap sort as stdsort for OS X
* Rewrap stdsort with specific types
* Switch to qsort
* Fix merge
* Improve cmp functions
* Fix realloc
* Fix realloc again
* Initialize span struct while retokenizing
* Temporarily skip retokenizing
* Revert "Move more of special case retokenize to cdef nogil"
This reverts commit 0b7e52c797cd8ff1548f214bd4186ebb3a7ce8b1.
* Revert "Switch to qsort"
This reverts commit a98d71a942fc9bca531cf5eb05cf89fa88153b60.
* Fix specials check while caching
* Modify URL test with emoticons
The multiple suffix tests result in the emoticon `:>`, which is now
retokenized into one token as a special case after the suffixes are
split off.
* Refactor _apply_special_cases()
* Use cdef ints for span info used in multiple spots
* Modify _filter_special_spans() to prefer earlier
Parallel to #4414, modify _filter_special_spans() so that the earlier
span is preferred for overlapping spans of the same length.
* Replace MatchStruct with Entity
Replace MatchStruct with Entity since the existing Entity struct is
nearly identical.
* Replace Entity with more general SpanC
* Replace MatchStruct with SpanC
* Add error in debug-data if no dev docs are available (see #4575)
* Update azure-pipelines.yml
* Revert "Update azure-pipelines.yml"
This reverts commit ed1060cf59e5895b5fe92ad5b894fd1078ec4c49.
* Use latest wasabi
* Reorganise install_requires
* add dframcy to universe.json (#4580)
* Update universe.json [ci skip]
* Fix multiprocessing for as_tuples=True (#4582)
* Fix conllu script (#4579)
* force extensions to avoid clash between example scripts
* fix arg order and default file encoding
* add example config for conllu script
* newline
* move extension definitions to main function
* few more encodings fixes
* Add load_from_docbin example [ci skip]
TODO: upload the file somewhere
* Update README.md
* Add warnings about 3.8 (resolves #4593) [ci skip]
* Fixed typo: Added space between "recognize" and "various" (#4600)
* Fix DocBin.merge() example (#4599)
* Replace function registries with catalogue (#4584)
* Replace functions registries with catalogue
* Update __init__.py
* Fix test
* Revert unrelated flag [ci skip]
* Bugfix/dep matcher issue 4590 (#4601)
* add contributor agreement for prilopes
* add test for issue #4590
* fix on_match params for DependencyMacther (#4590)
* Minor updates to language example sentences (#4608)
* Add punctuation to Spanish example sentences
* Combine multilanguage examples for lang xx
* Add punctuation to nb examples
* Always realloc to a larger size
Avoid potential (unlikely) edge case and cymem error seen in #4604.
* Add error in debug-data if no dev docs are available (see #4575)
* Update debug-data for GoldCorpus / Example
* Ignore None label in misaligned NER data
2019-11-13 20:24:35 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_tokenizer_special_cases_with_affixes(tokenizer):
|
|
|
|
text = '(((_SPECIAL_ A/B, A/B-A/B")'
|
|
|
|
tokenizer.add_special_case("_SPECIAL_", [{"orth": "_SPECIAL_"}])
|
|
|
|
tokenizer.add_special_case("A/B", [{"orth": "A/B"}])
|
|
|
|
doc = tokenizer(text)
|
2020-02-18 14:38:18 +00:00
|
|
|
assert [token.text for token in doc] == [
|
|
|
|
"(",
|
|
|
|
"(",
|
|
|
|
"(",
|
|
|
|
"_SPECIAL_",
|
|
|
|
"A/B",
|
|
|
|
",",
|
|
|
|
"A/B",
|
|
|
|
"-",
|
|
|
|
"A/B",
|
|
|
|
'"',
|
|
|
|
")",
|
|
|
|
]
|
Generalize handling of tokenizer special cases (#4259)
* Generalize handling of tokenizer special cases
Handle tokenizer special cases more generally by using the Matcher
internally to match special cases after the affix/token_match
tokenization is complete.
Instead of only matching special cases while processing balanced or
nearly balanced prefixes and suffixes, this recognizes special cases in
a wider range of contexts:
* Allows arbitrary numbers of prefixes/affixes around special cases
* Allows special cases separated by infixes
Existing tests/settings that couldn't be preserved as before:
* The emoticon '")' is no longer a supported special case
* The emoticon ':)' in "example:)" is a false positive again
When merged with #4258 (or the relevant cache bugfix), the affix and
token_match properties should be modified to flush and reload all
special cases to use the updated internal tokenization with the Matcher.
* Remove accidentally added test case
* Really remove accidentally added test
* Reload special cases when necessary
Reload special cases when affixes or token_match are modified. Skip
reloading during initialization.
* Update error code number
* Fix offset and whitespace in Matcher special cases
* Fix offset bugs when merging and splitting tokens
* Set final whitespace on final token in inserted special case
* Improve cache flushing in tokenizer
* Separate cache and specials memory (temporarily)
* Flush cache when adding special cases
* Repeated `self._cache = PreshMap()` and `self._specials = PreshMap()`
are necessary due to this bug:
https://github.com/explosion/preshed/issues/21
* Remove reinitialized PreshMaps on cache flush
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Use special Matcher only for cases with affixes
* Reinsert specials cache checks during normal tokenization for special
cases as much as possible
* Additionally include specials cache checks while splitting on infixes
* Since the special Matcher needs consistent affix-only tokenization
for the special cases themselves, introduce the argument
`with_special_cases` in order to do tokenization with or without
specials cache checks
* After normal tokenization, postprocess with special cases Matcher for
special cases containing affixes
* Replace PhraseMatcher with Aho-Corasick
Replace PhraseMatcher with the Aho-Corasick algorithm over numpy arrays
of the hash values for the relevant attribute. The implementation is
based on FlashText.
The speed should be similar to the previous PhraseMatcher. It is now
possible to easily remove match IDs and matches don't go missing with
large keyword lists / vocabularies.
Fixes #4308.
* Restore support for pickling
* Fix internal keyword add/remove for numpy arrays
* Add test for #4248, clean up test
* Improve efficiency of special cases handling
* Use PhraseMatcher instead of Matcher
* Improve efficiency of merging/splitting special cases in document
* Process merge/splits in one pass without repeated token shifting
* Merge in place if no splits
* Update error message number
* Remove UD script modifications
Only used for timing/testing, should be a separate PR
* Remove final traces of UD script modifications
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Add missing loop for match ID set in search loop
* Remove cruft in matching loop for partial matches
There was a bit of unnecessary code left over from FlashText in the
matching loop to handle partial token matches, which we don't have with
PhraseMatcher.
* Replace dict trie with MapStruct trie
* Fix how match ID hash is stored/added
* Update fix for match ID vocab
* Switch from map_get_unless_missing to map_get
* Switch from numpy array to Token.get_struct_attr
Access token attributes directly in Doc instead of making a copy of the
relevant values in a numpy array.
Add unsatisfactory warning for hash collision with reserved terminal
hash key. (Ideally it would change the reserved terminal hash and redo
the whole trie, but for now, I'm hoping there won't be collisions.)
* Restructure imports to export find_matches
* Implement full remove()
Remove unnecessary trie paths and free unused maps.
Parallel to Matcher, raise KeyError when attempting to remove a match ID
that has not been added.
* Switch to PhraseMatcher.find_matches
* Switch to local cdef functions for span filtering
* Switch special case reload threshold to variable
Refer to variable instead of hard-coded threshold
* Move more of special case retokenize to cdef nogil
Move as much of the special case retokenization to nogil as possible.
* Rewrap sort as stdsort for OS X
* Rewrap stdsort with specific types
* Switch to qsort
* Fix merge
* Improve cmp functions
* Fix realloc
* Fix realloc again
* Initialize span struct while retokenizing
* Temporarily skip retokenizing
* Revert "Move more of special case retokenize to cdef nogil"
This reverts commit 0b7e52c797cd8ff1548f214bd4186ebb3a7ce8b1.
* Revert "Switch to qsort"
This reverts commit a98d71a942fc9bca531cf5eb05cf89fa88153b60.
* Fix specials check while caching
* Modify URL test with emoticons
The multiple suffix tests result in the emoticon `:>`, which is now
retokenized into one token as a special case after the suffixes are
split off.
* Refactor _apply_special_cases()
* Use cdef ints for span info used in multiple spots
* Modify _filter_special_spans() to prefer earlier
Parallel to #4414, modify _filter_special_spans() so that the earlier
span is preferred for overlapping spans of the same length.
* Replace MatchStruct with Entity
Replace MatchStruct with Entity since the existing Entity struct is
nearly identical.
* Replace Entity with more general SpanC
* Replace MatchStruct with SpanC
* Add error in debug-data if no dev docs are available (see #4575)
* Update azure-pipelines.yml
* Revert "Update azure-pipelines.yml"
This reverts commit ed1060cf59e5895b5fe92ad5b894fd1078ec4c49.
* Use latest wasabi
* Reorganise install_requires
* add dframcy to universe.json (#4580)
* Update universe.json [ci skip]
* Fix multiprocessing for as_tuples=True (#4582)
* Fix conllu script (#4579)
* force extensions to avoid clash between example scripts
* fix arg order and default file encoding
* add example config for conllu script
* newline
* move extension definitions to main function
* few more encodings fixes
* Add load_from_docbin example [ci skip]
TODO: upload the file somewhere
* Update README.md
* Add warnings about 3.8 (resolves #4593) [ci skip]
* Fixed typo: Added space between "recognize" and "various" (#4600)
* Fix DocBin.merge() example (#4599)
* Replace function registries with catalogue (#4584)
* Replace functions registries with catalogue
* Update __init__.py
* Fix test
* Revert unrelated flag [ci skip]
* Bugfix/dep matcher issue 4590 (#4601)
* add contributor agreement for prilopes
* add test for issue #4590
* fix on_match params for DependencyMacther (#4590)
* Minor updates to language example sentences (#4608)
* Add punctuation to Spanish example sentences
* Combine multilanguage examples for lang xx
* Add punctuation to nb examples
* Always realloc to a larger size
Avoid potential (unlikely) edge case and cymem error seen in #4604.
* Add error in debug-data if no dev docs are available (see #4575)
* Update debug-data for GoldCorpus / Example
* Ignore None label in misaligned NER data
2019-11-13 20:24:35 +00:00
|
|
|
|
|
|
|
|
2020-12-08 06:25:56 +00:00
|
|
|
def test_tokenizer_special_cases_with_affixes_preserve_spacy():
|
|
|
|
tokenizer = English().tokenizer
|
|
|
|
# reset all special cases
|
|
|
|
tokenizer.rules = {}
|
|
|
|
|
|
|
|
# in-place modification (only merges)
|
|
|
|
text = "''a'' "
|
|
|
|
tokenizer.add_special_case("''", [{"ORTH": "''"}])
|
|
|
|
assert tokenizer(text).text == text
|
|
|
|
|
|
|
|
# not in-place (splits and merges)
|
|
|
|
tokenizer.add_special_case("ab", [{"ORTH": "a"}, {"ORTH": "b"}])
|
|
|
|
text = "ab ab ab ''ab ab'' ab'' ''ab"
|
|
|
|
assert tokenizer(text).text == text
|
|
|
|
|
|
|
|
|
Generalize handling of tokenizer special cases (#4259)
* Generalize handling of tokenizer special cases
Handle tokenizer special cases more generally by using the Matcher
internally to match special cases after the affix/token_match
tokenization is complete.
Instead of only matching special cases while processing balanced or
nearly balanced prefixes and suffixes, this recognizes special cases in
a wider range of contexts:
* Allows arbitrary numbers of prefixes/affixes around special cases
* Allows special cases separated by infixes
Existing tests/settings that couldn't be preserved as before:
* The emoticon '")' is no longer a supported special case
* The emoticon ':)' in "example:)" is a false positive again
When merged with #4258 (or the relevant cache bugfix), the affix and
token_match properties should be modified to flush and reload all
special cases to use the updated internal tokenization with the Matcher.
* Remove accidentally added test case
* Really remove accidentally added test
* Reload special cases when necessary
Reload special cases when affixes or token_match are modified. Skip
reloading during initialization.
* Update error code number
* Fix offset and whitespace in Matcher special cases
* Fix offset bugs when merging and splitting tokens
* Set final whitespace on final token in inserted special case
* Improve cache flushing in tokenizer
* Separate cache and specials memory (temporarily)
* Flush cache when adding special cases
* Repeated `self._cache = PreshMap()` and `self._specials = PreshMap()`
are necessary due to this bug:
https://github.com/explosion/preshed/issues/21
* Remove reinitialized PreshMaps on cache flush
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Use special Matcher only for cases with affixes
* Reinsert specials cache checks during normal tokenization for special
cases as much as possible
* Additionally include specials cache checks while splitting on infixes
* Since the special Matcher needs consistent affix-only tokenization
for the special cases themselves, introduce the argument
`with_special_cases` in order to do tokenization with or without
specials cache checks
* After normal tokenization, postprocess with special cases Matcher for
special cases containing affixes
* Replace PhraseMatcher with Aho-Corasick
Replace PhraseMatcher with the Aho-Corasick algorithm over numpy arrays
of the hash values for the relevant attribute. The implementation is
based on FlashText.
The speed should be similar to the previous PhraseMatcher. It is now
possible to easily remove match IDs and matches don't go missing with
large keyword lists / vocabularies.
Fixes #4308.
* Restore support for pickling
* Fix internal keyword add/remove for numpy arrays
* Add test for #4248, clean up test
* Improve efficiency of special cases handling
* Use PhraseMatcher instead of Matcher
* Improve efficiency of merging/splitting special cases in document
* Process merge/splits in one pass without repeated token shifting
* Merge in place if no splits
* Update error message number
* Remove UD script modifications
Only used for timing/testing, should be a separate PR
* Remove final traces of UD script modifications
* Update UD bin scripts
* Update imports for `bin/`
* Add all currently supported languages
* Update subtok merger for new Matcher validation
* Modify blinded check to look at tokens instead of lemmas (for corpora
with tokens but not lemmas like Telugu)
* Add missing loop for match ID set in search loop
* Remove cruft in matching loop for partial matches
There was a bit of unnecessary code left over from FlashText in the
matching loop to handle partial token matches, which we don't have with
PhraseMatcher.
* Replace dict trie with MapStruct trie
* Fix how match ID hash is stored/added
* Update fix for match ID vocab
* Switch from map_get_unless_missing to map_get
* Switch from numpy array to Token.get_struct_attr
Access token attributes directly in Doc instead of making a copy of the
relevant values in a numpy array.
Add unsatisfactory warning for hash collision with reserved terminal
hash key. (Ideally it would change the reserved terminal hash and redo
the whole trie, but for now, I'm hoping there won't be collisions.)
* Restructure imports to export find_matches
* Implement full remove()
Remove unnecessary trie paths and free unused maps.
Parallel to Matcher, raise KeyError when attempting to remove a match ID
that has not been added.
* Switch to PhraseMatcher.find_matches
* Switch to local cdef functions for span filtering
* Switch special case reload threshold to variable
Refer to variable instead of hard-coded threshold
* Move more of special case retokenize to cdef nogil
Move as much of the special case retokenization to nogil as possible.
* Rewrap sort as stdsort for OS X
* Rewrap stdsort with specific types
* Switch to qsort
* Fix merge
* Improve cmp functions
* Fix realloc
* Fix realloc again
* Initialize span struct while retokenizing
* Temporarily skip retokenizing
* Revert "Move more of special case retokenize to cdef nogil"
This reverts commit 0b7e52c797cd8ff1548f214bd4186ebb3a7ce8b1.
* Revert "Switch to qsort"
This reverts commit a98d71a942fc9bca531cf5eb05cf89fa88153b60.
* Fix specials check while caching
* Modify URL test with emoticons
The multiple suffix tests result in the emoticon `:>`, which is now
retokenized into one token as a special case after the suffixes are
split off.
* Refactor _apply_special_cases()
* Use cdef ints for span info used in multiple spots
* Modify _filter_special_spans() to prefer earlier
Parallel to #4414, modify _filter_special_spans() so that the earlier
span is preferred for overlapping spans of the same length.
* Replace MatchStruct with Entity
Replace MatchStruct with Entity since the existing Entity struct is
nearly identical.
* Replace Entity with more general SpanC
* Replace MatchStruct with SpanC
* Add error in debug-data if no dev docs are available (see #4575)
* Update azure-pipelines.yml
* Revert "Update azure-pipelines.yml"
This reverts commit ed1060cf59e5895b5fe92ad5b894fd1078ec4c49.
* Use latest wasabi
* Reorganise install_requires
* add dframcy to universe.json (#4580)
* Update universe.json [ci skip]
* Fix multiprocessing for as_tuples=True (#4582)
* Fix conllu script (#4579)
* force extensions to avoid clash between example scripts
* fix arg order and default file encoding
* add example config for conllu script
* newline
* move extension definitions to main function
* few more encodings fixes
* Add load_from_docbin example [ci skip]
TODO: upload the file somewhere
* Update README.md
* Add warnings about 3.8 (resolves #4593) [ci skip]
* Fixed typo: Added space between "recognize" and "various" (#4600)
* Fix DocBin.merge() example (#4599)
* Replace function registries with catalogue (#4584)
* Replace functions registries with catalogue
* Update __init__.py
* Fix test
* Revert unrelated flag [ci skip]
* Bugfix/dep matcher issue 4590 (#4601)
* add contributor agreement for prilopes
* add test for issue #4590
* fix on_match params for DependencyMacther (#4590)
* Minor updates to language example sentences (#4608)
* Add punctuation to Spanish example sentences
* Combine multilanguage examples for lang xx
* Add punctuation to nb examples
* Always realloc to a larger size
Avoid potential (unlikely) edge case and cymem error seen in #4604.
* Add error in debug-data if no dev docs are available (see #4575)
* Update debug-data for GoldCorpus / Example
* Ignore None label in misaligned NER data
2019-11-13 20:24:35 +00:00
|
|
|
def test_tokenizer_special_cases_with_period(tokenizer):
|
|
|
|
text = "_SPECIAL_."
|
|
|
|
tokenizer.add_special_case("_SPECIAL_", [{"orth": "_SPECIAL_"}])
|
|
|
|
doc = tokenizer(text)
|
|
|
|
assert [token.text for token in doc] == ["_SPECIAL_", "."]
|
2020-09-13 12:05:36 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_tokenizer_special_cases_idx(tokenizer):
|
|
|
|
text = "the _ID'X_"
|
|
|
|
tokenizer.add_special_case("_ID'X_", [{"orth": "_ID"}, {"orth": "'X_"}])
|
|
|
|
doc = tokenizer(text)
|
|
|
|
assert doc[1].idx == 4
|
|
|
|
assert doc[2].idx == 7
|
2021-01-06 04:05:10 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_tokenizer_special_cases_spaces(tokenizer):
|
|
|
|
assert [t.text for t in tokenizer("a b c")] == ["a", "b", "c"]
|
|
|
|
tokenizer.add_special_case("a b c", [{"ORTH": "a b c"}])
|
|
|
|
assert [t.text for t in tokenizer("a b c")] == ["a b c"]
|
2021-04-22 08:14:57 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_tokenizer_flush_cache(en_vocab):
|
|
|
|
suffix_re = re.compile(r"[\.]$")
|
|
|
|
tokenizer = Tokenizer(
|
|
|
|
en_vocab,
|
|
|
|
suffix_search=suffix_re.search,
|
|
|
|
)
|
|
|
|
assert [t.text for t in tokenizer("a.")] == ["a", "."]
|
|
|
|
tokenizer.suffix_search = None
|
|
|
|
assert [t.text for t in tokenizer("a.")] == ["a."]
|
|
|
|
|
|
|
|
|
|
|
|
def test_tokenizer_flush_specials(en_vocab):
|
|
|
|
suffix_re = re.compile(r"[\.]$")
|
|
|
|
rules = {"a a": [{"ORTH": "a a"}]}
|
|
|
|
tokenizer1 = Tokenizer(
|
|
|
|
en_vocab,
|
|
|
|
suffix_search=suffix_re.search,
|
|
|
|
rules=rules,
|
|
|
|
)
|
|
|
|
assert [t.text for t in tokenizer1("a a.")] == ["a a", "."]
|
|
|
|
tokenizer1.rules = {}
|
|
|
|
assert [t.text for t in tokenizer1("a a.")] == ["a", "a", "."]
|
2021-10-27 11:02:25 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_tokenizer_prefix_suffix_overlap_lookbehind(en_vocab):
|
|
|
|
# the prefix and suffix matches overlap in the suffix lookbehind
|
2021-11-05 08:56:26 +00:00
|
|
|
prefixes = ["a(?=.)"]
|
|
|
|
suffixes = [r"(?<=\w)\.", r"(?<=a)\d+\."]
|
2021-10-27 11:02:25 +00:00
|
|
|
prefix_re = compile_prefix_regex(prefixes)
|
|
|
|
suffix_re = compile_suffix_regex(suffixes)
|
|
|
|
tokenizer = Tokenizer(
|
|
|
|
en_vocab,
|
|
|
|
prefix_search=prefix_re.search,
|
|
|
|
suffix_search=suffix_re.search,
|
|
|
|
)
|
|
|
|
tokens = [t.text for t in tokenizer("a10.")]
|
|
|
|
assert tokens == ["a", "10", "."]
|
|
|
|
explain_tokens = [t[1] for t in tokenizer.explain("a10.")]
|
|
|
|
assert tokens == explain_tokens
|
2022-01-28 16:00:54 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_tokenizer_infix_prefix(en_vocab):
|
|
|
|
# the prefix and suffix matches overlap in the suffix lookbehind
|
|
|
|
infixes = ["±"]
|
|
|
|
suffixes = ["%"]
|
|
|
|
infix_re = compile_infix_regex(infixes)
|
|
|
|
suffix_re = compile_suffix_regex(suffixes)
|
|
|
|
tokenizer = Tokenizer(
|
|
|
|
en_vocab,
|
|
|
|
infix_finditer=infix_re.finditer,
|
|
|
|
suffix_search=suffix_re.search,
|
|
|
|
)
|
|
|
|
tokens = [t.text for t in tokenizer("±10%")]
|
|
|
|
assert tokens == ["±10", "%"]
|
|
|
|
explain_tokens = [t[1] for t in tokenizer.explain("±10%")]
|
|
|
|
assert tokens == explain_tokens
|
2022-03-11 09:50:47 +00:00
|
|
|
|
|
|
|
|
2022-03-24 12:21:32 +00:00
|
|
|
@pytest.mark.issue(10086)
|
|
|
|
def test_issue10086(en_tokenizer):
|
|
|
|
"""Test special case works when part of infix substring."""
|
|
|
|
text = "No--don't see"
|
|
|
|
|
|
|
|
# without heuristics: do n't
|
|
|
|
en_tokenizer.faster_heuristics = False
|
|
|
|
doc = en_tokenizer(text)
|
|
|
|
assert "n't" in [w.text for w in doc]
|
|
|
|
assert "do" in [w.text for w in doc]
|
|
|
|
|
|
|
|
# with (default) heuristics: don't
|
|
|
|
en_tokenizer.faster_heuristics = True
|
|
|
|
doc = en_tokenizer(text)
|
|
|
|
assert "don't" in [w.text for w in doc]
|
|
|
|
|
|
|
|
|
2022-03-11 09:50:47 +00:00
|
|
|
def test_tokenizer_initial_special_case_explain(en_vocab):
|
|
|
|
tokenizer = Tokenizer(
|
|
|
|
en_vocab,
|
|
|
|
token_match=re.compile("^id$").match,
|
|
|
|
rules={
|
|
|
|
"id": [{"ORTH": "i"}, {"ORTH": "d"}],
|
2022-03-21 08:21:24 +00:00
|
|
|
},
|
2022-03-11 09:50:47 +00:00
|
|
|
)
|
|
|
|
tokens = [t.text for t in tokenizer("id")]
|
|
|
|
explain_tokens = [t[1] for t in tokenizer.explain("id")]
|
2022-03-28 08:44:46 +00:00
|
|
|
assert tokens == explain_tokens
|