Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Better error handling

This commit is contained in:
Gyorgy Orosz 2017-01-14 22:09:29 +01:00
parent a45f22913f
commit b03a46792c
2 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/hu/punctuation.py
View File

@ -35,7 +35,7 @@ TOKENIZER_INFIXES = (
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
r'(?<=[{a}])--(?=[{a}])'.format(a=ALPHA),
r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),
r'(?<=[0-9{a}])(({q})|[\)\]])(?=\-[{a}])'.format(a=ALPHA, q=QUOTES),
r'(?<=[{a}])(({q})|[\)\]\(\[])(?=[\-{a}])'.format(a=ALPHA, q=QUOTES),
]
)
__all__ = ["TOKENIZER_PREFIXES", "TOKENIZER_SUFFIXES", "TOKENIZER_INFIXES"]

7
spacy/tests/hu/test_tokenizer.py
View File

@ -147,7 +147,7 @@ NUMBER_TESTS = [
('A 15.-ben.', ['A', '15.-ben', '.']),
('A 2002--2003. van.', ['A', '2002--2003.', 'van', '.']),
('A 2002--2003-ben van.', ['A', '2002--2003-ben', 'van', '.']),
('A 2002--2003-ben.', ['A', '2002--2003-ben', '.']),
('A 2002-2003-ben.', ['A', '2002-2003-ben', '.']),
('A +0,99% van.', ['A', '+0,99%', 'van', '.']),
('A -0,99% van.', ['A', '-0,99%', 'van', '.']),
('A -0,99%-ben van.', ['A', '-0,99%-ben', 'van', '.']),
@ -217,7 +217,7 @@ QUOTE_TESTS = [
('Az "Ime, hat"-ban irja.', ['Az', '"', 'Ime', ',', 'hat', '"', '-ban', 'irja', '.']),
('"Ime, hat"-ban irja.', ['"', 'Ime', ',', 'hat', '"', '-ban', 'irja', '.']),
('Az "Ime, hat".', ['Az', '"', 'Ime', ',', 'hat', '"', '.']),
('Egy 24"-os monitor.', ['Egy', '24', '"', '-os', 'monitor', '.']),
('Egy 24"-os monitor.', ['Egy', '24"-os', 'monitor', '.']),
# ("A don't van.", ['A', "don't", 'van', '.'])
]
@ -241,11 +241,14 @@ DOT_TESTS = [
WIKI_TESTS = [
('!"', ['!', '"']),
('lány"a', ['lány', '"', 'a']),
('lány"a', ['lány', '"', 'a']),
('!"-lel', ['!', '"', '-lel']),
('""-sorozat ', ['"', '"', '-sorozat']),
('"(Köszönöm', ['"', '(', 'Köszönöm']),
('(törvénykönyv)-ben ', ['(', 'törvénykönyv', ')', '-ben']),
('"(...)"–sokkal ', ['"', '(', '...', ')', '"', '–sokkal']),
('cérium(IV)-oxid', ['cérium', '(', 'IV', ')', '-oxid'])
]
TESTCASES = DEFAULT_TESTS + DOT_TESTS + QUOTE_TESTS + NUMBER_TESTS + HYPHEN_TESTS + WIKI_TESTS