spaCy/spacy/lang/tokenizer_exceptions.py

246 lines
3.1 KiB
Python
Raw Normal View History

2017-05-08 13:55:52 +00:00
# coding: utf8
from __future__ import unicode_literals
import re
from ..symbols import ORTH, POS, TAG, LEMMA, SPACE, PUNCT
2017-05-08 13:55:52 +00:00
2017-03-04 22:13:11 +00:00
# URL validation regex courtesy of: https://mathiasbynens.be/demo/url-regex
# A few minor mods to this regex to account for use cases represented in test_urls
2017-05-08 22:00:00 +00:00
URL_PATTERN = (
2017-03-04 22:13:11 +00:00
r"^"
# in order to support the prefix tokenization (see prefix test cases in test_urls).
r"(?=[\w])"
# protocol identifier
r"(?:(?:https?|ftp|mailto)://)?"
# user:pass authentication
r"(?:\S+(?::\S*)?@)?"
r"(?:"
# IP address exclusion
# private & local networks
r"(?!(?:10|127)(?:\.\d{1,3}){3})"
r"(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})"
r"(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})"
# IP address dotted notation octets
# excludes loopback network 0.0.0.0
# excludes reserved space >= 224.0.0.0
# excludes network & broadcast addresses
# (first & last IP address of each class)
# MH: Do we really need this? Seems excessive, and seems to have caused
# Issue #957
2017-03-04 22:13:11 +00:00
r"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])"
r"(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}"
r"(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))"
r"|"
# host name
r"(?:(?:[a-z0-9\-]*)?[a-z0-9]+)"
2017-03-04 22:13:11 +00:00
# domain name
r"(?:\.(?:[a-z0-9])(?:[a-z0-9\-])*[a-z0-9])?"
2017-03-04 22:13:11 +00:00
# TLD identifier
r"(?:\.(?:[a-z]{2,}))"
2017-03-04 22:13:11 +00:00
r")"
# port number
r"(?::\d{2,5})?"
# resource path
r"(?:/\S*)?"
# query parameters
r"\??(:?\S*)?"
# in order to support the suffix tokenization (see suffix test cases in test_urls),
r"(?<=[\w/])"
r"$"
).strip()
2017-05-08 22:34:31 +00:00
TOKEN_MATCH = re.compile(URL_PATTERN, re.UNICODE).match
2017-05-08 13:55:52 +00:00
BASE_EXCEPTIONS = {}
for exc_data in [
{ORTH: " ", POS: SPACE, TAG: "_SP"},
{ORTH: "\t", POS: SPACE, TAG: "_SP"},
{ORTH: "\\t", POS: SPACE, TAG: "_SP"},
{ORTH: "\n", POS: SPACE, TAG: "_SP"},
{ORTH: "\\n", POS: SPACE, TAG: "_SP"},
{ORTH: "\u2014"},
{ORTH: "\u00a0", POS: SPACE, LEMMA: " ", TAG: "_SP"},
]:
2017-11-01 22:02:45 +00:00
BASE_EXCEPTIONS[exc_data[ORTH]] = [exc_data]
2017-05-08 13:55:52 +00:00
for orth in [
"'",
'\\")',
"<space>",
"''",
"C++",
"a.",
"b.",
"c.",
"d.",
"e.",
"f.",
"g.",
"h.",
"i.",
"j.",
"k.",
"l.",
"m.",
"n.",
"o.",
"p.",
"q.",
"r.",
"s.",
"t.",
"u.",
"v.",
"w.",
"x.",
"y.",
"z.",
"ä.",
"ö.",
"ü.",
]:
2017-05-08 13:55:52 +00:00
BASE_EXCEPTIONS[orth] = [{ORTH: orth}]
emoticons = set(
"""
2017-05-08 13:55:52 +00:00
:)
:-)
:))
:-))
:)))
:-)))
(:
(-:
=)
(=
")
:]
:-]
[:
[-:
:o)
(o:
:}
:-}
8)
8-)
(-8
;)
;-)
(;
(-;
:(
:-(
:((
:-((
:(((
:-(((
):
)-:
=(
>:(
:')
:'-)
:'(
:'-(
:/
:-/
=/
=|
:|
:-|
:1
:P
:-P
:p
:-p
:O
:-O
:o
:-o
:0
:-0
:()
>:o
:*
:-*
:3
:-3
=3
:>
:->
:X
:-X
:x
:-x
:D
:-D
;D
;-D
=D
xD
XD
xDD
XDD
8D
8-D
^_^
^__^
^___^
>.<
>.>
<.<
._.
;_;
-_-
-__-
v.v
V.V
v_v
V_V
o_o
o_O
O_o
O_O
0_o
o_0
0_0
o.O
O.o
O.O
o.o
0.0
o.0
0.o
@_@
<3
<33
<333
</3
(^_^)
(-_-)
(._.)
(>_<)
(*_*)
(¬_¬)
ಠ_ಠ
(ಠ_ಠ)
¯\()/¯
(°°
><(((*>
""".split()
)
2017-05-08 13:55:52 +00:00
for orth in emoticons:
BASE_EXCEPTIONS[orth] = [{ORTH: orth}]