2014-07-07 05:36:43 +00:00
|
|
|
# cython: profile=True
|
2014-08-20 11:39:39 +00:00
|
|
|
# cython: embedsignature=True
|
|
|
|
"""Common classes and utilities across languages.
|
|
|
|
|
|
|
|
Provides the main implementation for the spacy tokenizer. Specific languages
|
|
|
|
subclass the Language class, over-writing the tokenization rules as necessary.
|
|
|
|
Special-case tokenization rules are read from data/<lang>/tokenization .
|
|
|
|
"""
|
2014-07-05 18:51:42 +00:00
|
|
|
from __future__ import unicode_literals
|
2014-07-07 02:21:06 +00:00
|
|
|
|
2014-07-07 06:05:29 +00:00
|
|
|
from libc.stdlib cimport calloc, free
|
|
|
|
|
2014-07-07 02:21:06 +00:00
|
|
|
from . import util
|
2014-08-27 15:15:39 +00:00
|
|
|
import json
|
2014-07-07 05:36:43 +00:00
|
|
|
from os import path
|
2014-08-16 17:59:38 +00:00
|
|
|
|
2014-07-07 14:58:48 +00:00
|
|
|
|
2014-07-07 10:47:21 +00:00
|
|
|
cdef class Language:
|
|
|
|
def __cinit__(self, name):
|
|
|
|
self.name = name
|
2014-08-27 15:15:39 +00:00
|
|
|
self.cache = {}
|
|
|
|
self.lexicon = Lexicon()
|
2014-07-07 10:47:21 +00:00
|
|
|
self.load_tokenization(util.read_tokenization(name))
|
2014-08-20 11:39:39 +00:00
|
|
|
|
2014-08-22 15:28:23 +00:00
|
|
|
cpdef list tokenize(self, unicode string):
|
2014-08-27 15:15:39 +00:00
|
|
|
"""Tokenize a string.
|
|
|
|
|
|
|
|
The tokenization rules are defined in two places:
|
2014-08-20 11:39:39 +00:00
|
|
|
|
2014-08-27 15:15:39 +00:00
|
|
|
* The data/<lang>/tokenization table, which handles special cases like contractions;
|
|
|
|
* The appropriate :py:meth:`find_split` function, which is used to split
|
|
|
|
off punctuation etc.
|
2014-07-07 10:47:21 +00:00
|
|
|
|
2014-08-20 11:39:39 +00:00
|
|
|
Args:
|
2014-08-27 15:15:39 +00:00
|
|
|
string (unicode): The string to be tokenized.
|
2014-08-20 11:39:39 +00:00
|
|
|
|
|
|
|
Returns:
|
2014-08-27 15:15:39 +00:00
|
|
|
tokens (Tokens): A Tokens object, giving access to a sequence of LexIDs.
|
2014-08-20 11:39:39 +00:00
|
|
|
"""
|
2014-08-22 15:28:23 +00:00
|
|
|
cdef list tokens = []
|
2014-08-18 18:23:54 +00:00
|
|
|
cdef size_t length = len(string)
|
2014-08-18 17:59:59 +00:00
|
|
|
cdef size_t start = 0
|
2014-08-18 18:48:48 +00:00
|
|
|
cdef size_t i = 0
|
|
|
|
for c in string:
|
2014-08-25 14:42:22 +00:00
|
|
|
if c == ' ':
|
2014-08-18 17:59:59 +00:00
|
|
|
if start < i:
|
2014-08-27 15:15:39 +00:00
|
|
|
tokens.extend(self._tokenize(string[start:i]))
|
2014-08-18 17:59:59 +00:00
|
|
|
start = i + 1
|
2014-08-18 18:48:48 +00:00
|
|
|
i += 1
|
2014-08-18 17:59:59 +00:00
|
|
|
if start < i:
|
2014-08-27 15:15:39 +00:00
|
|
|
tokens.extend(self._tokenize(string[start:]))
|
2014-07-07 10:47:21 +00:00
|
|
|
return tokens
|
|
|
|
|
2014-08-27 15:15:39 +00:00
|
|
|
cdef list _tokenize(self, unicode string):
|
|
|
|
if string in self.cache:
|
|
|
|
return self.cache[string]
|
|
|
|
cdef list lexemes = []
|
|
|
|
substrings = self._split(string)
|
2014-08-18 17:14:00 +00:00
|
|
|
for i, substring in enumerate(substrings):
|
2014-08-27 15:15:39 +00:00
|
|
|
lexemes.append(self.lookup(substring))
|
|
|
|
self.cache[string] = lexemes
|
|
|
|
return lexemes
|
|
|
|
|
|
|
|
cpdef list _split(self, unicode string):
|
|
|
|
"""Find how to split a contiguous span of non-space characters into substrings.
|
2014-08-20 11:39:39 +00:00
|
|
|
|
|
|
|
This method calls find_split repeatedly. Most languages will want to
|
2014-08-27 15:15:39 +00:00
|
|
|
override _split_one, but it may be useful to override this instead.
|
2014-08-20 11:39:39 +00:00
|
|
|
|
|
|
|
Args:
|
|
|
|
chunk (unicode): The string to be split, e.g. u"Mike's!"
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
substrings (list): The component substrings, e.g. [u"Mike", "'s", "!"].
|
|
|
|
"""
|
2014-08-18 17:14:00 +00:00
|
|
|
substrings = []
|
2014-08-27 15:15:39 +00:00
|
|
|
while string:
|
|
|
|
split = self._split_one(string)
|
2014-08-18 17:14:00 +00:00
|
|
|
if split == 0:
|
2014-08-27 15:15:39 +00:00
|
|
|
substrings.append(string)
|
2014-08-18 17:14:00 +00:00
|
|
|
break
|
2014-08-27 15:15:39 +00:00
|
|
|
substrings.append(string[:split])
|
|
|
|
string = string[split:]
|
2014-08-18 17:14:00 +00:00
|
|
|
return substrings
|
|
|
|
|
2014-08-27 15:15:39 +00:00
|
|
|
cpdef int _split_one(self, unicode word):
|
2014-08-18 17:14:00 +00:00
|
|
|
return len(word)
|
2014-08-16 01:22:03 +00:00
|
|
|
|
2014-08-27 15:15:39 +00:00
|
|
|
def load_special_tokenization(self, token_rules):
|
2014-08-20 11:39:39 +00:00
|
|
|
'''Load special-case tokenization rules.
|
|
|
|
|
2014-08-27 15:15:39 +00:00
|
|
|
Loads special-case tokenization rules into the Language.cache cache,
|
2014-08-20 11:39:39 +00:00
|
|
|
read from data/<lang>/tokenization . The special cases are loaded before
|
|
|
|
any language data is tokenized, giving these priority. For instance,
|
|
|
|
the English tokenization rules map "ain't" to ["are", "not"].
|
|
|
|
|
|
|
|
Args:
|
|
|
|
token_rules (list): A list of (chunk, tokens) pairs, where chunk is
|
|
|
|
a string and tokens is a list of strings.
|
|
|
|
'''
|
2014-08-27 15:15:39 +00:00
|
|
|
for string, substrings in token_rules:
|
|
|
|
lexemes = []
|
|
|
|
for i, substring in enumerate(substrings):
|
|
|
|
lexemes.append(self.lookup(substring))
|
|
|
|
self.cache[string] = lexemes
|
|
|
|
|
2014-08-18 17:59:59 +00:00
|
|
|
|
2014-08-27 15:15:39 +00:00
|
|
|
cdef class Lexicon:
|
|
|
|
def __cinit__(self):
|
|
|
|
self.flag_checkers = []
|
|
|
|
self.string_transforms = []
|
|
|
|
self.lexicon = {}
|
2014-08-20 11:39:39 +00:00
|
|
|
|
2014-08-27 15:15:39 +00:00
|
|
|
cpdef Lexeme lookup(self, unicode string):
|
|
|
|
"""Retrieve (or create, if not found) a Lexeme for a string, and return it.
|
|
|
|
|
|
|
|
Args:
|
|
|
|
string (unicode): The string to be looked up. Must be unicode, not bytes.
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
lexeme (Lexeme): A reference to a lexical type.
|
|
|
|
"""
|
|
|
|
assert len(string) != 0
|
|
|
|
if string in self.lexicon:
|
|
|
|
return self.lexicon[string]
|
|
|
|
|
|
|
|
prob = _pop_default(self.probs, string, 0.0)
|
|
|
|
cluster = _pop_default(self.clusters, string, 0.0)
|
|
|
|
case_stats = _pop_default(self.case_stats, string, {})
|
|
|
|
tag_stats = _pop_default(self.tag_stats, string, {})
|
|
|
|
|
|
|
|
cdef Lexeme word = Lexeme(string, prob, cluster, case_stats, tag_stats,
|
|
|
|
self.flag_checkers, self.string_transformers)
|
|
|
|
self.lexicon[string] = word
|
|
|
|
return word
|
|
|
|
|
|
|
|
def add_flag(self, flag_checker):
|
|
|
|
cdef unicode string
|
|
|
|
cdef Lexeme word
|
|
|
|
flag_id = len(self.flag_checkers)
|
|
|
|
for string, word in self.lexicon.items():
|
|
|
|
if flag_checker(string, word.prob, {}):
|
|
|
|
word.set_flag(flag_id)
|
|
|
|
self.flag_checkers.append(flag_checker)
|
|
|
|
return flag_id
|
|
|
|
|
|
|
|
def add_transform(self, string_transform):
|
|
|
|
self.string_transformers.append(string_transform)
|
|
|
|
return len(self.string_transformers) - 1
|
|
|
|
|
|
|
|
def load_probs(self, location):
|
|
|
|
"""Load unigram probabilities.
|
|
|
|
"""
|
|
|
|
self.probs = json.load(location)
|
|
|
|
|
|
|
|
cdef Lexeme word
|
|
|
|
cdef unicode string
|
|
|
|
|
|
|
|
for string, word in self.lexicon.items():
|
|
|
|
prob = _pop_default(self.probs, string, 0.0)
|
|
|
|
word.prob = prob
|
|
|
|
|
|
|
|
def load_clusters(self, location):
|
|
|
|
self.probs = json.load(location)
|
|
|
|
|
|
|
|
cdef Lexeme word
|
2014-08-19 00:40:37 +00:00
|
|
|
cdef unicode string
|
2014-08-27 15:15:39 +00:00
|
|
|
|
|
|
|
for string, word in self.lexicon.items():
|
|
|
|
cluster = _pop_default(self.cluster, string, 0)
|
|
|
|
word.cluster = cluster
|
|
|
|
|
|
|
|
def load_stats(self, location):
|
|
|
|
"""Load distributional stats.
|
|
|
|
"""
|
|
|
|
raise NotImplementedError
|
|
|
|
|
|
|
|
|
|
|
|
def _pop_default(dict d, key, default):
|
|
|
|
return d.pop(key) if key in d else default
|