spaCy/spacy/en.pyx

65 lines
1.8 KiB
Cython
Raw Normal View History

# cython: profile=True
'''Serve pointers to Lexeme structs, given strings. Maintain a reverse index,
so that strings can be retrieved from hashes. Use 64-bit hash values and
boldly assume no collisions.
'''
from __future__ import unicode_literals
from libc.stdlib cimport malloc, calloc, free
from libc.stdint cimport uint64_t
from libcpp.vector cimport vector
from spacy.string_tools cimport substr
from . import util
cimport spacy
cdef class English(spacy.Language):
cdef int find_split(self, unicode word, size_t length):
cdef int i = 0
# Contractions
if word.endswith("'s"):
return length - 2
# Leading punctuation
if is_punct(word, 0, length):
return 1
elif length >= 1:
# Split off all trailing punctuation characters
i = 0
while i < length and not is_punct(word, i, length):
i += 1
return i
2014-07-06 16:35:55 +00:00
2014-07-06 16:35:55 +00:00
cdef bint is_punct(unicode word, size_t i, size_t length):
# Don't count appostrophes as punct if the next char is a letter
if word[i] == "'" and i < (length - 1) and word[i+1].isalpha():
2014-07-07 21:26:01 +00:00
# ...Unless we're at 0
return i == 0
2014-07-23 16:35:18 +00:00
if word[i] == "-" and i < (length - 1) and word[i+1] == '-':
return False
# Don't count commas as punct if the next char is a number
if word[i] == "," and i < (length - 1) and word[i+1].isdigit():
return False
2014-07-23 16:35:18 +00:00
# Don't count periods as punct if the next char is not whitespace
if word[i] == "." and i < (length - 1) and not word[i+1].isspace():
return False
return not word[i].isalnum()
EN = English('en')
cpdef Tokens tokenize(unicode string):
return EN.tokenize(string)
cpdef Lexeme_addr lookup(unicode string) except 0:
2014-08-16 01:22:03 +00:00
return EN.lookup_chunk(string)
cpdef unicode unhash(StringHash hash_value):
return EN.unhash(hash_value)