spaCy/spacy/orth.py

114 lines
2.2 KiB
Python
Raw Normal View History

# -*- coding: utf8 -*-
from __future__ import unicode_literals
import unicodedata
from unidecode import unidecode
import math
2014-09-10 16:11:13 +00:00
TAGS = 'adj adp adv conj det noun num pdt pos pron prt punct verb'.upper().split()
2014-08-30 17:01:00 +00:00
# Binary string features
def is_alpha(string):
return string.isalpha()
2014-08-30 17:01:00 +00:00
def is_digit(string):
return string.isdigit()
2014-08-30 17:01:00 +00:00
def is_punct(string):
for c in string:
2014-09-01 21:41:31 +00:00
if not unicodedata.category(c).startswith('P'):
return False
else:
2014-09-01 21:41:31 +00:00
return True
2014-08-30 17:01:00 +00:00
def is_space(string):
return string.isspace()
2014-08-30 17:01:00 +00:00
def is_ascii(string):
for c in string:
2014-09-01 21:41:31 +00:00
if ord(c) >= 128:
return False
else:
return True
2014-08-30 17:01:00 +00:00
def is_title(string):
return string.istitle()
2014-08-30 17:01:00 +00:00
def is_lower(string):
return string.islower()
2014-08-30 17:01:00 +00:00
def is_upper(string):
return string.isupper()
2014-08-30 17:01:00 +00:00
# Statistics features
def oft_case(name, thresh):
def wrapped(string, prob, case_stats, tag_stats):
return string
return wrapped
2014-09-10 16:27:44 +00:00
def can_tag(name, thresh=0.5):
2014-08-30 17:01:00 +00:00
def wrapped(string, prob, case_stats, tag_stats):
return string
return wrapped
# String features
2014-10-29 12:19:38 +00:00
def canon_case(string, upper_pc=0.0, title_pc=0.0, lower_pc=0.0):
2014-08-30 18:57:43 +00:00
if upper_pc >= lower_pc and upper_pc >= title_pc:
return string.upper()
elif title_pc >= lower_pc:
return string.title()
else:
return string.lower()
2014-08-30 17:01:00 +00:00
2014-10-29 12:19:38 +00:00
def word_shape(string):
2014-08-30 17:01:00 +00:00
length = len(string)
shape = []
2014-08-30 17:01:00 +00:00
last = ""
shape_char = ""
seq = 0
for c in string:
if c.isalpha():
if c.isupper():
shape_char = "X"
else:
shape_char = "x"
elif c.isdigit():
shape_char = "d"
else:
shape_char = c
if shape_char == last:
seq += 1
else:
seq = 0
last = shape_char
if seq < 5:
shape.append(shape_char)
return ''.join(shape)
2014-08-30 17:01:00 +00:00
2014-10-29 12:19:38 +00:00
def non_sparse(string, prob, cluster, upper_pc, title_pc, lower_pc):
if is_alpha(string):
2014-10-29 12:19:38 +00:00
return canon_case(string, upper_pc, title_pc, lower_pc)
elif prob >= math.log(0.0001):
return string
else:
2014-10-29 12:19:38 +00:00
return word_shape(string)
2014-10-29 12:19:38 +00:00
def asciied(string):
ascii_string = unidecode(string)
return ascii_string.decode('ascii')