Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tidy up and improve docs and docstrings (#3370) 

<!--- Provide a general summary of your changes in the title. -->

## Description
* tidy up and adjust Cython code to code style
* improve docstrings and make calling `help()` nicer
* add URLs to new docs pages to docstrings wherever possible, mostly to user-facing objects
* fix various typos and inconsistencies in docs

### Types of change
enhancement, docs

## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
This commit is contained in:
Ines Montani 2019-03-08 11:42:26 +01:00 committed by GitHub
parent daaeeb7a2b
commit 296446a1c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 1942 additions and 1390 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
spacy/compat.py
View File

@ -1,4 +1,11 @@
# coding: utf8
"""
Helpers for Python and platform compatibility. To distinguish them from
the builtin functions, replacement functions are suffixed with an underscore,
e.g. `unicode_`.
DOCS: https://spacy.io/api/top-level#compat
"""
from __future__ import unicode_literals
import os
@ -64,19 +71,23 @@ elif is_python3:
def b_to_str(b_str):
"""Convert a bytes object to a string.
b_str (bytes): The object to convert.
RETURNS (unicode): The converted string.
"""
if is_python2:
return b_str
# important: if no encoding is set, string becomes "b'...'"
# Important: if no encoding is set, string becomes "b'...'"
return str(b_str, encoding="utf8")
def getattr_(obj, name, *default):
if is_python3 and isinstance(name, bytes):
name = name.decode("utf8")
return getattr(obj, name, *default)
def symlink_to(orig, dest):
"""Create a symlink. Used for model shortcut links.
orig (unicode / Path): The origin path.
dest (unicode / Path): The destination path of the symlink.
"""
if is_windows:
import subprocess
@ -86,6 +97,10 @@ def symlink_to(orig, dest):
def symlink_remove(link):
"""Remove a symlink. Used for model shortcut links.
link (unicode / Path): The path to the symlink.
"""
# https://stackoverflow.com/q/26554135/6400719
if os.path.isdir(path2str(link)) and is_windows:
# this should only be on Py2.7 and windows
@ -95,6 +110,18 @@ def symlink_remove(link):
def is_config(python2=None, python3=None, windows=None, linux=None, osx=None):
"""Check if a specific configuration of Python version and operating system
matches the user's setup. Mostly used to display targeted error messages.
python2 (bool): spaCy is executed with Python 2.x.
python3 (bool): spaCy is executed with Python 3.x.
windows (bool): spaCy is executed on Windows.
linux (bool): spaCy is executed on Linux.
osx (bool): spaCy is executed on OS X or macOS.
RETURNS (bool): Whether the configuration matches the user's platform.
DOCS: https://spacy.io/api/top-level#compat.is_config
"""
return (
python2 in (None, is_python2)
and python3 in (None, is_python3)
@ -104,19 +131,14 @@ def is_config(python2=None, python3=None, windows=None, linux=None, osx=None):
)
def normalize_string_keys(old):
"""Given a dictionary, make sure keys are unicode strings, not bytes."""
new = {}
for key, value in old.items():
if isinstance(key, bytes_):
new[key.decode("utf8")] = value
else:
new[key] = value
return new
def import_file(name, loc):
loc = str(loc)
"""Import module from a file. Used to load models from a directory.
name (unicode): Name of module to load.
loc (unicode / Path): Path to the file.
RETURNS: The loaded module.
"""
loc = path2str(loc)
if is_python_pre_3_5:
import imp

12
spacy/displacy/__init__.py
View File

@ -1,4 +1,10 @@
# coding: utf8
"""
spaCy's built in visualization suite for dependencies and named entities.
DOCS: https://spacy.io/api/top-level#displacy
USAGE: https://spacy.io/usage/visualizers
"""
from __future__ import unicode_literals
from .render import DependencyRenderer, EntityRenderer
@ -25,6 +31,9 @@ def render(
options (dict): Visualiser-specific options, e.g. colors.
manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts.
RETURNS (unicode): Rendered HTML markup.
DOCS: https://spacy.io/api/top-level#displacy.render
USAGE: https://spacy.io/usage/visualizers
"""
factories = {
"dep": (DependencyRenderer, parse_deps),
@ -71,6 +80,9 @@ def serve(
manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts.
port (int): Port to serve visualisation.
host (unicode): Host to serve visualisation.
DOCS: https://spacy.io/api/top-level#displacy.serve
USAGE: https://spacy.io/usage/visualizers
"""
from wsgiref import simple_server

11
spacy/errors.py
View File

@ -338,6 +338,17 @@ class Errors(object):
"or with a getter AND setter.")
E120 = ("Can't set custom extension attributes during retokenization. "
"Expected dict mapping attribute names to values, but got: {value}")
E121 = ("Can't bulk merge spans. Attribute length {attr_len} should be "
"equal to span length ({span_len}).")
E122 = ("Cannot find token to be split. Did it get merged?")
E123 = ("Cannot find head of token to be split. Did it get merged?")
E124 = ("Cannot read from file: {path}. Supported formats: .json, .msg")
E125 = ("Unexpected value: {value}")
E126 = ("Unexpected matcher predicate: '{bad}'. Expected one of: {good}. "
"This is likely a bug in spaCy, so feel free to open an issue.")
E127 = ("Cannot create phrase pattern representation for length 0. This "
"is likely a bug in spaCy.")
@add_codes

196
spacy/gold.pyx
View File

@ -14,34 +14,38 @@ from . import _align
from .syntax import nonproj
from .tokens import Doc, Span
from .errors import Errors
from .compat import path2str
from . import util
from .util import minibatch, itershuffle
from libc.stdio cimport FILE, fopen, fclose, fread, fwrite, feof, fseek
punct_re = re.compile(r"\W")
def tags_to_entities(tags):
entities = []
start = None
for i, tag in enumerate(tags):
if tag is None:
continue
if tag.startswith('O'):
if tag.startswith("O"):
# TODO: We shouldn't be getting these malformed inputs. Fix this.
if start is not None:
start = None
continue
elif tag == '-':
elif tag == "-":
continue
elif tag.startswith('I'):
elif tag.startswith("I"):
if start is None:
raise ValueError(Errors.E067.format(tags=tags[:i+1]))
raise ValueError(Errors.E067.format(tags=tags[:i + 1]))
continue
if tag.startswith('U'):
if tag.startswith("U"):
entities.append((tag[2:], i, i))
elif tag.startswith('B'):
elif tag.startswith("B"):
start = i
elif tag.startswith('L'):
elif tag.startswith("L"):
entities.append((tag[2:], start, i))
start = None
else:
@ -60,19 +64,18 @@ def merge_sents(sents):
m_deps[3].extend(head + i for head in heads)
m_deps[4].extend(labels)
m_deps[5].extend(ner)
m_brackets.extend((b['first'] + i, b['last'] + i, b['label'])
m_brackets.extend((b["first"] + i, b["last"] + i, b["label"])
for b in brackets)
i += len(ids)
return [(m_deps, m_brackets)]
punct_re = re.compile(r'\W')
def align(cand_words, gold_words):
if cand_words == gold_words:
alignment = numpy.arange(len(cand_words))
return 0, alignment, alignment, {}, {}
cand_words = [w.replace(' ', '').lower() for w in cand_words]
gold_words = [w.replace(' ', '').lower() for w in gold_words]
cand_words = [w.replace(" ", "").lower() for w in cand_words]
gold_words = [w.replace(" ", "").lower() for w in gold_words]
cost, i2j, j2i, matrix = _align.align(cand_words, gold_words)
i2j_multi, j2i_multi = _align.multi_align(i2j, j2i, [len(w) for w in cand_words],
[len(w) for w in gold_words])
@ -89,7 +92,10 @@ def align(cand_words, gold_words):
class GoldCorpus(object):
"""An annotated corpus, using the JSON file format. Manages
annotations for tagging, dependency parsing and NER."""
annotations for tagging, dependency parsing and NER.
DOCS: https://spacy.io/api/goldcorpus
"""
def __init__(self, train, dev, gold_preproc=False, limit=None):
"""Create a GoldCorpus.
@ -101,12 +107,10 @@ class GoldCorpus(object):
if isinstance(train, str) or isinstance(train, Path):
train = self.read_tuples(self.walk_corpus(train))
dev = self.read_tuples(self.walk_corpus(dev))
# Write temp directory with one doc per file, so we can shuffle
# and stream
# Write temp directory with one doc per file, so we can shuffle and stream
self.tmp_dir = Path(tempfile.mkdtemp())
self.write_msgpack(self.tmp_dir / 'train', train, limit=self.limit)
self.write_msgpack(self.tmp_dir / 'dev', dev, limit=self.limit)
self.write_msgpack(self.tmp_dir / "train", train, limit=self.limit)
self.write_msgpack(self.tmp_dir / "dev", dev, limit=self.limit)
def __del__(self):
shutil.rmtree(self.tmp_dir)
@ -117,7 +121,7 @@ class GoldCorpus(object):
directory.mkdir()
n = 0
for i, doc_tuple in enumerate(doc_tuples):
srsly.write_msgpack(directory / '{}.msg'.format(i), [doc_tuple])
srsly.write_msgpack(directory / "{}.msg".format(i), [doc_tuple])
n += len(doc_tuple[1])
if limit and n >= limit:
break
@ -134,11 +138,11 @@ class GoldCorpus(object):
if str(path) in seen:
continue
seen.add(str(path))
if path.parts[-1].startswith('.'):
if path.parts[-1].startswith("."):
continue
elif path.is_dir():
paths.extend(path.iterdir())
elif path.parts[-1].endswith('.json'):
elif path.parts[-1].endswith(".json"):
locs.append(path)
return locs
@ -147,13 +151,12 @@ class GoldCorpus(object):
i = 0
for loc in locs:
loc = util.ensure_path(loc)
if loc.parts[-1].endswith('json'):
if loc.parts[-1].endswith("json"):
gold_tuples = read_json_file(loc)
elif loc.parts[-1].endswith('msg'):
elif loc.parts[-1].endswith("msg"):
gold_tuples = srsly.read_msgpack(loc)
else:
msg = "Cannot read from file: %s. Supported formats: .json, .msg"
raise ValueError(msg % loc)
raise ValueError(Errors.E124.format(path=path2str(loc)))
for item in gold_tuples:
yield item
i += len(item[1])
@ -162,12 +165,12 @@ class GoldCorpus(object):
@property
def dev_tuples(self):
locs = (self.tmp_dir / 'dev').iterdir()
locs = (self.tmp_dir / "dev").iterdir()
yield from self.read_tuples(locs, limit=self.limit)
@property
def train_tuples(self):
locs = (self.tmp_dir / 'train').iterdir()
locs = (self.tmp_dir / "train").iterdir()
yield from self.read_tuples(locs, limit=self.limit)
def count_train(self):
@ -193,8 +196,7 @@ class GoldCorpus(object):
yield from gold_docs
def dev_docs(self, nlp, gold_preproc=False):
gold_docs = self.iter_gold_docs(nlp, self.dev_tuples,
gold_preproc=gold_preproc)
gold_docs = self.iter_gold_docs(nlp, self.dev_tuples, gold_preproc=gold_preproc)
yield from gold_docs
@classmethod
@ -205,32 +207,29 @@ class GoldCorpus(object):
raw_text = None
else:
paragraph_tuples = merge_sents(paragraph_tuples)
docs = cls._make_docs(nlp, raw_text, paragraph_tuples,
gold_preproc, noise_level=noise_level)
docs = cls._make_docs(nlp, raw_text, paragraph_tuples, gold_preproc,
noise_level=noise_level)
golds = cls._make_golds(docs, paragraph_tuples, make_projective)
for doc, gold in zip(docs, golds):
if (not max_length) or len(doc) < max_length:
yield doc, gold
@classmethod
def _make_docs(cls, nlp, raw_text, paragraph_tuples, gold_preproc,
noise_level=0.0):
def _make_docs(cls, nlp, raw_text, paragraph_tuples, gold_preproc, noise_level=0.0):
if raw_text is not None:
raw_text = add_noise(raw_text, noise_level)
return [nlp.make_doc(raw_text)]
else:
return [Doc(nlp.vocab,
words=add_noise(sent_tuples[1], noise_level))
return [Doc(nlp.vocab, words=add_noise(sent_tuples[1], noise_level))
for (sent_tuples, brackets) in paragraph_tuples]
@classmethod
def _make_golds(cls, docs, paragraph_tuples, make_projective):
if len(docs) != len(paragraph_tuples):
raise ValueError(Errors.E070.format(n_docs=len(docs),
n_annots=len(paragraph_tuples)))
n_annots = len(paragraph_tuples)
raise ValueError(Errors.E070.format(n_docs=len(docs), n_annots=n_annots))
if len(docs) == 1:
return [GoldParse.from_annot_tuples(docs[0],
paragraph_tuples[0][0],
return [GoldParse.from_annot_tuples(docs[0], paragraph_tuples[0][0],
make_projective=make_projective)]
else:
return [GoldParse.from_annot_tuples(doc, sent_tuples,
@ -247,18 +246,18 @@ def add_noise(orig, noise_level):
corrupted = [w for w in corrupted if w]
return corrupted
else:
return ''.join(_corrupt(c, noise_level) for c in orig)
return "".join(_corrupt(c, noise_level) for c in orig)
def _corrupt(c, noise_level):
if random.random() >= noise_level:
return c
elif c == ' ':
return '\n'
elif c == '\n':
return ' '
elif c in ['.', "'", "!", "?", ',']:
return ''
elif c == " ":
return "\n"
elif c == "\n":
return " "
elif c in [".", "'", "!", "?", ","]:
return ""
else:
return c.lower()
@ -284,30 +283,30 @@ def json_to_tuple(doc):
YIELDS (tuple): The reformatted data.
"""
paragraphs = []
for paragraph in doc['paragraphs']:
for paragraph in doc["paragraphs"]:
sents = []
for sent in paragraph['sentences']:
for sent in paragraph["sentences"]:
words = []
ids = []
tags = []
heads = []
labels = []
ner = []
for i, token in enumerate(sent['tokens']):
words.append(token['orth'])
for i, token in enumerate(sent["tokens"]):
words.append(token["orth"])
ids.append(i)
tags.append(token.get('tag', '-'))
heads.append(token.get('head', 0) + i)
labels.append(token.get('dep', ''))
tags.append(token.get('tag', "-"))
heads.append(token.get("head", 0) + i)
labels.append(token.get("dep", ""))
# Ensure ROOT label is case-insensitive
if labels[-1].lower() == 'root':
labels[-1] = 'ROOT'
ner.append(token.get('ner', '-'))
if labels[-1].lower() == "root":
labels[-1] = "ROOT"
ner.append(token.get("ner", "-"))
sents.append([
[ids, words, tags, heads, labels, ner],
sent.get('brackets', [])])
sent.get("brackets", [])])
if sents:
yield [paragraph.get('raw', None), sents]
yield [paragraph.get("raw", None), sents]
def read_json_file(loc, docs_filter=None, limit=None):
@ -329,7 +328,7 @@ def _json_iterate(loc):
# It's okay to read in the whole file -- just don't parse it into JSON.
cdef bytes py_raw
loc = util.ensure_path(loc)
with loc.open('rb') as file_:
with loc.open("rb") as file_:
py_raw = file_.read()
raw = <char*>py_raw
cdef int square_depth = 0
@ -339,11 +338,11 @@ def _json_iterate(loc):
cdef int start = -1
cdef char c
cdef char quote = ord('"')
cdef char backslash = ord('\\')
cdef char open_square = ord('[')
cdef char close_square = ord(']')
cdef char open_curly = ord('{')
cdef char close_curly = ord('}')
cdef char backslash = ord("\\")
cdef char open_square = ord("[")
cdef char close_square = ord("]")
cdef char open_curly = ord("{")
cdef char close_curly = ord("}")
for i in range(len(py_raw)):
c = raw[i]
if escape:
@ -368,7 +367,7 @@ def _json_iterate(loc):
elif c == close_curly:
curly_depth -= 1
if square_depth == 1 and curly_depth == 0:
py_str = py_raw[start : i+1].decode('utf8')
py_str = py_raw[start : i + 1].decode("utf8")
try:
yield srsly.json_loads(py_str)
except Exception:
@ -388,7 +387,7 @@ def iob_to_biluo(tags):
def _consume_os(tags):
while tags and tags[0] == 'O':
while tags and tags[0] == "O":
yield tags.pop(0)
@ -396,24 +395,27 @@ def _consume_ent(tags):
if not tags:
return []
tag = tags.pop(0)
target_in = 'I' + tag[1:]
target_last = 'L' + tag[1:]
target_in = "I" + tag[1:]
target_last = "L" + tag[1:]
length = 1
while tags and tags[0] in {target_in, target_last}:
length += 1
tags.pop(0)
label = tag[2:]
if length == 1:
return ['U-' + label]
return ["U-" + label]
else:
start = 'B-' + label
end = 'L-' + label
middle = ['I-%s' % label for _ in range(1, length - 1)]
start = "B-" + label
end = "L-" + label
middle = ["I-%s" % label for _ in range(1, length - 1)]
return [start] + middle + [end]
cdef class GoldParse:
"""Collection for training annotations."""
"""Collection for training annotations.
DOCS: https://spacy.io/api/goldparse
"""
@classmethod
def from_annot_tuples(cls, doc, annot_tuples, make_projective=False):
_, words, tags, heads, deps, entities = annot_tuples
@ -456,13 +458,13 @@ cdef class GoldParse:
if deps is None:
deps = [None for _ in doc]
if entities is None:
entities = ['-' for _ in doc]
entities = ["-" for _ in doc]
elif len(entities) == 0:
entities = ['O' for _ in doc]
entities = ["O" for _ in doc]
else:
# Translate the None values to '-', to make processing easier.
# See Issue #2603
entities = [(ent if ent is not None else '-') for ent in entities]
entities = [(ent if ent is not None else "-") for ent in entities]
if not isinstance(entities[0], basestring):
# Assume we have entities specified by character offset.
entities = biluo_tags_from_offsets(doc, entities)
@ -508,10 +510,10 @@ cdef class GoldParse:
for i, gold_i in enumerate(self.cand_to_gold):
if doc[i].text.isspace():
self.words[i] = doc[i].text
self.tags[i] = '_SP'
self.tags[i] = "_SP"
self.heads[i] = None
self.labels[i] = None
self.ner[i] = 'O'
self.ner[i] = "O"
if gold_i is None:
if i in i2j_multi:
self.words[i] = words[i2j_multi[i]]
@ -521,7 +523,7 @@ cdef class GoldParse:
# Set next word in multi-token span as head, until last
if not is_last:
self.heads[i] = i+1
self.labels[i] = 'subtok'
self.labels[i] = "subtok"
else:
self.heads[i] = self.gold_to_cand[heads[i2j_multi[i]]]
self.labels[i] = deps[i2j_multi[i]]
@ -530,24 +532,24 @@ cdef class GoldParse:
# BILOU tags. We can't have BB or LL etc.
# Case 1: O -- easy.
ner_tag = entities[i2j_multi[i]]
if ner_tag == 'O':
self.ner[i] = 'O'
if ner_tag == "O":
self.ner[i] = "O"
# Case 2: U. This has to become a B I* L sequence.
elif ner_tag.startswith('U-'):
elif ner_tag.startswith("U-"):
if is_first:
self.ner[i] = ner_tag.replace('U-', 'B-', 1)
self.ner[i] = ner_tag.replace("U-", "B-", 1)
elif is_last:
self.ner[i] = ner_tag.replace('U-', 'L-', 1)
self.ner[i] = ner_tag.replace("U-", "L-", 1)
else:
self.ner[i] = ner_tag.replace('U-', 'I-', 1)
self.ner[i] = ner_tag.replace("U-", "I-", 1)
# Case 3: L. If not last, change to I.
elif ner_tag.startswith('L-'):
elif ner_tag.startswith("L-"):
if is_last:
self.ner[i] = ner_tag
else:
self.ner[i] = ner_tag.replace('L-', 'I-', 1)
self.ner[i] = ner_tag.replace("L-", "I-", 1)
# Case 4: I. Stays correct
elif ner_tag.startswith('I-'):
elif ner_tag.startswith("I-"):
self.ner[i] = ner_tag
else:
self.words[i] = words[gold_i]
@ -608,7 +610,7 @@ def docs_to_json(docs, underscore=None):
return [doc.to_json(underscore=underscore) for doc in docs]
def biluo_tags_from_offsets(doc, entities, missing='O'):
def biluo_tags_from_offsets(doc, entities, missing="O"):
"""Encode labelled spans into per-token tags, using the
Begin/In/Last/Unit/Out scheme (BILUO).
@ -631,11 +633,11 @@ def biluo_tags_from_offsets(doc, entities, missing='O'):
>>> entities = [(len('I like '), len('I like London'), 'LOC')]
>>> doc = nlp.tokenizer(text)
>>> tags = biluo_tags_from_offsets(doc, entities)
>>> assert tags == ['O', 'O', 'U-LOC', 'O']
>>> assert tags == ["O", "O", 'U-LOC', "O"]
"""
starts = {token.idx: token.i for token in doc}
ends = {token.idx+len(token): token.i for token in doc}
biluo = ['-' for _ in doc]
ends = {token.idx + len(token): token.i for token in doc}
biluo = ["-" for _ in doc]
# Handle entity cases
for start_char, end_char, label in entities:
start_token = starts.get(start_char)
@ -643,19 +645,19 @@ def biluo_tags_from_offsets(doc, entities, missing='O'):
# Only interested if the tokenization is correct
if start_token is not None and end_token is not None:
if start_token == end_token:
biluo[start_token] = 'U-%s' % label
biluo[start_token] = "U-%s" % label
else:
biluo[start_token] = 'B-%s' % label
biluo[start_token] = "B-%s" % label
for i in range(start_token+1, end_token):
biluo[i] = 'I-%s' % label
biluo[end_token] = 'L-%s' % label
biluo[i] = "I-%s" % label
biluo[end_token] = "L-%s" % label
# Now distinguish the O cases from ones where we miss the tokenization
entity_chars = set()
for start_char, end_char, label in entities:
for i in range(start_char, end_char):
entity_chars.add(i)
for token in doc:
for i in range(token.idx, token.idx+len(token)):
for i in range(token.idx, token.idx + len(token)):
if i in entity_chars:
break
else:
@ -697,4 +699,4 @@ def offsets_from_biluo_tags(doc, tags):
def is_punct_label(label):
return label == 'P' or label.lower() == 'punct'
return label == "P" or label.lower() == "punct"

3
spacy/language.py
View File

@ -103,8 +103,9 @@ class Language(object):
Defaults (class): Settings, data and factory methods for creating the `nlp`
object and processing pipeline.
lang (unicode): Two-letter language ID, i.e. ISO code.
"""
DOCS: https://spacy.io/api/language
"""
Defaults = BaseDefaults
lang = None

7
spacy/lemmatizer.py
View File

@ -6,6 +6,13 @@ from .symbols import VerbForm_inf, VerbForm_none, Number_sing, Degree_pos
class Lemmatizer(object):
"""
The Lemmatizer supports simple part-of-speech-sensitive suffix rules and
lookup tables.
DOCS: https://spacy.io/api/lemmatizer
"""
@classmethod
def load(cls, path, index=None, exc=None, rules=None, lookup=None):
return cls(index, exc, rules, lookup)

20
spacy/lexeme.pyx
View File

@ -4,17 +4,19 @@ from __future__ import unicode_literals, print_function
# Compiler crashes on memory view coercion without this. Should report bug.
from cython.view cimport array as cvarray
from libc.string cimport memset
cimport numpy as np
np.import_array()
from libc.string cimport memset
import numpy
from thinc.neural.util import get_array_module
from .typedefs cimport attr_t, flags_t
from .attrs cimport IS_ALPHA, IS_ASCII, IS_DIGIT, IS_LOWER, IS_PUNCT, IS_SPACE
from .attrs cimport IS_TITLE, IS_UPPER, LIKE_URL, LIKE_NUM, LIKE_EMAIL, IS_STOP
from .attrs cimport IS_BRACKET, IS_QUOTE, IS_LEFT_PUNCT, IS_RIGHT_PUNCT, IS_CURRENCY, IS_OOV
from .attrs cimport PROB
from .attrs cimport IS_BRACKET, IS_QUOTE, IS_LEFT_PUNCT, IS_RIGHT_PUNCT
from .attrs cimport IS_CURRENCY, IS_OOV, PROB
from .attrs import intify_attrs
from .errors import Errors, Warnings, user_warning
@ -27,6 +29,8 @@ cdef class Lexeme:
word-type, as opposed to a word token. It therefore has no part-of-speech
tag, dependency parse, or lemma (lemmatization depends on the
part-of-speech tag).
DOCS: https://spacy.io/api/lexeme
"""
def __init__(self, Vocab vocab, attr_t orth):
"""Create a Lexeme object.
@ -115,15 +119,15 @@ cdef class Lexeme:
RETURNS (float): A scalar similarity score. Higher is more similar.
"""
# Return 1.0 similarity for matches
if hasattr(other, 'orth'):
if hasattr(other, "orth"):
if self.c.orth == other.orth:
return 1.0
elif hasattr(other, '__len__') and len(other) == 1 \
and hasattr(other[0], 'orth'):
elif hasattr(other, "__len__") and len(other) == 1 \
and hasattr(other[0], "orth"):
if self.c.orth == other[0].orth:
return 1.0
if self.vector_norm == 0 or other.vector_norm == 0:
user_warning(Warnings.W008.format(obj='Lexeme'))
user_warning(Warnings.W008.format(obj="Lexeme"))
return 0.0
vector = self.vector
xp = get_array_module(vector)
@ -136,7 +140,7 @@ cdef class Lexeme:
if (end-start) != sizeof(lex_data.data):
raise ValueError(Errors.E072.format(length=end-start,
bad_length=sizeof(lex_data.data)))
byte_string = b'\0' * sizeof(lex_data.data)
byte_string = b"\0" * sizeof(lex_data.data)
byte_chars = <char*>byte_string
for i in range(sizeof(lex_data.data)):
byte_chars[i] = lex_data.data[i]

8
spacy/matcher/__init__.py
View File

@ -1,6 +1,8 @@
# coding: utf8
from __future__ import unicode_literals
from .matcher import Matcher # noqa: F401
from .phrasematcher import PhraseMatcher # noqa: F401
from .dependencymatcher import DependencyTreeMatcher # noqa: F401
from .matcher import Matcher
from .phrasematcher import PhraseMatcher
from .dependencymatcher import DependencyTreeMatcher
__all__ = ["Matcher", "PhraseMatcher", "DependencyTreeMatcher"]

164
spacy/matcher/dependencymatcher.pyx
View File

@ -13,7 +13,7 @@ from .matcher import unpickle_matcher
from ..errors import Errors
DELIMITER = '||'
DELIMITER = "||"
INDEX_HEAD = 1
INDEX_RELOP = 0
@ -55,7 +55,8 @@ cdef class DependencyTreeMatcher:
return (unpickle_matcher, data, None, None)
def __len__(self):
"""Get the number of rules, which are edges ,added to the dependency tree matcher.
"""Get the number of rules, which are edges, added to the dependency
tree matcher.
RETURNS (int): The number of rules.
"""
@ -73,19 +74,30 @@ cdef class DependencyTreeMatcher:
idx = 0
visitedNodes = {}
for relation in pattern:
if 'PATTERN' not in relation or 'SPEC' not in relation:
if "PATTERN" not in relation or "SPEC" not in relation:
raise ValueError(Errors.E098.format(key=key))
if idx == 0:
if not('NODE_NAME' in relation['SPEC'] and 'NBOR_RELOP' not in relation['SPEC'] and 'NBOR_NAME' not in relation['SPEC']):
if not(
"NODE_NAME" in relation["SPEC"]
and "NBOR_RELOP" not in relation["SPEC"]
and "NBOR_NAME" not in relation["SPEC"]
):
raise ValueError(Errors.E099.format(key=key))
visitedNodes[relation['SPEC']['NODE_NAME']] = True
visitedNodes[relation["SPEC"]["NODE_NAME"]] = True
else:
if not('NODE_NAME' in relation['SPEC'] and 'NBOR_RELOP' in relation['SPEC'] and 'NBOR_NAME' in relation['SPEC']):
if not(
"NODE_NAME" in relation["SPEC"]
and "NBOR_RELOP" in relation["SPEC"]
and "NBOR_NAME" in relation["SPEC"]
):
raise ValueError(Errors.E100.format(key=key))
if relation['SPEC']['NODE_NAME'] in visitedNodes or relation['SPEC']['NBOR_NAME'] not in visitedNodes:
if (
relation["SPEC"]["NODE_NAME"] in visitedNodes
or relation["SPEC"]["NBOR_NAME"] not in visitedNodes
):
raise ValueError(Errors.E101.format(key=key))
visitedNodes[relation['SPEC']['NODE_NAME']] = True
visitedNodes[relation['SPEC']['NBOR_NAME']] = True
visitedNodes[relation["SPEC"]["NODE_NAME"]] = True
visitedNodes[relation["SPEC"]["NBOR_NAME"]] = True
idx = idx + 1
def add(self, key, on_match, *patterns):
@ -93,55 +105,46 @@ cdef class DependencyTreeMatcher:
if len(pattern) == 0:
raise ValueError(Errors.E012.format(key=key))
self.validateInput(pattern,key)
key = self._normalize_key(key)
_patterns = []
for pattern in patterns:
token_patterns = []
for i in range(len(pattern)):
token_pattern = [pattern[i]['PATTERN']]
token_pattern = [pattern[i]["PATTERN"]]
token_patterns.append(token_pattern)
# self.patterns.append(token_patterns)
_patterns.append(token_patterns)
self._patterns.setdefault(key, [])
self._callbacks[key] = on_match
self._patterns[key].extend(_patterns)
# Add each node pattern of all the input patterns individually to the matcher.
# This enables only a single instance of Matcher to be used.
# Add each node pattern of all the input patterns individually to the
# matcher. This enables only a single instance of Matcher to be used.
# Multiple adds are required to track each node pattern.
_keys_to_token_list = []
for i in range(len(_patterns)):
_keys_to_token = {}
# TODO : Better ways to hash edges in pattern?
# TODO: Better ways to hash edges in pattern?
for j in range(len(_patterns[i])):
k = self._normalize_key(unicode(key)+DELIMITER+unicode(i)+DELIMITER+unicode(j))
self.token_matcher.add(k,None,_patterns[i][j])
k = self._normalize_key(unicode(key) + DELIMITER + unicode(i) + DELIMITER + unicode(j))
self.token_matcher.add(k, None, _patterns[i][j])
_keys_to_token[k] = j
_keys_to_token_list.append(_keys_to_token)
self._keys_to_token.setdefault(key, [])
self._keys_to_token[key].extend(_keys_to_token_list)
_nodes_list = []
for pattern in patterns:
nodes = {}
for i in range(len(pattern)):
nodes[pattern[i]['SPEC']['NODE_NAME']]=i
nodes[pattern[i]["SPEC"]["NODE_NAME"]] = i
_nodes_list.append(nodes)
self._nodes.setdefault(key, [])
self._nodes[key].extend(_nodes_list)
# Create an object tree to traverse later on. This data structure
# enables easy tree pattern match. Doc-Token based tree cannot be
# reused since it is memory-heavy and tightly coupled with the Doc.
self.retrieve_tree(patterns, _nodes_list,key)
# Create an object tree to traverse later on.
# This datastructure enable easy tree pattern match.
# Doc-Token based tree cannot be reused since it is memory heavy and
# tightly coupled with doc
self.retrieve_tree(patterns,_nodes_list,key)
def retrieve_tree(self,patterns,_nodes_list,key):
def retrieve_tree(self, patterns, _nodes_list, key):
_heads_list = []
_root_list = []
for i in range(len(patterns)):
@ -149,31 +152,29 @@ cdef class DependencyTreeMatcher:
root = -1
for j in range(len(patterns[i])):
token_pattern = patterns[i][j]
if('NBOR_RELOP' not in token_pattern['SPEC']):
heads[j] = ('root',j)
if ("NBOR_RELOP" not in token_pattern["SPEC"]):
heads[j] = ('root', j)
root = j
else:
heads[j] = (token_pattern['SPEC']['NBOR_RELOP'],_nodes_list[i][token_pattern['SPEC']['NBOR_NAME']])
heads[j] = (
token_pattern["SPEC"]["NBOR_RELOP"],
_nodes_list[i][token_pattern["SPEC"]["NBOR_NAME"]]
)
_heads_list.append(heads)
_root_list.append(root)
_tree_list = []
for i in range(len(patterns)):
tree = {}
for j in range(len(patterns[i])):
if(_heads_list[i][j][INDEX_HEAD] == j):
continue
head = _heads_list[i][j][INDEX_HEAD]
if(head not in tree):
tree[head] = []
tree[head].append( (_heads_list[i][j][INDEX_RELOP],j) )
tree[head].append((_heads_list[i][j][INDEX_RELOP], j))
_tree_list.append(tree)
self._tree.setdefault(key, [])
self._tree[key].extend(_tree_list)
self._root.setdefault(key, [])
self._root[key].extend(_root_list)
@ -199,7 +200,6 @@ cdef class DependencyTreeMatcher:
def __call__(self, Doc doc):
matched_trees = []
matches = self.token_matcher(doc)
for key in list(self._patterns.keys()):
_patterns_list = self._patterns[key]
@ -216,39 +216,51 @@ cdef class DependencyTreeMatcher:
id_to_position = {}
for i in range(len(_nodes)):
id_to_position[i]=[]
# This could be taken outside to improve running time..?
# TODO: This could be taken outside to improve running time..?
for match_id, start, end in matches:
if match_id in _keys_to_token:
id_to_position[_keys_to_token[match_id]].append(start)
_node_operator_map = self.get_node_operator_map(doc,_tree,id_to_position,_nodes,_root)
_node_operator_map = self.get_node_operator_map(
doc,
_tree,
id_to_position,
_nodes,_root
)
length = len(_nodes)
if _root in id_to_position:
candidates = id_to_position[_root]
for candidate in candidates:
isVisited = {}
self.dfs(candidate,_root,_tree,id_to_position,doc,isVisited,_node_operator_map)
# To check if the subtree pattern is completely identified. This is a heuristic.
# This is done to reduce the complexity of exponential unordered subtree matching.
# Will give approximate matches in some cases.
self.dfs(
candidate,
_root,_tree,
id_to_position,
doc,
isVisited,
_node_operator_map
)
# To check if the subtree pattern is completely
# identified. This is a heuristic. This is done to
# reduce the complexity of exponential unordered subtree
# matching. Will give approximate matches in some cases.
if(len(isVisited) == length):
matched_trees.append((key,list(isVisited)))
for i, (ent_id, nodes) in enumerate(matched_trees):
on_match = self._callbacks.get(ent_id)
if on_match is not None:
on_match(self, doc, i, matches)
return matched_trees
def dfs(self,candidate,root,tree,id_to_position,doc,isVisited,_node_operator_map):
if(root in id_to_position and candidate in id_to_position[root]):
# color the node since it is valid
if (root in id_to_position and candidate in id_to_position[root]):
# Color the node since it is valid
isVisited[candidate] = True
if root in tree:
for root_child in tree[root]:
if candidate in _node_operator_map and root_child[INDEX_RELOP] in _node_operator_map[candidate]:
if (
candidate in _node_operator_map
and root_child[INDEX_RELOP] in _node_operator_map[candidate]
):
candidate_children = _node_operator_map[candidate][root_child[INDEX_RELOP]]
for candidate_child in candidate_children:
result = self.dfs(
@ -275,72 +287,68 @@ cdef class DependencyTreeMatcher:
for child in tree[node]:
all_operators.append(child[INDEX_RELOP])
all_operators = list(set(all_operators))
all_nodes = []
for node in all_node_indices:
all_nodes = all_nodes + id_to_position[node]
all_nodes = list(set(all_nodes))
for node in all_nodes:
_node_operator_map[node] = {}
for operator in all_operators:
_node_operator_map[node][operator] = []
# Used to invoke methods for each operator
switcher = {
'<':self.dep,
'>':self.gov,
'>>':self.dep_chain,
'<<':self.gov_chain,
'.':self.imm_precede,
'$+':self.imm_right_sib,
'$-':self.imm_left_sib,
'$++':self.right_sib,
'$--':self.left_sib
"<": self.dep,
">": self.gov,
">>": self.dep_chain,
"<<": self.gov_chain,
".": self.imm_precede,
"$+": self.imm_right_sib,
"$-": self.imm_left_sib,
"$++": self.right_sib,
"$--": self.left_sib
}
for operator in all_operators:
for node in all_nodes:
_node_operator_map[node][operator] = switcher.get(operator)(doc,node)
return _node_operator_map
def dep(self,doc,node):
def dep(self, doc, node):
return list(doc[node].head)
def gov(self,doc,node):
return list(doc[node].children)
def dep_chain(self,doc,node):
def dep_chain(self, doc, node):
return list(doc[node].ancestors)
def gov_chain(self,doc,node):
def gov_chain(self, doc, node):
return list(doc[node].subtree)
def imm_precede(self,doc,node):
if node>0:
return [doc[node-1]]
def imm_precede(self, doc, node):
if node > 0:
return [doc[node - 1]]
return []
def imm_right_sib(self,doc,node):
def imm_right_sib(self, doc, node):
for idx in range(list(doc[node].head.children)):
if idx == node-1:
if idx == node - 1:
return [doc[idx]]
return []
def imm_left_sib(self,doc,node):
def imm_left_sib(self, doc, node):
for idx in range(list(doc[node].head.children)):
if idx == node+1:
if idx == node + 1:
return [doc[idx]]
return []
def right_sib(self,doc,node):
def right_sib(self, doc, node):
candidate_children = []
for idx in range(list(doc[node].head.children)):
if idx < node:
candidate_children.append(doc[idx])
return candidate_children
def left_sib(self,doc,node):
def left_sib(self, doc, node):
candidate_children = []
for idx in range(list(doc[node].head.children)):
if idx > node:

1141
spacy/matcher/matcher.pyx
View File

File diff suppressed because it is too large Load Diff

42
spacy/matcher/phrasematcher.pyx
View File

@ -12,7 +12,7 @@ from ..vocab cimport Vocab
from ..tokens.doc cimport Doc, get_token_attr
from ..typedefs cimport attr_t, hash_t
from ..errors import Warnings, deprecation_warning, user_warning
from ..errors import Errors, Warnings, deprecation_warning, user_warning
from ..attrs import FLAG61 as U_ENT
from ..attrs import FLAG60 as B2_ENT
from ..attrs import FLAG59 as B3_ENT
@ -25,6 +25,13 @@ from ..attrs import FLAG41 as I4_ENT
cdef class PhraseMatcher:
"""Efficiently match large terminology lists. While the `Matcher` matches
sequences based on lists of token descriptions, the `PhraseMatcher` accepts
match patterns in the form of `Doc` objects.
DOCS: https://spacy.io/api/phrasematcher
USAGE: https://spacy.io/usage/rule-based-matching#phrasematcher
"""
cdef Pool mem
cdef Vocab vocab
cdef Matcher matcher
@ -36,7 +43,16 @@ cdef class PhraseMatcher:
cdef public object _docs
cdef public object _validate
def __init__(self, Vocab vocab, max_length=0, attr='ORTH', validate=False):
def __init__(self, Vocab vocab, max_length=0, attr="ORTH", validate=False):
"""Initialize the PhraseMatcher.
vocab (Vocab): The shared vocabulary.
attr (int / unicode): Token attribute to match on.
validate (bool): Perform additional validation when patterns are added.
RETURNS (PhraseMatcher): The newly constructed object.
DOCS: https://spacy.io/api/phrasematcher#init
"""
if max_length != 0:
deprecation_warning(Warnings.W010)
self.mem = Pool()
@ -54,7 +70,7 @@ cdef class PhraseMatcher:
[{B3_ENT: True}, {I3_ENT: True}, {L3_ENT: True}],
[{B4_ENT: True}, {I4_ENT: True}, {I4_ENT: True, "OP": "+"}, {L4_ENT: True}],
]
self.matcher.add('Candidate', None, *abstract_patterns)
self.matcher.add("Candidate", None, *abstract_patterns)
self._callbacks = {}
self._docs = {}
self._validate = validate
@ -65,6 +81,8 @@ cdef class PhraseMatcher:
number of individual patterns.
RETURNS (int): The number of rules.
DOCS: https://spacy.io/api/phrasematcher#len
"""
return len(self._docs)
@ -73,6 +91,8 @@ cdef class PhraseMatcher:
key (unicode): The match ID.
RETURNS (bool): Whether the matcher contains rules for this match ID.
DOCS: https://spacy.io/api/phrasematcher#contains
"""
cdef hash_t ent_id = self.matcher._normalize_key(key)
return ent_id in self._callbacks
@ -88,6 +108,8 @@ cdef class PhraseMatcher:
key (unicode): The match ID.
on_match (callable): Callback executed on match.
*docs (Doc): `Doc` objects representing match patterns.
DOCS: https://spacy.io/api/phrasematcher#add
"""
cdef Doc doc
cdef hash_t ent_id = self.matcher._normalize_key(key)
@ -112,8 +134,7 @@ cdef class PhraseMatcher:
lexeme = self.vocab[attr_value]
lexeme.set_flag(tag, True)
phrase_key[i] = lexeme.orth
phrase_hash = hash64(phrase_key,
length * sizeof(attr_t), 0)
phrase_hash = hash64(phrase_key, length * sizeof(attr_t), 0)
self.phrase_ids.set(phrase_hash, <void*>ent_id)
def __call__(self, Doc doc):
@ -123,6 +144,8 @@ cdef class PhraseMatcher:
RETURNS (list): A list of `(key, start, end)` tuples,
describing the matches. A match tuple describes a span
`doc[start:end]`. The `label_id` and `key` are both integers.
DOCS: https://spacy.io/api/phrasematcher#call
"""
matches = []
if self.attr == ORTH:
@ -158,6 +181,8 @@ cdef class PhraseMatcher:
If both return_matches and as_tuples are True, the output will
be a sequence of ((doc, matches), context) tuples.
YIELDS (Doc): Documents, in order.
DOCS: https://spacy.io/api/phrasematcher#pipe
"""
if as_tuples:
for doc, context in stream:
@ -180,8 +205,7 @@ cdef class PhraseMatcher:
phrase_key = <attr_t*>mem.alloc(end-start, sizeof(attr_t))
for i, j in enumerate(range(start, end)):
phrase_key[i] = doc.c[j].lex.orth
cdef hash_t key = hash64(phrase_key,
(end-start) * sizeof(attr_t), 0)
cdef hash_t key = hash64(phrase_key, (end-start) * sizeof(attr_t), 0)
ent_id = <hash_t>self.phrase_ids.get(key)
if ent_id == 0:
return None
@ -203,12 +227,12 @@ cdef class PhraseMatcher:
# Concatenate the attr name and value to not pollute lexeme space
# e.g. 'POS-VERB' instead of just 'VERB', which could otherwise
# create false positive matches
return 'matcher:{}-{}'.format(string_attr_name, string_attr_value)
return "matcher:{}-{}".format(string_attr_name, string_attr_value)
def get_bilou(length):
if length == 0:
raise ValueError("Length must be >= 1")
raise ValueError(Errors.E127)
elif length == 1:
return [U_ENT]
elif length == 2:

25
spacy/pipeline/__init__.py
View File

@ -1,8 +1,23 @@
# coding: utf8
from __future__ import unicode_literals
from .pipes import Tagger, DependencyParser, EntityRecognizer # noqa
from .pipes import TextCategorizer, Tensorizer, Pipe # noqa
from .entityruler import EntityRuler # noqa
from .hooks import SentenceSegmenter, SimilarityHook # noqa
from .functions import merge_entities, merge_noun_chunks, merge_subtokens # noqa
from .pipes import Tagger, DependencyParser, EntityRecognizer
from .pipes import TextCategorizer, Tensorizer, Pipe
from .entityruler import EntityRuler
from .hooks import SentenceSegmenter, SimilarityHook
from .functions import merge_entities, merge_noun_chunks, merge_subtokens
__all__ = [
"Tagger",
"DependencyParser",
"EntityRecognizer",
"TextCategorizer",
"Tensorizer",
"Pipe",
"EntityRuler",
"SentenceSegmenter",
"SimilarityHook",
"merge_entities",
"merge_noun_chunks",
"merge_subtokens",
]

30
spacy/pipeline/entityruler.py
View File

@ -12,10 +12,20 @@ from ..matcher import Matcher, PhraseMatcher
class EntityRuler(object):
"""The EntityRuler lets you add spans to the `Doc.ents` using token-based
rules or exact phrase matches. It can be combined with the statistical
`EntityRecognizer` to boost accuracy, or used on its own to implement a
purely rule-based entity recognition system. After initialization, the
component is typically added to the pipeline using `nlp.add_pipe`.
DOCS: https://spacy.io/api/entityruler
USAGE: https://spacy.io/usage/rule-based-matching#entityruler
"""
name = "entity_ruler"
def __init__(self, nlp, **cfg):
"""Initialise the entitiy ruler. If patterns are supplied here, they
"""Initialize the entitiy ruler. If patterns are supplied here, they
need to be a list of dictionaries with a `"label"` and `"pattern"`
key. A pattern can either be a token pattern (list) or a phrase pattern
(string). For example: `{'label': 'ORG', 'pattern': 'Apple'}`.
@ -29,6 +39,8 @@ class EntityRuler(object):
of a model pipeline, this will include all keyword arguments passed
to `spacy.load`.
RETURNS (EntityRuler): The newly constructed object.
DOCS: https://spacy.io/api/entityruler#init
"""
self.nlp = nlp
self.overwrite = cfg.get("overwrite_ents", False)
@ -55,6 +67,8 @@ class EntityRuler(object):
doc (Doc): The Doc object in the pipeline.
RETURNS (Doc): The Doc with added entities, if available.
DOCS: https://spacy.io/api/entityruler#call
"""
matches = list(self.matcher(doc)) + list(self.phrase_matcher(doc))
matches = set(
@ -83,6 +97,8 @@ class EntityRuler(object):
"""All labels present in the match patterns.
RETURNS (set): The string labels.
DOCS: https://spacy.io/api/entityruler#labels
"""
all_labels = set(self.token_patterns.keys())
all_labels.update(self.phrase_patterns.keys())
@ -93,6 +109,8 @@ class EntityRuler(object):
"""Get all patterns that were added to the entity ruler.
RETURNS (list): The original patterns, one dictionary per pattern.
DOCS: https://spacy.io/api/entityruler#patterns
"""
all_patterns = []
for label, patterns in self.token_patterns.items():
@ -110,6 +128,8 @@ class EntityRuler(object):
{'label': 'GPE', 'pattern': [{'lower': 'san'}, {'lower': 'francisco'}]}
patterns (list): The patterns to add.
DOCS: https://spacy.io/api/entityruler#add_patterns
"""
for entry in patterns:
label = entry["label"]
@ -131,6 +151,8 @@ class EntityRuler(object):
patterns_bytes (bytes): The bytestring to load.
**kwargs: Other config paramters, mostly for consistency.
RETURNS (EntityRuler): The loaded entity ruler.
DOCS: https://spacy.io/api/entityruler#from_bytes
"""
patterns = srsly.msgpack_loads(patterns_bytes)
self.add_patterns(patterns)
@ -140,6 +162,8 @@ class EntityRuler(object):
"""Serialize the entity ruler patterns to a bytestring.
RETURNS (bytes): The serialized patterns.
DOCS: https://spacy.io/api/entityruler#to_bytes
"""
return srsly.msgpack_dumps(self.patterns)
@ -150,6 +174,8 @@ class EntityRuler(object):
path (unicode / Path): The JSONL file to load.
**kwargs: Other config paramters, mostly for consistency.
RETURNS (EntityRuler): The loaded entity ruler.
DOCS: https://spacy.io/api/entityruler#from_disk
"""
path = ensure_path(path)
path = path.with_suffix(".jsonl")
@ -164,6 +190,8 @@ class EntityRuler(object):
path (unicode / Path): The JSONL file to load.
**kwargs: Other config paramters, mostly for consistency.
RETURNS (EntityRuler): The loaded entity ruler.
DOCS: https://spacy.io/api/entityruler
"""
path = ensure_path(path)
path = path.with_suffix(".jsonl")

14
spacy/pipeline/functions.py
View File

@ -9,6 +9,8 @@ def merge_noun_chunks(doc):
doc (Doc): The Doc object.
RETURNS (Doc): The Doc object with merged noun chunks.
DOCS: https://spacy.io/api/pipeline-functions#merge_noun_chunks
"""
if not doc.is_parsed:
return doc
@ -23,7 +25,9 @@ def merge_entities(doc):
"""Merge entities into a single token.
doc (Doc): The Doc object.
RETURNS (Doc): The Doc object with merged noun entities.
RETURNS (Doc): The Doc object with merged entities.
DOCS: https://spacy.io/api/pipeline-functions#merge_entities
"""
with doc.retokenize() as retokenizer:
for ent in doc.ents:
@ -33,6 +37,14 @@ def merge_entities(doc):
def merge_subtokens(doc, label="subtok"):
"""Merge subtokens into a single token.
doc (Doc): The Doc object.
label (unicode): The subtoken dependency label.
RETURNS (Doc): The Doc object with merged subtokens.
DOCS: https://spacy.io/api/pipeline-functions#merge_subtokens
"""
merger = Matcher(doc.vocab)
merger.add("SUBTOK", None, [{"DEP": label, "op": "+"}])
matches = merger(doc)

2
spacy/pipeline/hooks.py
View File

@ -15,6 +15,8 @@ class SentenceSegmenter(object):
initialization, or assign a new strategy to the .strategy attribute.
Sentence detection strategies should be generators that take `Doc` objects
and yield `Span` objects for each sentence.
DOCS: https://spacy.io/api/sentencesegmenter
"""
name = "sentencizer"

175
spacy/pipeline/pipes.pyx
View File

@ -6,9 +6,8 @@ from __future__ import unicode_literals
cimport numpy as np
import numpy
from collections import OrderedDict
import srsly
from collections import OrderedDict
from thinc.api import chain
from thinc.v2v import Affine, Maxout, Softmax
from thinc.misc import LayerNorm
@ -284,9 +283,7 @@ class Tensorizer(Pipe):
"""
for doc, tensor in zip(docs, tensors):
if tensor.shape[0] != len(doc):
raise ValueError(
Errors.E076.format(rows=tensor.shape[0], words=len(doc))
)
raise ValueError(Errors.E076.format(rows=tensor.shape[0], words=len(doc)))
doc.tensor = tensor
def update(self, docs, golds, state=None, drop=0.0, sgd=None, losses=None):
@ -346,14 +343,19 @@ class Tensorizer(Pipe):
class Tagger(Pipe):
name = 'tagger'
"""Pipeline component for part-of-speech tagging.
DOCS: https://spacy.io/api/tagger
"""
name = "tagger"
def __init__(self, vocab, model=True, **cfg):
self.vocab = vocab
self.model = model
self._rehearsal_model = None
self.cfg = OrderedDict(sorted(cfg.items()))
self.cfg.setdefault('cnn_maxout_pieces', 2)
self.cfg.setdefault("cnn_maxout_pieces", 2)
@property
def labels(self):
@ -404,7 +406,7 @@ class Tagger(Pipe):
cdef Vocab vocab = self.vocab
for i, doc in enumerate(docs):
doc_tag_ids = batch_tag_ids[i]
if hasattr(doc_tag_ids, 'get'):
if hasattr(doc_tag_ids, "get"):
doc_tag_ids = doc_tag_ids.get()
for j, tag_id in enumerate(doc_tag_ids):
# Don't clobber preset POS tags
@ -453,9 +455,9 @@ class Tagger(Pipe):
scores = self.model.ops.flatten(scores)
tag_index = {tag: i for i, tag in enumerate(self.labels)}
cdef int idx = 0
correct = numpy.zeros((scores.shape[0],), dtype='i')
correct = numpy.zeros((scores.shape[0],), dtype="i")
guesses = scores.argmax(axis=1)
known_labels = numpy.ones((scores.shape[0], 1), dtype='f')
known_labels = numpy.ones((scores.shape[0], 1), dtype="f")
for gold in golds:
for tag in gold.tags:
if tag is None:
@ -466,7 +468,7 @@ class Tagger(Pipe):
correct[idx] = 0
known_labels[idx] = 0.
idx += 1
correct = self.model.ops.xp.array(correct, dtype='i')
correct = self.model.ops.xp.array(correct, dtype="i")
d_scores = scores - to_categorical(correct, nb_classes=scores.shape[1])
d_scores *= self.model.ops.asarray(known_labels)
loss = (d_scores**2).sum()
@ -490,9 +492,9 @@ class Tagger(Pipe):
vocab.morphology = Morphology(vocab.strings, new_tag_map,
vocab.morphology.lemmatizer,
exc=vocab.morphology.exc)
self.cfg['pretrained_vectors'] = kwargs.get('pretrained_vectors')
self.cfg["pretrained_vectors"] = kwargs.get("pretrained_vectors")
if self.model is True:
for hp in ['token_vector_width', 'conv_depth']:
for hp in ["token_vector_width", "conv_depth"]:
if hp in kwargs:
self.cfg[hp] = kwargs[hp]
self.model = self.Model(self.vocab.morphology.n_tags, **self.cfg)
@ -503,7 +505,7 @@ class Tagger(Pipe):
@classmethod
def Model(cls, n_tags, **cfg):
if cfg.get('pretrained_dims') and not cfg.get('pretrained_vectors'):
if cfg.get("pretrained_dims") and not cfg.get("pretrained_vectors"):
raise ValueError(TempErrors.T008)
return build_tagger_model(n_tags, **cfg)
@ -538,25 +540,23 @@ class Tagger(Pipe):
def to_bytes(self, **exclude):
serialize = OrderedDict()
if self.model not in (None, True, False):
serialize['model'] = self.model.to_bytes
serialize['vocab'] = self.vocab.to_bytes
serialize['cfg'] = lambda: srsly.json_dumps(self.cfg)
serialize["model"] = self.model.to_bytes
serialize["vocab"] = self.vocab.to_bytes
serialize["cfg"] = lambda: srsly.json_dumps(self.cfg)
tag_map = OrderedDict(sorted(self.vocab.morphology.tag_map.items()))
serialize['tag_map'] = lambda: srsly.msgpack_dumps(tag_map)
serialize["tag_map"] = lambda: srsly.msgpack_dumps(tag_map)
return util.to_bytes(serialize, exclude)
def from_bytes(self, bytes_data, **exclude):
def load_model(b):
# TODO: Remove this once we don't have to handle previous models
if self.cfg.get('pretrained_dims') and 'pretrained_vectors' not in self.cfg:
self.cfg['pretrained_vectors'] = self.vocab.vectors.name
if self.cfg.get("pretrained_dims") and "pretrained_vectors" not in self.cfg:
self.cfg["pretrained_vectors"] = self.vocab.vectors.name
if self.model is True:
token_vector_width = util.env_opt(
'token_vector_width',
self.cfg.get('token_vector_width', 96))
self.model = self.Model(self.vocab.morphology.n_tags,
**self.cfg)
"token_vector_width",
self.cfg.get("token_vector_width", 96))
self.model = self.Model(self.vocab.morphology.n_tags, **self.cfg)
self.model.from_bytes(b)
def load_tag_map(b):
@ -567,10 +567,10 @@ class Tagger(Pipe):
exc=self.vocab.morphology.exc)
deserialize = OrderedDict((
('vocab', lambda b: self.vocab.from_bytes(b)),
('tag_map', load_tag_map),
('cfg', lambda b: self.cfg.update(srsly.json_loads(b))),
('model', lambda b: load_model(b)),
("vocab", lambda b: self.vocab.from_bytes(b)),
("tag_map", load_tag_map),
("cfg", lambda b: self.cfg.update(srsly.json_loads(b))),
("model", lambda b: load_model(b)),
))
util.from_bytes(bytes_data, deserialize, exclude)
return self
@ -580,7 +580,7 @@ class Tagger(Pipe):
serialize = OrderedDict((
('vocab', lambda p: self.vocab.to_disk(p)),
('tag_map', lambda p: srsly.write_msgpack(p, tag_map)),
('model', lambda p: p.open('wb').write(self.model.to_bytes())),
('model', lambda p: p.open("wb").write(self.model.to_bytes())),
('cfg', lambda p: srsly.write_json(p, self.cfg))
))
util.to_disk(path, serialize, exclude)
@ -588,11 +588,11 @@ class Tagger(Pipe):
def from_disk(self, path, **exclude):
def load_model(p):
# TODO: Remove this once we don't have to handle previous models
if self.cfg.get('pretrained_dims') and 'pretrained_vectors' not in self.cfg:
self.cfg['pretrained_vectors'] = self.vocab.vectors.name
if self.cfg.get("pretrained_dims") and "pretrained_vectors" not in self.cfg:
self.cfg["pretrained_vectors"] = self.vocab.vectors.name
if self.model is True:
self.model = self.Model(self.vocab.morphology.n_tags, **self.cfg)
with p.open('rb') as file_:
with p.open("rb") as file_:
self.model.from_bytes(file_.read())
def load_tag_map(p):
@ -603,10 +603,10 @@ class Tagger(Pipe):
exc=self.vocab.morphology.exc)
deserialize = OrderedDict((
('cfg', lambda p: self.cfg.update(_load_cfg(p))),
('vocab', lambda p: self.vocab.from_disk(p)),
('tag_map', load_tag_map),
('model', load_model),
("cfg", lambda p: self.cfg.update(_load_cfg(p))),
("vocab", lambda p: self.vocab.from_disk(p)),
("tag_map", load_tag_map),
("model", load_model),
))
util.from_disk(path, deserialize, exclude)
return self
@ -616,37 +616,38 @@ class MultitaskObjective(Tagger):
"""Experimental: Assist training of a parser or tagger, by training a
side-objective.
"""
name = 'nn_labeller'
name = "nn_labeller"
def __init__(self, vocab, model=True, target='dep_tag_offset', **cfg):
self.vocab = vocab
self.model = model
if target == 'dep':
if target == "dep":
self.make_label = self.make_dep
elif target == 'tag':
elif target == "tag":
self.make_label = self.make_tag
elif target == 'ent':
elif target == "ent":
self.make_label = self.make_ent
elif target == 'dep_tag_offset':
elif target == "dep_tag_offset":
self.make_label = self.make_dep_tag_offset
elif target == 'ent_tag':
elif target == "ent_tag":
self.make_label = self.make_ent_tag
elif target == 'sent_start':
elif target == "sent_start":
self.make_label = self.make_sent_start
elif hasattr(target, '__call__'):
elif hasattr(target, "__call__"):
self.make_label = target
else:
raise ValueError(Errors.E016)
self.cfg = dict(cfg)
self.cfg.setdefault('cnn_maxout_pieces', 2)
self.cfg.setdefault("cnn_maxout_pieces", 2)
@property
def labels(self):
return self.cfg.setdefault('labels', {})
return self.cfg.setdefault("labels", {})
@labels.setter
def labels(self, value):
self.cfg['labels'] = value
self.cfg["labels"] = value
def set_annotations(self, docs, dep_ids, tensors=None):
pass
@ -662,7 +663,7 @@ class MultitaskObjective(Tagger):
if label is not None and label not in self.labels:
self.labels[label] = len(self.labels)
if self.model is True:
token_vector_width = util.env_opt('token_vector_width')
token_vector_width = util.env_opt("token_vector_width")
self.model = self.Model(len(self.labels), tok2vec=tok2vec)
link_vectors_to_models(self.vocab)
if sgd is None:
@ -671,7 +672,7 @@ class MultitaskObjective(Tagger):
@classmethod
def Model(cls, n_tags, tok2vec=None, **cfg):
token_vector_width = util.env_opt('token_vector_width', 96)
token_vector_width = util.env_opt("token_vector_width", 96)
softmax = Softmax(n_tags, token_vector_width*2)
model = chain(
tok2vec,
@ -690,10 +691,10 @@ class MultitaskObjective(Tagger):
def get_loss(self, docs, golds, scores):
if len(docs) != len(golds):
raise ValueError(Errors.E077.format(value='loss', n_docs=len(docs),
raise ValueError(Errors.E077.format(value="loss", n_docs=len(docs),
n_golds=len(golds)))
cdef int idx = 0
correct = numpy.zeros((scores.shape[0],), dtype='i')
correct = numpy.zeros((scores.shape[0],), dtype="i")
guesses = scores.argmax(axis=1)
for i, gold in enumerate(golds):
for j in range(len(docs[i])):
@ -705,7 +706,7 @@ class MultitaskObjective(Tagger):
else:
correct[idx] = self.labels[label]
idx += 1
correct = self.model.ops.xp.array(correct, dtype='i')
correct = self.model.ops.xp.array(correct, dtype="i")
d_scores = scores - to_categorical(correct, nb_classes=scores.shape[1])
loss = (d_scores**2).sum()
return float(loss), d_scores
@ -733,25 +734,25 @@ class MultitaskObjective(Tagger):
offset = heads[i] - i
offset = min(offset, 2)
offset = max(offset, -2)
return '%s-%s:%d' % (deps[i], tags[i], offset)
return "%s-%s:%d" % (deps[i], tags[i], offset)
@staticmethod
def make_ent_tag(i, words, tags, heads, deps, ents):
if ents is None or ents[i] is None:
return None
else:
return '%s-%s' % (tags[i], ents[i])
return "%s-%s" % (tags[i], ents[i])
@staticmethod
def make_sent_start(target, words, tags, heads, deps, ents, cache=True, _cache={}):
'''A multi-task objective for representing sentence boundaries,
"""A multi-task objective for representing sentence boundaries,
using BILU scheme. (O is impossible)
The implementation of this method uses an internal cache that relies
on the identity of the heads array, to avoid requiring a new piece
of gold data. You can pass cache=False if you know the cache will
do the wrong thing.
'''
"""
assert len(words) == len(heads)
assert target < len(words), (target, len(words))
if cache:
@ -760,10 +761,10 @@ class MultitaskObjective(Tagger):
else:
for key in list(_cache.keys()):
_cache.pop(key)
sent_tags = ['I-SENT'] * len(words)
sent_tags = ["I-SENT"] * len(words)
_cache[id(heads)] = sent_tags
else:
sent_tags = ['I-SENT'] * len(words)
sent_tags = ["I-SENT"] * len(words)
def _find_root(child):
seen = set([child])
@ -781,10 +782,10 @@ class MultitaskObjective(Tagger):
sentences.setdefault(root, []).append(i)
for root, span in sorted(sentences.items()):
if len(span) == 1:
sent_tags[span[0]] = 'U-SENT'
sent_tags[span[0]] = "U-SENT"
else:
sent_tags[span[0]] = 'B-SENT'
sent_tags[span[-1]] = 'L-SENT'
sent_tags[span[0]] = "B-SENT"
sent_tags[span[-1]] = "L-SENT"
return sent_tags[target]
@ -854,6 +855,10 @@ class ClozeMultitask(Pipe):
class TextCategorizer(Pipe):
"""Pipeline component for text classification.
DOCS: https://spacy.io/api/textcategorizer
"""
name = 'textcat'
@classmethod
@ -863,7 +868,7 @@ class TextCategorizer(Pipe):
token_vector_width = cfg["token_vector_width"]
else:
token_vector_width = util.env_opt("token_vector_width", 96)
if cfg.get('architecture') == 'simple_cnn':
if cfg.get("architecture") == "simple_cnn":
tok2vec = Tok2Vec(token_vector_width, embed_size, **cfg)
return build_simple_cnn_text_classifier(tok2vec, nr_class, **cfg)
else:
@ -884,11 +889,11 @@ class TextCategorizer(Pipe):
@property
def labels(self):
return tuple(self.cfg.setdefault('labels', []))
return tuple(self.cfg.setdefault("labels", []))
@labels.setter
def labels(self, value):
self.cfg['labels'] = tuple(value)
self.cfg["labels"] = tuple(value)
def __call__(self, doc):
scores, tensors = self.predict([doc])
@ -934,8 +939,8 @@ class TextCategorizer(Pipe):
losses[self.name] += (gradient**2).sum()
def get_loss(self, docs, golds, scores):
truths = numpy.zeros((len(golds), len(self.labels)), dtype='f')
not_missing = numpy.ones((len(golds), len(self.labels)), dtype='f')
truths = numpy.zeros((len(golds), len(self.labels)), dtype="f")
not_missing = numpy.ones((len(golds), len(self.labels)), dtype="f")
for i, gold in enumerate(golds):
for j, label in enumerate(self.labels):
if label in gold.cats:
@ -956,20 +961,19 @@ class TextCategorizer(Pipe):
# This functionality was available previously, but was broken.
# The problem is that we resize the last layer, but the last layer
# is actually just an ensemble. We're not resizing the child layers
# -- a huge problem.
# - a huge problem.
raise ValueError(Errors.E116)
#smaller = self.model._layers[-1]
#larger = Affine(len(self.labels)+1, smaller.nI)
#copy_array(larger.W[:smaller.nO], smaller.W)
#copy_array(larger.b[:smaller.nO], smaller.b)
#self.model._layers[-1] = larger
# smaller = self.model._layers[-1]
# larger = Affine(len(self.labels)+1, smaller.nI)
# copy_array(larger.W[:smaller.nO], smaller.W)
# copy_array(larger.b[:smaller.nO], smaller.b)
# self.model._layers[-1] = larger
self.labels = tuple(list(self.labels) + [label])
return 1
def begin_training(self, get_gold_tuples=lambda: [], pipeline=None, sgd=None,
**kwargs):
def begin_training(self, get_gold_tuples=lambda: [], pipeline=None, sgd=None, **kwargs):
if self.model is True:
self.cfg['pretrained_vectors'] = kwargs.get('pretrained_vectors')
self.cfg["pretrained_vectors"] = kwargs.get("pretrained_vectors")
self.model = self.Model(len(self.labels), **self.cfg)
link_vectors_to_models(self.vocab)
if sgd is None:
@ -978,7 +982,12 @@ class TextCategorizer(Pipe):
cdef class DependencyParser(Parser):
name = 'parser'
"""Pipeline component for dependency parsing.
DOCS: https://spacy.io/api/dependencyparser
"""
name = "parser"
TransitionSystem = ArcEager
@property
@ -986,7 +995,7 @@ cdef class DependencyParser(Parser):
return [nonproj.deprojectivize]
def add_multitask_objective(self, target):
if target == 'cloze':
if target == "cloze":
cloze = ClozeMultitask(self.vocab)
self._multitasks.append(cloze)
else:
@ -1000,8 +1009,7 @@ cdef class DependencyParser(Parser):
tok2vec=tok2vec, sgd=sgd)
def __reduce__(self):
return (DependencyParser, (self.vocab, self.moves, self.model),
None, None)
return (DependencyParser, (self.vocab, self.moves, self.model), None, None)
@property
def labels(self):
@ -1010,6 +1018,11 @@ cdef class DependencyParser(Parser):
cdef class EntityRecognizer(Parser):
"""Pipeline component for named entity recognition.
DOCS: https://spacy.io/api/entityrecognizer
"""
name = "ner"
TransitionSystem = BiluoPushDown
nr_feature = 6
@ -1040,4 +1053,4 @@ cdef class EntityRecognizer(Parser):
if move[0] in ("B", "I", "L", "U")))
__all__ = ['Tagger', 'DependencyParser', 'EntityRecognizer', 'Tensorizer', 'TextCategorizer']
__all__ = ["Tagger", "DependencyParser", "EntityRecognizer", "Tensorizer", "TextCategorizer"]

21
spacy/strings.pyx
View File

@ -31,12 +31,12 @@ def get_string_id(key):
elif not key:
return 0
else:
chars = key.encode('utf8')
chars = key.encode("utf8")
return hash_utf8(chars, len(chars))
cpdef hash_t hash_string(unicode string) except 0:
chars = string.encode('utf8')
chars = string.encode("utf8")
return hash_utf8(chars, len(chars))
@ -51,9 +51,9 @@ cdef uint32_t hash32_utf8(char* utf8_string, int length) nogil:
cdef unicode decode_Utf8Str(const Utf8Str* string):
cdef int i, length
if string.s[0] < sizeof(string.s) and string.s[0] != 0:
return string.s[1:string.s[0]+1].decode('utf8')
return string.s[1:string.s[0]+1].decode("utf8")
elif string.p[0] < 255:
return string.p[1:string.p[0]+1].decode('utf8')
return string.p[1:string.p[0]+1].decode("utf8")
else:
i = 0
length = 0
@ -62,7 +62,7 @@ cdef unicode decode_Utf8Str(const Utf8Str* string):
length += 255
length += string.p[i]
i += 1
return string.p[i:length + i].decode('utf8')
return string.p[i:length + i].decode("utf8")
cdef Utf8Str* _allocate(Pool mem, const unsigned char* chars, uint32_t length) except *:
@ -91,7 +91,10 @@ cdef Utf8Str* _allocate(Pool mem, const unsigned char* chars, uint32_t length) e
cdef class StringStore:
"""Look up strings by 64-bit hashes."""
"""Look up strings by 64-bit hashes.
DOCS: https://spacy.io/api/stringstore
"""
def __init__(self, strings=None, freeze=False):
"""Create the StringStore.
@ -113,7 +116,7 @@ cdef class StringStore:
if isinstance(string_or_id, basestring) and len(string_or_id) == 0:
return 0
elif string_or_id == 0:
return u''
return ""
elif string_or_id in SYMBOLS_BY_STR:
return SYMBOLS_BY_STR[string_or_id]
@ -181,7 +184,7 @@ cdef class StringStore:
elif isinstance(string, unicode):
key = hash_string(string)
else:
string = string.encode('utf8')
string = string.encode("utf8")
key = hash_utf8(string, len(string))
if key < len(SYMBOLS_BY_INT):
return True
@ -296,7 +299,7 @@ cdef class StringStore:
cdef const Utf8Str* intern_unicode(self, unicode py_string):
# 0 means missing, but we don't bother offsetting the index.
cdef bytes byte_string = py_string.encode('utf8')
cdef bytes byte_string = py_string.encode("utf8")
return self._intern_utf8(byte_string, len(byte_string))
@cython.final

85
spacy/tokenizer.pyx
View File

@ -3,16 +3,18 @@
# coding: utf8
from __future__ import unicode_literals
from collections import OrderedDict
from cython.operator cimport dereference as deref
from cython.operator cimport preincrement as preinc
from cymem.cymem cimport Pool
from preshed.maps cimport PreshMap
import re
cimport cython
from collections import OrderedDict
import re
from .tokens.doc cimport Doc
from .strings cimport hash_string
from .errors import Errors, Warnings, deprecation_warning
from . import util
@ -20,6 +22,8 @@ from . import util
cdef class Tokenizer:
"""Segment text, and create Doc objects with the discovered segment
boundaries.
DOCS: https://spacy.io/api/tokenizer
"""
def __init__(self, Vocab vocab, rules=None, prefix_search=None,
suffix_search=None, infix_finditer=None, token_match=None):
@ -40,6 +44,8 @@ cdef class Tokenizer:
EXAMPLE:
>>> tokenizer = Tokenizer(nlp.vocab)
>>> tokenizer = English().Defaults.create_tokenizer(nlp)
DOCS: https://spacy.io/api/tokenizer#init
"""
self.mem = Pool()
self._cache = PreshMap()
@ -73,6 +79,8 @@ cdef class Tokenizer:
string (unicode): The string to tokenize.
RETURNS (Doc): A container for linguistic annotations.
DOCS: https://spacy.io/api/tokenizer#call
"""
if len(string) >= (2 ** 30):
raise ValueError(Errors.E025.format(length=len(string)))
@ -114,7 +122,7 @@ cdef class Tokenizer:
cache_hit = self._try_cache(key, doc)
if not cache_hit:
self._tokenize(doc, span, key)
doc.c[doc.length - 1].spacy = string[-1] == ' ' and not in_ws
doc.c[doc.length - 1].spacy = string[-1] == " " and not in_ws
return doc
def pipe(self, texts, batch_size=1000, n_threads=2):
@ -122,9 +130,9 @@ cdef class Tokenizer:
texts: A sequence of unicode texts.
batch_size (int): Number of texts to accumulate in an internal buffer.
n_threads (int): Number of threads to use, if the implementation
supports multi-threading. The default tokenizer is single-threaded.
YIELDS (Doc): A sequence of Doc objects, in order.
DOCS: https://spacy.io/api/tokenizer#pipe
"""
for text in texts:
yield self(text)
@ -235,7 +243,7 @@ cdef class Tokenizer:
if not matches:
tokens.push_back(self.vocab.get(tokens.mem, string), False)
else:
# let's say we have dyn-o-mite-dave - the regex finds the
# Let's say we have dyn-o-mite-dave - the regex finds the
# start and end positions of the hyphens
start = 0
start_before_infixes = start
@ -257,7 +265,6 @@ cdef class Tokenizer:
# https://github.com/explosion/spaCy/issues/768)
infix_span = string[infix_start:infix_end]
tokens.push_back(self.vocab.get(tokens.mem, infix_span), False)
start = infix_end
span = string[start:]
if span:
@ -274,7 +281,7 @@ cdef class Tokenizer:
for i in range(n):
if self.vocab._by_orth.get(tokens[i].lex.orth) == NULL:
return 0
# See https://github.com/explosion/spaCy/issues/1250
# See #1250
if has_special:
return 0
cached = <_Cached*>self.mem.alloc(1, sizeof(_Cached))
@ -293,6 +300,8 @@ cdef class Tokenizer:
RETURNS (list): A list of `re.MatchObject` objects that have `.start()`
and `.end()` methods, denoting the placement of internal segment
separators, e.g. hyphens.
DOCS: https://spacy.io/api/tokenizer#find_infix
"""
if self.infix_finditer is None:
return 0
@ -304,6 +313,8 @@ cdef class Tokenizer:
string (unicode): The string to segment.
RETURNS (int): The length of the prefix if present, otherwise `None`.
DOCS: https://spacy.io/api/tokenizer#find_prefix
"""
if self.prefix_search is None:
return 0
@ -316,6 +327,8 @@ cdef class Tokenizer:
string (unicode): The string to segment.
Returns (int): The length of the suffix if present, otherwise `None`.
DOCS: https://spacy.io/api/tokenizer#find_suffix
"""
if self.suffix_search is None:
return 0
@ -334,6 +347,8 @@ cdef class Tokenizer:
token_attrs (iterable): A sequence of dicts, where each dict describes
a token and its attributes. The `ORTH` fields of the attributes
must exactly match the string when they are concatenated.
DOCS: https://spacy.io/api/tokenizer#add_special_case
"""
substrings = list(substrings)
cached = <_Cached*>self.mem.alloc(1, sizeof(_Cached))
@ -350,8 +365,10 @@ cdef class Tokenizer:
path (unicode or Path): A path to a directory, which will be created if
it doesn't exist. Paths may be either strings or Path-like objects.
DOCS: https://spacy.io/api/tokenizer#to_disk
"""
with path.open('wb') as file_:
with path.open("wb") as file_:
file_.write(self.to_bytes(**exclude))
def from_disk(self, path, **exclude):
@ -361,8 +378,10 @@ cdef class Tokenizer:
path (unicode or Path): A path to a directory. Paths may be either
strings or `Path`-like objects.
RETURNS (Tokenizer): The modified `Tokenizer` object.
DOCS: https://spacy.io/api/tokenizer#from_disk
"""
with path.open('rb') as file_:
with path.open("rb") as file_:
bytes_data = file_.read()
self.from_bytes(bytes_data, **exclude)
return self
@ -372,14 +391,16 @@ cdef class Tokenizer:
**exclude: Named attributes to prevent from being serialized.
RETURNS (bytes): The serialized form of the `Tokenizer` object.
DOCS: https://spacy.io/api/tokenizer#to_bytes
"""
serializers = OrderedDict((
('vocab', lambda: self.vocab.to_bytes()),
('prefix_search', lambda: _get_regex_pattern(self.prefix_search)),
('suffix_search', lambda: _get_regex_pattern(self.suffix_search)),
('infix_finditer', lambda: _get_regex_pattern(self.infix_finditer)),
('token_match', lambda: _get_regex_pattern(self.token_match)),
('exceptions', lambda: OrderedDict(sorted(self._rules.items())))
("vocab", lambda: self.vocab.to_bytes()),
("prefix_search", lambda: _get_regex_pattern(self.prefix_search)),
("suffix_search", lambda: _get_regex_pattern(self.suffix_search)),
("infix_finditer", lambda: _get_regex_pattern(self.infix_finditer)),
("token_match", lambda: _get_regex_pattern(self.token_match)),
("exceptions", lambda: OrderedDict(sorted(self._rules.items())))
))
return util.to_bytes(serializers, exclude)
@ -389,26 +410,28 @@ cdef class Tokenizer:
bytes_data (bytes): The data to load from.
**exclude: Named attributes to prevent from being loaded.
RETURNS (Tokenizer): The `Tokenizer` object.
DOCS: https://spacy.io/api/tokenizer#from_bytes
"""
data = OrderedDict()
deserializers = OrderedDict((
('vocab', lambda b: self.vocab.from_bytes(b)),
('prefix_search', lambda b: data.setdefault('prefix_search', b)),
('suffix_search', lambda b: data.setdefault('suffix_search', b)),
('infix_finditer', lambda b: data.setdefault('infix_finditer', b)),
('token_match', lambda b: data.setdefault('token_match', b)),
('exceptions', lambda b: data.setdefault('rules', b))
("vocab", lambda b: self.vocab.from_bytes(b)),
("prefix_search", lambda b: data.setdefault("prefix_search", b)),
("suffix_search", lambda b: data.setdefault("suffix_search", b)),
("infix_finditer", lambda b: data.setdefault("infix_finditer", b)),
("token_match", lambda b: data.setdefault("token_match", b)),
("exceptions", lambda b: data.setdefault("rules", b))
))
msg = util.from_bytes(bytes_data, deserializers, exclude)
if data.get('prefix_search'):
self.prefix_search = re.compile(data['prefix_search']).search
if data.get('suffix_search'):
self.suffix_search = re.compile(data['suffix_search']).search
if data.get('infix_finditer'):
self.infix_finditer = re.compile(data['infix_finditer']).finditer
if data.get('token_match'):
self.token_match = re.compile(data['token_match']).match
for string, substrings in data.get('rules', {}).items():
if data.get("prefix_search"):
self.prefix_search = re.compile(data["prefix_search"]).search
if data.get("suffix_search"):
self.suffix_search = re.compile(data["suffix_search"]).search
if data.get("infix_finditer"):
self.infix_finditer = re.compile(data["infix_finditer"]).finditer
if data.get("token_match"):
self.token_match = re.compile(data["token_match"]).match
for string, substrings in data.get("rules", {}).items():
self.add_special_case(string, substrings)
return self

5
spacy/tokens/__init__.py
View File

@ -1,5 +1,8 @@
# coding: utf8
from __future__ import unicode_literals
from .doc import Doc
from .token import Token
from .span import Span
__all__ = ['Doc', 'Token', 'Span']
__all__ = ["Doc", "Token", "Span"]

35
spacy/tokens/_retokenize.pyx
View File

@ -6,11 +6,11 @@ from __future__ import unicode_literals
from libc.string cimport memcpy, memset
from libc.stdlib cimport malloc, free
import numpy
from cymem.cymem cimport Pool
from thinc.neural.util import get_array_module
import numpy
from .doc cimport Doc, set_children_from_heads, token_by_start, token_by_end
from .span cimport Span
from .token cimport Token
@ -26,11 +26,16 @@ from ..strings import get_string_id
cdef class Retokenizer:
"""Helper class for doc.retokenize() context manager."""
"""Helper class for doc.retokenize() context manager.
DOCS: https://spacy.io/api/doc#retokenize
USAGE: https://spacy.io/usage/linguistic-features#retokenization
"""
cdef Doc doc
cdef list merges
cdef list splits
cdef set tokens_to_merge
def __init__(self, doc):
self.doc = doc
self.merges = []
@ -40,6 +45,11 @@ cdef class Retokenizer:
def merge(self, Span span, attrs=SimpleFrozenDict()):
"""Mark a span for merging. The attrs will be applied to the resulting
token.
span (Span): The span to merge.
attrs (dict): Attributes to set on the merged token.
DOCS: https://spacy.io/api/doc#retokenizer.merge
"""
for token in span:
if token.i in self.tokens_to_merge:
@ -58,6 +68,16 @@ cdef class Retokenizer:
def split(self, Token token, orths, heads, attrs=SimpleFrozenDict()):
"""Mark a Token for splitting, into the specified orths. The attrs
will be applied to each subtoken.
token (Token): The token to split.
orths (list): The verbatim text of the split tokens. Needs to match the
text of the original token.
heads (list): List of token or `(token, subtoken)` tuples specifying the
tokens to attach the newly split subtokens to.
attrs (dict): Attributes to set on all split tokens. Attribute names
mapped to list of per-token attribute values.
DOCS: https://spacy.io/api/doc#retokenizer.split
"""
if ''.join(orths) != token.text:
raise ValueError(Errors.E117.format(new=''.join(orths), old=token.text))
@ -104,14 +124,12 @@ cdef class Retokenizer:
# referred to in the splits. If we merged these tokens previously, we
# have to raise an error
if token_index == -1:
raise IndexError(
"Cannot find token to be split. Did it get merged?")
raise IndexError(Errors.E122)
head_indices = []
for head_char, subtoken in heads:
head_index = token_by_start(self.doc.c, self.doc.length, head_char)
if head_index == -1:
raise IndexError(
"Cannot find head of token to be split. Did it get merged?")
raise IndexError(Errors.E123)
# We want to refer to the token index of the head *after* the
# mergery. We need to account for the extra tokens introduced.
# e.g., let's say we have [ab, c] and we want a and b to depend
@ -206,7 +224,6 @@ def _merge(Doc doc, int start, int end, attributes):
doc.c[i].head -= i
# Set the left/right children, left/right edges
set_children_from_heads(doc.c, doc.length)
# Clear the cached Python objects
# Return the merged Python object
return doc[start]
@ -336,7 +353,7 @@ def _bulk_merge(Doc doc, merges):
# Make sure ent_iob remains consistent
for (span, _) in merges:
if(span.end < len(offsets)):
#if it's not the last span
# If it's not the last span
token_after_span_position = offsets[span.end]
if doc.c[token_after_span_position].ent_iob == 1\
and doc.c[token_after_span_position - 1].ent_iob in (0, 2):

6
spacy/tokens/_serialize.py
View File

@ -1,3 +1,4 @@
# coding: utf8
from __future__ import unicode_literals
import numpy
@ -16,9 +17,8 @@ class Binder(object):
def __init__(self, attrs=None):
"""Create a Binder object, to hold serialized annotations.
attrs (list):
List of attributes to serialize. 'orth' and 'spacy' are always
serialized, so they're not required. Defaults to None.
attrs (list): List of attributes to serialize. 'orth' and 'spacy' are
always serialized, so they're not required. Defaults to None.
"""
attrs = attrs or []
self.attrs = list(attrs)

356
spacy/tokens/doc.pyx
View File

@ -7,28 +7,25 @@ from __future__ import unicode_literals
cimport cython
cimport numpy as np
from libc.string cimport memcpy, memset
from libc.math cimport sqrt
import numpy
import numpy.linalg
import struct
import srsly
from thinc.neural.util import get_array_module, copy_array
import srsly
from libc.string cimport memcpy, memset
from libc.math cimport sqrt
from .span cimport Span
from .token cimport Token
from .span cimport Span
from .token cimport Token
from ..lexeme cimport Lexeme, EMPTY_LEXEME
from ..typedefs cimport attr_t, flags_t
from ..attrs import intify_attrs, IDS
from ..attrs cimport attr_id_t
from ..attrs cimport ID, ORTH, NORM, LOWER, SHAPE, PREFIX, SUFFIX, CLUSTER
from ..attrs cimport LENGTH, POS, LEMMA, TAG, DEP, HEAD, SPACY, ENT_IOB
from ..attrs cimport ENT_TYPE, SENT_START
from ..attrs cimport ENT_TYPE, SENT_START, attr_id_t
from ..parts_of_speech cimport CCONJ, PUNCT, NOUN, univ_pos_t
from ..attrs import intify_attrs, IDS
from ..util import normalize_slice
from ..compat import is_config, copy_reg, pickle, basestring_
from ..errors import deprecation_warning, models_warning, user_warning
@ -37,6 +34,7 @@ from .. import util
from .underscore import Underscore, get_ext_args
from ._retokenize import Retokenizer
DEF PADDING = 5
@ -77,7 +75,7 @@ def _get_chunker(lang):
return None
except KeyError:
return None
return cls.Defaults.syntax_iterators.get(u'noun_chunks')
return cls.Defaults.syntax_iterators.get("noun_chunks")
cdef class Doc:
@ -94,23 +92,60 @@ cdef class Doc:
>>> from spacy.tokens import Doc
>>> doc = Doc(nlp.vocab, words=[u'hello', u'world', u'!'],
spaces=[True, False, False])
DOCS: https://spacy.io/api/doc
"""
@classmethod
def set_extension(cls, name, **kwargs):
if cls.has_extension(name) and not kwargs.get('force', False):
raise ValueError(Errors.E090.format(name=name, obj='Doc'))
"""Define a custom attribute which becomes available as `Doc._`.
name (unicode): Name of the attribute to set.
default: Optional default value of the attribute.
getter (callable): Optional getter function.
setter (callable): Optional setter function.
method (callable): Optional method for method extension.
force (bool): Force overwriting existing attribute.
DOCS: https://spacy.io/api/doc#set_extension
USAGE: https://spacy.io/usage/processing-pipelines#custom-components-attributes
"""
if cls.has_extension(name) and not kwargs.get("force", False):
raise ValueError(Errors.E090.format(name=name, obj="Doc"))
Underscore.doc_extensions[name] = get_ext_args(**kwargs)
@classmethod
def get_extension(cls, name):
"""Look up a previously registered extension by name.
name (unicode): Name of the extension.
RETURNS (tuple): A `(default, method, getter, setter)` tuple.
DOCS: https://spacy.io/api/doc#get_extension
"""
return Underscore.doc_extensions.get(name)
@classmethod
def has_extension(cls, name):
"""Check whether an extension has been registered.
name (unicode): Name of the extension.
RETURNS (bool): Whether the extension has been registered.
DOCS: https://spacy.io/api/doc#has_extension
"""
return name in Underscore.doc_extensions
@classmethod
def remove_extension(cls, name):
"""Remove a previously registered extension.
name (unicode): Name of the extension.
RETURNS (tuple): A `(default, method, getter, setter)` tuple of the
removed extension.
DOCS: https://spacy.io/api/doc#remove_extension
"""
if not cls.has_extension(name):
raise ValueError(Errors.E046.format(name=name))
return Underscore.doc_extensions.pop(name)
@ -128,6 +163,8 @@ cdef class Doc:
it is not. If `None`, defaults to `[True]*len(words)`
user_data (dict or None): Optional extra data to attach to the Doc.
RETURNS (Doc): The newly constructed object.
DOCS: https://spacy.io/api/doc#init
"""
self.vocab = vocab
size = 20
@ -151,7 +188,7 @@ cdef class Doc:
self.user_hooks = {}
self.user_token_hooks = {}
self.user_span_hooks = {}
self.tensor = numpy.zeros((0,), dtype='float32')
self.tensor = numpy.zeros((0,), dtype="float32")
self.user_data = {} if user_data is None else user_data
self._vector = None
self.noun_chunks_iterator = _get_chunker(self.vocab.lang)
@ -184,6 +221,7 @@ cdef class Doc:
@property
def _(self):
"""Custom extension attributes registered via `set_extension`."""
return Underscore(Underscore.doc_extensions, self)
@property
@ -195,7 +233,7 @@ cdef class Doc:
b) sent.is_parsed is set to True;
c) At least one token other than the first where sent_start is not None.
"""
if 'sents' in self.user_hooks:
if "sents" in self.user_hooks:
return True
if self.is_parsed:
return True
@ -227,11 +265,12 @@ cdef class Doc:
supported, as `Span` objects must be contiguous (cannot have gaps).
You can use negative indices and open-ended ranges, which have
their normal Python semantics.
DOCS: https://spacy.io/api/doc#getitem
"""
if isinstance(i, slice):
start, stop = normalize_slice(len(self), i.start, i.stop, i.step)
return Span(self, start, stop, label=0)
if i < 0:
i = self.length + i
bounds_check(i, self.length, PADDING)
@ -244,8 +283,7 @@ cdef class Doc:
than-Python speeds are required, you can instead access the annotations
as a numpy array, or access the underlying C data directly from Cython.
EXAMPLE:
>>> for token in doc
DOCS: https://spacy.io/api/doc#iter
"""
cdef int i
for i in range(self.length):
@ -256,16 +294,15 @@ cdef class Doc:
RETURNS (int): The number of tokens in the document.
EXAMPLE:
>>> len(doc)
DOCS: https://spacy.io/api/doc#len
"""
return self.length
def __unicode__(self):
return u''.join([t.text_with_ws for t in self])
return "".join([t.text_with_ws for t in self])
def __bytes__(self):
return u''.join([t.text_with_ws for t in self]).encode('utf-8')
return "".join([t.text_with_ws for t in self]).encode("utf-8")
def __str__(self):
if is_config(python3=True):
@ -290,6 +327,8 @@ cdef class Doc:
vector (ndarray[ndim=1, dtype='float32']): A meaning representation of
the span.
RETURNS (Span): The newly constructed object.
DOCS: https://spacy.io/api/doc#char_span
"""
if not isinstance(label, int):
label = self.vocab.strings.add(label)
@ -311,9 +350,11 @@ cdef class Doc:
other (object): The object to compare with. By default, accepts `Doc`,
`Span`, `Token` and `Lexeme` objects.
RETURNS (float): A scalar similarity score. Higher is more similar.
DOCS: https://spacy.io/api/doc#similarity
"""
if 'similarity' in self.user_hooks:
return self.user_hooks['similarity'](self, other)
if "similarity" in self.user_hooks:
return self.user_hooks["similarity"](self, other)
if isinstance(other, (Lexeme, Token)) and self.length == 1:
if self.c[0].lex.orth == other.orth:
return 1.0
@ -325,9 +366,9 @@ cdef class Doc:
else:
return 1.0
if self.vocab.vectors.n_keys == 0:
models_warning(Warnings.W007.format(obj='Doc'))
models_warning(Warnings.W007.format(obj="Doc"))
if self.vector_norm == 0 or other.vector_norm == 0:
user_warning(Warnings.W008.format(obj='Doc'))
user_warning(Warnings.W008.format(obj="Doc"))
return 0.0
vector = self.vector
xp = get_array_module(vector)
@ -338,10 +379,12 @@ cdef class Doc:
the object.
RETURNS (bool): Whether a word vector is associated with the object.
DOCS: https://spacy.io/api/doc#has_vector
"""
def __get__(self):
if 'has_vector' in self.user_hooks:
return self.user_hooks['has_vector'](self)
if "has_vector" in self.user_hooks:
return self.user_hooks["has_vector"](self)
elif self.vocab.vectors.data.size:
return True
elif self.tensor.size:
@ -355,15 +398,16 @@ cdef class Doc:
RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array
representing the document's semantics.
DOCS: https://spacy.io/api/doc#vector
"""
def __get__(self):
if 'vector' in self.user_hooks:
return self.user_hooks['vector'](self)
if "vector" in self.user_hooks:
return self.user_hooks["vector"](self)
if self._vector is not None:
return self._vector
elif not len(self):
self._vector = numpy.zeros((self.vocab.vectors_length,),
dtype='f')
self._vector = numpy.zeros((self.vocab.vectors_length,), dtype="f")
return self._vector
elif self.vocab.vectors.data.size > 0:
self._vector = sum(t.vector for t in self) / len(self)
@ -372,8 +416,7 @@ cdef class Doc:
self._vector = self.tensor.mean(axis=0)
return self._vector
else:
return numpy.zeros((self.vocab.vectors_length,),
dtype='float32')
return numpy.zeros((self.vocab.vectors_length,), dtype="float32")
def __set__(self, value):
self._vector = value
@ -382,10 +425,12 @@ cdef class Doc:
"""The L2 norm of the document's vector representation.
RETURNS (float): The L2 norm of the vector representation.
DOCS: https://spacy.io/api/doc#vector_norm
"""
def __get__(self):
if 'vector_norm' in self.user_hooks:
return self.user_hooks['vector_norm'](self)
if "vector_norm" in self.user_hooks:
return self.user_hooks["vector_norm"](self)
cdef float value
cdef double norm = 0
if self._vector_norm is None:
@ -404,7 +449,7 @@ cdef class Doc:
RETURNS (unicode): The original verbatim text of the document.
"""
def __get__(self):
return u''.join(t.text_with_ws for t in self)
return "".join(t.text_with_ws for t in self)
property text_with_ws:
"""An alias of `Doc.text`, provided for duck-type compatibility with
@ -416,21 +461,12 @@ cdef class Doc:
return self.text
property ents:
"""Iterate over the entities in the document. Yields named-entity
`Span` objects, if the entity recognizer has been applied to the
document.
"""The named entities in the document. Returns a tuple of named entity
`Span` objects, if the entity recognizer has been applied.
YIELDS (Span): Entities in the document.
RETURNS (tuple): Entities in the document, one `Span` per entity.
EXAMPLE: Iterate over the span to get individual Token objects,
or access the label:
>>> tokens = nlp(u'Mr. Best flew to New York on Saturday morning.')
>>> ents = list(tokens.ents)
>>> assert ents[0].label == 346
>>> assert ents[0].label_ == 'PERSON'
>>> assert ents[0].orth_ == 'Best'
>>> assert ents[0].text == 'Mr. Best'
DOCS: https://spacy.io/api/doc#ents
"""
def __get__(self):
cdef int i
@ -442,8 +478,8 @@ cdef class Doc:
token = &self.c[i]
if token.ent_iob == 1:
if start == -1:
seq = ['%s|%s' % (t.text, t.ent_iob_) for t in self[i-5:i+5]]
raise ValueError(Errors.E093.format(seq=' '.join(seq)))
seq = ["%s|%s" % (t.text, t.ent_iob_) for t in self[i-5:i+5]]
raise ValueError(Errors.E093.format(seq=" ".join(seq)))
elif token.ent_iob == 2 or token.ent_iob == 0:
if start != -1:
output.append(Span(self, start, i, label=label))
@ -465,7 +501,6 @@ cdef class Doc:
# prediction
# 3. Test basic data-driven ORTH gazetteer
# 4. Test more nuanced date and currency regex
tokens_in_ents = {}
cdef attr_t entity_type
cdef int ent_start, ent_end
@ -479,7 +514,6 @@ cdef class Doc:
self.vocab.strings[tokens_in_ents[token_index][2]]),
span2=(ent_start, ent_end, self.vocab.strings[entity_type])))
tokens_in_ents[token_index] = (ent_start, ent_end, entity_type)
cdef int i
for i in range(self.length):
self.c[i].ent_type = 0
@ -510,6 +544,8 @@ cdef class Doc:
clauses.
YIELDS (Span): Noun chunks in the document.
DOCS: https://spacy.io/api/doc#noun_chunks
"""
def __get__(self):
if not self.is_parsed:
@ -533,15 +569,15 @@ cdef class Doc:
dependency parse. If the parser is disabled, the `sents` iterator will
be unavailable.
EXAMPLE:
>>> doc = nlp("This is a sentence. Here's another...")
>>> assert [s.root.text for s in doc.sents] == ["is", "'s"]
YIELDS (Span): Sentences in the document.
DOCS: https://spacy.io/api/doc#sents
"""
def __get__(self):
if not self.is_sentenced:
raise ValueError(Errors.E030)
if 'sents' in self.user_hooks:
yield from self.user_hooks['sents'](self)
if "sents" in self.user_hooks:
yield from self.user_hooks["sents"](self)
else:
start = 0
for i in range(1, self.length):
@ -606,17 +642,16 @@ cdef class Doc:
if isinstance(py_attr_ids, basestring_):
# Handle inputs like doc.to_array('ORTH')
py_attr_ids = [py_attr_ids]
elif not hasattr(py_attr_ids, '__iter__'):
elif not hasattr(py_attr_ids, "__iter__"):
# Handle inputs like doc.to_array(ORTH)
py_attr_ids = [py_attr_ids]
# Allow strings, e.g. 'lemma' or 'LEMMA'
py_attr_ids = [(IDS[id_.upper()] if hasattr(id_, 'upper') else id_)
py_attr_ids = [(IDS[id_.upper()] if hasattr(id_, "upper") else id_)
for id_ in py_attr_ids]
# Make an array from the attributes --- otherwise our inner loop is
# Python dict iteration.
cdef np.ndarray attr_ids = numpy.asarray(py_attr_ids, dtype='i')
output = numpy.ndarray(shape=(self.length, len(attr_ids)),
dtype=numpy.uint64)
cdef np.ndarray attr_ids = numpy.asarray(py_attr_ids, dtype="i")
output = numpy.ndarray(shape=(self.length, len(attr_ids)), dtype=numpy.uint64)
c_output = <attr_t*>output.data
c_attr_ids = <attr_id_t*>attr_ids.data
cdef TokenC* token
@ -628,8 +663,7 @@ cdef class Doc:
# Handle 1d case
return output if len(attr_ids) >= 2 else output.reshape((self.length,))
def count_by(self, attr_id_t attr_id, exclude=None,
PreshCounter counts=None):
def count_by(self, attr_id_t attr_id, exclude=None, PreshCounter counts=None):
"""Count the frequencies of a given attribute. Produces a dict of
`{attribute (int): count (ints)}` frequencies, keyed by the values of
the given attribute ID.
@ -637,13 +671,7 @@ cdef class Doc:
attr_id (int): The attribute ID to key the counts.
RETURNS (dict): A dictionary mapping attributes to integer counts.
EXAMPLE:
>>> from spacy import attrs
>>> doc = nlp(u'apple apple orange banana')
>>> tokens.count_by(attrs.ORTH)
{12800L: 1, 11880L: 2, 7561L: 1}
>>> tokens.to_array([attrs.ORTH])
array([[11880], [11880], [7561], [12800]])
DOCS: https://spacy.io/api/doc#count_by
"""
cdef int i
cdef attr_t attr
@ -684,13 +712,21 @@ cdef class Doc:
cdef void set_parse(self, const TokenC* parsed) nogil:
# TODO: This method is fairly misleading atm. It's used by Parser
# to actually apply the parse calculated. Need to rethink this.
# Probably we should use from_array?
self.is_parsed = True
for i in range(self.length):
self.c[i] = parsed[i]
def from_array(self, attrs, array):
"""Load attributes from a numpy array. Write to a `Doc` object, from an
`(M, N)` array of attributes.
attrs (list) A list of attribute ID ints.
array (numpy.ndarray[ndim=2, dtype='int32']): The attribute values.
RETURNS (Doc): Itself.
DOCS: https://spacy.io/api/doc#from_array
"""
if SENT_START in attrs and HEAD in attrs:
raise ValueError(Errors.E032)
cdef int i, col
@ -714,10 +750,10 @@ cdef class Doc:
for i in range(length):
if array[i, col] != 0:
self.vocab.morphology.assign_tag(&tokens[i], array[i, col])
# set flags
# Set flags
self.is_parsed = bool(self.is_parsed or HEAD in attrs or DEP in attrs)
self.is_tagged = bool(self.is_tagged or TAG in attrs or POS in attrs)
# if document is parsed, set children
# If document is parsed, set children
if self.is_parsed:
set_children_from_heads(self.c, self.length)
return self
@ -729,6 +765,8 @@ cdef class Doc:
RETURNS (np.array[ndim=2, dtype=numpy.int32]): LCA matrix with shape
(n, n), where n = len(self).
DOCS: https://spacy.io/api/doc#get_lca_matrix
"""
return numpy.asarray(_get_lca_matrix(self, 0, len(self)))
@ -737,9 +775,11 @@ cdef class Doc:
path (unicode or Path): A path to a directory, which will be created if
it doesn't exist. Paths may be either strings or Path-like objects.
DOCS: https://spacy.io/api/doc#to_disk
"""
path = util.ensure_path(path)
with path.open('wb') as file_:
with path.open("wb") as file_:
file_.write(self.to_bytes(**exclude))
def from_disk(self, path, **exclude):
@ -749,9 +789,11 @@ cdef class Doc:
path (unicode or Path): A path to a directory. Paths may be either
strings or `Path`-like objects.
RETURNS (Doc): The modified `Doc` object.
DOCS: https://spacy.io/api/doc#from_disk
"""
path = util.ensure_path(path)
with path.open('rb') as file_:
with path.open("rb") as file_:
bytes_data = file_.read()
return self.from_bytes(bytes_data, **exclude)
@ -760,15 +802,16 @@ cdef class Doc:
RETURNS (bytes): A losslessly serialized copy of the `Doc`, including
all annotations.
DOCS: https://spacy.io/api/doc#to_bytes
"""
array_head = [LENGTH, SPACY, LEMMA, ENT_IOB, ENT_TYPE]
if self.is_tagged:
array_head.append(TAG)
# if doc parsed add head and dep attribute
# If doc parsed add head and dep attribute
if self.is_parsed:
array_head.extend([HEAD, DEP])
# otherwise add sent_start
# Otherwise add sent_start
else:
array_head.append(SENT_START)
# Msgpack doesn't distinguish between lists and tuples, which is
@ -776,17 +819,16 @@ cdef class Doc:
# keys, we must have tuples. In values we just have to hope
# users don't mind getting a list instead of a tuple.
serializers = {
'text': lambda: self.text,
'array_head': lambda: array_head,
'array_body': lambda: self.to_array(array_head),
'sentiment': lambda: self.sentiment,
'tensor': lambda: self.tensor,
"text": lambda: self.text,
"array_head": lambda: array_head,
"array_body": lambda: self.to_array(array_head),
"sentiment": lambda: self.sentiment,
"tensor": lambda: self.tensor,
}
if 'user_data' not in exclude and self.user_data:
if "user_data" not in exclude and self.user_data:
user_data_keys, user_data_values = list(zip(*self.user_data.items()))
serializers['user_data_keys'] = lambda: srsly.msgpack_dumps(user_data_keys)
serializers['user_data_values'] = lambda: srsly.msgpack_dumps(user_data_values)
serializers["user_data_keys"] = lambda: srsly.msgpack_dumps(user_data_keys)
serializers["user_data_values"] = lambda: srsly.msgpack_dumps(user_data_values)
return util.to_bytes(serializers, exclude)
def from_bytes(self, bytes_data, **exclude):
@ -794,42 +836,40 @@ cdef class Doc:
data (bytes): The string to load from.
RETURNS (Doc): Itself.
DOCS: https://spacy.io/api/doc#from_bytes
"""
if self.length != 0:
raise ValueError(Errors.E033.format(length=self.length))
deserializers = {
'text': lambda b: None,
'array_head': lambda b: None,
'array_body': lambda b: None,
'sentiment': lambda b: None,
'tensor': lambda b: None,
'user_data_keys': lambda b: None,
'user_data_values': lambda b: None,
"text": lambda b: None,
"array_head": lambda b: None,
"array_body": lambda b: None,
"sentiment": lambda b: None,
"tensor": lambda b: None,
"user_data_keys": lambda b: None,
"user_data_values": lambda b: None,
}
msg = util.from_bytes(bytes_data, deserializers, exclude)
# Msgpack doesn't distinguish between lists and tuples, which is
# vexing for user data. As a best guess, we *know* that within
# keys, we must have tuples. In values we just have to hope
# users don't mind getting a list instead of a tuple.
if 'user_data' not in exclude and 'user_data_keys' in msg:
user_data_keys = srsly.msgpack_loads(msg['user_data_keys'], use_list=False)
user_data_values = srsly.msgpack_loads(msg['user_data_values'])
if "user_data" not in exclude and "user_data_keys" in msg:
user_data_keys = srsly.msgpack_loads(msg["user_data_keys"], use_list=False)
user_data_values = srsly.msgpack_loads(msg["user_data_values"])
for key, value in zip(user_data_keys, user_data_values):
self.user_data[key] = value
cdef int i, start, end, has_space
if 'sentiment' not in exclude and 'sentiment' in msg:
self.sentiment = msg['sentiment']
if 'tensor' not in exclude and 'tensor' in msg:
self.tensor = msg['tensor']
if "sentiment" not in exclude and "sentiment" in msg:
self.sentiment = msg["sentiment"]
if "tensor" not in exclude and "tensor" in msg:
self.tensor = msg["tensor"]
start = 0
cdef const LexemeC* lex
cdef unicode orth_
text = msg['text']
attrs = msg['array_body']
text = msg["text"]
attrs = msg["array_body"]
for i in range(attrs.shape[0]):
end = start + attrs[i, 0]
has_space = attrs[i, 1]
@ -837,11 +877,11 @@ cdef class Doc:
lex = self.vocab.get(self.mem, orth_)
self.push_back(lex, has_space)
start = end + has_space
self.from_array(msg['array_head'][2:], attrs[:, 2:])
self.from_array(msg["array_head"][2:], attrs[:, 2:])
return self
def extend_tensor(self, tensor):
'''Concatenate a new tensor onto the doc.tensor object.
"""Concatenate a new tensor onto the doc.tensor object.
The doc.tensor attribute holds dense feature vectors
computed by the models in the pipeline. Let's say a
@ -849,7 +889,7 @@ cdef class Doc:
per word. doc.tensor.shape will be (30, 128). After
calling doc.extend_tensor with an array of shape (30, 64),
doc.tensor == (30, 192).
'''
"""
xp = get_array_module(self.tensor)
if self.tensor.size == 0:
self.tensor.resize(tensor.shape, refcheck=False)
@ -858,7 +898,7 @@ cdef class Doc:
self.tensor = xp.hstack((self.tensor, tensor))
def retokenize(self):
'''Context manager to handle retokenization of the Doc.
"""Context manager to handle retokenization of the Doc.
Modifications to the Doc's tokenization are stored, and then
made all at once when the context manager exits. This is
much more efficient, and less error-prone.
@ -866,7 +906,10 @@ cdef class Doc:
All views of the Doc (Span and Token) created before the
retokenization are invalidated, although they may accidentally
continue to work.
'''
DOCS: https://spacy.io/api/doc#retokenize
USAGE: https://spacy.io/usage/linguistic-features#retokenization
"""
return Retokenizer(self)
def _bulk_merge(self, spans, attributes):
@ -882,9 +925,10 @@ cdef class Doc:
RETURNS (Token): The first newly merged token.
"""
cdef unicode tag, lemma, ent_type
assert len(attributes) == len(spans), "attribute length should be equal to span length" + str(len(attributes)) +\
str(len(spans))
attr_len = len(attributes)
span_len = len(spans)
if not attr_len == span_len:
raise ValueError(Errors.E121.format(attr_len=attr_len, span_len=span_len))
with self.retokenize() as retokenizer:
for i, span in enumerate(spans):
fix_attributes(self, attributes[i])
@ -915,13 +959,10 @@ cdef class Doc:
elif not args:
fix_attributes(self, attributes)
elif args:
raise ValueError(Errors.E034.format(n_args=len(args),
args=repr(args),
raise ValueError(Errors.E034.format(n_args=len(args), args=repr(args),
kwargs=repr(attributes)))
remove_label_if_necessary(attributes)
attributes = intify_attrs(attributes, strings_map=self.vocab.strings)
cdef int start = token_by_start(self.c, self.length, start_idx)
if start == -1:
return None
@ -938,44 +979,47 @@ cdef class Doc:
raise ValueError(Errors.E105)
def to_json(self, underscore=None):
"""Convert a Doc to JSON. Produces the same format used by the spacy
train command.
"""Convert a Doc to JSON. The format it produces will be the new format
for the `spacy train` command (not implemented yet).
underscore (list): Optional list of string names of custom doc._.
attributes. Attribute values need to be JSON-serializable. Values will
be added to an "_" key in the data, e.g. "_": {"foo": "bar"}.
RETURNS (dict): The data in spaCy's JSON format.
DOCS: https://spacy.io/api/doc#to_json
"""
data = {'text': self.text}
data['ents'] = [{'start': ent.start_char, 'end': ent.end_char,
'label': ent.label_} for ent in self.ents]
data = {"text": self.text}
if self.ents:
data["ents"] = [{"start": ent.start_char, "end": ent.end_char,
"label": ent.label_} for ent in self.ents]
sents = list(self.sents)
if sents:
data['sents'] = [{'start': sent.start_char, 'end': sent.end_char}
data["sents"] = [{"start": sent.start_char, "end": sent.end_char}
for sent in sents]
if self.cats:
data['cats'] = self.cats
data['tokens'] = []
data["cats"] = self.cats
data["tokens"] = []
for token in self:
token_data = {'id': token.i, 'start': token.idx, 'end': token.idx + len(token)}
token_data = {"id": token.i, "start": token.idx, "end": token.idx + len(token)}
if token.pos_:
token_data['pos'] = token.pos_
token_data["pos"] = token.pos_
if token.tag_:
token_data['tag'] = token.tag_
token_data["tag"] = token.tag_
if token.dep_:
token_data['dep'] = token.dep_
token_data["dep"] = token.dep_
if token.head:
token_data['head'] = token.head.i
data['tokens'].append(token_data)
token_data["head"] = token.head.i
data["tokens"].append(token_data)
if underscore:
data['_'] = {}
data["_"] = {}
for attr in underscore:
if not self.has_extension(attr):
raise ValueError(Errors.E106.format(attr=attr, opts=underscore))
value = self._.get(attr)
if not srsly.is_json_serializable(value):
raise ValueError(Errors.E107.format(attr=attr, value=repr(value)))
data['_'][attr] = value
data["_"][attr] = value
return data
@ -1007,9 +1051,8 @@ cdef int set_children_from_heads(TokenC* tokens, int length) except -1:
tokens[i].r_kids = 0
tokens[i].l_edge = i
tokens[i].r_edge = i
# Three times, for non-projectivity
# See issue #3170. This isn't a very satisfying fix, but I think it's
# sufficient.
# Three times, for non-projectivity. See issue #3170. This isn't a very
# satisfying fix, but I think it's sufficient.
for loop_count in range(3):
# Set left edges
for i in range(length):
@ -1021,7 +1064,7 @@ cdef int set_children_from_heads(TokenC* tokens, int length) except -1:
head.l_edge = child.l_edge
if child.r_edge > head.r_edge:
head.r_edge = child.r_edge
# Set right edges --- same as above, but iterate in reverse
# Set right edges - same as above, but iterate in reverse
for i in range(length-1, -1, -1):
child = &tokens[i]
head = &tokens[i + child.head]
@ -1052,20 +1095,14 @@ cdef int _get_tokens_lca(Token token_j, Token token_k):
return token_k.i
elif token_k.head == token_j:
return token_j.i
token_j_ancestors = set(token_j.ancestors)
if token_k in token_j_ancestors:
return token_k.i
for token_k_ancestor in token_k.ancestors:
if token_k_ancestor == token_j:
return token_j.i
if token_k_ancestor in token_j_ancestors:
return token_k_ancestor.i
return -1
@ -1083,12 +1120,10 @@ cdef int [:,:] _get_lca_matrix(Doc doc, int start, int end):
with shape (n, n), where n = len(doc).
"""
cdef int [:,:] lca_matrix
n_tokens= end - start
lca_mat = numpy.empty((n_tokens, n_tokens), dtype=numpy.int32)
lca_mat.fill(-1)
lca_matrix = lca_mat
for j in range(n_tokens):
token_j = doc[start + j]
# the common ancestor of token and itself is itself:
@ -1109,7 +1144,6 @@ cdef int [:,:] _get_lca_matrix(Doc doc, int start, int end):
else:
lca_matrix[j, k] = lca - start
lca_matrix[k, j] = lca - start
return lca_matrix
@ -1123,8 +1157,7 @@ def pickle_doc(doc):
def unpickle_doc(vocab, hooks_and_data, bytes_data):
user_data, doc_hooks, span_hooks, token_hooks = srsly.pickle_loads(hooks_and_data)
doc = Doc(vocab, user_data=user_data).from_bytes(bytes_data,
exclude='user_data')
doc = Doc(vocab, user_data=user_data).from_bytes(bytes_data, exclude="user_data")
doc.user_hooks.update(doc_hooks)
doc.user_span_hooks.update(span_hooks)
doc.user_token_hooks.update(token_hooks)
@ -1133,19 +1166,22 @@ def unpickle_doc(vocab, hooks_and_data, bytes_data):
copy_reg.pickle(Doc, pickle_doc, unpickle_doc)
def remove_label_if_necessary(attributes):
# More deprecated attribute handling =/
if 'label' in attributes:
attributes['ent_type'] = attributes.pop('label')
if "label" in attributes:
attributes["ent_type"] = attributes.pop("label")
def fix_attributes(doc, attributes):
if 'label' in attributes and 'ent_type' not in attributes:
if isinstance(attributes['label'], int):
attributes[ENT_TYPE] = attributes['label']
if "label" in attributes and "ent_type" not in attributes:
if isinstance(attributes["label"], int):
attributes[ENT_TYPE] = attributes["label"]
else:
attributes[ENT_TYPE] = doc.vocab.strings[attributes['label']]
if 'ent_type' in attributes:
attributes[ENT_TYPE] = attributes['ent_type']
attributes[ENT_TYPE] = doc.vocab.strings[attributes["label"]]
if "ent_type" in attributes:
attributes[ENT_TYPE] = attributes["ent_type"]
def get_entity_info(ent_info):
if isinstance(ent_info, Span):

258
spacy/tokens/span.pyx
View File

@ -1,12 +1,13 @@
# coding: utf8
from __future__ import unicode_literals
from collections import defaultdict
cimport numpy as np
from libc.math cimport sqrt
import numpy
import numpy.linalg
from libc.math cimport sqrt
from thinc.neural.util import get_array_module
from collections import defaultdict
from .doc cimport token_by_start, token_by_end, get_token_attr, _get_lca_matrix
from .token cimport TokenC
@ -14,9 +15,10 @@ from ..structs cimport TokenC, LexemeC
from ..typedefs cimport flags_t, attr_t, hash_t
from ..attrs cimport attr_id_t
from ..parts_of_speech cimport univ_pos_t
from ..util import normalize_slice
from ..attrs cimport *
from ..lexeme cimport Lexeme
from ..util import normalize_slice
from ..compat import is_config, basestring_
from ..errors import Errors, TempErrors, Warnings, user_warning, models_warning
from ..errors import deprecation_warning
@ -24,29 +26,66 @@ from .underscore import Underscore, get_ext_args
cdef class Span:
"""A slice from a Doc object."""
"""A slice from a Doc object.
DOCS: https://spacy.io/api/span
"""
@classmethod
def set_extension(cls, name, **kwargs):
if cls.has_extension(name) and not kwargs.get('force', False):
raise ValueError(Errors.E090.format(name=name, obj='Span'))
"""Define a custom attribute which becomes available as `Span._`.
name (unicode): Name of the attribute to set.
default: Optional default value of the attribute.
getter (callable): Optional getter function.
setter (callable): Optional setter function.
method (callable): Optional method for method extension.
force (bool): Force overwriting existing attribute.
DOCS: https://spacy.io/api/span#set_extension
USAGE: https://spacy.io/usage/processing-pipelines#custom-components-attributes
"""
if cls.has_extension(name) and not kwargs.get("force", False):
raise ValueError(Errors.E090.format(name=name, obj="Span"))
Underscore.span_extensions[name] = get_ext_args(**kwargs)
@classmethod
def get_extension(cls, name):
"""Look up a previously registered extension by name.
name (unicode): Name of the extension.
RETURNS (tuple): A `(default, method, getter, setter)` tuple.
DOCS: https://spacy.io/api/span#get_extension
"""
return Underscore.span_extensions.get(name)
@classmethod
def has_extension(cls, name):
"""Check whether an extension has been registered.
name (unicode): Name of the extension.
RETURNS (bool): Whether the extension has been registered.
DOCS: https://spacy.io/api/span#has_extension
"""
return name in Underscore.span_extensions
@classmethod
def remove_extension(cls, name):
"""Remove a previously registered extension.
name (unicode): Name of the extension.
RETURNS (tuple): A `(default, method, getter, setter)` tuple of the
removed extension.
DOCS: https://spacy.io/api/span#remove_extension
"""
if not cls.has_extension(name):
raise ValueError(Errors.E046.format(name=name))
return Underscore.span_extensions.pop(name)
def __cinit__(self, Doc doc, int start, int end, label=0,
vector=None, vector_norm=None):
def __cinit__(self, Doc doc, int start, int end, label=0, vector=None,
vector_norm=None):
"""Create a `Span` object from the slice `doc[start : end]`.
doc (Doc): The parent document.
@ -56,6 +95,8 @@ cdef class Span:
vector (ndarray[ndim=1, dtype='float32']): A meaning representation
of the span.
RETURNS (Span): The newly constructed object.
DOCS: https://spacy.io/api/span#init
"""
if not (0 <= start <= end <= len(doc)):
raise IndexError(Errors.E035.format(start=start, end=end, length=len(doc)))
@ -102,6 +143,8 @@ cdef class Span:
"""Get the number of tokens in the span.
RETURNS (int): The number of tokens in the span.
DOCS: https://spacy.io/api/span#len
"""
self._recalculate_indices()
if self.end < self.start:
@ -111,7 +154,7 @@ cdef class Span:
def __repr__(self):
if is_config(python3=True):
return self.text
return self.text.encode('utf-8')
return self.text.encode("utf-8")
def __getitem__(self, object i):
"""Get a `Token` or a `Span` object
@ -120,9 +163,7 @@ cdef class Span:
the span to get.
RETURNS (Token or Span): The token at `span[i]`.
EXAMPLE:
>>> span[0]
>>> span[1:3]
DOCS: https://spacy.io/api/span#getitem
"""
self._recalculate_indices()
if isinstance(i, slice):
@ -138,6 +179,8 @@ cdef class Span:
"""Iterate over `Token` objects.
YIELDS (Token): A `Token` object.
DOCS: https://spacy.io/api/span#iter
"""
self._recalculate_indices()
for i in range(self.start, self.end):
@ -148,31 +191,32 @@ cdef class Span:
@property
def _(self):
"""User space for adding custom attribute extensions."""
"""Custom extension attributes registered via `set_extension`."""
return Underscore(Underscore.span_extensions, self,
start=self.start_char, end=self.end_char)
def as_doc(self):
# TODO: fix
"""Create a `Doc` object with a copy of the Span's data.
"""Create a `Doc` object with a copy of the `Span`'s data.
RETURNS (Doc): The `Doc` copy of the span.
DOCS: https://spacy.io/api/span#as_doc
"""
cdef Doc doc = Doc(self.doc.vocab,
words=[t.text for t in self],
spaces=[bool(t.whitespace_) for t in self])
# TODO: Fix!
words = [t.text for t in self]
spaces = [bool(t.whitespace_) for t in self]
cdef Doc doc = Doc(self.doc.vocab, words=words, spaces=spaces)
array_head = [LENGTH, SPACY, LEMMA, ENT_IOB, ENT_TYPE]
if self.doc.is_tagged:
array_head.append(TAG)
# if doc parsed add head and dep attribute
# If doc parsed add head and dep attribute
if self.doc.is_parsed:
array_head.extend([HEAD, DEP])
# otherwise add sent_start
# Otherwise add sent_start
else:
array_head.append(SENT_START)
array = self.doc.to_array(array_head)
doc.from_array(array_head, array[self.start : self.end])
doc.noun_chunks_iterator = self.doc.noun_chunks_iterator
doc.user_hooks = self.doc.user_hooks
doc.user_span_hooks = self.doc.user_span_hooks
@ -181,7 +225,7 @@ cdef class Span:
doc.vector_norm = self.vector_norm
doc.tensor = self.doc.tensor[self.start : self.end]
for key, value in self.doc.cats.items():
if hasattr(key, '__len__') and len(key) == 3:
if hasattr(key, "__len__") and len(key) == 3:
cat_start, cat_end, cat_label = key
if cat_start == self.start_char and cat_end == self.end_char:
doc.cats[cat_label] = value
@ -207,6 +251,8 @@ cdef class Span:
RETURNS (np.array[ndim=2, dtype=numpy.int32]): LCA matrix with shape
(n, n), where n = len(self).
DOCS: https://spacy.io/api/span#get_lca_matrix
"""
return numpy.asarray(_get_lca_matrix(self.doc, self.start, self.end))
@ -217,22 +263,24 @@ cdef class Span:
other (object): The object to compare with. By default, accepts `Doc`,
`Span`, `Token` and `Lexeme` objects.
RETURNS (float): A scalar similarity score. Higher is more similar.
DOCS: https://spacy.io/api/span#similarity
"""
if 'similarity' in self.doc.user_span_hooks:
self.doc.user_span_hooks['similarity'](self, other)
if len(self) == 1 and hasattr(other, 'orth'):
if "similarity" in self.doc.user_span_hooks:
self.doc.user_span_hooks["similarity"](self, other)
if len(self) == 1 and hasattr(other, "orth"):
if self[0].orth == other.orth:
return 1.0
elif hasattr(other, '__len__') and len(self) == len(other):
elif hasattr(other, "__len__") and len(self) == len(other):
for i in range(len(self)):
if self[i].orth != getattr(other[i], 'orth', None):
if self[i].orth != getattr(other[i], "orth", None):
break
else:
return 1.0
if self.vocab.vectors.n_keys == 0:
models_warning(Warnings.W007.format(obj='Span'))
models_warning(Warnings.W007.format(obj="Span"))
if self.vector_norm == 0.0 or other.vector_norm == 0.0:
user_warning(Warnings.W008.format(obj='Span'))
user_warning(Warnings.W008.format(obj="Span"))
return 0.0
vector = self.vector
xp = get_array_module(vector)
@ -251,8 +299,8 @@ cdef class Span:
cdef int i, j
cdef attr_id_t feature
cdef np.ndarray[attr_t, ndim=2] output
# Make an array from the attributes --- otherwise our inner loop is Python
# dict iteration.
# Make an array from the attributes - otherwise our inner loop is Python
# dict iteration
cdef np.ndarray[attr_t, ndim=1] attr_ids = numpy.asarray(py_attr_ids, dtype=numpy.uint64)
cdef int length = self.end - self.start
output = numpy.ndarray(shape=(length, len(attr_ids)), dtype=numpy.uint64)
@ -282,12 +330,11 @@ cdef class Span:
property sent:
"""RETURNS (Span): The sentence span that the span is a part of."""
def __get__(self):
if 'sent' in self.doc.user_span_hooks:
return self.doc.user_span_hooks['sent'](self)
# This should raise if we're not parsed
# or doesen't have any sbd component :)
if "sent" in self.doc.user_span_hooks:
return self.doc.user_span_hooks["sent"](self)
# This should raise if not parsed / no custom sentence boundaries
self.doc.sents
# if doc is parsed we can use the deps to find the sentence
# If doc is parsed we can use the deps to find the sentence
# otherwise we use the `sent_start` token attribute
cdef int n = 0
cdef int i
@ -300,11 +347,11 @@ cdef class Span:
raise RuntimeError(Errors.E038)
return self.doc[root.l_edge:root.r_edge + 1]
elif self.doc.is_sentenced:
# find start of the sentence
# Find start of the sentence
start = self.start
while self.doc.c[start].sent_start != 1 and start > 0:
start += -1
# find end of the sentence
# Find end of the sentence
end = self.end
n = 0
while end < self.doc.length and self.doc.c[end].sent_start != 1:
@ -315,7 +362,13 @@ cdef class Span:
return self.doc[start:end]
property ents:
"""RETURNS (list): A list of tokens that belong to the current span."""
"""The named entities in the span. Returns a tuple of named entity
`Span` objects, if the entity recognizer has been applied.
RETURNS (tuple): Entities in the span, one `Span` per entity.
DOCS: https://spacy.io/api/span#ents
"""
def __get__(self):
ents = []
for ent in self.doc.ents:
@ -324,11 +377,16 @@ cdef class Span:
return ents
property has_vector:
"""RETURNS (bool): Whether a word vector is associated with the object.
"""A boolean value indicating whether a word vector is associated with
the object.
RETURNS (bool): Whether a word vector is associated with the object.
DOCS: https://spacy.io/api/span#has_vector
"""
def __get__(self):
if 'has_vector' in self.doc.user_span_hooks:
return self.doc.user_span_hooks['has_vector'](self)
if "has_vector" in self.doc.user_span_hooks:
return self.doc.user_span_hooks["has_vector"](self)
elif self.vocab.vectors.data.size > 0:
return any(token.has_vector for token in self)
elif self.doc.tensor.size > 0:
@ -342,19 +400,26 @@ cdef class Span:
RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array
representing the span's semantics.
DOCS: https://spacy.io/api/span#vector
"""
def __get__(self):
if 'vector' in self.doc.user_span_hooks:
return self.doc.user_span_hooks['vector'](self)
if "vector" in self.doc.user_span_hooks:
return self.doc.user_span_hooks["vector"](self)
if self._vector is None:
self._vector = sum(t.vector for t in self) / len(self)
return self._vector
property vector_norm:
"""RETURNS (float): The L2 norm of the vector representation."""
"""The L2 norm of the span's vector representation.
RETURNS (float): The L2 norm of the vector representation.
DOCS: https://spacy.io/api/span#vector_norm
"""
def __get__(self):
if 'vector_norm' in self.doc.user_span_hooks:
return self.doc.user_span_hooks['vector'](self)
if "vector_norm" in self.doc.user_span_hooks:
return self.doc.user_span_hooks["vector"](self)
cdef float value
cdef double norm = 0
if self._vector_norm is None:
@ -369,8 +434,8 @@ cdef class Span:
negativity of the span.
"""
def __get__(self):
if 'sentiment' in self.doc.user_span_hooks:
return self.doc.user_span_hooks['sentiment'](self)
if "sentiment" in self.doc.user_span_hooks:
return self.doc.user_span_hooks["sentiment"](self)
else:
return sum([token.sentiment for token in self]) / len(self)
@ -390,7 +455,7 @@ cdef class Span:
whitespace).
"""
def __get__(self):
return u''.join([t.text_with_ws for t in self])
return "".join([t.text_with_ws for t in self])
property noun_chunks:
"""Yields base noun-phrase `Span` objects, if the document has been
@ -399,7 +464,9 @@ cdef class Span:
NP-level coordination, no prepositional phrases, and no relative
clauses.
YIELDS (Span): Base noun-phrase `Span` objects
YIELDS (Span): Base noun-phrase `Span` objects.
DOCS: https://spacy.io/api/span#noun_chunks
"""
def __get__(self):
if not self.doc.is_parsed:
@ -418,52 +485,18 @@ cdef class Span:
yield span
property root:
"""The token within the span that's highest in the parse tree.
If there's a tie, the earliest is prefered.
"""The token with the shortest path to the root of the
sentence (or the root itself). If multiple tokens are equally
high in the tree, the first token is taken.
RETURNS (Token): The root token.
EXAMPLE: The root token has the shortest path to the root of the
sentence (or is the root itself). If multiple words are equally
high in the tree, the first word is taken. For example:
>>> toks = nlp(u'I like New York in Autumn.')
Let's name the indices – easier than writing `toks[4]` etc.
>>> i, like, new, york, in_, autumn, dot = range(len(toks))
The head of 'new' is 'York', and the head of "York" is "like"
>>> toks[new].head.text
'York'
>>> toks[york].head.text
'like'
Create a span for "New York". Its root is "York".
>>> new_york = toks[new:york+1]
>>> new_york.root.text
'York'
Here's a more complicated case, raised by issue #214:
>>> toks = nlp(u'to, north and south carolina')
>>> to, north, and_, south, carolina = toks
>>> south.head.text, carolina.head.text
('north', 'to')
Here "south" is a child of "north", which is a child of "carolina".
Carolina is the root of the span:
>>> south_carolina = toks[-2:]
>>> south_carolina.root.text
'carolina'
DOCS: https://spacy.io/api/span#root
"""
def __get__(self):
self._recalculate_indices()
if 'root' in self.doc.user_span_hooks:
return self.doc.user_span_hooks['root'](self)
if "root" in self.doc.user_span_hooks:
return self.doc.user_span_hooks["root"](self)
# This should probably be called 'head', and the other one called
# 'gov'. But we went with 'head' elsehwhere, and now we're stuck =/
cdef int i
@ -495,10 +528,12 @@ cdef class Span:
return self.doc[root]
property lefts:
""" Tokens that are to the left of the span, whose head is within the
"""Tokens that are to the left of the span, whose head is within the
`Span`.
YIELDS (Token):A left-child of a token of the span.
DOCS: https://spacy.io/api/span#lefts
"""
def __get__(self):
for token in reversed(self): # Reverse, so we get tokens in order
@ -511,6 +546,8 @@ cdef class Span:
`Span`.
YIELDS (Token): A right-child of a token of the span.
DOCS: https://spacy.io/api/span#rights
"""
def __get__(self):
for token in self:
@ -519,15 +556,25 @@ cdef class Span:
yield right
property n_lefts:
"""RETURNS (int): The number of leftward immediate children of the
"""The number of tokens that are to the left of the span, whose
heads are within the span.
RETURNS (int): The number of leftward immediate children of the
span, in the syntactic dependency parse.
DOCS: https://spacy.io/api/span#n_lefts
"""
def __get__(self):
return len(list(self.lefts))
property n_rights:
"""RETURNS (int): The number of rightward immediate children of the
"""The number of tokens that are to the right of the span, whose
heads are within the span.
RETURNS (int): The number of rightward immediate children of the
span, in the syntactic dependency parse.
DOCS: https://spacy.io/api/span#n_rights
"""
def __get__(self):
return len(list(self.rights))
@ -536,6 +583,8 @@ cdef class Span:
"""Tokens within the span and tokens which descend from them.
YIELDS (Token): A token within the span, or a descendant from it.
DOCS: https://spacy.io/api/span#subtree
"""
def __get__(self):
for word in self.lefts:
@ -550,7 +599,7 @@ cdef class Span:
return self.root.ent_id
def __set__(self, hash_t key):
raise NotImplementedError(TempErrors.T007.format(attr='ent_id'))
raise NotImplementedError(TempErrors.T007.format(attr="ent_id"))
property ent_id_:
"""RETURNS (unicode): The (string) entity ID."""
@ -558,10 +607,10 @@ cdef class Span:
return self.root.ent_id_
def __set__(self, hash_t key):
raise NotImplementedError(TempErrors.T007.format(attr='ent_id_'))
raise NotImplementedError(TempErrors.T007.format(attr="ent_id_"))
property orth_:
"""Verbatim text content (identical to Span.text). Exists mostly for
"""Verbatim text content (identical to `Span.text`). Exists mostly for
consistency with other attributes.
RETURNS (unicode): The span's text."""
@ -571,27 +620,28 @@ cdef class Span:
property lemma_:
"""RETURNS (unicode): The span's lemma."""
def __get__(self):
return ' '.join([t.lemma_ for t in self]).strip()
return " ".join([t.lemma_ for t in self]).strip()
property upper_:
"""Deprecated. Use Span.text.upper() instead."""
"""Deprecated. Use `Span.text.upper()` instead."""
def __get__(self):
return ''.join([t.text_with_ws.upper() for t in self]).strip()
return "".join([t.text_with_ws.upper() for t in self]).strip()
property lower_:
"""Deprecated. Use Span.text.lower() instead."""
"""Deprecated. Use `Span.text.lower()` instead."""
def __get__(self):
return ''.join([t.text_with_ws.lower() for t in self]).strip()
return "".join([t.text_with_ws.lower() for t in self]).strip()
property string:
"""Deprecated: Use Span.text_with_ws instead."""
"""Deprecated: Use `Span.text_with_ws` instead."""
def __get__(self):
return ''.join([t.text_with_ws for t in self])
return "".join([t.text_with_ws for t in self])
property label_:
"""RETURNS (unicode): The span's label."""
def __get__(self):
return self.doc.vocab.strings[self.label]
def __set__(self, unicode label_):
self.label = self.doc.vocab.strings.add(label_)

203
spacy/tokens/token.pyx
View File

@ -8,42 +8,82 @@ from cpython.mem cimport PyMem_Malloc, PyMem_Free
from cython.view cimport array as cvarray
cimport numpy as np
np.import_array()
import numpy
from thinc.neural.util import get_array_module
from ..typedefs cimport hash_t
from ..lexeme cimport Lexeme
from .. import parts_of_speech
from ..attrs cimport IS_ALPHA, IS_ASCII, IS_DIGIT, IS_LOWER, IS_PUNCT, IS_SPACE
from ..attrs cimport IS_BRACKET, IS_QUOTE, IS_LEFT_PUNCT, IS_RIGHT_PUNCT
from ..attrs cimport IS_OOV, IS_TITLE, IS_UPPER, IS_CURRENCY, LIKE_URL, LIKE_NUM, LIKE_EMAIL
from ..attrs cimport IS_STOP, ID, ORTH, NORM, LOWER, SHAPE, PREFIX, SUFFIX
from ..attrs cimport LENGTH, CLUSTER, LEMMA, POS, TAG, DEP
from ..symbols cimport conj
from .. import parts_of_speech
from .. import util
from ..compat import is_config
from ..errors import Errors, Warnings, user_warning, models_warning
from .. import util
from .underscore import Underscore, get_ext_args
cdef class Token:
"""An individual token – i.e. a word, punctuation symbol, whitespace,
etc."""
etc.
DOCS: https://spacy.io/api/token
"""
@classmethod
def set_extension(cls, name, **kwargs):
if cls.has_extension(name) and not kwargs.get('force', False):
raise ValueError(Errors.E090.format(name=name, obj='Token'))
"""Define a custom attribute which becomes available as `Token._`.
name (unicode): Name of the attribute to set.
default: Optional default value of the attribute.
getter (callable): Optional getter function.
setter (callable): Optional setter function.
method (callable): Optional method for method extension.
force (bool): Force overwriting existing attribute.
DOCS: https://spacy.io/api/token#set_extension
USAGE: https://spacy.io/usage/processing-pipelines#custom-components-attributes
"""
if cls.has_extension(name) and not kwargs.get("force", False):
raise ValueError(Errors.E090.format(name=name, obj="Token"))
Underscore.token_extensions[name] = get_ext_args(**kwargs)
@classmethod
def get_extension(cls, name):
"""Look up a previously registered extension by name.
name (unicode): Name of the extension.
RETURNS (tuple): A `(default, method, getter, setter)` tuple.
DOCS: https://spacy.io/api/token#get_extension
"""
return Underscore.token_extensions.get(name)
@classmethod
def has_extension(cls, name):
"""Check whether an extension has been registered.
name (unicode): Name of the extension.
RETURNS (bool): Whether the extension has been registered.
DOCS: https://spacy.io/api/token#has_extension
"""
return name in Underscore.token_extensions
@classmethod
def remove_extension(cls, name):
"""Remove a previously registered extension.
name (unicode): Name of the extension.
RETURNS (tuple): A `(default, method, getter, setter)` tuple of the
removed extension.
DOCS: https://spacy.io/api/token#remove_extension
"""
if not cls.has_extension(name):
raise ValueError(Errors.E046.format(name=name))
return Underscore.token_extensions.pop(name)
@ -54,6 +94,8 @@ cdef class Token:
vocab (Vocab): A storage container for lexical types.
doc (Doc): The parent document.
offset (int): The index of the token within the document.
DOCS: https://spacy.io/api/token#init
"""
self.vocab = vocab
self.doc = doc
@ -67,6 +109,8 @@ cdef class Token:
"""The number of unicode characters in the token, i.e. `token.text`.
RETURNS (int): The number of unicode characters in the token.
DOCS: https://spacy.io/api/token#len
"""
return self.c.lex.length
@ -121,6 +165,7 @@ cdef class Token:
@property
def _(self):
"""Custom extension attributes registered via `set_extension`."""
return Underscore(Underscore.token_extensions, self,
start=self.idx, end=None)
@ -130,12 +175,7 @@ cdef class Token:
flag_id (int): The ID of the flag attribute.
RETURNS (bool): Whether the flag is set.
EXAMPLE:
>>> from spacy.attrs import IS_TITLE
>>> doc = nlp(u'Give it back! He pleaded.')
>>> token = doc[0]
>>> token.check_flag(IS_TITLE)
True
DOCS: https://spacy.io/api/token#check_flag
"""
return Lexeme.c_check_flag(self.c.lex, flag_id)
@ -144,6 +184,8 @@ cdef class Token:
i (int): The relative position of the token to get. Defaults to 1.
RETURNS (Token): The token at position `self.doc[self.i+i]`.
DOCS: https://spacy.io/api/token#nbor
"""
if self.i+i < 0 or (self.i+i >= len(self.doc)):
raise IndexError(Errors.E042.format(i=self.i, j=i, length=len(self.doc)))
@ -156,19 +198,21 @@ cdef class Token:
other (object): The object to compare with. By default, accepts `Doc`,
`Span`, `Token` and `Lexeme` objects.
RETURNS (float): A scalar similarity score. Higher is more similar.
DOCS: https://spacy.io/api/token#similarity
"""
if 'similarity' in self.doc.user_token_hooks:
return self.doc.user_token_hooks['similarity'](self)
if hasattr(other, '__len__') and len(other) == 1 and hasattr(other, "__getitem__"):
if self.c.lex.orth == getattr(other[0], 'orth', None):
if "similarity" in self.doc.user_token_hooks:
return self.doc.user_token_hooks["similarity"](self)
if hasattr(other, "__len__") and len(other) == 1 and hasattr(other, "__getitem__"):
if self.c.lex.orth == getattr(other[0], "orth", None):
return 1.0
elif hasattr(other, 'orth'):
elif hasattr(other, "orth"):
if self.c.lex.orth == other.orth:
return 1.0
if self.vocab.vectors.n_keys == 0:
models_warning(Warnings.W007.format(obj='Token'))
models_warning(Warnings.W007.format(obj="Token"))
if self.vector_norm == 0 or other.vector_norm == 0:
user_warning(Warnings.W008.format(obj='Token'))
user_warning(Warnings.W008.format(obj="Token"))
return 0.0
vector = self.vector
xp = get_array_module(vector)
@ -202,7 +246,7 @@ cdef class Token:
def __get__(self):
cdef unicode orth = self.vocab.strings[self.c.lex.orth]
if self.c.spacy:
return orth + u' '
return orth + " "
else:
return orth
@ -215,8 +259,8 @@ cdef class Token:
"""RETURNS (float): A scalar value indicating the positivity or
negativity of the token."""
def __get__(self):
if 'sentiment' in self.doc.user_token_hooks:
return self.doc.user_token_hooks['sentiment'](self)
if "sentiment" in self.doc.user_token_hooks:
return self.doc.user_token_hooks["sentiment"](self)
return self.c.lex.sentiment
property lang:
@ -298,6 +342,7 @@ cdef class Token:
"""RETURNS (uint64): ID of coarse-grained part-of-speech tag."""
def __get__(self):
return self.c.pos
def __set__(self, pos):
self.c.pos = pos
@ -322,10 +367,12 @@ cdef class Token:
the object.
RETURNS (bool): Whether a word vector is associated with the object.
DOCS: https://spacy.io/api/token#has_vector
"""
def __get__(self):
if 'has_vector' in self.doc.user_token_hooks:
return self.doc.user_token_hooks['has_vector'](self)
return self.doc.user_token_hooks["has_vector"](self)
if self.vocab.vectors.size == 0 and self.doc.tensor.size != 0:
return True
return self.vocab.has_vector(self.c.lex.orth)
@ -335,10 +382,12 @@ cdef class Token:
RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array
representing the token's semantics.
DOCS: https://spacy.io/api/token#vector
"""
def __get__(self):
if 'vector' in self.doc.user_token_hooks:
return self.doc.user_token_hooks['vector'](self)
return self.doc.user_token_hooks["vector"](self)
if self.vocab.vectors.size == 0 and self.doc.tensor.size != 0:
return self.doc.tensor[self.i]
else:
@ -348,23 +397,35 @@ cdef class Token:
"""The L2 norm of the token's vector representation.
RETURNS (float): The L2 norm of the vector representation.
DOCS: https://spacy.io/api/token#vector_norm
"""
def __get__(self):
if 'vector_norm' in self.doc.user_token_hooks:
return self.doc.user_token_hooks['vector_norm'](self)
return self.doc.user_token_hooks["vector_norm"](self)
vector = self.vector
return numpy.sqrt((vector ** 2).sum())
property n_lefts:
"""RETURNS (int): The number of leftward immediate children of the
"""The number of leftward immediate children of the word, in the
syntactic dependency parse.
RETURNS (int): The number of leftward immediate children of the
word, in the syntactic dependency parse.
DOCS: https://spacy.io/api/token#n_lefts
"""
def __get__(self):
return self.c.l_kids
property n_rights:
"""RETURNS (int): The number of rightward immediate children of the
"""The number of rightward immediate children of the word, in the
syntactic dependency parse.
RETURNS (int): The number of rightward immediate children of the
word, in the syntactic dependency parse.
DOCS: https://spacy.io/api/token#n_rights
"""
def __get__(self):
return self.c.r_kids
@ -373,7 +434,7 @@ cdef class Token:
"""RETURNS (Span): The sentence span that the token is a part of."""
def __get__(self):
if 'sent' in self.doc.user_token_hooks:
return self.doc.user_token_hooks['sent'](self)
return self.doc.user_token_hooks["sent"](self)
return self.doc[self.i : self.i+1].sent
property sent_start:
@ -390,8 +451,13 @@ cdef class Token:
self.is_sent_start = value
property is_sent_start:
"""RETURNS (bool / None): Whether the token starts a sentence.
"""A boolean value indicating whether the token starts a sentence.
`None` if unknown. Defaults to `True` for the first token in the `Doc`.
RETURNS (bool / None): Whether the token starts a sentence.
None if unknown.
DOCS: https://spacy.io/api/token#is_sent_start
"""
def __get__(self):
if self.c.sent_start == 0:
@ -418,6 +484,8 @@ cdef class Token:
dependency parse.
YIELDS (Token): A left-child of the token.
DOCS: https://spacy.io/api/token#lefts
"""
def __get__(self):
cdef int nr_iter = 0
@ -429,13 +497,15 @@ cdef class Token:
nr_iter += 1
# This is ugly, but it's a way to guard out infinite loops
if nr_iter >= 10000000:
raise RuntimeError(Errors.E045.format(attr='token.lefts'))
raise RuntimeError(Errors.E045.format(attr="token.lefts"))
property rights:
"""The rightward immediate children of the word, in the syntactic
dependency parse.
YIELDS (Token): A right-child of the token.
DOCS: https://spacy.io/api/token#rights
"""
def __get__(self):
cdef const TokenC* ptr = self.c + (self.c.r_edge - self.i)
@ -447,7 +517,7 @@ cdef class Token:
ptr -= 1
nr_iter += 1
if nr_iter >= 10000000:
raise RuntimeError(Errors.E045.format(attr='token.rights'))
raise RuntimeError(Errors.E045.format(attr="token.rights"))
tokens.reverse()
for t in tokens:
yield t
@ -455,7 +525,9 @@ cdef class Token:
property children:
"""A sequence of the token's immediate syntactic children.
YIELDS (Token): A child token such that child.head==self
YIELDS (Token): A child token such that `child.head==self`.
DOCS: https://spacy.io/api/token#children
"""
def __get__(self):
yield from self.lefts
@ -467,6 +539,8 @@ cdef class Token:
YIELDS (Token): A descendent token such that
`self.is_ancestor(descendent) or token == self`.
DOCS: https://spacy.io/api/token#subtree
"""
def __get__(self):
for word in self.lefts:
@ -496,11 +570,13 @@ cdef class Token:
YIELDS (Token): A sequence of ancestor tokens such that
`ancestor.is_ancestor(self)`.
DOCS: https://spacy.io/api/token#ancestors
"""
def __get__(self):
cdef const TokenC* head_ptr = self.c
# guard against infinite loop, no token can have
# more ancestors than tokens in the tree
# Guard against infinite loop, no token can have
# more ancestors than tokens in the tree.
cdef int i = 0
while head_ptr.head != 0 and i < self.doc.length:
head_ptr += head_ptr.head
@ -513,6 +589,8 @@ cdef class Token:
descendant (Token): Another token.
RETURNS (bool): Whether this token is the ancestor of the descendant.
DOCS: https://spacy.io/api/token#is_ancestor
"""
if self.doc is not descendant.doc:
return False
@ -528,34 +606,28 @@ cdef class Token:
return self.doc[self.i + self.c.head]
def __set__(self, Token new_head):
# this function sets the head of self to new_head
# and updates the counters for left/right dependents
# and left/right corner for the new and the old head
# do nothing if old head is new head
# This function sets the head of self to new_head and updates the
# counters for left/right dependents and left/right corner for the
# new and the old head
# Do nothing if old head is new head
if self.i + self.c.head == new_head.i:
return
cdef Token old_head = self.head
cdef int rel_newhead_i = new_head.i - self.i
# is the new head a descendant of the old head
# Is the new head a descendant of the old head
cdef bint is_desc = old_head.is_ancestor(new_head)
cdef int new_edge
cdef Token anc, child
# update number of deps of old head
# Update number of deps of old head
if self.c.head > 0: # left dependent
old_head.c.l_kids -= 1
if self.c.l_edge == old_head.c.l_edge:
# the token dominates the left edge so the left edge of
# The token dominates the left edge so the left edge of
# the head may change when the token is reattached, it may
# not change if the new head is a descendant of the current
# head
# head.
new_edge = self.c.l_edge
# the new l_edge is the left-most l_edge on any of the
# The new l_edge is the left-most l_edge on any of the
# other dependents where the l_edge is left of the head,
# otherwise it is the head
if not is_desc:
@ -566,21 +638,18 @@ cdef class Token:
if child.c.l_edge < new_edge:
new_edge = child.c.l_edge
old_head.c.l_edge = new_edge
# walk up the tree from old_head and assign new l_edge to
# Walk up the tree from old_head and assign new l_edge to
# ancestors until an ancestor already has an l_edge that's
# further left
for anc in old_head.ancestors:
if anc.c.l_edge <= new_edge:
break
anc.c.l_edge = new_edge
elif self.c.head < 0: # right dependent
old_head.c.r_kids -= 1
# do the same thing as for l_edge
# Do the same thing as for l_edge
if self.c.r_edge == old_head.c.r_edge:
new_edge = self.c.r_edge
if not is_desc:
new_edge = old_head.i
for child in old_head.children:
@ -589,16 +658,14 @@ cdef class Token:
if child.c.r_edge > new_edge:
new_edge = child.c.r_edge
old_head.c.r_edge = new_edge
for anc in old_head.ancestors:
if anc.c.r_edge >= new_edge:
break
anc.c.r_edge = new_edge
# update number of deps of new head
# Update number of deps of new head
if rel_newhead_i > 0: # left dependent
new_head.c.l_kids += 1
# walk up the tree from new head and set l_edge to self.l_edge
# Walk up the tree from new head and set l_edge to self.l_edge
# until you hit a token with an l_edge further to the left
if self.c.l_edge < new_head.c.l_edge:
new_head.c.l_edge = self.c.l_edge
@ -606,34 +673,33 @@ cdef class Token:
if anc.c.l_edge <= self.c.l_edge:
break
anc.c.l_edge = self.c.l_edge
elif rel_newhead_i < 0: # right dependent
new_head.c.r_kids += 1
# do the same as for l_edge
# Do the same as for l_edge
if self.c.r_edge > new_head.c.r_edge:
new_head.c.r_edge = self.c.r_edge
for anc in new_head.ancestors:
if anc.c.r_edge >= self.c.r_edge:
break
anc.c.r_edge = self.c.r_edge
# set new head
# Set new head
self.c.head = rel_newhead_i
property conjuncts:
"""A sequence of coordinated tokens, including the token itself.
YIELDS (Token): A coordinated token.
DOCS: https://spacy.io/api/token#conjuncts
"""
def __get__(self):
"""Get a list of conjoined words."""
cdef Token word
if 'conjuncts' in self.doc.user_token_hooks:
yield from self.doc.user_token_hooks['conjuncts'](self)
if "conjuncts" in self.doc.user_token_hooks:
yield from self.doc.user_token_hooks["conjuncts"](self)
else:
if self.dep_ != 'conj':
if self.dep != conj:
for word in self.rights:
if word.dep_ == 'conj':
if word.dep == conj:
yield word
yield from word.conjuncts
@ -670,7 +736,7 @@ cdef class Token:
RETURNS (unicode): IOB code of named entity tag.
"""
def __get__(self):
iob_strings = ('', 'I', 'O', 'B')
iob_strings = ("", "I", "O", "B")
return iob_strings[self.c.ent_iob]
property ent_id:
@ -697,7 +763,7 @@ cdef class Token:
"""RETURNS (unicode): The trailing whitespace character, if present.
"""
def __get__(self):
return ' ' if self.c.spacy else ''
return " " if self.c.spacy else ""
property orth_:
"""RETURNS (unicode): Verbatim text content (identical to
@ -770,6 +836,7 @@ cdef class Token:
"""RETURNS (unicode): Coarse-grained part-of-speech tag."""
def __get__(self):
return parts_of_speech.NAMES[self.c.pos]
def __set__(self, pos_name):
self.c.pos = parts_of_speech.IDS[pos_name]

128
spacy/vectors.pyx
View File

@ -1,30 +1,31 @@
# coding: utf8
from __future__ import unicode_literals
cimport numpy as np
from cython.operator cimport dereference as deref
from libcpp.set cimport set as cppset
import functools
import numpy
from collections import OrderedDict
import srsly
cimport numpy as np
from thinc.neural.util import get_array_module
from thinc.neural._classes.model import Model
from .strings cimport StringStore
from .strings import get_string_id
from .compat import basestring_, path2str
from .errors import Errors
from . import util
from cython.operator cimport dereference as deref
from libcpp.set cimport set as cppset
def unpickle_vectors(bytes_data):
return Vectors().from_bytes(bytes_data)
class GlobalRegistry(object):
'''Global store of vectors, to avoid repeatedly loading the data.'''
"""Global store of vectors, to avoid repeatedly loading the data."""
data = {}
@classmethod
@ -46,8 +47,10 @@ cdef class Vectors:
rows in the vectors.data table.
Multiple keys can be mapped to the same vector, and not all of the rows in
the table need to be assigned --- so len(list(vectors.keys())) may be
the table need to be assigned - so len(list(vectors.keys())) may be
greater or smaller than vectors.shape[0].
DOCS: https://spacy.io/api/vectors
"""
cdef public object name
cdef public object data
@ -62,12 +65,14 @@ cdef class Vectors:
keys (iterable): A sequence of keys, aligned with the data.
name (string): A name to identify the vectors table.
RETURNS (Vectors): The newly created object.
DOCS: https://spacy.io/api/vectors#init
"""
self.name = name
if data is None:
if shape is None:
shape = (0,0)
data = numpy.zeros(shape, dtype='f')
data = numpy.zeros(shape, dtype="f")
self.data = data
self.key2row = OrderedDict()
if self.data is not None:
@ -84,23 +89,40 @@ cdef class Vectors:
in the vector table.
RETURNS (tuple): A `(rows, dims)` pair.
DOCS: https://spacy.io/api/vectors#shape
"""
return self.data.shape
@property
def size(self):
"""RETURNS (int): rows*dims"""
"""The vector size i,e. rows * dims.
RETURNS (int): The vector size.
DOCS: https://spacy.io/api/vectors#size
"""
return self.data.shape[0] * self.data.shape[1]
@property
def is_full(self):
"""RETURNS (bool): `True` if no slots are available for new keys."""
"""Whether the vectors table is full.
RETURNS (bool): `True` if no slots are available for new keys.
DOCS: https://spacy.io/api/vectors#is_full
"""
return self._unset.size() == 0
@property
def n_keys(self):
"""RETURNS (int) The number of keys in the table. Note that this is the
number of all keys, not just unique vectors."""
"""Get the number of keys in the table. Note that this is the number
of all keys, not just unique vectors.
RETURNS (int): The number of keys in the table.
DOCS: https://spacy.io/api/vectors#n_keys
"""
return len(self.key2row)
def __reduce__(self):
@ -111,6 +133,8 @@ cdef class Vectors:
key (int): The key to get the vector for.
RETURNS (ndarray): The vector for the key.
DOCS: https://spacy.io/api/vectors#getitem
"""
i = self.key2row[key]
if i is None:
@ -123,6 +147,8 @@ cdef class Vectors:
key (int): The key to set the vector for.
vector (ndarray): The vector to set.
DOCS: https://spacy.io/api/vectors#setitem
"""
i = self.key2row[key]
self.data[i] = vector
@ -133,6 +159,8 @@ cdef class Vectors:
"""Iterate over the keys in the table.
YIELDS (int): A key in the table.
DOCS: https://spacy.io/api/vectors#iter
"""
yield from self.key2row
@ -140,6 +168,8 @@ cdef class Vectors:
"""Return the number of vectors in the table.
RETURNS (int): The number of vectors in the data.
DOCS: https://spacy.io/api/vectors#len
"""
return self.data.shape[0]
@ -148,6 +178,8 @@ cdef class Vectors:
key (int): The key to check.
RETURNS (bool): Whether the key has a vector entry.
DOCS: https://spacy.io/api/vectors#contains
"""
return key in self.key2row
@ -159,6 +191,12 @@ cdef class Vectors:
If the number of vectors is reduced, keys mapped to rows that have been
deleted are removed. These removed items are returned as a list of
`(key, row)` tuples.
shape (tuple): A `(rows, dims)` tuple.
inplace (bool): Reallocate the memory.
RETURNS (list): The removed items as a list of `(key, row)` tuples.
DOCS: https://spacy.io/api/vectors#resize
"""
if inplace:
self.data.resize(shape, refcheck=False)
@ -175,10 +213,7 @@ cdef class Vectors:
return removed_items
def keys(self):
"""A sequence of the keys in the table.
RETURNS (iterable): The keys.
"""
"""RETURNS (iterable): A sequence of keys in the table."""
return self.key2row.keys()
def values(self):
@ -188,6 +223,8 @@ cdef class Vectors:
returned may be less than the length of the vectors table.
YIELDS (ndarray): A vector in the table.
DOCS: https://spacy.io/api/vectors#values
"""
for row, vector in enumerate(range(self.data.shape[0])):
if not self._unset.count(row):
@ -197,6 +234,8 @@ cdef class Vectors:
"""Iterate over `(key, vector)` pairs.
YIELDS (tuple): A key/vector pair.
DOCS: https://spacy.io/api/vectors#items
"""
for key, row in self.key2row.items():
yield key, self.data[row]
@ -215,7 +254,7 @@ cdef class Vectors:
RETURNS: The requested key, keys, row or rows.
"""
if sum(arg is None for arg in (key, keys, row, rows)) != 3:
bad_kwargs = {'key': key, 'keys': keys, 'row': row, 'rows': rows}
bad_kwargs = {"key": key, "keys": keys, "row": row, "rows": rows}
raise ValueError(Errors.E059.format(kwargs=bad_kwargs))
xp = get_array_module(self.data)
if key is not None:
@ -224,7 +263,7 @@ cdef class Vectors:
elif keys is not None:
keys = [get_string_id(key) for key in keys]
rows = [self.key2row.get(key, -1.) for key in keys]
return xp.asarray(rows, dtype='i')
return xp.asarray(rows, dtype="i")
else:
targets = set()
if row is not None:
@ -236,7 +275,7 @@ cdef class Vectors:
if row in targets:
results.append(key)
targets.remove(row)
return xp.asarray(results, dtype='uint64')
return xp.asarray(results, dtype="uint64")
def add(self, key, *, vector=None, row=None):
"""Add a key to the table. Keys can be mapped to an existing vector
@ -246,6 +285,8 @@ cdef class Vectors:
vector (ndarray / None): A vector to add for the key.
row (int / None): The row number of a vector to map the key to.
RETURNS (int): The row the vector was added to.
DOCS: https://spacy.io/api/vectors#add
"""
key = get_string_id(key)
if row is None and key in self.key2row:
@ -292,11 +333,10 @@ cdef class Vectors:
sims = xp.dot(batch, vectors.T)
best_rows[i:i+batch_size] = sims.argmax(axis=1)
scores[i:i+batch_size] = sims.max(axis=1)
xp = get_array_module(self.data)
row2key = {row: key for key, row in self.key2row.items()}
keys = xp.asarray(
[row2key[row] for row in best_rows if row in row2key], dtype='uint64')
[row2key[row] for row in best_rows if row in row2key], dtype="uint64")
return (keys, best_rows, scores)
def from_glove(self, path):
@ -308,29 +348,30 @@ cdef class Vectors:
path (unicode / Path): The path to load the GloVe vectors from.
RETURNS: A `StringStore` object, holding the key-to-string mapping.
DOCS: https://spacy.io/api/vectors#from_glove
"""
path = util.ensure_path(path)
width = None
for name in path.iterdir():
if name.parts[-1].startswith('vectors'):
if name.parts[-1].startswith("vectors"):
_, dims, dtype, _2 = name.parts[-1].split('.')
width = int(dims)
break
else:
raise IOError(Errors.E061.format(filename=path))
bin_loc = path / 'vectors.{dims}.{dtype}.bin'.format(dims=dims,
dtype=dtype)
bin_loc = path / "vectors.{dims}.{dtype}.bin".format(dims=dims, dtype=dtype)
xp = get_array_module(self.data)
self.data = None
with bin_loc.open('rb') as file_:
with bin_loc.open("rb") as file_:
self.data = xp.fromfile(file_, dtype=dtype)
if dtype != 'float32':
self.data = xp.ascontiguousarray(self.data, dtype='float32')
if dtype != "float32":
self.data = xp.ascontiguousarray(self.data, dtype="float32")
if self.data.ndim == 1:
self.data = self.data.reshape((self.data.size//width, width))
n = 0
strings = StringStore()
with (path / 'vocab.txt').open('r') as file_:
with (path / "vocab.txt").open("r") as file_:
for i, line in enumerate(file_):
key = strings.add(line.strip())
self.add(key, row=i)
@ -341,16 +382,17 @@ cdef class Vectors:
path (unicode / Path): A path to a directory, which will be created if
it doesn't exists. Either a string or a Path-like object.
DOCS: https://spacy.io/api/vectors#to_disk
"""
xp = get_array_module(self.data)
if xp is numpy:
save_array = lambda arr, file_: xp.save(file_, arr,
allow_pickle=False)
save_array = lambda arr, file_: xp.save(file_, arr, allow_pickle=False)
else:
save_array = lambda arr, file_: xp.save(file_, arr)
serializers = OrderedDict((
('vectors', lambda p: save_array(self.data, p.open('wb'))),
('key2row', lambda p: srsly.write_msgpack(p, self.key2row))
("vectors", lambda p: save_array(self.data, p.open("wb"))),
("key2row", lambda p: srsly.write_msgpack(p, self.key2row))
))
return util.to_disk(path, serializers, exclude)
@ -360,6 +402,8 @@ cdef class Vectors:
path (unicode / Path): Directory path, string or Path-like object.
RETURNS (Vectors): The modified object.
DOCS: https://spacy.io/api/vectors#from_disk
"""
def load_key2row(path):
if path.exists():
@ -380,9 +424,9 @@ cdef class Vectors:
self.data = xp.load(str(path))
serializers = OrderedDict((
('key2row', load_key2row),
('keys', load_keys),
('vectors', load_vectors),
("key2row", load_key2row),
("keys", load_keys),
("vectors", load_vectors),
))
util.from_disk(path, serializers, exclude)
return self
@ -392,15 +436,17 @@ cdef class Vectors:
**exclude: Named attributes to prevent from being serialized.
RETURNS (bytes): The serialized form of the `Vectors` object.
DOCS: https://spacy.io/api/vectors#to_bytes
"""
def serialize_weights():
if hasattr(self.data, 'to_bytes'):
if hasattr(self.data, "to_bytes"):
return self.data.to_bytes()
else:
return srsly.msgpack_dumps(self.data)
serializers = OrderedDict((
('key2row', lambda: srsly.msgpack_dumps(self.key2row)),
('vectors', serialize_weights)
("key2row", lambda: srsly.msgpack_dumps(self.key2row)),
("vectors", serialize_weights)
))
return util.to_bytes(serializers, exclude)
@ -410,16 +456,18 @@ cdef class Vectors:
data (bytes): The data to load from.
**exclude: Named attributes to prevent from being loaded.
RETURNS (Vectors): The `Vectors` object.
DOCS: https://spacy.io/api/vectors#from_bytes
"""
def deserialize_weights(b):
if hasattr(self.data, 'from_bytes'):
if hasattr(self.data, "from_bytes"):
self.data.from_bytes()
else:
self.data = srsly.msgpack_loads(b)
deserializers = OrderedDict((
('key2row', lambda b: self.key2row.update(srsly.msgpack_loads(b))),
('vectors', deserialize_weights)
("key2row", lambda b: self.key2row.update(srsly.msgpack_loads(b))),
("vectors", deserialize_weights)
))
util.from_bytes(data, deserializers, exclude)
return self

93
spacy/vocab.pyx
View File

@ -4,9 +4,9 @@ from __future__ import unicode_literals
import numpy
import srsly
from collections import OrderedDict
from thinc.neural.util import get_array_module
from .lexeme cimport EMPTY_LEXEME
from .lexeme cimport Lexeme
from .typedefs cimport attr_t
@ -27,6 +27,8 @@ cdef class Vocab:
"""A look-up table that allows you to access `Lexeme` objects. The `Vocab`
instance also provides access to the `StringStore`, and owns underlying
C-data that is shared between `Doc` objects.
DOCS: https://spacy.io/api/vocab
"""
def __init__(self, lex_attr_getters=None, tag_map=None, lemmatizer=None,
strings=tuple(), oov_prob=-20., **deprecated_kwargs):
@ -62,7 +64,7 @@ cdef class Vocab:
langfunc = None
if self.lex_attr_getters:
langfunc = self.lex_attr_getters.get(LANG, None)
return langfunc('_') if langfunc else ''
return langfunc("_") if langfunc else ""
def __len__(self):
"""The current number of lexemes stored.
@ -87,11 +89,7 @@ cdef class Vocab:
available bit will be chosen.
RETURNS (int): The integer ID by which the flag value can be checked.
EXAMPLE:
>>> my_product_getter = lambda text: text in ['spaCy', 'dislaCy']
>>> MY_PRODUCT = nlp.vocab.add_flag(my_product_getter)
>>> doc = nlp(u'I like spaCy')
>>> assert doc[2].check_flag(MY_PRODUCT) == True
DOCS: https://spacy.io/api/vocab#add_flag
"""
if flag_id == -1:
for bit in range(1, 64):
@ -112,7 +110,7 @@ cdef class Vocab:
`Lexeme` if necessary using memory acquired from the given pool. If the
pool is the lexicon's own memory, the lexeme is saved in the lexicon.
"""
if string == u'':
if string == "":
return &EMPTY_LEXEME
cdef LexemeC* lex
cdef hash_t key = self.strings[string]
@ -176,10 +174,12 @@ cdef class Vocab:
string (unicode): The ID string.
RETURNS (bool) Whether the string has an entry in the vocabulary.
DOCS: https://spacy.io/api/vocab#contains
"""
cdef hash_t int_key
if isinstance(key, bytes):
int_key = self.strings[key.decode('utf8')]
int_key = self.strings[key.decode("utf8")]
elif isinstance(key, unicode):
int_key = self.strings[key]
else:
@ -191,6 +191,8 @@ cdef class Vocab:
"""Iterate over the lexemes in the vocabulary.
YIELDS (Lexeme): An entry in the vocabulary.
DOCS: https://spacy.io/api/vocab#iter
"""
cdef attr_t key
cdef size_t addr
@ -210,8 +212,10 @@ cdef class Vocab:
RETURNS (Lexeme): The lexeme indicated by the given ID.
EXAMPLE:
>>> apple = nlp.vocab.strings['apple']
>>> assert nlp.vocab[apple] == nlp.vocab[u'apple']
>>> apple = nlp.vocab.strings["apple"]
>>> assert nlp.vocab[apple] == nlp.vocab[u"apple"]
DOCS: https://spacy.io/api/vocab#getitem
"""
cdef attr_t orth
if isinstance(id_or_string, unicode):
@ -284,6 +288,8 @@ cdef class Vocab:
`(string, score)` tuples, where `string` is the entry the removed
word was mapped to, and `score` the similarity score between the
two words.
DOCS: https://spacy.io/api/vocab#prune_vectors
"""
xp = get_array_module(self.vectors.data)
# Make prob negative so it sorts by rank ascending
@ -291,16 +297,12 @@ cdef class Vocab:
priority = [(-lex.prob, self.vectors.key2row[lex.orth], lex.orth)
for lex in self if lex.orth in self.vectors.key2row]
priority.sort()
indices = xp.asarray([i for (prob, i, key) in priority], dtype='i')
keys = xp.asarray([key for (prob, i, key) in priority], dtype='uint64')
indices = xp.asarray([i for (prob, i, key) in priority], dtype="i")
keys = xp.asarray([key for (prob, i, key) in priority], dtype="uint64")
keep = xp.ascontiguousarray(self.vectors.data[indices[:nr_row]])
toss = xp.ascontiguousarray(self.vectors.data[indices[nr_row:]])
self.vectors = Vectors(data=keep, keys=keys)
syn_keys, syn_rows, scores = self.vectors.most_similar(toss, batch_size=batch_size)
remap = {}
for i, key in enumerate(keys[nr_row:]):
self.vectors.add(key, row=syn_rows[i])
@ -319,21 +321,22 @@ cdef class Vocab:
RETURNS (numpy.ndarray): A word vector. Size
and shape determined by the `vocab.vectors` instance. Usually, a
numpy ndarray of shape (300,) and dtype float32.
DOCS: https://spacy.io/api/vocab#get_vector
"""
if isinstance(orth, basestring_):
orth = self.strings.add(orth)
word = self[orth].orth_
if orth in self.vectors.key2row:
return self.vectors[orth]
# Assign default ngram limits to minn and maxn which is the length of the word.
if minn is None:
minn = len(word)
if maxn is None:
maxn = len(word)
vectors = numpy.zeros((self.vectors_length,), dtype='f')
# Fasttext's ngram computation taken from https://github.com/facebookresearch/fastText
vectors = numpy.zeros((self.vectors_length,), dtype="f")
# Fasttext's ngram computation taken from
# https://github.com/facebookresearch/fastText
ngrams_size = 0;
for i in range(len(word)):
ngram = ""
@ -356,12 +359,16 @@ cdef class Vocab:
n = n + 1
if ngrams_size > 0:
vectors = vectors * (1.0/ngrams_size)
return vectors
def set_vector(self, orth, vector):
"""Set a vector for a word in the vocabulary. Words can be referenced
by string or int ID.
orth (int / unicode): The word.
vector (numpy.ndarray[ndim=1, dtype='float32']): The vector to set.
DOCS: https://spacy.io/api/vocab#set_vector
"""
if isinstance(orth, basestring_):
orth = self.strings.add(orth)
@ -372,13 +379,19 @@ cdef class Vocab:
else:
width = self.vectors.shape[1]
self.vectors.resize((new_rows, width))
lex = self[orth] # Adds worse to vocab
lex = self[orth] # Adds words to vocab
self.vectors.add(orth, vector=vector)
self.vectors.add(orth, vector=vector)
def has_vector(self, orth):
"""Check whether a word has a vector. Returns False if no vectors have
been loaded. Words can be looked up by string or int ID."""
been loaded. Words can be looked up by string or int ID.
orth (int / unicode): The word.
RETURNS (bool): Whether the word has a vector.
DOCS: https://spacy.io/api/vocab#has_vector
"""
if isinstance(orth, basestring_):
orth = self.strings.add(orth)
return orth in self.vectors
@ -388,12 +401,14 @@ cdef class Vocab:
path (unicode or Path): A path to a directory, which will be created if
it doesn't exist. Paths may be either strings or Path-like objects.
DOCS: https://spacy.io/api/vocab#to_disk
"""
path = util.ensure_path(path)
if not path.exists():
path.mkdir()
self.strings.to_disk(path / 'strings.json')
with (path / 'lexemes.bin').open('wb') as file_:
self.strings.to_disk(path / "strings.json")
with (path / "lexemes.bin").open('wb') as file_:
file_.write(self.lexemes_to_bytes())
if self.vectors is not None:
self.vectors.to_disk(path)
@ -405,13 +420,15 @@ cdef class Vocab:
path (unicode or Path): A path to a directory. Paths may be either
strings or `Path`-like objects.
RETURNS (Vocab): The modified `Vocab` object.
DOCS: https://spacy.io/api/vocab#to_disk
"""
path = util.ensure_path(path)
self.strings.from_disk(path / 'strings.json')
with (path / 'lexemes.bin').open('rb') as file_:
self.strings.from_disk(path / "strings.json")
with (path / "lexemes.bin").open("rb") as file_:
self.lexemes_from_bytes(file_.read())
if self.vectors is not None:
self.vectors.from_disk(path, exclude='strings.json')
self.vectors.from_disk(path, exclude="strings.json")
if self.vectors.name is not None:
link_vectors_to_models(self)
return self
@ -421,6 +438,8 @@ cdef class Vocab:
**exclude: Named attributes to prevent from being serialized.
RETURNS (bytes): The serialized form of the `Vocab` object.
DOCS: https://spacy.io/api/vocab#to_bytes
"""
def deserialize_vectors():
if self.vectors is None:
@ -429,9 +448,9 @@ cdef class Vocab:
return self.vectors.to_bytes()
getters = OrderedDict((
('strings', lambda: self.strings.to_bytes()),
('lexemes', lambda: self.lexemes_to_bytes()),
('vectors', deserialize_vectors)
("strings", lambda: self.strings.to_bytes()),
("lexemes", lambda: self.lexemes_to_bytes()),
("vectors", deserialize_vectors)
))
return util.to_bytes(getters, exclude)
@ -441,6 +460,8 @@ cdef class Vocab:
bytes_data (bytes): The data to load from.
**exclude: Named attributes to prevent from being loaded.
RETURNS (Vocab): The `Vocab` object.
DOCS: https://spacy.io/api/vocab#from_bytes
"""
def serialize_vectors(b):
if self.vectors is None:
@ -448,9 +469,9 @@ cdef class Vocab:
else:
return self.vectors.from_bytes(b)
setters = OrderedDict((
('strings', lambda b: self.strings.from_bytes(b)),
('lexemes', lambda b: self.lexemes_from_bytes(b)),
('vectors', lambda b: serialize_vectors(b))
("strings", lambda b: self.strings.from_bytes(b)),
("lexemes", lambda b: self.lexemes_from_bytes(b)),
("vectors", lambda b: serialize_vectors(b))
))
util.from_bytes(bytes_data, setters, exclude)
if self.vectors.name is not None:
@ -467,7 +488,7 @@ cdef class Vocab:
if addr == 0:
continue
size += sizeof(lex_data.data)
byte_string = b'\0' * size
byte_string = b"\0" * size
byte_ptr = <unsigned char*>byte_string
cdef int j
cdef int i = 0

12
website/docs/api/dependencyparser.md
View File

@ -1,7 +1,7 @@
---
title: DependencyParser
tag: class
source: spacy/pipeline.pyx
source: spacy/pipeline/pipes.pyx
---
This class is a subclass of `Pipe` and follows the same API. The pipeline
@ -211,7 +211,7 @@ Modify the pipe's model, to use the given parameter values.
> ```python
> parser = DependencyParser(nlp.vocab)
> with parser.use_params():
> parser.to_disk('/best_model')
> parser.to_disk("/best_model")
> ```
| Name | Type | Description |
@ -226,7 +226,7 @@ Add a new label to the pipe.
>
> ```python
> parser = DependencyParser(nlp.vocab)
> parser.add_label('MY_LABEL')
> parser.add_label("MY_LABEL")
> ```
| Name | Type | Description |
@ -241,7 +241,7 @@ Serialize the pipe to disk.
>
> ```python
> parser = DependencyParser(nlp.vocab)
> parser.to_disk('/path/to/parser')
> parser.to_disk("/path/to/parser")
> ```
| Name | Type | Description |
@ -256,7 +256,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
>
> ```python
> parser = DependencyParser(nlp.vocab)
> parser.from_disk('/path/to/parser')
> parser.from_disk("/path/to/parser")
> ```
| Name | Type | Description |
@ -266,7 +266,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
## DependencyParser.to_bytes {#to_bytes tag="method"}
> #### example
> #### Example
>
> ```python
> parser = DependencyParser(nlp.vocab)

73
website/docs/api/doc.md
View File

@ -127,6 +127,7 @@ details, see the documentation on
| `method` | callable | Set a custom method on the object, for example `doc._.compare(other_doc)`. |
| `getter` | callable | Getter function that takes the object and returns an attribute value. Is called when the user accesses the `._` attribute. |
| `setter` | callable | Setter function that takes the `Doc` and a value, and modifies the object. Is called when the user writes to the `Doc._` attribute. |
| `force` | bool | Force overwriting existing attribute. |
## Doc.get_extension {#get_extension tag="classmethod" new="2"}
@ -263,6 +264,46 @@ ancestor is found, e.g. if span excludes a necessary ancestor.
| ----------- | -------------------------------------- | ----------------------------------------------- |
| **RETURNS** | `numpy.ndarray[ndim=2, dtype='int32']` | The lowest common ancestor matrix of the `Doc`. |
## Doc.to_json {#to_json, tag="method" new="2.1"}
Convert a Doc to JSON. The format it produces will be the new format for the
[`spacy train`](/api/cli#train) command (not implemented yet). If custom
underscore attributes are specified, their values need to be JSON-serializable.
They'll be added to an `"_"` key in the data, e.g. `"_": {"foo": "bar"}`.
> #### Example
>
> ```python
> doc = nlp(u"Hello")
> json_doc = doc.to_json()
> ```
>
> #### Result
>
> ```python
> {
> "text": "Hello",
> "ents": [],
> "sents": [{"start": 0, "end": 5}],
> "tokens": [{"id": 0, "start": 0, "end": 5, "pos": "INTJ", "tag": "UH", "dep": "ROOT", "head": 0}
> ]
> }
> ```
| Name | Type | Description |
| ------------ | ---- | ------------------------------------------------------------------------------ |
| `underscore` | list | Optional list of string names of custom JSON-serializable `doc._.` attributes. |
| **RETURNS** | dict | The JSON-formatted data. |
<Infobox title="Deprecation note" variant="warning">
spaCy previously implemented a `Doc.print_tree` method that returned a similar
JSON-formatted representation of a `Doc`. As of v2.1, this method is deprecated
in favor of `Doc.to_json`. If you need more complex nested representations, you
might want to write your own function to extract the data.
</Infobox>
## Doc.to_array {#to_array tag="method"}
Export given token attributes to a numpy `ndarray`. If `attr_ids` is a sequence
@ -310,7 +351,7 @@ array of attributes.
| Name | Type | Description |
| ----------- | -------------------------------------- | ----------------------------- |
| `attrs` | ints | A list of attribute ID ints. |
| `attrs` | list | A list of attribute ID ints. |
| `array` | `numpy.ndarray[ndim=2, dtype='int32']` | The attribute values to load. |
| **RETURNS** | `Doc` | Itself. |
@ -429,14 +470,16 @@ to specify how the new subtokens should be integrated into the dependency tree.
The list of per-token heads can either be a token in the original document, e.g.
`doc[2]`, or a tuple consisting of the token in the original document and its
subtoken index. For example, `(doc[3], 1)` will attach the subtoken to the
second subtoken of `doc[3]`. This mechanism allows attaching subtokens to other
newly created subtokens, without having to keep track of the changing token
indices. If the specified head token will be split within the retokenizer block
and no subtoken index is specified, it will default to `0`. Attributes to set on
subtokens can be provided as a list of values. They'll be applied to the
resulting token (if they're context-dependent token attributes like `LEMMA` or
`DEP`) or to the underlying lexeme (if they're context-independent lexical
attributes like `LOWER` or `IS_STOP`).
second subtoken of `doc[3]`.
This mechanism allows attaching subtokens to other newly created subtokens,
without having to keep track of the changing token indices. If the specified
head token will be split within the retokenizer block and no subtoken index is
specified, it will default to `0`. Attributes to set on subtokens can be
provided as a list of values. They'll be applied to the resulting token (if
they're context-dependent token attributes like `LEMMA` or `DEP`) or to the
underlying lexeme (if they're context-independent lexical attributes like
`LOWER` or `IS_STOP`).
> #### Example
>
@ -487,8 +530,8 @@ and end token boundaries, the document remains unchanged.
## Doc.ents {#ents tag="property" model="NER"}
Iterate over the entities in the document. Yields named-entity `Span` objects,
if the entity recognizer has been applied to the document.
The named entities in the document. Returns a tuple of named entity `Span`
objects, if the entity recognizer has been applied.
> #### Example
>
@ -501,8 +544,8 @@ if the entity recognizer has been applied to the document.
> ```
| Name | Type | Description |
| ---------- | ------ | ------------------------- |
| **YIELDS** | `Span` | Entities in the document. |
| ----------- | ----- | ------------------------------------------------ |
| **RETURNS** | tuple | Entities in the document, one `Span` per entity. |
## Doc.noun_chunks {#noun_chunks tag="property" model="parser"}
@ -542,8 +585,8 @@ will be unavailable.
> ```
| Name | Type | Description |
| ---------- | ---------------------------------- | ----------- |
| **YIELDS** | `Span | Sentences in the document. |
| ---------- | ------ | -------------------------- |
| **YIELDS** | `Span` | Sentences in the document. |
## Doc.has_vector {#has_vector tag="property" model="vectors"}

12
website/docs/api/entityrecognizer.md
View File

@ -1,7 +1,7 @@
---
title: EntityRecognizer
tag: class
source: spacy/pipeline.pyx
source: spacy/pipeline/pipes.pyx
---
This class is a subclass of `Pipe` and follows the same API. The pipeline
@ -211,7 +211,7 @@ Modify the pipe's model, to use the given parameter values.
> ```python
> ner = EntityRecognizer(nlp.vocab)
> with ner.use_params():
> ner.to_disk('/best_model')
> ner.to_disk("/best_model")
> ```
| Name | Type | Description |
@ -226,7 +226,7 @@ Add a new label to the pipe.
>
> ```python
> ner = EntityRecognizer(nlp.vocab)
> ner.add_label('MY_LABEL')
> ner.add_label("MY_LABEL")
> ```
| Name | Type | Description |
@ -241,7 +241,7 @@ Serialize the pipe to disk.
>
> ```python
> ner = EntityRecognizer(nlp.vocab)
> ner.to_disk('/path/to/ner')
> ner.to_disk("/path/to/ner")
> ```
| Name | Type | Description |
@ -256,7 +256,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
>
> ```python
> ner = EntityRecognizer(nlp.vocab)
> ner.from_disk('/path/to/ner')
> ner.from_disk("/path/to/ner")
> ```
| Name | Type | Description |
@ -266,7 +266,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
## EntityRecognizer.to_bytes {#to_bytes tag="method"}
> #### example
> #### Example
>
> ```python
> ner = EntityRecognizer(nlp.vocab)

6
website/docs/api/entityruler.md
View File

@ -1,7 +1,7 @@
---
title: EntityRuler
tag: class
source: spacy/pipeline.pyx
source: spacy/pipeline/entityruler.py
new: 2.1
---
@ -128,7 +128,7 @@ newline-delimited JSON (JSONL).
>
> ```python
> ruler = EntityRuler(nlp)
> ruler.to_disk('/path/to/rules.jsonl')
> ruler.to_disk("/path/to/rules.jsonl")
> ```
| Name | Type | Description |
@ -144,7 +144,7 @@ JSON (JSONL) with one entry per line.
>
> ```python
> ruler = EntityRuler(nlp)
> ruler.from_disk('/path/to/rules.jsonl')
> ruler.from_disk("/path/to/rules.jsonl")
> ```
| Name | Type | Description |

6
website/docs/api/pipeline-functions.md
View File

@ -1,7 +1,7 @@
---
title: Pipeline Functions
teaser: Other built-in pipeline components and helpers
source: spacy/pipeline.pyx
source: spacy/pipeline/functions.py
menu:
- ['merge_noun_chunks', 'merge_noun_chunks']
- ['merge_entities', 'merge_entities']
@ -73,10 +73,10 @@ components to the end of the pipeline and after all other components.
| `doc` | `Doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. |
| **RETURNS** | `Doc` | The modified `Doc` with merged entities. |
## merge_subtokens {#merge_entities tag="function" new="2.1"}
## merge_subtokens {#merge_subtokens tag="function" new="2.1"}
Merge subtokens into a single token. Also available via the string name
`"merge_entities"`. After initialization, the component is typically added to
`"merge_subtokens"`. After initialization, the component is typically added to
the processing pipeline using [`nlp.add_pipe`](/api/language#add_pipe).
As of v2.1, the parser is able to predict "subtokens" that should be merged into

2
website/docs/api/sentencesegmenter.md
View File

@ -1,7 +1,7 @@
---
title: SentenceSegmenter
tag: class
source: spacy/pipeline.pyx
source: spacy/pipeline/hooks.py
---
A simple spaCy hook, to allow custom sentence boundary detection logic that

13
website/docs/api/span.md
View File

@ -260,8 +260,8 @@ Retokenize the document, such that the span is merged into a single token.
## Span.ents {#ents tag="property" new="2.0.12" model="ner"}
Iterate over the entities in the span. Yields named-entity `Span` objects, if
the entity recognizer has been applied to the parent document.
The named entities in the span. Returns a tuple of named entity `Span` objects,
if the entity recognizer has been applied.
> #### Example
>
@ -275,8 +275,8 @@ the entity recognizer has been applied to the parent document.
> ```
| Name | Type | Description |
| ---------- | ------ | ------------------------- |
| **YIELDS** | `Span` | Entities in the document. |
| ----------- | ----- | -------------------------------------------- |
| **RETURNS** | tuple | Entities in the span, one `Span` per entity. |
## Span.as_doc {#as_doc tag="method"}
@ -297,8 +297,9 @@ Create a new `Doc` object corresponding to the `Span`, with a copy of the data.
## Span.root {#root tag="property" model="parser"}
The token within the span that's highest in the parse tree. If there's a tie,
the earliest is preferred.
The token with the shortest path to the root of the sentence (or the root
itself). If multiple tokens are equally high in the tree, the first token is
taken.
> #### Example
>

12
website/docs/api/tagger.md
View File

@ -1,7 +1,7 @@
---
title: Tagger
tag: class
source: spacy/pipeline.pyx
source: spacy/pipeline/pipes.pyx
---
This class is a subclass of `Pipe` and follows the same API. The pipeline
@ -209,7 +209,7 @@ Modify the pipe's model, to use the given parameter values.
> ```python
> tagger = Tagger(nlp.vocab)
> with tagger.use_params():
> tagger.to_disk('/best_model')
> tagger.to_disk("/best_model")
> ```
| Name | Type | Description |
@ -225,7 +225,7 @@ Add a new label to the pipe.
> ```python
> from spacy.symbols import POS
> tagger = Tagger(nlp.vocab)
> tagger.add_label('MY_LABEL', {POS: 'NOUN'})
> tagger.add_label("MY_LABEL", {POS: 'NOUN'})
> ```
| Name | Type | Description |
@ -241,7 +241,7 @@ Serialize the pipe to disk.
>
> ```python
> tagger = Tagger(nlp.vocab)
> tagger.to_disk('/path/to/tagger')
> tagger.to_disk("/path/to/tagger")
> ```
| Name | Type | Description |
@ -256,7 +256,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
>
> ```python
> tagger = Tagger(nlp.vocab)
> tagger.from_disk('/path/to/tagger')
> tagger.from_disk("/path/to/tagger")
> ```
| Name | Type | Description |
@ -266,7 +266,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
## Tagger.to_bytes {#to_bytes tag="method"}
> #### example
> #### Example
>
> ```python
> tagger = Tagger(nlp.vocab)

12
website/docs/api/textcategorizer.md
View File

@ -1,7 +1,7 @@
---
title: TextCategorizer
tag: class
source: spacy/pipeline.pyx
source: spacy/pipeline/pipes.pyx
new: 2
---
@ -227,7 +227,7 @@ Modify the pipe's model, to use the given parameter values.
> ```python
> textcat = TextCategorizer(nlp.vocab)
> with textcat.use_params():
> textcat.to_disk('/best_model')
> textcat.to_disk("/best_model")
> ```
| Name | Type | Description |
@ -242,7 +242,7 @@ Add a new label to the pipe.
>
> ```python
> textcat = TextCategorizer(nlp.vocab)
> textcat.add_label('MY_LABEL')
> textcat.add_label("MY_LABEL")
> ```
| Name | Type | Description |
@ -257,7 +257,7 @@ Serialize the pipe to disk.
>
> ```python
> textcat = TextCategorizer(nlp.vocab)
> textcat.to_disk('/path/to/textcat')
> textcat.to_disk("/path/to/textcat")
> ```
| Name | Type | Description |
@ -272,7 +272,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
>
> ```python
> textcat = TextCategorizer(nlp.vocab)
> textcat.from_disk('/path/to/textcat')
> textcat.from_disk("/path/to/textcat")
> ```
| Name | Type | Description |
@ -282,7 +282,7 @@ Load the pipe from disk. Modifies the object in place and returns it.
## TextCategorizer.to_bytes {#to_bytes tag="method"}
> #### example
> #### Example
>
> ```python
> textcat = TextCategorizer(nlp.vocab)

2
website/docs/api/token.md
View File

@ -324,7 +324,7 @@ A sequence containing the token and all the token's syntactic descendants.
## Token.is_sent_start {#is_sent_start tag="property" new="2"}
A boolean value indicating whether the token starts a sentence. `None` if
unknown. Defaults to `True` for the first token in the `doc`.
unknown. Defaults to `True` for the first token in the `Doc`.
> #### Example
>

66
website/docs/api/tokenizer.md
View File

@ -116,6 +116,72 @@ details and examples.
| `string` | unicode | The string to specially tokenize. |
| `token_attrs` | iterable | A sequence of dicts, where each dict describes a token and its attributes. The `ORTH` fields of the attributes must exactly match the string when they are concatenated. |
## Tokenizer.to_disk {#to_disk tag="method"}
Serialize the tokenizer to disk.
> #### Example
>
> ```python
> tokenizer = Tokenizer(nlp.vocab)
> tokenizer.to_disk("/path/to/tokenizer")
> ```
| Name | Type | Description |
| ------ | ---------------- | --------------------------------------------------------------------------------------------------------------------- |
| `path` | unicode / `Path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. |
## Tokenizer.from_disk {#from_disk tag="method"}
Load the tokenizer from disk. Modifies the object in place and returns it.
> #### Example
>
> ```python
> tokenizer = Tokenizer(nlp.vocab)
> tokenizer.from_disk("/path/to/tokenizer")
> ```
| Name | Type | Description |
| ----------- | ---------------- | -------------------------------------------------------------------------- |
| `path` | unicode / `Path` | A path to a directory. Paths may be either strings or `Path`-like objects. |
| **RETURNS** | `Tokenizer` | The modified `Tokenizer` object. |
## Tokenizer.to_bytes {#to_bytes tag="method"}
> #### Example
>
> ```python
> tokenizer = tokenizer(nlp.vocab)
> tokenizer_bytes = tokenizer.to_bytes()
> ```
Serialize the tokenizer to a bytestring.
| Name | Type | Description |
| ----------- | ----- | -------------------------------------------------- |
| `**exclude` | - | Named attributes to prevent from being serialized. |
| **RETURNS** | bytes | The serialized form of the `Tokenizer` object. |
## Tokenizer.from_bytes {#from_bytes tag="method"}
Load the tokenizer from a bytestring. Modifies the object in place and returns
it.
> #### Example
>
> ```python
> tokenizer_bytes = tokenizer.to_bytes()
> tokenizer = Tokenizer(nlp.vocab)
> tokenizer.from_bytes(tokenizer_bytes)
> ```
| Name | Type | Description |
| ------------ | ----------- | ---------------------------------------------- |
| `bytes_data` | bytes | The data to load from. |
| `**exclude` | - | Named attributes to prevent from being loaded. |
| **RETURNS** | `Tokenizer` | The `Tokenizer` object. |
## Attributes {#attributes}
| Name | Type | Description |

4
website/docs/api/top-level.md
View File

@ -642,7 +642,7 @@ All Python code is written in an **intersection of Python 2 and Python 3**. This
is easy in Cython, but somewhat ugly in Python. Logic that deals with Python or
platform compatibility only lives in `spacy.compat`. To distinguish them from
the builtin functions, replacement functions are suffixed with an underscore,
e.e `unicode_`.
e.g. `unicode_`.
> #### Example
>
@ -660,7 +660,7 @@ e.e `unicode_`.
| `compat.input_` | `raw_input` | `input` |
| `compat.path2str` | `str(path)` with `.decode('utf8')` | `str(path)` |
### compat.is_config {#is_config tag="function"}
### compat.is_config {#compat.is_config tag="function"}
Check if a specific configuration of Python version and operating system matches
the user's setup. Mostly used to display targeted error messages.

2
website/docs/usage/rule-based-matching.md
View File

@ -424,7 +424,7 @@ take a path to a JSON file containing the patterns. This lets you reuse the
component with different patterns, depending on your application:
```python
html_merger = BadHTMLMerger(nlp, path='/path/to/patterns.json')
html_merger = BadHTMLMerger(nlp, path="/path/to/patterns.json")
```
<Infobox title="📖 Processing pipelines">