2019-03-08 10:42:26 +00:00
|
|
|
# coding: utf8
|
2018-08-22 11:12:51 +00:00
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
|
|
import numpy
|
2019-10-03 12:48:45 +00:00
|
|
|
import zlib
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
import srsly
|
2018-08-22 11:12:51 +00:00
|
|
|
from thinc.neural.ops import NumpyOps
|
|
|
|
|
2018-09-28 13:23:14 +00:00
|
|
|
from ..compat import copy_reg
|
2018-09-28 12:27:24 +00:00
|
|
|
from ..tokens import Doc
|
2019-10-03 12:48:45 +00:00
|
|
|
from ..attrs import SPACY, ORTH, intify_attr
|
2019-09-18 18:23:21 +00:00
|
|
|
from ..errors import Errors
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
|
2019-09-18 13:15:37 +00:00
|
|
|
class DocBin(object):
|
|
|
|
"""Pack Doc objects for binary serialization.
|
2019-09-18 17:18:30 +00:00
|
|
|
|
2019-09-18 13:15:37 +00:00
|
|
|
The DocBin class lets you efficiently serialize the information from a
|
2019-09-18 11:25:47 +00:00
|
|
|
collection of Doc objects. You can control which information is serialized
|
|
|
|
by passing a list of attribute IDs, and optionally also specify whether the
|
2019-09-18 13:15:37 +00:00
|
|
|
user data is serialized. The DocBin is faster and produces smaller data
|
2019-09-18 11:25:47 +00:00
|
|
|
sizes than pickle, and allows you to deserialize without executing arbitrary
|
|
|
|
Python code.
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
2019-09-18 11:25:47 +00:00
|
|
|
The serialization format is gzipped msgpack, where the msgpack object has
|
|
|
|
the following structure:
|
2019-09-18 17:18:30 +00:00
|
|
|
|
2019-09-18 11:25:47 +00:00
|
|
|
{
|
|
|
|
"attrs": List[uint64], # e.g. [TAG, HEAD, ENT_IOB, ENT_TYPE]
|
|
|
|
"tokens": bytes, # Serialized numpy uint64 array with the token data
|
|
|
|
"spaces": bytes, # Serialized numpy boolean array with spaces data
|
|
|
|
"lengths": bytes, # Serialized numpy int32 array with the doc lengths
|
|
|
|
"strings": List[unicode] # List of unique strings in the token data
|
|
|
|
}
|
|
|
|
|
|
|
|
Strings for the words, tags, labels etc are represented by 64-bit hashes in
|
|
|
|
the token data, and every string that occurs at least once is passed via the
|
|
|
|
strings object. This means the storage is more efficient if you pack more
|
|
|
|
documents together, because you have less duplication in the strings.
|
|
|
|
|
|
|
|
A notable downside to this format is that you can't easily extract just one
|
2019-09-18 18:23:21 +00:00
|
|
|
document from the DocBin.
|
2019-09-18 11:25:47 +00:00
|
|
|
"""
|
2019-09-18 17:18:30 +00:00
|
|
|
|
2019-07-10 17:37:20 +00:00
|
|
|
def __init__(self, attrs=None, store_user_data=False):
|
2019-09-18 18:23:21 +00:00
|
|
|
"""Create a DocBin object to hold serialized annotations.
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
2019-03-08 10:42:26 +00:00
|
|
|
attrs (list): List of attributes to serialize. 'orth' and 'spacy' are
|
|
|
|
always serialized, so they're not required. Defaults to None.
|
2019-09-18 18:23:21 +00:00
|
|
|
store_user_data (bool): Whether to include the `Doc.user_data`.
|
|
|
|
RETURNS (DocBin): The newly constructed object.
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/docbin#init
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
"""
|
2018-08-22 11:12:51 +00:00
|
|
|
attrs = attrs or []
|
2019-10-03 12:48:45 +00:00
|
|
|
attrs = sorted([intify_attr(attr) for attr in attrs])
|
2019-07-10 17:37:20 +00:00
|
|
|
self.attrs = [attr for attr in attrs if attr != ORTH and attr != SPACY]
|
2019-09-18 18:23:21 +00:00
|
|
|
self.attrs.insert(0, ORTH) # Ensure ORTH is always attrs[0]
|
2018-08-22 11:12:51 +00:00
|
|
|
self.tokens = []
|
|
|
|
self.spaces = []
|
2019-07-10 17:37:20 +00:00
|
|
|
self.user_data = []
|
2018-08-22 11:12:51 +00:00
|
|
|
self.strings = set()
|
2019-07-10 17:37:20 +00:00
|
|
|
self.store_user_data = store_user_data
|
2018-08-22 11:12:51 +00:00
|
|
|
|
2019-09-18 18:23:21 +00:00
|
|
|
def __len__(self):
|
|
|
|
"""RETURNS: The number of Doc objects added to the DocBin."""
|
|
|
|
return len(self.tokens)
|
|
|
|
|
2018-08-22 11:12:51 +00:00
|
|
|
def add(self, doc):
|
2019-09-18 18:23:21 +00:00
|
|
|
"""Add a Doc's annotations to the DocBin for serialization.
|
|
|
|
|
|
|
|
doc (Doc): The Doc object to add.
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/docbin#add
|
|
|
|
"""
|
2018-08-22 11:12:51 +00:00
|
|
|
array = doc.to_array(self.attrs)
|
|
|
|
if len(array.shape) == 1:
|
|
|
|
array = array.reshape((array.shape[0], 1))
|
|
|
|
self.tokens.append(array)
|
|
|
|
spaces = doc.to_array(SPACY)
|
2019-09-18 18:23:21 +00:00
|
|
|
assert array.shape[0] == spaces.shape[0] # this should never happen
|
2018-08-22 11:12:51 +00:00
|
|
|
spaces = spaces.reshape((spaces.shape[0], 1))
|
|
|
|
self.spaces.append(numpy.asarray(spaces, dtype=bool))
|
|
|
|
self.strings.update(w.text for w in doc)
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data:
|
|
|
|
self.user_data.append(srsly.msgpack_dumps(doc.user_data))
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
def get_docs(self, vocab):
|
2019-09-18 18:23:21 +00:00
|
|
|
"""Recover Doc objects from the annotations, using the given vocab.
|
|
|
|
|
|
|
|
vocab (Vocab): The shared vocab.
|
|
|
|
YIELDS (Doc): The Doc objects.
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/docbin#get_docs
|
|
|
|
"""
|
2018-08-22 11:12:51 +00:00
|
|
|
for string in self.strings:
|
|
|
|
vocab[string]
|
|
|
|
orth_col = self.attrs.index(ORTH)
|
2019-07-10 17:37:20 +00:00
|
|
|
for i in range(len(self.tokens)):
|
|
|
|
tokens = self.tokens[i]
|
|
|
|
spaces = self.spaces[i]
|
2018-08-22 11:12:51 +00:00
|
|
|
words = [vocab.strings[orth] for orth in tokens[:, orth_col]]
|
|
|
|
doc = Doc(vocab, words=words, spaces=spaces)
|
|
|
|
doc = doc.from_array(self.attrs, tokens)
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data:
|
|
|
|
doc.user_data.update(srsly.msgpack_loads(self.user_data[i]))
|
2018-08-22 11:12:51 +00:00
|
|
|
yield doc
|
|
|
|
|
|
|
|
def merge(self, other):
|
2019-09-18 18:23:21 +00:00
|
|
|
"""Extend the annotations of this DocBin with the annotations from
|
|
|
|
another. Will raise an error if the pre-defined attrs of the two
|
|
|
|
DocBins don't match.
|
|
|
|
|
|
|
|
other (DocBin): The DocBin to merge into the current bin.
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/docbin#merge
|
|
|
|
"""
|
|
|
|
if self.attrs != other.attrs:
|
|
|
|
raise ValueError(Errors.E166.format(current=self.attrs, other=other.attrs))
|
2018-08-22 11:12:51 +00:00
|
|
|
self.tokens.extend(other.tokens)
|
|
|
|
self.spaces.extend(other.spaces)
|
|
|
|
self.strings.update(other.strings)
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data:
|
|
|
|
self.user_data.extend(other.user_data)
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
def to_bytes(self):
|
2019-09-18 18:23:21 +00:00
|
|
|
"""Serialize the DocBin's annotations to a bytestring.
|
|
|
|
|
|
|
|
RETURNS (bytes): The serialized DocBin.
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/docbin#to_bytes
|
|
|
|
"""
|
2018-08-22 11:12:51 +00:00
|
|
|
for tokens in self.tokens:
|
2019-09-18 18:23:21 +00:00
|
|
|
assert len(tokens.shape) == 2, tokens.shape # this should never happen
|
2018-08-22 11:12:51 +00:00
|
|
|
lengths = [len(tokens) for tokens in self.tokens]
|
|
|
|
msg = {
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
"attrs": self.attrs,
|
|
|
|
"tokens": numpy.vstack(self.tokens).tobytes("C"),
|
|
|
|
"spaces": numpy.vstack(self.spaces).tobytes("C"),
|
|
|
|
"lengths": numpy.asarray(lengths, dtype="int32").tobytes("C"),
|
|
|
|
"strings": list(self.strings),
|
2018-08-22 11:12:51 +00:00
|
|
|
}
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data:
|
|
|
|
msg["user_data"] = self.user_data
|
2019-10-03 12:48:45 +00:00
|
|
|
return zlib.compress(srsly.msgpack_dumps(msg))
|
2018-08-22 11:12:51 +00:00
|
|
|
|
2019-09-18 18:23:21 +00:00
|
|
|
def from_bytes(self, bytes_data):
|
|
|
|
"""Deserialize the DocBin's annotations from a bytestring.
|
|
|
|
|
|
|
|
bytes_data (bytes): The data to load from.
|
|
|
|
RETURNS (DocBin): The loaded DocBin.
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/docbin#from_bytes
|
|
|
|
"""
|
2019-10-03 12:48:45 +00:00
|
|
|
msg = srsly.msgpack_loads(zlib.decompress(bytes_data))
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
self.attrs = msg["attrs"]
|
|
|
|
self.strings = set(msg["strings"])
|
|
|
|
lengths = numpy.fromstring(msg["lengths"], dtype="int32")
|
|
|
|
flat_spaces = numpy.fromstring(msg["spaces"], dtype=bool)
|
|
|
|
flat_tokens = numpy.fromstring(msg["tokens"], dtype="uint64")
|
2018-08-22 11:12:51 +00:00
|
|
|
shape = (flat_tokens.size // len(self.attrs), len(self.attrs))
|
|
|
|
flat_tokens = flat_tokens.reshape(shape)
|
|
|
|
flat_spaces = flat_spaces.reshape((flat_spaces.size, 1))
|
|
|
|
self.tokens = NumpyOps().unflatten(flat_tokens, lengths)
|
|
|
|
self.spaces = NumpyOps().unflatten(flat_spaces, lengths)
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data and "user_data" in msg:
|
|
|
|
self.user_data = list(msg["user_data"])
|
2018-08-22 11:12:51 +00:00
|
|
|
for tokens in self.tokens:
|
2019-09-18 18:23:21 +00:00
|
|
|
assert len(tokens.shape) == 2, tokens.shape # this should never happen
|
2018-08-22 11:12:51 +00:00
|
|
|
return self
|
|
|
|
|
|
|
|
|
2019-09-18 13:15:37 +00:00
|
|
|
def merge_bins(bins):
|
2019-07-10 17:37:20 +00:00
|
|
|
merged = None
|
2019-09-18 13:15:37 +00:00
|
|
|
for byte_string in bins:
|
2019-07-10 17:37:20 +00:00
|
|
|
if byte_string is not None:
|
2019-09-18 13:15:37 +00:00
|
|
|
doc_bin = DocBin(store_user_data=True).from_bytes(byte_string)
|
2019-07-10 17:37:20 +00:00
|
|
|
if merged is None:
|
2019-09-18 13:15:37 +00:00
|
|
|
merged = doc_bin
|
2019-07-10 17:37:20 +00:00
|
|
|
else:
|
2019-09-18 13:15:37 +00:00
|
|
|
merged.merge(doc_bin)
|
2019-07-10 17:37:20 +00:00
|
|
|
if merged is not None:
|
|
|
|
return merged.to_bytes()
|
|
|
|
else:
|
2019-07-11 09:49:36 +00:00
|
|
|
return b""
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
|
2019-09-18 17:18:30 +00:00
|
|
|
def pickle_bin(doc_bin):
|
|
|
|
return (unpickle_bin, (doc_bin.to_bytes(),))
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
|
2019-09-18 13:15:37 +00:00
|
|
|
def unpickle_bin(byte_string):
|
|
|
|
return DocBin().from_bytes(byte_string)
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
|
2019-09-18 13:15:37 +00:00
|
|
|
copy_reg.pickle(DocBin, pickle_bin, unpickle_bin)
|
2019-07-10 17:37:20 +00:00
|
|
|
# Compatibility, as we had named it this previously.
|
2019-09-18 13:15:37 +00:00
|
|
|
Binder = DocBin
|
2019-07-10 17:37:20 +00:00
|
|
|
|
2019-09-18 13:15:37 +00:00
|
|
|
__all__ = ["DocBin"]
|