2019-03-08 10:42:26 +00:00
|
|
|
# coding: utf8
|
2018-08-22 11:12:51 +00:00
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
|
|
import numpy
|
|
|
|
import gzip
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
import srsly
|
2018-08-22 11:12:51 +00:00
|
|
|
from thinc.neural.ops import NumpyOps
|
|
|
|
|
2018-09-28 13:23:14 +00:00
|
|
|
from ..compat import copy_reg
|
2018-09-28 12:27:24 +00:00
|
|
|
from ..tokens import Doc
|
2018-08-22 11:12:51 +00:00
|
|
|
from ..attrs import SPACY, ORTH
|
|
|
|
|
|
|
|
|
2019-07-10 17:37:20 +00:00
|
|
|
class DocBox(object):
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
"""Serialize analyses from a collection of doc objects."""
|
|
|
|
|
2019-07-10 17:37:20 +00:00
|
|
|
def __init__(self, attrs=None, store_user_data=False):
|
|
|
|
"""Create a DocBox object, to hold serialized annotations.
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
2019-03-08 10:42:26 +00:00
|
|
|
attrs (list): List of attributes to serialize. 'orth' and 'spacy' are
|
|
|
|
always serialized, so they're not required. Defaults to None.
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
"""
|
2018-08-22 11:12:51 +00:00
|
|
|
attrs = attrs or []
|
|
|
|
# Ensure ORTH is always attrs[0]
|
2019-07-10 17:37:20 +00:00
|
|
|
self.attrs = [attr for attr in attrs if attr != ORTH and attr != SPACY]
|
2018-08-22 11:12:51 +00:00
|
|
|
self.attrs.insert(0, ORTH)
|
|
|
|
self.tokens = []
|
|
|
|
self.spaces = []
|
2019-07-10 17:37:20 +00:00
|
|
|
self.user_data = []
|
2018-08-22 11:12:51 +00:00
|
|
|
self.strings = set()
|
2019-07-10 17:37:20 +00:00
|
|
|
self.store_user_data = store_user_data
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
def add(self, doc):
|
2019-07-10 17:37:20 +00:00
|
|
|
"""Add a doc's annotations to the DocBox for serialization."""
|
2018-08-22 11:12:51 +00:00
|
|
|
array = doc.to_array(self.attrs)
|
|
|
|
if len(array.shape) == 1:
|
|
|
|
array = array.reshape((array.shape[0], 1))
|
|
|
|
self.tokens.append(array)
|
|
|
|
spaces = doc.to_array(SPACY)
|
|
|
|
assert array.shape[0] == spaces.shape[0]
|
|
|
|
spaces = spaces.reshape((spaces.shape[0], 1))
|
|
|
|
self.spaces.append(numpy.asarray(spaces, dtype=bool))
|
|
|
|
self.strings.update(w.text for w in doc)
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data:
|
|
|
|
self.user_data.append(srsly.msgpack_dumps(doc.user_data))
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
def get_docs(self, vocab):
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
"""Recover Doc objects from the annotations, using the given vocab."""
|
2018-08-22 11:12:51 +00:00
|
|
|
for string in self.strings:
|
|
|
|
vocab[string]
|
|
|
|
orth_col = self.attrs.index(ORTH)
|
2019-07-10 17:37:20 +00:00
|
|
|
for i in range(len(self.tokens)):
|
|
|
|
tokens = self.tokens[i]
|
|
|
|
spaces = self.spaces[i]
|
2018-08-22 11:12:51 +00:00
|
|
|
words = [vocab.strings[orth] for orth in tokens[:, orth_col]]
|
|
|
|
doc = Doc(vocab, words=words, spaces=spaces)
|
|
|
|
doc = doc.from_array(self.attrs, tokens)
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data:
|
|
|
|
doc.user_data.update(srsly.msgpack_loads(self.user_data[i]))
|
2018-08-22 11:12:51 +00:00
|
|
|
yield doc
|
|
|
|
|
|
|
|
def merge(self, other):
|
2019-07-10 17:37:20 +00:00
|
|
|
"""Extend the annotations of this DocBox with the annotations from another."""
|
2018-08-22 11:12:51 +00:00
|
|
|
assert self.attrs == other.attrs
|
|
|
|
self.tokens.extend(other.tokens)
|
|
|
|
self.spaces.extend(other.spaces)
|
|
|
|
self.strings.update(other.strings)
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data:
|
|
|
|
self.user_data.extend(other.user_data)
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
def to_bytes(self):
|
2019-07-10 17:37:20 +00:00
|
|
|
"""Serialize the DocBox's annotations into a byte string."""
|
2018-08-22 11:12:51 +00:00
|
|
|
for tokens in self.tokens:
|
|
|
|
assert len(tokens.shape) == 2, tokens.shape
|
|
|
|
lengths = [len(tokens) for tokens in self.tokens]
|
|
|
|
msg = {
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
"attrs": self.attrs,
|
|
|
|
"tokens": numpy.vstack(self.tokens).tobytes("C"),
|
|
|
|
"spaces": numpy.vstack(self.spaces).tobytes("C"),
|
|
|
|
"lengths": numpy.asarray(lengths, dtype="int32").tobytes("C"),
|
|
|
|
"strings": list(self.strings),
|
2018-08-22 11:12:51 +00:00
|
|
|
}
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data:
|
|
|
|
msg["user_data"] = self.user_data
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
return gzip.compress(srsly.msgpack_dumps(msg))
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
def from_bytes(self, string):
|
2019-07-10 17:37:20 +00:00
|
|
|
"""Deserialize the DocBox's annotations from a byte string."""
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
msg = srsly.msgpack_loads(gzip.decompress(string))
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
self.attrs = msg["attrs"]
|
|
|
|
self.strings = set(msg["strings"])
|
|
|
|
lengths = numpy.fromstring(msg["lengths"], dtype="int32")
|
|
|
|
flat_spaces = numpy.fromstring(msg["spaces"], dtype=bool)
|
|
|
|
flat_tokens = numpy.fromstring(msg["tokens"], dtype="uint64")
|
2018-08-22 11:12:51 +00:00
|
|
|
shape = (flat_tokens.size // len(self.attrs), len(self.attrs))
|
|
|
|
flat_tokens = flat_tokens.reshape(shape)
|
|
|
|
flat_spaces = flat_spaces.reshape((flat_spaces.size, 1))
|
|
|
|
self.tokens = NumpyOps().unflatten(flat_tokens, lengths)
|
|
|
|
self.spaces = NumpyOps().unflatten(flat_spaces, lengths)
|
2019-07-10 17:37:20 +00:00
|
|
|
if self.store_user_data and "user_data" in msg:
|
|
|
|
self.user_data = list(msg["user_data"])
|
2018-08-22 11:12:51 +00:00
|
|
|
for tokens in self.tokens:
|
|
|
|
assert len(tokens.shape) == 2, tokens.shape
|
|
|
|
return self
|
|
|
|
|
|
|
|
|
2019-07-10 17:37:20 +00:00
|
|
|
def merge_boxes(boxes):
|
|
|
|
merged = None
|
|
|
|
for byte_string in boxes:
|
|
|
|
if byte_string is not None:
|
|
|
|
box = DocBox(store_user_data=True).from_bytes(byte_string)
|
|
|
|
if merged is None:
|
|
|
|
merged = box
|
|
|
|
else:
|
|
|
|
merged.merge(box)
|
|
|
|
if merged is not None:
|
|
|
|
return merged.to_bytes()
|
|
|
|
else:
|
|
|
|
return b''
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
|
2019-07-10 17:37:20 +00:00
|
|
|
def pickle_box(box):
|
|
|
|
return (unpickle_box, (box.to_bytes(),))
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
|
2019-07-10 17:37:20 +00:00
|
|
|
def unpickle_box(byte_string):
|
|
|
|
return Box().from_bytes(byte_string)
|
2018-08-22 11:12:51 +00:00
|
|
|
|
|
|
|
|
2019-07-10 17:37:20 +00:00
|
|
|
copy_reg.pickle(Box, pickle_box, unpickle_box)
|
|
|
|
# Compatibility, as we had named it this previously.
|
|
|
|
Binder = DocBox
|
|
|
|
|
|
|
|
__all__ = ["DocBox"]
|