2017-04-15 10:05:47 +00:00
|
|
|
# coding: utf8
|
|
|
|
from __future__ import absolute_import, unicode_literals
|
2016-10-09 10:24:24 +00:00
|
|
|
from contextlib import contextmanager
|
2017-05-17 10:04:50 +00:00
|
|
|
import dill
|
2015-08-27 07:16:11 +00:00
|
|
|
|
2017-05-18 09:25:19 +00:00
|
|
|
import numpy
|
|
|
|
from thinc.neural import Model
|
|
|
|
from thinc.neural.ops import NumpyOps, CupyOps
|
|
|
|
|
2015-08-26 17:16:09 +00:00
|
|
|
from .tokenizer import Tokenizer
|
|
|
|
from .vocab import Vocab
|
|
|
|
from .tagger import Tagger
|
2016-09-25 13:37:33 +00:00
|
|
|
from .lemmatizer import Lemmatizer
|
2016-10-09 10:24:24 +00:00
|
|
|
from .train import Trainer
|
2016-09-26 09:57:54 +00:00
|
|
|
from .syntax.parser import get_templates
|
2016-10-09 10:24:24 +00:00
|
|
|
from .syntax.nonproj import PseudoProjectivity
|
2017-05-16 14:17:30 +00:00
|
|
|
from .pipeline import NeuralDependencyParser, EntityRecognizer
|
|
|
|
from .pipeline import TokenVectorEncoder, NeuralTagger, NeuralEntityRecognizer
|
2017-04-16 23:46:14 +00:00
|
|
|
from .compat import json_dumps
|
2017-04-15 10:05:47 +00:00
|
|
|
from .attrs import IS_STOP
|
2017-05-08 21:58:31 +00:00
|
|
|
from .lang.punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
|
|
|
|
from .lang.tokenizer_exceptions import TOKEN_MATCH
|
|
|
|
from .lang.tag_map import TAG_MAP
|
2017-05-08 22:58:10 +00:00
|
|
|
from .lang.lex_attrs import LEX_ATTRS
|
2017-04-15 10:05:47 +00:00
|
|
|
from . import util
|
2016-10-09 10:24:24 +00:00
|
|
|
|
2015-08-27 07:16:11 +00:00
|
|
|
|
2016-09-24 18:26:17 +00:00
|
|
|
class BaseDefaults(object):
|
2016-10-18 14:18:25 +00:00
|
|
|
@classmethod
|
|
|
|
def create_lemmatizer(cls, nlp=None):
|
2017-03-15 09:52:50 +00:00
|
|
|
return Lemmatizer(cls.lemma_index, cls.lemma_exc, cls.lemma_rules)
|
2016-10-18 14:18:25 +00:00
|
|
|
|
|
|
|
@classmethod
|
|
|
|
def create_vocab(cls, nlp=None):
|
|
|
|
lemmatizer = cls.create_lemmatizer(nlp)
|
2017-05-16 09:21:59 +00:00
|
|
|
lex_attr_getters = dict(cls.lex_attr_getters)
|
|
|
|
# This is messy, but it's the minimal working fix to Issue #639.
|
|
|
|
lex_attr_getters[IS_STOP] = lambda string: string.lower() in cls.stop_words
|
|
|
|
vocab = Vocab(lex_attr_getters=lex_attr_getters, tag_map=cls.tag_map,
|
|
|
|
lemmatizer=lemmatizer)
|
2017-03-15 14:24:40 +00:00
|
|
|
for tag_str, exc in cls.morph_rules.items():
|
|
|
|
for orth_str, attrs in exc.items():
|
|
|
|
vocab.morphology.add_special_case(tag_str, orth_str, attrs)
|
|
|
|
return vocab
|
2016-12-18 15:54:52 +00:00
|
|
|
|
2016-10-18 14:18:25 +00:00
|
|
|
@classmethod
|
|
|
|
def create_tokenizer(cls, nlp=None):
|
|
|
|
rules = cls.tokenizer_exceptions
|
2017-05-16 09:21:59 +00:00
|
|
|
token_match = cls.token_match
|
|
|
|
prefix_search = util.compile_prefix_regex(cls.prefixes).search \
|
|
|
|
if cls.prefixes else None
|
|
|
|
suffix_search = util.compile_suffix_regex(cls.suffixes).search \
|
|
|
|
if cls.suffixes else None
|
|
|
|
infix_finditer = util.compile_infix_regex(cls.infixes).finditer \
|
|
|
|
if cls.infixes else None
|
2016-10-18 14:18:25 +00:00
|
|
|
vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)
|
2016-11-26 11:36:04 +00:00
|
|
|
return Tokenizer(vocab, rules=rules,
|
2016-10-18 14:18:25 +00:00
|
|
|
prefix_search=prefix_search, suffix_search=suffix_search,
|
2017-01-03 17:17:57 +00:00
|
|
|
infix_finditer=infix_finditer, token_match=token_match)
|
2016-09-24 12:08:53 +00:00
|
|
|
|
2017-05-16 14:17:30 +00:00
|
|
|
@classmethod
|
|
|
|
def create_tagger(cls, nlp=None, **cfg):
|
|
|
|
if nlp is None:
|
|
|
|
return NeuralTagger(cls.create_vocab(nlp), **cfg)
|
|
|
|
else:
|
|
|
|
return NeuralTagger(nlp.vocab, **cfg)
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
def create_parser(cls, nlp=None, **cfg):
|
|
|
|
if nlp is None:
|
|
|
|
return NeuralDependencyParser(cls.create_vocab(nlp), **cfg)
|
|
|
|
else:
|
|
|
|
return NeuralDependencyParser(nlp.vocab, **cfg)
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
def create_entity(cls, nlp=None, **cfg):
|
|
|
|
if nlp is None:
|
|
|
|
return NeuralEntityRecognizer(cls.create_vocab(nlp), **cfg)
|
|
|
|
else:
|
|
|
|
return NeuralEntityRecognizer(nlp.vocab, **cfg)
|
|
|
|
|
2016-10-18 14:18:25 +00:00
|
|
|
@classmethod
|
2017-05-16 09:21:59 +00:00
|
|
|
def create_pipeline(cls, nlp=None):
|
|
|
|
meta = nlp.meta if nlp is not None else {}
|
|
|
|
# Resolve strings, like "cnn", "lstm", etc
|
2016-10-14 15:38:29 +00:00
|
|
|
pipeline = []
|
2017-05-16 09:21:59 +00:00
|
|
|
for entry in cls.pipeline:
|
|
|
|
factory = cls.Defaults.factories[entry]
|
2017-05-16 14:17:30 +00:00
|
|
|
pipeline.append(factory(nlp, **meta.get(entry, {})))
|
2016-10-09 10:24:24 +00:00
|
|
|
return pipeline
|
|
|
|
|
2017-05-16 09:21:59 +00:00
|
|
|
factories = {
|
|
|
|
'make_doc': create_tokenizer,
|
2017-05-16 14:17:30 +00:00
|
|
|
'token_vectors': lambda nlp, **cfg: TokenVectorEncoder(nlp.vocab, **cfg),
|
|
|
|
'tags': lambda nlp, **cfg: NeuralTagger(nlp.vocab, **cfg),
|
2017-05-16 09:21:59 +00:00
|
|
|
'dependencies': lambda nlp, **cfg: NeuralDependencyParser(nlp.vocab, **cfg),
|
|
|
|
'entities': lambda nlp, **cfg: NeuralEntityRecognizer(nlp.vocab, **cfg),
|
|
|
|
}
|
|
|
|
|
2017-05-08 21:58:31 +00:00
|
|
|
token_match = TOKEN_MATCH
|
|
|
|
prefixes = tuple(TOKENIZER_PREFIXES)
|
|
|
|
suffixes = tuple(TOKENIZER_SUFFIXES)
|
|
|
|
infixes = tuple(TOKENIZER_INFIXES)
|
|
|
|
tag_map = dict(TAG_MAP)
|
2016-10-09 10:24:24 +00:00
|
|
|
tokenizer_exceptions = {}
|
2016-09-26 09:57:54 +00:00
|
|
|
parser_features = get_templates('parser')
|
|
|
|
entity_features = get_templates('ner')
|
2016-10-18 14:18:25 +00:00
|
|
|
tagger_features = Tagger.feature_templates # TODO -- fix this
|
2016-09-24 18:26:17 +00:00
|
|
|
stop_words = set()
|
2016-12-18 14:50:09 +00:00
|
|
|
lemma_rules = {}
|
2017-03-15 09:52:50 +00:00
|
|
|
lemma_exc = {}
|
|
|
|
lemma_index = {}
|
2017-03-15 14:24:40 +00:00
|
|
|
morph_rules = {}
|
2017-05-08 22:58:10 +00:00
|
|
|
lex_attr_getters = LEX_ATTRS
|
2015-09-14 07:48:51 +00:00
|
|
|
|
2015-08-26 17:16:09 +00:00
|
|
|
|
2016-09-24 12:08:53 +00:00
|
|
|
class Language(object):
|
2017-05-18 21:57:38 +00:00
|
|
|
"""A text-processing pipeline. Usually you'll load this once per process,
|
|
|
|
and pass the instance around your application.
|
2017-05-19 16:47:24 +00:00
|
|
|
|
|
|
|
Defaults (class): Settings, data and factory methods for creating the `nlp`
|
|
|
|
object and processing pipeline.
|
|
|
|
lang (unicode): Two-letter language ID, i.e. ISO code.
|
2017-04-15 09:59:21 +00:00
|
|
|
"""
|
2016-09-24 18:26:17 +00:00
|
|
|
Defaults = BaseDefaults
|
2016-09-24 12:08:53 +00:00
|
|
|
lang = None
|
2015-08-25 13:37:17 +00:00
|
|
|
|
2017-05-16 09:21:59 +00:00
|
|
|
def __init__(self, vocab=True, make_doc=True, pipeline=None, meta={}):
|
2017-05-18 21:57:38 +00:00
|
|
|
"""Initialise a Language object.
|
|
|
|
|
|
|
|
vocab (Vocab): A `Vocab` object. If `True`, a vocab is created via
|
|
|
|
`Language.Defaults.create_vocab`.
|
2017-05-21 11:17:40 +00:00
|
|
|
make_doc (callable): A function that takes text and returns a `Doc`
|
2017-05-18 21:57:38 +00:00
|
|
|
object. Usually a `Tokenizer`.
|
|
|
|
pipeline (list): A list of annotation processes or IDs of annotation,
|
|
|
|
processes, e.g. a `Tagger` object, or `'tagger'`. IDs are looked
|
|
|
|
up in `Language.Defaults.factories`.
|
|
|
|
meta (dict): Custom meta data for the Language class. Is written to by
|
|
|
|
models to add model meta data.
|
|
|
|
RETURNS (Language): The newly constructed object.
|
|
|
|
"""
|
2017-05-16 09:21:59 +00:00
|
|
|
self.meta = dict(meta)
|
|
|
|
|
|
|
|
if vocab is True:
|
|
|
|
factory = self.Defaults.create_vocab
|
|
|
|
vocab = factory(self, **meta.get('vocab', {}))
|
|
|
|
self.vocab = vocab
|
|
|
|
if make_doc is True:
|
|
|
|
factory = self.Defaults.create_tokenizer
|
|
|
|
make_doc = factory(self, **meta.get('tokenizer', {}))
|
|
|
|
self.make_doc = make_doc
|
|
|
|
if pipeline is True:
|
|
|
|
self.pipeline = self.Defaults.create_pipeline(self)
|
|
|
|
elif pipeline:
|
|
|
|
self.pipeline = list(pipeline)
|
|
|
|
# Resolve strings, like "cnn", "lstm", etc
|
|
|
|
for i, entry in enumerate(self.pipeline):
|
|
|
|
if entry in self.Defaults.factories:
|
|
|
|
factory = self.Defaults.factories[entry]
|
|
|
|
self.pipeline[i] = factory(self, **meta.get(entry, {}))
|
2016-10-09 10:24:24 +00:00
|
|
|
else:
|
2017-05-16 09:21:59 +00:00
|
|
|
self.pipeline = []
|
2015-10-12 08:33:11 +00:00
|
|
|
|
2017-05-16 14:17:30 +00:00
|
|
|
def __call__(self, text, state=None, **disabled):
|
2017-05-18 21:57:38 +00:00
|
|
|
"""Apply the pipeline to some text. The text can span multiple sentences,
|
|
|
|
and can contain arbtrary whitespace. Alignment into the original string
|
2015-08-25 13:37:17 +00:00
|
|
|
is preserved.
|
2016-12-18 15:54:52 +00:00
|
|
|
|
2017-05-18 21:57:38 +00:00
|
|
|
text (unicode): The text to be processed.
|
|
|
|
**disabled: Elements of the pipeline that should not be run.
|
|
|
|
RETURNS (Doc): A container for accessing the annotations.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
2017-05-18 21:57:38 +00:00
|
|
|
EXAMPLE:
|
2016-11-01 11:25:36 +00:00
|
|
|
>>> tokens = nlp('An example sentence. Another example sentence.')
|
2017-05-18 21:57:38 +00:00
|
|
|
>>> tokens[0].text, tokens[0].head.tag_
|
2016-11-01 11:25:36 +00:00
|
|
|
('An', 'NN')
|
2015-08-25 13:37:17 +00:00
|
|
|
"""
|
2016-10-14 15:38:29 +00:00
|
|
|
doc = self.make_doc(text)
|
|
|
|
for proc in self.pipeline:
|
2017-05-16 09:21:59 +00:00
|
|
|
name = getattr(proc, 'name', None)
|
2017-05-16 14:17:30 +00:00
|
|
|
if name in disabled and not disabled[name]:
|
2017-05-16 09:21:59 +00:00
|
|
|
continue
|
2017-05-16 14:17:30 +00:00
|
|
|
state = proc(doc, state=state)
|
2016-05-17 14:55:42 +00:00
|
|
|
return doc
|
2015-08-25 13:37:17 +00:00
|
|
|
|
2017-05-16 14:17:30 +00:00
|
|
|
def update(self, docs, golds, state=None, drop=0., sgd=None):
|
2017-05-18 21:57:38 +00:00
|
|
|
"""Update the models in the pipeline.
|
|
|
|
|
|
|
|
docs (iterable): A batch of `Doc` objects.
|
|
|
|
golds (iterable): A batch of `GoldParse` objects.
|
|
|
|
drop (float): The droput rate.
|
2017-05-21 11:17:40 +00:00
|
|
|
sgd (callable): An optimizer.
|
2017-05-18 21:57:38 +00:00
|
|
|
RETURNS (dict): Results from the update.
|
|
|
|
|
|
|
|
EXAMPLE:
|
|
|
|
>>> with nlp.begin_training(gold, use_gpu=True) as (trainer, optimizer):
|
|
|
|
>>> for epoch in trainer.epochs(gold):
|
|
|
|
>>> for docs, golds in epoch:
|
|
|
|
>>> state = nlp.update(docs, golds, sgd=optimizer)
|
|
|
|
"""
|
2017-05-16 14:17:30 +00:00
|
|
|
grads = {}
|
|
|
|
def get_grads(W, dW, key=None):
|
|
|
|
grads[key] = (W, dW)
|
|
|
|
state = {} if state is None else state
|
|
|
|
for process in self.pipeline:
|
|
|
|
if hasattr(process, 'update'):
|
2017-05-18 21:57:38 +00:00
|
|
|
state = process.update(docs, golds, state=state, drop=drop,
|
|
|
|
sgd=get_grads)
|
2017-05-16 14:17:30 +00:00
|
|
|
else:
|
|
|
|
process(docs, state=state)
|
|
|
|
if sgd is not None:
|
|
|
|
for key, (W, dW) in grads.items():
|
2017-05-18 09:25:19 +00:00
|
|
|
# TODO: Unhack this when thinc improves
|
|
|
|
if isinstance(W, numpy.ndarray):
|
|
|
|
sgd.ops = NumpyOps()
|
|
|
|
else:
|
|
|
|
sgd.ops = CupyOps()
|
2017-05-16 14:17:30 +00:00
|
|
|
sgd(W, dW, key=key)
|
|
|
|
return state
|
|
|
|
|
2017-05-16 09:21:59 +00:00
|
|
|
@contextmanager
|
|
|
|
def begin_training(self, gold_tuples, **cfg):
|
2017-05-18 21:57:38 +00:00
|
|
|
"""Allocate models, pre-process training data and acquire a trainer and
|
|
|
|
optimizer. Used as a contextmanager.
|
|
|
|
|
|
|
|
gold_tuples (iterable): Gold-standard training data.
|
|
|
|
**cfg: Config parameters.
|
|
|
|
YIELDS (tuple): A trainer and an optimizer.
|
|
|
|
|
|
|
|
EXAMPLE:
|
|
|
|
>>> with nlp.begin_training(gold, use_gpu=True) as (trainer, optimizer):
|
|
|
|
>>> for epoch in trainer.epochs(gold):
|
|
|
|
>>> for docs, golds in epoch:
|
|
|
|
>>> state = nlp.update(docs, golds, sgd=optimizer)
|
|
|
|
"""
|
2017-05-17 10:04:50 +00:00
|
|
|
# Populate vocab
|
|
|
|
for _, annots_brackets in gold_tuples:
|
|
|
|
for annots, _ in annots_brackets:
|
|
|
|
for word in annots[1]:
|
|
|
|
_ = self.vocab[word]
|
|
|
|
# Handle crossing dependencies
|
|
|
|
gold_tuples = PseudoProjectivity.preprocess_training_data(gold_tuples)
|
2017-05-16 09:21:59 +00:00
|
|
|
contexts = []
|
2017-05-18 09:25:19 +00:00
|
|
|
if cfg.get('use_gpu'):
|
|
|
|
Model.ops = CupyOps()
|
|
|
|
Model.Ops = CupyOps
|
|
|
|
print("Use GPU")
|
2017-05-16 09:21:59 +00:00
|
|
|
for proc in self.pipeline:
|
|
|
|
if hasattr(proc, 'begin_training'):
|
2017-05-17 10:04:50 +00:00
|
|
|
context = proc.begin_training(gold_tuples,
|
|
|
|
pipeline=self.pipeline)
|
2017-05-16 09:21:59 +00:00
|
|
|
contexts.append(context)
|
|
|
|
trainer = Trainer(self, gold_tuples, **cfg)
|
|
|
|
yield trainer, trainer.optimizer
|
|
|
|
|
2017-05-18 09:25:19 +00:00
|
|
|
@contextmanager
|
|
|
|
def use_params(self, params, **cfg):
|
2017-05-18 21:57:38 +00:00
|
|
|
"""Replace weights of models in the pipeline with those provided in the
|
|
|
|
params dictionary. Can be used as a contextmanager, in which case,
|
|
|
|
models go back to their original weights after the block.
|
|
|
|
|
|
|
|
params (dict): A dictionary of parameters keyed by model ID.
|
|
|
|
**cfg: Config parameters.
|
|
|
|
|
|
|
|
EXAMPLE:
|
|
|
|
>>> with nlp.use_params(optimizer.averages):
|
|
|
|
>>> nlp.to_disk('/tmp/checkpoint')
|
|
|
|
"""
|
2017-05-18 13:30:59 +00:00
|
|
|
contexts = [pipe.use_params(params) for pipe
|
|
|
|
in self.pipeline if hasattr(pipe, 'use_params')]
|
|
|
|
# TODO: Having trouble with contextlib
|
|
|
|
# Workaround: these aren't actually context managers atm.
|
|
|
|
for context in contexts:
|
|
|
|
try:
|
|
|
|
next(context)
|
|
|
|
except StopIteration:
|
|
|
|
pass
|
2017-05-18 09:25:19 +00:00
|
|
|
yield
|
|
|
|
for context in contexts:
|
|
|
|
try:
|
2017-05-18 13:30:59 +00:00
|
|
|
next(context)
|
2017-05-18 09:25:19 +00:00
|
|
|
except StopIteration:
|
|
|
|
pass
|
|
|
|
|
2017-05-16 09:21:59 +00:00
|
|
|
def pipe(self, texts, n_threads=2, batch_size=1000, **disabled):
|
2017-05-18 21:57:38 +00:00
|
|
|
"""Process texts as a stream, and yield `Doc` objects in order. Supports
|
|
|
|
GIL-free multi-threading.
|
|
|
|
|
|
|
|
texts (iterator): A sequence of texts to process.
|
|
|
|
n_threads (int): The number of worker threads to use. If -1, OpenMP will
|
|
|
|
decide how many to use at run time. Default is 2.
|
|
|
|
batch_size (int): The number of texts to buffer.
|
|
|
|
**disabled: Pipeline components to exclude.
|
|
|
|
YIELDS (Doc): Documents in the order of the original text.
|
|
|
|
|
|
|
|
EXAMPLE:
|
|
|
|
>>> texts = [u'One document.', u'...', u'Lots of documents']
|
|
|
|
>>> for doc in nlp.pipe(texts, batch_size=50, n_threads=4):
|
|
|
|
>>> assert doc.is_parsed
|
2017-04-15 09:59:21 +00:00
|
|
|
"""
|
2017-05-18 13:30:59 +00:00
|
|
|
#stream = ((self.make_doc(text), None) for text in texts)
|
|
|
|
stream = ((doc, {}) for doc in texts)
|
2016-10-14 15:38:29 +00:00
|
|
|
for proc in self.pipeline:
|
2017-05-16 09:21:59 +00:00
|
|
|
name = getattr(proc, 'name', None)
|
2017-05-16 14:17:30 +00:00
|
|
|
if name in disabled and not disabled[name]:
|
2017-05-16 09:21:59 +00:00
|
|
|
continue
|
|
|
|
if hasattr(proc, 'pipe'):
|
|
|
|
stream = proc.pipe(stream, n_threads=n_threads, batch_size=batch_size)
|
|
|
|
else:
|
2017-05-16 14:17:30 +00:00
|
|
|
stream = (proc(doc, state) for doc, state in stream)
|
|
|
|
for doc, state in stream:
|
2016-02-03 01:04:55 +00:00
|
|
|
yield doc
|
2016-02-01 08:01:13 +00:00
|
|
|
|
2017-05-17 10:04:50 +00:00
|
|
|
def to_disk(self, path, **exclude):
|
|
|
|
"""Save the current state to a directory.
|
2017-04-16 23:40:26 +00:00
|
|
|
|
2017-05-18 21:57:38 +00:00
|
|
|
path (unicode or Path): A path to a directory, which will be created if
|
|
|
|
it doesn't exist. Paths may be either strings or `Path`-like objects.
|
|
|
|
**exclude: Named attributes to prevent from being saved.
|
|
|
|
|
|
|
|
EXAMPLE:
|
|
|
|
>>> nlp.to_disk('/path/to/models')
|
2017-05-17 10:04:50 +00:00
|
|
|
"""
|
|
|
|
path = util.ensure_path(path)
|
|
|
|
if not path.exists():
|
|
|
|
path.mkdir()
|
|
|
|
if not path.is_dir():
|
|
|
|
raise IOError("Output path must be a directory")
|
|
|
|
props = {}
|
|
|
|
for name, value in self.__dict__.items():
|
|
|
|
if name in exclude:
|
|
|
|
continue
|
|
|
|
if hasattr(value, 'to_disk'):
|
|
|
|
value.to_disk(path / name)
|
|
|
|
else:
|
|
|
|
props[name] = value
|
|
|
|
with (path / 'props.pickle').open('wb') as file_:
|
|
|
|
dill.dump(props, file_)
|
|
|
|
|
|
|
|
def from_disk(self, path, **exclude):
|
2017-05-18 21:57:38 +00:00
|
|
|
"""Loads state from a directory. Modifies the object in place and
|
|
|
|
returns it.
|
|
|
|
|
|
|
|
path (unicode or Path): A path to a directory. Paths may be either
|
|
|
|
strings or `Path`-like objects.
|
|
|
|
**exclude: Named attributes to prevent from being loaded.
|
|
|
|
RETURNS (Language): The modified `Language` object.
|
2017-05-17 10:04:50 +00:00
|
|
|
|
2017-05-18 21:57:38 +00:00
|
|
|
EXAMPLE:
|
|
|
|
>>> from spacy.language import Language
|
|
|
|
>>> nlp = Language().from_disk('/path/to/models')
|
2017-05-17 10:04:50 +00:00
|
|
|
"""
|
|
|
|
path = util.ensure_path(path)
|
|
|
|
for name in path.iterdir():
|
|
|
|
if name not in exclude and hasattr(self, str(name)):
|
|
|
|
getattr(self, name).from_disk(path / name)
|
|
|
|
with (path / 'props.pickle').open('rb') as file_:
|
|
|
|
bytes_data = file_.read()
|
|
|
|
self.from_bytes(bytes_data, **exclude)
|
|
|
|
return self
|
|
|
|
|
|
|
|
def to_bytes(self, **exclude):
|
|
|
|
"""Serialize the current state to a binary string.
|
2016-12-18 15:54:52 +00:00
|
|
|
|
2017-05-18 21:57:38 +00:00
|
|
|
**exclude: Named attributes to prevent from being serialized.
|
|
|
|
RETURNS (bytes): The serialized form of the `Language` object.
|
2017-05-17 10:04:50 +00:00
|
|
|
"""
|
|
|
|
props = dict(self.__dict__)
|
|
|
|
for key in exclude:
|
|
|
|
if key in props:
|
|
|
|
props.pop(key)
|
|
|
|
return dill.dumps(props, -1)
|
2017-04-15 10:05:47 +00:00
|
|
|
|
2017-05-17 10:04:50 +00:00
|
|
|
def from_bytes(self, bytes_data, **exclude):
|
|
|
|
"""Load state from a binary string.
|
|
|
|
|
2017-05-18 21:57:38 +00:00
|
|
|
bytes_data (bytes): The data to load from.
|
|
|
|
**exclude: Named attributes to prevent from being loaded.
|
|
|
|
RETURNS (Language): The `Language` object.
|
2017-05-17 10:04:50 +00:00
|
|
|
"""
|
|
|
|
props = dill.loads(bytes_data)
|
|
|
|
for key, value in props.items():
|
|
|
|
if key not in exclude:
|
|
|
|
setattr(self, key, value)
|
|
|
|
return self
|