mirror of https://github.com/explosion/spaCy.git
Make doc pickling support hooks
This commit is contained in:
parent
fe844148f6
commit
394633efce
|
@ -939,12 +939,19 @@ cdef int set_children_from_heads(TokenC* tokens, int length) except -1:
|
||||||
|
|
||||||
def pickle_doc(doc):
|
def pickle_doc(doc):
|
||||||
bytes_data = doc.to_bytes(vocab=False, user_data=False)
|
bytes_data = doc.to_bytes(vocab=False, user_data=False)
|
||||||
return (unpickle_doc, (doc.vocab, doc.user_data, bytes_data))
|
hooks_and_data = (doc.user_data, doc.user_hooks, doc.user_span_hooks,
|
||||||
|
doc.user_token_hooks)
|
||||||
|
return (unpickle_doc, (doc.vocab, dill.dumps(hooks_and_data), bytes_data))
|
||||||
|
|
||||||
|
|
||||||
def unpickle_doc(vocab, user_data, bytes_data):
|
def unpickle_doc(vocab, hooks_and_data, bytes_data):
|
||||||
|
user_data, doc_hooks, span_hooks, token_hooks = dill.loads(hooks_and_data)
|
||||||
|
|
||||||
doc = Doc(vocab, user_data=user_data).from_bytes(bytes_data,
|
doc = Doc(vocab, user_data=user_data).from_bytes(bytes_data,
|
||||||
exclude='user_data')
|
exclude='user_data')
|
||||||
|
doc.user_hooks.update(doc_hooks)
|
||||||
|
doc.user_span_hooks.update(span_hooks)
|
||||||
|
doc.user_token_hooks.update(token_hooks)
|
||||||
return doc
|
return doc
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue