explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-11 04:08:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

Make doc pickling support hooks

Browse Source
This commit is contained in:
Matthew Honnibal 2017-10-17 19:44:09 +02:00
parent fe844148f6
commit 394633efce
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
spacy/tokens/doc.pyx
View File

@ -939,12 +939,19 @@ cdef int set_children_from_heads(TokenC* tokens, int length) except -1:
def pickle_doc(doc):
bytes_data = doc.to_bytes(vocab=False, user_data=False)
return (unpickle_doc, (doc.vocab, doc.user_data, bytes_data))
hooks_and_data = (doc.user_data, doc.user_hooks, doc.user_span_hooks,
doc.user_token_hooks)
return (unpickle_doc, (doc.vocab, dill.dumps(hooks_and_data), bytes_data))
def unpickle_doc(vocab, user_data, bytes_data):
def unpickle_doc(vocab, hooks_and_data, bytes_data):
user_data, doc_hooks, span_hooks, token_hooks = dill.loads(hooks_and_data)
doc = Doc(vocab, user_data=user_data).from_bytes(bytes_data,
exclude='user_data')
doc.user_hooks.update(doc_hooks)
doc.user_span_hooks.update(span_hooks)
doc.user_token_hooks.update(token_hooks)
return doc