explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 09:56:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Serialize lex_attr_getters with dill, for better pickle support

Browse Source
This commit is contained in:
Matthew Honnibal 2017-10-17 18:17:45 +02:00
parent 45d1dd90b1
commit 0d57b9748a
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
spacy/vocab.pyx
View File

@ -5,6 +5,7 @@ import bz2
import ujson import ujson
import re import re
import numpy import numpy
import dill
from libc.string cimport memset, memcpy from libc.string cimport memset, memcpy
from libc.stdint cimport int32_t from libc.stdint cimport int32_t
@ -419,7 +420,7 @@ def pickle_vocab(vocab):
morph = vocab.morphology morph = vocab.morphology
length = vocab.length length = vocab.length
data_dir = vocab.data_dir data_dir = vocab.data_dir
lex_attr_getters = vocab.lex_attr_getters lex_attr_getters = dill.dumps(vocab.lex_attr_getters)
lexemes_data = vocab.lexemes_to_bytes() lexemes_data = vocab.lexemes_to_bytes()
@ -435,7 +436,7 @@ def unpickle_vocab(sstore, morphology, data_dir,
vocab.strings = sstore vocab.strings = sstore
vocab.morphology = morphology vocab.morphology = morphology
vocab.data_dir = data_dir vocab.data_dir = data_dir
vocab.lex_attr_getters = lex_attr_getters vocab.lex_attr_getters = dill.loads(lex_attr_getters)
vocab.lexemes_from_bytes(lexemes_data) vocab.lexemes_from_bytes(lexemes_data)
vocab.length = length vocab.length = length
link_vectors_to_models(vocab) link_vectors_to_models(vocab)