Pickle vectors with vocab

This commit is contained in:
Matthew Honnibal 2017-11-23 17:19:50 +01:00
parent b60d92aca8
commit 79f11d4f85

View File

@ -470,26 +470,27 @@ cdef class Vocab:
def pickle_vocab(vocab): def pickle_vocab(vocab):
sstore = vocab.strings sstore = vocab.strings
vectors = vocab.vectors
morph = vocab.morphology morph = vocab.morphology
length = vocab.length length = vocab.length
data_dir = vocab.data_dir data_dir = vocab.data_dir
lex_attr_getters = dill.dumps(vocab.lex_attr_getters) lex_attr_getters = dill.dumps(vocab.lex_attr_getters)
lexemes_data = vocab.lexemes_to_bytes() lexemes_data = vocab.lexemes_to_bytes()
return (unpickle_vocab, return (unpickle_vocab,
(sstore, morph, data_dir, lex_attr_getters, lexemes_data, length)) (sstore, vectors, morph, data_dir, lex_attr_getters, lexemes_data, length))
def unpickle_vocab(sstore, morphology, data_dir, def unpickle_vocab(sstore, vectors, morphology, data_dir,
lex_attr_getters, bytes lexemes_data, int length): lex_attr_getters, bytes lexemes_data, int length):
cdef Vocab vocab = Vocab() cdef Vocab vocab = Vocab()
vocab.length = length vocab.length = length
vocab.vectors = vectors
vocab.strings = sstore vocab.strings = sstore
vocab.morphology = morphology vocab.morphology = morphology
vocab.data_dir = data_dir vocab.data_dir = data_dir
vocab.lex_attr_getters = dill.loads(lex_attr_getters) vocab.lex_attr_getters = dill.loads(lex_attr_getters)
vocab.lexemes_from_bytes(lexemes_data) vocab.lexemes_from_bytes(lexemes_data)
vocab.length = length vocab.length = length
link_vectors_to_models(vocab)
return vocab return vocab