explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 18:06:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Load serializer freqs in vocab

Browse Source
This commit is contained in:
Matthew Honnibal 2015-09-10 15:22:48 +02:00
parent 094440f9f5
commit e285ca7d6c
2 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
spacy/vocab.pxd
View File

@ -34,6 +34,7 @@ cdef class Vocab:
cdef public object data_dir cdef public object data_dir
cdef public object get_lex_attr cdef public object get_lex_attr
cdef public object pos_tags cdef public object pos_tags
cdef public object serializer_freqs
cdef const LexemeC* get(self, Pool mem, unicode string) except NULL cdef const LexemeC* get(self, Pool mem, unicode string) except NULL
cdef const LexemeC* get_by_orth(self, Pool mem, attr_t orth) except NULL cdef const LexemeC* get_by_orth(self, Pool mem, attr_t orth) except NULL

14
spacy/vocab.pyx
View File

@ -47,16 +47,17 @@ cdef class Vocab:
tag_map = json.load(open(path.join(data_dir, 'tag_map.json'))) tag_map = json.load(open(path.join(data_dir, 'tag_map.json')))
lemmatizer = Lemmatizer.from_dir(path.join(data_dir, '..')) lemmatizer = Lemmatizer.from_dir(path.join(data_dir, '..'))
serializer_freqs = json.load(open(path.join(data_dir, 'serializer.json')))
cdef Vocab self = cls(get_lex_attr=get_lex_attr, vectors=vectors, tag_map=tag_map, cdef Vocab self = cls(get_lex_attr=get_lex_attr, vectors=vectors, tag_map=tag_map,
lemmatizer=lemmatizer) lemmatizer=lemmatizer, serializer_freqs=serializer_freqs)
self.load_lexemes(path.join(data_dir, 'strings.txt'), path.join(data_dir, 'lexemes.bin')) self.load_lexemes(path.join(data_dir, 'strings.txt'), path.join(data_dir, 'lexemes.bin'))
if vectors is None and path.exists(path.join(data_dir, 'vec.bin')): if vectors is None and path.exists(path.join(data_dir, 'vec.bin')):
self.repvec_length = self.load_rep_vectors(path.join(data_dir, 'vec.bin')) self.repvec_length = self.load_rep_vectors(path.join(data_dir, 'vec.bin'))
return self return self
def __init__(self, get_lex_attr=None, tag_map=None, vectors=None, lemmatizer=None): def __init__(self, get_lex_attr=None, tag_map=None, vectors=None, lemmatizer=None,
serializer_freqs=None):
if tag_map is None: if tag_map is None:
tag_map = {} tag_map = {}
if lemmatizer is None: if lemmatizer is None:
@ -67,6 +68,7 @@ cdef class Vocab:
self.strings = StringStore() self.strings = StringStore()
self.get_lex_attr = get_lex_attr self.get_lex_attr = get_lex_attr
self.morphology = Morphology(self.strings, tag_map, lemmatizer) self.morphology = Morphology(self.strings, tag_map, lemmatizer)
self.serializer_freqs = serializer_freqs
self.length = 1 self.length = 1
self._serializer = None self._serializer = None
@ -75,11 +77,7 @@ cdef class Vocab:
def __get__(self): def __get__(self):
if self._serializer is None: if self._serializer is None:
freqs = [] freqs = []
if self.data_dir is not None: self._serializer = Packer(self, self.serializer_freqs)
freqs_loc = path.join(self.data_dir, 'serializer.json')
if path.exists(freqs_loc):
freqs = json.load(open(freqs_loc))
self._serializer = Packer(self, freqs)
return self._serializer return self._serializer
def __len__(self): def __len__(self):