explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-13 02:36:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

Patch deserialisation for invalid loads, to avoid model failure

Browse Source
This commit is contained in:
Matthew Honnibal 2017-10-21 00:51:42 +02:00
parent d8391b1c4d
commit 8f8bccecb9
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy/vocab.pyx
View File

@ -400,6 +400,7 @@ cdef class Vocab:
cdef int j = 0 cdef int j = 0
cdef SerializedLexemeC lex_data cdef SerializedLexemeC lex_data
chunk_size = sizeof(lex_data.data) chunk_size = sizeof(lex_data.data)
cdef void* ptr
cdef unsigned char* bytes_ptr = bytes_data cdef unsigned char* bytes_ptr = bytes_data
for i in range(0, len(bytes_data), chunk_size): for i in range(0, len(bytes_data), chunk_size):
lexeme = <LexemeC*>self.mem.alloc(1, sizeof(LexemeC)) lexeme = <LexemeC*>self.mem.alloc(1, sizeof(LexemeC))
@ -407,6 +408,9 @@ cdef class Vocab:
lex_data.data[j] = bytes_ptr[i+j] lex_data.data[j] = bytes_ptr[i+j]
Lexeme.c_from_bytes(lexeme, lex_data) Lexeme.c_from_bytes(lexeme, lex_data)
ptr = self.strings._map.get(lexeme.orth)
if ptr == NULL:
continue
py_str = self.strings[lexeme.orth] py_str = self.strings[lexeme.orth]
assert self.strings[py_str] == lexeme.orth, (py_str, lexeme.orth) assert self.strings[py_str] == lexeme.orth, (py_str, lexeme.orth)
key = hash_string(py_str) key = hash_string(py_str)