explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 17:36:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Add serialize/deserialize functions for lexeme, transport to/from python dict.

Browse Source
This commit is contained in:
Matthew Honnibal 2014-10-09 14:10:46 +11:00
parent d73d89a2de
commit 51d75b244b
2 changed files with 31 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spacy/lexeme.pxd
View File

@ -19,3 +19,6 @@ cdef LexemeC* lexeme_init(Pool mem, size_t i, unicode string, double prob, size_
cdef bint lexeme_check_flag(LexemeC* lexeme, size_t flag_id) cdef bint lexeme_check_flag(LexemeC* lexeme, size_t flag_id)
cdef unicode lexeme_string_view(LexemeC* lexeme, size_t view_id) cdef unicode lexeme_string_view(LexemeC* lexeme, size_t view_id)
cdef dict lexeme_pack(LexemeC* lexeme)

28
spacy/lexeme.pyx
View File

@ -34,3 +34,31 @@ cdef bint lexeme_check_flag(LexemeC* lexeme, size_t flag_id):
cdef unicode lexeme_string_view(LexemeC* lexeme, size_t view_id): cdef unicode lexeme_string_view(LexemeC* lexeme, size_t view_id):
cdef bytes byte_string = lexeme.views[view_id] cdef bytes byte_string = lexeme.views[view_id]
return byte_string.decode('utf8') return byte_string.decode('utf8')
cdef dict lexeme_pack(LexemeC* lexeme):
cdef dict packed = {}
packed['i'] = lexeme.i
packed['length'] = lexeme.length
packed['prob'] = lexeme.prob
packed['cluster'] = lexeme.cluster
packed['string'] = lexeme.string.decode('utf8')
packed['views'] = []
cdef size_t i = 0
while lexeme.views[i] != NULL:
packed['views'].append(lexeme.views[i].decode('utf8'))
i += 1
packed['flags'] = lexeme.flags
return packed
cdef int lexeme_unpack(LexemeC* lex, dict p) except -1:
cdef size_t length
lex.i = p['i']
lex.length = p['length']
lex.prob = p['prob']
lex.cluster = p['cluster']
lex.string = intern_and_encode(p['string'], &length)
for i, view in enumerate(p['views']):
lex.views[i] = intern_and_encode(view, &length)
lex.flags = p['flags']