explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-13 02:36:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Fix memory error in cache, where entry wasn't being null-terminated. Various other changes, some good for performance

Browse Source
This commit is contained in:
Matthew Honnibal 2014-09-15 06:33:53 +02:00
parent 7959141d36
commit 0bb547ab98
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/lang.pxd
View File

@ -42,5 +42,5 @@ cdef class Language:
cpdef Tokens tokenize(self, unicode text) cpdef Tokens tokenize(self, unicode text)
cpdef Lexeme lookup(self, unicode text) cpdef Lexeme lookup(self, unicode text)
cdef int _tokenize(self, vector[LexemeC*] *tokens_v, String* string) cdef int _tokenize(self, vector[LexemeC*] *tokens_v, String* string) except -1
cdef int _split_one(self, Py_UNICODE* characters, size_t length) cdef int _split_one(self, Py_UNICODE* characters, size_t length)

10
spacy/lang.pyx
View File

@ -190,7 +190,7 @@ cdef class Language:
self._tokenize(tokens.v, &span) self._tokenize(tokens.v, &span)
return tokens return tokens
cdef int _tokenize(self, vector[LexemeC*] *tokens_v, String* string): cdef int _tokenize(self, vector[LexemeC*] *tokens_v, String* string) except -1:
cdef LexemeC** lexemes = <LexemeC**>self.cache.get(string.key) cdef LexemeC** lexemes = <LexemeC**>self.cache.get(string.key)
cdef size_t i cdef size_t i
if lexemes != NULL: if lexemes != NULL:
@ -204,6 +204,7 @@ cdef class Language:
cdef int split cdef int split
cdef int remaining = string.n cdef int remaining = string.n
cdef String prefix cdef String prefix
cdef LexemeC* lexeme
while remaining >= 1: while remaining >= 1:
split = self._split_one(string.chars, string.n) split = self._split_one(string.chars, string.n)
remaining -= split remaining -= split
@ -215,11 +216,13 @@ cdef class Language:
tokens_v.push_back(lexemes[i]) tokens_v.push_back(lexemes[i])
i += 1 i += 1
else: else:
tokens_v.push_back(<LexemeC*>self.lexicon.get(&prefix)) lexeme = <LexemeC*>self.lexicon.get(&prefix)
lexemes = <LexemeC**>calloc(tokens_v.size() - first_token, sizeof(LexemeC*)) tokens_v.push_back(lexeme)
lexemes = <LexemeC**>calloc((tokens_v.size() - first_token) + 1, sizeof(LexemeC*))
cdef size_t j cdef size_t j
for i, j in enumerate(range(first_token, tokens_v.size())): for i, j in enumerate(range(first_token, tokens_v.size())):
lexemes[i] = tokens_v[0][j] lexemes[i] = tokens_v[0][j]
lexemes[i+1] = NULL
self.cache.set(key, lexemes) self.cache.set(key, lexemes)
cdef int _split_one(self, Py_UNICODE* characters, size_t length): cdef int _split_one(self, Py_UNICODE* characters, size_t length):
@ -310,7 +313,6 @@ cdef class Lexicon:
cdef void string_from_unicode(String* s, unicode uni): cdef void string_from_unicode(String* s, unicode uni):
Py_INCREF(uni)
cdef Py_UNICODE* c_uni = <Py_UNICODE*>uni cdef Py_UNICODE* c_uni = <Py_UNICODE*>uni
string_from_slice(s, c_uni, 0, len(uni)) string_from_slice(s, c_uni, 0, len(uni))