mirror of
https://github.com/explosion/spaCy.git
synced 2025-07-10 16:22:29 +03:00
Make vocab always own lexemes
This commit is contained in:
parent
4048ca01eb
commit
6117adcd6d
|
@ -161,8 +161,11 @@ cdef class Vocab:
|
||||||
return self._new_lexeme(mem, self.strings[orth])
|
return self._new_lexeme(mem, self.strings[orth])
|
||||||
|
|
||||||
cdef const LexemeC* _new_lexeme(self, Pool mem, unicode string) except NULL:
|
cdef const LexemeC* _new_lexeme(self, Pool mem, unicode string) except NULL:
|
||||||
if len(string) < 3 or self.length < 10000:
|
#if len(string) < 3 or self.length < 10000:
|
||||||
mem = self.mem
|
# mem = self.mem
|
||||||
|
# TODO: Experiment with never allowing the Doc to own lexemes, to see
|
||||||
|
# if it solves the Doc.copy() issue.
|
||||||
|
mem = self.mem
|
||||||
cdef bint is_oov = mem is not self.mem
|
cdef bint is_oov = mem is not self.mem
|
||||||
lex = <LexemeC*>mem.alloc(1, sizeof(LexemeC))
|
lex = <LexemeC*>mem.alloc(1, sizeof(LexemeC))
|
||||||
lex.orth = self.strings.add(string)
|
lex.orth = self.strings.add(string)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user