explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-13 05:07:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

Allow the vocabulary to grow to 10,000, to prevent cold-start problem.

Browse Source
This commit is contained in:
Matthew Honnibal 2016-12-27 21:03:45 +01:00
parent 0c7720e162
commit ce4539dafd
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy/vocab.pyx
View File

@ -262,9 +262,9 @@ cdef class Vocab:
cdef const LexemeC* _new_lexeme(self, Pool mem, unicode string) except NULL: cdef const LexemeC* _new_lexeme(self, Pool mem, unicode string) except NULL:
cdef hash_t key cdef hash_t key
cdef bint is_oov = mem is not self.mem if len(string) < 3 or self.length < 10000:
if len(string) < 3:
mem = self.mem mem = self.mem
cdef bint is_oov = mem is not self.mem
lex = <LexemeC*>mem.alloc(sizeof(LexemeC), 1) lex = <LexemeC*>mem.alloc(sizeof(LexemeC), 1)
lex.orth = self.strings[string] lex.orth = self.strings[string]
lex.length = len(string) lex.length = len(string)