explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-11 04:08:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Add string length cap in Tokenizer.__call__

Browse Source
This commit is contained in:
Matthew Honnibal 2015-10-16 04:54:16 +11:00
parent 17fffb4c57
commit 3ba66f2dc7
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy/tokenizer.pyx
View File

@ -72,6 +72,10 @@ cdef class Tokenizer:
Returns: Returns:
tokens (Doc): A Doc object, giving access to a sequence of LexemeCs. tokens (Doc): A Doc object, giving access to a sequence of LexemeCs.
""" """
if len(string) >= (2 ** 30):
raise ValueError(
"String is too long: %d characters. Max is 2**30." % len(string)
)
cdef int length = len(string) cdef int length = len(string)
cdef Doc tokens = Doc(self.vocab) cdef Doc tokens = Doc(self.vocab)
if length == 0: if length == 0: