explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 09:26:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

* More slight cleaning for lang.pyx

Browse Source
This commit is contained in:
Matthew Honnibal 2014-10-10 20:11:22 +11:00
parent 3d82ed1e5e
commit ff79dbac2e
1 changed files with 2 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
spacy/lang.pyx
View File

@ -176,18 +176,12 @@ cdef class Language:
cdef int _find_prefix(self, Py_UNICODE* chars, size_t length) except -1:
cdef unicode string = chars[:length]
match = self.prefix_re.search(string)
if match is None:
return 0
else:
return match.end() - match.start()
return (match.end() - match.start()) if match is not None else 0
cdef int _find_suffix(self, Py_UNICODE* chars, size_t length):
cdef unicode string = chars[:length]
match = self.suffix_re.search(string)
if match is None:
return 0
else:
return match.end() - match.start()
return (match.end() - match.start()) if match is not None else 0
def _load_special_tokenization(self, token_rules):
'''Load special-case tokenization rules.