explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-26 01:04:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

* If final token is whitespace, don't mark it as owning a trailing space. Fixes Issue #154

Browse Source
This commit is contained in:
Matthew Honnibal 2016-01-16 17:08:59 +01:00
parent 223d2b3484
commit 3e9961d2c4
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/tokenizer.pyx
View File

@ -130,7 +130,7 @@ cdef class Tokenizer:
cache_hit = self._try_cache(key, tokens) cache_hit = self._try_cache(key, tokens)
if not cache_hit: if not cache_hit:
self._tokenize(tokens, span, key) self._tokenize(tokens, span, key)
tokens.c[tokens.length - 1].spacy = string[-1] == ' ' tokens.c[tokens.length - 1].spacy = string[-1] == ' ' and not in_ws
return tokens return tokens
cdef int _try_cache(self, hash_t key, Doc tokens) except -1: cdef int _try_cache(self, hash_t key, Doc tokens) except -1: