explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-09-21 03:19:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixed issue of infix capturing prefixes

Browse Source
This commit is contained in:
Felix Sonntag 2017-11-28 17:17:12 +01:00
parent 33b0f86de3
commit 724ae7dc55
2 changed files with 21 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
spacy/tests/regression/test_issue1494.py
View File

@ -10,8 +10,14 @@ from ...tokenizer import Tokenizer
def test_issue1494():
infix_re = re.compile(r'''[^a-z]''')
text_to_tokenize = 'token 123test'
expected_tokens = ['token', '1', '2', '3', 'test']
text_to_tokenize1 = 'token 123test'
expected_tokens1 = ['token', '1', '2', '3', 'test']
text_to_tokenize2 = 'token 1test'
expected_tokens2 = ['token', '1test']
text_to_tokenize3 = 'hello...test'
expected_tokens3 = ['hello', '.', '.', '.', 'test']
def my_tokenizer(nlp):
return Tokenizer(nlp.vocab,
@ -22,6 +28,12 @@ def test_issue1494():
nlp = English()
nlp.tokenizer = my_tokenizer(nlp)
tokenized_words = [token.text for token in nlp(text_to_tokenize)]
print(tokenized_words)
assert tokenized_words == expected_tokens
tokenized_words1 = [token.text for token in nlp(text_to_tokenize1)]
assert tokenized_words1 == expected_tokens1
tokenized_words2 = [token.text for token in nlp(text_to_tokenize2)]
assert tokenized_words2 == expected_tokens2
tokenized_words3 = [token.text for token in nlp(text_to_tokenize3)]
assert tokenized_words3 == expected_tokens3

4
spacy/tokenizer.pyx
View File

@ -238,10 +238,14 @@ cdef class Tokenizer:
# let's say we have dyn-o-mite-dave - the regex finds the
# start and end positions of the hyphens
start = 0
start_before_infixes = start
for match in matches:
infix_start = match.start()
infix_end = match.end()
if infix_start == start_before_infixes:
continue
if infix_start != start:
span = string[start:infix_start]
tokens.push_back(self.vocab.get(tokens.mem, span), False)