mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-25 01:16:28 +03:00
Port over changes from #1294
This commit is contained in:
parent
38c756fd85
commit
cd6a29dce7
42
spacy/tests/lang/en/test_customized_tokenizer.py
Normal file
42
spacy/tests/lang/en/test_customized_tokenizer.py
Normal file
|
@ -0,0 +1,42 @@
|
|||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import pytest
|
||||
|
||||
from ....lang.en import English
|
||||
from ....tokenizer import Tokenizer
|
||||
from .... import util
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def custom_en_tokenizer(en_vocab):
|
||||
prefix_re = util.compile_prefix_regex(English.Defaults.prefixes)
|
||||
suffix_re = util.compile_suffix_regex(English.Defaults.suffixes)
|
||||
custom_infixes = ['\.\.\.+',
|
||||
'(?<=[0-9])-(?=[0-9])',
|
||||
# '(?<=[0-9]+),(?=[0-9]+)',
|
||||
'[0-9]+(,[0-9]+)+',
|
||||
'[\[\]!&:,()\*—–\/-]']
|
||||
|
||||
infix_re = util.compile_infix_regex(custom_infixes)
|
||||
return Tokenizer(en_vocab,
|
||||
English.Defaults.tokenizer_exceptions,
|
||||
prefix_re.search,
|
||||
suffix_re.search,
|
||||
infix_re.finditer,
|
||||
token_match=None)
|
||||
|
||||
|
||||
def test_customized_tokenizer_handles_infixes(custom_en_tokenizer):
|
||||
sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion."
|
||||
context = [word.text for word in custom_en_tokenizer(sentence)]
|
||||
assert context == ['The', '8', 'and', '10', '-', 'county', 'definitions',
|
||||
'are', 'not', 'used', 'for', 'the', 'greater',
|
||||
'Southern', 'California', 'Megaregion', '.']
|
||||
|
||||
# the trailing '-' may cause Assertion Error
|
||||
sentence = "The 8- and 10-county definitions are not used for the greater Southern California Megaregion."
|
||||
context = [word.text for word in custom_en_tokenizer(sentence)]
|
||||
assert context == ['The', '8', '-', 'and', '10', '-', 'county',
|
||||
'definitions', 'are', 'not', 'used', 'for', 'the',
|
||||
'greater', 'Southern', 'California', 'Megaregion', '.']
|
|
@ -248,7 +248,8 @@ cdef class Tokenizer:
|
|||
|
||||
start = infix_end
|
||||
span = string[start:]
|
||||
tokens.push_back(self.vocab.get(tokens.mem, span), False)
|
||||
if span:
|
||||
tokens.push_back(self.vocab.get(tokens.mem, span), False)
|
||||
cdef vector[const LexemeC*].reverse_iterator it = suffixes.rbegin()
|
||||
while it != suffixes.rend():
|
||||
lexeme = deref(it)
|
||||
|
|
Loading…
Reference in New Issue
Block a user