explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-28 19:06:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update test to make it slightly more direct

Browse Source 
The `nlp` container should be unnecessary here. If so, we can test the tokenizer class just a little more directly.
This commit is contained in:
Matthew Honnibal 2017-09-01 21:16:56 +02:00 committed by GitHub
parent a6d9fb5bb6
commit 9bffcaa73d
1 changed files with 20 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
spacy/tests/tokenizer/test_customized_tokenizer.py
View File

@ -1,15 +1,14 @@
# coding: utf-8 # coding: utf-8
from __future__ import unicode_literals from __future__ import unicode_literals
from ... import load from ...lang.en import English
from ...tokenizer import Tokenizer from ...tokenizer import Tokenizer
from ... import util from ... import util
import pytest import pytest
@pytest.fixture
def test_customized_tokenizer_handles_infixes(): def tokenizer(en_vocab):
def custom_tokenizer(nlp_model):
prefix_re = util.compile_prefix_regex(nlp_model.Defaults.prefixes) prefix_re = util.compile_prefix_regex(nlp_model.Defaults.prefixes)
suffix_re = util.compile_suffix_regex(nlp_model.Defaults.suffixes) suffix_re = util.compile_suffix_regex(nlp_model.Defaults.suffixes)
custom_infixes = ['\.\.\.+', custom_infixes = ['\.\.\.+',
@ -19,28 +18,23 @@ def test_customized_tokenizer_handles_infixes():
u'[\[\]!&:,()\*—–\/-]'] u'[\[\]!&:,()\*—–\/-]']
infix_re = util.compile_infix_regex(custom_infixes) infix_re = util.compile_infix_regex(custom_infixes)
return Tokenizer(en_vocab,
# infix_re = re.compile(ur'[\[\]!&:,()]') English.Defaults.tokenizer_exceptions,
tokenizer = Tokenizer(nlp_model.vocab,
nlp_model.Defaults.tokenizer_exceptions,
prefix_re.search, prefix_re.search,
suffix_re.search, suffix_re.search,
infix_re.finditer, infix_re.finditer,
token_match=None) token_match=None)
return lambda text: tokenizer(text)
nlp = load('en', create_make_doc=custom_tokenizer)
def test_customized_tokenizer_handles_infixes(tokenizer):
sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion." sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion."
context = [word.text for word in nlp(sentence)] context = [word.text for word in tokenizer(sentence)]
assert context == [u'The', u'8', u'and', u'10', u'-', u'county', u'definitions', u'are', u'not', u'used', assert context == [u'The', u'8', u'and', u'10', u'-', u'county', u'definitions', u'are', u'not', u'used',
u'for', u'for',
u'the', u'greater', u'Southern', u'California', u'Megaregion', u'.'] u'the', u'greater', u'Southern', u'California', u'Megaregion', u'.']
# the trailing '-' may cause Assertion Error # the trailing '-' may cause Assertion Error
sentence = "The 8- and 10-county definitions are not used for the greater Southern California Megaregion." sentence = "The 8- and 10-county definitions are not used for the greater Southern California Megaregion."
context = [word.text for word in nlp(sentence)] context = [word.text for word in tokenizer(sentence)]
assert context == [u'The', u'8', u'-', u'and', u'10', u'-', u'county', u'definitions', u'are', u'not', u'used', assert context == [u'The', u'8', u'-', u'and', u'10', u'-', u'county', u'definitions', u'are', u'not', u'used',
u'for', u'for',
u'the', u'greater', u'Southern', u'California', u'Megaregion', u'.'] u'the', u'greater', u'Southern', u'California', u'Megaregion', u'.']