mirror of
https://github.com/explosion/spaCy.git
synced 2025-10-31 16:07:41 +03:00
db55577c45
* Remove unicode declarations * Remove Python 3.5 and 2.7 from CI * Don't require pathlib * Replace compat helpers * Remove OrderedDict * Use f-strings * Set Cython compiler language level * Fix typo * Re-add OrderedDict for Table * Update setup.cfg * Revert CONTRIBUTING.md * Revert lookups.md * Revert top-level.md * Small adjustments and docs [ci skip]
154 lines
3.7 KiB
Python
154 lines
3.7 KiB
Python
import pytest
|
|
import re
|
|
from spacy.lang.en import English
|
|
from spacy.tokenizer import Tokenizer
|
|
from spacy.util import compile_prefix_regex, compile_suffix_regex
|
|
from spacy.util import compile_infix_regex
|
|
|
|
|
|
@pytest.fixture
|
|
def custom_en_tokenizer(en_vocab):
|
|
prefix_re = compile_prefix_regex(English.Defaults.prefixes)
|
|
suffix_re = compile_suffix_regex(English.Defaults.suffixes)
|
|
custom_infixes = [
|
|
r"\.\.\.+",
|
|
r"(?<=[0-9])-(?=[0-9])",
|
|
r"[0-9]+(,[0-9]+)+",
|
|
r"[\[\]!&:,()\*—–\/-]",
|
|
]
|
|
infix_re = compile_infix_regex(custom_infixes)
|
|
token_match_re = re.compile("a-b")
|
|
return Tokenizer(
|
|
en_vocab,
|
|
English.Defaults.tokenizer_exceptions,
|
|
prefix_re.search,
|
|
suffix_re.search,
|
|
infix_re.finditer,
|
|
token_match=token_match_re.match,
|
|
)
|
|
|
|
|
|
def test_en_customized_tokenizer_handles_infixes(custom_en_tokenizer):
|
|
sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion."
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"are",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
]
|
|
# the trailing '-' may cause Assertion Error
|
|
sentence = "The 8- and 10-county definitions are not used for the greater Southern California Megaregion."
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"-",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"are",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
]
|
|
|
|
|
|
def test_en_customized_tokenizer_handles_token_match(custom_en_tokenizer):
|
|
sentence = "The 8 and 10-county definitions a-b not used for the greater Southern California Megaregion."
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"a-b",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
]
|
|
|
|
|
|
def test_en_customized_tokenizer_handles_rules(custom_en_tokenizer):
|
|
sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion. :)"
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"are",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
":)",
|
|
]
|
|
|
|
|
|
def test_en_customized_tokenizer_handles_rules_property(custom_en_tokenizer):
|
|
sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion. :)"
|
|
rules = custom_en_tokenizer.rules
|
|
del rules[":)"]
|
|
custom_en_tokenizer.rules = rules
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"are",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
":",
|
|
")",
|
|
]
|