mirror of
https://github.com/explosion/spaCy.git
synced 2025-01-15 20:16:23 +03:00
5861308910
Handle tokenizer special cases more generally by using the Matcher internally to match special cases after the affix/token_match tokenization is complete. Instead of only matching special cases while processing balanced or nearly balanced prefixes and suffixes, this recognizes special cases in a wider range of contexts: * Allows arbitrary numbers of prefixes/affixes around special cases * Allows special cases separated by infixes Existing tests/settings that couldn't be preserved as before: * The emoticon '")' is no longer a supported special case * The emoticon ':)' in "example:)" is a false positive again When merged with #4258 (or the relevant cache bugfix), the affix and token_match properties should be modified to flush and reload all special cases to use the updated internal tokenization with the Matcher.
138 lines
4.3 KiB
Python
138 lines
4.3 KiB
Python
# coding: utf-8
|
|
from __future__ import unicode_literals
|
|
|
|
import pytest
|
|
from spacy.vocab import Vocab
|
|
from spacy.tokenizer import Tokenizer
|
|
from spacy.util import ensure_path
|
|
|
|
|
|
def test_tokenizer_handles_no_word(tokenizer):
|
|
tokens = tokenizer("")
|
|
assert len(tokens) == 0
|
|
|
|
|
|
@pytest.mark.parametrize("text", ["lorem"])
|
|
def test_tokenizer_handles_single_word(tokenizer, text):
|
|
tokens = tokenizer(text)
|
|
assert tokens[0].text == text
|
|
|
|
|
|
def test_tokenizer_handles_punct(tokenizer):
|
|
text = "Lorem, ipsum."
|
|
tokens = tokenizer(text)
|
|
assert len(tokens) == 4
|
|
assert tokens[0].text == "Lorem"
|
|
assert tokens[1].text == ","
|
|
assert tokens[2].text == "ipsum"
|
|
assert tokens[1].text != "Lorem"
|
|
|
|
|
|
def test_tokenizer_handles_punct_braces(tokenizer):
|
|
text = "Lorem, (ipsum)."
|
|
tokens = tokenizer(text)
|
|
assert len(tokens) == 6
|
|
|
|
|
|
def test_tokenizer_handles_digits(tokenizer):
|
|
exceptions = ["hu", "bn"]
|
|
text = "Lorem ipsum: 1984."
|
|
tokens = tokenizer(text)
|
|
|
|
if tokens[0].lang_ not in exceptions:
|
|
assert len(tokens) == 5
|
|
assert tokens[0].text == "Lorem"
|
|
assert tokens[3].text == "1984"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"text",
|
|
["google.com", "python.org", "spacy.io", "explosion.ai", "http://www.google.com"],
|
|
)
|
|
def test_tokenizer_keep_urls(tokenizer, text):
|
|
tokens = tokenizer(text)
|
|
assert len(tokens) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("text", ["NASDAQ:GOOG"])
|
|
def test_tokenizer_colons(tokenizer, text):
|
|
tokens = tokenizer(text)
|
|
assert len(tokens) == 3
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"text", ["hello123@example.com", "hi+there@gmail.it", "matt@explosion.ai"]
|
|
)
|
|
def test_tokenizer_keeps_email(tokenizer, text):
|
|
tokens = tokenizer(text)
|
|
assert len(tokens) == 1
|
|
|
|
|
|
def test_tokenizer_handles_long_text(tokenizer):
|
|
text = """Lorem ipsum dolor sit amet, consectetur adipiscing elit
|
|
|
|
Cras egestas orci non porttitor maximus.
|
|
Maecenas quis odio id dolor rhoncus dignissim. Curabitur sed velit at orci ultrices sagittis. Nulla commodo euismod arcu eget vulputate.
|
|
|
|
Phasellus tincidunt, augue quis porta finibus, massa sapien consectetur augue, non lacinia enim nibh eget ipsum. Vestibulum in bibendum mauris.
|
|
|
|
"Nullam porta fringilla enim, a dictum orci consequat in." Mauris nec malesuada justo."""
|
|
|
|
tokens = tokenizer(text)
|
|
assert len(tokens) > 5
|
|
|
|
|
|
@pytest.mark.parametrize("file_name", ["sun.txt"])
|
|
def test_tokenizer_handle_text_from_file(tokenizer, file_name):
|
|
loc = ensure_path(__file__).parent / file_name
|
|
text = loc.open("r", encoding="utf8").read()
|
|
assert len(text) != 0
|
|
tokens = tokenizer(text)
|
|
assert len(tokens) > 100
|
|
|
|
|
|
def test_tokenizer_suspected_freeing_strings(tokenizer):
|
|
text1 = "Lorem dolor sit amet, consectetur adipiscing elit."
|
|
text2 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."
|
|
tokens1 = tokenizer(text1)
|
|
tokens2 = tokenizer(text2)
|
|
assert tokens1[0].text == "Lorem"
|
|
assert tokens2[0].text == "Lorem"
|
|
|
|
|
|
@pytest.mark.parametrize("text,tokens", [("lorem", [{"orth": "lo"}, {"orth": "rem"}])])
|
|
def test_tokenizer_add_special_case(tokenizer, text, tokens):
|
|
tokenizer.add_special_case(text, tokens)
|
|
doc = tokenizer(text)
|
|
assert doc[0].text == tokens[0]["orth"]
|
|
assert doc[1].text == tokens[1]["orth"]
|
|
|
|
|
|
@pytest.mark.parametrize("text,tokens", [("lorem", [{"orth": "lo"}, {"orth": "re"}])])
|
|
def test_tokenizer_validate_special_case(tokenizer, text, tokens):
|
|
with pytest.raises(ValueError):
|
|
tokenizer.add_special_case(text, tokens)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"text,tokens", [("lorem", [{"orth": "lo", "tag": "NN"}, {"orth": "rem"}])]
|
|
)
|
|
def test_tokenizer_add_special_case_tag(text, tokens):
|
|
vocab = Vocab(tag_map={"NN": {"pos": "NOUN"}})
|
|
tokenizer = Tokenizer(vocab, {}, None, None, None)
|
|
tokenizer.add_special_case(text, tokens)
|
|
doc = tokenizer(text)
|
|
assert doc[0].text == tokens[0]["orth"]
|
|
assert doc[0].tag_ == tokens[0]["tag"]
|
|
assert doc[0].pos_ == "NOUN"
|
|
assert doc[1].text == tokens[1]["orth"]
|
|
|
|
|
|
def test_tokenizer_special_cases_with_affixes(tokenizer):
|
|
text = '(((_SPECIAL_ A/B, A/B-A/B")'
|
|
tokenizer.add_special_case("_SPECIAL_", [{"orth": "_SPECIAL_"}])
|
|
tokenizer.add_special_case("A/B", [{"orth": "A/B"}])
|
|
doc = tokenizer(text)
|
|
print([token.text for token in doc])
|
|
assert [token.text for token in doc] == ["(", "(", "(", "_SPECIAL_", "A/B", ",", "A/B", "-", "A/B", '"', ")"]
|