From e74963acd4ffe2c2660c8366f34691315e27a617 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 20 Sep 2019 09:20:57 +0200 Subject: [PATCH] Add test for #4248, clean up test --- spacy/tests/tokenizer/test_tokenizer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spacy/tests/tokenizer/test_tokenizer.py b/spacy/tests/tokenizer/test_tokenizer.py index 2c8ef73c4..5ac681c5e 100644 --- a/spacy/tests/tokenizer/test_tokenizer.py +++ b/spacy/tests/tokenizer/test_tokenizer.py @@ -133,5 +133,11 @@ def test_tokenizer_special_cases_with_affixes(tokenizer): tokenizer.add_special_case("_SPECIAL_", [{"orth": "_SPECIAL_"}]) tokenizer.add_special_case("A/B", [{"orth": "A/B"}]) doc = tokenizer(text) - print([token.text for token in doc]) assert [token.text for token in doc] == ["(", "(", "(", "_SPECIAL_", "A/B", ",", "A/B", "-", "A/B", '"', ")"] + + +def test_tokenizer_special_cases_with_period(tokenizer): + text = "_SPECIAL_." + tokenizer.add_special_case("_SPECIAL_", [{"orth": "_SPECIAL_"}]) + doc = tokenizer(text) + assert [token.text for token in doc] == ["_SPECIAL_", "."]