explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-14 18:40:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add test for #4248, clean up test

Browse Source
This commit is contained in:
Adriane Boyd 2019-09-20 09:20:57 +02:00
parent e7e7c942c7
commit e74963acd4
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
spacy/tests/tokenizer/test_tokenizer.py
View File

@ -133,5 +133,11 @@ def test_tokenizer_special_cases_with_affixes(tokenizer):
tokenizer.add_special_case("_SPECIAL_", [{"orth": "_SPECIAL_"}])
tokenizer.add_special_case("A/B", [{"orth": "A/B"}])
doc = tokenizer(text)
print([token.text for token in doc])
assert [token.text for token in doc] == ["(", "(", "(", "_SPECIAL_", "A/B", ",", "A/B", "-", "A/B", '"', ")"]
def test_tokenizer_special_cases_with_period(tokenizer):
text = "_SPECIAL_."
tokenizer.add_special_case("_SPECIAL_", [{"orth": "_SPECIAL_"}])
doc = tokenizer(text)
assert [token.text for token in doc] == ["_SPECIAL_", "."]