explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-25 00:34:20 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Add more infix tests

Browse Source
This commit is contained in:
Matthew Honnibal 2015-07-26 17:30:34 +02:00
parent bd608559bc
commit 460b4c3207
1 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tests/tokenizer/test_infix.py
View File

@ -12,3 +12,19 @@ def test_period(en_tokenizer):
assert len(tokens) == 3
tokens = en_tokenizer('zombo.com')
assert len(tokens) == 1
def test_ellipsis(en_tokenizer):
tokens = en_tokenizer('best...Known')
assert len(tokens) == 3
tokens = en_tokenizer('best...known')
assert len(tokens) == 3
def test_email(en_tokenizer):
tokens = en_tokenizer('hello@example.com')
assert len(tokens) == 3
tokens = en_tokenizer('hi+there@gmail.it')
assert len(tokens) == 3