From 460b4c3207744f78991efdb5a4e939280e4d35e1 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 26 Jul 2015 17:30:34 +0200 Subject: [PATCH] * Add more infix tests --- tests/tokenizer/test_infix.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/tokenizer/test_infix.py b/tests/tokenizer/test_infix.py index 2e9fbe1e4..33648a752 100644 --- a/tests/tokenizer/test_infix.py +++ b/tests/tokenizer/test_infix.py @@ -12,3 +12,19 @@ def test_period(en_tokenizer): assert len(tokens) == 3 tokens = en_tokenizer('zombo.com') assert len(tokens) == 1 + + +def test_ellipsis(en_tokenizer): + tokens = en_tokenizer('best...Known') + assert len(tokens) == 3 + tokens = en_tokenizer('best...known') + assert len(tokens) == 3 + + +def test_email(en_tokenizer): + tokens = en_tokenizer('hello@example.com') + assert len(tokens) == 3 + tokens = en_tokenizer('hi+there@gmail.it') + assert len(tokens) == 3 + +