spaCy/tests/test_infix.py

18 lines
322 B
Python
Raw Normal View History

2014-10-14 13:26:16 +04:00
from __future__ import unicode_literals
import pytest
from spacy.en import EN
def test_hyphen():
tokens = EN.tokenize('best-known')
assert len(tokens) == 3
def test_period():
tokens = EN.tokenize('best.Known')
assert len(tokens) == 3
tokens = EN.tokenize('zombo.com')
assert len(tokens) == 1