spaCy/tests/test_tokens_from_list.py

17 lines
305 B
Python
Raw Normal View History

2014-11-12 15:24:32 +03:00
from __future__ import unicode_literals
import pytest
2014-11-12 15:24:32 +03:00
from spacy.en import English
2014-11-12 15:24:32 +03:00
@pytest.fixture
def EN():
return English()
def test1(EN):
2014-11-12 15:24:32 +03:00
words = ['JAPAN', 'GET', 'LUCKY']
tokens = EN.tokenizer.tokens_from_list(words)
2014-11-12 15:24:32 +03:00
assert len(tokens) == 3
assert tokens[0].string == 'JAPAN'