mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-13 13:17:06 +03:00
25 lines
434 B
Python
25 lines
434 B
Python
|
from __future__ import unicode_literals
|
||
|
|
|
||
|
|
from spacy.en import unhash
|
||
|
|
from spacy import lex_of
|
||
|
|
from spacy import en
|
||
|
|
from spacy.util import utf8open
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
import os
|
||
|
|
from os import path
|
||
|
|
|
||
|
|
|
||
|
|
HERE = path.dirname(__file__)
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def sun_txt():
|
||
|
|
loc = path.join(HERE, 'sun.txt')
|
||
|
|
return utf8open(loc).read()
|
||
|
|
|
||
|
|
|
||
|
|
def test_tokenize(sun_txt):
|
||
|
|
assert len(sun_txt) != 0
|
||
|
|
tokens = en.tokenize(sun_txt)
|
||
|
|
assert True
|