spaCy/tests/test_pre_punct.py

54 lines
1.1 KiB
Python
Raw Normal View History

from __future__ import unicode_literals
from spacy.en import English
import pytest
@pytest.fixture
def open_puncts():
return ['(', '[', '{', '*']
@pytest.fixture
def EN():
2014-12-30 13:34:09 +03:00
return English()
def test_open(open_puncts, EN):
word_str = 'Hello'
for p in open_puncts:
string = p + word_str
tokens = EN(string)
assert len(tokens) == 2
assert tokens[0].orth_ == p
assert tokens[1].orth_ == word_str
def test_two_different_open(open_puncts, EN):
word_str = 'Hello'
for p in open_puncts:
string = p + "`" + word_str
tokens = EN(string)
assert len(tokens) == 3
assert tokens[0].orth_ == p
assert tokens[1].orth_ == "`"
assert tokens[2].orth_ == word_str
def test_three_same_open(open_puncts, EN):
word_str = 'Hello'
for p in open_puncts:
string = p + p + p + word_str
tokens = EN(string)
assert len(tokens) == 4
assert tokens[0].orth_ == p
assert tokens[3].orth_ == word_str
2014-07-08 01:24:20 +04:00
def test_open_appostrophe(EN):
2014-07-08 01:24:20 +04:00
string = "'The"
tokens = EN(string)
2014-07-08 01:24:20 +04:00
assert len(tokens) == 2
assert tokens[0].orth_ == "'"