spaCy/tests/test_contractions.py

58 lines
1.2 KiB
Python
Raw Normal View History

from __future__ import unicode_literals
2014-12-21 12:41:13 +03:00
import pytest
2014-12-21 12:41:13 +03:00
from spacy.en import English
2014-12-21 12:41:13 +03:00
@pytest.fixture
def EN():
2014-12-30 13:34:09 +03:00
return English()
2014-12-21 12:41:13 +03:00
def test_possess(EN):
tokens = EN("Mike's")
2015-01-05 05:15:46 +03:00
assert EN.vocab.strings[tokens[0].sic] == b"Mike"
assert EN.vocab.strings[tokens[1].sic] == b"'s"
assert len(tokens) == 2
2014-12-21 12:41:13 +03:00
def test_apostrophe(EN):
tokens = EN("schools'")
assert len(tokens) == 2
2015-01-05 09:54:54 +03:00
assert tokens[1].string == b"'"
assert tokens[0].string == b"schools"
2014-12-21 12:41:13 +03:00
def test_LL(EN):
tokens = EN("we'll")
assert len(tokens) == 2
2015-01-05 09:54:54 +03:00
assert tokens[1].string == b"'ll"
assert tokens[1].lemma == b"will"
assert tokens[0].string == b"we"
2014-12-21 12:41:13 +03:00
def test_aint(EN):
tokens = EN("ain't")
assert len(tokens) == 2
2015-01-05 09:54:54 +03:00
assert tokens[0].string == b"ai"
assert tokens[0].lemma == b"be"
assert tokens[1].string == b"n't"
assert tokens[1].lemma == b"not"
2014-12-21 12:41:13 +03:00
def test_capitalized(EN):
tokens = EN("can't")
assert len(tokens) == 2
2014-12-21 12:41:13 +03:00
tokens = EN("Can't")
assert len(tokens) == 2
2014-12-21 12:41:13 +03:00
tokens = EN("Ain't")
assert len(tokens) == 2
2015-01-05 09:54:54 +03:00
assert tokens[0].string == b"Ai"
assert tokens[0].lemma == b"be"
2014-12-07 14:08:04 +03:00
2014-12-21 12:41:13 +03:00
def test_punct(EN):
tokens = EN("We've")
2014-12-07 14:08:04 +03:00
assert len(tokens) == 2
2014-12-21 12:41:13 +03:00
tokens = EN("``We've")
2014-12-07 14:08:04 +03:00
assert len(tokens) == 3