spaCy/spacy/tests/regression/test_issue801.py

20 lines
734 B
Python
Raw Normal View History

2017-02-02 17:33:52 +03:00
# coding: utf-8
from __future__ import unicode_literals
import pytest
@pytest.mark.parametrize('text,tokens', [
('"deserve,"--and', ['"', "deserve", ',"--', "and"]),
("exception;--exclusive", ["exception", ";--", "exclusive"]),
("day.--Is", ["day", ".--", "Is"]),
("refinement:--just", ["refinement", ":--", "just"]),
("memories?--To", ["memories", "?--", "To"]),
("Useful.=--Therefore", ["Useful", ".=--", "Therefore"]),
("=Hope.=--Pandora", ["=", "Hope", ".=--", "Pandora"])])
2017-02-02 17:33:52 +03:00
def test_issue801(en_tokenizer, text, tokens):
"""Test that special characters + hyphens are split correctly."""
doc = en_tokenizer(text)
assert len(doc) == len(tokens)
assert [t.text for t in doc] == tokens