mirror of
https://github.com/explosion/spaCy.git
synced 2025-01-12 02:06:31 +03:00
19 lines
514 B
Python
19 lines
514 B
Python
# coding: utf-8
|
|
"""Test that tokenizer exceptions are parsed correctly."""
|
|
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.mark.parametrize('text,norms,lemmas', [("ім.", ["імені"], ["ім'я"]),
|
|
("проф.", ["професор"], ["професор"])])
|
|
def test_uk_tokenizer_abbrev_exceptions(uk_tokenizer, text, norms, lemmas):
|
|
tokens = uk_tokenizer(text)
|
|
assert len(tokens) == 1
|
|
assert [token.norm_ for token in tokens] == norms
|
|
|
|
|
|
|