mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-15 06:09:01 +03:00
e2b70df012
* Use isort with Black profile * isort all the things * Fix import cycles as a result of import sorting * Add DOCBIN_ALL_ATTRS type definition * Add isort to requirements * Remove isort from build dependencies check * Typo
22 lines
733 B
Python
22 lines
733 B
Python
import pytest
|
|
|
|
TEST_CASES = [
|
|
(
|
|
"Adresa este str. Principală nr. 5.",
|
|
["Adresa", "este", "str.", "Principală", "nr.", "5", "."],
|
|
),
|
|
("Teste, etc.", ["Teste", ",", "etc."]),
|
|
("Lista, ș.a.m.d.", ["Lista", ",", "ș.a.m.d."]),
|
|
("Și d.p.d.v. al...", ["Și", "d.p.d.v.", "al", "..."]),
|
|
# number tests
|
|
("Clasa a 4-a.", ["Clasa", "a", "4-a", "."]),
|
|
("Al 12-lea ceas.", ["Al", "12-lea", "ceas", "."]),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("text,expected_tokens", TEST_CASES)
|
|
def test_ro_tokenizer_handles_testcases(ro_tokenizer, text, expected_tokens):
|
|
tokens = ro_tokenizer(text)
|
|
token_list = [token.text for token in tokens if not token.is_space]
|
|
assert expected_tokens == token_list
|