mirror of
https://github.com/explosion/spaCy.git
synced 2025-11-04 01:48:04 +03:00
db55577c45
* Remove unicode declarations * Remove Python 3.5 and 2.7 from CI * Don't require pathlib * Replace compat helpers * Remove OrderedDict * Use f-strings * Set Cython compiler language level * Fix typo * Re-add OrderedDict for Table * Update setup.cfg * Revert CONTRIBUTING.md * Revert lookups.md * Revert top-level.md * Small adjustments and docs [ci skip]
23 lines
734 B
Python
23 lines
734 B
Python
import pytest
|
|
|
|
|
|
TEST_CASES = [
|
|
(
|
|
"Adresa este str. Principală nr. 5.",
|
|
["Adresa", "este", "str.", "Principală", "nr.", "5", "."],
|
|
),
|
|
("Teste, etc.", ["Teste", ",", "etc."]),
|
|
("Lista, ș.a.m.d.", ["Lista", ",", "ș.a.m.d."]),
|
|
("Și d.p.d.v. al...", ["Și", "d.p.d.v.", "al", "..."]),
|
|
# number tests
|
|
("Clasa a 4-a.", ["Clasa", "a", "4-a", "."]),
|
|
("Al 12-lea ceas.", ["Al", "12-lea", "ceas", "."]),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("text,expected_tokens", TEST_CASES)
|
|
def test_ro_tokenizer_handles_testcases(ro_tokenizer, text, expected_tokens):
|
|
tokens = ro_tokenizer(text)
|
|
token_list = [token.text for token in tokens if not token.is_space]
|
|
assert expected_tokens == token_list
|