1
1
mirror of https://github.com/explosion/spaCy.git synced 2025-02-22 22:40:32 +03:00
spaCy/spacy/tests/lang/tl/test_indices.py
Lj Miranda f1bc655a38
Add initial Tagalog (tl) tests ()
* Add tl_tokenizer to test fixtures

* Add tagalog tests
2021-11-02 08:35:49 +01:00

9 lines
257 B
Python

def test_tl_simple_punct(tl_tokenizer):
text = "Sige, punta ka dito"
tokens = tl_tokenizer(text)
assert tokens[0].idx == 0
assert tokens[1].idx == 4
assert tokens[2].idx == 6
assert tokens[3].idx == 12
assert tokens[4].idx == 15