explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-04-22 01:51:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix token len

Browse Source
This commit is contained in:
sani 2023-05-08 00:54:12 +08:00
parent 86d78d9730
commit 82ce3cc3f0
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/tests/lang/ms/test_prefix_suffix_infix.py
View File

@ -61,7 +61,7 @@ def test_ms_tokenizer_splits_uneven_wrap_interact(id_tokenizer, text):
@pytest.mark.parametrize(
"text,length", [("kerana", 1), ("Mahathir-Anwar", 3), ("Tun Dr. Ismail-Abdul Rahman", 4)]
"text,length", [("kerana", 1), ("Mahathir-Anwar", 3), ("Tun Dr. Ismail-Abdul Rahman", 6)]
)
def test_my_tokenizer_splits_hyphens(ms_tokenizer, text, length):
tokens = ms_tokenizer(text)