2017-01-24 12:55:02 +03:00
|
|
|
|
# coding: utf-8
|
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
|
|
|
|
import pytest
|
2018-07-24 21:09:06 +03:00
|
|
|
|
|
2017-01-24 12:55:02 +03:00
|
|
|
|
|
2018-11-27 03:09:36 +03:00
|
|
|
|
@pytest.mark.parametrize(
|
2018-12-18 15:48:10 +03:00
|
|
|
|
"text",
|
|
|
|
|
[
|
2019-02-01 10:05:22 +03:00
|
|
|
|
u"aujourd'hui",
|
|
|
|
|
u"Aujourd'hui",
|
|
|
|
|
u"prud'hommes",
|
|
|
|
|
u"prud’hommal",
|
|
|
|
|
u"audio-numérique",
|
|
|
|
|
u"Audio-numérique",
|
|
|
|
|
u"entr'amis",
|
|
|
|
|
u"entr'abat",
|
|
|
|
|
u"rentr'ouvertes",
|
|
|
|
|
u"grand'hamien",
|
|
|
|
|
u"Châteauneuf-la-Forêt",
|
|
|
|
|
u"Château-Guibert",
|
|
|
|
|
u"11-septembre",
|
|
|
|
|
u"11-Septembre",
|
|
|
|
|
u"refox-trottâmes",
|
|
|
|
|
u"K-POP",
|
|
|
|
|
u"K-Pop",
|
|
|
|
|
u"K-pop",
|
|
|
|
|
u"z'yeutes",
|
|
|
|
|
u"black-outeront",
|
|
|
|
|
u"états-unienne",
|
|
|
|
|
u"courtes-pattes",
|
|
|
|
|
u"court-pattes",
|
|
|
|
|
u"saut-de-ski",
|
|
|
|
|
u"Écourt-Saint-Quentin",
|
|
|
|
|
u"Bout-de-l'Îlien",
|
|
|
|
|
u"pet-en-l'air",
|
2018-12-18 15:48:10 +03:00
|
|
|
|
],
|
2018-11-27 03:09:36 +03:00
|
|
|
|
)
|
2018-07-25 00:38:44 +03:00
|
|
|
|
def test_fr_tokenizer_infix_exceptions(fr_tokenizer, text):
|
2017-01-24 12:55:02 +03:00
|
|
|
|
tokens = fr_tokenizer(text)
|
|
|
|
|
assert len(tokens) == 1
|
|
|
|
|
|
|
|
|
|
|
2018-11-27 03:09:36 +03:00
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
|
"text,lemma",
|
|
|
|
|
[
|
|
|
|
|
("janv.", "janvier"),
|
|
|
|
|
("juill.", "juillet"),
|
|
|
|
|
("Dr.", "docteur"),
|
|
|
|
|
("av.", "avant"),
|
|
|
|
|
("sept.", "septembre"),
|
|
|
|
|
],
|
|
|
|
|
)
|
2018-07-25 00:38:44 +03:00
|
|
|
|
def test_fr_tokenizer_handles_abbr(fr_tokenizer, text, lemma):
|
2017-01-24 12:55:02 +03:00
|
|
|
|
tokens = fr_tokenizer(text)
|
|
|
|
|
assert len(tokens) == 1
|
|
|
|
|
assert tokens[0].lemma_ == lemma
|
|
|
|
|
|
|
|
|
|
|
2018-07-25 00:38:44 +03:00
|
|
|
|
def test_fr_tokenizer_handles_exc_in_text(fr_tokenizer):
|
2017-01-24 12:55:02 +03:00
|
|
|
|
text = "Je suis allé au mois de janv. aux prud’hommes."
|
|
|
|
|
tokens = fr_tokenizer(text)
|
|
|
|
|
assert len(tokens) == 10
|
|
|
|
|
assert tokens[6].text == "janv."
|
|
|
|
|
assert tokens[6].lemma_ == "janvier"
|
|
|
|
|
assert tokens[8].text == "prud’hommes"
|
2017-02-10 15:17:05 +03:00
|
|
|
|
|
|
|
|
|
|
2018-07-25 00:38:44 +03:00
|
|
|
|
def test_fr_tokenizer_handles_exc_in_text_2(fr_tokenizer):
|
2017-02-10 15:17:05 +03:00
|
|
|
|
text = "Cette après-midi, je suis allé dans un restaurant italo-mexicain."
|
|
|
|
|
tokens = fr_tokenizer(text)
|
|
|
|
|
assert len(tokens) == 11
|
|
|
|
|
assert tokens[1].text == "après-midi"
|
2017-04-27 11:59:38 +03:00
|
|
|
|
assert tokens[9].text == "italo-mexicain"
|
|
|
|
|
|
2018-07-25 00:38:44 +03:00
|
|
|
|
|
|
|
|
|
def test_fr_tokenizer_handles_title(fr_tokenizer):
|
2017-04-27 11:59:38 +03:00
|
|
|
|
text = "N'est-ce pas génial?"
|
|
|
|
|
tokens = fr_tokenizer(text)
|
|
|
|
|
assert len(tokens) == 6
|
|
|
|
|
assert tokens[0].text == "N'"
|
|
|
|
|
assert tokens[0].lemma_ == "ne"
|
2017-04-27 12:52:14 +03:00
|
|
|
|
assert tokens[1].text == "est"
|
|
|
|
|
assert tokens[1].lemma_ == "être"
|
2017-04-27 11:59:38 +03:00
|
|
|
|
assert tokens[2].text == "-ce"
|
|
|
|
|
assert tokens[2].lemma_ == "ce"
|
|
|
|
|
|
2018-07-25 00:38:44 +03:00
|
|
|
|
|
2018-11-27 03:09:36 +03:00
|
|
|
|
@pytest.mark.xfail
|
2018-07-25 00:38:44 +03:00
|
|
|
|
def test_fr_tokenizer_handles_title_2(fr_tokenizer):
|
2017-04-27 11:59:38 +03:00
|
|
|
|
text = "Est-ce pas génial?"
|
|
|
|
|
tokens = fr_tokenizer(text)
|
|
|
|
|
assert len(tokens) == 6
|
|
|
|
|
assert tokens[0].text == "Est"
|
|
|
|
|
assert tokens[0].lemma_ == "être"
|
|
|
|
|
|
2018-07-25 00:38:44 +03:00
|
|
|
|
|
2018-11-27 03:09:36 +03:00
|
|
|
|
def test_fr_tokenizer_handles_title_3(fr_tokenizer):
|
2017-04-27 11:59:38 +03:00
|
|
|
|
text = "Qu'est-ce que tu fais?"
|
|
|
|
|
tokens = fr_tokenizer(text)
|
|
|
|
|
assert len(tokens) == 7
|
|
|
|
|
assert tokens[0].text == "Qu'"
|
2018-07-25 00:38:44 +03:00
|
|
|
|
assert tokens[0].lemma_ == "que"
|