spaCy/spacy/tests/lang/nl/test_lemmatizer.py

144 lines
3.7 KiB
Python
Raw Normal View History

# coding: utf-8
from __future__ import unicode_literals
import pytest
# Calling the Lemmatizer directly
# Imitates behavior of:
# Tagger.set_annotations()
# -> vocab.morphology.assign_tag_id()
# -> vocab.morphology.assign_tag_id()
# -> Token.tag.__set__
# -> vocab.morphology.assign_tag(...)
# -> ... -> Morphology.assign_tag(...)
# -> self.lemmatize(analysis.tag.pos, token.lex.orth,
noun_irreg_lemmatization_cases = [
("volkeren", "volk"),
("vaatje", "vat"),
("verboden", "verbod"),
("ijsje", "ijsje"),
("slagen", "slag"),
("verdragen", "verdrag"),
("verloven", "verlof"),
("gebeden", "gebed"),
("gaten", "gat"),
("staven", "staf"),
("aquariums", "aquarium"),
("podia", "podium"),
("holen", "hol"),
("lammeren", "lam"),
("bevelen", "bevel"),
("wegen", "weg"),
("moeilijkheden", "moeilijkheid"),
("aanwezigheden", "aanwezigheid"),
("goden", "god"),
("loten", "lot"),
("kaarsen", "kaars"),
("leden", "lid"),
("glaasje", "glas"),
("eieren", "ei"),
("vatten", "vat"),
("kalveren", "kalf"),
("padden", "pad"),
("smeden", "smid"),
("genen", "gen"),
("beenderen", "been"),
]
verb_irreg_lemmatization_cases = [
("liep", "lopen"),
("hief", "heffen"),
("begon", "beginnen"),
("sla", "slaan"),
("aangekomen", "aankomen"),
("sproot", "spruiten"),
("waart", "zijn"),
("snoof", "snuiven"),
("spoot", "spuiten"),
("ontbeet", "ontbijten"),
("gehouwen", "houwen"),
("afgewassen", "afwassen"),
("deed", "doen"),
("schoven", "schuiven"),
("gelogen", "liegen"),
("woog", "wegen"),
("gebraden", "braden"),
("smolten", "smelten"),
("riep", "roepen"),
("aangedaan", "aandoen"),
("vermeden", "vermijden"),
("stootten", "stoten"),
("ging", "gaan"),
("geschoren", "scheren"),
("gesponnen", "spinnen"),
("reden", "rijden"),
("zochten", "zoeken"),
("leed", "lijden"),
("verzonnen", "verzinnen"),
]
@pytest.mark.parametrize("text,lemma", noun_irreg_lemmatization_cases)
def test_nl_lemmatizer_noun_lemmas_irreg(nl_lemmatizer, text, lemma):
pos = "noun"
lemmas_pred = nl_lemmatizer(text, pos)
assert lemma == sorted(lemmas_pred)[0]
@pytest.mark.parametrize("text,lemma", verb_irreg_lemmatization_cases)
def test_nl_lemmatizer_verb_lemmas_irreg(nl_lemmatizer, text, lemma):
pos = "verb"
lemmas_pred = nl_lemmatizer(text, pos)
assert lemma == sorted(lemmas_pred)[0]
@pytest.mark.skip
@pytest.mark.parametrize("text,lemma", [])
def test_nl_lemmatizer_verb_lemmas_reg(nl_lemmatizer, text, lemma):
# TODO: add test
pass
@pytest.mark.skip
@pytest.mark.parametrize("text,lemma", [])
def test_nl_lemmatizer_adjective_lemmas(nl_lemmatizer, text, lemma):
# TODO: add test
pass
@pytest.mark.skip
@pytest.mark.parametrize("text,lemma", [])
def test_nl_lemmatizer_determiner_lemmas(nl_lemmatizer, text, lemma):
# TODO: add test
pass
@pytest.mark.skip
@pytest.mark.parametrize("text,lemma", [])
def test_nl_lemmatizer_adverb_lemmas(nl_lemmatizer, text, lemma):
# TODO: add test
pass
@pytest.mark.parametrize("text,lemma", [])
def test_nl_lemmatizer_pronoun_lemmas(nl_lemmatizer, text, lemma):
# TODO: add test
pass
# Using the lemma lookup table only
@pytest.mark.parametrize("text,lemma", noun_irreg_lemmatization_cases)
def test_nl_lemmatizer_lookup_noun(nl_lemmatizer, text, lemma):
lemma_pred = nl_lemmatizer.lookup(text)
assert lemma_pred in (lemma, text)
@pytest.mark.parametrize("text,lemma", verb_irreg_lemmatization_cases)
def test_nl_lemmatizer_lookup_verb(nl_lemmatizer, text, lemma):
lemma_pred = nl_lemmatizer.lookup(text)
assert lemma_pred in (lemma, text)