This commit is contained in:
Adriane Boyd 2024-04-09 06:19:58 +08:00 committed by GitHub
commit afa9707674
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 1 deletions

View File

@ -1,3 +1,4 @@
import re
from typing import Callable, Dict, List, Optional, Tuple
from thinc.api import Model
@ -7,6 +8,7 @@ from ...pipeline.lemmatizer import lemmatizer_score
from ...symbols import POS
from ...tokens import Token
from ...vocab import Vocab
from ..char_classes import COMBINING_DIACRITICS
PUNCT_RULES = {"«": '"', "»": '"'}
@ -51,9 +53,10 @@ class RussianLemmatizer(Lemmatizer):
super().__init__(
vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer
)
self._diacritics_re = re.compile(f"[{COMBINING_DIACRITICS}]")
def _pymorphy_lemmatize(self, token: Token) -> List[str]:
string = token.text
string = self._diacritics_re.sub("", token.text)
univ_pos = token.pos_
morphology = token.morph.to_dict()
if univ_pos == "PUNCT":

View File

@ -65,6 +65,8 @@ def test_ru_lemmatizer_works_with_different_pos_homonyms(
("гвоздики", "Gender=Masc", "гвоздик"),
("вина", "Gender=Fem", "вина"),
("вина", "Gender=Neut", "вино"),
("жену", "Gender=Fem", "жена"),
("жену́", "Gender=Fem", "жена"),
],
)
def test_ru_lemmatizer_works_with_noun_homonyms(ru_lemmatizer, text, morph, lemma):