From 5fa8f4faca966fe58c5c8de861900724c7659f25 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 22 Aug 2022 11:27:14 +0200 Subject: [PATCH] Switch ru and uk lemmatizers to pymorphy3 (#11345) * Switch ru and uk lemmatizers to pymorphy3 * Switch to pymorphy3 in tests --- spacy/lang/ru/__init__.py | 2 +- spacy/lang/ru/lemmatizer.py | 15 ++++++++++++++- spacy/lang/uk/__init__.py | 2 +- spacy/lang/uk/lemmatizer.py | 13 ++++++++++++- spacy/tests/conftest.py | 10 +++++----- website/docs/api/lemmatizer.md | 6 +++--- website/meta/languages.json | 8 ++++---- 7 files changed, 40 insertions(+), 16 deletions(-) diff --git a/spacy/lang/ru/__init__.py b/spacy/lang/ru/__init__.py index c118c26ff..7d17628c4 100644 --- a/spacy/lang/ru/__init__.py +++ b/spacy/lang/ru/__init__.py @@ -28,7 +28,7 @@ class Russian(Language): assigns=["token.lemma"], default_config={ "model": None, - "mode": "pymorphy2", + "mode": "pymorphy3", "overwrite": False, "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, }, diff --git a/spacy/lang/ru/lemmatizer.py b/spacy/lang/ru/lemmatizer.py index 85180b1e4..720d3a8cb 100644 --- a/spacy/lang/ru/lemmatizer.py +++ b/spacy/lang/ru/lemmatizer.py @@ -19,7 +19,7 @@ class RussianLemmatizer(Lemmatizer): model: Optional[Model], name: str = "lemmatizer", *, - mode: str = "pymorphy2", + mode: str = "pymorphy3", overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: @@ -33,6 +33,16 @@ class RussianLemmatizer(Lemmatizer): ) from None if getattr(self, "_morph", None) is None: self._morph = MorphAnalyzer() + elif mode == "pymorphy3": + try: + from pymorphy3 import MorphAnalyzer + except ImportError: + raise ImportError( + "The Russian lemmatizer mode 'pymorphy3' requires the " + "pymorphy3 library. Install it with: pip install pymorphy3" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer() super().__init__( vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer ) @@ -104,6 +114,9 @@ class RussianLemmatizer(Lemmatizer): return [analyses[0].normal_form] return [string] + def pymorphy3_lemmatize(self, token: Token) -> List[str]: + return self.pymorphy2_lemmatize(token) + def oc2ud(oc_tag: str) -> Tuple[str, Dict[str, str]]: gram_map = { diff --git a/spacy/lang/uk/__init__.py b/spacy/lang/uk/__init__.py index 737243b66..bfea9ff69 100644 --- a/spacy/lang/uk/__init__.py +++ b/spacy/lang/uk/__init__.py @@ -29,7 +29,7 @@ class Ukrainian(Language): assigns=["token.lemma"], default_config={ "model": None, - "mode": "pymorphy2", + "mode": "pymorphy3", "overwrite": False, "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, }, diff --git a/spacy/lang/uk/lemmatizer.py b/spacy/lang/uk/lemmatizer.py index a8bc56057..97ee80479 100644 --- a/spacy/lang/uk/lemmatizer.py +++ b/spacy/lang/uk/lemmatizer.py @@ -14,7 +14,7 @@ class UkrainianLemmatizer(RussianLemmatizer): model: Optional[Model], name: str = "lemmatizer", *, - mode: str = "pymorphy2", + mode: str = "pymorphy3", overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: @@ -29,6 +29,17 @@ class UkrainianLemmatizer(RussianLemmatizer): ) from None if getattr(self, "_morph", None) is None: self._morph = MorphAnalyzer(lang="uk") + elif mode == "pymorphy3": + try: + from pymorphy3 import MorphAnalyzer + except ImportError: + raise ImportError( + "The Ukrainian lemmatizer mode 'pymorphy3' requires the " + "pymorphy3 library and dictionaries. Install them with: " + "pip install pymorphy3 pymorphy3-dicts-uk" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer(lang="uk") super().__init__( vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer ) diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index eb643ec2f..76de8f373 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -323,13 +323,13 @@ def ro_tokenizer(): @pytest.fixture(scope="session") def ru_tokenizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("ru")().tokenizer @pytest.fixture def ru_lemmatizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("ru")().add_pipe("lemmatizer") @@ -401,14 +401,14 @@ def ky_tokenizer(): @pytest.fixture(scope="session") def uk_tokenizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("uk")().tokenizer @pytest.fixture def uk_lemmatizer(): - pytest.importorskip("pymorphy2") - pytest.importorskip("pymorphy2_dicts_uk") + pytest.importorskip("pymorphy3") + pytest.importorskip("pymorphy3_dicts_uk") return get_lang_class("uk")().add_pipe("lemmatizer") diff --git a/website/docs/api/lemmatizer.md b/website/docs/api/lemmatizer.md index 422f34040..905096338 100644 --- a/website/docs/api/lemmatizer.md +++ b/website/docs/api/lemmatizer.md @@ -70,7 +70,7 @@ lemmatizer is available. The lemmatizer modes `rule` and `pos_lookup` require [`token.pos`](/api/token) from a previous pipeline component (see example pipeline configurations in the [pretrained pipeline design details](/models#design-cnn)) or rely on third-party -libraries (`pymorphy2`). +libraries (`pymorphy3`). | Language | Default Mode | | -------- | ------------ | @@ -86,9 +86,9 @@ libraries (`pymorphy2`). | `nb` | `rule` | | `nl` | `rule` | | `pl` | `pos_lookup` | -| `ru` | `pymorphy2` | +| `ru` | `pymorphy3` | | `sv` | `rule` | -| `uk` | `pymorphy2` | +| `uk` | `pymorphy3` | ```python %%GITHUB_SPACY/spacy/pipeline/lemmatizer.py diff --git a/website/meta/languages.json b/website/meta/languages.json index 6bc2309ed..5305ceffc 100644 --- a/website/meta/languages.json +++ b/website/meta/languages.json @@ -369,8 +369,8 @@ "has_examples": true, "dependencies": [ { - "name": "pymorphy2", - "url": "https://github.com/kmike/pymorphy2" + "name": "pymorphy3", + "url": "https://github.com/no-plagiarism/pymorphy3" } ], "models": [ @@ -469,8 +469,8 @@ "has_examples": true, "dependencies": [ { - "name": "pymorphy2", - "url": "https://github.com/kmike/pymorphy2" + "name": "pymorphy3", + "url": "https://github.com/no-plagiarism/pymorphy3" } ] },