2020-07-22 14:42:59 +03:00
|
|
|
from typing import Optional, Callable, List, Dict
|
|
|
|
|
|
|
|
from .lookups import Lookups
|
2020-05-20 10:56:56 +03:00
|
|
|
from .parts_of_speech import NAMES as UPOS_NAMES
|
2020-07-22 16:59:37 +03:00
|
|
|
from .util import registry
|
2020-07-22 14:42:59 +03:00
|
|
|
|
|
|
|
|
|
|
|
@registry.lemmatizers("spacy.Lemmatizer.v1")
|
2020-07-22 16:59:37 +03:00
|
|
|
def create_lemmatizer(data: Dict[str, str] = {}) -> "Lemmatizer":
|
|
|
|
return Lemmatizer(data=data)
|
2015-08-27 10:16:11 +03:00
|
|
|
|
2014-12-23 07:16:57 +03:00
|
|
|
|
2020-07-12 15:03:23 +03:00
|
|
|
class Lemmatizer:
|
2019-03-08 13:42:26 +03:00
|
|
|
"""
|
|
|
|
The Lemmatizer supports simple part-of-speech-sensitive suffix rules and
|
|
|
|
lookup tables.
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/lemmatizer
|
|
|
|
"""
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
lookups: Optional[Lookups] = None,
|
2020-07-22 16:59:37 +03:00
|
|
|
data: Dict[str, dict] = {},
|
2020-07-22 14:42:59 +03:00
|
|
|
is_base_form: Optional[Callable] = None,
|
|
|
|
) -> None:
|
2019-10-01 22:36:04 +03:00
|
|
|
"""Initialize a Lemmatizer.
|
2014-12-23 07:16:57 +03:00
|
|
|
|
2019-10-01 22:36:04 +03:00
|
|
|
lookups (Lookups): The lookups object containing the (optional) tables
|
|
|
|
"lemma_rules", "lemma_index", "lemma_exc" and "lemma_lookup".
|
|
|
|
RETURNS (Lemmatizer): The newly constructed object.
|
|
|
|
"""
|
2020-07-22 14:42:59 +03:00
|
|
|
self.lookups = lookups if lookups is not None else Lookups()
|
2020-07-22 16:59:37 +03:00
|
|
|
for name, table in data.items():
|
|
|
|
if table is not None:
|
|
|
|
self.lookups.add_table(name, table)
|
2020-06-29 15:16:57 +03:00
|
|
|
self.is_base_form = is_base_form
|
2015-08-25 16:46:19 +03:00
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def __call__(
|
|
|
|
self, string: str, univ_pos: str, morphology: Optional[dict] = None
|
|
|
|
) -> List[str]:
|
2019-10-01 22:36:04 +03:00
|
|
|
"""Lemmatize a string.
|
|
|
|
|
2020-05-24 18:20:58 +03:00
|
|
|
string (str): The string to lemmatize, e.g. the token text.
|
|
|
|
univ_pos (str / int): The token's universal part-of-speech tag.
|
2019-10-01 22:36:04 +03:00
|
|
|
morphology (dict): The token's morphological features following the
|
|
|
|
Universal Dependencies scheme.
|
|
|
|
RETURNS (list): The available lemmas for the string.
|
|
|
|
"""
|
|
|
|
lookup_table = self.lookups.get_table("lemma_lookup", {})
|
|
|
|
if "lemma_rules" not in self.lookups:
|
|
|
|
return [lookup_table.get(string, string)]
|
2020-05-20 10:56:56 +03:00
|
|
|
if isinstance(univ_pos, int):
|
|
|
|
univ_pos = UPOS_NAMES.get(univ_pos, "X")
|
|
|
|
univ_pos = univ_pos.lower()
|
|
|
|
if univ_pos in ("", "eol", "space"):
|
2018-03-27 20:23:02 +03:00
|
|
|
return [string.lower()]
|
2016-09-27 14:52:11 +03:00
|
|
|
# See Issue #435 for example of where this logic is requied.
|
2020-06-29 15:16:57 +03:00
|
|
|
if callable(self.is_base_form) and self.is_base_form(univ_pos, morphology):
|
2018-03-27 20:23:02 +03:00
|
|
|
return [string.lower()]
|
2019-10-01 22:36:04 +03:00
|
|
|
index_table = self.lookups.get_table("lemma_index", {})
|
|
|
|
exc_table = self.lookups.get_table("lemma_exc", {})
|
|
|
|
rules_table = self.lookups.get_table("lemma_rules", {})
|
2020-06-20 15:15:04 +03:00
|
|
|
if not any(
|
|
|
|
(
|
|
|
|
index_table.get(univ_pos),
|
|
|
|
exc_table.get(univ_pos),
|
|
|
|
rules_table.get(univ_pos),
|
|
|
|
)
|
|
|
|
):
|
2020-05-20 16:35:08 +03:00
|
|
|
if univ_pos == "propn":
|
|
|
|
return [string]
|
|
|
|
else:
|
|
|
|
return [string.lower()]
|
2019-10-01 22:36:04 +03:00
|
|
|
lemmas = self.lemmatize(
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
string,
|
2019-10-01 22:36:04 +03:00
|
|
|
index_table.get(univ_pos, {}),
|
|
|
|
exc_table.get(univ_pos, {}),
|
|
|
|
rules_table.get(univ_pos, []),
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
)
|
2015-09-08 16:38:03 +03:00
|
|
|
return lemmas
|
2014-12-23 07:16:57 +03:00
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def noun(self, string: str, morphology: Optional[dict] = None) -> List[str]:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
return self(string, "noun", morphology)
|
2014-12-23 07:16:57 +03:00
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def verb(self, string: str, morphology: Optional[dict] = None) -> List[str]:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
return self(string, "verb", morphology)
|
2014-12-23 07:16:57 +03:00
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def adj(self, string: str, morphology: Optional[dict] = None) -> List[str]:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
return self(string, "adj", morphology)
|
2016-09-27 14:52:11 +03:00
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def det(self, string: str, morphology: Optional[dict] = None) -> List[str]:
|
2019-10-01 22:36:04 +03:00
|
|
|
return self(string, "det", morphology)
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def pron(self, string: str, morphology: Optional[dict] = None) -> List[str]:
|
2019-10-01 22:36:04 +03:00
|
|
|
return self(string, "pron", morphology)
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def adp(self, string: str, morphology: Optional[dict] = None) -> List[str]:
|
2019-10-01 22:36:04 +03:00
|
|
|
return self(string, "adp", morphology)
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def num(self, string: str, morphology: Optional[dict] = None) -> List[str]:
|
2019-10-01 22:36:04 +03:00
|
|
|
return self(string, "num", morphology)
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def punct(self, string: str, morphology: Optional[dict] = None) -> List[str]:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 19:03:03 +03:00
|
|
|
return self(string, "punct", morphology)
|
2015-10-09 10:44:21 +03:00
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def lookup(self, string: str, orth: Optional[int] = None) -> str:
|
2019-09-15 23:08:13 +03:00
|
|
|
"""Look up a lemma in the table, if available. If no lemma is found,
|
|
|
|
the original string is returned.
|
|
|
|
|
2020-05-24 18:20:58 +03:00
|
|
|
string (str): The original string.
|
2019-09-15 23:08:13 +03:00
|
|
|
orth (int): Optional hash of the string to look up. If not set, the
|
|
|
|
string will be used and hashed.
|
2020-05-24 18:20:58 +03:00
|
|
|
RETURNS (str): The lemma if the string was found, otherwise the
|
2019-09-15 23:08:13 +03:00
|
|
|
original string.
|
|
|
|
"""
|
2019-10-01 22:36:04 +03:00
|
|
|
lookup_table = self.lookups.get_table("lemma_lookup", {})
|
2019-09-15 23:08:13 +03:00
|
|
|
key = orth if orth is not None else string
|
2019-10-01 22:36:04 +03:00
|
|
|
if key in lookup_table:
|
|
|
|
return lookup_table[key]
|
2017-10-11 14:25:51 +03:00
|
|
|
return string
|
|
|
|
|
2020-07-22 14:42:59 +03:00
|
|
|
def lemmatize(
|
|
|
|
self,
|
|
|
|
string: str,
|
|
|
|
index: Dict[str, List[str]],
|
|
|
|
exceptions: Dict[str, Dict[str, List[str]]],
|
|
|
|
rules: Dict[str, List[List[str]]],
|
|
|
|
) -> List[str]:
|
2019-10-01 22:36:04 +03:00
|
|
|
orig = string
|
|
|
|
string = string.lower()
|
|
|
|
forms = []
|
|
|
|
oov_forms = []
|
|
|
|
for old, new in rules:
|
|
|
|
if string.endswith(old):
|
|
|
|
form = string[: len(string) - len(old)] + new
|
|
|
|
if not form:
|
|
|
|
pass
|
|
|
|
elif form in index or not form.isalpha():
|
|
|
|
forms.append(form)
|
|
|
|
else:
|
|
|
|
oov_forms.append(form)
|
|
|
|
# Remove duplicates but preserve the ordering of applied "rules"
|
2019-12-22 03:53:56 +03:00
|
|
|
forms = list(dict.fromkeys(forms))
|
2019-10-01 22:36:04 +03:00
|
|
|
# Put exceptions at the front of the list, so they get priority.
|
|
|
|
# This is a dodgy heuristic -- but it's the best we can do until we get
|
|
|
|
# frequencies on this. We can at least prune out problematic exceptions,
|
|
|
|
# if they shadow more frequent analyses.
|
|
|
|
for form in exceptions.get(string, []):
|
|
|
|
if form not in forms:
|
|
|
|
forms.insert(0, form)
|
|
|
|
if not forms:
|
|
|
|
forms.extend(oov_forms)
|
|
|
|
if not forms:
|
|
|
|
forms.append(orig)
|
|
|
|
return forms
|