explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-24 17:06:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix lookup usage in French/Catalan (fix #11347) (#11382)

Browse Source 
* Fix lookup usage (fix #11347)

Before using the lookups table in the French (and Catalan) lemmatizers,
there's a check to see if the current term is in the table. But it's
checking a string against hashes, so it's always false. Also the table
lookup function is designed so you don't have to do that anyway.

* Use the lookup table directly

* Use string, not token
This commit is contained in:
Paul O'Leary McCann 2022-08-29 17:32:38 +09:00 committed by GitHub
parent 6fd3b4d9d6
commit aafee5e1b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spacy/lang/ca/lemmatizer.py
View File

@ -72,10 +72,10 @@ class CatalanLemmatizer(Lemmatizer):
oov_forms.append(form) oov_forms.append(form)
if not forms: if not forms:
forms.extend(oov_forms) forms.extend(oov_forms)
if not forms and string in lookup_table.keys():
forms.append(self.lookup_lemmatize(token)[0]) # use lookups, and fall back to the token itself
if not forms: if not forms:
forms.append(string) forms.append(lookup_table.get(string, [string])[0])
forms = list(dict.fromkeys(forms)) forms = list(dict.fromkeys(forms))
self.cache[cache_key] = forms self.cache[cache_key] = forms
return forms return forms

13
spacy/lang/fr/lemmatizer.py
View File

@ -53,11 +53,16 @@ class FrenchLemmatizer(Lemmatizer):
rules = rules_table.get(univ_pos, []) rules = rules_table.get(univ_pos, [])
string = string.lower() string = string.lower()
forms = [] forms = []
# first try lookup in table based on upos
if string in index: if string in index:
forms.append(string) forms.append(string)
self.cache[cache_key] = forms self.cache[cache_key] = forms
return forms return forms
# then add anything in the exceptions table
forms.extend(exceptions.get(string, [])) forms.extend(exceptions.get(string, []))
# if nothing found yet, use the rules
oov_forms = [] oov_forms = []
if not forms: if not forms:
for old, new in rules: for old, new in rules:
@ -69,12 +74,14 @@ class FrenchLemmatizer(Lemmatizer):
forms.append(form) forms.append(form)
else: else:
oov_forms.append(form) oov_forms.append(form)
# if still nothing, add the oov forms from rules
if not forms: if not forms:
forms.extend(oov_forms) forms.extend(oov_forms)
if not forms and string in lookup_table.keys():
forms.append(self.lookup_lemmatize(token)[0]) # use lookups, which fall back to the token itself
if not forms: if not forms:
forms.append(string) forms.append(lookup_table.get(string, [string])[0])
forms = list(dict.fromkeys(forms)) forms = list(dict.fromkeys(forms))
self.cache[cache_key] = forms self.cache[cache_key] = forms
return forms return forms