explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-10-31 07:57:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix tokenizer exceptions

Browse Source
This commit is contained in:
Ines Montani 2021-01-27 22:11:42 +11:00
parent 80ba9eaf7d
commit 615dba9d99
1 changed files with 24 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
spacy/lang/ky/tokenizer_exceptions.py
View File

@ -1,39 +1,39 @@
from ..tokenizer_exceptions import BASE_EXCEPTIONS from ..tokenizer_exceptions import BASE_EXCEPTIONS
from ...symbols import ORTH, LEMMA, NORM from ...symbols import ORTH, NORM
from ...util import update_exc from ...util import update_exc
_exc = {} _exc = {}
_abbrev_exc = [ _abbrev_exc = [
# Weekdays abbreviations # Weekdays abbreviations
{ORTH: "дүй", LEMMA: "дүйшөмбү"}, {ORTH: "дүй", NORM: "дүйшөмбү"},
{ORTH: "шей", LEMMA: "шейшемби"}, {ORTH: "шей", NORM: "шейшемби"},
{ORTH: "шар", LEMMA: "шаршемби"}, {ORTH: "шар", NORM: "шаршемби"},
{ORTH: "бей", LEMMA: "бейшемби"}, {ORTH: "бей", NORM: "бейшемби"},
{ORTH: "жум", LEMMA: "жума"}, {ORTH: "жум", NORM: "жума"},
{ORTH: "ишм", LEMMA: "ишемби"}, {ORTH: "ишм", NORM: "ишемби"},
{ORTH: "жек", LEMMA: "жекшемби"}, {ORTH: "жек", NORM: "жекшемби"},
# Months abbreviations # Months abbreviations
{ORTH: "янв", LEMMA: "январь"}, {ORTH: "янв", NORM: "январь"},
{ORTH: "фев", LEMMA: "февраль"}, {ORTH: "фев", NORM: "февраль"},
{ORTH: "мар", LEMMA: "март"}, {ORTH: "мар", NORM: "март"},
{ORTH: "апр", LEMMA: "апрель"}, {ORTH: "апр", NORM: "апрель"},
{ORTH: "июн", LEMMA: "июнь"}, {ORTH: "июн", NORM: "июнь"},
{ORTH: "июл", LEMMA: "июль"}, {ORTH: "июл", NORM: "июль"},
{ORTH: "авг", LEMMA: "август"}, {ORTH: "авг", NORM: "август"},
{ORTH: "сен", LEMMA: "сентябрь"}, {ORTH: "сен", NORM: "сентябрь"},
{ORTH: "окт", LEMMA: "октябрь"}, {ORTH: "окт", NORM: "октябрь"},
{ORTH: "ноя", LEMMA: "ноябрь"}, {ORTH: "ноя", NORM: "ноябрь"},
{ORTH: "дек", LEMMA: "декабрь"}, {ORTH: "дек", NORM: "декабрь"},
# Number abbreviations # Number abbreviations
{ORTH: "млрд", LEMMA: "миллиард"}, {ORTH: "млрд", NORM: "миллиард"},
{ORTH: "млн", LEMMA: "миллион"}, {ORTH: "млн", NORM: "миллион"},
] ]
for abbr in _abbrev_exc: for abbr in _abbrev_exc:
for orth in (abbr[ORTH], abbr[ORTH].capitalize(), abbr[ORTH].upper()): for orth in (abbr[ORTH], abbr[ORTH].capitalize(), abbr[ORTH].upper()):
_exc[orth] = [{ORTH: orth, LEMMA: abbr[LEMMA], NORM: abbr[LEMMA]}] _exc[orth] = [{ORTH: orth, NORM: abbr[NORM]}]
_exc[orth + "."] = [{ORTH: orth + ".", LEMMA: abbr[LEMMA], NORM: abbr[LEMMA]}] _exc[orth + "."] = [{ORTH: orth + ".", NORM: abbr[NORM]}]
for exc_data in [ # "etc." abbreviations for exc_data in [ # "etc." abbreviations
{ORTH: "ж.б.у.с.", NORM: "жана башка ушул сыяктуу"}, {ORTH: "ж.б.у.с.", NORM: "жана башка ушул сыяктуу"},
@ -47,8 +47,6 @@ for exc_data in [ # "etc." abbreviations
{ORTH: "көч.", NORM: "көчөсү"}, {ORTH: "көч.", NORM: "көчөсү"},
{ORTH: "м-н", NORM: "менен"}, {ORTH: "м-н", NORM: "менен"},
{ORTH: "б-ча", NORM: "боюнча"}, {ORTH: "б-ча", NORM: "боюнча"},
]: ]: _exc[exc_data[ORTH]] = [exc_data]
exc_data[LEMMA] = exc_data[NORM]
_exc[exc_data[ORTH]] = [exc_data]
TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc)