mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-14 13:47:13 +03:00
407 lines
15 KiB
Cython
407 lines
15 KiB
Cython
# cython: infer_types
|
|
from __future__ import unicode_literals
|
|
|
|
from os import path
|
|
|
|
from libc.string cimport memset
|
|
|
|
from .lemmatizer import Lemmatizer
|
|
|
|
try:
|
|
import ujson as json
|
|
except ImportError:
|
|
import json
|
|
|
|
from .parts_of_speech cimport ADJ, VERB, NOUN, PUNCT
|
|
from .attrs cimport POS, IS_SPACE
|
|
from .parts_of_speech import IDS as POS_IDS
|
|
from .lexeme cimport Lexeme
|
|
from .attrs import LEMMA, intify_attrs
|
|
|
|
|
|
def _normalize_props(props):
|
|
'''Transform deprecated string keys to correct names.'''
|
|
out = {}
|
|
for key, value in props.items():
|
|
if key == POS:
|
|
if hasattr(value, 'upper'):
|
|
value = value.upper()
|
|
if value in POS_IDS:
|
|
value = POS_IDS[value]
|
|
out[key] = value
|
|
elif key.lower() == 'pos':
|
|
out[POS] = POS_IDS[value.upper()]
|
|
else:
|
|
out[key] = value
|
|
return out
|
|
|
|
|
|
|
|
cdef class Morphology:
|
|
def __init__(self, StringStore string_store, tag_map, lemmatizer):
|
|
self.mem = Pool()
|
|
self.strings = string_store
|
|
self.tag_map = tag_map
|
|
self.lemmatizer = lemmatizer
|
|
self.n_tags = len(tag_map) + 1
|
|
self.tag_names = tuple(sorted(tag_map.keys()))
|
|
self.reverse_index = {}
|
|
|
|
self.rich_tags = <RichTagC*>self.mem.alloc(self.n_tags, sizeof(RichTagC))
|
|
for i, (tag_str, attrs) in enumerate(sorted(tag_map.items())):
|
|
attrs = _normalize_props(attrs)
|
|
attrs = intify_attrs(attrs, self.strings, _do_deprecated=True)
|
|
self.rich_tags[i].id = i
|
|
self.rich_tags[i].name = self.strings[tag_str]
|
|
self.rich_tags[i].morph = 0
|
|
self.rich_tags[i].pos = attrs[POS]
|
|
self.reverse_index[self.rich_tags[i].name] = i
|
|
self._cache = PreshMapArray(self.n_tags)
|
|
|
|
def __reduce__(self):
|
|
return (Morphology, (self.strings, self.tag_map, self.lemmatizer), None, None)
|
|
|
|
cdef int assign_tag(self, TokenC* token, tag) except -1:
|
|
if isinstance(tag, basestring):
|
|
tag_id = self.reverse_index[self.strings[tag]]
|
|
else:
|
|
tag_id = self.reverse_index[tag]
|
|
self.assign_tag_id(token, tag_id)
|
|
|
|
cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1:
|
|
if tag_id >= self.n_tags:
|
|
raise ValueError("Unknown tag ID: %s" % tag_id)
|
|
# TODO: It's pretty arbitrary to put this logic here. I guess the justification
|
|
# is that this is where the specific word and the tag interact. Still,
|
|
# we should have a better way to enforce this rule, or figure out why
|
|
# the statistical model fails.
|
|
# Related to Issue #220
|
|
if Lexeme.c_check_flag(token.lex, IS_SPACE):
|
|
tag_id = self.reverse_index[self.strings['SP']]
|
|
analysis = <MorphAnalysisC*>self._cache.get(tag_id, token.lex.orth)
|
|
if analysis is NULL:
|
|
analysis = <MorphAnalysisC*>self.mem.alloc(1, sizeof(MorphAnalysisC))
|
|
analysis.tag = self.rich_tags[tag_id]
|
|
tag_str = self.strings[self.rich_tags[tag_id].name]
|
|
analysis.lemma = self.lemmatize(analysis.tag.pos, token.lex.orth,
|
|
self.tag_map.get(tag_str, {}))
|
|
self._cache.set(tag_id, token.lex.orth, analysis)
|
|
token.lemma = analysis.lemma
|
|
token.pos = analysis.tag.pos
|
|
token.tag = analysis.tag.name
|
|
token.morph = analysis.tag.morph
|
|
|
|
cdef int assign_feature(self, uint64_t* flags, univ_morph_t flag_id, bint value) except -1:
|
|
cdef flags_t one = 1
|
|
if value:
|
|
flags[0] |= one << flag_id
|
|
else:
|
|
flags[0] &= ~(one << flag_id)
|
|
|
|
def add_special_case(self, unicode tag_str, unicode orth_str, attrs, force=False):
|
|
'''Add a special-case rule to the morphological analyser. Tokens whose
|
|
tag and orth match the rule will receive the specified properties.
|
|
|
|
Arguments:
|
|
tag (unicode): The part-of-speech tag to key the exception.
|
|
orth (unicode): The word-form to key the exception.
|
|
'''
|
|
tag = self.strings[tag_str]
|
|
tag_id = self.reverse_index[tag]
|
|
orth = self.strings[orth_str]
|
|
cdef RichTagC rich_tag = self.rich_tags[tag_id]
|
|
attrs = intify_attrs(attrs, self.strings, _do_deprecated=True)
|
|
cached = <MorphAnalysisC*>self._cache.get(tag_id, orth)
|
|
if cached is NULL:
|
|
cached = <MorphAnalysisC*>self.mem.alloc(1, sizeof(MorphAnalysisC))
|
|
elif force:
|
|
memset(cached, 0, sizeof(cached[0]))
|
|
else:
|
|
msg = ("Conflicting morphology exception for (%s, %s). Use force=True "
|
|
"to overwrite.")
|
|
msg = msg % (tag_str, orth_str)
|
|
raise ValueError(msg)
|
|
|
|
cached.tag = rich_tag
|
|
# TODO: Refactor this to take arbitrary attributes.
|
|
for name_id, value_id in attrs.items():
|
|
if name_id == LEMMA:
|
|
cached.lemma = value_id
|
|
else:
|
|
self.assign_feature(&cached.tag.morph, name_id, value_id)
|
|
if cached.lemma == 0:
|
|
cached.lemma = self.lemmatize(rich_tag.pos, orth,
|
|
self.tag_map.get(tag_str, {}))
|
|
self._cache.set(tag_id, orth, <void*>cached)
|
|
|
|
def load_morph_exceptions(self, dict exc):
|
|
# Map (form, pos) to (lemma, rich tag)
|
|
for tag_str, entries in exc.items():
|
|
for form_str, attrs in entries.items():
|
|
self.add_special_case(tag_str, form_str, attrs)
|
|
|
|
def lemmatize(self, const univ_pos_t univ_pos, attr_t orth, morphology):
|
|
cdef unicode py_string = self.strings[orth]
|
|
if self.lemmatizer is None:
|
|
return self.strings[py_string.lower()]
|
|
if univ_pos not in (NOUN, VERB, ADJ, PUNCT):
|
|
return self.strings[py_string.lower()]
|
|
cdef set lemma_strings
|
|
cdef unicode lemma_string
|
|
lemma_strings = self.lemmatizer(py_string, univ_pos, morphology)
|
|
lemma_string = sorted(lemma_strings)[0]
|
|
lemma = self.strings[lemma_string]
|
|
return lemma
|
|
|
|
|
|
IDS = {
|
|
"Animacy_anim": Animacy_anim,
|
|
"Animacy_inam": Animacy_inam,
|
|
"Aspect_freq": Aspect_freq,
|
|
"Aspect_imp": Aspect_imp,
|
|
"Aspect_mod": Aspect_mod,
|
|
"Aspect_none": Aspect_none,
|
|
"Aspect_perf": Aspect_perf,
|
|
"Case_abe": Case_abe,
|
|
"Case_abl": Case_abl,
|
|
"Case_abs": Case_abs,
|
|
"Case_acc": Case_acc,
|
|
"Case_ade": Case_ade,
|
|
"Case_all": Case_all,
|
|
"Case_cau": Case_cau,
|
|
"Case_com": Case_com,
|
|
"Case_dat": Case_dat,
|
|
"Case_del": Case_del,
|
|
"Case_dis": Case_dis,
|
|
"Case_ela": Case_ela,
|
|
"Case_ess": Case_ess,
|
|
"Case_gen": Case_gen,
|
|
"Case_ill": Case_ill,
|
|
"Case_ine": Case_ine,
|
|
"Case_ins": Case_ins,
|
|
"Case_loc": Case_loc,
|
|
"Case_lat": Case_lat,
|
|
"Case_nom": Case_nom,
|
|
"Case_par": Case_par,
|
|
"Case_sub": Case_sub,
|
|
"Case_sup": Case_sup,
|
|
"Case_tem": Case_tem,
|
|
"Case_ter": Case_ter,
|
|
"Case_tra": Case_tra,
|
|
"Case_voc": Case_voc,
|
|
"Definite_two": Definite_two,
|
|
"Definite_def": Definite_def,
|
|
"Definite_red": Definite_red,
|
|
"Definite_ind": Definite_ind,
|
|
"Degree_cmp": Degree_cmp,
|
|
"Degree_comp": Degree_comp,
|
|
"Degree_none": Degree_none,
|
|
"Degree_pos": Degree_pos,
|
|
"Degree_sup": Degree_sup,
|
|
"Degree_abs": Degree_abs,
|
|
"Degree_com": Degree_com,
|
|
"Degree_dim ": Degree_dim, # du
|
|
"Gender_com": Gender_com,
|
|
"Gender_fem": Gender_fem,
|
|
"Gender_masc": Gender_masc,
|
|
"Gender_neut": Gender_neut,
|
|
"Mood_cnd": Mood_cnd,
|
|
"Mood_imp": Mood_imp,
|
|
"Mood_ind": Mood_ind,
|
|
"Mood_n": Mood_n,
|
|
"Mood_pot": Mood_pot,
|
|
"Mood_sub": Mood_sub,
|
|
"Mood_opt": Mood_opt,
|
|
"Negative_neg": Negative_neg,
|
|
"Negative_pos": Negative_pos,
|
|
"Negative_yes": Negative_yes,
|
|
"Number_com": Number_com,
|
|
"Number_dual": Number_dual,
|
|
"Number_none": Number_none,
|
|
"Number_plur": Number_plur,
|
|
"Number_sing": Number_sing,
|
|
"Number_ptan ": Number_ptan, # bg
|
|
"Number_count ": Number_count, # bg
|
|
"NumType_card": NumType_card,
|
|
"NumType_dist": NumType_dist,
|
|
"NumType_frac": NumType_frac,
|
|
"NumType_gen": NumType_gen,
|
|
"NumType_mult": NumType_mult,
|
|
"NumType_none": NumType_none,
|
|
"NumType_ord": NumType_ord,
|
|
"NumType_sets": NumType_sets,
|
|
"Person_one": Person_one,
|
|
"Person_two": Person_two,
|
|
"Person_three": Person_three,
|
|
"Person_none": Person_none,
|
|
"Poss_yes": Poss_yes,
|
|
"PronType_advPart": PronType_advPart,
|
|
"PronType_art": PronType_art,
|
|
"PronType_default": PronType_default,
|
|
"PronType_dem": PronType_dem,
|
|
"PronType_ind": PronType_ind,
|
|
"PronType_int": PronType_int,
|
|
"PronType_neg": PronType_neg,
|
|
"PronType_prs": PronType_prs,
|
|
"PronType_rcp": PronType_rcp,
|
|
"PronType_rel": PronType_rel,
|
|
"PronType_tot": PronType_tot,
|
|
"PronType_clit": PronType_clit,
|
|
"PronType_exc ": PronType_exc, # es, ca, it, fa,
|
|
"Reflex_yes": Reflex_yes,
|
|
"Tense_fut": Tense_fut,
|
|
"Tense_imp": Tense_imp,
|
|
"Tense_past": Tense_past,
|
|
"Tense_pres": Tense_pres,
|
|
"VerbForm_fin": VerbForm_fin,
|
|
"VerbForm_ger": VerbForm_ger,
|
|
"VerbForm_inf": VerbForm_inf,
|
|
"VerbForm_none": VerbForm_none,
|
|
"VerbForm_part": VerbForm_part,
|
|
"VerbForm_partFut": VerbForm_partFut,
|
|
"VerbForm_partPast": VerbForm_partPast,
|
|
"VerbForm_partPres": VerbForm_partPres,
|
|
"VerbForm_sup": VerbForm_sup,
|
|
"VerbForm_trans": VerbForm_trans,
|
|
"VerbForm_gdv ": VerbForm_gdv, # la,
|
|
"Voice_act": Voice_act,
|
|
"Voice_cau": Voice_cau,
|
|
"Voice_pass": Voice_pass,
|
|
"Voice_mid ": Voice_mid, # gkc,
|
|
"Voice_int ": Voice_int, # hb,
|
|
"Abbr_yes ": Abbr_yes, # cz, fi, sl, U,
|
|
"AdpType_prep ": AdpType_prep, # cz, U,
|
|
"AdpType_post ": AdpType_post, # U,
|
|
"AdpType_voc ": AdpType_voc, # cz,
|
|
"AdpType_comprep ": AdpType_comprep, # cz,
|
|
"AdpType_circ ": AdpType_circ, # U,
|
|
"AdvType_man": AdvType_man,
|
|
"AdvType_loc": AdvType_loc,
|
|
"AdvType_tim": AdvType_tim,
|
|
"AdvType_deg": AdvType_deg,
|
|
"AdvType_cau": AdvType_cau,
|
|
"AdvType_mod": AdvType_mod,
|
|
"AdvType_sta": AdvType_sta,
|
|
"AdvType_ex": AdvType_ex,
|
|
"AdvType_adadj": AdvType_adadj,
|
|
"ConjType_oper ": ConjType_oper, # cz, U,
|
|
"ConjType_comp ": ConjType_comp, # cz, U,
|
|
"Connegative_yes ": Connegative_yes, # fi,
|
|
"Derivation_minen ": Derivation_minen, # fi,
|
|
"Derivation_sti ": Derivation_sti, # fi,
|
|
"Derivation_inen ": Derivation_inen, # fi,
|
|
"Derivation_lainen ": Derivation_lainen, # fi,
|
|
"Derivation_ja ": Derivation_ja, # fi,
|
|
"Derivation_ton ": Derivation_ton, # fi,
|
|
"Derivation_vs ": Derivation_vs, # fi,
|
|
"Derivation_ttain ": Derivation_ttain, # fi,
|
|
"Derivation_ttaa ": Derivation_ttaa, # fi,
|
|
"Echo_rdp ": Echo_rdp, # U,
|
|
"Echo_ech ": Echo_ech, # U,
|
|
"Foreign_foreign ": Foreign_foreign, # cz, fi, U,
|
|
"Foreign_fscript ": Foreign_fscript, # cz, fi, U,
|
|
"Foreign_tscript ": Foreign_tscript, # cz, U,
|
|
"Foreign_yes ": Foreign_yes, # sl,
|
|
"Gender_dat_masc ": Gender_dat_masc, # bq, U,
|
|
"Gender_dat_fem ": Gender_dat_fem, # bq, U,
|
|
"Gender_erg_masc ": Gender_erg_masc, # bq,
|
|
"Gender_erg_fem ": Gender_erg_fem, # bq,
|
|
"Gender_psor_masc ": Gender_psor_masc, # cz, sl, U,
|
|
"Gender_psor_fem ": Gender_psor_fem, # cz, sl, U,
|
|
"Gender_psor_neut ": Gender_psor_neut, # sl,
|
|
"Hyph_yes ": Hyph_yes, # cz, U,
|
|
"InfForm_one ": InfForm_one, # fi,
|
|
"InfForm_two ": InfForm_two, # fi,
|
|
"InfForm_three ": InfForm_three, # fi,
|
|
"NameType_geo ": NameType_geo, # U, cz,
|
|
"NameType_prs ": NameType_prs, # U, cz,
|
|
"NameType_giv ": NameType_giv, # U, cz,
|
|
"NameType_sur ": NameType_sur, # U, cz,
|
|
"NameType_nat ": NameType_nat, # U, cz,
|
|
"NameType_com ": NameType_com, # U, cz,
|
|
"NameType_pro ": NameType_pro, # U, cz,
|
|
"NameType_oth ": NameType_oth, # U, cz,
|
|
"NounType_com ": NounType_com, # U,
|
|
"NounType_prop ": NounType_prop, # U,
|
|
"NounType_class ": NounType_class, # U,
|
|
"Number_abs_sing ": Number_abs_sing, # bq, U,
|
|
"Number_abs_plur ": Number_abs_plur, # bq, U,
|
|
"Number_dat_sing ": Number_dat_sing, # bq, U,
|
|
"Number_dat_plur ": Number_dat_plur, # bq, U,
|
|
"Number_erg_sing ": Number_erg_sing, # bq, U,
|
|
"Number_erg_plur ": Number_erg_plur, # bq, U,
|
|
"Number_psee_sing ": Number_psee_sing, # U,
|
|
"Number_psee_plur ": Number_psee_plur, # U,
|
|
"Number_psor_sing ": Number_psor_sing, # cz, fi, sl, U,
|
|
"Number_psor_plur ": Number_psor_plur, # cz, fi, sl, U,
|
|
"NumForm_digit ": NumForm_digit, # cz, sl, U,
|
|
"NumForm_roman ": NumForm_roman, # cz, sl, U,
|
|
"NumForm_word ": NumForm_word, # cz, sl, U,
|
|
"NumValue_one ": NumValue_one, # cz, U,
|
|
"NumValue_two ": NumValue_two, # cz, U,
|
|
"NumValue_three ": NumValue_three, # cz, U,
|
|
"PartForm_pres ": PartForm_pres, # fi,
|
|
"PartForm_past ": PartForm_past, # fi,
|
|
"PartForm_agt ": PartForm_agt, # fi,
|
|
"PartForm_neg ": PartForm_neg, # fi,
|
|
"PartType_mod ": PartType_mod, # U,
|
|
"PartType_emp ": PartType_emp, # U,
|
|
"PartType_res ": PartType_res, # U,
|
|
"PartType_inf ": PartType_inf, # U,
|
|
"PartType_vbp ": PartType_vbp, # U,
|
|
"Person_abs_one ": Person_abs_one, # bq, U,
|
|
"Person_abs_two ": Person_abs_two, # bq, U,
|
|
"Person_abs_three ": Person_abs_three, # bq, U,
|
|
"Person_dat_one ": Person_dat_one, # bq, U,
|
|
"Person_dat_two ": Person_dat_two, # bq, U,
|
|
"Person_dat_three ": Person_dat_three, # bq, U,
|
|
"Person_erg_one ": Person_erg_one, # bq, U,
|
|
"Person_erg_two ": Person_erg_two, # bq, U,
|
|
"Person_erg_three ": Person_erg_three, # bq, U,
|
|
"Person_psor_one ": Person_psor_one, # fi, U,
|
|
"Person_psor_two ": Person_psor_two, # fi, U,
|
|
"Person_psor_three ": Person_psor_three, # fi, U,
|
|
"Polite_inf ": Polite_inf, # bq, U,
|
|
"Polite_pol ": Polite_pol, # bq, U,
|
|
"Polite_abs_inf ": Polite_abs_inf, # bq, U,
|
|
"Polite_abs_pol ": Polite_abs_pol, # bq, U,
|
|
"Polite_erg_inf ": Polite_erg_inf, # bq, U,
|
|
"Polite_erg_pol ": Polite_erg_pol, # bq, U,
|
|
"Polite_dat_inf ": Polite_dat_inf, # bq, U,
|
|
"Polite_dat_pol ": Polite_dat_pol, # bq, U,
|
|
"Prefix_yes ": Prefix_yes, # U,
|
|
"PrepCase_npr ": PrepCase_npr, # cz,
|
|
"PrepCase_pre ": PrepCase_pre, # U,
|
|
"PunctSide_ini ": PunctSide_ini, # U,
|
|
"PunctSide_fin ": PunctSide_fin, # U,
|
|
"PunctType_peri ": PunctType_peri, # U,
|
|
"PunctType_qest ": PunctType_qest, # U,
|
|
"PunctType_excl ": PunctType_excl, # U,
|
|
"PunctType_quot ": PunctType_quot, # U,
|
|
"PunctType_brck ": PunctType_brck, # U,
|
|
"PunctType_comm ": PunctType_comm, # U,
|
|
"PunctType_colo ": PunctType_colo, # U,
|
|
"PunctType_semi ": PunctType_semi, # U,
|
|
"PunctType_dash ": PunctType_dash, # U,
|
|
"Style_arch ": Style_arch, # cz, fi, U,
|
|
"Style_rare ": Style_rare, # cz, fi, U,
|
|
"Style_poet ": Style_poet, # cz, U,
|
|
"Style_norm ": Style_norm, # cz, U,
|
|
"Style_coll ": Style_coll, # cz, U,
|
|
"Style_vrnc ": Style_vrnc, # cz, U,
|
|
"Style_sing ": Style_sing, # cz, U,
|
|
"Style_expr ": Style_expr, # cz, U,
|
|
"Style_derg ": Style_derg, # cz, U,
|
|
"Style_vulg ": Style_vulg, # cz, U,
|
|
"Style_yes ": Style_yes, # fi, U,
|
|
"StyleVariant_styleShort ": StyleVariant_styleShort, # cz,
|
|
"StyleVariant_styleBound ": StyleVariant_styleBound, # cz, sl,
|
|
"VerbType_aux ": VerbType_aux, # U,
|
|
"VerbType_cop ": VerbType_cop, # U,
|
|
"VerbType_mod ": VerbType_mod, # U,
|
|
"VerbType_light ": VerbType_light, # U,
|
|
}
|
|
|
|
|
|
NAMES = [key for key, value in sorted(IDS.items(), key=lambda item: item[1])]
|