2015-08-28 03:02:33 +03:00
|
|
|
from cymem.cymem cimport Pool
|
2018-09-25 01:35:59 +03:00
|
|
|
from preshed.maps cimport PreshMap, PreshMapArray
|
2015-08-28 03:02:33 +03:00
|
|
|
from libc.stdint cimport uint64_t
|
2018-09-25 00:57:41 +03:00
|
|
|
from murmurhash cimport mrmr
|
2015-08-28 03:02:33 +03:00
|
|
|
|
2015-08-26 20:17:35 +03:00
|
|
|
from .structs cimport TokenC
|
2015-08-27 10:16:11 +03:00
|
|
|
from .strings cimport StringStore
|
2018-09-25 00:57:41 +03:00
|
|
|
from .typedefs cimport hash_t, attr_t, flags_t
|
2015-08-28 03:02:33 +03:00
|
|
|
from .parts_of_speech cimport univ_pos_t
|
|
|
|
|
|
2015-10-10 14:10:58 +03:00
|
|
|
from . cimport symbols
|
2015-08-28 03:02:33 +03:00
|
|
|
|
2015-08-26 20:17:35 +03:00
|
|
|
cdef class Morphology:
|
2015-08-28 03:02:33 +03:00
|
|
|
cdef readonly Pool mem
|
2015-08-28 04:44:54 +03:00
|
|
|
cdef readonly StringStore strings
|
2018-09-25 00:57:41 +03:00
|
|
|
cdef PreshMap tags # Keyed by hash, value is pointer to tag
|
|
|
|
|
|
2015-08-27 10:16:11 +03:00
|
|
|
cdef public object lemmatizer
|
2015-10-12 07:27:47 +03:00
|
|
|
cdef readonly object tag_map
|
2018-09-25 01:35:59 +03:00
|
|
|
cdef readonly object tag_names
|
|
|
|
|
cdef readonly object reverse_index
|
|
|
|
|
cdef readonly object exc
|
2018-09-25 11:57:33 +03:00
|
|
|
cdef readonly PreshMapArray _cache
|
2018-09-25 01:35:59 +03:00
|
|
|
cdef readonly int n_tags
|
2015-08-28 03:02:33 +03:00
|
|
|
|
2018-09-25 22:07:08 +03:00
|
|
|
cpdef update(self, hash_t morph, features)
|
2018-09-25 00:57:41 +03:00
|
|
|
cdef hash_t insert(self, RichTagC tag) except 0
|
|
|
|
|
|
2017-10-11 04:22:49 +03:00
|
|
|
cdef int assign_untagged(self, TokenC* token) except -1
|
2015-08-28 03:02:33 +03:00
|
|
|
cdef int assign_tag(self, TokenC* token, tag) except -1
|
2016-11-04 21:19:09 +03:00
|
|
|
cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1
|
2015-08-26 20:17:35 +03:00
|
|
|
|
2018-09-25 16:18:21 +03:00
|
|
|
cdef int _assign_tag_from_exceptions(self, TokenC* token, int tag_id) except -1
|
|
|
|
|
|
2017-10-20 14:59:57 +03:00
|
|
|
cdef enum univ_morph_t:
|
2015-10-10 14:10:58 +03:00
|
|
|
NIL = 0
|
2018-09-25 18:28:13 +03:00
|
|
|
begin_Abbr
|
|
|
|
|
Abbr_yes # cz, fi, sl, U
|
|
|
|
|
end_Abbr
|
|
|
|
|
begin_AdpType
|
|
|
|
|
AdpType_circ # U
|
|
|
|
|
AdpType_comprep # cz
|
|
|
|
|
AdpType_prep # cz, U
|
|
|
|
|
AdpType_post # U
|
|
|
|
|
AdpType_voc # cz
|
|
|
|
|
end_AdpType
|
|
|
|
|
begin_AdvType
|
|
|
|
|
AdvType_adadj
|
|
|
|
|
AdvType_cau
|
|
|
|
|
AdvType_deg
|
|
|
|
|
AdvType_ex
|
|
|
|
|
AdvType_loc
|
|
|
|
|
AdvType_man
|
|
|
|
|
AdvType_mod
|
|
|
|
|
AdvType_sta
|
|
|
|
|
AdvType_tim
|
|
|
|
|
end_AdvType
|
|
|
|
|
begin_Animacy
|
2015-10-10 14:10:58 +03:00
|
|
|
Animacy_anim = symbols.Animacy_anim
|
2018-03-27 20:23:02 +03:00
|
|
|
Animacy_hum
|
2018-09-25 18:28:13 +03:00
|
|
|
Animacy_inan
|
2018-03-27 20:23:02 +03:00
|
|
|
Animacy_nhum
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Animacy
|
|
|
|
|
begin_Aspect
|
2015-10-10 14:10:58 +03:00
|
|
|
Aspect_freq
|
|
|
|
|
Aspect_imp
|
|
|
|
|
Aspect_mod
|
|
|
|
|
Aspect_none
|
|
|
|
|
Aspect_perf
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Aspect
|
|
|
|
|
begin_Case
|
2015-10-10 14:10:58 +03:00
|
|
|
Case_abe
|
|
|
|
|
Case_abl
|
|
|
|
|
Case_abs
|
|
|
|
|
Case_acc
|
|
|
|
|
Case_ade
|
|
|
|
|
Case_all
|
|
|
|
|
Case_cau
|
|
|
|
|
Case_com
|
|
|
|
|
Case_dat
|
|
|
|
|
Case_del
|
|
|
|
|
Case_dis
|
|
|
|
|
Case_ela
|
|
|
|
|
Case_ess
|
|
|
|
|
Case_gen
|
|
|
|
|
Case_ill
|
|
|
|
|
Case_ine
|
|
|
|
|
Case_ins
|
|
|
|
|
Case_loc
|
|
|
|
|
Case_lat
|
|
|
|
|
Case_nom
|
|
|
|
|
Case_par
|
|
|
|
|
Case_sub
|
|
|
|
|
Case_sup
|
|
|
|
|
Case_tem
|
|
|
|
|
Case_ter
|
|
|
|
|
Case_tra
|
|
|
|
|
Case_voc
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Case
|
|
|
|
|
begin_ConjType
|
|
|
|
|
ConjType_comp # cz, U
|
|
|
|
|
ConjType_oper # cz, U
|
|
|
|
|
end_ConjType
|
|
|
|
|
begin_Connegative
|
|
|
|
|
Connegative_yes # fi
|
|
|
|
|
end_Connegative
|
|
|
|
|
begin_Definite
|
2017-02-27 00:27:11 +03:00
|
|
|
Definite_cons # U20
|
2018-09-25 18:28:13 +03:00
|
|
|
Definite_def
|
2015-10-10 14:10:58 +03:00
|
|
|
Definite_ind
|
2018-09-25 18:28:13 +03:00
|
|
|
Definite_red
|
|
|
|
|
Definite_two
|
|
|
|
|
end_Definite
|
|
|
|
|
begin_Degree
|
|
|
|
|
Degree_abs
|
2015-10-10 14:10:58 +03:00
|
|
|
Degree_cmp
|
|
|
|
|
Degree_comp
|
|
|
|
|
Degree_none
|
|
|
|
|
Degree_pos
|
|
|
|
|
Degree_sup
|
|
|
|
|
Degree_com
|
|
|
|
|
Degree_dim # du
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Degree
|
|
|
|
|
begin_Gender
|
2015-10-10 14:10:58 +03:00
|
|
|
Gender_com
|
|
|
|
|
Gender_fem
|
|
|
|
|
Gender_masc
|
|
|
|
|
Gender_neut
|
2018-09-25 18:28:13 +03:00
|
|
|
Gender_dat_masc # bq, U
|
|
|
|
|
Gender_dat_fem # bq, U
|
|
|
|
|
Gender_erg_masc # bq
|
|
|
|
|
Gender_erg_fem # bq
|
|
|
|
|
Gender_psor_masc # cz, sl, U
|
|
|
|
|
Gender_psor_fem # cz, sl, U
|
|
|
|
|
Gender_psor_neut # sl
|
|
|
|
|
|
|
|
|
|
end_Gender
|
|
|
|
|
begin_Mood
|
2015-10-10 14:10:58 +03:00
|
|
|
Mood_cnd
|
|
|
|
|
Mood_imp
|
|
|
|
|
Mood_ind
|
|
|
|
|
Mood_n
|
|
|
|
|
Mood_pot
|
|
|
|
|
Mood_sub
|
|
|
|
|
Mood_opt
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Mood
|
|
|
|
|
begin_Negative
|
2015-10-10 14:10:58 +03:00
|
|
|
Negative_neg
|
|
|
|
|
Negative_pos
|
|
|
|
|
Negative_yes
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Negative
|
|
|
|
|
begin_Polarity
|
2017-02-27 00:27:11 +03:00
|
|
|
Polarity_neg # U20
|
|
|
|
|
Polarity_pos # U20
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Polarity
|
|
|
|
|
begin_Number
|
2015-10-10 14:10:58 +03:00
|
|
|
Number_com
|
|
|
|
|
Number_dual
|
|
|
|
|
Number_none
|
|
|
|
|
Number_plur
|
|
|
|
|
Number_sing
|
|
|
|
|
Number_ptan # bg
|
|
|
|
|
Number_count # bg
|
2018-09-25 18:28:13 +03:00
|
|
|
Number_abs_sing # bq, U
|
|
|
|
|
Number_abs_plur # bq, U
|
|
|
|
|
Number_dat_sing # bq, U
|
|
|
|
|
Number_dat_plur # bq, U
|
|
|
|
|
Number_erg_sing # bq, U
|
|
|
|
|
Number_erg_plur # bq, U
|
|
|
|
|
Number_psee_sing # U
|
|
|
|
|
Number_psee_plur # U
|
|
|
|
|
Number_psor_sing # cz, fi, sl, U
|
|
|
|
|
Number_psor_plur # cz, fi, sl, U
|
|
|
|
|
|
|
|
|
|
end_Number
|
|
|
|
|
begin_NumType
|
2015-10-10 14:10:58 +03:00
|
|
|
NumType_card
|
|
|
|
|
NumType_dist
|
|
|
|
|
NumType_frac
|
|
|
|
|
NumType_gen
|
|
|
|
|
NumType_mult
|
|
|
|
|
NumType_none
|
|
|
|
|
NumType_ord
|
|
|
|
|
NumType_sets
|
2018-09-25 18:28:13 +03:00
|
|
|
end_NumType
|
|
|
|
|
begin_Person
|
2015-10-10 14:10:58 +03:00
|
|
|
Person_one
|
|
|
|
|
Person_two
|
|
|
|
|
Person_three
|
|
|
|
|
Person_none
|
2018-09-25 18:28:13 +03:00
|
|
|
Person_abs_one # bq, U
|
|
|
|
|
Person_abs_two # bq, U
|
|
|
|
|
Person_abs_three # bq, U
|
|
|
|
|
Person_dat_one # bq, U
|
|
|
|
|
Person_dat_two # bq, U
|
|
|
|
|
Person_dat_three # bq, U
|
|
|
|
|
Person_erg_one # bq, U
|
|
|
|
|
Person_erg_two # bq, U
|
|
|
|
|
Person_erg_three # bq, U
|
|
|
|
|
Person_psor_one # fi, U
|
|
|
|
|
Person_psor_two # fi, U
|
|
|
|
|
Person_psor_three # fi, U
|
|
|
|
|
end_Person
|
|
|
|
|
begin_Poss
|
2015-10-10 14:10:58 +03:00
|
|
|
Poss_yes
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Poss
|
|
|
|
|
begin_PronType
|
2015-10-10 14:10:58 +03:00
|
|
|
PronType_advPart
|
|
|
|
|
PronType_art
|
|
|
|
|
PronType_default
|
|
|
|
|
PronType_dem
|
|
|
|
|
PronType_ind
|
|
|
|
|
PronType_int
|
|
|
|
|
PronType_neg
|
|
|
|
|
PronType_prs
|
|
|
|
|
PronType_rcp
|
|
|
|
|
PronType_rel
|
|
|
|
|
PronType_tot
|
|
|
|
|
PronType_clit
|
|
|
|
|
PronType_exc # es, ca, it, fa
|
2018-09-25 18:28:13 +03:00
|
|
|
end_PronType
|
|
|
|
|
begin_Reflex
|
2015-10-10 14:10:58 +03:00
|
|
|
Reflex_yes
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Reflex
|
|
|
|
|
begin_Tense
|
2015-10-10 14:10:58 +03:00
|
|
|
Tense_fut
|
|
|
|
|
Tense_imp
|
|
|
|
|
Tense_past
|
|
|
|
|
Tense_pres
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Tense
|
|
|
|
|
begin_VerbForm
|
2015-10-10 14:10:58 +03:00
|
|
|
VerbForm_fin
|
|
|
|
|
VerbForm_ger
|
|
|
|
|
VerbForm_inf
|
|
|
|
|
VerbForm_none
|
|
|
|
|
VerbForm_part
|
|
|
|
|
VerbForm_partFut
|
|
|
|
|
VerbForm_partPast
|
|
|
|
|
VerbForm_partPres
|
|
|
|
|
VerbForm_sup
|
|
|
|
|
VerbForm_trans
|
2017-02-27 00:27:11 +03:00
|
|
|
VerbForm_conv # U20
|
2015-10-10 14:10:58 +03:00
|
|
|
VerbForm_gdv # la
|
2018-09-25 18:28:13 +03:00
|
|
|
end_VerbForm
|
|
|
|
|
begin_Voice
|
2015-10-10 14:10:58 +03:00
|
|
|
Voice_act
|
|
|
|
|
Voice_cau
|
|
|
|
|
Voice_pass
|
|
|
|
|
Voice_mid # gkc
|
|
|
|
|
Voice_int # hb
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Voice
|
|
|
|
|
begin_Derivation
|
2015-10-10 14:10:58 +03:00
|
|
|
Derivation_minen # fi
|
|
|
|
|
Derivation_sti # fi
|
|
|
|
|
Derivation_inen # fi
|
|
|
|
|
Derivation_lainen # fi
|
|
|
|
|
Derivation_ja # fi
|
|
|
|
|
Derivation_ton # fi
|
|
|
|
|
Derivation_vs # fi
|
|
|
|
|
Derivation_ttain # fi
|
|
|
|
|
Derivation_ttaa # fi
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Derivation
|
|
|
|
|
begin_Echo
|
2015-10-10 14:10:58 +03:00
|
|
|
Echo_rdp # U
|
|
|
|
|
Echo_ech # U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Echo
|
|
|
|
|
begin_Foreign
|
2015-10-10 14:10:58 +03:00
|
|
|
Foreign_foreign # cz, fi, U
|
|
|
|
|
Foreign_fscript # cz, fi, U
|
|
|
|
|
Foreign_tscript # cz, U
|
|
|
|
|
Foreign_yes # sl
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Foreign
|
|
|
|
|
begin_Hyph
|
2015-10-10 14:10:58 +03:00
|
|
|
Hyph_yes # cz, U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Hyph
|
|
|
|
|
begin_InfForm
|
2015-10-10 14:10:58 +03:00
|
|
|
InfForm_one # fi
|
|
|
|
|
InfForm_two # fi
|
|
|
|
|
InfForm_three # fi
|
2018-09-25 18:28:13 +03:00
|
|
|
end_InfForm
|
|
|
|
|
begin_NameType
|
2015-10-10 14:10:58 +03:00
|
|
|
NameType_geo # U, cz
|
|
|
|
|
NameType_prs # U, cz
|
|
|
|
|
NameType_giv # U, cz
|
|
|
|
|
NameType_sur # U, cz
|
|
|
|
|
NameType_nat # U, cz
|
|
|
|
|
NameType_com # U, cz
|
|
|
|
|
NameType_pro # U, cz
|
|
|
|
|
NameType_oth # U, cz
|
2018-09-25 18:28:13 +03:00
|
|
|
end_NameType
|
|
|
|
|
begin_NounType
|
2015-10-10 14:10:58 +03:00
|
|
|
NounType_com # U
|
|
|
|
|
NounType_prop # U
|
|
|
|
|
NounType_class # U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_NounType
|
|
|
|
|
begin_NumForm
|
2015-10-10 14:10:58 +03:00
|
|
|
NumForm_digit # cz, sl, U
|
|
|
|
|
NumForm_roman # cz, sl, U
|
|
|
|
|
NumForm_word # cz, sl, U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_NumForm
|
|
|
|
|
begin_NumValue
|
2015-10-10 14:10:58 +03:00
|
|
|
NumValue_one # cz, U
|
|
|
|
|
NumValue_two # cz, U
|
|
|
|
|
NumValue_three # cz, U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_NumValue
|
|
|
|
|
begin_PartForm
|
2015-10-10 14:10:58 +03:00
|
|
|
PartForm_pres # fi
|
|
|
|
|
PartForm_past # fi
|
|
|
|
|
PartForm_agt # fi
|
|
|
|
|
PartForm_neg # fi
|
2018-09-25 18:28:13 +03:00
|
|
|
end_PartForm
|
|
|
|
|
begin_PartType
|
2015-10-10 14:10:58 +03:00
|
|
|
PartType_mod # U
|
|
|
|
|
PartType_emp # U
|
|
|
|
|
PartType_res # U
|
|
|
|
|
PartType_inf # U
|
|
|
|
|
PartType_vbp # U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_PartType
|
|
|
|
|
begin_Polite
|
2015-10-10 14:10:58 +03:00
|
|
|
Polite_inf # bq, U
|
|
|
|
|
Polite_pol # bq, U
|
|
|
|
|
Polite_abs_inf # bq, U
|
|
|
|
|
Polite_abs_pol # bq, U
|
|
|
|
|
Polite_erg_inf # bq, U
|
|
|
|
|
Polite_erg_pol # bq, U
|
|
|
|
|
Polite_dat_inf # bq, U
|
|
|
|
|
Polite_dat_pol # bq, U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Polite
|
|
|
|
|
begin_Prefix
|
2015-10-10 14:10:58 +03:00
|
|
|
Prefix_yes # U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Prefix
|
|
|
|
|
begin_PrepCase
|
2015-10-10 14:10:58 +03:00
|
|
|
PrepCase_npr # cz
|
|
|
|
|
PrepCase_pre # U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_PrepCase
|
|
|
|
|
begin_PunctSide
|
2015-10-10 14:10:58 +03:00
|
|
|
PunctSide_ini # U
|
|
|
|
|
PunctSide_fin # U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_PunctSide
|
|
|
|
|
begin_PunctType
|
2015-10-10 14:10:58 +03:00
|
|
|
PunctType_peri # U
|
|
|
|
|
PunctType_qest # U
|
|
|
|
|
PunctType_excl # U
|
|
|
|
|
PunctType_quot # U
|
|
|
|
|
PunctType_brck # U
|
|
|
|
|
PunctType_comm # U
|
|
|
|
|
PunctType_colo # U
|
|
|
|
|
PunctType_semi # U
|
|
|
|
|
PunctType_dash # U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_PunctType
|
|
|
|
|
begin_Style
|
2015-10-10 14:10:58 +03:00
|
|
|
Style_arch # cz, fi, U
|
|
|
|
|
Style_rare # cz, fi, U
|
|
|
|
|
Style_poet # cz, U
|
|
|
|
|
Style_norm # cz, U
|
|
|
|
|
Style_coll # cz, U
|
|
|
|
|
Style_vrnc # cz, U
|
|
|
|
|
Style_sing # cz, U
|
|
|
|
|
Style_expr # cz, U
|
|
|
|
|
Style_derg # cz, U
|
|
|
|
|
Style_vulg # cz, U
|
|
|
|
|
Style_yes # fi, U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_Style
|
|
|
|
|
begin_StyleVariant
|
2015-10-10 14:10:58 +03:00
|
|
|
StyleVariant_styleShort # cz
|
|
|
|
|
StyleVariant_styleBound # cz, sl
|
2018-09-25 18:28:13 +03:00
|
|
|
end_StyleVariant
|
|
|
|
|
begin_VerbType
|
2015-10-10 14:10:58 +03:00
|
|
|
VerbType_aux # U
|
|
|
|
|
VerbType_cop # U
|
|
|
|
|
VerbType_mod # U
|
|
|
|
|
VerbType_light # U
|
2018-09-25 18:28:13 +03:00
|
|
|
end_VerbType
|
2015-08-26 20:17:35 +03:00
|
|
|
|
2018-09-25 00:57:41 +03:00
|
|
|
cdef struct RichTagC:
|
|
|
|
|
univ_pos_t pos
|
|
|
|
|
|
|
|
|
|
univ_morph_t abbr
|
|
|
|
|
univ_morph_t adp_type
|
|
|
|
|
univ_morph_t adv_type
|
|
|
|
|
univ_morph_t animacy
|
|
|
|
|
univ_morph_t aspect
|
|
|
|
|
univ_morph_t case
|
|
|
|
|
univ_morph_t conj_type
|
|
|
|
|
univ_morph_t connegative
|
|
|
|
|
univ_morph_t definite
|
|
|
|
|
univ_morph_t degree
|
|
|
|
|
univ_morph_t derivation
|
|
|
|
|
univ_morph_t echo
|
|
|
|
|
univ_morph_t foreign
|
|
|
|
|
univ_morph_t gender
|
|
|
|
|
univ_morph_t hyph
|
|
|
|
|
univ_morph_t inf_form
|
|
|
|
|
univ_morph_t mood
|
|
|
|
|
univ_morph_t negative
|
|
|
|
|
univ_morph_t number
|
|
|
|
|
univ_morph_t name_type
|
|
|
|
|
univ_morph_t num_form
|
|
|
|
|
univ_morph_t num_type
|
|
|
|
|
univ_morph_t num_value
|
|
|
|
|
univ_morph_t part_form
|
|
|
|
|
univ_morph_t part_type
|
|
|
|
|
univ_morph_t person
|
|
|
|
|
univ_morph_t polite
|
|
|
|
|
univ_morph_t polarity
|
|
|
|
|
univ_morph_t poss
|
|
|
|
|
univ_morph_t prefix
|
|
|
|
|
univ_morph_t prep_case
|
|
|
|
|
univ_morph_t pron_type
|
|
|
|
|
univ_morph_t punct_side
|
|
|
|
|
univ_morph_t punct_type
|
|
|
|
|
univ_morph_t reflex
|
|
|
|
|
univ_morph_t style
|
|
|
|
|
univ_morph_t style_variant
|
|
|
|
|
univ_morph_t tense
|
|
|
|
|
univ_morph_t verb_form
|
|
|
|
|
univ_morph_t voice
|
|
|
|
|
univ_morph_t verb_type
|
