mirror of
https://github.com/explosion/spaCy.git
synced 2025-11-10 21:07:53 +03:00
a5cd203284
* Reduce stored lexemes data, move feats to lookups
* Move non-derivable lexemes features (`norm / cluster / prob`) to
`spacy-lookups-data` as lookups
* Get/set `norm` in both lookups and `LexemeC`, serialize in lookups
* Remove `cluster` and `prob` from `LexemesC`, get/set/serialize in
lookups only
* Remove serialization of lexemes data as `vocab/lexemes.bin`
* Remove `SerializedLexemeC`
* Remove `Lexeme.to_bytes/from_bytes`
* Modify normalization exception loading:
* Always create `Vocab.lookups` table `lexeme_norm` for
normalization exceptions
* Load base exceptions from `lang.norm_exceptions`, but load
language-specific exceptions from lookups
* Set `lex_attr_getter[NORM]` including new lookups table in
`BaseDefaults.create_vocab()` and when deserializing `Vocab`
* Remove all cached lexemes when deserializing vocab to override
existing normalizations with the new normalizations (as a replacement
for the previous step that replaced all lexemes data with the
deserialized data)
* Skip English normalization test
Skip English normalization test because the data is now in
`spacy-lookups-data`.
* Remove norm exceptions
Moved to spacy-lookups-data.
* Move norm exceptions test to spacy-lookups-data
* Load extra lookups from spacy-lookups-data lazily
Load extra lookups (currently for cluster and prob) lazily from the
entry point `lg_extra` as `Vocab.lookups_extra`.
* Skip creating lexeme cache on load
To improve model loading times, do not create the full lexeme cache when
loading. The lexemes will be created on demand when processing.
* Identify numeric values in Lexeme.set_attrs()
With the removal of a special case for `PROB`, also identify `float` to
avoid trying to convert it with the `StringStore`.
* Skip lexeme cache init in from_bytes
* Unskip and update lookups tests for python3.6+
* Update vocab pickle to include lookups_extra
* Update vocab serialization tests
Check strings rather than lexemes since lexemes aren't initialized
automatically, account for addition of "_SP".
* Re-skip lookups test because of python3.5
* Skip PROB/float values in Lexeme.set_attrs
* Convert is_oov from lexeme flag to lex in vectors
Instead of storing `is_oov` as a lexeme flag, `is_oov` reports whether
the lexeme has a vector.
Co-authored-by: Matthew Honnibal <honnibal+gh@gmail.com>
136 lines
2.8 KiB
Cython
136 lines
2.8 KiB
Cython
from libc.stdint cimport uint8_t, uint32_t, int32_t, uint64_t
|
|
|
|
from .typedefs cimport flags_t, attr_t, hash_t
|
|
from .parts_of_speech cimport univ_pos_t
|
|
|
|
from libcpp.vector cimport vector
|
|
from libc.stdint cimport int32_t, int64_t
|
|
|
|
|
|
|
|
cdef struct LexemeC:
|
|
flags_t flags
|
|
|
|
attr_t lang
|
|
|
|
attr_t id
|
|
attr_t length
|
|
|
|
attr_t orth
|
|
attr_t lower
|
|
attr_t norm
|
|
attr_t shape
|
|
attr_t prefix
|
|
attr_t suffix
|
|
|
|
|
|
cdef struct SpanC:
|
|
hash_t id
|
|
int start
|
|
int end
|
|
int start_char
|
|
int end_char
|
|
attr_t label
|
|
attr_t kb_id
|
|
|
|
|
|
cdef struct TokenC:
|
|
const LexemeC* lex
|
|
uint64_t morph
|
|
univ_pos_t pos
|
|
bint spacy
|
|
attr_t tag
|
|
int idx
|
|
attr_t lemma
|
|
attr_t norm
|
|
int head
|
|
attr_t dep
|
|
|
|
uint32_t l_kids
|
|
uint32_t r_kids
|
|
uint32_t l_edge
|
|
uint32_t r_edge
|
|
|
|
int sent_start
|
|
int ent_iob
|
|
attr_t ent_type # TODO: Is there a better way to do this? Multiple sources of truth..
|
|
attr_t ent_kb_id
|
|
hash_t ent_id
|
|
|
|
|
|
cdef struct MorphAnalysisC:
|
|
univ_pos_t pos
|
|
int length
|
|
|
|
attr_t abbr
|
|
attr_t adp_type
|
|
attr_t adv_type
|
|
attr_t animacy
|
|
attr_t aspect
|
|
attr_t case
|
|
attr_t conj_type
|
|
attr_t connegative
|
|
attr_t definite
|
|
attr_t degree
|
|
attr_t derivation
|
|
attr_t echo
|
|
attr_t foreign
|
|
attr_t gender
|
|
attr_t hyph
|
|
attr_t inf_form
|
|
attr_t mood
|
|
attr_t negative
|
|
attr_t number
|
|
attr_t name_type
|
|
attr_t noun_type
|
|
attr_t num_form
|
|
attr_t num_type
|
|
attr_t num_value
|
|
attr_t part_form
|
|
attr_t part_type
|
|
attr_t person
|
|
attr_t polite
|
|
attr_t polarity
|
|
attr_t poss
|
|
attr_t prefix
|
|
attr_t prep_case
|
|
attr_t pron_type
|
|
attr_t punct_side
|
|
attr_t punct_type
|
|
attr_t reflex
|
|
attr_t style
|
|
attr_t style_variant
|
|
attr_t tense
|
|
attr_t typo
|
|
attr_t verb_form
|
|
attr_t voice
|
|
attr_t verb_type
|
|
|
|
# Internal struct, for storage and disambiguation of entities.
|
|
cdef struct KBEntryC:
|
|
|
|
# The hash of this entry's unique ID/name in the kB
|
|
hash_t entity_hash
|
|
|
|
# Allows retrieval of the entity vector, as an index into a vectors table of the KB.
|
|
# Can be expanded later to refer to multiple rows (compositional model to reduce storage footprint).
|
|
int32_t vector_index
|
|
|
|
# Allows retrieval of a struct of non-vector features.
|
|
# This is currently not implemented and set to -1 for the common case where there are no features.
|
|
int32_t feats_row
|
|
|
|
# log probability of entity, based on corpus frequency
|
|
float freq
|
|
|
|
|
|
# Each alias struct stores a list of Entry pointers with their prior probabilities
|
|
# for this specific mention/alias.
|
|
cdef struct AliasC:
|
|
|
|
# All entry candidates for this alias
|
|
vector[int64_t] entry_indices
|
|
|
|
# Prior probability P(entity|alias) - should sum up to (at most) 1.
|
|
vector[float] probs
|