mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-16 06:37:04 +03:00
a5cd203284
* Reduce stored lexemes data, move feats to lookups
* Move non-derivable lexemes features (`norm / cluster / prob`) to
`spacy-lookups-data` as lookups
* Get/set `norm` in both lookups and `LexemeC`, serialize in lookups
* Remove `cluster` and `prob` from `LexemesC`, get/set/serialize in
lookups only
* Remove serialization of lexemes data as `vocab/lexemes.bin`
* Remove `SerializedLexemeC`
* Remove `Lexeme.to_bytes/from_bytes`
* Modify normalization exception loading:
* Always create `Vocab.lookups` table `lexeme_norm` for
normalization exceptions
* Load base exceptions from `lang.norm_exceptions`, but load
language-specific exceptions from lookups
* Set `lex_attr_getter[NORM]` including new lookups table in
`BaseDefaults.create_vocab()` and when deserializing `Vocab`
* Remove all cached lexemes when deserializing vocab to override
existing normalizations with the new normalizations (as a replacement
for the previous step that replaced all lexemes data with the
deserialized data)
* Skip English normalization test
Skip English normalization test because the data is now in
`spacy-lookups-data`.
* Remove norm exceptions
Moved to spacy-lookups-data.
* Move norm exceptions test to spacy-lookups-data
* Load extra lookups from spacy-lookups-data lazily
Load extra lookups (currently for cluster and prob) lazily from the
entry point `lg_extra` as `Vocab.lookups_extra`.
* Skip creating lexeme cache on load
To improve model loading times, do not create the full lexeme cache when
loading. The lexemes will be created on demand when processing.
* Identify numeric values in Lexeme.set_attrs()
With the removal of a special case for `PROB`, also identify `float` to
avoid trying to convert it with the `StringStore`.
* Skip lexeme cache init in from_bytes
* Unskip and update lookups tests for python3.6+
* Update vocab pickle to include lookups_extra
* Update vocab serialization tests
Check strings rather than lexemes since lexemes aren't initialized
automatically, account for addition of "_SP".
* Re-skip lookups test because of python3.5
* Skip PROB/float values in Lexeme.set_attrs
* Convert is_oov from lexeme flag to lex in vectors
Instead of storing `is_oov` as a lexeme flag, `is_oov` reports whether
the lexeme has a vector.
Co-authored-by: Matthew Honnibal <honnibal+gh@gmail.com>
173 lines
5.0 KiB
Cython
173 lines
5.0 KiB
Cython
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
IDS = {
|
|
"": NULL_ATTR,
|
|
"IS_ALPHA": IS_ALPHA,
|
|
"IS_ASCII": IS_ASCII,
|
|
"IS_DIGIT": IS_DIGIT,
|
|
"IS_LOWER": IS_LOWER,
|
|
"IS_PUNCT": IS_PUNCT,
|
|
"IS_SPACE": IS_SPACE,
|
|
"IS_TITLE": IS_TITLE,
|
|
"IS_UPPER": IS_UPPER,
|
|
"LIKE_URL": LIKE_URL,
|
|
"LIKE_NUM": LIKE_NUM,
|
|
"LIKE_EMAIL": LIKE_EMAIL,
|
|
"IS_STOP": IS_STOP,
|
|
"IS_OOV_DEPRECATED": IS_OOV_DEPRECATED,
|
|
"IS_BRACKET": IS_BRACKET,
|
|
"IS_QUOTE": IS_QUOTE,
|
|
"IS_LEFT_PUNCT": IS_LEFT_PUNCT,
|
|
"IS_RIGHT_PUNCT": IS_RIGHT_PUNCT,
|
|
"IS_CURRENCY": IS_CURRENCY,
|
|
"FLAG19": FLAG19,
|
|
"FLAG20": FLAG20,
|
|
"FLAG21": FLAG21,
|
|
"FLAG22": FLAG22,
|
|
"FLAG23": FLAG23,
|
|
"FLAG24": FLAG24,
|
|
"FLAG25": FLAG25,
|
|
"FLAG26": FLAG26,
|
|
"FLAG27": FLAG27,
|
|
"FLAG28": FLAG28,
|
|
"FLAG29": FLAG29,
|
|
"FLAG30": FLAG30,
|
|
"FLAG31": FLAG31,
|
|
"FLAG32": FLAG32,
|
|
"FLAG33": FLAG33,
|
|
"FLAG34": FLAG34,
|
|
"FLAG35": FLAG35,
|
|
"FLAG36": FLAG36,
|
|
"FLAG37": FLAG37,
|
|
"FLAG38": FLAG38,
|
|
"FLAG39": FLAG39,
|
|
"FLAG40": FLAG40,
|
|
"FLAG41": FLAG41,
|
|
"FLAG42": FLAG42,
|
|
"FLAG43": FLAG43,
|
|
"FLAG44": FLAG44,
|
|
"FLAG45": FLAG45,
|
|
"FLAG46": FLAG46,
|
|
"FLAG47": FLAG47,
|
|
"FLAG48": FLAG48,
|
|
"FLAG49": FLAG49,
|
|
"FLAG50": FLAG50,
|
|
"FLAG51": FLAG51,
|
|
"FLAG52": FLAG52,
|
|
"FLAG53": FLAG53,
|
|
"FLAG54": FLAG54,
|
|
"FLAG55": FLAG55,
|
|
"FLAG56": FLAG56,
|
|
"FLAG57": FLAG57,
|
|
"FLAG58": FLAG58,
|
|
"FLAG59": FLAG59,
|
|
"FLAG60": FLAG60,
|
|
"FLAG61": FLAG61,
|
|
"FLAG62": FLAG62,
|
|
"FLAG63": FLAG63,
|
|
|
|
"ID": ID,
|
|
"ORTH": ORTH,
|
|
"LOWER": LOWER,
|
|
"NORM": NORM,
|
|
"SHAPE": SHAPE,
|
|
"PREFIX": PREFIX,
|
|
"SUFFIX": SUFFIX,
|
|
|
|
"LENGTH": LENGTH,
|
|
"CLUSTER": CLUSTER,
|
|
"LEMMA": LEMMA,
|
|
"POS": POS,
|
|
"TAG": TAG,
|
|
"DEP": DEP,
|
|
"ENT_IOB": ENT_IOB,
|
|
"ENT_TYPE": ENT_TYPE,
|
|
"ENT_ID": ENT_ID,
|
|
"ENT_KB_ID": ENT_KB_ID,
|
|
"HEAD": HEAD,
|
|
"SENT_START": SENT_START,
|
|
"SENT_END": SENT_END,
|
|
"SPACY": SPACY,
|
|
"PROB": PROB,
|
|
"LANG": LANG,
|
|
"IDX": IDX
|
|
}
|
|
|
|
|
|
# ATTR IDs, in order of the symbol
|
|
NAMES = [key for key, value in sorted(IDS.items(), key=lambda item: item[1])]
|
|
locals().update(IDS)
|
|
|
|
|
|
def intify_attrs(stringy_attrs, strings_map=None, _do_deprecated=False):
|
|
"""
|
|
Normalize a dictionary of attributes, converting them to ints.
|
|
|
|
stringy_attrs (dict): Dictionary keyed by attribute string names. Values
|
|
can be ints or strings.
|
|
strings_map (StringStore): Defaults to None. If provided, encodes string
|
|
values into ints.
|
|
RETURNS (dict): Attributes dictionary with keys and optionally values
|
|
converted to ints.
|
|
"""
|
|
inty_attrs = {}
|
|
if _do_deprecated:
|
|
if 'F' in stringy_attrs:
|
|
stringy_attrs["ORTH"] = stringy_attrs.pop("F")
|
|
if 'L' in stringy_attrs:
|
|
stringy_attrs["LEMMA"] = stringy_attrs.pop("L")
|
|
if 'pos' in stringy_attrs:
|
|
stringy_attrs["TAG"] = stringy_attrs.pop("pos")
|
|
if 'morph' in stringy_attrs:
|
|
morphs = stringy_attrs.pop('morph')
|
|
if 'number' in stringy_attrs:
|
|
stringy_attrs.pop('number')
|
|
if 'tenspect' in stringy_attrs:
|
|
stringy_attrs.pop('tenspect')
|
|
morph_keys = [
|
|
'PunctType', 'PunctSide', 'Other', 'Degree', 'AdvType', 'Number',
|
|
'VerbForm', 'PronType', 'Aspect', 'Tense', 'PartType', 'Poss',
|
|
'Hyph', 'ConjType', 'NumType', 'Foreign', 'VerbType', 'NounType',
|
|
'Gender', 'Mood', 'Negative', 'Tense', 'Voice', 'Abbr',
|
|
'Derivation', 'Echo', 'Foreign', 'NameType', 'NounType', 'NumForm',
|
|
'NumValue', 'PartType', 'Polite', 'StyleVariant',
|
|
'PronType', 'AdjType', 'Person', 'Variant', 'AdpType',
|
|
'Reflex', 'Negative', 'Mood', 'Aspect', 'Case',
|
|
'Polarity', 'PrepCase', 'Animacy' # U20
|
|
]
|
|
for key in morph_keys:
|
|
if key in stringy_attrs:
|
|
stringy_attrs.pop(key)
|
|
elif key.lower() in stringy_attrs:
|
|
stringy_attrs.pop(key.lower())
|
|
elif key.upper() in stringy_attrs:
|
|
stringy_attrs.pop(key.upper())
|
|
for name, value in stringy_attrs.items():
|
|
int_key = intify_attr(name)
|
|
if int_key is not None:
|
|
if strings_map is not None and isinstance(value, basestring):
|
|
if hasattr(strings_map, 'add'):
|
|
value = strings_map.add(value)
|
|
else:
|
|
value = strings_map[value]
|
|
inty_attrs[int_key] = value
|
|
return inty_attrs
|
|
|
|
|
|
def intify_attr(name):
|
|
"""
|
|
Normalize an attribute name, converting it to int.
|
|
|
|
stringy_attr (string): Attribute string name. Can also be int (will then be left unchanged)
|
|
RETURNS (int): int representation of the attribute, or None if it couldn't be converted.
|
|
"""
|
|
if isinstance(name, int):
|
|
return name
|
|
elif name in IDS:
|
|
return IDS[name]
|
|
elif name.upper() in IDS:
|
|
return IDS[name.upper()]
|
|
return None
|