mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-11 12:18:04 +03:00
a5cd203284
* Reduce stored lexemes data, move feats to lookups
* Move non-derivable lexemes features (`norm / cluster / prob`) to
`spacy-lookups-data` as lookups
* Get/set `norm` in both lookups and `LexemeC`, serialize in lookups
* Remove `cluster` and `prob` from `LexemesC`, get/set/serialize in
lookups only
* Remove serialization of lexemes data as `vocab/lexemes.bin`
* Remove `SerializedLexemeC`
* Remove `Lexeme.to_bytes/from_bytes`
* Modify normalization exception loading:
* Always create `Vocab.lookups` table `lexeme_norm` for
normalization exceptions
* Load base exceptions from `lang.norm_exceptions`, but load
language-specific exceptions from lookups
* Set `lex_attr_getter[NORM]` including new lookups table in
`BaseDefaults.create_vocab()` and when deserializing `Vocab`
* Remove all cached lexemes when deserializing vocab to override
existing normalizations with the new normalizations (as a replacement
for the previous step that replaced all lexemes data with the
deserialized data)
* Skip English normalization test
Skip English normalization test because the data is now in
`spacy-lookups-data`.
* Remove norm exceptions
Moved to spacy-lookups-data.
* Move norm exceptions test to spacy-lookups-data
* Load extra lookups from spacy-lookups-data lazily
Load extra lookups (currently for cluster and prob) lazily from the
entry point `lg_extra` as `Vocab.lookups_extra`.
* Skip creating lexeme cache on load
To improve model loading times, do not create the full lexeme cache when
loading. The lexemes will be created on demand when processing.
* Identify numeric values in Lexeme.set_attrs()
With the removal of a special case for `PROB`, also identify `float` to
avoid trying to convert it with the `StringStore`.
* Skip lexeme cache init in from_bytes
* Unskip and update lookups tests for python3.6+
* Update vocab pickle to include lookups_extra
* Update vocab serialization tests
Check strings rather than lexemes since lexemes aren't initialized
automatically, account for addition of "_SP".
* Re-skip lookups test because of python3.5
* Skip PROB/float values in Lexeme.set_attrs
* Convert is_oov from lexeme flag to lex in vectors
Instead of storing `is_oov` as a lexeme flag, `is_oov` reports whether
the lexeme has a vector.
Co-authored-by: Matthew Honnibal <honnibal+gh@gmail.com>
49 lines
1.5 KiB
Python
49 lines
1.5 KiB
Python
# coding: utf8
|
||
from __future__ import unicode_literals
|
||
|
||
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
||
from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
|
||
from .punctuation import TOKENIZER_INFIXES
|
||
from .tag_map import TAG_MAP
|
||
from .stop_words import STOP_WORDS
|
||
from .syntax_iterators import SYNTAX_ITERATORS
|
||
|
||
from ..tokenizer_exceptions import BASE_EXCEPTIONS
|
||
from ...language import Language
|
||
from ...attrs import LANG
|
||
from ...util import update_exc
|
||
|
||
|
||
class GermanDefaults(Language.Defaults):
|
||
lex_attr_getters = dict(Language.Defaults.lex_attr_getters)
|
||
lex_attr_getters[LANG] = lambda text: "de"
|
||
tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS)
|
||
prefixes = TOKENIZER_PREFIXES
|
||
suffixes = TOKENIZER_SUFFIXES
|
||
infixes = TOKENIZER_INFIXES
|
||
tag_map = TAG_MAP
|
||
stop_words = STOP_WORDS
|
||
syntax_iterators = SYNTAX_ITERATORS
|
||
single_orth_variants = [
|
||
{"tags": ["$("], "variants": ["…", "..."]},
|
||
{"tags": ["$("], "variants": ["-", "—", "–", "--", "---", "——"]},
|
||
]
|
||
paired_orth_variants = [
|
||
{
|
||
"tags": ["$("],
|
||
"variants": [("'", "'"), (",", "'"), ("‚", "‘"), ("›", "‹"), ("‹", "›")],
|
||
},
|
||
{
|
||
"tags": ["$("],
|
||
"variants": [("``", "''"), ('"', '"'), ("„", "“"), ("»", "«"), ("«", "»")],
|
||
},
|
||
]
|
||
|
||
|
||
class German(Language):
|
||
lang = "de"
|
||
Defaults = GermanDefaults
|
||
|
||
|
||
__all__ = ["German"]
|