mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-11 20:28:20 +03:00
417d45f5d0
Don't create lookup lemmatizer within Language class and just pass in the data so it can be set on Token creation
38 lines
1.2 KiB
Python
38 lines
1.2 KiB
Python
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
|
from .norm_exceptions import NORM_EXCEPTIONS
|
|
from .punctuation import TOKENIZER_INFIXES
|
|
from .tag_map import TAG_MAP
|
|
from .stop_words import STOP_WORDS
|
|
from .lemmatizer import LOOKUP
|
|
from .syntax_iterators import SYNTAX_ITERATORS
|
|
|
|
from ..tokenizer_exceptions import BASE_EXCEPTIONS
|
|
from ..norm_exceptions import BASE_NORMS
|
|
from ...language import Language
|
|
from ...attrs import LANG, NORM
|
|
from ...util import update_exc, add_lookups
|
|
|
|
|
|
class GermanDefaults(Language.Defaults):
|
|
lex_attr_getters = dict(Language.Defaults.lex_attr_getters)
|
|
lex_attr_getters[LANG] = lambda text: 'de'
|
|
lex_attr_getters[NORM] = add_lookups(Language.Defaults.lex_attr_getters[NORM],
|
|
NORM_EXCEPTIONS, BASE_NORMS)
|
|
tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS)
|
|
infixes = tuple(TOKENIZER_INFIXES)
|
|
tag_map = dict(TAG_MAP)
|
|
stop_words = set(STOP_WORDS)
|
|
syntax_iterators = dict(SYNTAX_ITERATORS)
|
|
lemma_lookup = dict(LOOKUP)
|
|
|
|
|
|
class German(Language):
|
|
lang = 'de'
|
|
Defaults = GermanDefaults
|
|
|
|
|
|
__all__ = ['German']
|