mirror of
https://github.com/explosion/spaCy.git
synced 2025-02-23 23:20:52 +03:00
2f981d5af1
Remove corpus-specific tag maps from the language data for languages without custom tokenizers. For languages with custom word segmenters that also provide tags (Japanese and Korean), the tag maps for the custom tokenizers are kept as the default. The default tag maps for languages without custom tokenizers are now the default tag map from `lang/tag_map/py`, UPOS -> UPOS.
28 lines
803 B
Python
28 lines
803 B
Python
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS
|
|
from .stop_words import STOP_WORDS
|
|
from .lex_attrs import LEX_ATTRS
|
|
|
|
from ..tokenizer_exceptions import BASE_EXCEPTIONS
|
|
from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES
|
|
from ...language import Language
|
|
from ...attrs import LANG
|
|
from ...util import update_exc
|
|
|
|
|
|
class PortugueseDefaults(Language.Defaults):
|
|
lex_attr_getters = dict(Language.Defaults.lex_attr_getters)
|
|
lex_attr_getters[LANG] = lambda text: "pt"
|
|
lex_attr_getters.update(LEX_ATTRS)
|
|
tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS)
|
|
stop_words = STOP_WORDS
|
|
infixes = TOKENIZER_INFIXES
|
|
prefixes = TOKENIZER_PREFIXES
|
|
|
|
|
|
class Portuguese(Language):
|
|
lang = "pt"
|
|
Defaults = PortugueseDefaults
|
|
|
|
|
|
__all__ = ["Portuguese"]
|