mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-15 14:17:58 +03:00
c5de9b463a
* Fix incorrect pickling of Japanese and Korean pipelines, which led to the entire pipeline being reset if pickled * Enable pickling of Vietnamese tokenizer * Update tokenizer APIs for Chinese, Japanese, Korean, Thai, and Vietnamese so that only the `Vocab` is required for initialization
55 lines
1.3 KiB
Python
55 lines
1.3 KiB
Python
from .stop_words import STOP_WORDS
|
|
from .lex_attrs import LEX_ATTRS
|
|
from ...language import Language
|
|
from ...tokens import Doc
|
|
from ...util import DummyTokenizer, registry, load_config_from_str
|
|
from ...vocab import Vocab
|
|
|
|
|
|
DEFAULT_CONFIG = """
|
|
[nlp]
|
|
|
|
[nlp.tokenizer]
|
|
@tokenizers = "spacy.th.ThaiTokenizer"
|
|
"""
|
|
|
|
|
|
@registry.tokenizers("spacy.th.ThaiTokenizer")
|
|
def create_thai_tokenizer():
|
|
def thai_tokenizer_factory(nlp):
|
|
return ThaiTokenizer(nlp.vocab)
|
|
|
|
return thai_tokenizer_factory
|
|
|
|
|
|
class ThaiTokenizer(DummyTokenizer):
|
|
def __init__(self, vocab: Vocab) -> None:
|
|
try:
|
|
from pythainlp.tokenize import word_tokenize
|
|
except ImportError:
|
|
raise ImportError(
|
|
"The Thai tokenizer requires the PyThaiNLP library: "
|
|
"https://github.com/PyThaiNLP/pythainlp"
|
|
) from None
|
|
self.word_tokenize = word_tokenize
|
|
self.vocab = vocab
|
|
|
|
def __call__(self, text: str) -> Doc:
|
|
words = list(self.word_tokenize(text))
|
|
spaces = [False] * len(words)
|
|
return Doc(self.vocab, words=words, spaces=spaces)
|
|
|
|
|
|
class ThaiDefaults(Language.Defaults):
|
|
config = load_config_from_str(DEFAULT_CONFIG)
|
|
lex_attr_getters = LEX_ATTRS
|
|
stop_words = STOP_WORDS
|
|
|
|
|
|
class Thai(Language):
|
|
lang = "th"
|
|
Defaults = ThaiDefaults
|
|
|
|
|
|
__all__ = ["Thai"]
|