mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-22 20:09:18 +03:00
39ebcd9ec9
* Refactor Chinese tokenizer configuration Refactor `ChineseTokenizer` configuration so that it uses a single `segmenter` setting to choose between character segmentation, jieba, and pkuseg. * replace `use_jieba`, `use_pkuseg`, `require_pkuseg` with the setting `segmenter` with the supported values: `char`, `jieba`, `pkuseg` * make the default segmenter plain character segmentation `char` (no additional libraries required) * Fix Chinese serialization test to use char default * Warn if attempting to customize other segmenter Add a warning if `Chinese.pkuseg_update_user_dict` is called when another segmenter is selected.
41 lines
1.1 KiB
Python
41 lines
1.1 KiB
Python
import pytest
|
|
from spacy.lang.zh import Chinese
|
|
from ...util import make_tempdir
|
|
|
|
|
|
def zh_tokenizer_serialize(zh_tokenizer):
|
|
tokenizer_bytes = zh_tokenizer.to_bytes()
|
|
nlp = Chinese()
|
|
nlp.tokenizer.from_bytes(tokenizer_bytes)
|
|
assert tokenizer_bytes == nlp.tokenizer.to_bytes()
|
|
|
|
with make_tempdir() as d:
|
|
file_path = d / "tokenizer"
|
|
zh_tokenizer.to_disk(file_path)
|
|
nlp = Chinese()
|
|
nlp.tokenizer.from_disk(file_path)
|
|
assert tokenizer_bytes == nlp.tokenizer.to_bytes()
|
|
|
|
|
|
def test_zh_tokenizer_serialize_char(zh_tokenizer_char):
|
|
zh_tokenizer_serialize(zh_tokenizer_char)
|
|
|
|
|
|
def test_zh_tokenizer_serialize_jieba(zh_tokenizer_jieba):
|
|
zh_tokenizer_serialize(zh_tokenizer_jieba)
|
|
|
|
|
|
@pytest.mark.slow
|
|
def test_zh_tokenizer_serialize_pkuseg_with_processors(zh_tokenizer_pkuseg):
|
|
nlp = Chinese(
|
|
meta={
|
|
"tokenizer": {
|
|
"config": {
|
|
"segmenter": "pkuseg",
|
|
"pkuseg_model": "medicine",
|
|
}
|
|
}
|
|
}
|
|
)
|
|
zh_tokenizer_serialize(nlp.tokenizer)
|