mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-25 09:26:27 +03:00
f7471abd82
* Add pkuseg and serialization support for Chinese
Add support for pkuseg alongside jieba
* Specify model through `Language` meta:
* split on characters (if no word segmentation packages are installed)
```
Chinese(meta={"tokenizer": {"config": {"use_jieba": False, "use_pkuseg": False}}})
```
* jieba (remains the default tokenizer if installed)
```
Chinese()
Chinese(meta={"tokenizer": {"config": {"use_jieba": True}}}) # explicit
```
* pkuseg
```
Chinese(meta={"tokenizer": {"config": {"pkuseg_model": "default", "use_jieba": False, "use_pkuseg": True}}})
```
* The new tokenizer setting `require_pkuseg` is used to override
`use_jieba` default, which is intended for models that provide a pkuseg
model:
```
nlp_pkuseg = Chinese(meta={"tokenizer": {"config": {"pkuseg_model": "default", "require_pkuseg": True}}})
nlp = Chinese() # has `use_jieba` as `True` by default
nlp.from_bytes(nlp_pkuseg.to_bytes()) # `require_pkuseg` overrides `use_jieba` when calling the tokenizer
```
Add support for serialization of tokenizer settings and pkuseg model, if
loaded
* Add sorting for `Language.to_bytes()` serialization of `Language.meta`
so that the (emptied, but still present) tokenizer metadata is in a
consistent position in the serialized data
Extend tests to cover all three tokenizer configurations and
serialization
* Fix from_disk and tests without jieba or pkuseg
* Load cfg first and only show error if `use_pkuseg`
* Fix blank/default initialization in serialization tests
* Explicitly initialize jieba's cache on init
* Add serialization for pkuseg pre/postprocessors
* Reformat pkuseg install message
46 lines
1.6 KiB
Python
46 lines
1.6 KiB
Python
# coding: utf-8
|
||
from __future__ import unicode_literals
|
||
|
||
import pytest
|
||
|
||
|
||
# fmt: off
|
||
TEXTS = ("作为语言而言，为世界使用人数最多的语言，目前世界有五分之一人口做为母语。",)
|
||
JIEBA_TOKENIZER_TESTS = [
|
||
(TEXTS[0],
|
||
['作为', '语言', '而言', '，', '为', '世界', '使用', '人', '数最多',
|
||
'的', '语言', '，', '目前', '世界', '有', '五分之一', '人口', '做',
|
||
'为', '母语', '。']),
|
||
]
|
||
PKUSEG_TOKENIZER_TESTS = [
|
||
(TEXTS[0],
|
||
['作为', '语言', '而言', '，', '为', '世界', '使用', '人数', '最多',
|
||
'的', '语言', '，', '目前', '世界', '有', '五分之一', '人口', '做为',
|
||
'母语', '。']),
|
||
]
|
||
# fmt: on
|
||
|
||
|
||
@pytest.mark.parametrize("text", TEXTS)
|
||
def test_zh_tokenizer_char(zh_tokenizer_char, text):
|
||
tokens = [token.text for token in zh_tokenizer_char(text)]
|
||
assert tokens == list(text)
|
||
|
||
|
||
@pytest.mark.parametrize("text,expected_tokens", JIEBA_TOKENIZER_TESTS)
|
||
def test_zh_tokenizer_jieba(zh_tokenizer_jieba, text, expected_tokens):
|
||
tokens = [token.text for token in zh_tokenizer_jieba(text)]
|
||
assert tokens == expected_tokens
|
||
|
||
|
||
@pytest.mark.parametrize("text,expected_tokens", PKUSEG_TOKENIZER_TESTS)
|
||
def test_zh_tokenizer_pkuseg(zh_tokenizer_pkuseg, text, expected_tokens):
|
||
tokens = [token.text for token in zh_tokenizer_pkuseg(text)]
|
||
assert tokens == expected_tokens
|
||
|
||
|
||
def test_extra_spaces(zh_tokenizer_char):
|
||
# note: three spaces after "I"
|
||
tokens = zh_tokenizer_char("I like cheese.")
|
||
assert tokens[1].orth_ == " "
|