explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-04 21:50:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Use tokens from Jieba library

Browse Source
This commit is contained in:
Matthew Honnibal 2016-04-28 14:32:27 +02:00
parent 3186379253
commit 1ede19c75a
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spacy/zh/__init__.py
View File

@ -23,10 +23,11 @@ class CharacterTokenizer(Tokenizer):
def __call__(self, text):
return self.tokens_from_list(list(text))
class Chinese(Language):
lang = u'zh'
@classmethod
def default_tokenizer(cls, package, vocab):
'''Return Jieba-wrapper tokenizer.'''
return CharacterTokenizer.from_package(package, vocab)
return JiebaTokenizer.from_package(package, vocab)