explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 18:06:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

[ja] Use user_details instead of a wrapper class

Browse Source 
Instead of using a JapaneseDoc wrapper class to store Mecab output,
stash it in `user_data`. -POLM
This commit is contained in:
Paul O'Leary McCann 2017-10-16 00:24:34 +09:00
parent 43eedf73f2
commit 71ae8013ec
1 changed files with 6 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
spacy/ja/__init__.py
View File

@ -16,12 +16,7 @@ from collections import namedtuple
ShortUnitWord = namedtuple('ShortUnitWord', ['surface', 'base_form', 'part_of_speech']) ShortUnitWord = namedtuple('ShortUnitWord', ['surface', 'base_form', 'part_of_speech'])
class JapaneseDoc(Doc): DETAILS_KEY = 'mecab_details'
def __init__(self, detailed_tokens, vocab, words=None, spaces=None, orths_and_spaces=None):
super(JapaneseDoc, self).__init__(vocab, words, spaces, orths_and_spaces)
# This saves tokenizer output so mecab doesn't have to be called again
# when determining POS tags.
self.detailed_tokens = detailed_tokens
def try_mecab_import(): def try_mecab_import():
"""Mecab is required for Japanese support, so check for it. """Mecab is required for Japanese support, so check for it.
@ -43,7 +38,10 @@ class JapaneseTokenizer(object):
def __call__(self, text): def __call__(self, text):
dtokens = detailed_tokens(self.tokenizer, text) dtokens = detailed_tokens(self.tokenizer, text)
words = [x.surface for x in dtokens] words = [x.surface for x in dtokens]
return JapaneseDoc(dtokens, self.vocab, words=words, spaces=[False]*len(words)) doc = Doc(self.vocab, words=words, spaces=[False]*len(words))
# stash details tokens for tagger to use
doc.user_data[DETAILS_KEY] = dtokens
return doc
def resolve_pos(token): def resolve_pos(token):
"""If necessary, add a field to the POS tag for UD mapping. """If necessary, add a field to the POS tag for UD mapping.
@ -99,7 +97,7 @@ class JapaneseTagger(object):
# 1. get raw JP tags # 1. get raw JP tags
# 2. add features to tags as necessary for UD # 2. add features to tags as necessary for UD
dtokens = tokens.detailed_tokens dtokens = tokens.user_data[DETAILS_KEY]
rawtags = list(map(resolve_pos, dtokens)) rawtags = list(map(resolve_pos, dtokens))
self.tagger.tag_from_strings(tokens, rawtags) self.tagger.tag_from_strings(tokens, rawtags)