explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-27 17:54:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update tagger & tokenizer tests

Browse Source 
Tagger is now parametrized and has two sentences with more tag coverage.

The tokenizer tests are updated to reflect differences in tokenization
between IPAdic and Unidic. -POLM
This commit is contained in:
Paul O'Leary McCann 2017-08-22 00:03:11 +09:00
parent adfd987316
commit bcf2b9b4f5
2 changed files with 30 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
spacy/tests/ja/test_tagger.py
View File

@ -3,8 +3,31 @@ from __future__ import unicode_literals
import pytest import pytest
def test_japanese_tagger(japanese): TAGGER_TESTS = [
doc = japanese.make_doc("このファイルには小さなテストが入っているよ") ('あれならそこにあるよ',
# note these both have the same raw tag, '連体詞,*,*,*' (('代名詞,*,*,*', 'PRON'),
assert doc[0].pos_ == "DET" ('助動詞,*,*,*', 'AUX'),
assert doc[4].pos_ == "ADJ" ('代名詞,*,*,*', 'PRON'),
('助詞,格助詞,*,*', 'ADP'),
('動詞,非自立可能,*,*', 'VERB'),
('助詞,終助詞,*,*', 'PART'))),
('このファイルには小さなテストが入っているよ',
(('連体詞,*,*,*,DET', 'DET'),
('名詞,普通名詞,サ変可能,*', 'NOUN'),
('助詞,格助詞,*,*', 'ADP'),
('助詞,係助詞,*,*', 'ADP'),
('連体詞,*,*,*,ADJ', 'ADJ'),
('名詞,普通名詞,サ変可能,*', 'NOUN'),
('助詞,格助詞,*,*', 'ADP'),
('動詞,一般,*,*', 'VERB'),
('助詞,接続助詞,*,*', 'SCONJ'),
('動詞,非自立可能,*,*', 'VERB'),
('助詞,終助詞,*,*', 'PART')))
]
@pytest.mark.parametrize('text,expected_tags', TAGGER_TESTS)
def test_japanese_tagger(japanese, text, expected_tags):
tokens = japanese.make_doc(text)
assert len(tokens) == len(expected_tags)
for token, res in zip(tokens, expected_tags):
assert token.tag_ == res[0] and token.pos_ == res[1]

4
spacy/tests/ja/test_tokenizer.py
View File

@ -4,10 +4,10 @@ from __future__ import unicode_literals
import pytest import pytest
TOKENIZER_TESTS = [ TOKENIZER_TESTS = [
("日本語だよ", ['日本語', 'だ', 'よ']), ("日本語だよ", ['日本', '語', 'だ', 'よ']),
("東京タワーの近くに住んでいます。", ['東京', 'タワー', 'の', '近く', 'に', '住ん', 'で', 'い', 'ます', '。']), ("東京タワーの近くに住んでいます。", ['東京', 'タワー', 'の', '近く', 'に', '住ん', 'で', 'い', 'ます', '。']),
("吾輩は猫である。", ['吾輩', 'は', '猫', 'で', 'ある', '。']), ("吾輩は猫である。", ['吾輩', 'は', '猫', 'で', 'ある', '。']),
("月に代わって、お仕置きよ!", ['月', 'に', '代わっ', 'て', '、', 'お仕置き', 'よ', '!']), ("月に代わって、お仕置きよ!", ['月', 'に', '代わっ', 'て', '、', 'お', '仕置き', 'よ', '!']),
("すもももももももものうち", ['すもも', 'も', 'もも', 'も', 'もも', 'の', 'うち']) ("すもももももももものうち", ['すもも', 'も', 'もも', 'も', 'もも', 'の', 'うち'])
] ]