explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-05-29 18:23:06 +03:00
Code Issues Packages Projects Releases Wiki Activity

Modernise noun chunks tests and don't depend on models

Browse Source
This commit is contained in:
Ines Montani 2017-01-13 02:01:00 +01:00
parent 3ee97b5686
commit 49febd8c62
1 changed files with 61 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
spacy/tests/parser/test_noun_chunks.py
View File

@ -1,137 +1,76 @@
# coding: utf-8 # coding: utf-8
from __future__ import unicode_literals from __future__ import unicode_literals
from ..util import get_doc
import pytest import pytest
import numpy
from ...attrs import HEAD, DEP
@pytest.mark.models def test_parser_noun_chunks_standard(en_tokenizer):
class TestNounChunks: text = "A base phrase should be recognized."
@pytest.fixture(scope="class") heads = [2, 1, 3, 2, 1, 0, -1]
def ex1_en(self, EN): tags = ['DT', 'JJ', 'NN', 'MD', 'VB', 'VBN', '.']
example = EN.tokenizer.tokens_from_list('A base phrase should be recognized .'.split(' ')) deps = ['det', 'amod', 'nsubjpass', 'aux', 'auxpass', 'ROOT', 'punct']
EN.tagger.tag_from_strings(example, 'DT NN NN MD VB VBN .'.split(' '))
det,compound,nsubjpass,aux,auxpass,root,punct = tuple( EN.vocab.strings[l] for l in ['det','compound','nsubjpass','aux','auxpass','root','punct'] )
example.from_array([HEAD, DEP],
numpy.asarray(
[
[2, det],
[1, compound],
[3, nsubjpass],
[2, aux],
[1, auxpass],
[0, root],
[-1, punct]
], dtype='int32'))
return example
@pytest.fixture(scope="class") tokens = en_tokenizer(text)
def ex2_en(self, EN): doc = get_doc(tokens.vocab, [t.text for t in tokens], tags=tags, deps=deps, heads=heads)
example = EN.tokenizer.tokens_from_list('A base phrase and a good phrase are often the same .'.split(' ')) chunks = list(doc.noun_chunks)
EN.tagger.tag_from_strings(example, 'DT NN NN CC DT JJ NN VBP RB DT JJ .'.split(' ')) assert len(chunks) == 1
det,compound,nsubj,cc,amod,conj,root,advmod,attr,punct = tuple( EN.vocab.strings[l] for l in ['det','compound','nsubj','cc','amod','conj','root','advmod','attr','punct'] ) assert chunks[0].text_with_ws == "A base phrase "
example.from_array([HEAD, DEP],
numpy.asarray(
[
[2, det],
[1, compound],
[5, nsubj],
[-1, cc],
[1, det],
[1, amod],
[-4, conj],
[0, root],
[-1, advmod],
[1, det],
[-3, attr],
[-4, punct]
], dtype='int32'))
return example
@pytest.fixture(scope="class")
def ex3_en(self, EN):
example = EN.tokenizer.tokens_from_list('A phrase with another phrase occurs .'.split(' '))
EN.tagger.tag_from_strings(example, 'DT NN IN DT NN VBZ .'.split(' '))
det,nsubj,prep,pobj,root,punct = tuple( EN.vocab.strings[l] for l in ['det','nsubj','prep','pobj','root','punct'] )
example.from_array([HEAD, DEP],
numpy.asarray(
[
[1, det],
[4, nsubj],
[-1, prep],
[1, det],
[-2, pobj],
[0, root],
[-1, punct]
], dtype='int32'))
return example
@pytest.fixture(scope="class") def test_parser_noun_chunks_coordinated(en_tokenizer):
def ex1_de(self, DE): text = "A base phrase and a good phrase are often the same."
example = DE.tokenizer.tokens_from_list('Eine Tasse steht auf dem Tisch .'.split(' ')) heads = [2, 1, 5, -1, 2, 1, -4, 0, -1, 1, -3, -4]
DE.tagger.tag_from_strings(example, 'ART NN VVFIN APPR ART NN $.'.split(' ')) tags = ['DT', 'NN', 'NN', 'CC', 'DT', 'JJ', 'NN', 'VBP', 'RB', 'DT', 'JJ', '.']
nk,sb,root,mo,punct = tuple( DE.vocab.strings[l] for l in ['nk','sb','root','mo','punct']) deps = ['det', 'compound', 'nsubj', 'cc', 'det', 'amod', 'conj', 'ROOT', 'advmod', 'det', 'attr', 'punct']
example.from_array([HEAD, DEP],
numpy.asarray(
[
[1, nk],
[1, sb],
[0, root],
[-1, mo],
[1, nk],
[-2, nk],
[-3, punct]
], dtype='int32'))
return example
@pytest.fixture(scope="class") tokens = en_tokenizer(text)
def ex2_de(self, DE): doc = get_doc(tokens.vocab, [t.text for t in tokens], tags=tags, deps=deps, heads=heads)
example = DE.tokenizer.tokens_from_list('Die Sängerin singt mit einer Tasse Kaffee Arien .'.split(' ')) chunks = list(doc.noun_chunks)
DE.tagger.tag_from_strings(example, 'ART NN VVFIN APPR ART NN NN NN $.'.split(' ')) assert len(chunks) == 2
nk,sb,root,mo,punct,oa = tuple( DE.vocab.strings[l] for l in ['nk','sb','root','mo','punct','oa']) assert chunks[0].text_with_ws == "A base phrase "
example.from_array([HEAD, DEP], assert chunks[1].text_with_ws == "a good phrase "
numpy.asarray(
[
[1, nk],
[1, sb],
[0, root],
[-1, mo],
[1, nk],
[-2, nk],
[-1, nk],
[-5, oa],
[-6, punct]
], dtype='int32'))
return example
def test_en_standard_chunk(self, ex1_en):
chunks = list(ex1_en.noun_chunks)
assert len(chunks) == 1
assert chunks[0].string == 'A base phrase '
def test_en_coordinated_chunks(self, ex2_en): def test_parser_noun_chunks_pp_chunks(en_tokenizer):
chunks = list(ex2_en.noun_chunks) text = "A phrase with another phrase occurs."
assert len(chunks) == 2 heads = [1, 4, -1, 1, -2, 0, -1]
assert chunks[0].string == 'A base phrase ' tags = ['DT', 'NN', 'IN', 'DT', 'NN', 'VBZ', '.']
assert chunks[1].string == 'a good phrase ' deps = ['det', 'nsubj', 'prep', 'det', 'pobj', 'ROOT', 'punct']
def test_en_pp_chunks(self, ex3_en): tokens = en_tokenizer(text)
chunks = list(ex3_en.noun_chunks) doc = get_doc(tokens.vocab, [t.text for t in tokens], tags=tags, deps=deps, heads=heads)
assert len(chunks) == 2 chunks = list(doc.noun_chunks)
assert chunks[0].string == 'A phrase ' assert len(chunks) == 2
assert chunks[1].string == 'another phrase ' assert chunks[0].text_with_ws == "A phrase "
assert chunks[1].text_with_ws == "another phrase "
def test_de_standard_chunk(self, ex1_de):
chunks = list(ex1_de.noun_chunks)
assert len(chunks) == 2
assert chunks[0].string == 'Eine Tasse '
assert chunks[1].string == 'dem Tisch '
def test_de_extended_chunk(self, ex2_de): def test_parser_noun_chunks_standard_de(de_tokenizer):
chunks = list(ex2_de.noun_chunks) text = "Eine Tasse steht auf dem Tisch."
assert len(chunks) == 3 heads = [1, 1, 0, -1, 1, -2, -4]
assert chunks[0].string == 'Die Sängerin ' tags = ['ART', 'NN', 'VVFIN', 'APPR', 'ART', 'NN', '$.']
assert chunks[1].string == 'einer Tasse Kaffee ' deps = ['nk', 'sb', 'ROOT', 'mo', 'nk', 'nk', 'punct']
assert chunks[2].string == 'Arien '
tokens = de_tokenizer(text)
doc = get_doc(tokens.vocab, [t.text for t in tokens], tags=tags, deps=deps, heads=heads)
chunks = list(doc.noun_chunks)
assert len(chunks) == 2
assert chunks[0].text_with_ws == "Eine Tasse "
assert chunks[1].text_with_ws == "dem Tisch "
def test_de_extended_chunk(de_tokenizer):
text = "Die Sängerin singt mit einer Tasse Kaffee Arien."
heads = [1, 1, 0, -1, 1, -2, -1, -5, -6]
tags = ['ART', 'NN', 'VVFIN', 'APPR', 'ART', 'NN', 'NN', 'NN', '$.']
deps = ['nk', 'sb', 'ROOT', 'mo', 'nk', 'nk', 'nk', 'oa', 'punct']
tokens = de_tokenizer(text)
doc = get_doc(tokens.vocab, [t.text for t in tokens], tags=tags, deps=deps, heads=heads)
chunks = list(doc.noun_chunks)
assert len(chunks) == 3
assert chunks[0].text_with_ws == "Die Sängerin "
assert chunks[1].text_with_ws == "einer Tasse Kaffee "
assert chunks[2].text_with_ws == "Arien "