From 6e883f4c0051af765cd32653d6880363162e8a31 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Wed, 11 Jan 2017 18:05:36 +0100 Subject: [PATCH] Modernise Doc API tests and don't depend on models --- spacy/tests/tokens/test_doc_api.py | 169 ++++++++++++++--------------- 1 file changed, 82 insertions(+), 87 deletions(-) diff --git a/spacy/tests/tokens/test_doc_api.py b/spacy/tests/tokens/test_doc_api.py index c9b57e689..e66c1da58 100644 --- a/spacy/tests/tokens/test_doc_api.py +++ b/spacy/tests/tokens/test_doc_api.py @@ -1,39 +1,37 @@ +# coding: utf-8 from __future__ import unicode_literals -from spacy.tokens import Doc -from spacy.en import English -import numpy -from spacy.attrs import HEAD -from ...vocab import Vocab -from ...tokens.doc import Doc +from ..util import get_doc import pytest +import numpy -def test_tokens_compare_by_string_position(): - vocab = Vocab() - doc = Doc(vocab, [u'one', u'two', u'three']) + +@pytest.mark.parametrize('text', [["one", "two", "three"]]) +def test_doc_api_compare_by_string_position(en_vocab, text): + doc = get_doc(en_vocab, text) # Get the tokens in this order, so their ID ordering doesn't match the idx - three = doc[-1] - two = doc[-2] - one = doc[-1] - one, two, three = doc - assert one < two < three - assert not one > two - assert two > one - assert two <= three - assert three >= one + token3 = doc[-1] + token2 = doc[-2] + token1 = doc[-1] + token1, token2, token3 = doc + assert token1 < token2 < token3 + assert not token1 > token2 + assert token2 > token1 + assert token2 <= token3 + assert token3 >= token1 -@pytest.mark.models -def test_getitem(EN): - tokens = EN(u'Give it back! He pleaded.') - assert tokens[0].orth_ == 'Give' - assert tokens[-1].orth_ == '.' +def test_doc_api_getitem(en_tokenizer): + text = "Give it back! He pleaded." + tokens = en_tokenizer(text) + assert tokens[0].text == 'Give' + assert tokens[-1].text == '.' with pytest.raises(IndexError): tokens[len(tokens)] def to_str(span): - return '/'.join(token.orth_ for token in span) + return '/'.join(token.text for token in span) span = tokens[1:1] assert not to_str(span) @@ -99,70 +97,56 @@ def test_getitem(EN): assert subspan.start == subspan.end == 4 and not to_str(subspan) -@pytest.mark.models -def test_serialize(EN): - tokens = EN(u'Give it back! He pleaded.') - packed = tokens.to_bytes() - new_tokens = Doc(EN.vocab).from_bytes(packed) +@pytest.mark.parametrize('text', ["Give it back! He pleaded.", + " Give it back! He pleaded. "]) +def test_doc_api_serialize(en_tokenizer, text): + tokens = en_tokenizer(text) + new_tokens = get_doc(tokens.vocab).from_bytes(tokens.to_bytes()) assert tokens.string == new_tokens.string - assert [t.orth_ for t in tokens] == [t.orth_ for t in new_tokens] + assert [t.text for t in tokens] == [t.text for t in new_tokens] assert [t.orth for t in tokens] == [t.orth for t in new_tokens] -@pytest.mark.models -def test_serialize_whitespace(EN): - tokens = EN(u' Give it back! He pleaded. ') - packed = tokens.to_bytes() - new_tokens = Doc(EN.vocab).from_bytes(packed) - assert tokens.string == new_tokens.string - assert [t.orth_ for t in tokens] == [t.orth_ for t in new_tokens] - assert [t.orth for t in tokens] == [t.orth for t in new_tokens] - - -def test_set_ents(EN): - tokens = EN.tokenizer(u'I use goggle chrone to surf the web') +def test_doc_api_set_ents(en_tokenizer): + text = "I use goggle chrone to surf the web" + tokens = en_tokenizer(text) assert len(tokens.ents) == 0 - tokens.ents = [(EN.vocab.strings['PRODUCT'], 2, 4)] + tokens.ents = [(tokens.vocab.strings['PRODUCT'], 2, 4)] assert len(list(tokens.ents)) == 1 assert [t.ent_iob for t in tokens] == [0, 0, 3, 1, 0, 0, 0, 0] - ent = tokens.ents[0] - assert ent.label_ == 'PRODUCT' - assert ent.start == 2 - assert ent.end == 4 + assert tokens.ents[0].label_ == 'PRODUCT' + assert tokens.ents[0].start == 2 + assert tokens.ents[0].end == 4 -def test_merge(EN): - doc = EN('WKRO played songs by the beach boys all night') +def test_doc_api_merge(en_tokenizer): + text = "WKRO played songs by the beach boys all night" - assert len(doc) == 9 # merge 'The Beach Boys' + doc = en_tokenizer(text) + assert len(doc) == 9 doc.merge(doc[4].idx, doc[6].idx + len(doc[6]), 'NAMED', 'LEMMA', 'TYPE') assert len(doc) == 7 - assert doc[4].text == 'the beach boys' assert doc[4].text_with_ws == 'the beach boys ' assert doc[4].tag_ == 'NAMED' - -def test_merge_end_string(EN): - doc = EN('WKRO played songs by the beach boys all night') - + # merge 'all night' + doc = en_tokenizer(text) assert len(doc) == 9 - # merge 'The Beach Boys' doc.merge(doc[7].idx, doc[8].idx + len(doc[8]), 'NAMED', 'LEMMA', 'TYPE') assert len(doc) == 8 - assert doc[7].text == 'all night' assert doc[7].text_with_ws == 'all night' -@pytest.mark.models -def test_merge_children(EN): +def test_doc_api_merge_children(en_tokenizer): """Test that attachments work correctly after merging.""" - doc = EN('WKRO played songs by the beach boys all night') - # merge 'The Beach Boys' + text = "WKRO played songs by the beach boys all night" + doc = en_tokenizer(text) + assert len(doc) == 9 doc.merge(doc[4].idx, doc[6].idx + len(doc[6]), 'NAMED', 'LEMMA', 'TYPE') - + for word in doc: if word.i < word.head.i: assert word in list(word.head.lefts) @@ -170,28 +154,31 @@ def test_merge_children(EN): assert word in list(word.head.rights) -def test_merge_hang(): - text = 'through North and South Carolina' - EN = English(parser=False) - doc = EN(text, tag=True) - heads = numpy.asarray([[0, 3, -1, -2, -4]], dtype='int32') - doc.from_array([HEAD], heads.T) +def test_doc_api_merge_hang(en_tokenizer): + text = "through North and South Carolina" + doc = en_tokenizer(text) doc.merge(18, 32, '', '', 'ORG') doc.merge(8, 32, '', '', 'ORG') -def test_sents_empty_string(EN): - doc = EN(u'') +def test_doc_api_sents_empty_string(en_tokenizer): + doc = en_tokenizer("") doc.is_parsed = True sents = list(doc.sents) assert len(sents) == 0 -@pytest.mark.models -def test_runtime_error(EN): +def test_doc_api_runtime_error(en_tokenizer): # Example that caused run-time error while parsing Reddit - text = u'67% of black households are single parent \n\n72% of all black babies born out of wedlock \n\n50% of all black kids don\u2019t finish high school' - doc = EN(text) + text = "67% of black households are single parent \n\n72% of all black babies born out of wedlock \n\n50% of all black kids don\u2019t finish high school" + deps = ['nsubj', 'prep', 'amod', 'pobj', 'ROOT', 'amod', 'attr', '', + 'nummod', 'prep', 'det', 'amod', 'pobj', 'acl', 'prep', 'prep', + 'pobj', '', 'nummod', 'prep', 'det', 'amod', 'pobj', 'aux', 'neg', + 'ROOT', 'amod', 'dobj'] + + tokens = en_tokenizer(text) + doc = get_doc(tokens.vocab, [t.text for t in tokens], deps=deps) + nps = [] for np in doc.noun_chunks: while len(np) > 1 and np[0].dep_ not in ('advmod', 'amod', 'compound'): @@ -205,20 +192,28 @@ def test_runtime_error(EN): doc.merge(*np) -@pytest.mark.models -def test_right_edge(EN): +def test_doc_api_right_edge(en_tokenizer): # Test for bug occurring from Unshift action, causing incorrect right edge - doc = EN(u'''I have proposed to myself, for the sake of such as live ''' - u'''under the government of the Romans, to translate those books ''' - u'''into the Greek tongue.''') - token = doc[6] - assert token.text == u'for' - subtree = [w.text for w in token.subtree] - assert subtree == [u'for' , u'the', u'sake', u'of', u'such', u'as', u'live', u'under', u'the', u'government', u'of', u'the', u'Romans', u','] - assert token.right_edge.text == u',' + text = "I have proposed to myself, for the sake of such as live under the government of the Romans, to translate those books into the Greek tongue." + heads = [2, 1, 0, -1, -1, -3, 15, 1, -2, -1, 1, -3, -1, -1, 1, -2, -1, 1, + -2, -7, 1, -19, 1, -2, -3, 2, 1, -3, -26] + + tokens = en_tokenizer(text) + doc = get_doc(tokens.vocab, [t.text for t in tokens], heads=heads) + assert doc[6].text == 'for' + subtree = [w.text for w in doc[6].subtree] + assert subtree == ['for' , 'the', 'sake', 'of', 'such', 'as', 'live', 'under', 'the', 'government', 'of', 'the', 'Romans', ','] + assert doc[6].right_edge.text == ',' -@pytest.mark.vectors -def test_has_vector(EN): - doc = EN(u'''apple orange pear''') +@pytest.mark.parametrize('text,vectors', [ + ("apple orange pear", ["apple -1 -1 -1", "orange -1 -1 0", "pear -1 0 -1"]) +]) +def test_doc_api_has_vector(en_tokenizer, text_file, text, vectors): + text_file.write('\n'.join(vectors)) + text_file.seek(0) + vector_length = en_tokenizer.vocab.load_vectors(text_file) + assert vector_length == 3 + + doc = en_tokenizer(text) assert doc.has_vector