diff --git a/spacy/tests/gold/test_misaligned_gold.py b/spacy/tests/gold/test_misaligned_gold.py index f8733e5fa..24b07545a 100644 --- a/spacy/tests/gold/test_misaligned_gold.py +++ b/spacy/tests/gold/test_misaligned_gold.py @@ -2,27 +2,40 @@ align in tokenization.''' from __future__ import unicode_literals import pytest +from collections import Counter from ...tokens import Doc from ...gold import GoldParse -from ...gold import _flatten_fused_heads from ...vocab import Vocab -@pytest.mark.parametrize('fused,flat', [ - ([ [(0, 1), 1], 1], [1, 2, 2]), - ([1, 1, [1, 3], 1], [1, 1, 1, 4, 1]) -]) -def test_flatten_fused_heads(fused, flat): - assert _flatten_fused_heads(fused) == flat - def test_over_segmented(): - doc = Doc(Vocab(), words=['a', 'b', 'c']) + doc = Doc(Vocab(), words=['a', 'b', 'c']) gold = GoldParse(doc, words=['ab', 'c'], heads=[1,1]) - assert gold.heads == [1, 2, 2] + assert gold._alignment._y2t == [(0, 0), (0, 1), 1] assert gold.labels == ['subtok', None, None] - + assert gold.heads == [1, 2, 2] + + def test_under_segmented(): - doc = Doc(Vocab(), words=['ab', 'c']) + doc = Doc(Vocab(), words=['ab', 'c']) gold = GoldParse(doc, words=['a', 'b', 'c'], heads=[2,2,2]) assert gold.heads == [[1,1], 1] assert gold.labels == [[None, None], None] + +def test_over_segmented_heads(): + doc = Doc(Vocab(), words=['a', 'b', 'c', 'd', 'e']) + gold = GoldParse(doc, words=['a', 'bc', 'd', 'e'], heads=[2,2,2,2]) + assert gold._alignment._y2t == [0, (1, 0), (1, 1), 2, 3] + assert gold._alignment._t2y == [0, [1, 2], 3, 4] + assert gold.labels == [None, 'subtok', None, None, None] + assert gold.heads == [3, 2, 3, 3, 3] + +def test_under_segmented_attach_inside_fused(): + '''Test arcs point ing into the fused token, + e.g. "its good" + ''' + doc = Doc(Vocab(), words=['ab', 'c']) + gold = GoldParse(doc, words=['a', 'b', 'c'], heads=[1,1,1]) + assert gold.heads == [[(0, 1), (0, 1)], (0, 1)] + assert gold.labels == [[None, None], None] + diff --git a/spacy/tests/parser/test_arc_eager_oracle.py b/spacy/tests/parser/test_arc_eager_oracle.py index bffe9621b..a7b973206 100644 --- a/spacy/tests/parser/test_arc_eager_oracle.py +++ b/spacy/tests/parser/test_arc_eager_oracle.py @@ -173,6 +173,21 @@ def test_oracle_at_sentence_break(arc_eager, vocab): assert c3['B-ROOT'] == 0.0 assert c3['D'] == 9000.0 + +def test_split_oracle(arc_eager, vocab): + gold_words = ['a', 'b', 'c'] + doc = Doc(vocab, words=['ab', 'c']) + heads = [2, 2, 2] + deps = ['dep', 'dep', 'ROOT'] + actions = ['P-1', 'S', 'L-dep', 'S', 'B-ROOT'] + gold = GoldParse(doc, words=gold_words, heads=heads, deps=deps) + assert gold.heads == [[1, 1], 1] + assert gold.labels == [['dep', 'dep'], 'ROOT'] + state = StateClass(doc) + M = arc_eager + M.preprocess_gold(gold) + + annot_tuples = [ (0, 'When', 'WRB', 11, 'advmod', 'O'), (1, 'Walter', 'NNP', 2, 'compound', 'B-PERSON'), diff --git a/spacy/tests/test_align.py b/spacy/tests/test_align.py index a917617b7..a7440a52a 100644 --- a/spacy/tests/test_align.py +++ b/spacy/tests/test_align.py @@ -1,6 +1,8 @@ from __future__ import unicode_literals import pytest -from .._align import align, multi_align +from numpy.testing import assert_array_equal +from .._align import levenshtein_align, multi_align, Alignment +from .._align import _get_regions, _get_many2one @pytest.mark.parametrize('string1,string2,cost', [ @@ -11,7 +13,7 @@ from .._align import align, multi_align ('t', 'catsie', 5), ]) def test_align_costs(string1, string2, cost): - output_cost, i2j, j2i, matrix = align(string1, string2) + output_cost, i2j, j2i, matrix = levenshtein_align(string1, string2) assert output_cost == cost @@ -23,7 +25,7 @@ def test_align_costs(string1, string2, cost): ('t', 'catsie', [2]), ]) def test_align_i2j(string1, string2, i2j): - output_cost, output_i2j, j2i, matrix = align(string1, string2) + output_cost, output_i2j, j2i, matrix = levenshtein_align(string1, string2) assert list(output_i2j) == i2j @@ -35,25 +37,54 @@ def test_align_i2j(string1, string2, i2j): ('t', 'catsie', [-1, -1, 0, -1, -1, -1]), ]) def test_align_i2j(string1, string2, j2i): - output_cost, output_i2j, output_j2i, matrix = align(string1, string2) + output_cost, output_i2j, output_j2i, matrix = levenshtein_align(string1, string2) assert list(output_j2i) == j2i def test_align_strings(): words1 = ['hello', 'this', 'is', 'test!'] words2 = ['hellothis', 'is', 'test', '!'] - cost, i2j, j2i, matrix = align(words1, words2) + cost, i2j, j2i, matrix = levenshtein_align(words1, words2) assert cost == 4 assert list(i2j) == [-1, -1, 1, -1] assert list(j2i) == [-1, 2, -1, -1] +def test_levenshtein_align_is_symmetric(): + words1 = ['a', 'bc', 'd'] + words2 = ['a', 'b', 'c', 'd'] + i_lengths = [len(w) for w in words1] + j_lengths = [len(w) for w in words2] + cost, i2j, j2i, matrix = levenshtein_align(words1, words2) + new_cost, new_j2i, new_i2j, new_matrix = levenshtein_align(words2, words1) + assert cost == new_cost + assert list(i2j) == list(new_i2j) + assert list(j2i) == list(new_j2i) + assert_array_equal(matrix, new_matrix.T) + + +def test_get_many_to_one_undersegment(): + words1 = ['a', 'bc', 'd'] + words2 = ['a', 'b', 'c', 'd'] + i_lengths = [len(w) for w in words1] + j_lengths = [len(w) for w in words2] + cost, i2j, j2i, matrix = levenshtein_align(words1, words2) + i2j_miss = _get_regions(i2j, i_lengths) + j2i_miss = _get_regions(j2i, j_lengths) + i2j_many2one = _get_many2one(i2j_miss, j2i_miss, i_lengths, j_lengths) + assert i2j_many2one == {} + j2i_many2one = _get_many2one(j2i_miss, i2j_miss, j_lengths, i_lengths) + assert j2i_many2one == {1: 1, 2: 1} + + def test_align_many_to_one(): words1 = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'] words2 = ['ab', 'bc', 'e', 'fg', 'h'] - cost, i2j, j2i, matrix = align(words1, words2) + cost, i2j, j2i, matrix = levenshtein_align(words1, words2) assert list(i2j) == [-1, -1, -1, -1, 2, -1, -1, 4] lengths1 = [len(w) for w in words1] lengths2 = [len(w) for w in words2] - i2j_multi, j2i_multi = multi_align(i2j, j2i, lengths1, lengths2) + i2j_miss = _get_regions(i2j, lengths1) + j2i_miss = _get_regions(j2i, lengths2) + i2j_multi = _get_many2one(i2j_miss, j2i_miss, lengths1, lengths2) assert i2j_multi[0] == 0 assert i2j_multi[1] == 0 assert i2j_multi[2] == 1 @@ -62,18 +93,29 @@ def test_align_many_to_one(): assert i2j_multi[5] == 3 assert i2j_multi[6] == 3 - assert j2i_multi[0] == 1 - assert j2i_multi[1] == 3 -def test_align_one_to_many(): +def test_alignment_class_oversegment(): + words1 = ['a', 'b', 'c', 'd'] + words2 = ['a', 'bc', 'd'] + A = Alignment(words1, words2) + B = Alignment(words2, words1) + assert A._y2t == [0, (1, 0), (1, 1), 2] + assert A._t2y == [0, [1, 2], 3] + + +def test_alignment_class_undersegment(): words1 = ['a', 'bc', 'd'] words2 = ['a', 'b', 'c', 'd'] - cost, i2j, j2i, matrix = align(words1, words2) - assert list(i2j) == [0, -1, 3] - lengths1 = [len(w) for w in words1] - lengths2 = [len(w) for w in words2] - multi_j2i, multi_i2j = multi_align(j2i, i2j, lengths2, lengths1) - unsegmented = {} - for j, i in multi_j2i.items(): - unsegmented.setdefault(i, []).append(j) - assert unsegmented[1] == [1, 2] + A = Alignment(words1, words2) + assert A._y2t == [0, [1, 2], 3] + assert A._t2y == [0, (1, 0), (1, 1), 2] + + +@pytest.mark.parametrize('fused,flat', [ + ([ [(0, 1), 1], 1], [1, 2, 2]), + ([1, 1, [1, 3], 1], [1, 1, 1, 4, 1]) +]) +def test_flatten_fused_heads(fused, flat): + assert Alignment.flatten(fused) == flat + +