explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-13 18:56:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Restore vectors tests

Browse Source
This commit is contained in:
Matthew Honnibal 2017-08-19 20:34:58 +02:00
parent 8cfeeb4884
commit 1391f9da37
2 changed files with 128 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
spacy/tests/vectors/test_similarity.py
View File

@ -14,10 +14,9 @@ def vectors():
@pytest.fixture() @pytest.fixture()
def vocab(en_vocab, vectors): def vocab(en_vocab, vectors):
#return add_vecs_to_vocab(en_vocab, vectors) add_vecs_to_vocab(en_vocab, vectors)
return None return en_vocab
@pytest.mark.xfail
def test_vectors_similarity_LL(vocab, vectors): def test_vectors_similarity_LL(vocab, vectors):
[(word1, vec1), (word2, vec2)] = vectors [(word1, vec1), (word2, vec2)] = vectors
lex1 = vocab[word1] lex1 = vocab[word1]
@ -31,7 +30,6 @@ def test_vectors_similarity_LL(vocab, vectors):
assert numpy.isclose(lex2.similarity(lex2), lex1.similarity(lex1)) assert numpy.isclose(lex2.similarity(lex2), lex1.similarity(lex1))
@pytest.mark.xfail
def test_vectors_similarity_TT(vocab, vectors): def test_vectors_similarity_TT(vocab, vectors):
[(word1, vec1), (word2, vec2)] = vectors [(word1, vec1), (word2, vec2)] = vectors
doc = get_doc(vocab, words=[word1, word2]) doc = get_doc(vocab, words=[word1, word2])
@ -44,21 +42,18 @@ def test_vectors_similarity_TT(vocab, vectors):
assert numpy.isclose(doc[1].similarity(doc[0]), doc[0].similarity(doc[1])) assert numpy.isclose(doc[1].similarity(doc[0]), doc[0].similarity(doc[1]))
@pytest.mark.xfail
def test_vectors_similarity_TD(vocab, vectors): def test_vectors_similarity_TD(vocab, vectors):
[(word1, vec1), (word2, vec2)] = vectors [(word1, vec1), (word2, vec2)] = vectors
doc = get_doc(vocab, words=[word1, word2]) doc = get_doc(vocab, words=[word1, word2])
assert doc.similarity(doc[0]) == doc[0].similarity(doc) assert doc.similarity(doc[0]) == doc[0].similarity(doc)
@pytest.mark.xfail
def test_vectors_similarity_DS(vocab, vectors): def test_vectors_similarity_DS(vocab, vectors):
[(word1, vec1), (word2, vec2)] = vectors [(word1, vec1), (word2, vec2)] = vectors
doc = get_doc(vocab, words=[word1, word2]) doc = get_doc(vocab, words=[word1, word2])
assert doc.similarity(doc[:2]) == doc[:2].similarity(doc) assert doc.similarity(doc[:2]) == doc[:2].similarity(doc)
@pytest.mark.xfail
def test_vectors_similarity_TS(vocab, vectors): def test_vectors_similarity_TS(vocab, vectors):
[(word1, vec1), (word2, vec2)] = vectors [(word1, vec1), (word2, vec2)] = vectors
doc = get_doc(vocab, words=[word1, word2]) doc = get_doc(vocab, words=[word1, word2])

248
spacy/tests/vectors/test_vectors.py
View File

@ -2,6 +2,8 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from ...vectors import Vectors from ...vectors import Vectors
from ...tokenizer import Tokenizer
from ..util import add_vecs_to_vocab, get_doc
import numpy import numpy
import pytest import pytest
@ -11,11 +13,27 @@ import pytest
def strings(): def strings():
return ["apple", "orange"] return ["apple", "orange"]
@pytest.fixture
def vectors():
return [
("apple", [1, 2, 3]),
("orange", [-1, -2, -3]),
('and', [-1, -1, -1]),
('juice', [5, 5, 10]),
('pie', [7, 6.3, 8.9])]
@pytest.fixture @pytest.fixture
def data(): def data():
return numpy.asarray([[0.0, 1.0, 2.0], [3.0, -2.0, 4.0]], dtype='f') return numpy.asarray([[0.0, 1.0, 2.0], [3.0, -2.0, 4.0]], dtype='f')
@pytest.fixture()
def vocab(en_vocab, vectors):
add_vecs_to_vocab(en_vocab, vectors)
return en_vocab
def test_init_vectors_with_data(strings, data): def test_init_vectors_with_data(strings, data):
v = Vectors(strings, data) v = Vectors(strings, data)
assert v.shape == data.shape assert v.shape == data.shape
@ -42,125 +60,111 @@ def test_set_vector(strings, data):
assert list(v[strings[0]]) != list(orig[0]) assert list(v[strings[0]]) != list(orig[0])
#
#@pytest.fixture() @pytest.fixture()
#def tokenizer_v(vocab): def tokenizer_v(vocab):
# return Tokenizer(vocab, {}, None, None, None) return Tokenizer(vocab, {}, None, None, None)
#
#
#@pytest.mark.xfail @pytest.mark.parametrize('text', ["apple and orange"])
#@pytest.mark.parametrize('text', ["apple and orange"]) def test_vectors_token_vector(tokenizer_v, vectors, text):
#def test_vectors_token_vector(tokenizer_v, vectors, text): doc = tokenizer_v(text)
# doc = tokenizer_v(text) assert vectors[0] == (doc[0].text, list(doc[0].vector))
# assert vectors[0] == (doc[0].text, list(doc[0].vector)) assert vectors[1] == (doc[2].text, list(doc[2].vector))
# assert vectors[1] == (doc[2].text, list(doc[2].vector))
#
# @pytest.mark.parametrize('text', ["apple", "orange"])
#@pytest.mark.xfail def test_vectors_lexeme_vector(vocab, text):
#@pytest.mark.parametrize('text', ["apple", "orange"]) lex = vocab[text]
#def test_vectors_lexeme_vector(vocab, text): assert list(lex.vector)
# lex = vocab[text] assert lex.vector_norm
# assert list(lex.vector)
# assert lex.vector_norm
# @pytest.mark.parametrize('text', [["apple", "and", "orange"]])
# def test_vectors_doc_vector(vocab, text):
#@pytest.mark.xfail doc = get_doc(vocab, text)
#@pytest.mark.parametrize('text', [["apple", "and", "orange"]]) assert list(doc.vector)
#def test_vectors_doc_vector(vocab, text): assert doc.vector_norm
# doc = get_doc(vocab, text)
# assert list(doc.vector)
# assert doc.vector_norm @pytest.mark.parametrize('text', [["apple", "and", "orange"]])
# def test_vectors_span_vector(vocab, text):
# span = get_doc(vocab, text)[0:2]
#@pytest.mark.xfail assert list(span.vector)
#@pytest.mark.parametrize('text', [["apple", "and", "orange"]]) assert span.vector_norm
#def test_vectors_span_vector(vocab, text):
# span = get_doc(vocab, text)[0:2]
# assert list(span.vector) @pytest.mark.parametrize('text', ["apple orange"])
# assert span.vector_norm def test_vectors_token_token_similarity(tokenizer_v, text):
# doc = tokenizer_v(text)
# assert doc[0].similarity(doc[1]) == doc[1].similarity(doc[0])
#@pytest.mark.xfail assert -1. < doc[0].similarity(doc[1]) < 1.0
#@pytest.mark.parametrize('text', ["apple orange"])
#def test_vectors_token_token_similarity(tokenizer_v, text):
# doc = tokenizer_v(text) @pytest.mark.parametrize('text1,text2', [("apple", "orange")])
# assert doc[0].similarity(doc[1]) == doc[1].similarity(doc[0]) def test_vectors_token_lexeme_similarity(tokenizer_v, vocab, text1, text2):
# assert 0.0 < doc[0].similarity(doc[1]) < 1.0 token = tokenizer_v(text1)
# lex = vocab[text2]
# assert token.similarity(lex) == lex.similarity(token)
#@pytest.mark.xfail assert -1. < token.similarity(lex) < 1.0
#@pytest.mark.parametrize('text1,text2', [("apple", "orange")])
#def test_vectors_token_lexeme_similarity(tokenizer_v, vocab, text1, text2):
# token = tokenizer_v(text1) @pytest.mark.parametrize('text', [["apple", "orange", "juice"]])
# lex = vocab[text2] def test_vectors_token_span_similarity(vocab, text):
# assert token.similarity(lex) == lex.similarity(token) doc = get_doc(vocab, text)
# assert 0.0 < token.similarity(lex) < 1.0 assert doc[0].similarity(doc[1:3]) == doc[1:3].similarity(doc[0])
# assert -1. < doc[0].similarity(doc[1:3]) < 1.0
#
#@pytest.mark.xfail
#@pytest.mark.parametrize('text', [["apple", "orange", "juice"]]) @pytest.mark.parametrize('text', [["apple", "orange", "juice"]])
#def test_vectors_token_span_similarity(vocab, text): def test_vectors_token_doc_similarity(vocab, text):
# doc = get_doc(vocab, text) doc = get_doc(vocab, text)
# assert doc[0].similarity(doc[1:3]) == doc[1:3].similarity(doc[0]) assert doc[0].similarity(doc) == doc.similarity(doc[0])
# assert 0.0 < doc[0].similarity(doc[1:3]) < 1.0 assert -1. < doc[0].similarity(doc) < 1.0
#
#
#@pytest.mark.xfail @pytest.mark.parametrize('text', [["apple", "orange", "juice"]])
#@pytest.mark.parametrize('text', [["apple", "orange", "juice"]]) def test_vectors_lexeme_span_similarity(vocab, text):
#def test_vectors_token_doc_similarity(vocab, text): doc = get_doc(vocab, text)
# doc = get_doc(vocab, text) lex = vocab[text[0]]
# assert doc[0].similarity(doc) == doc.similarity(doc[0]) assert lex.similarity(doc[1:3]) == doc[1:3].similarity(lex)
# assert 0.0 < doc[0].similarity(doc) < 1.0 assert -1. < doc.similarity(doc[1:3]) < 1.0
#
#
#@pytest.mark.xfail @pytest.mark.parametrize('text1,text2', [("apple", "orange")])
#@pytest.mark.parametrize('text', [["apple", "orange", "juice"]]) def test_vectors_lexeme_lexeme_similarity(vocab, text1, text2):
#def test_vectors_lexeme_span_similarity(vocab, text): lex1 = vocab[text1]
# doc = get_doc(vocab, text) lex2 = vocab[text2]
# lex = vocab[text[0]] assert lex1.similarity(lex2) == lex2.similarity(lex1)
# assert lex.similarity(doc[1:3]) == doc[1:3].similarity(lex) assert -1. < lex1.similarity(lex2) < 1.0
# assert 0.0 < doc.similarity(doc[1:3]) < 1.0
#
# @pytest.mark.parametrize('text', [["apple", "orange", "juice"]])
#@pytest.mark.xfail def test_vectors_lexeme_doc_similarity(vocab, text):
#@pytest.mark.parametrize('text1,text2', [("apple", "orange")]) doc = get_doc(vocab, text)
#def test_vectors_lexeme_lexeme_similarity(vocab, text1, text2): lex = vocab[text[0]]
# lex1 = vocab[text1] assert lex.similarity(doc) == doc.similarity(lex)
# lex2 = vocab[text2] assert -1. < lex.similarity(doc) < 1.0
# assert lex1.similarity(lex2) == lex2.similarity(lex1)
# assert 0.0 < lex1.similarity(lex2) < 1.0
# @pytest.mark.parametrize('text', [["apple", "orange", "juice"]])
# def test_vectors_span_span_similarity(vocab, text):
#@pytest.mark.xfail doc = get_doc(vocab, text)
#@pytest.mark.parametrize('text', [["apple", "orange", "juice"]]) assert doc[0:2].similarity(doc[1:3]) == doc[1:3].similarity(doc[0:2])
#def test_vectors_lexeme_doc_similarity(vocab, text): assert -1. < doc[0:2].similarity(doc[1:3]) < 1.0
# doc = get_doc(vocab, text)
# lex = vocab[text[0]]
# assert lex.similarity(doc) == doc.similarity(lex) @pytest.mark.parametrize('text', [["apple", "orange", "juice"]])
# assert 0.0 < lex.similarity(doc) < 1.0 def test_vectors_span_doc_similarity(vocab, text):
# doc = get_doc(vocab, text)
# assert doc[0:2].similarity(doc) == doc.similarity(doc[0:2])
#@pytest.mark.xfail assert -1. < doc[0:2].similarity(doc) < 1.0
#@pytest.mark.parametrize('text', [["apple", "orange", "juice"]])
#def test_vectors_span_span_similarity(vocab, text):
# doc = get_doc(vocab, text) @pytest.mark.parametrize('text1,text2', [
# assert doc[0:2].similarity(doc[1:3]) == doc[1:3].similarity(doc[0:2]) (["apple", "and", "apple", "pie"], ["orange", "juice"])])
# assert 0.0 < doc[0:2].similarity(doc[1:3]) < 1.0 def test_vectors_doc_doc_similarity(vocab, text1, text2):
# doc1 = get_doc(vocab, text1)
# doc2 = get_doc(vocab, text2)
#@pytest.mark.xfail assert doc1.similarity(doc2) == doc2.similarity(doc1)
#@pytest.mark.parametrize('text', [["apple", "orange", "juice"]]) assert -1. < doc1.similarity(doc2) < 1.0
#def test_vectors_span_doc_similarity(vocab, text):
# doc = get_doc(vocab, text)
# assert doc[0:2].similarity(doc) == doc.similarity(doc[0:2])
# assert 0.0 < doc[0:2].similarity(doc) < 1.0
#
#
#@pytest.mark.xfail
#@pytest.mark.parametrize('text1,text2', [
# (["apple", "and", "apple", "pie"], ["orange", "juice"])])
#def test_vectors_doc_doc_similarity(vocab, text1, text2):
# doc1 = get_doc(vocab, text1)
# doc2 = get_doc(vocab, text2)
# assert doc1.similarity(doc2) == doc2.similarity(doc1)
# assert 0.0 < doc1.similarity(doc2) < 1.0