explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-11 17:56:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Changes to test for new string-store"

Browse Source 
This reverts commit 21e90d7d0b.
This commit is contained in:
Matthew Honnibal 2016-09-30 20:11:01 +02:00
parent 1f1cd5013f
commit bd7fe6420c
6 changed files with 28 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
spacy/tests/test_matcher.py
View File

@ -30,7 +30,6 @@ def test_no_match(matcher):
def test_match_start(matcher):
doc = Doc(matcher.vocab, ['JavaScript', 'is', 'good'])
print([(t.text, t.orth) for t in doc])
assert matcher(doc) == [(matcher.vocab.strings['JS'],
matcher.vocab.strings['PRODUCT'], 0, 1)]

11
spacy/tests/tokenizer/test_contractions.py
View File

@ -3,10 +3,10 @@ import pytest
def test_possess(en_tokenizer):
doc = en_tokenizer("Mike's")
assert en_tokenizer.vocab.strings.decode_int(doc[0].orth, mem=doc.mem) == "Mike"
assert en_tokenizer.vocab.strings.decode_int(doc[1].orth, mem=doc.mem) == "'s"
assert len(doc) == 2
tokens = en_tokenizer("Mike's")
assert en_tokenizer.vocab.strings[tokens[0].orth] == "Mike"
assert en_tokenizer.vocab.strings[tokens[1].orth] == "'s"
assert len(tokens) == 2
def test_apostrophe(en_tokenizer):
@ -50,8 +50,9 @@ def test_punct(en_tokenizer):
assert len(tokens) == 3
@pytest.mark.xfail
def test_therell(en_tokenizer):
tokens = en_tokenizer("there'll")
assert len(tokens) == 2
assert tokens[0].text == "there"
assert tokens[1].text == "'ll"
assert tokens[1].text == "there"

6
spacy/tests/tokenizer/test_tokenizer.py
View File

@ -43,12 +43,10 @@ def test_punct(en_tokenizer):
def test_digits(en_tokenizer):
The = en_tokenizer.vocab['The'].orth
nineteen_84 = en_tokenizer.vocab['1984'].orth
tokens = en_tokenizer('The year: 1984.')
assert len(tokens) == 5
assert tokens[0].orth == The
assert tokens[3].orth == nineteen_84
assert tokens[0].orth == en_tokenizer.vocab['The'].orth
assert tokens[3].orth == en_tokenizer.vocab['1984'].orth
def test_contraction(en_tokenizer):

1
spacy/tests/tokens/test_token_api.py
View File

@ -44,7 +44,6 @@ def test_str_builtin(EN):
assert str(tokens[1]) == u'two'
@pytest.mark.models
def test_is_properties(EN):
Hi, comma, my, email, is_, addr = EN(u'Hi, my email is test@me.com')
assert Hi.is_title

55
spacy/tests/vocab/test_intern.py
View File

@ -3,7 +3,6 @@ from __future__ import unicode_literals
import pickle
from spacy.strings import StringStore
from cymem.cymem import Pool
import pytest
import tempfile
@ -17,27 +16,25 @@ def sstore():
def test_save_bytes(sstore):
Hello_i = sstore.intern(b'Hello')
Hello_i = sstore[b'Hello']
assert Hello_i == 1
assert sstore.intern(b'Hello') == 1
assert sstore[b'Hello'] == 1
assert sstore.intern(b'goodbye') != Hello_i
assert sstore.intern(b'hello') != Hello_i
assert sstore[b'goodbye'] != Hello_i
assert sstore[b'hello'] != Hello_i
assert Hello_i == 1
def test_save_unicode(sstore):
Hello_i = sstore.intern(u'Hello')
Hello_i = sstore[u'Hello']
assert Hello_i == 1
assert sstore.intern(u'Hello') == 1
assert sstore[u'Hello'] == 1
assert sstore.intern(u'goodbye') != Hello_i
assert sstore.intern(u'hello') != Hello_i
assert sstore[u'goodbye'] != Hello_i
assert sstore[u'hello'] != Hello_i
assert Hello_i == 1
def test_retrieve_id(sstore):
A_i = sstore.intern(b'A')
A_i = sstore[b'A']
assert sstore.size == 1
assert sstore[1] == 'A'
with pytest.raises(IndexError):
@ -45,64 +42,48 @@ def test_retrieve_id(sstore):
def test_med_string(sstore):
nine_char_string = sstore.intern(b'0123456789')
nine_char_string = sstore[b'0123456789']
assert sstore[nine_char_string] == u'0123456789'
dummy = sstore.intern(b'A')
dummy = sstore[b'A']
assert sstore[b'0123456789'] == nine_char_string
def test_long_string(sstore):
url = u'INFORMATIVE](http://www.google.com/search?as_q=RedditMonkey&hl=en&num=50&btnG=Google+Search&as_epq=&as_oq=&as_eq=&lr=&as_ft=i&as_filetype=&as_qdr=all&as_nlo=&as_nhi=&as_occt=any&as_dt=i&as_sitesearch=&as_rights=&safe=off'
orth = sstore.intern(url)
orth = sstore[url]
assert sstore[orth] == url
def test_254_string(sstore):
s254 = 'a' * 254
orth = sstore.intern(s254)
orth = sstore[s254]
assert sstore[orth] == s254
def test_255_string(sstore):
s255 = 'b' * 255
orth = sstore.intern(s255)
orth = sstore[s255]
assert sstore[orth] == s255
def test_256_string(sstore):
s256 = 'c' * 256
orth = sstore.intern(s256)
orth = sstore[s256]
assert sstore[orth] == s256
def test_massive_strings(sstore):
s511 = 'd' * 511
orth = sstore.intern(s511)
orth = sstore[s511]
assert sstore[orth] == s511
s512 = 'e' * 512
orth = sstore.intern(s512)
orth = sstore[s512]
assert sstore[orth] == s512
s513 = '1' * 513
orth = sstore.intern(s513)
orth = sstore[s513]
assert sstore[orth] == s513
def test_intern_oov(sstore):
mem1 = Pool()
mem2 = Pool()
one = sstore.intern(u'Pool1', mem=mem1)
two = sstore.intern(u'Pool2', mem=mem2)
assert sstore.decode_int(one, mem=mem1) == 'Pool1'
assert sstore.decode_int(two, mem=mem2) == 'Pool2'
with pytest.raises(IndexError):
sstore.decode_int(one, mem=mem2)
sstore.decode_int(two, mem=mem1)
sstore.remove_oov_map(mem1)
with pytest.raises(IndexError):
sstore.decode_int(one, mem=mem1)
def test_pickle_string_store(sstore):
hello_id = sstore.intern(u'Hi')
hello_id = sstore[u'Hi']
string_file = io.BytesIO()
pickle.dump(sstore, string_file)
@ -114,7 +95,7 @@ def test_pickle_string_store(sstore):
def test_dump_load(sstore):
id_ = sstore.intern(u'qqqqq')
id_ = sstore[u'qqqqq']
with tempfile.TemporaryFile('w+t') as file_:
sstore.dump(file_)
file_.seek(0)

4
spacy/tests/vocab/test_lexeme.py
View File

@ -9,9 +9,9 @@ def test_lexeme_eq(en_vocab):
'''Test Issue #361: Equality of lexemes'''
cat1 = en_vocab['cat']
#cat2 = en_vocab['cat']
cat2 = en_vocab['cat']
#assert cat1 == cat2
assert cat1 == cat2
def test_lexeme_neq(en_vocab):
'''Inequality of lexemes'''