explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-11-04 01:48:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Update serializer tests

Browse Source
This commit is contained in:
Matthew Honnibal 2015-07-18 22:46:40 +02:00
parent 6b13e7227c
commit 0973e2f107
2 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/serialize/test_codecs.py
View File

@ -62,11 +62,11 @@ def test_vocab_codec():
vocab['the'] = get_lex_props('the', 0.05) vocab['the'] = get_lex_props('the', 0.05)
vocab['jumped'] = get_lex_props('jumped', 0.005) vocab['jumped'] = get_lex_props('jumped', 0.005)
codec = make_vocab_codec(vocab) codec = _AttributeCodec([(lex.orth, lex.prob) for lex in vocab])
bits = BitArray() bits = BitArray()
ids = [vocab[s].id for s in ('the', 'dog', 'jumped')] ids = [vocab[s].orth for s in ('the', 'dog', 'jumped')]
msg = numpy.array(ids, dtype=numpy.int32) msg = numpy.array(ids, dtype=numpy.int32)
msg_list = list(msg) msg_list = list(msg)
codec.encode(msg, bits) codec.encode(msg, bits)

12
tests/serialize/test_packer.py
View File

@ -5,7 +5,7 @@ import numpy
from spacy.vocab import Vocab from spacy.vocab import Vocab
from spacy.tokens.doc import Doc from spacy.tokens.doc import Doc
from spacy.attrs import ID, SPACY, TAG, DEP, HEAD from spacy.attrs import ORTH, SPACY, TAG, DEP, HEAD
from spacy.serialize.packer import Packer from spacy.serialize.packer import Packer
from spacy.serialize.bits import BitArray from spacy.serialize.bits import BitArray
@ -30,6 +30,7 @@ def get_lex_props(string, prob=-22):
def vocab(): def vocab():
vocab = Vocab(get_lex_props=get_lex_props) vocab = Vocab(get_lex_props=get_lex_props)
vocab['dog'] = get_lex_props('dog', 0.001) vocab['dog'] = get_lex_props('dog', 0.001)
assert vocab[vocab.strings['dog']].orth_ == 'dog'
vocab['the'] = get_lex_props('the', 0.01) vocab['the'] = get_lex_props('the', 0.01)
vocab['quick'] = get_lex_props('quick', 0.005) vocab['quick'] = get_lex_props('quick', 0.005)
vocab['jumped'] = get_lex_props('jumped', 0.007) vocab['jumped'] = get_lex_props('jumped', 0.007)
@ -37,9 +38,10 @@ def vocab():
def test_packer_unannotated(vocab): def test_packer_unannotated(vocab):
packer = Packer(vocab, [(ID, {}), (SPACY, {})]) packer = Packer(vocab, [(ORTH, [(lex.orth, lex.prob) for lex in vocab]),
(SPACY, [])])
ids = [vocab[w].id for w in 'the dog jumped'.split()] ids = [vocab[w].orth for w in 'the dog jumped'.split()]
msg = Doc.from_ids(vocab, ids, [1, 1, 0]) msg = Doc.from_ids(vocab, ids, [1, 1, 0])
assert msg.string == 'the dog jumped' assert msg.string == 'the dog jumped'
@ -62,7 +64,7 @@ def test_packer_annotated(vocab):
root = vocab.strings['ROOT'] root = vocab.strings['ROOT']
attr_freqs = [ attr_freqs = [
(ID, []), (ORTH, [(lex.orth, lex.prob) for lex in vocab]),
(SPACY, []), (SPACY, []),
(TAG, [(nn, 0.1), (dt, 0.2), (jj, 0.01), (vbd, 0.05)]), (TAG, [(nn, 0.1), (dt, 0.2), (jj, 0.01), (vbd, 0.05)]),
(DEP, {det: 0.2, nsubj: 0.1, adj: 0.05, root: 0.1}.items()), (DEP, {det: 0.2, nsubj: 0.1, adj: 0.05, root: 0.1}.items()),
@ -71,7 +73,7 @@ def test_packer_annotated(vocab):
packer = Packer(vocab, attr_freqs) packer = Packer(vocab, attr_freqs)
ids = [vocab[w].id for w in 'the dog jumped'.split()] ids = [vocab[w].orth for w in 'the dog jumped'.split()]
msg = Doc.from_ids(vocab, ids, [1, 1, 0]) msg = Doc.from_ids(vocab, ids, [1, 1, 0])
msg.from_array( msg.from_array(
[TAG, DEP, HEAD], [TAG, DEP, HEAD],