spaCy/tests/serialize/test_codecs.py

61 lines
1.6 KiB
Python
Raw Normal View History

2015-07-17 22:21:10 +03:00
from __future__ import unicode_literals
import pytest
import numpy
from spacy.vocab import Vocab
from spacy.serialize.packer import _BinaryCodec
2015-07-20 02:38:15 +03:00
from spacy.serialize.huffman import HuffmanCodec
2015-07-17 22:21:10 +03:00
from spacy.serialize.bits import BitArray
def test_binary():
codec = _BinaryCodec()
bits = BitArray()
msg = numpy.array([0, 1, 0, 1, 1], numpy.int32)
codec.encode(msg, bits)
result = numpy.array([0, 0, 0, 0, 0], numpy.int32)
2015-07-17 22:31:44 +03:00
bits.seek(0)
codec.decode(bits, result)
2015-07-17 22:21:10 +03:00
assert list(msg) == list(result)
def test_attribute():
freqs = {'the': 10, 'quick': 3, 'brown': 4, 'fox': 1, 'jumped': 5, 'over': 8,
'lazy': 1, 'dog': 2, '.': 9}
int_map = {'the': 0, 'quick': 1, 'brown': 2, 'fox': 3, 'jumped': 4, 'over': 5,
'lazy': 6, 'dog': 7, '.': 8}
2015-07-20 02:38:15 +03:00
codec = HuffmanCodec([(int_map[string], freq) for string, freq in freqs.items()])
2015-07-17 22:21:10 +03:00
bits = BitArray()
msg = numpy.array([1, 7], dtype=numpy.int32)
msg_list = list(msg)
codec.encode(msg, bits)
result = numpy.array([0, 0], dtype=numpy.int32)
2015-07-17 22:31:44 +03:00
bits.seek(0)
2015-07-17 22:21:10 +03:00
codec.decode(bits, result)
assert msg_list == list(result)
def test_vocab_codec():
vocab = Vocab()
lex = vocab['dog']
lex = vocab['the']
lex = vocab['jumped']
2015-07-17 22:21:10 +03:00
2015-07-20 02:38:15 +03:00
codec = HuffmanCodec([(lex.orth, lex.prob) for lex in vocab])
2015-07-17 22:21:10 +03:00
bits = BitArray()
2015-07-18 23:46:40 +03:00
ids = [vocab[s].orth for s in ('the', 'dog', 'jumped')]
2015-07-17 22:21:10 +03:00
msg = numpy.array(ids, dtype=numpy.int32)
msg_list = list(msg)
codec.encode(msg, bits)
result = numpy.array(range(len(msg)), dtype=numpy.int32)
2015-07-17 22:31:44 +03:00
bits.seek(0)
2015-07-17 22:21:10 +03:00
codec.decode(bits, result)
assert msg_list == list(result)