explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-04 21:50:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

Modernise serializer codecs tests

Browse Source
This commit is contained in:
Ines Montani 2017-01-12 21:58:55 +01:00
parent 5dbc6e59f6
commit 61f1ca09c2
1 changed files with 28 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
spacy/tests/serialize/test_codecs.py
View File

@ -1,60 +1,51 @@
# coding: utf-8
from __future__ import unicode_literals from __future__ import unicode_literals
import pytest
from ...serialize.packer import _BinaryCodec
from ...serialize.huffman import HuffmanCodec
from ...serialize.bits import BitArray
import numpy import numpy
import pytest
from spacy.vocab import Vocab
from spacy.serialize.packer import _BinaryCodec
from spacy.serialize.huffman import HuffmanCodec
from spacy.serialize.bits import BitArray
def test_binary(): def test_serialize_codecs_binary():
codec = _BinaryCodec() codec = _BinaryCodec()
bits = BitArray() bits = BitArray()
msg = numpy.array([0, 1, 0, 1, 1], numpy.int32) array = numpy.array([0, 1, 0, 1, 1], numpy.int32)
codec.encode(msg, bits) codec.encode(array, bits)
result = numpy.array([0, 0, 0, 0, 0], numpy.int32) result = numpy.array([0, 0, 0, 0, 0], numpy.int32)
bits.seek(0) bits.seek(0)
codec.decode(bits, result) codec.decode(bits, result)
assert list(msg) == list(result) assert list(array) == list(result)
def test_attribute(): def test_serialize_codecs_attribute():
freqs = {'the': 10, 'quick': 3, 'brown': 4, 'fox': 1, 'jumped': 5, 'over': 8, freqs = {'the': 10, 'quick': 3, 'brown': 4, 'fox': 1, 'jumped': 5,
'lazy': 1, 'dog': 2, '.': 9} 'over': 8, 'lazy': 1, 'dog': 2, '.': 9}
int_map = {'the': 0, 'quick': 1, 'brown': 2, 'fox': 3, 'jumped': 4,
int_map = {'the': 0, 'quick': 1, 'brown': 2, 'fox': 3, 'jumped': 4, 'over': 5, 'over': 5, 'lazy': 6, 'dog': 7, '.': 8}
'lazy': 6, 'dog': 7, '.': 8}
codec = HuffmanCodec([(int_map[string], freq) for string, freq in freqs.items()]) codec = HuffmanCodec([(int_map[string], freq) for string, freq in freqs.items()])
bits = BitArray() bits = BitArray()
array = numpy.array([1, 7], dtype=numpy.int32)
msg = numpy.array([1, 7], dtype=numpy.int32) codec.encode(array, bits)
msg_list = list(msg)
codec.encode(msg, bits)
result = numpy.array([0, 0], dtype=numpy.int32) result = numpy.array([0, 0], dtype=numpy.int32)
bits.seek(0) bits.seek(0)
codec.decode(bits, result) codec.decode(bits, result)
assert msg_list == list(result) assert list(array) == list(result)
def test_vocab_codec(): def test_serialize_codecs_vocab(en_vocab):
vocab = Vocab() words = ["the", "dog", "jumped"]
lex = vocab['dog'] for word in words:
lex = vocab['the'] _ = en_vocab[word]
lex = vocab['jumped'] codec = HuffmanCodec([(lex.orth, lex.prob) for lex in en_vocab])
codec = HuffmanCodec([(lex.orth, lex.prob) for lex in vocab])
bits = BitArray() bits = BitArray()
ids = [en_vocab[s].orth for s in words]
ids = [vocab[s].orth for s in ('the', 'dog', 'jumped')] array = numpy.array(ids, dtype=numpy.int32)
msg = numpy.array(ids, dtype=numpy.int32) codec.encode(array, bits)
msg_list = list(msg) result = numpy.array(range(len(array)), dtype=numpy.int32)
codec.encode(msg, bits)
result = numpy.array(range(len(msg)), dtype=numpy.int32)
bits.seek(0) bits.seek(0)
codec.decode(bits, result) codec.decode(bits, result)
assert msg_list == list(result) assert list(array) == list(result)