* Remove vector[bint], in favor of simple Code struct.

This commit is contained in:
Matthew Honnibal 2015-07-12 17:58:27 +02:00
parent aa7bfd932b
commit 3fb9de2d13

View File

@ -1,8 +1,12 @@
from libcpp.vector cimport vector from libcpp.vector cimport vector
from libc.stdint cimport uint32_t from libc.stdint cimport uint32_t
from libc.stdint cimport int64_t from libc.stdint cimport int64_t
from libc.stdint cimport int32_t
from libc.stdint cimport uint64_t from libc.stdint cimport uint64_t
from preshed.maps cimport PreshMap
from murmurhash.mrmr cimport hash64
import numpy import numpy
cimport cython cimport cython
@ -24,33 +28,61 @@ cimport cython
cdef struct Node: cdef struct Node:
float prob float prob
int left int32_t left
int right int32_t right
cdef struct Code:
uint64_t bits
char length
# Note that we're setting the most significant bits here first, when in practice
# we're actually wanting the last bit to be most significant (for Huffman coding,
# anyway).
cdef Code bit_append(Code code, bint bit) nogil:
cdef uint64_t one = 1
if bit:
code.bits |= one << code.length
else:
code.bits &= ~(one << code.length)
code.length += 1
return code
cdef class HuffmanCodec: cdef class HuffmanCodec:
cdef vector[Node] nodes cdef vector[Node] nodes
cdef vector[vector[bint]] codes cdef vector[Code] codes
cdef vector[bint] oov_code
cdef uint64_t oov_symbol
cdef float[:] probs cdef float[:] probs
cdef dict table cdef PreshMap table
def __init__(self, symbols, probs): def __init__(self, symbols, probs):
self.table = {} self.table = PreshMap()
cdef bytes symb_str
cdef uint64_t key
cdef uint64_t i
for i, symbol in enumerate(symbols): for i, symbol in enumerate(symbols):
self.table[symbol] = i if type(symbol) == unicode or type(symbol) == bytes:
symb_str = symbol.encode('utf8')
key = hash64(<unsigned char*>symb_str, len(symb_str), 0)
else:
key = int(symbol)
self.table[key] = i
self.probs = probs self.probs = probs
self.codes.resize(len(probs)) self.codes.resize(len(probs))
for i in range(len(self.codes)):
self.codes[i].bits = 0
self.codes[i].length = 0
populate_nodes(self.nodes, probs) populate_nodes(self.nodes, probs)
cdef vector[bint] path cdef Code path
path.bits = 0
path.length = 0
assign_codes(self.nodes, self.codes, len(self.nodes) - 1, path) assign_codes(self.nodes, self.codes, len(self.nodes) - 1, path)
def encode(self, uint64_t[:] sequence): def encode(self, uint64_t[:] sequence):
bits = [] cdef vector[bint] bits
cdef uint64_t symbol cdef uint64_t symbol
for symbol in sequence: for symbol in sequence:
i = <int>self.table.get(symbol) i = <size_t>self.table.get(symbol)
if i == 0: if i == 0:
raise Exception("Unseen symbol: %s" % symbol) raise Exception("Unseen symbol: %s" % symbol)
else: else:
@ -77,14 +109,13 @@ cdef class HuffmanCodec:
def __get__(self): def __get__(self):
output = [] output = []
cdef int i, j cdef int i, j
cdef bytes string
cdef Code code
for i in range(self.codes.size()): for i in range(self.codes.size()):
code = [] code = self.codes[i]
for j in range(self.codes[i].size()): string = b'{0:b}'.format(code.bits).rjust(code.length, '0')
if self.codes[i][j]: string = string[::-1]
code += '1' output.append(string)
else:
code += '0'
output.append(code)
return output return output
@ -149,12 +180,10 @@ cdef int _cover_two_words(vector[Node]& nodes, int id1, int id2, float prob) nog
nodes.push_back(node) nodes.push_back(node)
cdef int assign_codes(vector[Node]& nodes, vector[vector[bint]]& codes, int i, cdef int assign_codes(vector[Node]& nodes, vector[Code]& codes, int i, Code path) except -1:
vector[bint] path) except -1: cdef Code left_path = bit_append(path, 0)
cdef vector[bint] left_path = path cdef Code right_path = bit_append(path, 1)
cdef vector[bint] right_path = path
left_path.push_back(0)
right_path.push_back(1)
# Assign down left branch # Assign down left branch
if nodes[i].left >= 0: if nodes[i].left >= 0:
assign_codes(nodes, codes, nodes[i].left, left_path) assign_codes(nodes, codes, nodes[i].left, left_path)