explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-07-15 10:42:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Refactor to use vector[bint]

Browse Source
This commit is contained in:
Matthew Honnibal 2015-07-12 05:27:47 +02:00
parent 6a6e852a39
commit 14eafcab15
1 changed files with 23 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
spacy/serialize.pyx
View File

@ -28,14 +28,9 @@ cdef struct Node:
int right int right
cdef struct Code:
uint64_t bits
int length
cdef class HuffmanCodec: cdef class HuffmanCodec:
cdef vector[Node] nodes cdef vector[Node] nodes
cdef vector[Code] codes cdef vector[vector[bint]] codes
cdef float[:] probs cdef float[:] probs
cdef dict table cdef dict table
def __init__(self, symbols, probs): def __init__(self, symbols, probs):
@ -46,7 +41,8 @@ cdef class HuffmanCodec:
self.codes.resize(len(probs)) self.codes.resize(len(probs))
populate_nodes(self.nodes, probs) populate_nodes(self.nodes, probs)
assign_codes(self.nodes, self.codes, len(self.nodes) - 1, b'') cdef vector[bint] path
assign_codes(self.nodes, self.codes, len(self.nodes) - 1, path)
def encode(self, sequence): def encode(self, sequence):
bits = [] bits = []
@ -69,11 +65,20 @@ cdef class HuffmanCodec:
return symbols return symbols
property strings: property strings:
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.nonecheck(False)
def __get__(self): def __get__(self):
output = [] output = []
for i in range(len(self.codes)): cdef int i, j
string = '{0:b}'.format(self.codes[i].bits).rjust(self.codes[i].length, '0') for i in range(self.codes.size()):
output.append(string) code = []
for j in range(self.codes[i].size()):
if self.codes[i][j]:
code += '1'
else:
code += '0'
output.append(code)
return output return output
@ -138,22 +143,23 @@ cdef int _cover_two_words(vector[Node]& nodes, int id1, int id2, float prob) nog
nodes.push_back(node) nodes.push_back(node)
cdef int assign_codes(vector[Node]& nodes, vector[Code]& codes, int i, bytes path) except -1: cdef int assign_codes(vector[Node]& nodes, vector[vector[bint]]& codes, int i,
left_path = path + b'0' vector[bint] path) except -1:
right_path = path + b'1' cdef vector[bint] left_path = path
cdef vector[bint] right_path = path
left_path.push_back(0)
right_path.push_back(1)
# Assign down left branch # Assign down left branch
if nodes[i].left >= 0: if nodes[i].left >= 0:
assign_codes(nodes, codes, nodes[i].left, left_path) assign_codes(nodes, codes, nodes[i].left, left_path)
else: else:
# Leaf on left # Leaf on left
id_ = -(nodes[i].left + 1) id_ = -(nodes[i].left + 1)
codes[id_].length = len(left_path) codes[id_] = left_path
codes[id_].bits = <uint64_t>int(left_path, 2)
# Assign down right branch # Assign down right branch
if nodes[i].right >= 0: if nodes[i].right >= 0:
assign_codes(nodes, codes, nodes[i].right, right_path) assign_codes(nodes, codes, nodes[i].right, right_path)
else: else:
# Leaf on right # Leaf on right
id_ = -(nodes[i].right + 1) id_ = -(nodes[i].right + 1)
codes[id_].length = len(right_path) codes[id_] = right_path
codes[id_].bits = <uint64_t>int(right_path, 2)