mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 19:39:13 +03:00
121 lines
4.1 KiB
Cython
121 lines
4.1 KiB
Cython
"""Create a term-document matrix"""
|
|
cimport cython
|
|
from libc.stdint cimport int64_t
|
|
from libc.string cimport memmove
|
|
|
|
from cymem.cymem cimport Address
|
|
|
|
from .lexeme cimport Lexeme, get_attr
|
|
from .tokens cimport TokenC
|
|
from .typedefs cimport hash_t
|
|
|
|
from preshed.maps cimport MapStruct, Cell, map_get, map_set, map_init
|
|
from murmurhash.mrmr cimport hash64
|
|
|
|
|
|
cdef class Index:
|
|
def __init__(self, attr_id_t attr_id):
|
|
self.attr_id = attr_id
|
|
self.max_value = 0
|
|
|
|
cpdef int count(self, Tokens tokens) except -1:
|
|
cdef PreshCounter counts = PreshCounter(2 ** 8)
|
|
cdef attr_id_t attr_id = self.attr_id
|
|
cdef attr_t term
|
|
cdef int i
|
|
for i in range(tokens.length):
|
|
term = get_attr(tokens.data[i].lex, attr_id)
|
|
counts.inc(term, 1)
|
|
if term > self.max_value:
|
|
self.max_value = term
|
|
cdef count_t count
|
|
cdef count_vector_t doc_counts
|
|
for term, count in counts:
|
|
doc_counts.push_back(pair[id_t, count_t](term, count))
|
|
self.counts.push_back(doc_counts)
|
|
|
|
|
|
cdef class DecisionMemory:
|
|
def __init__(self, class_names):
|
|
self.class_names = class_names
|
|
self.n_classes = len(class_names)
|
|
self.mem = Pool()
|
|
self._counts = PreshCounter()
|
|
self._class_counts = PreshCounter()
|
|
self.memos = PreshMap()
|
|
|
|
def load(self, loc, thresh=50):
|
|
cdef:
|
|
count_t freq
|
|
hash_t key
|
|
int clas
|
|
for line in open(loc):
|
|
freq, key, clas = [int(p) for p in line.split()]
|
|
if thresh == 0 or freq >= thresh:
|
|
self.memos.set(key, <void*>(clas+1))
|
|
|
|
def __getitem__(self, ids):
|
|
cdef id_t[2] context
|
|
context[0] = context[0]
|
|
context[1] = context[1]
|
|
cdef hash_t context_key = hash64(context, 2 * sizeof(id_t), 0)
|
|
cdef hash_t[2] class_context
|
|
class_context[0] = context_key
|
|
counts = {}
|
|
cdef id_t i
|
|
for i, clas in enumerate(self.clas_names):
|
|
class_context[1] = <hash_t>i
|
|
key = hash64(class_context, sizeof(hash_t) * 2, 0)
|
|
count = self._class_counts[key]
|
|
counts[clas] = count
|
|
return counts
|
|
|
|
@cython.cdivision(True)
|
|
def iter_contexts(self, float min_acc=0.99, count_t min_freq=10):
|
|
cdef Address counts_addr = Address(self.n_classes, sizeof(count_t))
|
|
cdef count_t* counts = <count_t*>counts_addr.ptr
|
|
cdef MapStruct* context_counts = self._counts.c_map
|
|
cdef hash_t context_key
|
|
cdef count_t context_freq
|
|
cdef int best_class
|
|
cdef float acc
|
|
|
|
cdef int i
|
|
for i in range(context_counts.length):
|
|
context_key = context_counts.cells[i].key
|
|
context_freq = <count_t>context_counts.cells[i].value
|
|
if context_key != 0 and context_freq >= min_freq:
|
|
best_class = self.find_best_class(counts, context_key)
|
|
acc = counts[best_class] / context_freq
|
|
if acc >= min_acc:
|
|
yield counts[best_class], context_key, best_class
|
|
|
|
cdef int inc(self, hash_t context_key, hash_t clas, count_t inc) except -1:
|
|
cdef hash_t context_and_class_key
|
|
cdef hash_t[2] context_and_class
|
|
context_and_class[0] = context_key
|
|
context_and_class[1] = clas
|
|
context_and_class_key = hash64(context_and_class, 2 * sizeof(hash_t), 0)
|
|
self._counts.inc(context_key, inc)
|
|
self._class_counts.inc(context_and_class_key, inc)
|
|
|
|
cdef int find_best_class(self, count_t* counts, hash_t context_key) except -1:
|
|
cdef hash_t[2] unhashed_key
|
|
unhashed_key[0] = context_key
|
|
|
|
cdef count_t total = 0
|
|
cdef hash_t key
|
|
cdef int clas
|
|
cdef int best
|
|
cdef int mode = 0
|
|
for clas in range(self.n_classes):
|
|
unhashed_key[1] = <hash_t>clas
|
|
key = hash64(unhashed_key, sizeof(hash_t) * 2, 0)
|
|
count = self._class_counts[key]
|
|
counts[clas] = count
|
|
if count >= mode:
|
|
mode = count
|
|
best = clas
|
|
total += count
|
|
return best
|