create candidate object from entry pointer (not fully functional yet)

This commit is contained in:
svlandeg 2019-03-21 00:04:06 +01:00
parent b7ca3de358
commit 81a9030ab7
3 changed files with 65 additions and 8 deletions

View File

@ -13,11 +13,14 @@ from .typedefs cimport hash_t
# of bits we need to keep track of the answers. # of bits we need to keep track of the answers.
cdef struct _EntryC: cdef struct _EntryC:
# The hash of this entry's unique ID
hash_t entity_key
# Allows retrieval of one or more vectors. # Allows retrieval of one or more vectors.
# Each element of vector_rows should be an index into a vectors table. # Each element of vector_rows should be an index into a vectors table.
# Every entry should have the same number of vectors, so we can avoid storing # Every entry should have the same number of vectors, so we can avoid storing
# the number of vectors in each knowledge-base struct # the number of vectors in each knowledge-base struct
const int32_t* vector_rows int32_t* vector_rows
# Allows retrieval of a struct of non-vector features. We could make this a # Allows retrieval of a struct of non-vector features. We could make this a
# pointer, but we have 32 bits left over in the struct after prob, so we'd # pointer, but we have 32 bits left over in the struct after prob, so we'd
@ -40,6 +43,17 @@ cdef struct _AliasC:
vector[float] probs vector[float] probs
# TODO: document
cdef class Candidate:
cdef _EntryC* entity
cdef hash_t alias_hash
cdef float prior_prob
@staticmethod
cdef Candidate from_entry(_EntryC* entity, hash_t alias_hash, float prior_prob)
cdef class KnowledgeBase: cdef class KnowledgeBase:
cdef Pool mem cdef Pool mem
cpdef readonly StringStore strings cpdef readonly StringStore strings
@ -85,7 +99,7 @@ cdef class KnowledgeBase:
cdef object _features_table cdef object _features_table
cdef inline int64_t c_add_entity(self, hash_t entity_key, float prob, const int32_t* vector_rows, cdef inline int64_t c_add_entity(self, hash_t entity_key, float prob, int32_t* vector_rows,
int feats_row): int feats_row):
"""Add an entry to the knowledge base.""" """Add an entry to the knowledge base."""
# This is what we'll map the hash key to. It's where the entry will sit # This is what we'll map the hash key to. It's where the entry will sit
@ -93,6 +107,7 @@ cdef class KnowledgeBase:
cdef int64_t entity_index = self._entries.size() cdef int64_t entity_index = self._entries.size()
self._entries.push_back( self._entries.push_back(
_EntryC( _EntryC(
entity_key=entity_key,
vector_rows=vector_rows, vector_rows=vector_rows,
feats_row=feats_row, feats_row=feats_row,
prob=prob prob=prob
@ -121,6 +136,7 @@ cdef class KnowledgeBase:
cdef int32_t dummy_value = 0 cdef int32_t dummy_value = 0
self._entries.push_back( self._entries.push_back(
_EntryC( _EntryC(
entity_key=self.strings.add(""),
vector_rows=&dummy_value, vector_rows=&dummy_value,
feats_row=dummy_value, feats_row=dummy_value,
prob=dummy_value prob=dummy_value

View File

@ -2,6 +2,35 @@
# coding: utf8 # coding: utf8
from spacy.errors import user_warning from spacy.errors import user_warning
cdef class Candidate:
# def inline __cinit__(self, _EntryC entity, hash_t alias_hash, float prior_prob):
# self.alias_hash = alias_hash
# self.entity = entity
# self.prior_prob = prior_prob
@staticmethod
cdef Candidate from_entry(_EntryC* entity, hash_t alias_hash, float prior_prob):
"""Factory function to create Candidate objects from entity entries."""
# Call to __new__ bypasses __init__ constructor
cdef Candidate candidate = Candidate.__new__(Candidate)
candidate.entity = entity
candidate.alias_hash = alias_hash
candidate.prior_prob = prior_prob
return candidate
def __str__(self):
return "alias=" + self.strings[self.alias_hash] + \
" prior_prob=" + str(self.prior_prob)
#" entry=" + self.strings[self.entity_hash] + \
def __repr__(self):
return self.__str__()
cdef class KnowledgeBase: cdef class KnowledgeBase:
def __init__(self): def __init__(self):
@ -74,7 +103,19 @@ cdef class KnowledgeBase:
def get_candidates(self, unicode alias): def get_candidates(self, unicode alias):
cdef hash_t alias_hash = self.strings.add(alias) cdef hash_t alias_hash = self.strings[alias]
alias_index = <int64_t>self._alias_index.get(alias_hash) alias_index = <int64_t>self._alias_index.get(alias_hash)
return self._aliases_table[alias_index] alias_entry = self._aliases_table[alias_index]
for (entry_index, prob) in zip(alias_entry.entry_indices, alias_entry.probs):
entity = <_EntryC>self._entries[entry_index]
# candidate = Candidate(entity=entity, alias_hash=alias_hash, prior_prob=prob)
candidate = Candidate.from_entry(entity=&entity, alias_hash=alias_hash, prior_prob=prob)
print(candidate)
# return [Candidate(entity=<_EntryC>self._entries[<int64_t>self._entry_index[entry_index]],
# alias_hash=alias_hash,
# prior_prob=prob)
# for (entry_index, prob) in zip(alias_entry.entry_indices, alias_entry.probs)]

View File

@ -25,16 +25,16 @@ def create_kb():
# adding aliases # adding aliases
alias1 = "douglassss" alias1 = "douglassss"
print(" adding alias", alias1) print(" adding alias", alias1, "to Q42 and Q5301561")
mykb.add_alias(alias=alias1, entities=["Q42", "Q5301561"], probabilities=[0.8, 0.2]) mykb.add_alias(alias=alias1, entities=["Q42", "Q5301561"], probabilities=[0.8, 0.2])
alias2 = "johny" alias2 = "johny"
print(" adding alias", alias2) print(" adding alias", alias2, "to Q0, Q42 and Q5301561")
mykb.add_alias(alias=alias2, entities=["Q0", "Q42", "Q5301561"], probabilities=[0.3, 0.1, 0.4]) mykb.add_alias(alias=alias2, entities=["Q0", "Q42", "Q5301561"], probabilities=[0.3, 0.1, 0.4])
alias3 = "adam" alias3 = "adam"
print(" adding alias", alias3) print(" adding alias", alias3, "to Q42")
mykb.add_alias(alias=alias3, entities=["Q42"], probabilities=[1.0]) mykb.add_alias(alias=alias3, entities=["Q42"], probabilities=[0.9])
print("kb size", len(mykb), mykb.get_size_entities(), mykb.get_size_aliases()) print("kb size", len(mykb), mykb.get_size_entities(), mykb.get_size_aliases())