From 81a9030ab7922beffcc307ccbf00bcf993353335 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Thu, 21 Mar 2019 00:04:06 +0100 Subject: [PATCH] create candidate object from entry pointer (not fully functional yet) --- spacy/kb.pxd | 20 ++++++++++-- spacy/kb.pyx | 45 ++++++++++++++++++++++++-- spacy/sandbox_test_sofie/testing_el.py | 8 ++--- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/spacy/kb.pxd b/spacy/kb.pxd index d0f31ebb4..c409cf1b4 100644 --- a/spacy/kb.pxd +++ b/spacy/kb.pxd @@ -13,11 +13,14 @@ from .typedefs cimport hash_t # of bits we need to keep track of the answers. cdef struct _EntryC: + # The hash of this entry's unique ID + hash_t entity_key + # Allows retrieval of one or more vectors. # Each element of vector_rows should be an index into a vectors table. # Every entry should have the same number of vectors, so we can avoid storing # the number of vectors in each knowledge-base struct - const int32_t* vector_rows + int32_t* vector_rows # Allows retrieval of a struct of non-vector features. We could make this a # pointer, but we have 32 bits left over in the struct after prob, so we'd @@ -40,6 +43,17 @@ cdef struct _AliasC: vector[float] probs +# TODO: document +cdef class Candidate: + + cdef _EntryC* entity + cdef hash_t alias_hash + cdef float prior_prob + + @staticmethod + cdef Candidate from_entry(_EntryC* entity, hash_t alias_hash, float prior_prob) + + cdef class KnowledgeBase: cdef Pool mem cpdef readonly StringStore strings @@ -85,7 +99,7 @@ cdef class KnowledgeBase: cdef object _features_table - cdef inline int64_t c_add_entity(self, hash_t entity_key, float prob, const int32_t* vector_rows, + cdef inline int64_t c_add_entity(self, hash_t entity_key, float prob, int32_t* vector_rows, int feats_row): """Add an entry to the knowledge base.""" # This is what we'll map the hash key to. It's where the entry will sit @@ -93,6 +107,7 @@ cdef class KnowledgeBase: cdef int64_t entity_index = self._entries.size() self._entries.push_back( _EntryC( + entity_key=entity_key, vector_rows=vector_rows, feats_row=feats_row, prob=prob @@ -121,6 +136,7 @@ cdef class KnowledgeBase: cdef int32_t dummy_value = 0 self._entries.push_back( _EntryC( + entity_key=self.strings.add(""), vector_rows=&dummy_value, feats_row=dummy_value, prob=dummy_value diff --git a/spacy/kb.pyx b/spacy/kb.pyx index ba694ce61..38bc48c7f 100644 --- a/spacy/kb.pyx +++ b/spacy/kb.pyx @@ -2,6 +2,35 @@ # coding: utf8 from spacy.errors import user_warning + +cdef class Candidate: + + + # def inline __cinit__(self, _EntryC entity, hash_t alias_hash, float prior_prob): + # self.alias_hash = alias_hash + # self.entity = entity + # self.prior_prob = prior_prob + + @staticmethod + cdef Candidate from_entry(_EntryC* entity, hash_t alias_hash, float prior_prob): + """Factory function to create Candidate objects from entity entries.""" + # Call to __new__ bypasses __init__ constructor + cdef Candidate candidate = Candidate.__new__(Candidate) + candidate.entity = entity + candidate.alias_hash = alias_hash + candidate.prior_prob = prior_prob + return candidate + + def __str__(self): + return "alias=" + self.strings[self.alias_hash] + \ + " prior_prob=" + str(self.prior_prob) + + #" entry=" + self.strings[self.entity_hash] + \ + + def __repr__(self): + return self.__str__() + + cdef class KnowledgeBase: def __init__(self): @@ -74,7 +103,19 @@ cdef class KnowledgeBase: def get_candidates(self, unicode alias): - cdef hash_t alias_hash = self.strings.add(alias) + cdef hash_t alias_hash = self.strings[alias] alias_index = self._alias_index.get(alias_hash) - return self._aliases_table[alias_index] + alias_entry = self._aliases_table[alias_index] + + for (entry_index, prob) in zip(alias_entry.entry_indices, alias_entry.probs): + entity = <_EntryC>self._entries[entry_index] + # candidate = Candidate(entity=entity, alias_hash=alias_hash, prior_prob=prob) + candidate = Candidate.from_entry(entity=&entity, alias_hash=alias_hash, prior_prob=prob) + print(candidate) + + # return [Candidate(entity=<_EntryC>self._entries[self._entry_index[entry_index]], + # alias_hash=alias_hash, + # prior_prob=prob) + # for (entry_index, prob) in zip(alias_entry.entry_indices, alias_entry.probs)] + diff --git a/spacy/sandbox_test_sofie/testing_el.py b/spacy/sandbox_test_sofie/testing_el.py index 76151f27e..c96c5552f 100644 --- a/spacy/sandbox_test_sofie/testing_el.py +++ b/spacy/sandbox_test_sofie/testing_el.py @@ -25,16 +25,16 @@ def create_kb(): # adding aliases alias1 = "douglassss" - print(" adding alias", alias1) + print(" adding alias", alias1, "to Q42 and Q5301561") mykb.add_alias(alias=alias1, entities=["Q42", "Q5301561"], probabilities=[0.8, 0.2]) alias2 = "johny" - print(" adding alias", alias2) + print(" adding alias", alias2, "to Q0, Q42 and Q5301561") mykb.add_alias(alias=alias2, entities=["Q0", "Q42", "Q5301561"], probabilities=[0.3, 0.1, 0.4]) alias3 = "adam" - print(" adding alias", alias3) - mykb.add_alias(alias=alias3, entities=["Q42"], probabilities=[1.0]) + print(" adding alias", alias3, "to Q42") + mykb.add_alias(alias=alias3, entities=["Q42"], probabilities=[0.9]) print("kb size", len(mykb), mykb.get_size_entities(), mykb.get_size_aliases())