mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-25 09:26:27 +03:00
* Restore dense_hash_map for cache dictionary. Seems to double efficiency
This commit is contained in:
parent
3c928fb5e0
commit
073ee0de63
|
@ -224,7 +224,7 @@ cdef class English(Language):
|
|||
fl_is_digit = Flag_IsDigit
|
||||
v_shape = View_WordShape
|
||||
def __cinit__(self, name, user_string_features, user_flag_features):
|
||||
self.cache = {}
|
||||
self.cache.set_empty_key(0)
|
||||
lang_data = util.read_lang_data(name)
|
||||
rules, words, probs, clusters, case_stats, tag_stats = lang_data
|
||||
self.lexicon = lang.Lexicon(words, probs, clusters, case_stats, tag_stats,
|
||||
|
|
|
@ -3,14 +3,52 @@ from libc.stdint cimport uint64_t
|
|||
from spacy.word cimport Lexeme
|
||||
from spacy.tokens cimport Tokens
|
||||
|
||||
|
||||
cdef struct Flags:
|
||||
size_t is_alpha
|
||||
size_t can_noun
|
||||
from libcpp.utility cimport pair
|
||||
from libcpp.vector cimport vector
|
||||
from libc.stdint cimport uint64_t, int64_t
|
||||
|
||||
|
||||
cdef struct ViewIDs:
|
||||
size_t canon_form
|
||||
cdef extern from "sparsehash/dense_hash_map" namespace "google":
|
||||
cdef cppclass dense_hash_map[K, D]:
|
||||
K& key_type
|
||||
D& data_type
|
||||
pair[K, D]& value_type
|
||||
uint64_t size_type
|
||||
cppclass iterator:
|
||||
pair[K, D]& operator*() nogil
|
||||
iterator operator++() nogil
|
||||
iterator operator--() nogil
|
||||
bint operator==(iterator) nogil
|
||||
bint operator!=(iterator) nogil
|
||||
iterator begin()
|
||||
iterator end()
|
||||
uint64_t size()
|
||||
uint64_t max_size()
|
||||
bint empty()
|
||||
uint64_t bucket_count()
|
||||
uint64_t bucket_size(uint64_t i)
|
||||
uint64_t bucket(K& key)
|
||||
double max_load_factor()
|
||||
void max_load_vactor(double new_grow)
|
||||
double min_load_factor()
|
||||
double min_load_factor(double new_grow)
|
||||
void set_resizing_parameters(double shrink, double grow)
|
||||
void resize(uint64_t n)
|
||||
void rehash(uint64_t n)
|
||||
dense_hash_map()
|
||||
dense_hash_map(uint64_t n)
|
||||
void swap(dense_hash_map&)
|
||||
pair[iterator, bint] insert(pair[K, D]) nogil
|
||||
void set_empty_key(K&)
|
||||
void set_deleted_key(K& key)
|
||||
void clear_deleted_key()
|
||||
void erase(iterator pos)
|
||||
uint64_t erase(K& k)
|
||||
void erase(iterator first, iterator last)
|
||||
void clear()
|
||||
void clear_no_resize()
|
||||
pair[iterator, iterator] equal_range(K& k)
|
||||
D& operator[](K&) nogil
|
||||
|
||||
|
||||
cdef class Lexicon:
|
||||
|
@ -27,13 +65,14 @@ cdef class Lexicon:
|
|||
|
||||
cdef class Language:
|
||||
cdef unicode name
|
||||
cdef dict cache
|
||||
cdef dense_hash_map[uint64_t, size_t] cache
|
||||
cdef size_t cache_size
|
||||
cpdef readonly Lexicon lexicon
|
||||
cpdef readonly object tokens_class
|
||||
|
||||
cpdef Tokens tokenize(self, unicode text)
|
||||
cpdef Lexeme lookup(self, unicode text)
|
||||
|
||||
cdef _tokenize(self, Tokens tokens, unicode string)
|
||||
cdef _tokenize(self, Tokens tokens, Py_UNICODE* characters, size_t length)
|
||||
cdef list _split(self, unicode string)
|
||||
cdef int _split_one(self, unicode word)
|
||||
|
|
|
@ -40,7 +40,8 @@ cdef class Language:
|
|||
if string_features is None:
|
||||
string_features = []
|
||||
self.name = name
|
||||
self.cache = {}
|
||||
self.cache.set_empty_key(0)
|
||||
self.cache_size = 0
|
||||
lang_data = read_lang_data(name)
|
||||
rules, words, probs, clusters, case_stats, tag_stats = lang_data
|
||||
self.lexicon = Lexicon(words, probs, clusters, case_stats, tag_stats,
|
||||
|
@ -102,11 +103,10 @@ cdef class Language:
|
|||
cdef _tokenize(self, Tokens tokens, Py_UNICODE* characters, size_t length):
|
||||
cdef uint64_t hashed = hash64(characters, length * sizeof(Py_UNICODE), 0)
|
||||
cdef unicode string
|
||||
cdef LexemeC** lexemes
|
||||
cdef bint free_chunk = False
|
||||
cdef size_t i = 0
|
||||
if hashed in self.cache:
|
||||
lexemes = <LexemeC**><size_t>self.cache[hashed]
|
||||
cdef LexemeC** lexemes = <LexemeC**>self.cache[hashed]
|
||||
if lexemes is not NULL:
|
||||
while lexemes[i] != NULL:
|
||||
tokens.push_back(lexemes[i])
|
||||
i += 1
|
||||
|
@ -121,8 +121,9 @@ cdef class Language:
|
|||
# The intuition here is that if an element belongs in the cache, it
|
||||
# has several chances to get in. And if the cache is large, we less
|
||||
# believe that the element belongs there.
|
||||
if not self.cache or random.random() < (100000.0 / len(self.cache)):
|
||||
if self.cache_size == 0 or random.random() < (100000.0 / self.cache_size):
|
||||
self.cache[hashed] = <size_t>lexemes
|
||||
self.cache_size += 1
|
||||
else:
|
||||
free(lexemes)
|
||||
|
||||
|
@ -172,6 +173,7 @@ cdef class Language:
|
|||
lexemes[i + 1] = NULL
|
||||
hashed = hash64(<Py_UNICODE*>string, len(string) * sizeof(Py_UNICODE), 0)
|
||||
self.cache[hashed] = <size_t>lexemes
|
||||
self.cache_size += 1
|
||||
|
||||
|
||||
cdef class Lexicon:
|
||||
|
|
|
@ -5,7 +5,7 @@ cdef class Tokens:
|
|||
cdef size_t size
|
||||
|
||||
cdef LexemeC** lexemes
|
||||
cdef push_back(self, LexemeC* lexeme)
|
||||
cdef int push_back(self, LexemeC* lexeme) except -1
|
||||
|
||||
cpdef unicode string(self, size_t i)
|
||||
cpdef double prob(self, size_t i)
|
||||
|
|
|
@ -44,7 +44,7 @@ cdef class Tokens:
|
|||
def append(self, Lexeme lexeme):
|
||||
self.push_back(lexeme._c)
|
||||
|
||||
cdef push_back(self, LexemeC* lexeme):
|
||||
cdef int push_back(self, LexemeC* lexeme) except -1:
|
||||
if (self.size + 1) == self.length:
|
||||
self.size *= 2
|
||||
self.lexemes = <LexemeC**>realloc(self.lexemes, self.size * sizeof(LexemeC*))
|
||||
|
|
Loading…
Reference in New Issue
Block a user