explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 17:36:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove obsolete is_frozen functionality from StringStore

Browse Source
This commit is contained in:
Matthew Honnibal 2017-10-16 19:23:10 +02:00
parent 5c14f3f033
commit 3e037054c8
2 changed files with 1 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/strings.pxd
View File

@ -21,11 +21,9 @@ ctypedef union Utf8Str:
cdef class StringStore: cdef class StringStore:
cdef Pool mem cdef Pool mem
cdef bint is_frozen
cdef vector[hash_t] keys cdef vector[hash_t] keys
cdef public PreshMap _map cdef public PreshMap _map
cdef public PreshMap _oov
cdef const Utf8Str* intern_unicode(self, unicode py_string) cdef const Utf8Str* intern_unicode(self, unicode py_string)
cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length) cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length)

21
spacy/strings.pyx
View File

@ -86,8 +86,6 @@ cdef class StringStore:
""" """
self.mem = Pool() self.mem = Pool()
self._map = PreshMap() self._map = PreshMap()
self._oov = PreshMap()
self.is_frozen = freeze
if strings is not None: if strings is not None:
for string in strings: for string in strings:
self.add(string) self.add(string)
@ -243,21 +241,12 @@ cdef class StringStore:
self.add(word) self.add(word)
return self return self
def set_frozen(self, bint is_frozen): def _reset_and_load(self, strings):
# TODO
self.is_frozen = is_frozen
def flush_oov(self):
self._oov = PreshMap()
def _reset_and_load(self, strings, freeze=False):
self.mem = Pool() self.mem = Pool()
self._map = PreshMap() self._map = PreshMap()
self._oov = PreshMap()
self.keys.clear() self.keys.clear()
for string in strings: for string in strings:
self.add(string) self.add(string)
self.is_frozen = freeze
cdef const Utf8Str* intern_unicode(self, unicode py_string): cdef const Utf8Str* intern_unicode(self, unicode py_string):
# 0 means missing, but we don't bother offsetting the index. # 0 means missing, but we don't bother offsetting the index.
@ -275,14 +264,6 @@ cdef class StringStore:
value = <Utf8Str*>self._oov.get(key) value = <Utf8Str*>self._oov.get(key)
if value is not NULL: if value is not NULL:
return value return value
if self.is_frozen:
# OOV store uses 32 bit hashes. Pretty ugly :(
key32 = hash32_utf8(utf8_string, length)
# Important: Make the OOV store own the memory. That way it's trivial
# to flush them all.
value = _allocate(self._oov.mem, <unsigned char*>utf8_string, length)
self._oov.set(key32, value)
return NULL
value = _allocate(self.mem, <unsigned char*>utf8_string, length) value = _allocate(self.mem, <unsigned char*>utf8_string, length)
self._map.set(key, value) self._map.set(key, value)