explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-13 18:56:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Try again to do selective remove cache

Browse Source
This commit is contained in:
Roman Domrachev 2017-11-15 19:11:12 +03:00
parent b3311100c7
commit 61d28d03e4
2 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/language.py
View File

@ -561,7 +561,7 @@ class Language(object):
old_refs, recent_refs = recent_refs, old_refs old_refs, recent_refs = recent_refs, old_refs
keys, strings = self.vocab.strings._cleanup_stale_strings(original_strings_data) keys, strings = self.vocab.strings._cleanup_stale_strings(original_strings_data)
self.vocab._reset_cache(keys, strings) self.vocab._reset_cache(keys, strings)
self.tokenizer._reset_cache() self.tokenizer._reset_cache(keys)
nr_seen = 0 nr_seen = 0
def to_disk(self, path, disable=tuple()): def to_disk(self, path, disable=tuple()):

9
spacy/tokenizer.pyx
View File

@ -133,11 +133,10 @@ cdef class Tokenizer:
for text in texts: for text in texts:
yield self(text) yield self(text)
def _reset_cache(self): def _reset_cache(self, keys):
# We cannot do selective cache cleanup because cache can be different than words for k in keys:
# saved in StringStore (prefixes/suffixes/etc). del self._cache[k]
self._cache = PreshMap() del self._specials[k]
self._specials = PreshMap()
cdef int _try_cache(self, hash_t key, Doc tokens) except -1: cdef int _try_cache(self, hash_t key, Doc tokens) except -1:
cached = <_Cached*>self._cache.get(key) cached = <_Cached*>self._cache.get(key)