explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-09-21 03:19:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Try again to do selective remove cache

Browse Source
This commit is contained in:
Roman Domrachev 2017-11-15 19:11:12 +03:00
parent b3311100c7
commit 61d28d03e4
2 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/language.py
View File

@ -561,7 +561,7 @@ class Language(object):
old_refs, recent_refs = recent_refs, old_refs
keys, strings = self.vocab.strings._cleanup_stale_strings(original_strings_data)
self.vocab._reset_cache(keys, strings)
self.tokenizer._reset_cache()
self.tokenizer._reset_cache(keys)
nr_seen = 0
def to_disk(self, path, disable=tuple()):

9
spacy/tokenizer.pyx
View File

@ -133,11 +133,10 @@ cdef class Tokenizer:
for text in texts:
yield self(text)
def _reset_cache(self):
# We cannot do selective cache cleanup because cache can be different than words
# saved in StringStore (prefixes/suffixes/etc).
self._cache = PreshMap()
self._specials = PreshMap()
def _reset_cache(self, keys):
for k in keys:
del self._cache[k]
del self._specials[k]
cdef int _try_cache(self, hash_t key, Doc tokens) except -1:
cached = <_Cached*>self._cache.get(key)