mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-14 21:57:15 +03:00
5d0d2de955
Add a context manage nlp.memory_zone(), which will begin memory_zone() blocks on the vocab, string store, and potentially other components. Once the memory_zone() block expires, spaCy will free any shared resources that were allocated for the text-processing that occurred within the memory_zone. If you create Doc objects within a memory zone, it's invalid to access them once the memory zone is expired. The purpose of this is that spaCy creates and stores Lexeme objects in the Vocab that can be shared between multiple Doc objects. It also interns strings. Normally, spaCy can't know when all Doc objects using a Lexeme are out-of-scope, so new Lexemes accumulate in the vocab, causing memory pressure. Memory zones solve this problem by telling spaCy "okay none of the documents allocated within this block will be accessed again". This lets spaCy free all new Lexeme objects and other data that were created during the block. The mechanism is general, so memory_zone() context managers can be added to other components that could benefit from them, e.g. pipeline components. I experimented with adding memory zone support to the tokenizer as well, for its cache. However, this seems unnecessarily complicated. It makes more sense to just stick a limit on the cache size. This lets spaCy benefit from the efficiency advantage of the cache better, because we can maintain a (bounded) cache even if only small batches of documents are being processed.
84 lines
2.1 KiB
Cython
84 lines
2.1 KiB
Cython
from cymem.cymem cimport Pool
|
|
from libcpp.vector cimport vector
|
|
from preshed.maps cimport PreshMap
|
|
|
|
from .matcher.phrasematcher cimport PhraseMatcher
|
|
from .structs cimport LexemeC, SpanC, TokenC
|
|
from .tokens.doc cimport Doc
|
|
from .typedefs cimport hash_t
|
|
from .vocab cimport LexemesOrTokens, Vocab, _Cached
|
|
|
|
|
|
cdef class Tokenizer:
|
|
cdef Pool mem
|
|
cdef PreshMap _cache
|
|
cdef PreshMap _specials
|
|
cdef readonly Vocab vocab
|
|
|
|
cdef object _token_match
|
|
cdef object _url_match
|
|
cdef object _prefix_search
|
|
cdef object _suffix_search
|
|
cdef object _infix_finditer
|
|
cdef object _rules
|
|
cdef PhraseMatcher _special_matcher
|
|
cdef bint _faster_heuristics
|
|
cdef public int max_cache_size
|
|
|
|
cdef Doc _tokenize_affixes(self, str string, bint with_special_cases)
|
|
cdef int _apply_special_cases(self, Doc doc) except -1
|
|
cdef void _filter_special_spans(
|
|
self,
|
|
vector[SpanC] &original,
|
|
vector[SpanC] &filtered,
|
|
int doc_len,
|
|
) nogil
|
|
cdef object _prepare_special_spans(
|
|
self,
|
|
Doc doc,
|
|
vector[SpanC] &filtered,
|
|
)
|
|
cdef int _retokenize_special_spans(
|
|
self,
|
|
Doc doc,
|
|
TokenC* tokens,
|
|
object span_data,
|
|
)
|
|
cdef int _try_specials_and_cache(
|
|
self,
|
|
hash_t key,
|
|
Doc tokens,
|
|
int* has_special,
|
|
bint with_special_cases,
|
|
) except -1
|
|
cdef int _tokenize(
|
|
self,
|
|
Doc tokens,
|
|
str span,
|
|
hash_t key,
|
|
int* has_special,
|
|
bint with_special_cases,
|
|
) except -1
|
|
cdef str _split_affixes(
|
|
self,
|
|
str string,
|
|
vector[LexemeC*] *prefixes,
|
|
vector[LexemeC*] *suffixes, int* has_special,
|
|
bint with_special_cases,
|
|
)
|
|
cdef int _attach_tokens(
|
|
self,
|
|
Doc tokens,
|
|
str string,
|
|
vector[LexemeC*] *prefixes,
|
|
vector[LexemeC*] *suffixes, int* has_special,
|
|
bint with_special_cases,
|
|
) except -1
|
|
cdef int _save_cached(
|
|
self,
|
|
const TokenC* tokens,
|
|
hash_t key,
|
|
int* has_special,
|
|
int n,
|
|
) except -1
|