This commit is contained in:
Matthew Honnibal 2024-09-08 14:39:58 +02:00
parent 6a90330ff4
commit 80b1005ab1
3 changed files with 7 additions and 9 deletions

View File

@ -27,8 +27,6 @@ cdef class StringStore:
cdef const Utf8Str* intern_unicode(self, str py_string, bint allow_transient)
cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length, hash_t* precalculated_hash, bint allow_transient)
cdef vector[hash_t] _transient_keys
cdef PreshMap _transient_map
cdef Pool _non_temp_mem

View File

@ -3,7 +3,7 @@
cimport cython
from contextlib import contextmanager
from typing import Iterable, Iterator, List, Optional, Tuple, Union
from typing import Iterator, List, Optional
from libc.stdint cimport uint32_t
from libc.string cimport memcpy
@ -196,7 +196,7 @@ cdef class StringStore:
return self._keys.size() + self._transient_keys.size()
@contextmanager
def memory_zone(self, mem: Optional[Pool]=None) -> Pool:
def memory_zone(self, mem: Optional[Pool] = None) -> Pool:
"""Begin a block where all resources allocated during the block will
be freed at the end of it. If a resources was created within the
memory zone block, accessing it outside the block is invalid.

View File

@ -401,7 +401,7 @@ cdef class Tokenizer:
with_special_cases)
if len(self._cache) < self.max_cache_size:
self._save_cached(&tokens.c[orig_size], orig_key, has_special,
tokens.length - orig_size)
tokens.length - orig_size)
cdef str _split_affixes(
self,