explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-06-21 13:33:08 +03:00
Code Issues Packages Projects Releases Wiki Activity

strings: Make hash_utf8 and decode_Utf8Str private

Browse Source
This commit is contained in:
shadeMe 2022-08-17 12:12:09 +02:00
parent e032250d42
commit e7effa0c26
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
spacy/strings.pyx
View File

@ -39,7 +39,7 @@ def get_string_id(key):
return symbol return symbol
else: else:
chars = key.encode("utf8") chars = key.encode("utf8")
return hash_utf8(chars, len(chars)) return _hash_utf8(chars, len(chars))
elif _try_coerce_to_hash(key, &str_hash): elif _try_coerce_to_hash(key, &str_hash):
# Coerce the integral key to the expected primitive hash type. # Coerce the integral key to the expected primitive hash type.
# This ensures that custom/overloaded "primitive" data types # This ensures that custom/overloaded "primitive" data types
@ -54,15 +54,15 @@ def get_string_id(key):
cpdef hash_t hash_string(str string) except 0: cpdef hash_t hash_string(str string) except 0:
chars = string.encode("utf8") chars = string.encode("utf8")
return hash_utf8(chars, len(chars)) return _hash_utf8(chars, len(chars))
cdef hash_t hash_utf8(char* utf8_string, int length) nogil: cdef hash_t _hash_utf8(char* utf8_string, int length) nogil:
return hash64(utf8_string, length, 1) return hash64(utf8_string, length, 1)
cdef str decode_Utf8Str(const Utf8Str* string): cdef str _decode_Utf8Str(const Utf8Str* string):
cdef int i, length cdef int i, length
if string.s[0] < sizeof(string.s) and string.s[0] != 0: if string.s[0] < sizeof(string.s) and string.s[0] != 0:
return string.s[1:string.s[0]+1].decode("utf8") return string.s[1:string.s[0]+1].decode("utf8")
@ -140,7 +140,7 @@ cdef class StringStore:
else: else:
return hash_string(string_or_id) return hash_string(string_or_id)
elif isinstance(string_or_id, bytes): elif isinstance(string_or_id, bytes):
return hash_utf8(string_or_id, len(string_or_id)) return _hash_utf8(string_or_id, len(string_or_id))
elif _try_coerce_to_hash(string_or_id, &str_hash): elif _try_coerce_to_hash(string_or_id, &str_hash):
if str_hash == 0: if str_hash == 0:
return "" return ""
@ -155,7 +155,7 @@ cdef class StringStore:
if utf8str is NULL: if utf8str is NULL:
raise KeyError(Errors.E018.format(hash_value=string_or_id)) raise KeyError(Errors.E018.format(hash_value=string_or_id))
else: else:
return decode_Utf8Str(utf8str) return _decode_Utf8Str(utf8str)
def as_int(self, key): def as_int(self, key):
"""If key is an int, return it; otherwise, get the int value.""" """If key is an int, return it; otherwise, get the int value."""
@ -183,12 +183,12 @@ cdef class StringStore:
return SYMBOLS_BY_STR[string] return SYMBOLS_BY_STR[string]
string = string.encode("utf8") string = string.encode("utf8")
str_hash = hash_utf8(string, len(string)) str_hash = _hash_utf8(string, len(string))
self._intern_utf8(string, len(string), &str_hash) self._intern_utf8(string, len(string), &str_hash)
elif isinstance(string, bytes): elif isinstance(string, bytes):
if string in SYMBOLS_BY_STR: if string in SYMBOLS_BY_STR:
return SYMBOLS_BY_STR[string] return SYMBOLS_BY_STR[string]
str_hash = hash_utf8(string, len(string)) str_hash = _hash_utf8(string, len(string))
self._intern_utf8(string, len(string), &str_hash) self._intern_utf8(string, len(string), &str_hash)
else: else:
raise TypeError(Errors.E017.format(value_type=type(string))) raise TypeError(Errors.E017.format(value_type=type(string)))
@ -235,7 +235,7 @@ cdef class StringStore:
for i in range(self.keys.size()): for i in range(self.keys.size()):
key = self.keys[i] key = self.keys[i]
utf8str = <Utf8Str*>self._map.get(key) utf8str = <Utf8Str*>self._map.get(key)
yield decode_Utf8Str(utf8str) yield _decode_Utf8Str(utf8str)
# TODO: Iterate OOV here? # TODO: Iterate OOV here?
def __reduce__(self): def __reduce__(self):
@ -304,7 +304,7 @@ cdef class StringStore:
cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length, hash_t* precalculated_hash): cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length, hash_t* precalculated_hash):
# TODO: This function's API/behaviour is an unholy mess... # TODO: This function's API/behaviour is an unholy mess...
# 0 means missing, but we don't bother offsetting the index. # 0 means missing, but we don't bother offsetting the index.
cdef hash_t key = precalculated_hash[0] if precalculated_hash is not NULL else hash_utf8(utf8_string, length) cdef hash_t key = precalculated_hash[0] if precalculated_hash is not NULL else _hash_utf8(utf8_string, length)
cdef Utf8Str* value = <Utf8Str*>self._map.get(key) cdef Utf8Str* value = <Utf8Str*>self._map.get(key)
if value is not NULL: if value is not NULL:
return value return value