mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 19:39:13 +03:00
35 lines
704 B
Cython
35 lines
704 B
Cython
from preshed.maps cimport PreshMap
|
|
from cymem.cymem cimport Pool
|
|
from murmurhash.mrmr cimport hash64
|
|
|
|
from .typedefs cimport utf8_t, id_t, hash_t
|
|
|
|
|
|
cdef struct Utf8Str:
|
|
id_t i
|
|
hash_t key
|
|
utf8_t chars
|
|
int length
|
|
|
|
|
|
cdef struct UniStr:
|
|
Py_UNICODE* chars
|
|
size_t n
|
|
hash_t key
|
|
|
|
|
|
cdef inline void slice_unicode(UniStr* s, Py_UNICODE* chars, int start, int end) nogil:
|
|
s.chars = &chars[start]
|
|
s.n = end - start
|
|
s.key = hash64(s.chars, s.n * sizeof(Py_UNICODE), 0)
|
|
|
|
|
|
cdef class StringStore:
|
|
cdef Pool mem
|
|
cdef PreshMap _map
|
|
cdef Utf8Str* strings
|
|
cdef int size
|
|
cdef int _resize_at
|
|
|
|
cdef Utf8Str* intern(self, char* chars, int length) except NULL
|