* Move String struct stuff into the utf8string module, from spacy.lang

This commit is contained in:
Matthew Honnibal 2014-12-03 11:06:00 +11:00
parent e170faf5b0
commit e600f7b327

View File

@ -1,5 +1,6 @@
from preshed.maps cimport PreshMap
from cymem.cymem cimport Pool
from murmurhash.mrmr cimport hash64
from .typedefs cimport utf8_t, id_t, hash_t
@ -11,6 +12,18 @@ cdef struct Utf8Str:
int length
cdef struct UniStr:
Py_UNICODE* chars
size_t n
hash_t key
cdef inline void slice_unicode(UniStr* s, Py_UNICODE* chars, int start, int end) nogil:
s.chars = &chars[start]
s.n = end - start
s.key = hash64(s.chars, s.n * sizeof(Py_UNICODE), 0)
cdef class StringStore:
cdef Pool mem
cdef PreshMap _map