diff --git a/spacy/strings.pxd b/spacy/strings.pxd index 2556a1be9..abc1d5a28 100644 --- a/spacy/strings.pxd +++ b/spacy/strings.pxd @@ -1,7 +1,14 @@ from cymem.cymem cimport Pool from preshed.maps cimport PreshMap +from murmurhash.mrmr cimport hash64 -from .structs cimport Utf8Str +from .structs cimport Utf8Str, UniStr + + +cdef inline void slice_unicode(UniStr* s, Py_UNICODE* chars, int start, int end) nogil: + s.chars = &chars[start] + s.n = end - start + s.key = hash64(s.chars, s.n * sizeof(Py_UNICODE), 0) cdef class StringStore: diff --git a/spacy/strings.pyx b/spacy/strings.pyx index e6f0b2db0..24c233cfb 100644 --- a/spacy/strings.pyx +++ b/spacy/strings.pyx @@ -2,7 +2,6 @@ import codecs from libc.string cimport memcpy -from murmurhash.mrmr cimport hash64 from .typedefs cimport hash_t