explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-11-04 01:48:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of ssh://github.com/honnibal/spaCy

Browse Source
This commit is contained in:
Matthew Honnibal 2015-11-06 00:26:15 +11:00
commit e18bdff23a
2 changed files with 26 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spacy/strings.pxd
View File

@ -20,7 +20,7 @@ cdef class StringStore:
cdef Utf8Str* c cdef Utf8Str* c
cdef int64_t size cdef int64_t size
cdef PreshMap _map cdef public PreshMap _map
cdef size_t _resize_at cdef int64_t _resize_at
cdef const Utf8Str* intern(self, unsigned char* chars, int length) except NULL cdef const Utf8Str* intern(self, unicode py_string) except NULL

41
spacy/strings.pyx
View File

@ -4,6 +4,8 @@ import codecs
from libc.string cimport memcpy from libc.string cimport memcpy
from murmurhash.mrmr cimport hash64 from murmurhash.mrmr cimport hash64
from preshed.maps cimport map_iter, key_t
from cpython cimport PyUnicode_AS_DATA from cpython cimport PyUnicode_AS_DATA
from cpython cimport PyUnicode_GET_DATA_SIZE from cpython cimport PyUnicode_GET_DATA_SIZE
@ -76,7 +78,7 @@ cdef class StringStore:
def __init__(self, strings=None): def __init__(self, strings=None):
self.mem = Pool() self.mem = Pool()
self._map = PreshMap() self._map = PreshMap()
self._resize_at = 10000 self._resize_at = 10
self.c = <Utf8Str*>self.mem.alloc(self._resize_at, sizeof(Utf8Str)) self.c = <Utf8Str*>self.mem.alloc(self._resize_at, sizeof(Utf8Str))
self.size = 1 self.size = 1
if strings is not None: if strings is not None:
@ -92,6 +94,7 @@ cdef class StringStore:
def __getitem__(self, object string_or_id): def __getitem__(self, object string_or_id):
cdef bytes byte_string cdef bytes byte_string
cdef unicode py_string
cdef const Utf8Str* utf8str cdef const Utf8Str* utf8str
cdef int id_ cdef int id_
if isinstance(string_or_id, int) or isinstance(string_or_id, long): if isinstance(string_or_id, int) or isinstance(string_or_id, long):
@ -104,13 +107,14 @@ cdef class StringStore:
elif isinstance(string_or_id, bytes): elif isinstance(string_or_id, bytes):
if len(string_or_id) == 0: if len(string_or_id) == 0:
return 0 return 0
utf8str = self.intern(<unsigned char*>string_or_id, len(string_or_id)) py_string = string_or_id.decode('utf8')
utf8str = self.intern(py_string)
return utf8str - self.c return utf8str - self.c
elif isinstance(string_or_id, unicode): elif isinstance(string_or_id, unicode):
if len(string_or_id) == 0: if len(string_or_id) == 0:
return 0 return 0
byte_string = string_or_id.encode('utf8') py_string = string_or_id
utf8str = self.intern(<unsigned char*>byte_string, len(byte_string)) utf8str = self.intern(py_string)
return utf8str - self.c return utf8str - self.c
else: else:
raise TypeError(type(string_or_id)) raise TypeError(type(string_or_id))
@ -132,16 +136,17 @@ cdef class StringStore:
strings.append(py_string) strings.append(py_string)
return (StringStore, (strings,), None, None, None) return (StringStore, (strings,), None, None, None)
cdef const Utf8Str* intern(self, unsigned char* chars, int length) except NULL: cdef const Utf8Str* intern(self, unicode py_string) except NULL:
# 0 means missing, but we don't bother offsetting the index. # 0 means missing, but we don't bother offsetting the index.
key = hash64(chars, length * sizeof(char), 0) cdef hash_t key = hash_string(py_string)
value = <Utf8Str*>self._map.get(key) value = <Utf8Str*>self._map.get(key)
if value != NULL: if value != NULL:
return value return value
if self.size == self._resize_at: if self.size == self._resize_at:
self._realloc() self._realloc()
self.c[self.size] = _allocate(self.mem, chars, length) cdef bytes byte_string = py_string.encode('utf8')
self.c[self.size] = _allocate(self.mem, <unsigned char*>byte_string, len(byte_string))
self._map.set(key, <void*>&self.c[self.size]) self._map.set(key, <void*>&self.c[self.size])
self.size += 1 self.size += 1
return &self.c[self.size-1] return &self.c[self.size-1]
@ -157,26 +162,25 @@ cdef class StringStore:
if strings == ['']: if strings == ['']:
return None return None
cdef unicode string cdef unicode string
cdef bytes byte_string
for string in strings: for string in strings:
if string: if string:
byte_string = string.encode('utf8') self.intern(string)
self.intern(byte_string, len(byte_string))
def _realloc(self): def _realloc(self):
# We want to map straight to pointers, but they'll be invalidated if # We want to map straight to pointers, but they'll be invalidated if
# we resize our array. So, first we remap to indices, then we resize, # we resize our array. So, first we remap to indices, then we resize,
# then we can acquire the new pointers. # then we can acquire the new pointers.
cdef Pool tmp_mem = Pool() cdef Pool tmp_mem = Pool()
keys = <hash_t*>tmp_mem.alloc(self.size, sizeof(hash_t)) keys = <key_t*>tmp_mem.alloc(self.size, sizeof(key_t))
cdef hash_t key cdef key_t key
cdef size_t addr cdef void* value
cdef const Utf8Str ptr cdef const Utf8Str ptr
cdef size_t i cdef int i = 0
for key, addr in self._map.items(): cdef size_t offset
while map_iter(self._map.c_map, &i, &key, &value):
# Find array index with pointer arithmetic # Find array index with pointer arithmetic
i = (<Utf8Str*>addr) - self.c offset = ((<Utf8Str*>value) - self.c)
keys[i] = key keys[offset] = key
self._resize_at *= 2 self._resize_at *= 2
cdef size_t new_size = self._resize_at * sizeof(Utf8Str) cdef size_t new_size = self._resize_at * sizeof(Utf8Str)
@ -184,4 +188,5 @@ cdef class StringStore:
self._map = PreshMap(self.size) self._map = PreshMap(self.size)
for i in range(self.size): for i in range(self.size):
self._map.set(keys[i], &self.c[i]) if keys[i]:
self._map.set(keys[i], &self.c[i])