explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 18:06:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Remove unnecessary key and id properties from Utf8String.

Browse Source
This commit is contained in:
Matthew Honnibal 2015-07-16 19:29:02 +02:00
parent 5b0a7190c9
commit d1cb30dbc4
4 changed files with 10 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spacy/en/pos.pyx
View File

@ -332,8 +332,7 @@ cdef class EnPosTagger:
cdef unicode lemma_string cdef unicode lemma_string
lemma_strings = self.lemmatizer(py_string, pos) lemma_strings = self.lemmatizer(py_string, pos)
lemma_string = sorted(lemma_strings)[0] lemma_string = sorted(lemma_strings)[0]
bytes_string = lemma_string.encode('utf8') lemma = self.strings[lemma_string]
lemma = self.strings.intern(bytes_string, len(bytes_string)).i
return lemma return lemma
def load_morph_exceptions(self, dict exc): def load_morph_exceptions(self, dict exc):

2
spacy/strings.pxd
View File

@ -22,4 +22,4 @@ cdef class StringStore:
cdef PreshMap _map cdef PreshMap _map
cdef size_t _resize_at cdef size_t _resize_at
cdef const Utf8Str* intern(self, char* chars, int length) except NULL cdef const Utf8Str* intern(self, char* chars, int length, int* id_) except NULL

15
spacy/strings.pyx
View File

@ -67,6 +67,7 @@ cdef class StringStore:
def __getitem__(self, object string_or_id): def __getitem__(self, object string_or_id):
cdef bytes byte_string cdef bytes byte_string
cdef const Utf8Str* utf8str cdef const Utf8Str* utf8str
cdef int id_
if isinstance(string_or_id, int) or isinstance(string_or_id, long): if isinstance(string_or_id, int) or isinstance(string_or_id, long):
if string_or_id == 0: if string_or_id == 0:
return u'' return u''
@ -75,16 +76,16 @@ cdef class StringStore:
utf8str = &self.strings[<int>string_or_id] utf8str = &self.strings[<int>string_or_id]
return utf8str.chars[:utf8str.length].decode('utf8') return utf8str.chars[:utf8str.length].decode('utf8')
elif isinstance(string_or_id, bytes): elif isinstance(string_or_id, bytes):
utf8str = self.intern(<char*>string_or_id, len(string_or_id)) utf8str = self.intern(<char*>string_or_id, len(string_or_id), &id_)
return utf8str.i return id_
elif isinstance(string_or_id, unicode): elif isinstance(string_or_id, unicode):
byte_string = string_or_id.encode('utf8') byte_string = string_or_id.encode('utf8')
utf8str = self.intern(<char*>byte_string, len(byte_string)) utf8str = self.intern(<char*>byte_string, len(byte_string), &id_)
return utf8str.i return id_
else: else:
raise TypeError(type(string_or_id)) raise TypeError(type(string_or_id))
cdef const Utf8Str* intern(self, char* chars, int length) except NULL: cdef const Utf8Str* intern(self, char* chars, int length, int* id_) except NULL:
# 0 means missing, but we don't bother offsetting the index. We waste # 0 means missing, but we don't bother offsetting the index. We waste
# slot 0 to simplify the code, because it doesn't matter. # slot 0 to simplify the code, because it doesn't matter.
assert length != 0 assert length != 0
@ -97,7 +98,6 @@ cdef class StringStore:
self.strings = <Utf8Str*>self.mem.realloc(self.strings, self._resize_at * sizeof(Utf8Str)) self.strings = <Utf8Str*>self.mem.realloc(self.strings, self._resize_at * sizeof(Utf8Str))
i = self.size i = self.size
self.strings[i].i = self.size self.strings[i].i = self.size
self.strings[i].key = key
self.strings[i].chars = <unsigned char*>self.mem.alloc(length, sizeof(char)) self.strings[i].chars = <unsigned char*>self.mem.alloc(length, sizeof(char))
memcpy(self.strings[i].chars, chars, length) memcpy(self.strings[i].chars, chars, length)
self.strings[i].length = length self.strings[i].length = length
@ -123,6 +123,7 @@ cdef class StringStore:
strings = file_.read().split(SEPARATOR) strings = file_.read().split(SEPARATOR)
cdef unicode string cdef unicode string
cdef bytes byte_string cdef bytes byte_string
cdef int id_
for string in strings[1:]: for string in strings[1:]:
byte_string = string.encode('utf8') byte_string = string.encode('utf8')
self.intern(byte_string, len(byte_string)) self.intern(byte_string, len(byte_string), &id_)

2
spacy/structs.pxd
View File

@ -80,8 +80,6 @@ cdef struct TokenC:
cdef struct Utf8Str: cdef struct Utf8Str:
id_t i
hash_t key
unsigned char* chars unsigned char* chars
int length int length