explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-10-31 07:57:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Remove need for confusing _data pointer to be stored on Tokens

Browse Source
This commit is contained in:
Matthew Honnibal 2014-12-05 16:31:30 +11:00
parent 1c9253701d
commit e27b912ef9
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
spacy/tokens.pxd
View File

@ -23,7 +23,6 @@ cdef class Tokens:
cdef Pool mem cdef Pool mem
cdef StringStore _string_store cdef StringStore _string_store
cdef TokenC* _data
cdef TokenC* data cdef TokenC* data
cdef int length cdef int length

11
spacy/tokens.pyx
View File

@ -40,11 +40,11 @@ cdef class Tokens:
# Guarantee self.lex[i-x], for any i >= 0 and x < padding is in bounds # Guarantee self.lex[i-x], for any i >= 0 and x < padding is in bounds
# However, we need to remember the true starting places, so that we can # However, we need to remember the true starting places, so that we can
# realloc. # realloc.
self._data = <TokenC*>self.mem.alloc(size + (PADDING*2), sizeof(TokenC)) data_start = <TokenC*>self.mem.alloc(size + (PADDING*2), sizeof(TokenC))
cdef int i cdef int i
for i in range(size + (PADDING*2)): for i in range(size + (PADDING*2)):
self._data[i] = EMPTY_TOKEN data_start[i] = EMPTY_TOKEN
self.data = self._data + PADDING self.data = data_start + PADDING
self.max_length = size self.max_length = size
self.length = 0 self.length = 0
@ -116,8 +116,9 @@ cdef class Tokens:
def _realloc(self, new_size): def _realloc(self, new_size):
self.max_length = new_size self.max_length = new_size
n = new_size + (PADDING * 2) n = new_size + (PADDING * 2)
self._data = <TokenC*>self.mem.realloc(self._data, n * sizeof(TokenC)) cdef TokenC* data_start = self.data - PADDING
self.data = self._data + PADDING data_start = <TokenC*>self.mem.realloc(data_start, n * sizeof(TokenC))
self.data = data_start + PADDING
cdef int i cdef int i
for i in range(self.length, self.max_length + PADDING): for i in range(self.length, self.max_length + PADDING):
self.data[i] = EMPTY_TOKEN self.data[i] = EMPTY_TOKEN