explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 17:36:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Try not holding a reference to Pool, since that seems to confuse the GC

Browse Source
This commit is contained in:
Matthew Honnibal 2015-01-31 22:10:22 +11:00
parent ce3ae8b5d9
commit 7de00c5a79
2 changed files with 9 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
spacy/tokens.pxd
View File

@ -53,7 +53,6 @@ cdef class Tokens:
cdef class Token:
cdef Vocab vocab
cdef Pool mem
cdef unicode _string
cdef const TokenC* c
@ -66,14 +65,14 @@ cdef class Token:
cdef tuple _dep_strings
@staticmethod
cdef inline Token cinit(Pool mem, Vocab vocab, unicode string,
cdef inline Token cinit(Vocab vocab, unicode string,
const TokenC* token, int offset, int array_len,
list py_tokens, tuple tag_strings, tuple dep_strings):
assert offset >= 0 and offset < array_len
if py_tokens[offset] is not None:
return py_tokens[offset]
cdef Token self = Token.__new__(Token, mem, vocab, string)
cdef Token self = Token.__new__(Token, vocab, string)
self.c = token
self.i = offset

15
spacy/tokens.pyx
View File

@ -104,7 +104,7 @@ cdef class Tokens:
if i < 0:
i = self.length - i
bounds_check(i, self.length, PADDING)
return Token.cinit(self.mem, self.vocab, self._string,
return Token.cinit(self.vocab, self._string,
&self.data[i], i, self.length,
self._py_tokens, self._tag_strings, self._dep_strings)
@ -115,7 +115,7 @@ cdef class Tokens:
token (Token):
"""
for i in range(self.length):
yield Token.cinit(self.mem, self.vocab, self._string,
yield Token.cinit(self.vocab, self._string,
&self.data[i], i, self.length,
self._py_tokens, self._tag_strings, self._dep_strings)
@ -233,8 +233,7 @@ cdef class Tokens:
cdef class Token:
"""An individual token."""
def __cinit__(self, Pool mem, Vocab vocab, unicode string):
self.mem = mem
def __cinit__(self, Vocab vocab, unicode string):
self.vocab = vocab
self._string = string
@ -242,7 +241,7 @@ cdef class Token:
return self.c.lex.length
def nbor(self, int i=1):
return Token.cinit(self.mem, self.vocab, self._string,
return Token.cinit(self.vocab, self._string,
self.c, self.i, self.array_len,
self._py, self._tag_strings, self._dep_strings)
@ -343,7 +342,7 @@ cdef class Token:
ptr += ptr.head
elif ptr + ptr.head == self.c:
yield Token.cinit(self.mem, self.vocab, self._string,
yield Token.cinit(self.vocab, self._string,
ptr, ptr - (self.c - self.i), self.array_len,
self._py, self._tag_strings, self._dep_strings)
ptr += 1
@ -362,7 +361,7 @@ cdef class Token:
if (ptr.head < 0) and ((ptr + ptr.head) > self.c):
ptr += ptr.head
elif ptr + ptr.head == self.c:
yield Token.cinit(self.mem, self.vocab, self._string,
yield Token.cinit(self.vocab, self._string,
ptr, ptr - (self.c - self.i), self.array_len,
self._py, self._tag_strings, self._dep_strings)
ptr -= 1
@ -372,7 +371,7 @@ cdef class Token:
@property
def head(self):
"""The token predicted by the parser to be the head of the current token."""
return Token.cinit(self.mem, self.vocab, self._string,
return Token.cinit(self.vocab, self._string,
self.c + self.c.head, self.i + self.c.head, self.array_len,
self._py, self._tag_strings, self._dep_strings)