explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 01:46:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Try not holding a reference to Pool, since that seems to confuse the GC

Browse Source
This commit is contained in:
Matthew Honnibal 2015-01-31 22:10:22 +11:00
parent ce3ae8b5d9
commit 7de00c5a79
2 changed files with 9 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
spacy/tokens.pxd
View File

@ -53,7 +53,6 @@ cdef class Tokens:
cdef class Token: cdef class Token:
cdef Vocab vocab cdef Vocab vocab
cdef Pool mem
cdef unicode _string cdef unicode _string
cdef const TokenC* c cdef const TokenC* c
@ -66,14 +65,14 @@ cdef class Token:
cdef tuple _dep_strings cdef tuple _dep_strings
@staticmethod @staticmethod
cdef inline Token cinit(Pool mem, Vocab vocab, unicode string, cdef inline Token cinit(Vocab vocab, unicode string,
const TokenC* token, int offset, int array_len, const TokenC* token, int offset, int array_len,
list py_tokens, tuple tag_strings, tuple dep_strings): list py_tokens, tuple tag_strings, tuple dep_strings):
assert offset >= 0 and offset < array_len assert offset >= 0 and offset < array_len
if py_tokens[offset] is not None: if py_tokens[offset] is not None:
return py_tokens[offset] return py_tokens[offset]
cdef Token self = Token.__new__(Token, mem, vocab, string) cdef Token self = Token.__new__(Token, vocab, string)
self.c = token self.c = token
self.i = offset self.i = offset

15
spacy/tokens.pyx
View File

@ -104,7 +104,7 @@ cdef class Tokens:
if i < 0: if i < 0:
i = self.length - i i = self.length - i
bounds_check(i, self.length, PADDING) bounds_check(i, self.length, PADDING)
return Token.cinit(self.mem, self.vocab, self._string, return Token.cinit(self.vocab, self._string,
&self.data[i], i, self.length, &self.data[i], i, self.length,
self._py_tokens, self._tag_strings, self._dep_strings) self._py_tokens, self._tag_strings, self._dep_strings)
@ -115,7 +115,7 @@ cdef class Tokens:
token (Token): token (Token):
""" """
for i in range(self.length): for i in range(self.length):
yield Token.cinit(self.mem, self.vocab, self._string, yield Token.cinit(self.vocab, self._string,
&self.data[i], i, self.length, &self.data[i], i, self.length,
self._py_tokens, self._tag_strings, self._dep_strings) self._py_tokens, self._tag_strings, self._dep_strings)
@ -233,8 +233,7 @@ cdef class Tokens:
cdef class Token: cdef class Token:
"""An individual token.""" """An individual token."""
def __cinit__(self, Pool mem, Vocab vocab, unicode string): def __cinit__(self, Vocab vocab, unicode string):
self.mem = mem
self.vocab = vocab self.vocab = vocab
self._string = string self._string = string
@ -242,7 +241,7 @@ cdef class Token:
return self.c.lex.length return self.c.lex.length
def nbor(self, int i=1): def nbor(self, int i=1):
return Token.cinit(self.mem, self.vocab, self._string, return Token.cinit(self.vocab, self._string,
self.c, self.i, self.array_len, self.c, self.i, self.array_len,
self._py, self._tag_strings, self._dep_strings) self._py, self._tag_strings, self._dep_strings)
@ -343,7 +342,7 @@ cdef class Token:
ptr += ptr.head ptr += ptr.head
elif ptr + ptr.head == self.c: elif ptr + ptr.head == self.c:
yield Token.cinit(self.mem, self.vocab, self._string, yield Token.cinit(self.vocab, self._string,
ptr, ptr - (self.c - self.i), self.array_len, ptr, ptr - (self.c - self.i), self.array_len,
self._py, self._tag_strings, self._dep_strings) self._py, self._tag_strings, self._dep_strings)
ptr += 1 ptr += 1
@ -362,7 +361,7 @@ cdef class Token:
if (ptr.head < 0) and ((ptr + ptr.head) > self.c): if (ptr.head < 0) and ((ptr + ptr.head) > self.c):
ptr += ptr.head ptr += ptr.head
elif ptr + ptr.head == self.c: elif ptr + ptr.head == self.c:
yield Token.cinit(self.mem, self.vocab, self._string, yield Token.cinit(self.vocab, self._string,
ptr, ptr - (self.c - self.i), self.array_len, ptr, ptr - (self.c - self.i), self.array_len,
self._py, self._tag_strings, self._dep_strings) self._py, self._tag_strings, self._dep_strings)
ptr -= 1 ptr -= 1
@ -372,7 +371,7 @@ cdef class Token:
@property @property
def head(self): def head(self):
"""The token predicted by the parser to be the head of the current token.""" """The token predicted by the parser to be the head of the current token."""
return Token.cinit(self.mem, self.vocab, self._string, return Token.cinit(self.vocab, self._string,
self.c + self.c.head, self.i + self.c.head, self.array_len, self.c + self.c.head, self.i + self.c.head, self.array_len,
self._py, self._tag_strings, self._dep_strings) self._py, self._tag_strings, self._dep_strings)