* Fix Issue #16: tokens recurse when printing

This commit is contained in:
Matthew Honnibal 2015-01-30 19:47:50 +11:00
parent cb95ef6934
commit 1a7a1c2771

View File

@ -144,9 +144,6 @@ cdef class Tokens:
cdef const TokenC* last = &self.data[self.length - 1] cdef const TokenC* last = &self.data[self.length - 1]
return self._string[:last.idx + last.lex.length] return self._string[:last.idx + last.lex.length]
def __str__(self):
return unidecode(unicode(self))
cdef int push_back(self, int idx, LexemeOrToken lex_or_tok) except -1: cdef int push_back(self, int idx, LexemeOrToken lex_or_tok) except -1:
if self.length == self.max_length: if self.length == self.max_length:
self._realloc(self.length * 2) self._realloc(self.length * 2)