* Fiddle with token features

This commit is contained in:
Matthew Honnibal 2014-09-12 15:49:55 +02:00
parent 5aa591106b
commit 7eab281194

View File

@ -80,6 +80,7 @@ cdef class Language:
Returns: Returns:
tokens (Tokens): A Tokens object, giving access to a sequence of LexIDs. tokens (Tokens): A Tokens object, giving access to a sequence of LexIDs.
""" """
print repr(string)
cdef size_t length = len(string) cdef size_t length = len(string)
cdef Tokens tokens = self.tokens_class(length) cdef Tokens tokens = self.tokens_class(length)
if length == 0: if length == 0: