mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 19:39:13 +03:00
26 lines
793 B
Cython
26 lines
793 B
Cython
from ..vocab cimport Vocab
|
|
from ..structs cimport TokenC
|
|
from ..attrs cimport attr_id_t
|
|
from .doc cimport Doc
|
|
|
|
|
|
cdef class Token:
|
|
cdef Vocab vocab
|
|
cdef const TokenC* c
|
|
cdef readonly int i
|
|
cdef int array_len
|
|
cdef readonly Doc doc
|
|
|
|
@staticmethod
|
|
cdef inline Token cinit(Vocab vocab, const TokenC* token, int offset, Doc doc):
|
|
if offset < 0 or offset >= doc.length:
|
|
msg = "Attempt to access token at %d, max length %d"
|
|
raise IndexError(msg % (offset, doc.length))
|
|
if doc._py_tokens[offset] != None:
|
|
return doc._py_tokens[offset]
|
|
cdef Token self = Token.__new__(Token, vocab, doc, offset)
|
|
doc._py_tokens[offset] = self
|
|
return self
|
|
|
|
cpdef bint check_flag(self, attr_id_t flag_id) except -1
|