mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 19:39:13 +03:00
d99a9cbce9
space tokens are now always attached to the previous non-space token there are two exceptions: leading space tokens are attached to the first following non-space token in input that consists exclusively of space tokens, the last space token is the head of all others.
34 lines
938 B
Cython
34 lines
938 B
Cython
from libc.string cimport memcpy, memset
|
|
from libc.stdint cimport uint32_t
|
|
from ..vocab cimport EMPTY_LEXEME
|
|
from ..structs cimport Entity
|
|
from ..lexeme cimport Lexeme
|
|
from ..symbols cimport punct
|
|
from ..attrs cimport IS_SPACE
|
|
|
|
|
|
cdef class StateClass:
|
|
def __init__(self, int length):
|
|
cdef Pool mem = Pool()
|
|
self.mem = mem
|
|
|
|
def __dealloc__(self):
|
|
del self.c
|
|
|
|
@property
|
|
def stack(self):
|
|
return {self.S(i) for i in range(self.c._s_i)}
|
|
|
|
@property
|
|
def queue(self):
|
|
return {self.B(i) for i in range(self.c._b_i)}
|
|
|
|
def print_state(self, words):
|
|
words = list(words) + ['_']
|
|
top = words[self.S(0)] + '_%d' % self.S_(0).head
|
|
second = words[self.S(1)] + '_%d' % self.S_(1).head
|
|
third = words[self.S(2)] + '_%d' % self.S_(2).head
|
|
n0 = words[self.B(0)]
|
|
n1 = words[self.B(1)]
|
|
return ' '.join((third, second, top, '|', n0, n1))
|