mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-25 17:36:30 +03:00
* Have tokens track tuples that record the start offset and pos tag as well as a lexeme pointer
This commit is contained in:
Matthew Honnibal
parent
65d3ead4fd
commit
2805068ca8
|
|
@ -2,19 +2,28 @@ from spacy.lexeme cimport LexemeC
|
|||
from libcpp.vector cimport vector
|
||||
|
||||
|
||||
cdef struct Token:
|
||||
int i
|
||||
int pos
|
||||
LexemeC* lex
|
||||
|
||||
|
||||
cdef class Tokens:
|
||||
cdef vector[LexemeC*] *v
|
||||
cdef vector[Token] v
|
||||
|
||||
cdef int extend(self, int i, LexemeC** lexemes, int n) except -1
|
||||
cdef int push_back(self, int i, LexemeC* lexeme) except -1
|
||||
|
||||
cpdef int id(self, size_t i) except -1
|
||||
cpdef unicode string(self, size_t i)
|
||||
cpdef float prob(self, size_t i) except 1
|
||||
cpdef int cluster(self, size_t i) except *
|
||||
cpdef bint check_orth_flag(self, size_t i, size_t flag_id) except *
|
||||
cpdef bint check_dist_flag(self, size_t i, size_t flag_id) except *
|
||||
cpdef unicode string_view(self, size_t i, size_t view_id)
|
||||
|
||||
cpdef unicode casefix(self, size_t i)
|
||||
cpdef unicode shape(self, size_t i)
|
||||
cpdef unicode orig(self, size_t i)
|
||||
cpdef unicode norm(self, size_t i)
|
||||
cpdef unicode shape(self, size_t i)
|
||||
cpdef unicode unsparse(self, size_t i)
|
||||
cpdef unicode asciied(self, size_t i)
|
||||
cpdef bint is_alpha(self, size_t i) except *
|
||||
|
|
|
|||
106
spacy/tokens.pyx
106
spacy/tokens.pyx
|
|
@ -25,127 +25,141 @@ cdef class Tokens:
|
|||
"""
|
||||
def __cinit__(self, string_length=0):
|
||||
size = int(string_length / 3) if string_length >= 3 else 1
|
||||
self.v = new vector[LexemeC*]()
|
||||
self.v = vector[Token]()
|
||||
self.v.reserve(size)
|
||||
|
||||
def __getitem__(self, i):
|
||||
return Lexeme(<size_t>self.v.at(i))
|
||||
return Lexeme(<size_t>self.v.at(i).lex)
|
||||
|
||||
def __len__(self):
|
||||
return self.v.size()
|
||||
|
||||
def __dealloc__(self):
|
||||
del self.v
|
||||
cdef int push_back(self, int idx, LexemeC* lexeme) except -1:
|
||||
self.v.push_back(Token(idx, 0, lexeme))
|
||||
return idx + lexeme.ints[<int>LexInt_length]
|
||||
|
||||
def append(self, Lexeme lexeme):
|
||||
self.v.push_back(lexeme._c)
|
||||
cdef int extend(self, int idx, LexemeC** lexemes, int n) except -1:
|
||||
cdef int i
|
||||
if lexemes == NULL:
|
||||
return idx
|
||||
elif n == 0:
|
||||
i = 0
|
||||
while lexemes[i] != NULL:
|
||||
idx = self.push_back(idx, lexemes[i])
|
||||
else:
|
||||
for i in range(n):
|
||||
idx = self.push_back(idx, lexemes[i])
|
||||
return idx
|
||||
|
||||
cpdef int id(self, size_t i) except -1:
|
||||
return self.v.at(i).ints[<int>LexInt_i]
|
||||
|
||||
cpdef unicode string(self, size_t i):
|
||||
cdef bytes utf8_string = self.v.at(i).strings[<int>LexStr_orig]
|
||||
cdef unicode string = utf8_string.decode('utf8')
|
||||
return string
|
||||
return self.v.at(i).lex.ints[<int>LexInt_id]
|
||||
|
||||
cpdef float prob(self, size_t i) except 1:
|
||||
return self.v.at(i).floats[<int>LexFloat_prob]
|
||||
return self.v.at(i).lex.floats[<int>LexFloat_prob]
|
||||
|
||||
cpdef int cluster(self, size_t i) except *:
|
||||
return self.v.at(i).ints[<int>LexInt_cluster]
|
||||
return self.v.at(i).lex.ints[<int>LexInt_cluster]
|
||||
|
||||
cpdef bint check_orth_flag(self, size_t i, size_t flag_id) except *:
|
||||
return lexeme_check_orth_flag(self.v.at(i), flag_id)
|
||||
return lexeme_check_orth_flag(self.v.at(i).lex, flag_id)
|
||||
|
||||
cpdef bint check_dist_flag(self, size_t i, size_t flag_id) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), flag_id)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, flag_id)
|
||||
|
||||
cpdef unicode string_view(self, size_t i, size_t view_id):
|
||||
return lexeme_get_string(self.v.at(i), view_id)
|
||||
return lexeme_get_string(self.v.at(i).lex, view_id)
|
||||
|
||||
# Provide accessor methods for the features supported by the language.
|
||||
# Without these, clients have to use the underlying string_view and check_flag
|
||||
# methods, which requires them to know the IDs.
|
||||
cpdef unicode casefix(self, size_t i):
|
||||
return lexeme_get_string(self.v.at(i), LexStr_casefix)
|
||||
|
||||
cpdef unicode orig(self, size_t i):
|
||||
cdef bytes utf8_string = self.v.at(i).lex.strings[<int>LexStr_orig]
|
||||
cdef unicode string = utf8_string.decode('utf8')
|
||||
return string
|
||||
|
||||
cpdef unicode norm(self, size_t i):
|
||||
cdef bytes utf8_string = self.v.at(i).lex.strings[<int>LexStr_norm]
|
||||
cdef unicode string = utf8_string.decode('utf8')
|
||||
return string
|
||||
|
||||
cpdef unicode shape(self, size_t i):
|
||||
return lexeme_get_string(self.v.at(i), LexStr_shape)
|
||||
return lexeme_get_string(self.v.at(i).lex, LexStr_shape)
|
||||
|
||||
cpdef unicode unsparse(self, size_t i):
|
||||
return lexeme_get_string(self.v.at(i), LexStr_unsparse)
|
||||
return lexeme_get_string(self.v.at(i).lex, LexStr_unsparse)
|
||||
|
||||
cpdef unicode asciied(self, size_t i):
|
||||
return lexeme_get_string(self.v.at(i), LexStr_asciied)
|
||||
return lexeme_get_string(self.v.at(i).lex, LexStr_asciied)
|
||||
|
||||
cpdef bint is_alpha(self, size_t i) except *:
|
||||
return lexeme_check_orth_flag(self.v.at(i), LexOrth_alpha)
|
||||
return lexeme_check_orth_flag(self.v.at(i).lex, LexOrth_alpha)
|
||||
|
||||
cpdef bint is_ascii(self, size_t i) except *:
|
||||
return lexeme_check_orth_flag(self.v.at(i), LexOrth_ascii)
|
||||
return lexeme_check_orth_flag(self.v.at(i).lex, LexOrth_ascii)
|
||||
|
||||
cpdef bint is_digit(self, size_t i) except *:
|
||||
return lexeme_check_orth_flag(self.v.at(i), LexOrth_digit)
|
||||
return lexeme_check_orth_flag(self.v.at(i).lex, LexOrth_digit)
|
||||
|
||||
cpdef bint is_lower(self, size_t i) except *:
|
||||
return lexeme_check_orth_flag(self.v.at(i), LexOrth_lower)
|
||||
return lexeme_check_orth_flag(self.v.at(i).lex, LexOrth_lower)
|
||||
|
||||
cpdef bint is_punct(self, size_t i) except *:
|
||||
return lexeme_check_orth_flag(self.v.at(i), LexOrth_punct)
|
||||
return lexeme_check_orth_flag(self.v.at(i).lex, LexOrth_punct)
|
||||
|
||||
cpdef bint is_space(self, size_t i) except *:
|
||||
return lexeme_check_orth_flag(self.v.at(i), LexOrth_space)
|
||||
return lexeme_check_orth_flag(self.v.at(i).lex, LexOrth_space)
|
||||
|
||||
cpdef bint is_title(self, size_t i) except *:
|
||||
return lexeme_check_orth_flag(self.v.at(i), LexOrth_title)
|
||||
return lexeme_check_orth_flag(self.v.at(i).lex, LexOrth_title)
|
||||
|
||||
cpdef bint is_upper(self, size_t i) except *:
|
||||
return lexeme_check_orth_flag(self.v.at(i), LexOrth_upper)
|
||||
return lexeme_check_orth_flag(self.v.at(i).lex, LexOrth_upper)
|
||||
|
||||
cpdef bint can_adj(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_adj)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_adj)
|
||||
|
||||
cpdef bint can_adp(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_adp)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_adp)
|
||||
|
||||
cpdef bint can_adv(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_adv)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_adv)
|
||||
|
||||
cpdef bint can_conj(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_conj)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_conj)
|
||||
|
||||
cpdef bint can_det(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_det)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_det)
|
||||
|
||||
cpdef bint can_noun(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_noun)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_noun)
|
||||
|
||||
cpdef bint can_num(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_num)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_num)
|
||||
|
||||
cpdef bint can_pdt(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_pdt)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_pdt)
|
||||
|
||||
cpdef bint can_pos(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_pos)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_pos)
|
||||
|
||||
cpdef bint can_pron(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_pron)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_pron)
|
||||
|
||||
cpdef bint can_prt(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_prt)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_prt)
|
||||
|
||||
cpdef bint can_punct(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_punct)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_punct)
|
||||
|
||||
cpdef bint can_verb(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_verb)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_verb)
|
||||
|
||||
cpdef bint oft_lower(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_lower)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_lower)
|
||||
|
||||
cpdef bint oft_title(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_title)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_title)
|
||||
|
||||
cpdef bint oft_upper(self, size_t i) except *:
|
||||
return lexeme_check_dist_flag(self.v.at(i), LexDist_upper)
|
||||
return lexeme_check_dist_flag(self.v.at(i).lex, LexDist_upper)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user