explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-09-21 11:29:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Upd Tokens to use vector, with bounds checking.

Browse Source
This commit is contained in:
Matthew Honnibal 2014-09-15 03:22:40 +02:00
parent 0f6bf2a2ee
commit f77b7098c0
5 changed files with 125 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
spacy/lang.pyx
View File

@ -18,6 +18,8 @@ from .util import read_lang_data
from spacy.tokens import Tokens from spacy.tokens import Tokens
from spacy.lexeme cimport LexemeC, lexeme_init from spacy.lexeme cimport LexemeC, lexeme_init
from murmurhash.mrmr cimport hash64 from murmurhash.mrmr cimport hash64
from cpython.ref cimport Py_INCREF
from spacy._hashing cimport PointerHash from spacy._hashing cimport PointerHash
from spacy import orth from spacy import orth
@ -194,11 +196,11 @@ cdef class Language:
if lexemes != NULL: if lexemes != NULL:
i = 0 i = 0
while lexemes[i] != NULL: while lexemes[i] != NULL:
tokens.push_back(lexemes[i]) tokens.v.push_back(lexemes[i])
i += 1 i += 1
return 0 return 0
cdef uint64_t key = string.key cdef uint64_t key = string.key
cdef size_t first_token = tokens.length cdef size_t first_token = len(tokens)
cdef int split cdef int split
cdef int remaining = string.n cdef int remaining = string.n
cdef String prefix cdef String prefix
@ -210,14 +212,14 @@ cdef class Language:
if lexemes != NULL: if lexemes != NULL:
i = 0 i = 0
while lexemes[i] != NULL: while lexemes[i] != NULL:
tokens.push_back(lexemes[i]) tokens.v.push_back(lexemes[i])
i += 1 i += 1
else: else:
tokens.push_back(<LexemeC*>self.lexicon.get(&prefix)) tokens.v.push_back(<LexemeC*>self.lexicon.get(&prefix))
lexemes = <LexemeC**>calloc(tokens.length - first_token, sizeof(LexemeC*)) lexemes = <LexemeC**>calloc(len(tokens) - first_token, sizeof(LexemeC*))
cdef size_t j cdef size_t j
for i, j in enumerate(range(first_token, tokens.length)): for i, j in enumerate(range(first_token, tokens.v.size())):
lexemes[i] = tokens.lexemes[j] lexemes[i] = tokens.v[j]
self.cache.set(key, lexemes) self.cache.set(key, lexemes)
cdef int _split_one(self, Py_UNICODE* characters, size_t length): cdef int _split_one(self, Py_UNICODE* characters, size_t length):
@ -307,10 +309,8 @@ cdef class Lexicon:
return Lexeme(<size_t>lexeme) return Lexeme(<size_t>lexeme)
_unicodes = set()
cdef void string_from_unicode(String* s, unicode uni): cdef void string_from_unicode(String* s, unicode uni):
global _unicodes Py_INCREF(uni)
_unicodes.add(uni)
cdef Py_UNICODE* c_uni = <Py_UNICODE*>uni cdef Py_UNICODE* c_uni = <Py_UNICODE*>uni
string_from_slice(s, c_uni, 0, len(uni)) string_from_slice(s, c_uni, 0, len(uni))

1
spacy/lexeme.pxd
View File

@ -9,7 +9,6 @@ cdef struct LexemeC:
char* string char* string
char** views char** views
flag_t flags flag_t flags

8
spacy/lexeme.pyx
View File

@ -1,4 +1,6 @@
from libc.stdlib cimport calloc, free from libc.stdlib cimport calloc, free
from cpython.ref cimport Py_INCREF
cdef LexemeC* lexeme_init(unicode string, double prob, size_t cluster, cdef LexemeC* lexeme_init(unicode string, double prob, size_t cluster,
list views, set flags): list views, set flags):
@ -21,13 +23,11 @@ cdef int lexeme_free(LexemeC* lexeme) except -1:
free(lexeme) free(lexeme)
cdef set _strings = set()
cdef char* intern_and_encode(unicode string, size_t* length): cdef char* intern_and_encode(unicode string, size_t* length):
global _strings global _strings
cdef bytes decoded = string.encode('utf8') cdef bytes utf8_string = intern(string.encode('utf8'))
cdef bytes utf8_string = intern(decoded) Py_INCREF(utf8_string)
length[0] = len(utf8_string) length[0] = len(utf8_string)
_strings.add(utf8_string)
return <char*>utf8_string return <char*>utf8_string

74
spacy/tokens.pxd
View File

@ -1,50 +1,46 @@
from spacy.lexeme cimport LexemeC from spacy.lexeme cimport LexemeC
from libcpp.vector cimport vector
cdef class Tokens: cdef class Tokens:
cdef size_t length cdef vector[LexemeC*] v
cdef size_t size
cdef LexemeC** lexemes cpdef size_t id(self, size_t i) except 0
cdef int push_back(self, LexemeC* lexeme) except 01
cpdef size_t id(self, size_t i)
cpdef unicode string(self, size_t i) cpdef unicode string(self, size_t i)
cpdef double prob(self, size_t i) cpdef double prob(self, size_t i) except 1
cpdef size_t cluster(self, size_t i) cpdef size_t cluster(self, size_t i) except *
cpdef bint check_flag(self, size_t i, size_t flag_id) cpdef bint check_flag(self, size_t i, size_t flag_id) except *
cpdef unicode string_view(self, size_t i, size_t view_id) cpdef unicode string_view(self, size_t i, size_t view_id)
cpdef size_t canon(self, size_t i) cpdef size_t canon(self, size_t i) except 0
cpdef size_t shape(self, size_t i) cpdef size_t shape(self, size_t i) except 0
cpdef size_t non_sparse(self, size_t i) cpdef size_t non_sparse(self, size_t i) except 0
cpdef size_t asciied(self, size_t i) cpdef size_t asciied(self, size_t i) except 0
cpdef unicode canon_string(self, size_t i) cpdef unicode canon_string(self, size_t i)
cpdef unicode shape_string(self, size_t i) cpdef unicode shape_string(self, size_t i)
cpdef unicode non_sparse_string(self, size_t i) cpdef unicode non_sparse_string(self, size_t i)
cpdef unicode asciied_string(self, size_t i) cpdef unicode asciied_string(self, size_t i)
cpdef bint is_alpha(self, size_t i) cpdef bint is_alpha(self, size_t i) except *
cpdef bint is_ascii(self, size_t i) cpdef bint is_ascii(self, size_t i) except *
cpdef bint is_digit(self, size_t i) cpdef bint is_digit(self, size_t i) except *
cpdef bint is_lower(self, size_t i) cpdef bint is_lower(self, size_t i) except *
cpdef bint is_punct(self, size_t i) cpdef bint is_punct(self, size_t i) except *
cpdef bint is_space(self, size_t i) cpdef bint is_space(self, size_t i) except *
cpdef bint is_title(self, size_t i) cpdef bint is_title(self, size_t i) except *
cpdef bint is_upper(self, size_t i) cpdef bint is_upper(self, size_t i) except *
cpdef bint can_adj(self, size_t i) cpdef bint can_adj(self, size_t i) except *
cpdef bint can_adp(self, size_t i) cpdef bint can_adp(self, size_t i) except *
cpdef bint can_adv(self, size_t i) cpdef bint can_adv(self, size_t i) except *
cpdef bint can_conj(self, size_t i) cpdef bint can_conj(self, size_t i) except *
cpdef bint can_det(self, size_t i) cpdef bint can_det(self, size_t i) except *
cpdef bint can_noun(self, size_t i) cpdef bint can_noun(self, size_t i) except *
cpdef bint can_num(self, size_t i) cpdef bint can_num(self, size_t i) except *
cpdef bint can_pdt(self, size_t i) cpdef bint can_pdt(self, size_t i) except *
cpdef bint can_pos(self, size_t i) cpdef bint can_pos(self, size_t i) except *
cpdef bint can_pron(self, size_t i) cpdef bint can_pron(self, size_t i) except *
cpdef bint can_prt(self, size_t i) cpdef bint can_prt(self, size_t i) except *
cpdef bint can_punct(self, size_t i) cpdef bint can_punct(self, size_t i) except *
cpdef bint can_verb(self, size_t i) cpdef bint can_verb(self, size_t i) except *
cpdef bint oft_lower(self, size_t i) cpdef bint oft_lower(self, size_t i) except *
cpdef bint oft_title(self, size_t i) cpdef bint oft_title(self, size_t i) except *
cpdef bint oft_upper(self, size_t i) cpdef bint oft_upper(self, size_t i) except *

164
spacy/tokens.pyx
View File

@ -65,148 +65,132 @@ cdef class Tokens:
""" """
def __cinit__(self, string_length=0): def __cinit__(self, string_length=0):
size = int(string_length / 3) if string_length >= 3 else 1 size = int(string_length / 3) if string_length >= 3 else 1
self.lexemes = <LexemeC**>calloc(size, sizeof(LexemeC*)) self.v.reserve(size)
self.size = size
self.length = 0
def __dealloc__(self):
free(self.lexemes)
def __getitem__(self, i): def __getitem__(self, i):
if i >= self.length: return Lexeme(<size_t>self.v.at(i))
raise IndexError
return Lexeme(<size_t>self.lexemes[i])
def __len__(self): def __len__(self):
return self.length return self.v.size()
def append(self, Lexeme lexeme): def append(self, Lexeme lexeme):
self.push_back(lexeme._c) self.v.push_back(lexeme._c)
cdef int push_back(self, LexemeC* lexeme) except -1:
if (self.length + 1) == self.size:
self.size *= 2
self.lexemes = <LexemeC**>realloc(self.lexemes, self.size * sizeof(LexemeC*))
self.lexemes[self.length] = lexeme
self.length += 1
cpdef unicode string(self, size_t i): cpdef unicode string(self, size_t i):
cdef bytes utf8_string = self.lexemes[i].string[:self.lexemes[i].length] cdef bytes utf8_string = self.v[i].string[:self.v[i].length]
cdef unicode string = utf8_string.decode('utf8') cdef unicode string = utf8_string.decode('utf8')
return string return string
cpdef size_t id(self, size_t i): cpdef size_t id(self, size_t i) except 0:
return id(self.lexemes[i].string) return <size_t>&self.v.at(i).string
cpdef double prob(self, size_t i): cpdef double prob(self, size_t i) except 1:
return self.lexemes[i].prob return self.v.at(i).prob
cpdef size_t cluster(self, size_t i): cpdef size_t cluster(self, size_t i) except *:
return self.lexemes[i].cluster return self.v.at(i).cluster
cpdef bint check_flag(self, size_t i, size_t flag_id): cpdef bint check_flag(self, size_t i, size_t flag_id) except *:
return lexeme_check_flag(self.lexemes[i], flag_id) return lexeme_check_flag(self.v[i], flag_id)
cpdef unicode string_view(self, size_t i, size_t view_id): cpdef unicode string_view(self, size_t i, size_t view_id):
return lexeme_string_view(self.lexemes[i], view_id) return lexeme_string_view(self.v.at(i), view_id)
# Provide accessor methods for the features supported by the language. # Provide accessor methods for the features supported by the language.
# Without these, clients have to use the underlying string_view and check_flag # Without these, clients have to use the underlying string_view and check_flag
# methods, which requires them to know the IDs. # methods, which requires them to know the IDs.
cpdef unicode canon_string(self, size_t i): cpdef unicode canon_string(self, size_t i):
return lexeme_string_view(self.lexemes[i], View_CanonForm) return lexeme_string_view(self.v.at(i), View_CanonForm)
cpdef unicode shape_string(self, size_t i): cpdef unicode shape_string(self, size_t i):
return lexeme_string_view(self.lexemes[i], View_WordShape) return lexeme_string_view(self.v.at(i), View_WordShape)
cpdef unicode non_sparse_string(self, size_t i): cpdef unicode non_sparse_string(self, size_t i):
return lexeme_string_view(self.lexemes[i], View_NonSparse) return lexeme_string_view(self.v.at(i), View_NonSparse)
cpdef unicode asciied_string(self, size_t i): cpdef unicode asciied_string(self, size_t i):
return lexeme_string_view(self.lexemes[i], View_Asciied) return lexeme_string_view(self.v.at(i), View_Asciied)
cpdef size_t canon(self, size_t i): cpdef size_t canon(self, size_t i) except *:
return id(self.lexemes[i].views[<size_t>View_CanonForm]) return id(self.v.at(i).views[<size_t>View_CanonForm])
cpdef size_t shape(self, size_t i): cpdef size_t shape(self, size_t i) except *:
return id(self.lexemes[i].views[<size_t>View_WordShape]) return id(self.v.at(i).views[<size_t>View_WordShape])
cpdef size_t non_sparse(self, size_t i): cpdef size_t non_sparse(self, size_t i) except *:
return id(self.lexemes[i].views[<size_t>View_NonSparse]) return id(self.v.at(i).views[<size_t>View_NonSparse])
cpdef size_t asciied(self, size_t i): cpdef size_t asciied(self, size_t i) except *:
return id(self.lexemes[i].views[<size_t>View_Asciied]) return id(self.v.at(i).views[<size_t>View_Asciied])
cpdef bint is_alpha(self, size_t i): cpdef bint is_alpha(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_IsAlpha) return lexeme_check_flag(self.v.at(i), Flag_IsAlpha)
cpdef bint is_ascii(self, size_t i): cpdef bint is_ascii(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_IsAscii) return lexeme_check_flag(self.v.at(i), Flag_IsAscii)
cpdef bint is_digit(self, size_t i): cpdef bint is_digit(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_IsDigit) return lexeme_check_flag(self.v.at(i), Flag_IsDigit)
cpdef bint is_lower(self, size_t i): cpdef bint is_lower(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_IsLower) return lexeme_check_flag(self.v.at(i), Flag_IsLower)
cpdef bint is_punct(self, size_t i): cpdef bint is_punct(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_IsPunct) return lexeme_check_flag(self.v.at(i), Flag_IsPunct)
cpdef bint is_space(self, size_t i): cpdef bint is_space(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_IsSpace) return lexeme_check_flag(self.v.at(i), Flag_IsSpace)
cpdef bint is_title(self, size_t i): cpdef bint is_title(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_IsTitle) return lexeme_check_flag(self.v.at(i), Flag_IsTitle)
cpdef bint is_upper(self, size_t i): cpdef bint is_upper(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_IsUpper) return lexeme_check_flag(self.v.at(i), Flag_IsUpper)
cpdef bint can_adj(self, size_t i): cpdef bint can_adj(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanAdj) return lexeme_check_flag(self.v.at(i), Flag_CanAdj)
cpdef bint can_adp(self, size_t i): cpdef bint can_adp(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanAdp) return lexeme_check_flag(self.v.at(i), Flag_CanAdp)
cpdef bint can_adv(self, size_t i): cpdef bint can_adv(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanAdv) return lexeme_check_flag(self.v.at(i), Flag_CanAdv)
cpdef bint can_conj(self, size_t i): cpdef bint can_conj(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanConj) return lexeme_check_flag(self.v.at(i), Flag_CanConj)
cpdef bint can_det(self, size_t i): cpdef bint can_det(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanDet) return lexeme_check_flag(self.v.at(i), Flag_CanDet)
cpdef bint can_noun(self, size_t i): cpdef bint can_noun(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanNoun) return lexeme_check_flag(self.v.at(i), Flag_CanNoun)
cpdef bint can_num(self, size_t i): cpdef bint can_num(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanNum) return lexeme_check_flag(self.v.at(i), Flag_CanNum)
cpdef bint can_pdt(self, size_t i): cpdef bint can_pdt(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanPdt) return lexeme_check_flag(self.v.at(i), Flag_CanPdt)
cpdef bint can_pos(self, size_t i): cpdef bint can_pos(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanPos) return lexeme_check_flag(self.v.at(i), Flag_CanPos)
cpdef bint can_pron(self, size_t i): cpdef bint can_pron(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanPron) return lexeme_check_flag(self.v.at(i), Flag_CanPron)
cpdef bint can_prt(self, size_t i): cpdef bint can_prt(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanPrt) return lexeme_check_flag(self.v.at(i), Flag_CanPrt)
cpdef bint can_punct(self, size_t i): cpdef bint can_punct(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanPunct) return lexeme_check_flag(self.v.at(i), Flag_CanPunct)
cpdef bint can_verb(self, size_t i): cpdef bint can_verb(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_CanVerb) return lexeme_check_flag(self.v.at(i), Flag_CanVerb)
cpdef bint oft_lower(self, size_t i): cpdef bint oft_lower(self, size_t i) except *:
return lexeme_check_flag(self.lexemes[i], Flag_OftLower) return lexeme_check_flag(self.v.at(i), Flag_OftLower)
cpdef bint oft_title(self, size_t i):
return lexeme_check_flag(self.lexemes[i], Flag_OftTitle)
cpdef bint oft_upper(self, size_t i):
return lexeme_check_flag(self.lexemes[i], Flag_OftUpper)
cpdef bint oft_title(self, size_t i) except *:
return lexeme_check_flag(self.v.at(i), Flag_OftTitle)
cpdef bint oft_upper(self, size_t i) except *:
return lexeme_check_flag(self.v.at(i), Flag_OftUpper)