explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-11 04:08:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Introduce first attempt at const-ness

Browse Source
This commit is contained in:
Matthew Honnibal 2014-12-03 15:44:25 +11:00
parent d0d812c548
commit d70d31aa45
6 changed files with 34 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy/lang.pxd
View File

@ -18,7 +18,7 @@ cdef class Lexicon:
cpdef readonly StringStore strings cpdef readonly StringStore strings
cdef vector[Lexeme*] lexemes cdef vector[Lexeme*] lexemes
cdef Lexeme* get(self, UniStr* s) except NULL cdef const Lexeme* get(self, UniStr* s) except NULL
cdef PreshMap _map cdef PreshMap _map
@ -45,5 +45,5 @@ cdef class Language:
cdef int _find_prefix(self, Py_UNICODE* characters, size_t length) except -1 cdef int _find_prefix(self, Py_UNICODE* characters, size_t length) except -1
cdef int _find_suffix(self, Py_UNICODE* characters, size_t length) except -1 cdef int _find_suffix(self, Py_UNICODE* characters, size_t length) except -1
cdef int _find_infix(self, Py_UNICODE* characters, size_t length) except -1 cdef int _find_infix(self, Py_UNICODE* characters, size_t length) except -1
cdef int _save_cached(self, Lexeme** tokens, hash_t key, int n) except -1 cdef int _save_cached(self, const Lexeme* const* tokens, hash_t key, int n) except -1

40
spacy/lang.pyx
View File

@ -37,11 +37,12 @@ cdef class Language:
self._suffix_re = re.compile(suffix) self._suffix_re = re.compile(suffix)
self._infix_re = re.compile(infix) self._infix_re = re.compile(infix)
self.lexicon = Lexicon(self.set_flags) self.lexicon = Lexicon(self.set_flags)
if path.exists(path.join(util.DATA_DIR, name, 'lexemes')):
self.lexicon.load(path.join(util.DATA_DIR, name, 'lexemes'))
self.lexicon.strings.load(path.join(util.DATA_DIR, name, 'strings'))
self._load_special_tokenization(rules) self._load_special_tokenization(rules)
def load(self):
self.lexicon.load(path.join(util.DATA_DIR, self.name, 'lexemes'))
self.lexicon.strings.load(path.join(util.DATA_DIR, self.name, 'strings'))
cpdef Tokens tokens_from_list(self, list strings): cpdef Tokens tokens_from_list(self, list strings):
cdef int length = sum([len(s) for s in strings]) cdef int length = sum([len(s) for s in strings])
cdef Tokens tokens = Tokens(self.lexicon.strings, length) cdef Tokens tokens = Tokens(self.lexicon.strings, length)
@ -84,7 +85,7 @@ cdef class Language:
if Py_UNICODE_ISSPACE(chars[i]) != in_ws: if Py_UNICODE_ISSPACE(chars[i]) != in_ws:
if start < i: if start < i:
slice_unicode(&span, chars, start, i) slice_unicode(&span, chars, start, i)
lexemes = <Lexeme**>self._cache.get(span.key) lexemes = <const Lexeme* const*>self._cache.get(span.key)
if lexemes != NULL: if lexemes != NULL:
tokens.extend(start, lexemes, 0) tokens.extend(start, lexemes, 0)
else: else:
@ -96,7 +97,7 @@ cdef class Language:
i += 1 i += 1
if start < i: if start < i:
slice_unicode(&span, chars, start, i) slice_unicode(&span, chars, start, i)
lexemes = <Lexeme**>self._cache.get(span.key) lexemes = <const Lexeme* const*>self._cache.get(span.key)
if lexemes != NULL: if lexemes != NULL:
tokens.extend(start, lexemes, 0) tokens.extend(start, lexemes, 0)
else: else:
@ -114,8 +115,8 @@ cdef class Language:
self._attach_tokens(tokens, start, span, &prefixes, &suffixes) self._attach_tokens(tokens, start, span, &prefixes, &suffixes)
self._save_cached(&tokens.lex[orig_size], orig_key, tokens.length - orig_size) self._save_cached(&tokens.lex[orig_size], orig_key, tokens.length - orig_size)
cdef UniStr* _split_affixes(self, UniStr* string, vector[Lexeme*] *prefixes, cdef UniStr* _split_affixes(self, UniStr* string, vector[const Lexeme*] *prefixes,
vector[Lexeme*] *suffixes) except NULL: vector[const Lexeme*] *suffixes) except NULL:
cdef size_t i cdef size_t i
cdef UniStr prefix cdef UniStr prefix
cdef UniStr suffix cdef UniStr suffix
@ -158,17 +159,17 @@ cdef class Language:
cdef int _attach_tokens(self, Tokens tokens, cdef int _attach_tokens(self, Tokens tokens,
int idx, UniStr* string, int idx, UniStr* string,
vector[Lexeme*] *prefixes, vector[const Lexeme*] *prefixes,
vector[Lexeme*] *suffixes) except -1: vector[const Lexeme*] *suffixes) except -1:
cdef int split cdef int split
cdef Lexeme** lexemes cdef const Lexeme* const* lexemes
cdef Lexeme* lexeme cdef Lexeme* lexeme
cdef UniStr span cdef UniStr span
if prefixes.size(): if prefixes.size():
idx = tokens.extend(idx, prefixes.data(), prefixes.size()) idx = tokens.extend(idx, prefixes.data(), prefixes.size())
if string.n != 0: if string.n != 0:
lexemes = <Lexeme**>self._cache.get(string.key) lexemes = <const Lexeme* const*>self._cache.get(string.key)
if lexemes != NULL: if lexemes != NULL:
idx = tokens.extend(idx, lexemes, 0) idx = tokens.extend(idx, lexemes, 0)
else: else:
@ -182,13 +183,13 @@ cdef class Language:
idx = tokens.push_back(idx, self.lexicon.get(&span)) idx = tokens.push_back(idx, self.lexicon.get(&span))
slice_unicode(&span, string.chars, split + 1, string.n) slice_unicode(&span, string.chars, split + 1, string.n)
idx = tokens.push_back(idx, self.lexicon.get(&span)) idx = tokens.push_back(idx, self.lexicon.get(&span))
cdef vector[Lexeme*].reverse_iterator it = suffixes.rbegin() cdef vector[const Lexeme*].reverse_iterator it = suffixes.rbegin()
while it != suffixes.rend(): while it != suffixes.rend():
idx = tokens.push_back(idx, deref(it)) idx = tokens.push_back(idx, deref(it))
preinc(it) preinc(it)
cdef int _save_cached(self, Lexeme** tokens, hash_t key, int n) except -1: cdef int _save_cached(self, const Lexeme* const* tokens, hash_t key, int n) except -1:
lexemes = <Lexeme**>self.mem.alloc(n + 1, sizeof(Lexeme**)) lexemes = <const Lexeme**>self.mem.alloc(n + 1, sizeof(Lexeme**))
cdef int i cdef int i
for i in range(n): for i in range(n):
lexemes[i] = tokens[i] lexemes[i] = tokens[i]
@ -249,7 +250,7 @@ cdef class Lexicon:
self.size = 1 self.size = 1
self.set_flags = set_flags self.set_flags = set_flags
cdef Lexeme* get(self, UniStr* string) except NULL: cdef const Lexeme* get(self, UniStr* string) except NULL:
'''Retrieve a pointer to a Lexeme from the lexicon.''' '''Retrieve a pointer to a Lexeme from the lexicon.'''
cdef Lexeme* lex cdef Lexeme* lex
lex = <Lexeme*>self._map.get(string.key) lex = <Lexeme*>self._map.get(string.key)
@ -289,14 +290,14 @@ cdef class Lexicon:
return self.lexemes.at(id_or_string)[0] return self.lexemes.at(id_or_string)[0]
cdef UniStr string cdef UniStr string
slice_unicode(&string, id_or_string, 0, len(id_or_string)) slice_unicode(&string, id_or_string, 0, len(id_or_string))
cdef Lexeme* lexeme = self.get(&string) cdef const Lexeme* lexeme = self.get(&string)
return lexeme[0] return lexeme[0]
def __setitem__(self, unicode uni_string, dict props): def __setitem__(self, unicode uni_string, dict props):
cdef UniStr s cdef UniStr s
slice_unicode(&s, uni_string, 0, len(uni_string)) slice_unicode(&s, uni_string, 0, len(uni_string))
cdef Lexeme* lex = self.get(&s) cdef const Lexeme* lex = self.get(&s)
lex[0] = lexeme_init(lex.id, s.chars[:s.n], s.key, self.strings, props) self.lexemes[lex.id][0] = lexeme_init(lex.id, s.chars[:s.n], s.key, self.strings, props)
def dump(self, loc): def dump(self, loc):
if path.exists(loc): if path.exists(loc):
@ -319,7 +320,8 @@ cdef class Lexicon:
assert st == 0 assert st == 0
def load(self, loc): def load(self, loc):
assert path.exists(loc) if not path.exists(loc):
raise IOError('Lexemes file not found at %s' % loc)
cdef bytes bytes_loc = loc.encode('utf8') if type(loc) == unicode else loc cdef bytes bytes_loc = loc.encode('utf8') if type(loc) == unicode else loc
cdef FILE* fp = fopen(<char*>bytes_loc, 'rb') cdef FILE* fp = fopen(<char*>bytes_loc, 'rb')
assert fp != NULL assert fp != NULL

4
spacy/lexeme.pxd
View File

@ -112,8 +112,8 @@ cpdef Lexeme init(id_t i, unicode string, hash_t hashed, StringStore store,
dict props) except * dict props) except *
cdef inline bint check_flag(Lexeme* lexeme, attr_id_t flag_id) nogil: cdef inline bint check_flag(const Lexeme* lexeme, attr_id_t flag_id) nogil:
return lexeme.flags & (1 << flag_id) return lexeme.flags & (1 << flag_id)
cdef attr_t get_attr(Lexeme* lex, attr_id_t attr_id) cdef attr_t get_attr(const Lexeme* lex, attr_id_t attr_id)

2
spacy/lexeme.pyx
View File

@ -35,7 +35,7 @@ cpdef Lexeme init(id_t i, unicode string, hash_t hashed,
return lex return lex
cdef attr_t get_attr(Lexeme* lex, attr_id_t feat_name): cdef attr_t get_attr(const Lexeme* lex, attr_id_t feat_name):
if feat_name < (sizeof(flags_t) * 8): if feat_name < (sizeof(flags_t) * 8):
return check_flag(lex, feat_name) return check_flag(lex, feat_name)
elif feat_name == ID: elif feat_name == ID:

8
spacy/tokens.pxd
View File

@ -14,11 +14,11 @@ cdef class Tokens:
cdef Pool mem cdef Pool mem
cdef StringStore _string_store cdef StringStore _string_store
cdef Lexeme** _lex_ptr cdef const Lexeme** _lex_ptr
cdef int* _idx_ptr cdef int* _idx_ptr
cdef int* _pos_ptr cdef int* _pos_ptr
cdef int* _ner_ptr cdef int* _ner_ptr
cdef Lexeme** lex cdef const Lexeme** lex
cdef int* idx cdef int* idx
cdef int* pos cdef int* pos
cdef int* ner cdef int* ner
@ -26,8 +26,8 @@ cdef class Tokens:
cdef int length cdef int length
cdef int max_length cdef int max_length
cdef int extend(self, int i, Lexeme** lexemes, int n) except -1 cdef int extend(self, int i, const Lexeme* const* lexemes, int n) except -1
cdef int push_back(self, int i, Lexeme* lexeme) except -1 cdef int push_back(self, int i, const Lexeme* lexeme) except -1
cpdef int set_tag(self, int i, int tag_type, int tag) except -1 cpdef int set_tag(self, int i, int tag_type, int tag) except -1
cpdef np.ndarray[atom_t, ndim=2] get_array(self, list features) cpdef np.ndarray[atom_t, ndim=2] get_array(self, list features)

8
spacy/tokens.pyx
View File

@ -44,7 +44,7 @@ cdef class Tokens:
# Guarantee self.lex[i-x], for any i >= 0 and x < padding is in bounds # Guarantee self.lex[i-x], for any i >= 0 and x < padding is in bounds
# However, we need to remember the true starting places, so that we can # However, we need to remember the true starting places, so that we can
# realloc. # realloc.
self._lex_ptr = <Lexeme**>self.mem.alloc(size + (PADDING*2), sizeof(Lexeme*)) self._lex_ptr = <const Lexeme**>self.mem.alloc(size + (PADDING*2), sizeof(Lexeme*))
self._idx_ptr = <int*>self.mem.alloc(size + (PADDING*2), sizeof(int)) self._idx_ptr = <int*>self.mem.alloc(size + (PADDING*2), sizeof(int))
self._pos_ptr = <int*>self.mem.alloc(size + (PADDING*2), sizeof(int)) self._pos_ptr = <int*>self.mem.alloc(size + (PADDING*2), sizeof(int))
self._ner_ptr = <int*>self.mem.alloc(size + (PADDING*2), sizeof(int)) self._ner_ptr = <int*>self.mem.alloc(size + (PADDING*2), sizeof(int))
@ -74,7 +74,7 @@ cdef class Tokens:
def __len__(self): def __len__(self):
return self.length return self.length
cdef int push_back(self, int idx, Lexeme* lexeme) except -1: cdef int push_back(self, int idx, const Lexeme* lexeme) except -1:
if self.length == self.max_length: if self.length == self.max_length:
self._realloc(self.length * 2) self._realloc(self.length * 2)
self.lex[self.length] = lexeme self.lex[self.length] = lexeme
@ -84,7 +84,7 @@ cdef class Tokens:
self.length += 1 self.length += 1
return idx + lexeme.length return idx + lexeme.length
cdef int extend(self, int idx, Lexeme** lexemes, int n) except -1: cdef int extend(self, int idx, const Lexeme* const* lexemes, int n) except -1:
cdef int i cdef int i
if lexemes == NULL: if lexemes == NULL:
return idx return idx
@ -116,7 +116,7 @@ cdef class Tokens:
def _realloc(self, new_size): def _realloc(self, new_size):
self.max_length = new_size self.max_length = new_size
n = new_size + (PADDING * 2) n = new_size + (PADDING * 2)
self._lex_ptr = <Lexeme**>self.mem.realloc(self._lex_ptr, n * sizeof(Lexeme*)) self._lex_ptr = <const Lexeme**>self.mem.realloc(self._lex_ptr, n * sizeof(Lexeme*))
self._idx_ptr = <int*>self.mem.realloc(self._idx_ptr, n * sizeof(int)) self._idx_ptr = <int*>self.mem.realloc(self._idx_ptr, n * sizeof(int))
self._pos_ptr = <int*>self.mem.realloc(self._pos_ptr, n * sizeof(int)) self._pos_ptr = <int*>self.mem.realloc(self._pos_ptr, n * sizeof(int))
self._ner_ptr = <int*>self.mem.realloc(self._ner_ptr, n * sizeof(int)) self._ner_ptr = <int*>self.mem.realloc(self._ner_ptr, n * sizeof(int))