diff --git a/spacy/lang.pxd b/spacy/lang.pxd index 9e4bc7b5d..d4b587a6b 100644 --- a/spacy/lang.pxd +++ b/spacy/lang.pxd @@ -18,7 +18,7 @@ cdef class Lexicon: cpdef readonly StringStore strings cdef vector[Lexeme*] lexemes - cdef Lexeme* get(self, UniStr* s) except NULL + cdef const Lexeme* get(self, UniStr* s) except NULL cdef PreshMap _map @@ -45,5 +45,5 @@ cdef class Language: cdef int _find_prefix(self, Py_UNICODE* characters, size_t length) except -1 cdef int _find_suffix(self, Py_UNICODE* characters, size_t length) except -1 cdef int _find_infix(self, Py_UNICODE* characters, size_t length) except -1 - cdef int _save_cached(self, Lexeme** tokens, hash_t key, int n) except -1 + cdef int _save_cached(self, const Lexeme* const* tokens, hash_t key, int n) except -1 diff --git a/spacy/lang.pyx b/spacy/lang.pyx index 2a284b9df..8d4ea7802 100644 --- a/spacy/lang.pyx +++ b/spacy/lang.pyx @@ -37,11 +37,12 @@ cdef class Language: self._suffix_re = re.compile(suffix) self._infix_re = re.compile(infix) self.lexicon = Lexicon(self.set_flags) - if path.exists(path.join(util.DATA_DIR, name, 'lexemes')): - self.lexicon.load(path.join(util.DATA_DIR, name, 'lexemes')) - self.lexicon.strings.load(path.join(util.DATA_DIR, name, 'strings')) self._load_special_tokenization(rules) + def load(self): + self.lexicon.load(path.join(util.DATA_DIR, self.name, 'lexemes')) + self.lexicon.strings.load(path.join(util.DATA_DIR, self.name, 'strings')) + cpdef Tokens tokens_from_list(self, list strings): cdef int length = sum([len(s) for s in strings]) cdef Tokens tokens = Tokens(self.lexicon.strings, length) @@ -84,7 +85,7 @@ cdef class Language: if Py_UNICODE_ISSPACE(chars[i]) != in_ws: if start < i: slice_unicode(&span, chars, start, i) - lexemes = self._cache.get(span.key) + lexemes = self._cache.get(span.key) if lexemes != NULL: tokens.extend(start, lexemes, 0) else: @@ -96,7 +97,7 @@ cdef class Language: i += 1 if start < i: slice_unicode(&span, chars, start, i) - lexemes = self._cache.get(span.key) + lexemes = self._cache.get(span.key) if lexemes != NULL: tokens.extend(start, lexemes, 0) else: @@ -114,8 +115,8 @@ cdef class Language: self._attach_tokens(tokens, start, span, &prefixes, &suffixes) self._save_cached(&tokens.lex[orig_size], orig_key, tokens.length - orig_size) - cdef UniStr* _split_affixes(self, UniStr* string, vector[Lexeme*] *prefixes, - vector[Lexeme*] *suffixes) except NULL: + cdef UniStr* _split_affixes(self, UniStr* string, vector[const Lexeme*] *prefixes, + vector[const Lexeme*] *suffixes) except NULL: cdef size_t i cdef UniStr prefix cdef UniStr suffix @@ -158,17 +159,17 @@ cdef class Language: cdef int _attach_tokens(self, Tokens tokens, int idx, UniStr* string, - vector[Lexeme*] *prefixes, - vector[Lexeme*] *suffixes) except -1: + vector[const Lexeme*] *prefixes, + vector[const Lexeme*] *suffixes) except -1: cdef int split - cdef Lexeme** lexemes + cdef const Lexeme* const* lexemes cdef Lexeme* lexeme cdef UniStr span if prefixes.size(): idx = tokens.extend(idx, prefixes.data(), prefixes.size()) if string.n != 0: - lexemes = self._cache.get(string.key) + lexemes = self._cache.get(string.key) if lexemes != NULL: idx = tokens.extend(idx, lexemes, 0) else: @@ -182,13 +183,13 @@ cdef class Language: idx = tokens.push_back(idx, self.lexicon.get(&span)) slice_unicode(&span, string.chars, split + 1, string.n) idx = tokens.push_back(idx, self.lexicon.get(&span)) - cdef vector[Lexeme*].reverse_iterator it = suffixes.rbegin() + cdef vector[const Lexeme*].reverse_iterator it = suffixes.rbegin() while it != suffixes.rend(): idx = tokens.push_back(idx, deref(it)) preinc(it) - cdef int _save_cached(self, Lexeme** tokens, hash_t key, int n) except -1: - lexemes = self.mem.alloc(n + 1, sizeof(Lexeme**)) + cdef int _save_cached(self, const Lexeme* const* tokens, hash_t key, int n) except -1: + lexemes = self.mem.alloc(n + 1, sizeof(Lexeme**)) cdef int i for i in range(n): lexemes[i] = tokens[i] @@ -249,7 +250,7 @@ cdef class Lexicon: self.size = 1 self.set_flags = set_flags - cdef Lexeme* get(self, UniStr* string) except NULL: + cdef const Lexeme* get(self, UniStr* string) except NULL: '''Retrieve a pointer to a Lexeme from the lexicon.''' cdef Lexeme* lex lex = self._map.get(string.key) @@ -289,14 +290,14 @@ cdef class Lexicon: return self.lexemes.at(id_or_string)[0] cdef UniStr string slice_unicode(&string, id_or_string, 0, len(id_or_string)) - cdef Lexeme* lexeme = self.get(&string) + cdef const Lexeme* lexeme = self.get(&string) return lexeme[0] def __setitem__(self, unicode uni_string, dict props): cdef UniStr s slice_unicode(&s, uni_string, 0, len(uni_string)) - cdef Lexeme* lex = self.get(&s) - lex[0] = lexeme_init(lex.id, s.chars[:s.n], s.key, self.strings, props) + cdef const Lexeme* lex = self.get(&s) + self.lexemes[lex.id][0] = lexeme_init(lex.id, s.chars[:s.n], s.key, self.strings, props) def dump(self, loc): if path.exists(loc): @@ -319,7 +320,8 @@ cdef class Lexicon: assert st == 0 def load(self, loc): - assert path.exists(loc) + if not path.exists(loc): + raise IOError('Lexemes file not found at %s' % loc) cdef bytes bytes_loc = loc.encode('utf8') if type(loc) == unicode else loc cdef FILE* fp = fopen(bytes_loc, 'rb') assert fp != NULL diff --git a/spacy/lexeme.pxd b/spacy/lexeme.pxd index 9d5dddd6d..a998aeedb 100644 --- a/spacy/lexeme.pxd +++ b/spacy/lexeme.pxd @@ -112,8 +112,8 @@ cpdef Lexeme init(id_t i, unicode string, hash_t hashed, StringStore store, dict props) except * -cdef inline bint check_flag(Lexeme* lexeme, attr_id_t flag_id) nogil: +cdef inline bint check_flag(const Lexeme* lexeme, attr_id_t flag_id) nogil: return lexeme.flags & (1 << flag_id) -cdef attr_t get_attr(Lexeme* lex, attr_id_t attr_id) +cdef attr_t get_attr(const Lexeme* lex, attr_id_t attr_id) diff --git a/spacy/lexeme.pyx b/spacy/lexeme.pyx index 888edc07b..2090ece50 100644 --- a/spacy/lexeme.pyx +++ b/spacy/lexeme.pyx @@ -35,7 +35,7 @@ cpdef Lexeme init(id_t i, unicode string, hash_t hashed, return lex -cdef attr_t get_attr(Lexeme* lex, attr_id_t feat_name): +cdef attr_t get_attr(const Lexeme* lex, attr_id_t feat_name): if feat_name < (sizeof(flags_t) * 8): return check_flag(lex, feat_name) elif feat_name == ID: diff --git a/spacy/tokens.pxd b/spacy/tokens.pxd index 2c97a3163..f91aa16ba 100644 --- a/spacy/tokens.pxd +++ b/spacy/tokens.pxd @@ -14,11 +14,11 @@ cdef class Tokens: cdef Pool mem cdef StringStore _string_store - cdef Lexeme** _lex_ptr + cdef const Lexeme** _lex_ptr cdef int* _idx_ptr cdef int* _pos_ptr cdef int* _ner_ptr - cdef Lexeme** lex + cdef const Lexeme** lex cdef int* idx cdef int* pos cdef int* ner @@ -26,8 +26,8 @@ cdef class Tokens: cdef int length cdef int max_length - cdef int extend(self, int i, Lexeme** lexemes, int n) except -1 - cdef int push_back(self, int i, Lexeme* lexeme) except -1 + cdef int extend(self, int i, const Lexeme* const* lexemes, int n) except -1 + cdef int push_back(self, int i, const Lexeme* lexeme) except -1 cpdef int set_tag(self, int i, int tag_type, int tag) except -1 cpdef np.ndarray[atom_t, ndim=2] get_array(self, list features) diff --git a/spacy/tokens.pyx b/spacy/tokens.pyx index e8e016944..7f79dcda9 100644 --- a/spacy/tokens.pyx +++ b/spacy/tokens.pyx @@ -44,7 +44,7 @@ cdef class Tokens: # Guarantee self.lex[i-x], for any i >= 0 and x < padding is in bounds # However, we need to remember the true starting places, so that we can # realloc. - self._lex_ptr = self.mem.alloc(size + (PADDING*2), sizeof(Lexeme*)) + self._lex_ptr = self.mem.alloc(size + (PADDING*2), sizeof(Lexeme*)) self._idx_ptr = self.mem.alloc(size + (PADDING*2), sizeof(int)) self._pos_ptr = self.mem.alloc(size + (PADDING*2), sizeof(int)) self._ner_ptr = self.mem.alloc(size + (PADDING*2), sizeof(int)) @@ -74,7 +74,7 @@ cdef class Tokens: def __len__(self): return self.length - cdef int push_back(self, int idx, Lexeme* lexeme) except -1: + cdef int push_back(self, int idx, const Lexeme* lexeme) except -1: if self.length == self.max_length: self._realloc(self.length * 2) self.lex[self.length] = lexeme @@ -84,7 +84,7 @@ cdef class Tokens: self.length += 1 return idx + lexeme.length - cdef int extend(self, int idx, Lexeme** lexemes, int n) except -1: + cdef int extend(self, int idx, const Lexeme* const* lexemes, int n) except -1: cdef int i if lexemes == NULL: return idx @@ -116,7 +116,7 @@ cdef class Tokens: def _realloc(self, new_size): self.max_length = new_size n = new_size + (PADDING * 2) - self._lex_ptr = self.mem.realloc(self._lex_ptr, n * sizeof(Lexeme*)) + self._lex_ptr = self.mem.realloc(self._lex_ptr, n * sizeof(Lexeme*)) self._idx_ptr = self.mem.realloc(self._idx_ptr, n * sizeof(int)) self._pos_ptr = self.mem.realloc(self._pos_ptr, n * sizeof(int)) self._ner_ptr = self.mem.realloc(self._ner_ptr, n * sizeof(int))