explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-27 17:54:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Remove POS cache for now

Browse Source
This commit is contained in:
Matthew Honnibal 2014-12-20 03:49:32 +11:00
parent ff252dd535
commit b066102d2d
3 changed files with 2 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
spacy/en.pyx
View File

@ -139,20 +139,6 @@ cdef class English(Language):
name (unicode): The two letter code used by Wikipedia for the language. name (unicode): The two letter code used by Wikipedia for the language.
lexicon (Lexicon): The lexicon. Exposes the lookup method. lexicon (Lexicon): The lexicon. Exposes the lookup method.
""" """
def load_pos_cache(self, loc):
cdef int i = 0
cdef hash_t key
cdef int pos
with open(loc) as file_:
for line in file_:
pieces = line.split()
if i >= 500000:
break
i += 1
key = int(pieces[1])
pos = int(pieces[2])
self._pos_cache.set(key, <void*>pos)
def get_props(self, unicode string): def get_props(self, unicode string):
return {'flags': self.set_flags(string), 'dense': orth.word_shape(string)} return {'flags': self.set_flags(string), 'dense': orth.word_shape(string)}
@ -181,16 +167,8 @@ cdef class English(Language):
assert self.morphologizer is not None assert self.morphologizer is not None
cdef dict tagdict = self.pos_tagger.tagdict cdef dict tagdict = self.pos_tagger.tagdict
for i in range(tokens.length): for i in range(tokens.length):
if USE_POS_CACHE: fill_pos_context(context, i, t)
bigram[0] = tokens.data[i].lex.sic t[i].pos = self.pos_tagger.predict(context)
bigram[1] = tokens.data[i-1].lex.sic
cache_key = hash64(bigram, sizeof(id_t) * 2, 0)
cached = self._pos_cache.get(cache_key)
if cached != NULL:
t[i].pos = <int><size_t>cached
else:
fill_pos_context(context, i, t)
t[i].pos = self.pos_tagger.predict(context)
self.morphologizer.set_morph(i, t) self.morphologizer.set_morph(i, t)
def train_pos(self, Tokens tokens, golds): def train_pos(self, Tokens tokens, golds):

1
spacy/lang.pxd
View File

@ -46,7 +46,6 @@ cdef class Language:
cpdef readonly Morphologizer morphologizer cpdef readonly Morphologizer morphologizer
cpdef readonly GreedyParser parser cpdef readonly GreedyParser parser
cdef PreshMap _pos_cache
cdef object _prefix_re cdef object _prefix_re
cdef object _suffix_re cdef object _suffix_re
cdef object _infix_re cdef object _infix_re

1
spacy/lang.pyx
View File

@ -34,7 +34,6 @@ cdef class Language:
self.mem = Pool() self.mem = Pool()
self._cache = PreshMap(2 ** 25) self._cache = PreshMap(2 ** 25)
self._specials = PreshMap(2 ** 16) self._specials = PreshMap(2 ** 16)
self._pos_cache = PreshMap(2 ** 16)
rules, prefix, suffix, infix = util.read_lang_data(name) rules, prefix, suffix, infix = util.read_lang_data(name)
self._prefix_re = re.compile(prefix) self._prefix_re = re.compile(prefix)
self._suffix_re = re.compile(suffix) self._suffix_re = re.compile(suffix)