From 2fe98b8a9a75ff3613b71f850d4e0772674df82e Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 26 Jun 2015 13:51:39 +0200 Subject: [PATCH] * Prepare for new models to be plugged in by using Example class --- setup.py | 3 +- spacy/_ml.pxd | 5 ++++ spacy/_ml.pyx | 18 +++++++----- spacy/_theano.pyx | 64 ++++++++++++++++++++--------------------- spacy/syntax/parser.pyx | 12 ++++---- 5 files changed, 56 insertions(+), 46 deletions(-) diff --git a/setup.py b/setup.py index 24b88dea9..a86e0f98d 100644 --- a/setup.py +++ b/setup.py @@ -151,7 +151,8 @@ MOD_NAMES = ['spacy.parts_of_speech', 'spacy.strings', 'spacy.lexeme', 'spacy.vocab', 'spacy.tokens', 'spacy.spans', 'spacy.morphology', 'spacy.syntax.stateclass', - 'spacy._ml', 'spacy.tokenizer', 'spacy.en.attrs', + 'spacy._ml', 'spacy._theano', + 'spacy.tokenizer', 'spacy.en.attrs', 'spacy.en.pos', 'spacy.syntax.parser', 'spacy.syntax.transition_system', 'spacy.syntax.arc_eager', diff --git a/spacy/_ml.pxd b/spacy/_ml.pxd index add162e69..3562b4a32 100644 --- a/spacy/_ml.pxd +++ b/spacy/_ml.pxd @@ -14,9 +14,14 @@ from .tokens cimport Tokens cdef int arg_max(const weight_t* scores, const int n_classes) nogil +cdef int arg_max_if_true(const weight_t* scores, const int* is_valid, int n_classes) nogil + +cdef int arg_max_if_zero(const weight_t* scores, const int* costs, int n_classes) nogil + cdef class Model: cdef int n_classes + cdef int n_feats cdef const weight_t* score(self, atom_t* context) except NULL cdef int set_scores(self, weight_t* scores, atom_t* context) except -1 diff --git a/spacy/_ml.pyx b/spacy/_ml.pyx index df66a1791..993d1a8ac 100644 --- a/spacy/_ml.pyx +++ b/spacy/_ml.pyx @@ -24,7 +24,7 @@ cdef int arg_max(const weight_t* scores, const int n_classes) nogil: return best -cdef int arg_max_if_true(const weight_t* scores, const bint* is_valid, +cdef int arg_max_if_true(const weight_t* scores, const int* is_valid, const int n_classes) nogil: cdef int i cdef int best = 0 @@ -54,21 +54,25 @@ cdef class Model: model_loc = path.join(model_loc, 'model') self.n_classes = n_classes self._extractor = Extractor(templates) + self.n_feats = self._extractor.n_templ self._model = LinearModel(n_classes, self._extractor.n_templ) self.model_loc = model_loc if self.model_loc and path.exists(self.model_loc): self._model.load(self.model_loc, freq_thresh=0) def predict(self, Example eg): - self.set_scores(eg.scores, eg.atoms) - eg.guess = arg_max_if_true(eg.scores, eg.is_valid, self.n_classes) + self.set_scores(eg.scores.data, eg.atoms.data) + eg.guess = arg_max_if_true(eg.scores.data, eg.is_valid.data, + self.n_classes) def train(self, Example eg): - self.set_scores(eg.scores, eg.atoms) - eg.guess = arg_max_if_true(eg.scores, eg.is_valid, self.n_classes) - eg.best = arg_max_if_zero(eg.scores, eg.costs, self.n_classes) + self.set_scores(eg.scores.data, eg.atoms.data) + eg.guess = arg_max_if_true(eg.scores.data, + eg.is_valid.data, self.n_classes) + eg.best = arg_max_if_zero(eg.scores.data, eg.costs.data, + self.n_classes) eg.cost = eg.costs[eg.guess] - self.update(eg.atoms, eg.guess, eg.best, eg.cost) + self.update(eg.atoms.data, eg.guess, eg.best, eg.cost) cdef const weight_t* score(self, atom_t* context) except NULL: cdef int n_feats diff --git a/spacy/_theano.pyx b/spacy/_theano.pyx index 1a1224596..702208d18 100644 --- a/spacy/_theano.pyx +++ b/spacy/_theano.pyx @@ -1,44 +1,44 @@ -from thinc.example cimport Example +from thinc.api cimport Example +from thinc.typedefs cimport weight_t + +from ._ml cimport arg_max_if_true +from ._ml cimport arg_max_if_zero + +import numpy +from os import path cdef class TheanoModel(Model): - def __init__(self, n_classes, input_layer, train_func, predict_func, model_loc=None): + def __init__(self, n_classes, input_spec, train_func, predict_func, model_loc=None): if model_loc is not None and path.isdir(model_loc): model_loc = path.join(model_loc, 'model') - self.n_classes = n_classes - - tables = [] - lengths = [] - for window_size, n_dims, vocab_size in input_structure: - tables.append(EmbeddingTable(n_dims, vocab_size, initializer)) - lengths.append(window_size) - - self.input_layer = InputLayer(lengths, tables) + self.eta = 0.001 + self.mu = 0.9 + self.t = 1 + initializer = lambda: 0.2 * numpy.random.uniform(-1.0, 1.0) + self.input_layer = InputLayer(input_spec, initializer) self.train_func = train_func self.predict_func = predict_func + self.n_classes = n_classes + self.n_feats = len(self.input_layer) self.model_loc = model_loc - if self.model_loc and path.exists(self.model_loc): - self._model.load(self.model_loc, freq_thresh=0) - - def train(self, Instance eg): - pass - - def predict(self, Instance eg): - - cdef const weight_t* score(self, atom_t* context) except NULL: - self.set_scores(self._scores, context) - return self._scores - - cdef int set_scores(self, weight_t* scores, atom_t* context) except -1: - # TODO f(context) --> Values - self._input_layer.fill(self._x, self._values, use_avg=False) - theano_scores = self._predict(self._x) + + def predict(self, Example eg): + self.input_layer.fill(eg.embeddings, eg.atoms) + theano_scores = self.predict_func(eg.embeddings) + cdef int i for i in range(self.n_classes): - output[i] = theano_scores[i] + eg.scores[i] = theano_scores[i] + eg.guess = arg_max_if_true(eg.scores.data, eg.is_valid.data, + self.n_classes) - cdef int update(self, atom_t* context, class_t guess, class_t gold, int cost) except -1: - # TODO f(context) --> Values - self._input_layer.fill(self._x, self._values, use_avg=False) - + def train(self, Example eg): + self.predict(eg) + update, t, eta, mu = self.train_func(eg.embeddings, eg.scores, eg.costs) + self.input_layer.update(eg.atoms, update, self.t, self.eta, self.mu) + eg.best = arg_max_if_zero(eg.scores.data, eg.costs.data, + self.n_classes) + eg.cost = eg.costs[eg.guess] + self.t += 1 diff --git a/spacy/syntax/parser.pyx b/spacy/syntax/parser.pyx index 4bfb0eeb1..33ae5b497 100644 --- a/spacy/syntax/parser.pyx +++ b/spacy/syntax/parser.pyx @@ -69,11 +69,11 @@ cdef class Parser: cdef StateClass stcls = StateClass.init(tokens.data, tokens.length) self.moves.initialize_state(stcls) - cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE) + cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE, self.model.n_feats) while not stcls.is_final(): eg.wipe() - fill_context(eg.atoms, stcls) - self.moves.set_valid(eg.is_valid, stcls) + fill_context(eg.atoms.data, stcls) + self.moves.set_valid(eg.is_valid.data, stcls) self.model.predict(eg) @@ -85,12 +85,12 @@ cdef class Parser: self.moves.preprocess_gold(gold) cdef StateClass stcls = StateClass.init(tokens.data, tokens.length) self.moves.initialize_state(stcls) - cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE) + cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE, self.model.n_feats) cdef int cost = 0 while not stcls.is_final(): eg.wipe() - fill_context(eg.atoms, stcls) - self.moves.set_costs(eg.is_valid, eg.costs, stcls, gold) + fill_context(eg.atoms.data, stcls) + self.moves.set_costs(eg.is_valid.data, eg.costs.data, stcls, gold) self.model.train(eg)