* Prepare for new models to be plugged in by using Example class

setup.py

@@ -151,7 +151,8 @@ MOD_NAMES = ['spacy.parts_of_speech', 'spacy.strings',
              'spacy.lexeme', 'spacy.vocab', 'spacy.tokens', 'spacy.spans',
              'spacy.morphology', 
              'spacy.syntax.stateclass', 
-             'spacy._ml', 'spacy.tokenizer', 'spacy.en.attrs',
+             'spacy._ml', 'spacy._theano',
+             'spacy.tokenizer', 'spacy.en.attrs',
              'spacy.en.pos', 'spacy.syntax.parser', 
              'spacy.syntax.transition_system',
              'spacy.syntax.arc_eager',

spacy/_ml.pxd

@@ -14,9 +14,14 @@ from .tokens cimport Tokens
 
 cdef int arg_max(const weight_t* scores, const int n_classes) nogil
 
+cdef int arg_max_if_true(const weight_t* scores, const int* is_valid,  int n_classes) nogil
+
+cdef int arg_max_if_zero(const weight_t* scores, const int* costs, int n_classes) nogil
+
 
 cdef class Model:
     cdef int n_classes
+    cdef int n_feats
     
     cdef const weight_t* score(self, atom_t* context) except NULL
     cdef int set_scores(self, weight_t* scores, atom_t* context) except -1

spacy/_ml.pyx

@@ -24,7 +24,7 @@ cdef int arg_max(const weight_t* scores, const int n_classes) nogil:
     return best
 
 
-cdef int arg_max_if_true(const weight_t* scores, const bint* is_valid,
+cdef int arg_max_if_true(const weight_t* scores, const int* is_valid,
                          const int n_classes) nogil:
     cdef int i
     cdef int best = 0
@@ -54,21 +54,25 @@ cdef class Model:
             model_loc = path.join(model_loc, 'model')
         self.n_classes = n_classes
         self._extractor = Extractor(templates)
+        self.n_feats = self._extractor.n_templ
         self._model = LinearModel(n_classes, self._extractor.n_templ)
         self.model_loc = model_loc
         if self.model_loc and path.exists(self.model_loc):
             self._model.load(self.model_loc, freq_thresh=0)
 
     def predict(self, Example eg):
-        self.set_scores(eg.scores, eg.atoms)
+        self.set_scores(<weight_t*>eg.scores.data, <atom_t*>eg.atoms.data)
-        eg.guess = arg_max_if_true(eg.scores, eg.is_valid, self.n_classes)
+        eg.guess = arg_max_if_true(<weight_t*>eg.scores.data, <int*>eg.is_valid.data,
+                                   self.n_classes)
 
     def train(self, Example eg):
-        self.set_scores(eg.scores, eg.atoms)
+        self.set_scores(<weight_t*>eg.scores.data, <atom_t*>eg.atoms.data)
-        eg.guess = arg_max_if_true(eg.scores, eg.is_valid, self.n_classes)
+        eg.guess = arg_max_if_true(<weight_t*>eg.scores.data,
-        eg.best = arg_max_if_zero(eg.scores, eg.costs, self.n_classes)
+                                   <int*>eg.is_valid.data, self.n_classes)
+        eg.best = arg_max_if_zero(<weight_t*>eg.scores.data, <int*>eg.costs.data,
+                                  self.n_classes)
         eg.cost = eg.costs[eg.guess]
-        self.update(eg.atoms, eg.guess, eg.best, eg.cost)
+        self.update(<atom_t*>eg.atoms.data, eg.guess, eg.best, eg.cost)
 
     cdef const weight_t* score(self, atom_t* context) except NULL:
         cdef int n_feats

spacy/_theano.pyx

@@ -1,44 +1,44 @@
-from thinc.example cimport Example
+from thinc.api cimport Example
+from thinc.typedefs cimport weight_t
+
+from ._ml cimport arg_max_if_true
+from ._ml cimport arg_max_if_zero
+
+import numpy
+from os import path
 
 
 cdef class TheanoModel(Model):
-    def __init__(self, n_classes, input_layer, train_func, predict_func, model_loc=None):
+    def __init__(self, n_classes, input_spec, train_func, predict_func, model_loc=None):
         if model_loc is not None and path.isdir(model_loc):
             model_loc = path.join(model_loc, 'model')
-        self.n_classes = n_classes
-        
-        tables = []
-        lengths = []
-        for window_size, n_dims, vocab_size in input_structure:
-            tables.append(EmbeddingTable(n_dims, vocab_size, initializer))
-            lengths.append(window_size)
-
-        self.input_layer = InputLayer(lengths, tables)
 
+        self.eta = 0.001
+        self.mu = 0.9
+        self.t = 1
+        initializer = lambda: 0.2 * numpy.random.uniform(-1.0, 1.0)
+        self.input_layer = InputLayer(input_spec, initializer)
         self.train_func = train_func
         self.predict_func = predict_func
 
+        self.n_classes = n_classes
+        self.n_feats = len(self.input_layer)
         self.model_loc = model_loc
-        if self.model_loc and path.exists(self.model_loc):
-            self._model.load(self.model_loc, freq_thresh=0)
         
-    def train(self, Instance eg):
+    def predict(self, Example eg):
-        pass
+        self.input_layer.fill(eg.embeddings, eg.atoms)
-
+        theano_scores = self.predict_func(eg.embeddings)
-    def predict(self, Instance eg):
+        cdef int i
-
-    cdef const weight_t* score(self, atom_t* context) except NULL:
-        self.set_scores(self._scores, context)
-        return self._scores
-    
-    cdef int set_scores(self, weight_t* scores, atom_t* context) except -1:
-        # TODO f(context) --> Values
-        self._input_layer.fill(self._x, self._values, use_avg=False)
-        theano_scores = self._predict(self._x)
         for i in range(self.n_classes):
-            output[i] = theano_scores[i]
+            eg.scores[i] = theano_scores[i]
-
+        eg.guess = arg_max_if_true(<weight_t*>eg.scores.data, <int*>eg.is_valid.data,
-    cdef int update(self, atom_t* context, class_t guess, class_t gold, int cost) except -1:
+                                   self.n_classes)
-        # TODO f(context) --> Values
-        self._input_layer.fill(self._x, self._values, use_avg=False)
 
+    def train(self, Example eg):
+        self.predict(eg)
+        update, t, eta, mu = self.train_func(eg.embeddings, eg.scores, eg.costs)
+        self.input_layer.update(eg.atoms, update, self.t, self.eta, self.mu)
+        eg.best = arg_max_if_zero(<weight_t*>eg.scores.data, <int*>eg.costs.data,
+                                  self.n_classes)
+        eg.cost = eg.costs[eg.guess]
+        self.t += 1

spacy/syntax/parser.pyx

@@ -68,11 +68,11 @@ cdef class Parser:
         cdef StateClass stcls = StateClass.init(tokens.data, tokens.length)
         self.moves.initialize_state(stcls)
 
-        cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE)
+        cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE, self.model.n_feats)
         while not stcls.is_final():
             eg.wipe()
-            fill_context(eg.atoms, stcls)
+            fill_context(<atom_t*>eg.atoms.data, stcls)
-            self.moves.set_valid(eg.is_valid, stcls)
+            self.moves.set_valid(<bint*>eg.is_valid.data, stcls)
 
             self.model.predict(eg)
 
@@ -84,12 +84,12 @@ cdef class Parser:
         self.moves.preprocess_gold(gold)
         cdef StateClass stcls = StateClass.init(tokens.data, tokens.length)
         self.moves.initialize_state(stcls)
-        cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE)
+        cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE, self.model.n_feats)
         cdef int cost = 0
         while not stcls.is_final():
             eg.wipe()
-            fill_context(eg.atoms, stcls)
+            fill_context(<atom_t*>eg.atoms.data, stcls)
-            self.moves.set_costs(eg.is_valid, eg.costs, stcls, gold)
+            self.moves.set_costs(<bint*>eg.is_valid.data, <int*>eg.costs.data, stcls, gold)
 
             self.model.train(eg)