Learns things

bin/parser/train_ud.py

@@ -137,8 +137,8 @@ def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None):
 
     
     Xs, ys = organize_data(vocab, train_sents)
-    Xs = Xs[:1]
+    Xs = Xs[:10]
-    ys = ys[:1]
+    ys = ys[:10]
     with encoder.model.begin_training(Xs[:100], ys[:100]) as (trainer, optimizer):
         docs = list(Xs)
         for doc in docs:
@@ -151,8 +151,8 @@ def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None):
             print('%d:\t%.3f\t%.3f\t%.3f' % (itn, nn_loss[-1], scorer.uas, scorer.tags_acc))
             nn_loss.append(0.)
         trainer.each_epoch.append(track_progress)
-        trainer.batch_size = 1
+        trainer.batch_size = 2
-        trainer.nb_epoch = 100
+        trainer.nb_epoch = 10000
         for docs, golds in trainer.iterate(Xs, ys, progress_bar=False):
             docs = [Doc(vocab, words=[w.text for w in doc]) for doc in docs]
             tokvecs, upd_tokvecs = encoder.begin_update(docs)

spacy/_ml.py

@@ -5,7 +5,7 @@ from thinc.neural._classes.hash_embed import HashEmbed
 from thinc.neural._classes.convolution import ExtractWindow
 from thinc.neural._classes.static_vectors import StaticVectors
 
-from .attrs import ID, PREFIX, SUFFIX, SHAPE, TAG, DEP
+from .attrs import ID, LOWER, PREFIX, SUFFIX, SHAPE, TAG, DEP
 
 
 def get_col(idx):
@@ -147,19 +147,20 @@ def flatten(seqs, drop=0.):
 
 
 def build_tok2vec(lang, width, depth=2, embed_size=1000):
-    cols = [ID, PREFIX, SUFFIX, SHAPE]
+    cols = [ID, LOWER, PREFIX, SUFFIX, SHAPE, TAG]
     with Model.define_operators({'>>': chain, '|': concatenate, '**': clone}):
         #static = get_col(cols.index(ID))     >> StaticVectors(lang, width)
-        lower = get_col(cols.index(ID))     >> HashEmbed(width, embed_size)
+        lower = get_col(cols.index(LOWER))     >> HashEmbed(width, embed_size)
         prefix = get_col(cols.index(PREFIX)) >> HashEmbed(width, embed_size)
         suffix = get_col(cols.index(SUFFIX)) >> HashEmbed(width, embed_size)
         shape = get_col(cols.index(SHAPE))   >> HashEmbed(width, embed_size)
+        tag = get_col(cols.index(TAG))   >> HashEmbed(width, embed_size)
         tok2vec = (
             doc2feats(cols)
             >> with_flatten(
                 #(static | prefix | suffix | shape)
-                (lower | prefix | suffix | shape)
+                (lower | prefix | suffix | shape | tag)
-                >> Maxout(width, width*4)
+                >> Maxout(width, width*5)
                 >> (ExtractWindow(nW=1) >> Maxout(width, width*3))
                 >> (ExtractWindow(nW=1) >> Maxout(width, width*3))
             )

spacy/syntax/parser.pyx

@@ -113,7 +113,7 @@ cdef class Parser:
     def __reduce__(self):
         return (Parser, (self.vocab, self.moves, self.model), None, None)
 
-    def build_model(self, width=8, nr_vector=1000, nF=1, nB=1, nS=1, nL=1, nR=1, **_):
+    def build_model(self, width=32, nr_vector=1000, nF=1, nB=1, nS=1, nL=1, nR=1, **_):
         state2vec = build_debug_state2vec(width, nr_vector, nF, nB, nL, nR)
         model = build_debug_model(state2vec, width, 2, self.moves.n_moves)
         return model
@@ -214,11 +214,12 @@ cdef class Parser:
         output = list(d_tokens)
         todo = zip(states, tokvecs, golds, d_tokens)
         assert len(states) == len(todo)
-        loss = 0.
+        losses = []
         while todo:
             states, tokvecs, golds, d_tokens = zip(*todo)
             scores, finish_update = self._begin_update(states, tokvecs)
-            token_ids, batch_token_grads = finish_update(golds, sgd=sgd)
+            token_ids, batch_token_grads = finish_update(golds, sgd=sgd, losses=losses,
+                                                         force_gold=False)
             for i, tok_i in enumerate(token_ids):
                 d_tokens[i][tok_i] += batch_token_grads[i]
 
@@ -226,7 +227,7 @@ cdef class Parser:
 
             # Get unfinished states (and their matching gold and token gradients)
             todo = filter(lambda sp: not sp[0].py_is_final(), todo)
-        return output, loss
+        return output, sum(losses)
 
     def _begin_update(self, states, tokvecs, drop=0.):
         nr_class = self.moves.n_moves
@@ -240,14 +241,17 @@ cdef class Parser:
         self._validate_batch(is_valid, states)
         softmaxed = self.model.ops.softmax(scores)
         softmaxed *= is_valid
-        softmaxed /= softmaxed.sum(axis=1)
+        softmaxed /= softmaxed.sum(axis=1).reshape((softmaxed.shape[0], 1))
-        print('Scores', softmaxed[0])
+        def backward(golds, sgd=None, losses=[], force_gold=False):
-        def backward(golds, sgd=None):
+            nonlocal softmaxed
             costs = self.model.ops.allocate((len(states), nr_class), dtype='f')
             d_scores = self.model.ops.allocate((len(states), nr_class), dtype='f')
 
             self._cost_batch(costs, is_valid, states, golds)
             self._set_gradient(d_scores, scores, is_valid, costs)
+            losses.append(numpy.abs(d_scores).sum())
+            if force_gold:
+                softmaxed *= costs <= 0
             return finish_update(d_scores, sgd=sgd)
         return softmaxed, backward
 
@@ -298,17 +302,16 @@ cdef class Parser:
     def _set_gradient(self, gradients, scores, is_valid, costs):
         """Do multi-label log loss"""
         cdef double Z, gZ, max_, g_max
+        n = gradients.shape[0]
         scores = scores * is_valid
         g_scores = scores * is_valid * (costs <= 0.)
-        exps = numpy.exp(scores - scores.max(axis=1))
+        exps = numpy.exp(scores - scores.max(axis=1).reshape((n, 1)))
         exps *= is_valid
-        g_exps = numpy.exp(g_scores - g_scores.max(axis=1))
+        g_exps = numpy.exp(g_scores - g_scores.max(axis=1).reshape((n, 1)))
         g_exps *= costs <= 0.
         g_exps *= is_valid
-        gradients[:] = exps / exps.sum(axis=1)
+        gradients[:] = exps / exps.sum(axis=1).reshape((n, 1))
-        gradients -= g_exps / g_exps.sum(axis=1)
+        gradients -= g_exps / g_exps.sum(axis=1).reshape((n, 1))
-        print('Gradient', gradients[0])
-        print('Costs', costs[0])
 
     def step_through(self, Doc doc, GoldParse gold=None):
         """

spacy/syntax/stateclass.pyx

@@ -47,18 +47,18 @@ cdef class StateClass:
         return ' '.join((third, second, top, '|', n0, n1))
 
     def nr_context_tokens(self, int nF, int nB, int nS, int nL, int nR):
-        return 3
+        return 8
-        #return 1+nF+nB+nS + nL + (nS * nL) + (nS * nR)
 
     def set_context_tokens(self, int[:] output, nF=1, nB=0, nS=2,
             nL=2, nR=2):
         output[0] = self.B(0)
-        output[1] = self.S(0)
+        output[1] = self.B(1)
-        output[2] = self.S(1)
+        output[2] = self.S(0)
-        #output[3] = self.L(self.S(0), 1)
+        output[3] = self.S(1)
-        #output[4] = self.L(self.S(0), 2)
+        output[4] = self.L(self.S(0), 1)
-        #output[5] = self.R(self.S(0), 1)
+        output[5] = self.L(self.S(0), 2)
-        #output[6] = self.R(self.S(0), 2)
+        output[6] = self.R(self.S(0), 1)
+        output[7] = self.R(self.S(0), 2)
         #output[7] = self.L(self.S(1), 1)
         #output[8] = self.L(self.S(1), 2)
         #output[9] = self.R(self.S(1), 1)