From 2a91d641e6ccafb888121c50b41c4fd90a00a816 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Thu, 30 Mar 2017 02:36:33 +0200 Subject: [PATCH] Add dropout to parser --- spacy/syntax/parser.pyx | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/spacy/syntax/parser.pyx b/spacy/syntax/parser.pyx index dc157d13d..cffa96423 100644 --- a/spacy/syntax/parser.pyx +++ b/spacy/syntax/parser.pyx @@ -193,13 +193,11 @@ cdef class Parser: elif 'features' not in cfg: cfg['features'] = self.feature_templates self.model = ParserModel(self.moves.n_moves, cfg['features'], - size=2**18, + size=2**14, learn_rate=cfg.get('learn_rate', 0.001)) - #self.model.l1_penalty = cfg.get('L1', 1e-8) - #self.model.learn_rate = cfg.get('learn_rate', 0.001) + #self.model.l1_penalty = cfg.get('L1', 0.0) - self.optimizer = SGD(NumpyOps(), cfg.get('learn_rate', 0.001), - momentum=0.9) + self.optimizer = Adam(NumpyOps(), cfg.get('learn_rate', 0.001)) self.cfg = cfg @@ -337,9 +335,19 @@ cdef class Parser: cdef Transition action words = [w.text for w in tokens] + cdef int i + cdef double[::1] py_dropout + cdef double* dropout while not stcls.is_final(): nr_feat = self.model.set_featuresC(context, features, stcls.c) + py_dropout = numpy.random.uniform(0., 1., nr_feat) + dropout = &py_dropout[0] + for i in range(nr_feat): + if dropout[i] < 0.5: + features[i].value = 0 + else: + features[i].value *= 2 self.moves.set_costs(is_valid, costs, stcls, gold) self.model.set_scoresC(scores, features, nr_feat) @@ -347,6 +355,9 @@ cdef class Parser: best = arg_max_if_gold(scores, costs, nr_class) self.model.regression_lossC(d_scores, scores, costs) + for i in range(nr_class): + if not is_valid[i]: + d_scores[i] = 0 self.model.set_gradientC(d_scores, features, nr_feat) action = self.moves.c[guess] @@ -354,7 +365,7 @@ cdef class Parser: #print(scores[guess], scores[best], d_scores[guess], costs[guess], # self.moves.move_name(action.move, action.label), stcls.print_state(words)) - loss += scores[guess] + loss += abs(scores[guess] + costs[guess]) memset(context, 0, sizeof(context)) memset(features, 0, sizeof(features[0]) * nr_feat) memset(scores, 0, sizeof(scores[0]) * nr_class) @@ -363,8 +374,7 @@ cdef class Parser: for i in range(nr_class): is_valid[i] = 1 #if itn % 100 == 0: - # self.optimizer(self.model.model[0].ravel(), - # self.model.model[1].ravel(), key=1) + # self.model.finish_update(self.optimizer) return loss def step_through(self, Doc doc):