diff --git a/bin/parser/train_ud.py b/bin/parser/train_ud.py index e9dc2a443..610fe4b94 100644 --- a/bin/parser/train_ud.py +++ b/bin/parser/train_ud.py @@ -137,8 +137,8 @@ def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None): Xs, ys = organize_data(vocab, train_sents) - Xs = Xs[:1] - ys = ys[:1] + Xs = Xs[:10] + ys = ys[:10] with encoder.model.begin_training(Xs[:100], ys[:100]) as (trainer, optimizer): docs = list(Xs) for doc in docs: @@ -151,8 +151,8 @@ def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None): print('%d:\t%.3f\t%.3f\t%.3f' % (itn, nn_loss[-1], scorer.uas, scorer.tags_acc)) nn_loss.append(0.) trainer.each_epoch.append(track_progress) - trainer.batch_size = 1 - trainer.nb_epoch = 100 + trainer.batch_size = 2 + trainer.nb_epoch = 10000 for docs, golds in trainer.iterate(Xs, ys, progress_bar=False): docs = [Doc(vocab, words=[w.text for w in doc]) for doc in docs] tokvecs, upd_tokvecs = encoder.begin_update(docs) diff --git a/spacy/_ml.py b/spacy/_ml.py index 460c2a2c8..39f2a654b 100644 --- a/spacy/_ml.py +++ b/spacy/_ml.py @@ -5,7 +5,7 @@ from thinc.neural._classes.hash_embed import HashEmbed from thinc.neural._classes.convolution import ExtractWindow from thinc.neural._classes.static_vectors import StaticVectors -from .attrs import ID, PREFIX, SUFFIX, SHAPE, TAG, DEP +from .attrs import ID, LOWER, PREFIX, SUFFIX, SHAPE, TAG, DEP def get_col(idx): @@ -147,19 +147,20 @@ def flatten(seqs, drop=0.): def build_tok2vec(lang, width, depth=2, embed_size=1000): - cols = [ID, PREFIX, SUFFIX, SHAPE] + cols = [ID, LOWER, PREFIX, SUFFIX, SHAPE, TAG] with Model.define_operators({'>>': chain, '|': concatenate, '**': clone}): #static = get_col(cols.index(ID)) >> StaticVectors(lang, width) - lower = get_col(cols.index(ID)) >> HashEmbed(width, embed_size) + lower = get_col(cols.index(LOWER)) >> HashEmbed(width, embed_size) prefix = get_col(cols.index(PREFIX)) >> HashEmbed(width, embed_size) suffix = get_col(cols.index(SUFFIX)) >> HashEmbed(width, embed_size) shape = get_col(cols.index(SHAPE)) >> HashEmbed(width, embed_size) + tag = get_col(cols.index(TAG)) >> HashEmbed(width, embed_size) tok2vec = ( doc2feats(cols) >> with_flatten( #(static | prefix | suffix | shape) - (lower | prefix | suffix | shape) - >> Maxout(width, width*4) + (lower | prefix | suffix | shape | tag) + >> Maxout(width, width*5) >> (ExtractWindow(nW=1) >> Maxout(width, width*3)) >> (ExtractWindow(nW=1) >> Maxout(width, width*3)) ) diff --git a/spacy/syntax/parser.pyx b/spacy/syntax/parser.pyx index bd1c1650b..5d64048bf 100644 --- a/spacy/syntax/parser.pyx +++ b/spacy/syntax/parser.pyx @@ -113,7 +113,7 @@ cdef class Parser: def __reduce__(self): return (Parser, (self.vocab, self.moves, self.model), None, None) - def build_model(self, width=8, nr_vector=1000, nF=1, nB=1, nS=1, nL=1, nR=1, **_): + def build_model(self, width=32, nr_vector=1000, nF=1, nB=1, nS=1, nL=1, nR=1, **_): state2vec = build_debug_state2vec(width, nr_vector, nF, nB, nL, nR) model = build_debug_model(state2vec, width, 2, self.moves.n_moves) return model @@ -197,7 +197,7 @@ cdef class Parser: attr_names = self.model.ops.allocate((2,), dtype='i') attr_names[0] = TAG attr_names[1] = DEP - + features = self._get_features(states, tokvecs, attr_names) self.model.begin_training(features) @@ -214,11 +214,12 @@ cdef class Parser: output = list(d_tokens) todo = zip(states, tokvecs, golds, d_tokens) assert len(states) == len(todo) - loss = 0. + losses = [] while todo: states, tokvecs, golds, d_tokens = zip(*todo) scores, finish_update = self._begin_update(states, tokvecs) - token_ids, batch_token_grads = finish_update(golds, sgd=sgd) + token_ids, batch_token_grads = finish_update(golds, sgd=sgd, losses=losses, + force_gold=False) for i, tok_i in enumerate(token_ids): d_tokens[i][tok_i] += batch_token_grads[i] @@ -226,7 +227,7 @@ cdef class Parser: # Get unfinished states (and their matching gold and token gradients) todo = filter(lambda sp: not sp[0].py_is_final(), todo) - return output, loss + return output, sum(losses) def _begin_update(self, states, tokvecs, drop=0.): nr_class = self.moves.n_moves @@ -240,14 +241,17 @@ cdef class Parser: self._validate_batch(is_valid, states) softmaxed = self.model.ops.softmax(scores) softmaxed *= is_valid - softmaxed /= softmaxed.sum(axis=1) - print('Scores', softmaxed[0]) - def backward(golds, sgd=None): + softmaxed /= softmaxed.sum(axis=1).reshape((softmaxed.shape[0], 1)) + def backward(golds, sgd=None, losses=[], force_gold=False): + nonlocal softmaxed costs = self.model.ops.allocate((len(states), nr_class), dtype='f') d_scores = self.model.ops.allocate((len(states), nr_class), dtype='f') self._cost_batch(costs, is_valid, states, golds) self._set_gradient(d_scores, scores, is_valid, costs) + losses.append(numpy.abs(d_scores).sum()) + if force_gold: + softmaxed *= costs <= 0 return finish_update(d_scores, sgd=sgd) return softmaxed, backward @@ -298,17 +302,16 @@ cdef class Parser: def _set_gradient(self, gradients, scores, is_valid, costs): """Do multi-label log loss""" cdef double Z, gZ, max_, g_max + n = gradients.shape[0] scores = scores * is_valid g_scores = scores * is_valid * (costs <= 0.) - exps = numpy.exp(scores - scores.max(axis=1)) + exps = numpy.exp(scores - scores.max(axis=1).reshape((n, 1))) exps *= is_valid - g_exps = numpy.exp(g_scores - g_scores.max(axis=1)) + g_exps = numpy.exp(g_scores - g_scores.max(axis=1).reshape((n, 1))) g_exps *= costs <= 0. g_exps *= is_valid - gradients[:] = exps / exps.sum(axis=1) - gradients -= g_exps / g_exps.sum(axis=1) - print('Gradient', gradients[0]) - print('Costs', costs[0]) + gradients[:] = exps / exps.sum(axis=1).reshape((n, 1)) + gradients -= g_exps / g_exps.sum(axis=1).reshape((n, 1)) def step_through(self, Doc doc, GoldParse gold=None): """ diff --git a/spacy/syntax/stateclass.pyx b/spacy/syntax/stateclass.pyx index 0e0b63d09..5c535c899 100644 --- a/spacy/syntax/stateclass.pyx +++ b/spacy/syntax/stateclass.pyx @@ -47,18 +47,18 @@ cdef class StateClass: return ' '.join((third, second, top, '|', n0, n1)) def nr_context_tokens(self, int nF, int nB, int nS, int nL, int nR): - return 3 - #return 1+nF+nB+nS + nL + (nS * nL) + (nS * nR) + return 8 def set_context_tokens(self, int[:] output, nF=1, nB=0, nS=2, nL=2, nR=2): output[0] = self.B(0) - output[1] = self.S(0) - output[2] = self.S(1) - #output[3] = self.L(self.S(0), 1) - #output[4] = self.L(self.S(0), 2) - #output[5] = self.R(self.S(0), 1) - #output[6] = self.R(self.S(0), 2) + output[1] = self.B(1) + output[2] = self.S(0) + output[3] = self.S(1) + output[4] = self.L(self.S(0), 1) + output[5] = self.L(self.S(0), 2) + output[6] = self.R(self.S(0), 1) + output[7] = self.R(self.S(0), 2) #output[7] = self.L(self.S(1), 1) #output[8] = self.L(self.S(1), 2) #output[9] = self.R(self.S(1), 1)