diff --git a/spacy/syntax/_neural.pyx b/spacy/syntax/_neural.pyx index 90bee8253..d14e2d3ac 100644 --- a/spacy/syntax/_neural.pyx +++ b/spacy/syntax/_neural.pyx @@ -22,6 +22,8 @@ from ._parse_features cimport fill_context from ._parse_features cimport CONTEXT_SIZE from ._parse_features cimport fill_context from ._parse_features cimport * +from .transition_system cimport TransitionSystem +from ..tokens.doc cimport Doc cdef class ParserPerceptron(AveragedPerceptron): @@ -51,6 +53,23 @@ cdef class ParserPerceptron(AveragedPerceptron): fill_context(eg.atoms, state) eg.nr_feat = self.extracter.set_features(eg.features, eg.atoms) + def _update_from_history(self, TransitionSystem moves, Doc doc, history, weight_t grad): + cdef Pool mem = Pool() + cdef atom_t[CONTEXT_SIZE] context + features = mem.alloc(self.nr_feat, sizeof(FeatureC)) + + cdef StateClass stcls = StateClass.init(doc.c, doc.length) + moves.initialize_state(stcls.c) + + cdef class_t clas + self.time += 1 + for clas in history: + fill_context(context, stcls.c) + nr_feat = self.extracter.set_features(features, context) + for feat in features[:nr_feat]: + self.update_weight(feat.key, clas, feat.value * grad) + moves.c[clas].do(stcls.c, moves.c[clas].label) + cdef class ParserNeuralNet(NeuralNet): def __init__(self, shape, **kwargs): @@ -123,6 +142,41 @@ cdef class ParserNeuralNet(NeuralNet): cdef void _softmaxC(self, weight_t* out) nogil: pass + def _update_from_history(self, TransitionSystem moves, Doc doc, history, weight_t grad): + cdef Example py_eg = Example(nr_class=moves.n_moves, nr_atom=CONTEXT_SIZE, + nr_feat=self.nr_feat, widths=self.widths) + stcls = StateClass.init(doc.c, doc.length) + moves.initialize_state(stcls.c) + cdef uint64_t[2] key + key[0] = hash64(doc.c, sizeof(TokenC) * doc.length, 0) + key[1] = 0 + cdef uint64_t clas + for clas in history: + self.set_featuresC(py_eg.c, stcls.c) + moves.set_valid(py_eg.c.is_valid, stcls.c) + # Update with a sparse gradient: everything's 0, except our class. + # Remember, this is a component of the global update. It's not our + # "job" here to think about the other beam candidates. We just want + # to work on this sequence. However, other beam candidates will + # have gradients that refer to the same state. + # We therefore have a key that indicates the current sequence, so that + # the model can merge updates that refer to the same state together, + # by summing their gradients. + memset(py_eg.c.costs, 0, self.moves.n_moves) + py_eg.c.costs[clas] = grad + self.updateC( + py_eg.c.features, py_eg.c.nr_feat, True, py_eg.c.costs, py_eg.c.is_valid, + False, key=key[0]) + moves.c[clas].do(stcls.c, self.moves.c[clas].label) + py_eg.c.reset() + # Build a hash of the state sequence. + # Position 0 represents the previous sequence, position 1 the new class. + # So we want to do: + # key.prev = hash((key.prev, key.new)) + # key.new = clas + key[1] = clas + key[0] = hash64(key, sizeof(key), 0) + cdef inline FeatureC* _add_token(FeatureC* feats, int slot, const TokenC* token, weight_t value) nogil: diff --git a/spacy/syntax/beam_parser.pyx b/spacy/syntax/beam_parser.pyx index 7cf8e45c3..cb4f2d05d 100644 --- a/spacy/syntax/beam_parser.pyx +++ b/spacy/syntax/beam_parser.pyx @@ -113,33 +113,12 @@ cdef class BeamParser(Parser): break else: violn.check_crf(pred, gold) - if isinstance(self.model, ParserNeuralNet): - min_grad = 0.1 ** (itn+1) - for grad, hist in zip(violn.p_probs, violn.p_hist): - assert not math.isnan(grad) and not math.isinf(grad) - if abs(grad) >= min_grad: - self._update_dense(tokens, hist, grad) - for grad, hist in zip(violn.g_probs, violn.g_hist): - assert not math.isnan(grad) and not math.isinf(grad) - if abs(grad) >= min_grad: - self._update_dense(tokens, hist, grad) - else: - self.model.time += 1 - #min_grad = 0.01 ** (itn+1) - #for grad, hist in zip(violn.p_probs, violn.p_hist): - # assert not math.isnan(grad) - # assert not math.isinf(grad) - # if abs(grad) >= min_grad: - # self._update(tokens, hist, -grad) - #for grad, hist in zip(violn.g_probs, violn.g_hist): - # assert not math.isnan(grad) - # assert not math.isinf(grad) - # if abs(grad) >= min_grad: - # self._update(tokens, hist, -grad) - if violn.p_hist: - self._update(tokens, violn.p_hist[0], -1.0) - if violn.g_hist: - self._update(tokens, violn.g_hist[0], 1.0) + min_grad = 0.1 ** (itn+1) + histories = zip(violn.p_probs, violn.p_hist) + zip(violn.g_probs, violn.g_hist) + for grad, hist in histories: + assert not math.isnan(grad) and not math.isinf(grad) + if abs(grad) >= min_grad: + self._update_from_history(self.moves, tokens, hist, grad) _cleanup(pred) _cleanup(gold) return pred.loss @@ -149,16 +128,11 @@ cdef class BeamParser(Parser): nr_feat=self.model.nr_feat, widths=self.model.widths) cdef ExampleC* eg = py_eg.c - cdef ParserNeuralNet nn_model - cdef ParserPerceptron ap_model for i in range(beam.size): py_eg.reset() stcls = beam.at(i) if not stcls.c.is_final(): - if isinstance(self.model, ParserNeuralNet): - ParserNeuralNet.set_featuresC(self.model, eg, stcls.c) - else: - ParserPerceptron.set_featuresC(self.model, eg, stcls.c) + self.model.set_featuresC(eg, stcls.c) self.model.set_scoresC(beam.scores[i], eg.features, eg.nr_feat, 1) self.moves.set_valid(beam.is_valid[i], stcls.c) if gold is not None: @@ -173,59 +147,6 @@ cdef class BeamParser(Parser): beam.advance(_transition_state, _hash_state, self.moves.c) beam.check_done(_check_final_state, NULL) - def _update_dense(self, Doc doc, history, weight_t loss): - cdef Example py_eg = Example(nr_class=self.moves.n_moves, nr_atom=CONTEXT_SIZE, - nr_feat=self.model.nr_feat, widths=self.model.widths) - cdef ExampleC* eg = py_eg.c - cdef ParserNeuralNet model = self.model - stcls = StateClass.init(doc.c, doc.length) - self.moves.initialize_state(stcls.c) - cdef uint64_t[2] key - key[0] = hash64(doc.c, sizeof(TokenC) * doc.length, 0) - key[1] = 0 - cdef uint64_t clas - for clas in history: - model.set_featuresC(eg, stcls.c) - self.moves.set_valid(eg.is_valid, stcls.c) - # Update with a sparse gradient: everything's 0, except our class. - # Remember, this is a component of the global update. It's not our - # "job" here to think about the other beam candidates. We just want - # to work on this sequence. However, other beam candidates will - # have gradients that refer to the same state. - # We therefore have a key that indicates the current sequence, so that - # the model can merge updates that refer to the same state together, - # by summing their gradients. - memset(eg.costs, 0, self.moves.n_moves) - eg.costs[clas] = loss - model.updateC( - eg.features, eg.nr_feat, True, eg.costs, eg.is_valid, False, key=key[0]) - self.moves.c[clas].do(stcls.c, self.moves.c[clas].label) - py_eg.reset() - # Build a hash of the state sequence. - # Position 0 represents the previous sequence, position 1 the new class. - # So we want to do: - # key.prev = hash((key.prev, key.new)) - # key.new = clas - key[1] = clas - key[0] = hash64(key, sizeof(key), 0) - - def _update(self, Doc tokens, list hist, weight_t inc): - cdef Pool mem = Pool() - cdef atom_t[CONTEXT_SIZE] context - features = mem.alloc(self.model.nr_feat, sizeof(FeatureC)) - - cdef StateClass stcls = StateClass.init(tokens.c, tokens.length) - self.moves.initialize_state(stcls.c) - - cdef class_t clas - cdef ParserPerceptron model = self.model - for clas in hist: - fill_context(context, stcls.c) - nr_feat = model.extracter.set_features(features, context) - for feat in features[:nr_feat]: - model.update_weight(feat.key, clas, feat.value * inc) - self.moves.c[clas].do(stcls.c, self.moves.c[clas].label) - # These are passed as callbacks to thinc.search.Beam cdef int _transition_state(void* _dest, void* _src, class_t clas, void* _moves) except -1: @@ -261,32 +182,3 @@ def _cleanup(Beam beam): cdef hash_t _hash_state(void* _state, void* _) except 0: state = _state return state.c.hash() - - -# def _early_update(self, Doc doc, Beam pred, Beam gold): -# # Gather the partition function --- Z --- by which we can normalize the -# # scores into a probability distribution. The simple idea here is that -# # we clip the probability of all parses outside the beam to 0. -# cdef long double Z = 0.0 -# for i in range(pred.size): -# # Make sure we've only got negative examples here. -# # Otherwise, we might double-count the gold. -# if pred._states[i].loss > 0: -# Z += exp(pred._states[i].score) -# cdef weight_t grad -# if Z > 0: # If no negative examples, don't update. -# Z += exp(gold.score) -# for i, hist in enumerate(pred.histories): -# if pred._states[i].loss > 0: -# # Update with the negative example. -# # Gradient of loss is P(parse) - 0 -# grad = exp(pred._states[i].score) / Z -# if abs(grad) >= 0.01: -# self._update_dense(doc, hist, grad) -# # Update with the positive example. -# # Gradient of loss is P(parse) - 1 -# grad = (exp(gold.score) / Z) - 1 -# if abs(grad) >= 0.01: -# self._update_dense(doc, gold.histories[0], grad) -# -#