From ac63274e15eb0f958d8929775460391d26fbde81 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Wed, 27 Jul 2016 02:56:36 +0200 Subject: [PATCH] Tmp --- bin/parser/conll_train.py | 63 ++++++++----- setup.py | 4 +- spacy/syntax/beam_parser.pyx | 8 +- spacy/syntax/parser.pxd | 32 +------ spacy/syntax/parser.pyx | 169 +---------------------------------- 5 files changed, 55 insertions(+), 221 deletions(-) diff --git a/bin/parser/conll_train.py b/bin/parser/conll_train.py index dc0166b86..1342357b1 100755 --- a/bin/parser/conll_train.py +++ b/bin/parser/conll_train.py @@ -163,26 +163,43 @@ def train(Language, gold_tuples, model_dir, dev_loc, n_iter=15, feat_set=u'basic nr_trimmed = 0 eg_seen = 0 loss = 0 + micro_eval = gold_tuples[:50] for itn in range(n_iter): - random.shuffle(gold_tuples) - for _, sents in gold_tuples: - for annot_tuples, _ in sents: - tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1]) - nlp.tagger.tag_from_strings(tokens, annot_tuples[2]) - gold = GoldParse(tokens, annot_tuples) - loss += nlp.parser.train(tokens, gold) - - eg_seen += 1 - if eg_seen % 10000 == 0: - dev_uas = score_file(nlp, dev_loc).uas - train_uas = score_sents(nlp, gold_tuples[:1000]).uas - size = nlp.parser.model.mem.size - print('%d:\t%d\t%.3f\t%.3f\t%.3f\t%d' % (itn, int(loss), nr_trimmed, - train_uas, dev_uas, size)) - loss = 0 - nlp.end_training(model_dir) + try: + eg_seen = _train_epoch(nlp, gold_tuples, eg_seen, itn, + dev_loc, micro_eval) + except KeyboardInterrupt: + print("Saving model...") + break + #nlp.end_training(model_dir) + nlp.parser.model.end_training() + print("Saved. Evaluating...") return nlp +def _train_epoch(nlp, gold_tuples, eg_seen, itn, dev_loc, micro_eval): + random.shuffle(gold_tuples) + loss = 0 + nr_trimmed = 0 + for _, sents in gold_tuples: + for annot_tuples, _ in sents: + tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1]) + nlp.tagger.tag_from_strings(tokens, annot_tuples[2]) + gold = GoldParse(tokens, annot_tuples) + loss += nlp.parser.train(tokens, gold, itn=itn) + eg_seen += 1 + if eg_seen % 1000 == 0: + if eg_seen % 20000 == 0: + dev_uas = score_file(nlp, dev_loc).uas + else: + dev_uas = 0.0 + train_uas = score_sents(nlp, micro_eval).uas + size = nlp.parser.model.mem.size + nr_upd = nlp.parser.model.time + print('%d,%d:\t%d\t%.3f\t%.3f\t%.3f\t%d' % (itn, nr_upd, int(loss), nr_trimmed, + train_uas, dev_uas, size)) + loss = 0 + return eg_seen + @plac.annotations( train_loc=("Location of CoNLL 09 formatted training file"), @@ -206,11 +223,13 @@ def main(train_loc, dev_loc, model_dir, n_iter=15, neural=False, batch_norm=Fals batch_norm=batch_norm, learn_rate=learn_rate, update_step=update_step) - scorer = Scorer() - with io.open(dev_loc, 'r', encoding='utf8') as file_: - for _, sents in read_conll(file_): - for annot_tuples, _ in sents: - score_model(scorer, nlp, None, annot_tuples) + + scorer = score_file(nlp, dev_loc) + #scorer = Scorer() + #with io.open(dev_loc, 'r', encoding='utf8') as file_: + # for _, sents in read_conll(file_): + # for annot_tuples, _ in sents: + # score_model(scorer, nlp, None, annot_tuples) print('TOK', scorer.token_acc) print('POS', scorer.tags_acc) print('UAS', scorer.uas) diff --git a/setup.py b/setup.py index 3871432cc..0080926f2 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ MOD_NAMES = [ 'spacy.syntax.stateclass', 'spacy.syntax._state', 'spacy.tokenizer', + 'spacy.syntax._neural', 'spacy.syntax.parser', 'spacy.syntax.beam_parser', 'spacy.syntax.nonproj', @@ -174,7 +175,8 @@ def setup_package(): mod_path = mod_name.replace('.', '/') + '.cpp' ext_modules.append( Extension(mod_name, [mod_path], - language='c++', include_dirs=include_dirs)) + language='c++', include_dirs=include_dirs, + libraries=['/Users/matt/blis/lib/blis'])) if not is_source_release(root): generate_cython(root, 'spacy') diff --git a/spacy/syntax/beam_parser.pyx b/spacy/syntax/beam_parser.pyx index 6624f3b5d..7cf8e45c3 100644 --- a/spacy/syntax/beam_parser.pyx +++ b/spacy/syntax/beam_parser.pyx @@ -114,15 +114,13 @@ cdef class BeamParser(Parser): else: violn.check_crf(pred, gold) if isinstance(self.model, ParserNeuralNet): - min_grad = 0.01 ** (itn+1) + min_grad = 0.1 ** (itn+1) for grad, hist in zip(violn.p_probs, violn.p_hist): - assert not math.isnan(grad) - assert not math.isinf(grad) + assert not math.isnan(grad) and not math.isinf(grad) if abs(grad) >= min_grad: self._update_dense(tokens, hist, grad) for grad, hist in zip(violn.g_probs, violn.g_hist): - assert not math.isnan(grad) - assert not math.isinf(grad) + assert not math.isnan(grad) and not math.isinf(grad) if abs(grad) >= min_grad: self._update_dense(tokens, hist, grad) else: diff --git a/spacy/syntax/parser.pxd b/spacy/syntax/parser.pxd index 956c178a7..a103e549b 100644 --- a/spacy/syntax/parser.pxd +++ b/spacy/syntax/parser.pxd @@ -1,35 +1,11 @@ -from thinc.linear.avgtron cimport AveragedPerceptron -from thinc.neural.nn cimport NeuralNet -from thinc.linear.features cimport ConjunctionExtracter -from thinc.base cimport Model -from thinc.extra.eg cimport Example -from thinc.typedefs cimport weight_t -from thinc.structs cimport FeatureC - -from .stateclass cimport StateClass from .arc_eager cimport TransitionSystem -from ..tokens.doc cimport Doc -from ..structs cimport TokenC -from thinc.structs cimport NeuralNetC, ExampleC from ._state cimport StateC +from ..structs cimport TokenC + +from thinc.base cimport Model +from thinc.linalg cimport * -cdef class ParserNeuralNet(NeuralNet): - cdef ConjunctionExtracter extracter - cdef void set_featuresC(self, ExampleC* eg, const void* _state) nogil - - -cdef class ParserPerceptron(AveragedPerceptron): - cdef void set_featuresC(self, ExampleC* eg, const void* _state) nogil - - -cdef class ParserNeuralNetEnsemble(ParserNeuralNet): - cdef object _models - cdef NeuralNetC** _models_c - cdef int** _masks - cdef int _nr_model - - cdef class Parser: cdef readonly Model model cdef readonly TransitionSystem moves diff --git a/spacy/syntax/parser.pyx b/spacy/syntax/parser.pyx index 8af41c918..784e2fcb5 100644 --- a/spacy/syntax/parser.pyx +++ b/spacy/syntax/parser.pyx @@ -24,10 +24,13 @@ import random from cymem.cymem cimport Pool, Address from murmurhash.mrmr cimport hash64 + from thinc.typedefs cimport weight_t, class_t, feat_t, atom_t, hash_t, idx_t from thinc.linear.avgtron cimport AveragedPerceptron from thinc.linalg cimport VecVec from thinc.structs cimport NeuralNetC, SparseArrayC, ExampleC +from thinc.extra.eg cimport Example + from preshed.maps cimport MapStruct from preshed.maps cimport map_get from thinc.structs cimport FeatureC @@ -50,6 +53,7 @@ from ._parse_features cimport fill_context from ._parse_features cimport * from .stateclass cimport StateClass from ._state cimport StateC +from ._neural cimport ParserNeuralNet, ParserPerceptron DEBUG = False @@ -77,171 +81,6 @@ def ParserFactory(transition_system): return lambda strings, dir_: Parser(strings, dir_, transition_system) -cdef class ParserPerceptron(AveragedPerceptron): - @property - def widths(self): - return (self.extracter.nr_templ,) - - def update(self, Example eg): - '''Does regression on negative cost. Sort of cute?''' - self.time += 1 - cdef weight_t loss = 0.0 - best = eg.best - for clas in range(eg.c.nr_class): - if not eg.c.is_valid[clas]: - continue - if eg.c.scores[clas] < eg.c.scores[best]: - continue - loss += (-eg.c.costs[clas] - eg.c.scores[clas]) ** 2 - d_loss = 2 * (-eg.c.costs[clas] - eg.c.scores[clas]) - step = d_loss * 0.001 - for feat in eg.c.features[:eg.c.nr_feat]: - self.update_weight(feat.key, clas, feat.value * step) - return int(loss) - - cdef void set_featuresC(self, ExampleC* eg, const void* _state) nogil: - state = _state - fill_context(eg.atoms, state) - eg.nr_feat = self.extracter.set_features(eg.features, eg.atoms) - - -cdef class ParserNeuralNet(NeuralNet): - def __init__(self, shape, **kwargs): - vector_widths = [4] * 76 - slots = [0, 1, 2, 3] # S0 - slots += [4, 5, 6, 7] # S1 - slots += [8, 9, 10, 11] # S2 - slots += [12, 13, 14, 15] # S3+ - slots += [16, 17, 18, 19] # B0 - slots += [20, 21, 22, 23] # B1 - slots += [24, 25, 26, 27] # B2 - slots += [28, 29, 30, 31] # B3+ - slots += [32, 33, 34, 35] * 2 # S0l, S0r - slots += [36, 37, 38, 39] * 2 # B0l, B0r - slots += [40, 41, 42, 43] * 2 # S1l, S1r - slots += [44, 45, 46, 47] * 2 # S2l, S2r - slots += [48, 49, 50, 51, 52, 53, 54, 55] - slots += [53, 54, 55, 56] - input_length = sum(vector_widths[slot] for slot in slots) - widths = [input_length] + shape - NeuralNet.__init__(self, widths, embed=(vector_widths, slots), **kwargs) - - @property - def nr_feat(self): - return 2000 - - cdef void set_featuresC(self, ExampleC* eg, const void* _state) nogil: - memset(eg.features, 0, 2000 * sizeof(FeatureC)) - state = _state - fill_context(eg.atoms, state) - feats = eg.features - - feats = _add_token(feats, 0, state.S_(0), 1.0) - feats = _add_token(feats, 4, state.S_(1), 1.0) - feats = _add_token(feats, 8, state.S_(2), 1.0) - # Rest of the stack, with exponential decay - for i in range(3, state.stack_depth()): - feats = _add_token(feats, 12, state.S_(i), 1.0 * 0.5**(i-2)) - feats = _add_token(feats, 16, state.B_(0), 1.0) - feats = _add_token(feats, 20, state.B_(1), 1.0) - feats = _add_token(feats, 24, state.B_(2), 1.0) - # Rest of the buffer, with exponential decay - for i in range(3, min(8, state.buffer_length())): - feats = _add_token(feats, 28, state.B_(i), 1.0 * 0.5**(i-2)) - feats = _add_subtree(feats, 32, state, state.S(0)) - feats = _add_subtree(feats, 40, state, state.B(0)) - feats = _add_subtree(feats, 48, state, state.S(1)) - feats = _add_subtree(feats, 56, state, state.S(2)) - feats = _add_pos_bigram(feats, 64, state.S_(0), state.B_(0)) - feats = _add_pos_bigram(feats, 65, state.S_(1), state.S_(0)) - feats = _add_pos_bigram(feats, 66, state.S_(1), state.B_(0)) - feats = _add_pos_bigram(feats, 67, state.S_(0), state.B_(1)) - feats = _add_pos_bigram(feats, 68, state.S_(0), state.R_(state.S(0), 1)) - feats = _add_pos_bigram(feats, 69, state.S_(0), state.R_(state.S(0), 2)) - feats = _add_pos_bigram(feats, 70, state.S_(0), state.L_(state.S(0), 1)) - feats = _add_pos_bigram(feats, 71, state.S_(0), state.L_(state.S(0), 2)) - feats = _add_pos_trigram(feats, 72, state.S_(1), state.S_(0), state.B_(0)) - feats = _add_pos_trigram(feats, 73, state.S_(0), state.B_(0), state.B_(1)) - feats = _add_pos_trigram(feats, 74, state.S_(0), state.R_(state.S(0), 1), - state.R_(state.S(0), 2)) - feats = _add_pos_trigram(feats, 75, state.S_(0), state.L_(state.S(0), 1), - state.L_(state.S(0), 2)) - eg.nr_feat = feats - eg.features - - cdef void _set_delta_lossC(self, weight_t* delta_loss, - const weight_t* Zs, const weight_t* scores) nogil: - for i in range(self.c.widths[self.c.nr_layer-1]): - delta_loss[i] = Zs[i] - - cdef void _softmaxC(self, weight_t* out) nogil: - pass - - -cdef inline FeatureC* _add_token(FeatureC* feats, - int slot, const TokenC* token, weight_t value) nogil: - # Word - feats.i = slot - feats.key = token.lex.norm - feats.value = value - feats += 1 - # POS tag - feats.i = slot+1 - feats.key = token.tag - feats.value = value - feats += 1 - # Dependency label - feats.i = slot+2 - feats.key = token.dep - feats.value = value - feats += 1 - # Word, label, tag - feats.i = slot+3 - cdef uint64_t key[3] - key[0] = token.lex.cluster - key[1] = token.tag - key[2] = token.dep - feats.key = hash64(key, sizeof(key), 0) - feats.value = value - feats += 1 - return feats - - -cdef inline FeatureC* _add_subtree(FeatureC* feats, int slot, const StateC* state, int t) nogil: - value = 1.0 - for i in range(state.n_R(t)): - feats = _add_token(feats, slot, state.R_(t, i+1), value) - value *= 0.5 - slot += 4 - value = 1.0 - for i in range(state.n_L(t)): - feats = _add_token(feats, slot, state.L_(t, i+1), value) - value *= 0.5 - return feats - - -cdef inline FeatureC* _add_pos_bigram(FeatureC* feat, int slot, - const TokenC* t1, const TokenC* t2) nogil: - cdef uint64_t[2] key - key[0] = t1.tag - key[1] = t2.tag - feat.i = slot - feat.key = hash64(key, sizeof(key), slot) - feat.value = 1.0 - return feat+1 - - -cdef inline FeatureC* _add_pos_trigram(FeatureC* feat, int slot, - const TokenC* t1, const TokenC* t2, const TokenC* t3) nogil: - cdef uint64_t[3] key - key[0] = t1.tag - key[1] = t2.tag - key[2] = t3.tag - feat.i = slot - feat.key = hash64(key, sizeof(key), slot) - feat.value = 1.0 - return feat+1 - - cdef class Parser: def __init__(self, StringStore strings, transition_system, model): self.moves = transition_system