From 6bbdcc5db5bf8a96e7110db3bc64a51306b86073 Mon Sep 17 00:00:00 2001
From: Matthew Honnibal <honnibal@gmail.com>
Date: Sat, 30 May 2015 05:23:02 +0200
Subject: [PATCH] * Fix gold_preproc flag in train.py

---
 bin/parser/train.py | 36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/bin/parser/train.py b/bin/parser/train.py
index 4d6744937..7b9fbb9af 100755
--- a/bin/parser/train.py
+++ b/bin/parser/train.py
@@ -51,6 +51,22 @@ def score_model(scorer, nlp, raw_text, annot_tuples):
     scorer.score(tokens, gold, verbose=False)
 
 
+def _merge_sents(sents):
+    m_deps = [[], [], [], [], [], []]
+    m_brackets = []
+    i = 0
+    for (ids, words, tags, heads, labels, ner), brackets in sents:
+        m_deps[0].extend(id_ + i for id_ in ids)
+        m_deps[1].extend(words)
+        m_deps[2].extend(tags)
+        m_deps[3].extend(head + i for head in heads)
+        m_deps[4].extend(labels)
+        m_deps[5].extend(ner)
+        m_brackets.extend((b['first'] + i, b['last'] + i, b['label']) for b in brackets)
+        i += len(ids)
+    return [(m_deps, m_brackets)]
+        
+
 def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0,
           gold_preproc=False, n_sents=0, corruption_level=0):
     dep_model_dir = path.join(model_dir, 'deps')
@@ -82,11 +98,13 @@ def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0
         scorer = Scorer()
         loss = 0
         for raw_text, sents in gold_tuples:
-            if not gold_preproc:
+            if gold_preproc:
+                raw_text = None
+            else:
                 sents = _merge_sents(sents)
             for annot_tuples, ctnt in sents:
                 score_model(scorer, nlp, raw_text, annot_tuples)
-                if raw_text is None or gold_preproc:
+                if raw_text is None:
                     tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1])
                 else:
                     tokens = nlp.tokenizer(raw_text)
@@ -106,12 +124,16 @@ def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0
     nlp.vocab.strings.dump(path.join(model_dir, 'vocab', 'strings.txt'))
 
 
-def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=True):
+def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=False):
     nlp = Language(data_dir=model_dir)
     scorer = Scorer()
     for raw_text, sents in gold_tuples:
+        if gold_preproc:
+            raw_text = None
+        else:
+            sents = _merge_sents(sents)
         for annot_tuples, brackets in sents:
-            if raw_text is None or gold_preproc:
+            if raw_text is None:
                 tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1])
                 nlp.tagger(tokens)
                 nlp.entity(tokens)
@@ -120,8 +142,6 @@ def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=True)
                 tokens = nlp(raw_text, merge_mwes=False)
             gold = GoldParse(tokens, annot_tuples)
             scorer.score(tokens, gold, verbose=verbose)
-            for t in tokens:
-                print t.orth_, t.dep_, t.head.orth_, t.ent_type_
     return scorer
 
 
@@ -158,8 +178,8 @@ def main(train_loc, dev_loc, model_dir, n_sents=0, n_iter=15, out_loc="", verbos
           feat_set='basic' if not debug else 'debug',
           gold_preproc=gold_preproc, n_sents=n_sents,
           corruption_level=corruption_level, n_iter=n_iter)
-    #if out_loc:
-    #    write_parses(English, dev_loc, model_dir, out_loc)
+    if out_loc:
+        write_parses(English, dev_loc, model_dir, out_loc)
     scorer = evaluate(English, list(read_json_file(dev_loc)),
                       model_dir, gold_preproc=gold_preproc, verbose=verbose)
     print 'TOK', 100-scorer.token_acc