explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-07-04 11:53:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Fix evaluate method in train.py, to use sentences which don't have raw text

Browse Source
This commit is contained in:
Matthew Honnibal 2015-05-28 16:30:23 +02:00
parent d25d31442d
commit f42dc1f7d8
1 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
bin/parser/train.py
View File

@ -111,7 +111,13 @@ def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=True)
nlp = Language(data_dir=model_dir) nlp = Language(data_dir=model_dir)
scorer = Scorer() scorer = Scorer()
for raw_text, annot_tuples, brackets in gold_tuples: for raw_text, annot_tuples, brackets in gold_tuples:
if raw_text is not None:
tokens = nlp(raw_text, merge_mwes=False) tokens = nlp(raw_text, merge_mwes=False)
else:
tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1])
nlp.tagger(tokens)
nlp.entity(tokens)
nlp.parser(tokens)
gold = GoldParse(tokens, annot_tuples) gold = GoldParse(tokens, annot_tuples)
scorer.score(tokens, gold, verbose=verbose) scorer.score(tokens, gold, verbose=verbose)
return scorer return scorer
@ -144,13 +150,13 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
) )
def main(train_loc, dev_loc, model_dir, n_sents=0, n_iter=15, out_loc="", verbose=False, def main(train_loc, dev_loc, model_dir, n_sents=0, n_iter=15, out_loc="", verbose=False,
debug=False, corruption_level=0.0): debug=False, corruption_level=0.0):
print 'reading gold' #print 'reading gold'
gold_train = list(read_json_file(train_loc)) #gold_train = list(read_json_file(train_loc))
print 'done' #print 'done'
train(English, gold_train, model_dir, #train(English, gold_train, model_dir,
feat_set='basic' if not debug else 'debug', # feat_set='basic' if not debug else 'debug',
gold_preproc=False, n_sents=n_sents, # gold_preproc=False, n_sents=n_sents,
corruption_level=corruption_level, n_iter=n_iter) # corruption_level=corruption_level, n_iter=n_iter)
if out_loc: if out_loc:
write_parses(English, dev_loc, model_dir, out_loc) write_parses(English, dev_loc, model_dir, out_loc)
scorer = evaluate(English, list(read_json_file(dev_loc)), scorer = evaluate(English, list(read_json_file(dev_loc)),