explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 01:46:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove neptune refs from new train example

Browse Source
This commit is contained in:
Matthew Honnibal 2017-04-15 16:13:17 +02:00
parent 45464d065e
commit b9c26aae11
1 changed files with 2 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
examples/train_ner_standalone.py
View File

@ -32,8 +32,6 @@ from spacy.gold import GoldParse
from spacy.gold import _iob_to_biluo as iob_to_biluo from spacy.gold import _iob_to_biluo as iob_to_biluo
from spacy.scorer import Scorer from spacy.scorer import Scorer
from deepsense import neptune
try: try:
unicode unicode
except NameError: except NameError:
@ -180,24 +178,6 @@ class Pipeline(object):
def train(nlp, train_examples, dev_examples, ctx, nr_epoch=5): def train(nlp, train_examples, dev_examples, ctx, nr_epoch=5):
channels = {}
channels['loss'] = ctx.job.create_channel(
name='loss',
channel_type=neptune.ChannelType.NUMERIC)
channels['f'] = ctx.job.create_channel(
name='F-Measure',
channel_type=neptune.ChannelType.NUMERIC)
channels['p'] = ctx.job.create_channel(
name='Precision',
channel_type=neptune.ChannelType.NUMERIC)
channels['r'] = ctx.job.create_channel(
name='Recall',
channel_type=neptune.ChannelType.NUMERIC)
channels['log'] = ctx.job.create_channel(
name='logs',
channel_type=neptune.ChannelType.TEXT)
next_epoch = train_examples next_epoch = train_examples
print("Iter", "Loss", "P", "R", "F") print("Iter", "Loss", "P", "R", "F")
for i in range(nr_epoch): for i in range(nr_epoch):
@ -210,25 +190,17 @@ def train(nlp, train_examples, dev_examples, ctx, nr_epoch=5):
next_epoch.append((input_, annot)) next_epoch.append((input_, annot))
random.shuffle(next_epoch) random.shuffle(next_epoch)
scores = nlp.evaluate(dev_examples) scores = nlp.evaluate(dev_examples)
report_scores(channels, i, loss, scores) report_scores(i, loss, scores)
nlp.average_weights() nlp.average_weights()
scores = nlp.evaluate(dev_examples) scores = nlp.evaluate(dev_examples)
report_scores(channels, i+1, loss, scores) report_scores(channels, i+1, loss, scores)
def report_scores(channels, i, loss, scores): def report_scores(i, loss, scores):
precision = '%.2f' % scores['ents_p'] precision = '%.2f' % scores['ents_p']
recall = '%.2f' % scores['ents_r'] recall = '%.2f' % scores['ents_r']
f_measure = '%.2f' % scores['ents_f'] f_measure = '%.2f' % scores['ents_f']
print('%d %s %s %s' % (int(loss), precision, recall, f_measure)) print('%d %s %s %s' % (int(loss), precision, recall, f_measure))
channels['log'].send(x=i, y='%d %s %s %s' % (int(loss), precision, recall,
f_measure))
channels['f'].send(x=i, y=scores['ents_f'])
channels['p'].send(x=i, y=scores['ents_p'])
channels['r'].send(x=i, y=scores['ents_r'])
channels['loss'].send(x=i, y=loss)
def read_examples(path): def read_examples(path):
@ -258,11 +230,6 @@ def read_examples(path):
) )
def main(model_dir=Path('/home/matt/repos/spaCy/spacy/data/de-1.0.0'), def main(model_dir=Path('/home/matt/repos/spaCy/spacy/data/de-1.0.0'),
train_loc=None, dev_loc=None, nr_epoch=30): train_loc=None, dev_loc=None, nr_epoch=30):
ctx = neptune.Context()
train_loc = Path(ctx.params.train_loc)
dev_loc = Path(ctx.params.dev_loc)
model_dir = model_dir.resolve()
train_examples = read_examples(train_loc) train_examples = read_examples(train_loc)
dev_examples = read_examples(dev_loc) dev_examples = read_examples(dev_loc)