explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-03 21:24:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Evaluate accuracy at multiple beam widths

Browse Source
This commit is contained in:
Matthew Honnibal 2019-03-15 15:18:28 +01:00
parent 0703f5986b
commit f762c36e61
1 changed files with 55 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
spacy/cli/train.py
View File

@ -200,9 +200,9 @@ def train(
msg.text("Loaded pretrained tok2vec for: {}".format(components)) msg.text("Loaded pretrained tok2vec for: {}".format(components))
# fmt: off # fmt: off
row_head = ("Itn", "Dep Loss", "NER Loss", "UAS", "NER P", "NER R", "NER F", "Tag %", "Token %", "CPU WPS", "GPU WPS") row_head = ("Itn", "Beam Width", "Dep Loss", "NER Loss", "UAS", "NER P", "NER R", "NER F", "Tag %", "Token %", "CPU WPS", "GPU WPS")
row_settings = { row_settings = {
"widths": (3, 10, 10, 7, 7, 7, 7, 7, 7, 7, 7), "widths": (3, 10, 10, 10, 7, 7, 7, 7, 7, 7, 7, 7),
"aligns": tuple(["r" for i in row_head]), "aligns": tuple(["r" for i in row_head]),
"spacing": 2 "spacing": 2
} }
@ -247,51 +247,61 @@ def train(
epoch_model_path = output_path / ("model%d" % i) epoch_model_path = output_path / ("model%d" % i)
nlp.to_disk(epoch_model_path) nlp.to_disk(epoch_model_path)
nlp_loaded = util.load_model_from_path(epoch_model_path) nlp_loaded = util.load_model_from_path(epoch_model_path)
dev_docs = list(corpus.dev_docs(nlp_loaded, gold_preproc=gold_preproc)) for beam_width in [1, 4, 16, 128]:
nwords = sum(len(doc_gold[0]) for doc_gold in dev_docs) for name, component in nlp_loaded.pipeline:
start_time = timer() if hasattr(component, "cfg"):
scorer = nlp_loaded.evaluate(dev_docs, debug) component.cfg["beam_width"] = beam_width
end_time = timer() dev_docs = list(corpus.dev_docs(nlp_loaded, gold_preproc=gold_preproc))
if use_gpu < 0: nwords = sum(len(doc_gold[0]) for doc_gold in dev_docs)
gpu_wps = None start_time = timer()
cpu_wps = nwords / (end_time - start_time) scorer = nlp_loaded.evaluate(dev_docs, debug)
else: end_time = timer()
gpu_wps = nwords / (end_time - start_time) if use_gpu < 0:
with Model.use_device("cpu"): gpu_wps = None
nlp_loaded = util.load_model_from_path(epoch_model_path)
dev_docs = list(
corpus.dev_docs(nlp_loaded, gold_preproc=gold_preproc)
)
start_time = timer()
scorer = nlp_loaded.evaluate(dev_docs)
end_time = timer()
cpu_wps = nwords / (end_time - start_time) cpu_wps = nwords / (end_time - start_time)
acc_loc = output_path / ("model%d" % i) / "accuracy.json" else:
srsly.write_json(acc_loc, scorer.scores) gpu_wps = nwords / (end_time - start_time)
with Model.use_device("cpu"):
nlp_loaded = util.load_model_from_path(epoch_model_path)
nlp_loaded.parser.cfg["beam_width"]
dev_docs = list(
corpus.dev_docs(nlp_loaded, gold_preproc=gold_preproc)
)
start_time = timer()
scorer = nlp_loaded.evaluate(dev_docs)
end_time = timer()
cpu_wps = nwords / (end_time - start_time)
acc_loc = output_path / ("model%d" % i) / "accuracy.json"
srsly.write_json(acc_loc, scorer.scores)
# Update model meta.json # Update model meta.json
meta["lang"] = nlp.lang meta["lang"] = nlp.lang
meta["pipeline"] = nlp.pipe_names meta["pipeline"] = nlp.pipe_names
meta["spacy_version"] = ">=%s" % about.__version__ meta["spacy_version"] = ">=%s" % about.__version__
meta["accuracy"] = scorer.scores if beam_width == 1:
meta["speed"] = {"nwords": nwords, "cpu": cpu_wps, "gpu": gpu_wps} meta["speed"] = {"nwords": nwords, "cpu": cpu_wps, "gpu": gpu_wps}
meta["vectors"] = { meta["accuracy"] = scorer.scores
"width": nlp.vocab.vectors_length, else:
"vectors": len(nlp.vocab.vectors), meta.setdefault("beam_accuracy", {})
"keys": nlp.vocab.vectors.n_keys, meta.setdefault("beam_speed", {})
"name": nlp.vocab.vectors.name meta["beam_accuracy"][beam_width] = scorer.scores
meta["beam_speed"][beam_width] = {"nwords": nwords, "cpu": cpu_wps, "gpu": gpu_wps}
meta["vectors"] = {
"width": nlp.vocab.vectors_length,
"vectors": len(nlp.vocab.vectors),
"keys": nlp.vocab.vectors.n_keys,
"name": nlp.vocab.vectors.name
} }
meta.setdefault("name", "model%d" % i) meta.setdefault("name", "model%d" % i)
meta.setdefault("version", version) meta.setdefault("version", version)
meta_loc = output_path / ("model%d" % i) / "meta.json" meta_loc = output_path / ("model%d" % i) / "meta.json"
srsly.write_json(meta_loc, meta) srsly.write_json(meta_loc, meta)
util.set_env_log(verbose)
util.set_env_log(verbose) progress = _get_progress(
i, beam_width, losses, scorer.scores, cpu_wps=cpu_wps, gpu_wps=gpu_wps
progress = _get_progress( )
i, losses, scorer.scores, cpu_wps=cpu_wps, gpu_wps=gpu_wps msg.row(progress, **row_settings)
)
msg.row(progress, **row_settings)
finally: finally:
with nlp.use_params(optimizer.averages): with nlp.use_params(optimizer.averages):
final_model_path = output_path / "model-final" final_model_path = output_path / "model-final"
@ -377,7 +387,7 @@ def _get_metrics(component):
return ("token_acc",) return ("token_acc",)
def _get_progress(itn, losses, dev_scores, cpu_wps=0.0, gpu_wps=0.0): def _get_progress(itn, beam_width, losses, dev_scores, cpu_wps=0.0, gpu_wps=0.0):
scores = {} scores = {}
for col in [ for col in [
"dep_loss", "dep_loss",
@ -400,6 +410,7 @@ def _get_progress(itn, losses, dev_scores, cpu_wps=0.0, gpu_wps=0.0):
scores["gpu_wps"] = gpu_wps or 0.0 scores["gpu_wps"] = gpu_wps or 0.0
return [ return [
itn, itn,
beam_width,
"{:.3f}".format(scores["dep_loss"]), "{:.3f}".format(scores["dep_loss"]),
"{:.3f}".format(scores["ner_loss"]), "{:.3f}".format(scores["ner_loss"]),
"{:.3f}".format(scores["uas"]), "{:.3f}".format(scores["uas"]),