explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-05-28 17:53:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add eval_beam_widths argument to spacy train

Browse Source
This commit is contained in:
Matthew Honnibal 2019-03-16 15:02:39 +01:00
parent b13b2aeb54
commit daa8c3787a
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
spacy/cli/train.py
View File

@ -58,6 +58,7 @@ from .. import about
str, str,
), ),
noise_level=("Amount of corruption for data augmentation", "option", "nl", float), noise_level=("Amount of corruption for data augmentation", "option", "nl", float),
eval_beam_widths=("Beam widths to evaluate, e.g. 4,8", "option", "bw", str),
gold_preproc=("Use gold preprocessing", "flag", "G", bool), gold_preproc=("Use gold preprocessing", "flag", "G", bool),
learn_tokens=("Make parser learn gold-standard tokenization", "flag", "T", bool), learn_tokens=("Make parser learn gold-standard tokenization", "flag", "T", bool),
verbose=("Display more information for debug", "flag", "VV", bool), verbose=("Display more information for debug", "flag", "VV", bool),
@ -81,6 +82,7 @@ def train(
parser_multitasks="", parser_multitasks="",
entity_multitasks="", entity_multitasks="",
noise_level=0.0, noise_level=0.0,
eval_beam_widths="",
gold_preproc=False, gold_preproc=False,
learn_tokens=False, learn_tokens=False,
verbose=False, verbose=False,
@ -134,6 +136,14 @@ def train(
util.env_opt("batch_compound", 1.001), util.env_opt("batch_compound", 1.001),
) )
if not eval_beam_widths:
eval_beam_widths = [1]
else:
eval_beam_widths = [int(bw) for bw in eval_beam_widths.split(",")]
if 1 not in eval_beam_widths:
eval_beam_widths.append(1)
eval_beam_widths.sort()
# Set up the base model and pipeline. If a base model is specified, load # Set up the base model and pipeline. If a base model is specified, load
# the model and make sure the pipeline matches the pipeline setting. If # the model and make sure the pipeline matches the pipeline setting. If
# training starts from a blank model, intitalize the language class. # training starts from a blank model, intitalize the language class.
@ -247,7 +257,7 @@ def train(
epoch_model_path = output_path / ("model%d" % i) epoch_model_path = output_path / ("model%d" % i)
nlp.to_disk(epoch_model_path) nlp.to_disk(epoch_model_path)
nlp_loaded = util.load_model_from_path(epoch_model_path) nlp_loaded = util.load_model_from_path(epoch_model_path)
for beam_width in [1, 4, 16, 128]: for beam_width in eval_beam_widths:
for name, component in nlp_loaded.pipeline: for name, component in nlp_loaded.pipeline:
if hasattr(component, "cfg"): if hasattr(component, "cfg"):
component.cfg["beam_width"] = beam_width component.cfg["beam_width"] = beam_width