explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-06 06:30:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update train.py

Browse Source
This commit is contained in:
Matthew Honnibal 2020-06-20 21:49:06 +02:00
parent 49145b9ec1
commit 17efd6bfec
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
spacy/cli/train.py
View File

@ -210,7 +210,8 @@ def train(
nlp.resume_training() nlp.resume_training()
else: else:
msg.info(f"Initializing the nlp pipeline: {nlp.pipe_names}") msg.info(f"Initializing the nlp pipeline: {nlp.pipe_names}")
nlp.begin_training(lambda: corpus.train_dataset(nlp)) train_examples = list(corpus.train_dataset(nlp, shuffle=False))
nlp.begin_training(lambda: train_examples)
# Update tag map with provided mapping # Update tag map with provided mapping
nlp.vocab.morphology.tag_map.update(tag_map) nlp.vocab.morphology.tag_map.update(tag_map)
@ -280,11 +281,14 @@ def train(
eg.reference = None eg.reference = None
eg.predicted = None eg.predicted = None
except Exception as e: except Exception as e:
if output_path is not None:
msg.warn( msg.warn(
f"Aborting and saving the final best model. " f"Aborting and saving the final best model. "
f"Encountered exception: {str(e)}", f"Encountered exception: {str(e)}",
exits=1, exits=1,
) )
else:
raise e
finally: finally:
if output_path is not None: if output_path is not None:
final_model_path = output_path / "model-final" final_model_path = output_path / "model-final"
@ -300,7 +304,6 @@ def create_train_batches(nlp, corpus, cfg):
epochs_todo = cfg.get("max_epochs", 0) epochs_todo = cfg.get("max_epochs", 0)
while True: while True:
train_examples = list(corpus.train_dataset(nlp)) train_examples = list(corpus.train_dataset(nlp))
if len(train_examples) == 0: if len(train_examples) == 0:
raise ValueError(Errors.E988) raise ValueError(Errors.E988)
random.shuffle(train_examples) random.shuffle(train_examples)