diff --git a/spacy/cli/train.py b/spacy/cli/train.py
index b1e9446ed..a2c06c571 100644
--- a/spacy/cli/train.py
+++ b/spacy/cli/train.py
@@ -88,8 +88,11 @@ def train(cmd, lang, output_dir, train_data, dev_data, n_iter=20, n_sents=0,
             with nlp.use_params(optimizer.averages):
                 with (output_path / ('model%d.pickle' % i)).open('wb') as file_:
                     dill.dump(nlp, file_, -1)
-                with (output_path / ('model%d.pickle' % i)).open('rb') as file_:
-                    nlp_loaded = dill.load(file_)
+                with (output_path / ('model%d.bin' % i)).open('wb') as file_:
+                    file_.write(nlp.to_bytes())
+                with (output_path / ('model%d.bin' % i)).open('rb') as file_:
+                    nlp_loaded = lang_class(pipeline=pipeline)
+                    nlp_loaded.from_bytes(file_.read())
                 scorer = nlp_loaded.evaluate(corpus.dev_docs(nlp_loaded, gold_preproc=False))
             print_progress(i, losses, scorer.scores)
     finally: