diff --git a/spacy/cli/train.py b/spacy/cli/train.py
index e5157bc68..b4ea41114 100644
--- a/spacy/cli/train.py
+++ b/spacy/cli/train.py
@@ -203,7 +203,7 @@ def _collate_best_model(meta, output_path, components):
     best_dest = output_path / 'model-best'
     shutil.copytree(output_path / 'model-final', best_dest)
     for component, best_component_src in bests.items():
-        shutil.rmtree(best_dir / component)
+        shutil.rmtree(best_dest / component)
         shutil.copytree(best_component_src, best_dest / component)
         with (best_component_src / 'accuracy.json').open() as file_:
             accs = json.load(file_)