Add custom error when evaluation throws a KeyError

spacy/errors.py

@@ -456,6 +456,9 @@ class Errors:
             "issue tracker: http://github.com/explosion/spaCy/issues")
 
     # TODO: fix numbering after merging develop into master
+    E900 = ("Could not run the full 'nlp' pipeline for evaluation. If you specified "
+            "frozen components, make sure they were already trained and initialized. "
+            "You can also consider moving them to the 'disabled' list instead.")
     E901 = ("Failed to remove existing output directory: {path}. If your "
             "config and the components you train change between runs, a "
             "non-empty output directory can lead to stale pipeline data. To "

spacy/training/loop.py

@@ -249,7 +249,10 @@ def create_evaluation_callback(
 
     def evaluate() -> Tuple[float, Dict[str, float]]:
         dev_examples = list(dev_corpus(nlp))
-        scores = nlp.evaluate(dev_examples)
+        try:
+            scores = nlp.evaluate(dev_examples)
+        except KeyError as e:
+            raise KeyError(Errors.E900) from e
         # Calculate a weighted sum based on score_weights for the main score.
         # We can only consider scores that are ints/floats, not dicts like
         # entity scores per type etc.