diff --git a/spacy/training/loop.py b/spacy/training/loop.py
index 448b1a2c2..f29261b7e 100644
--- a/spacy/training/loop.py
+++ b/spacy/training/loop.py
@@ -197,9 +197,9 @@ def train_while_improving(
         if not (step % eval_frequency):
             if optimizer.averages:
                 with nlp.use_params(optimizer.averages):
-                    score, other_scores = evaluate(step)
+                    score, other_scores = evaluate()
             else:
-                score, other_scores = evaluate(step)
+                score, other_scores = evaluate()
             results.append((score, step))
             is_best_checkpoint = score == max(results)[0]
         else:
@@ -248,10 +248,8 @@ def create_evaluation_callback(
 ) -> Callable[[], Tuple[float, Dict[str, float]]]:
     weights = {key: value for key, value in weights.items() if value is not None}
 
-    def evaluate(step) -> Tuple[float, Dict[str, float]]:
-        # Limit dev_examples by steps, so we don't spend longer on
-        # the estimation than we have training.
-        dev_examples = list(itertools.islice(dev_corpus(nlp), step))
+    def evaluate() -> Tuple[float, Dict[str, float]]:
+        dev_examples = list(dev_corpus(nlp))
         try:
             scores = nlp.evaluate(dev_examples)
         except KeyError as e: