diff --git a/spacy/training/loop.py b/spacy/training/loop.py
index dacd2dba4..55919014b 100644
--- a/spacy/training/loop.py
+++ b/spacy/training/loop.py
@@ -230,7 +230,10 @@ def train_while_improving(
         if is_best_checkpoint is not None:
             losses = {}
         # Stop if no improvement in `patience` updates (if specified)
-        best_score, best_step = max(results)
+        # Negate step value so that the earliest best step is chosen for the
+        # same score, i.e. (1.0, 100) is chosen over (1.0, 200)
+        best_result = max((r_score, -r_step) for r_score, r_step in results)
+        best_step = -best_result[1]
         if patience and (step - best_step) >= patience:
             break
         # Stop if we've exhausted our max steps (if specified)