diff --git a/spacy/pipeline/spancat_exclusive.py b/spacy/pipeline/spancat_exclusive.py
index 5dcbacb67..18fc6c426 100644
--- a/spacy/pipeline/spancat_exclusive.py
+++ b/spacy/pipeline/spancat_exclusive.py
@@ -18,6 +18,12 @@ spancat_excl_default_config = """
 @architectures = "spacy.SpanCategorizer.v1"
 scorer = {"@layers": "Softmax.v2"}
 
+[model.scorer.init_W]
+@initializers = "zero_init.v1"
+
+[model.scorer.init_b]
+@initializers = "zero_init.v1"
+
 [model.reducer]
 @layers = spacy.mean_max_reducer.v1
 hidden_size = 128