diff --git a/spacy/pipeline/spancat_exclusive.py b/spacy/pipeline/spancat_exclusive.py
index 75c4446af..adf3741c1 100644
--- a/spacy/pipeline/spancat_exclusive.py
+++ b/spacy/pipeline/spancat_exclusive.py
@@ -18,12 +18,6 @@ spancat_exclusive_default_config = """
 @architectures = "spacy.SpanCategorizer.v1"
 scorer = {"@layers": "Softmax.v2"}
 
-[model.scorer.init_W]
-@initializers = "zero_init.v1"
-
-[model.scorer.init_b]
-@initializers = "zero_init.v1"
-
 [model.reducer]
 @layers = spacy.mean_max_reducer.v1
 hidden_size = 128