diff --git a/spacy/errors.py b/spacy/errors.py
index db1a886aa..a2f8ca85c 100644
--- a/spacy/errors.py
+++ b/spacy/errors.py
@@ -981,6 +981,8 @@ class Errors(metaclass=ErrorsWithCodes):
              " 'min_length': {min_length}, 'max_length': {max_length}")
     E1054 = ("The text, including whitespace, must match between reference and "
              "predicted docs when training {component}.")
+    E1055 = ("The 'replace_listener' callback expects {num_params} parameters, "
+             "but only callbacks with one or three parameters are supported")
 
 
 # Deprecated model shortcuts, only used in errors and warnings
diff --git a/spacy/language.py b/spacy/language.py
index 2326a44ef..6a848bf9a 100644
--- a/spacy/language.py
+++ b/spacy/language.py
@@ -2041,14 +2041,13 @@ class Language:
                     num_params = len(
                         inspect.signature(replace_listener_func).parameters
                     )
-                    assert num_params in (
-                        1,
-                        3,
-                    ), f"'replace_listener' callback expects {num_params} parameters, but we only support one or three"
                     if num_params == 1:
                         new_model = replace_listener_func(new_model)
-                    else:
+                    elif num_params == 3:
                         new_model = replace_listener_func(new_model, listener, tok2vec)
+                    else:
+                        raise ValueError(Errors.E1055.format(num_params=num_params))
+
                 util.replace_model_node(pipe.model, listener, new_model)  # type: ignore[attr-defined]
                 tok2vec.remove_listener(listener, pipe_name)