diff --git a/spacy/language.py b/spacy/language.py
index 44beb6811..d639df7ac 100644
--- a/spacy/language.py
+++ b/spacy/language.py
@@ -396,8 +396,6 @@ class Language:
         if name not in self._pipe_configs:
             raise ValueError(Errors.E960.format(name=name))
         pipe_config = self._pipe_configs[name]
-        pipe_config.pop("nlp", None)
-        pipe_config.pop("name", None)
         return pipe_config
 
     @classmethod
@@ -650,6 +648,10 @@ class Language:
         filled = Config(filled[factory_name])
         filled["factory"] = factory_name
         filled.pop("@factories", None)
+        # Remove the extra values we added because we don't want to keep passing
+        # them around, copying them etc.
+        filled.pop("nlp", None)
+        filled.pop("name", None)
         # Merge the final filled config with the raw config (including non-
         # interpolated variables)
         if raw_config:
diff --git a/spacy/tests/pipeline/test_pipe_factories.py b/spacy/tests/pipeline/test_pipe_factories.py
index 380312bb4..1cf06d97f 100644
--- a/spacy/tests/pipeline/test_pipe_factories.py
+++ b/spacy/tests/pipeline/test_pipe_factories.py
@@ -470,3 +470,21 @@ def test_pipe_factories_decorator_idempotent():
     nlp = Language()
     nlp.add_pipe(name)
     Language.component(name2, func=func2)
+
+
+def test_pipe_factories_config_excludes_nlp():
+    """Test that the extra values we temporarily add to component config
+    blocks/functions are removed and not copied around.
+    """
+    name = "test_pipe_factories_config_excludes_nlp"
+    func = lambda nlp, name: lambda doc: doc
+    Language.factory(name, func=func)
+    config = {
+        "nlp": {"lang": "en", "pipeline": [name]},
+        "components": {name: {"factory": name}},
+    }
+    nlp = English.from_config(config)
+    assert nlp.pipe_names == [name]
+    pipe_cfg = nlp.get_pipe_config(name)
+    pipe_cfg == {"factory": name}
+    assert nlp._pipe_configs[name] == {"factory": name}