diff --git a/spacy/errors.py b/spacy/errors.py
index 1a576faee..a66861667 100644
--- a/spacy/errors.py
+++ b/spacy/errors.py
@@ -136,11 +136,10 @@ class Errors:
             "after (component name or index), first (True) or last (True). "
             "Invalid configuration: {args}. Existing components: {opts}")
     E007 = ("'{name}' already exists in pipeline. Existing names: {opts}")
-    E008 = ("Some current components would be lost when restoring previous "
-            "pipeline state. If you added components after calling "
-            "`nlp.select_pipes()`, you should remove them explicitly with "
-            "`nlp.remove_pipe()` before the pipeline is restored. Names of "
-            "the new components: {names}")
+    E008 = ("Can't restore disabled pipeline component '{name}' because it "
+            "doesn't exist in the pipeline anymore. If you want to remove "
+            "components from the pipeline, you should do it before calling "
+            "`nlp.select_pipes()` or after restoring the disabled components.")
     E010 = ("Word vectors set to length 0. This may be because you don't have "
             "a model installed or loaded, or because your model doesn't "
             "include word vectors. For more info, see the docs:\n"
diff --git a/spacy/language.py b/spacy/language.py
index 33a1f2d9d..1a76a4c8e 100644
--- a/spacy/language.py
+++ b/spacy/language.py
@@ -1720,9 +1720,9 @@ class DisabledPipes(list):
     def restore(self) -> None:
         """Restore the pipeline to its state when DisabledPipes was created."""
         for name in self.names:
+            if name not in self.nlp._pipe_names:
+                raise ValueError(Errors.E008.format(name=name))
             self.nlp.enable_pipe(name)
-        # TODO: maybe add some more checks / catch errors that may occur if
-        # user removes a disabled pipe in the with block
         self[:] = []
 
 
diff --git a/spacy/tests/pipeline/test_pipe_methods.py b/spacy/tests/pipeline/test_pipe_methods.py
index cb91f546f..f91486358 100644
--- a/spacy/tests/pipeline/test_pipe_methods.py
+++ b/spacy/tests/pipeline/test_pipe_methods.py
@@ -181,6 +181,11 @@ def test_select_pipes_errors(nlp):
     with pytest.raises(ValueError):
         nlp.select_pipes(enable=[], disable=["c3"])
 
+    disabled = nlp.select_pipes(disable=["c2"])
+    nlp.remove_pipe("c2")
+    with pytest.raises(ValueError):
+        disabled.restore()
+
 
 @pytest.mark.parametrize("n_pipes", [100])
 def test_add_lots_of_pipes(nlp, n_pipes):