diff --git a/spacy/language.py b/spacy/language.py
index 05838f21b..d53710f58 100644
--- a/spacy/language.py
+++ b/spacy/language.py
@@ -402,9 +402,10 @@ class Language(object):
         """
         if name not in self.pipe_names:
             raise ValueError(Errors.E001.format(name=name, opts=self.pipe_names))
+        removed = self.pipeline.pop(self.pipe_names.index(name))
         if ENABLE_PIPELINE_ANALYSIS:
             analyze_all_pipes(self.pipeline)
-        return self.pipeline.pop(self.pipe_names.index(name))
+        return removed
 
     def __call__(self, text, disable=[], component_cfg=None):
         """Apply the pipeline to some text. The text can span multiple sentences,
diff --git a/spacy/tests/pipeline/test_analysis.py b/spacy/tests/pipeline/test_analysis.py
index 6f8b93a6e..198f11bcd 100644
--- a/spacy/tests/pipeline/test_analysis.py
+++ b/spacy/tests/pipeline/test_analysis.py
@@ -145,3 +145,24 @@ def test_analysis_validate_attrs_valid():
 def test_analysis_validate_attrs_invalid(attr):
     with pytest.raises(ValueError):
         validate_attrs([attr])
+
+
+def test_analysis_validate_attrs_remove_pipe():
+    """Test that attributes are validated correctly on remove."""
+    spacy.language.ENABLE_PIPELINE_ANALYSIS = True
+
+    @component("c1", assigns=["token.tag"])
+    def c1(doc):
+        return doc
+
+    @component("c2", requires=["token.pos"])
+    def c2(doc):
+        return doc
+
+    nlp = Language()
+    nlp.add_pipe(c1)
+    with pytest.warns(UserWarning):
+        nlp.add_pipe(c2)
+    with pytest.warns(None) as record:
+        nlp.remove_pipe("c2")
+    assert not record.list