Add as_example to Sentencizer pipe() (#4933)

spacy/pipeline/pipes.pyx

@@ -1711,11 +1711,23 @@ class Sentencizer(Pipe):
             return example
         return doc
 
-    def pipe(self, stream, batch_size=128, n_threads=-1):
+    def pipe(self, stream, batch_size=128, n_threads=-1, as_example=False):
-        for docs in util.minibatch(stream, size=batch_size):
+        for examples in util.minibatch(stream, size=batch_size):
-            docs = list(docs)
+            docs = [self._get_doc(ex) for ex in examples]
-            tag_ids = self.predict(docs)
+            predictions = self.predict(docs)
-            self.set_annotations(docs, tag_ids)
+            if isinstance(predictions, tuple) and len(tuple) == 2:
+                scores, tensors = predictions
+                self.set_annotations(docs, scores, tensors=tensors)
+            else:
+                self.set_annotations(docs, predictions)
+
+            if as_example:
+                annotated_examples = []
+                for ex, doc in zip(examples, docs):
+                    ex.doc = doc
+                    annotated_examples.append(ex)
+                yield from annotated_examples
+            else:
                 yield from docs
 
     def predict(self, docs):

spacy/tests/pipeline/test_sentencizer.py

@@ -24,6 +24,12 @@ def test_sentencizer_pipe():
         sent_starts = [t.is_sent_start for t in doc]
         assert sent_starts == [True, False, True, False, False, False, False]
         assert len(list(doc.sents)) == 2
+    for ex in nlp.pipe(texts, as_example=True):
+        doc = ex.doc
+        assert doc.is_sentenced
+        sent_starts = [t.is_sent_start for t in doc]
+        assert sent_starts == [True, False, True, False, False, False, False]
+        assert len(list(doc.sents)) == 2
 
 
 @pytest.mark.parametrize(