diff --git a/spacy/training/__init__.py b/spacy/training/__init__.py
index 71d1fa775..a6f873f05 100644
--- a/spacy/training/__init__.py
+++ b/spacy/training/__init__.py
@@ -1,4 +1,4 @@
-from .corpus import Corpus, JsonlCorpus  # noqa: F401
+from .corpus import Corpus, JsonlCorpus, PlainTextCorpus  # noqa: F401
 from .example import Example, validate_examples, validate_get_examples  # noqa: F401
 from .alignment import Alignment  # noqa: F401
 from .augment import dont_augment, orth_variants_augmenter  # noqa: F401
diff --git a/spacy/training/corpus.py b/spacy/training/corpus.py
index b9f929fcd..609733792 100644
--- a/spacy/training/corpus.py
+++ b/spacy/training/corpus.py
@@ -58,6 +58,17 @@ def read_labels(path: Path, *, require: bool = False):
     return srsly.read_json(path)
 
 
+@util.registry.readers("spacy.PlainTextCorpus.v1")
+def create_plain_text_reader(
+    path: Optional[Path],
+    min_length: int = 0,
+    max_length: int = 0,
+) -> Callable[["Language"], Iterable[Doc]]:
+    if path is None:
+        raise ValueError(Errors.E913)
+    return PlainTextCorpus(path, min_length=min_length, max_length=max_length)
+
+
 def walk_corpus(path: Union[str, Path], file_type) -> List[Path]:
     path = util.ensure_path(path)
     if not path.is_dir() and path.parts[-1].endswith(file_type):
@@ -257,3 +268,54 @@ class JsonlCorpus:
                     # We don't *need* an example here, but it seems nice to
                     # make it match the Corpus signature.
                     yield Example(doc, Doc(nlp.vocab, words=words, spaces=spaces))
+
+
+class PlainTextCorpus:
+    """Iterate Example objects from a file or directory of plain text
+    UTF-8 files with one line per doc.
+
+    path (Path): The directory or filename to read from.
+    min_length (int): Minimum document length (in tokens). Shorter documents
+        will be skipped. Defaults to 0, which indicates no limit.
+    max_length (int): Maximum document length (in tokens). Longer documents will
+        be skipped. Defaults to 0, which indicates no limit.
+
+    DOCS: https://spacy.io/api/corpus#plaintextcorpus
+    """
+
+    file_type = "txt"
+
+    def __init__(
+        self,
+        path: Optional[Union[str, Path]],
+        *,
+        min_length: int = 0,
+        max_length: int = 0,
+    ) -> None:
+        self.path = util.ensure_path(path)
+        self.min_length = min_length
+        self.max_length = max_length
+
+    def __call__(self, nlp: "Language") -> Iterator[Example]:
+        """Yield examples from the data.
+
+        nlp (Language): The current nlp object.
+        YIELDS (Example): The example objects.
+
+        DOCS: https://spacy.io/api/corpus#plaintextcorpus-call
+        """
+        for loc in walk_corpus(self.path, ".txt"):
+            with open(loc, encoding="utf-8") as f:
+                for text in f:
+                    text = text.strip()
+                    if len(text):
+                        doc = nlp.make_doc(text)
+                        if self.min_length >= 1 and len(doc) < self.min_length:
+                            continue
+                        elif self.max_length >= 1 and len(doc) > self.max_length:
+                            continue
+                        words = [w.text for w in doc]
+                        spaces = [bool(w.whitespace_) for w in doc]
+                        # We don't *need* an example here, but it seems nice to
+                        # make it match the Corpus signature.
+                        yield Example(doc, Doc(nlp.vocab, words=words, spaces=spaces))
diff --git a/website/docs/api/corpus.mdx b/website/docs/api/corpus.mdx
index c58723e82..37713af4b 100644
--- a/website/docs/api/corpus.mdx
+++ b/website/docs/api/corpus.mdx
@@ -175,3 +175,68 @@ Yield examples from the data.
 | ---------- | -------------------------------------- |
 | `nlp`      | The current `nlp` object. ~~Language~~ |
 | **YIELDS** | The examples. ~~Example~~              |
+
+## PlainTextCorpus {id="plaintextcorpus",tag="class"}
+
+Iterate over documents from a plain text file. Can be used to read the raw text
+corpus for language model
+[pretraining](/usage/embeddings-transformers#pretraining). The expected file
+format is:
+
+- UTF-8 encoding
+- One document per line
+- Blank lines are ignored.
+
+```text {title="Example"}
+Can I ask where you work now and what you do, and if you enjoy it?
+They may just pull out of the Seattle market completely, at least until they have autonomous vehicles.
+My cynical view on this is that it will never be free to the public. Reason: what would be the draw of joining the military? Right now their selling point is free Healthcare and Education. Ironically both are run horribly and most, that I've talked to, come out wishing they never went in.
+```
+
+### PlainTextCorpus.\_\_init\_\_ {id="plaintextcorpus-init",tag="method"}
+
+Initialize the reader.
+
+> #### Example
+>
+> ```python
+> from spacy.training import PlainTextCorpus
+>
+> corpus = PlainTextCorpus("./data/docs.txt")
+> ```
+>
+> ```ini
+> ### Example config
+> [corpora.pretrain]
+> @readers = "spacy.PlainTextCorpus.v1"
+> path = "corpus/raw_text.txt"
+> min_length = 0
+> max_length = 0
+> ```
+
+| Name           | Description                                                                                                                |
+| -------------- | -------------------------------------------------------------------------------------------------------------------------- |
+| `path`         | The directory or filename to read from. Expects newline-delimited documents in UTF8 format. ~~Union[str, Path]~~           |
+| _keyword-only_ |                                                                                                                            |
+| `min_length`   | Minimum document length (in tokens). Shorter documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ |
+| `max_length`   | Maximum document length (in tokens). Longer documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~  |
+
+### PlainTextCorpus.\_\_call\_\_ {id="plaintextcorpus-call",tag="method"}
+
+Yield examples from the data.
+
+> #### Example
+>
+> ```python
+> from spacy.training import PlainTextCorpus
+> import spacy
+>
+> corpus = PlainTextCorpus("./docs.txt")
+> nlp = spacy.blank("en")
+> data = corpus(nlp)
+> ```
+
+| Name       | Description                            |
+| ---------- | -------------------------------------- |
+| `nlp`      | The current `nlp` object. ~~Language~~ |
+| **YIELDS** | The examples. ~~Example~~              |