rewrite train_corpus to corpus.train in config

extra/experiments/onto-joint/defaults.cfg

@@ -21,14 +21,16 @@ eval_frequency = 200
 score_weights = {"dep_las": 0.4, "ents_f": 0.4, "tag_acc": 0.2}
 frozen_components = []
 
-[training.train_corpus]
+[training.corpus]
+
+[training.corpus.train]
 @readers = "spacy.Corpus.v1"
 path = ${paths:train}
 gold_preproc = true
 max_length = 0
 limit = 0
 
-[training.dev_corpus]
+[training.corpus.dev]
 @readers = "spacy.Corpus.v1"
 path = ${paths:dev}
 gold_preproc = ${training.read_train:gold_preproc}

extra/experiments/ptb-joint-pos-dep/defaults.cfg

@@ -20,14 +20,16 @@ patience = 10000
 eval_frequency = 200
 score_weights = {"dep_las": 0.8, "tag_acc": 0.2}
 
-[training.read_train]
+[training.corpus]
+
+[training.corpus.train]
 @readers = "spacy.Corpus.v1"
 path = ${paths:train}
 gold_preproc = true
 max_length = 0
 limit = 0
 
-[training.read_dev]
+[training.corpus.dev]
 @readers = "spacy.Corpus.v1"
 path = ${paths:dev}
 gold_preproc = ${training.read_train:gold_preproc}

spacy/cli/templates/quickstart_training.jinja

@@ -195,12 +195,14 @@ total_steps = 20000
 initial_rate = 5e-5
 {% endif %}
 
-[training.train_corpus]
+[training.corpus]
+
+[training.corpus.train]
 @readers = "spacy.Corpus.v1"
 path = ${paths.train}
 max_length = {{ 500 if hardware == "gpu" else 2000 }}
 
-[training.dev_corpus]
+[training.corpus.dev]
 @readers = "spacy.Corpus.v1"
 path = ${paths.dev}
 max_length = 0

spacy/cli/train.py

@@ -92,8 +92,8 @@ def train(
     raw_text, tag_map, morph_rules, weights_data = load_from_paths(config)
     T_cfg = config["training"]
     optimizer = T_cfg["optimizer"]
-    train_corpus = T_cfg["train_corpus"]
+    train_corpus = T_cfg["corpus"]["train"]
-    dev_corpus = T_cfg["dev_corpus"]
+    dev_corpus = T_cfg["corpus"]["dev"]
     batcher = T_cfg["batcher"]
     train_logger = T_cfg["logger"]
     # Components that shouldn't be updated during training

spacy/default_config.cfg

@@ -44,7 +44,9 @@ frozen_components = []
 [training.logger]
 @loggers = "spacy.ConsoleLogger.v1"
 
-[training.train_corpus]
+[training.corpus]
+
+[training.corpus.train]
 @readers = "spacy.Corpus.v1"
 path = ${paths.train}
 # Whether to train on sequences with 'gold standard' sentence boundaries
@@ -56,7 +58,7 @@ max_length = 0
 # Limitation on number of training examples
 limit = 0
 
-[training.dev_corpus]
+[training.corpus.dev]
 @readers = "spacy.Corpus.v1"
 path = ${paths.dev}
 # Whether to train on sequences with 'gold standard' sentence boundaries

spacy/schemas.py

@@ -198,8 +198,7 @@ class ModelMetaSchema(BaseModel):
 class ConfigSchemaTraining(BaseModel):
     # fmt: off
     vectors: Optional[StrictStr] = Field(..., title="Path to vectors")
-    train_corpus: Reader = Field(..., title="Reader for the training data")
+    corpus: Reader = Field(..., title="Reader for the training and dev data")
-    dev_corpus: Reader = Field(..., title="Reader for the dev data")
     batcher: Batcher = Field(..., title="Batcher for the training data")
     dropout: StrictFloat = Field(..., title="Dropout rate")
     patience: StrictInt = Field(..., title="How many steps to continue without improvement in evaluation score")

spacy/tests/serialize/test_serialize_config.py

@@ -19,11 +19,13 @@ dev = ""
 
 [training]
 
-[training.train_corpus]
+[training.corpus]
+
+[training.corpus.train]
 @readers = "spacy.Corpus.v1"
 path = ${paths.train}
 
-[training.dev_corpus]
+[training.corpus.dev]
 @readers = "spacy.Corpus.v1"
 path = ${paths.dev}
 
@@ -300,20 +302,20 @@ def test_config_overrides():
 
 def test_config_interpolation():
     config = Config().from_str(nlp_config_string, interpolate=False)
-    assert config["training"]["train_corpus"]["path"] == "${paths.train}"
+    assert config["training"]["corpus"]["train"]["path"] == "${paths.train}"
     interpolated = config.interpolate()
-    assert interpolated["training"]["train_corpus"]["path"] == ""
+    assert interpolated["training"]["corpus"]["train"]["path"] == ""
     nlp = English.from_config(config)
-    assert nlp.config["training"]["train_corpus"]["path"] == "${paths.train}"
+    assert nlp.config["training"]["corpus"]["train"]["path"] == "${paths.train}"
     # Ensure that variables are preserved in nlp config
     width = "${components.tok2vec.model.width}"
     assert config["components"]["tagger"]["model"]["tok2vec"]["width"] == width
     assert nlp.config["components"]["tagger"]["model"]["tok2vec"]["width"] == width
     interpolated2 = nlp.config.interpolate()
-    assert interpolated2["training"]["train_corpus"]["path"] == ""
+    assert interpolated2["training"]["corpus"]["train"]["path"] == ""
     assert interpolated2["components"]["tagger"]["model"]["tok2vec"]["width"] == 342
     nlp2 = English.from_config(interpolated)
-    assert nlp2.config["training"]["train_corpus"]["path"] == ""
+    assert nlp2.config["training"]["corpus"]["train"]["path"] == ""
     assert nlp2.config["components"]["tagger"]["model"]["tok2vec"]["width"] == 342
 
 

website/docs/api/corpus.md

@@ -26,7 +26,7 @@ streaming.
 > [paths]
 > train = "corpus/train.spacy"
 >
-> [training.train_corpus]
+> [training.corpus.train]
 > @readers = "spacy.Corpus.v1"
 > path = ${paths.train}
 > gold_preproc = false

website/docs/api/data-formats.md

@@ -127,10 +127,10 @@ This section defines settings and controls for the training and evaluation
 process that are used when you run [`spacy train`](/api/cli#train).
 
 | Name                  | Description                                                                                                                                                                                                                                           |
-| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `accumulate_gradient` | Whether to divide the batch up into substeps. Defaults to `1`. ~~int~~                                                                                                                                                                                |
 | `batcher`             | Callable that takes an iterator of [`Doc`](/api/doc) objects and yields batches of `Doc`s. Defaults to [`batch_by_words`](/api/top-level#batch_by_words). ~~Callable[[Iterator[Doc], Iterator[List[Doc]]]]~~                                          |
-| `dev_corpus`          | Callable that takes the current `nlp` object and yields [`Example`](/api/example) objects. Defaults to [`Corpus`](/api/top-level#Corpus). ~~Callable[[Language], Iterator[Example]]~~                        |
+| `corpus`              | Dictionary with `train` and `develop` keys, each referring to a callable that takes the current `nlp` object and yields [`Example`](/api/example) objects. Defaults to [`Corpus`](/api/top-level#Corpus). ~~Callable[[Language], Iterator[Example]]~~ |
 | `dropout`             | The dropout rate. Defaults to `0.1`. ~~float~~                                                                                                                                                                                                        |
 | `eval_frequency`      | How often to evaluate during training (steps). Defaults to `200`. ~~int~~                                                                                                                                                                             |
 | `frozen_components`   | Pipeline component names that are "frozen" and shouldn't be updated during training. See [here](/usage/training#config-components) for details. Defaults to `[]`. ~~List[str]~~                                                                       |
@@ -142,7 +142,6 @@ process that are used when you run [`spacy train`](/api/cli#train).
 | `raw_text`            | Optional path to a jsonl file with unlabelled text documents for a [rehearsal](/api/language#rehearse) step. Defaults to variable `${paths.raw}`. ~~Optional[str]~~                                                                                   |
 | `score_weights`       | Score names shown in metrics mapped to their weight towards the final weighted score. See [here](/usage/training#metrics) for details. Defaults to `{}`. ~~Dict[str, float]~~                                                                         |
 | `seed`                | The random seed. Defaults to variable `${system.seed}`. ~~int~~                                                                                                                                                                                       |
-| `train_corpus`        | Callable that takes the current `nlp` object and yields [`Example`](/api/example) objects. Defaults to [`Corpus`](/api/top-level#Corpus). ~~Callable[[Language], Iterator[Example]]~~                        |
 | `vectors`             | Name or path of pipeline containing pretrained word vectors to use, e.g. created with [`init vocab`](/api/cli#init-vocab). Defaults to `null`. ~~Optional[str]~~                                                                                      |
 
 ### pretraining {#config-pretraining tag="section,optional"}

website/docs/api/top-level.md

@@ -448,7 +448,7 @@ remain in the config file stored on your local system.
 > [training.logger]
 > @loggers = "spacy.WandbLogger.v1"
 > project_name = "monitor_spacy_training"
-> remove_config_values = ["paths.train", "paths.dev", "training.dev_corpus.path", "training.train_corpus.path"]
+> remove_config_values = ["paths.train", "paths.dev", "training.corpus.train.path", "training.corpus.dev.path"]
 > ```
 
 | Name                   | Description                                                                                                                           |
@@ -478,7 +478,7 @@ the [`Corpus`](/api/corpus) class.
 > [paths]
 > train = "corpus/train.spacy"
 >
-> [training.train_corpus]
+> [training.corpus.train]
 > @readers = "spacy.Corpus.v1"
 > path = ${paths.train}
 > gold_preproc = false

website/docs/usage/projects.md

@@ -969,7 +969,7 @@ your results.
 > [training.logger]
 > @loggers = "spacy.WandbLogger.v1"
 > project_name = "monitor_spacy_training"
-> remove_config_values = ["paths.train", "paths.dev", "training.dev_corpus.path", "training.train_corpus.path"]
+> remove_config_values = ["paths.train", "paths.dev", "training.corpus.train.path", "training.corpus.dev.path"]
 > ```
 
 ![Screenshot: Visualized training results](../images/wandb1.jpg)

website/docs/usage/training.md

@@ -746,7 +746,7 @@ as **config settings** – in this case, `source`.
 > #### config.cfg
 >
 > ```ini
-> [training.train_corpus]
+> [training.corpus.train]
 > @readers = "corpus_variants.v1"
 > source = "s3://your_bucket/path/data.csv"
 > ```