diff --git a/spacy/about.py b/spacy/about.py
index 4fb6dfff1..ec3c168a5 100644
--- a/spacy/about.py
+++ b/spacy/about.py
@@ -1,6 +1,6 @@
# fmt: off
__title__ = "spacy-nightly"
-__version__ = "3.0.0a19"
+__version__ = "3.0.0a20"
__release__ = True
__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py
index c67863ef1..797a701b9 100644
--- a/spacy/cli/_util.py
+++ b/spacy/cli/_util.py
@@ -6,15 +6,16 @@ from wasabi import msg
import srsly
import hashlib
import typer
-import subprocess
from click import NoSuchOption
+from click.parser import split_arg_string
from typer.main import get_command
from contextlib import contextmanager
from thinc.config import Config, ConfigValidationError
from configparser import InterpolationError
+import os
from ..schemas import ProjectConfigSchema, validate
-from ..util import import_file, run_command, make_tempdir, registry
+from ..util import import_file, run_command, make_tempdir, registry, logger
if TYPE_CHECKING:
from pathy import Pathy # noqa: F401
@@ -38,6 +39,7 @@ commands to check and validate your config files, training and evaluation data,
and custom model implementations.
"""
INIT_HELP = """Commands for initializing configs and pipeline packages."""
+OVERRIDES_ENV_VAR = "SPACY_CONFIG_OVERRIDES"
# Wrappers for Typer's annotations. Initially created to set defaults and to
# keep the names short, but not needed at the moment.
@@ -62,24 +64,41 @@ def setup_cli() -> None:
command(prog_name=COMMAND)
-def parse_config_overrides(args: List[str]) -> Dict[str, Any]:
+def parse_config_overrides(
+ args: List[str], env_var: Optional[str] = OVERRIDES_ENV_VAR
+) -> Dict[str, Any]:
"""Generate a dictionary of config overrides based on the extra arguments
provided on the CLI, e.g. --training.batch_size to override
"training.batch_size". Arguments without a "." are considered invalid,
since the config only allows top-level sections to exist.
- args (List[str]): The extra arguments from the command line.
+ env_vars (Optional[str]): Optional environment variable to read from.
RETURNS (Dict[str, Any]): The parsed dict, keyed by nested config setting.
"""
+ env_string = os.environ.get(env_var, "") if env_var else ""
+ env_overrides = _parse_overrides(split_arg_string(env_string))
+ cli_overrides = _parse_overrides(args, is_cli=True)
+ if cli_overrides:
+ keys = [k for k in cli_overrides if k not in env_overrides]
+ logger.debug(f"Config overrides from CLI: {keys}")
+ if env_overrides:
+ logger.debug(f"Config overrides from env variables: {list(env_overrides)}")
+ return {**cli_overrides, **env_overrides}
+
+
+def _parse_overrides(args: List[str], is_cli: bool = False) -> Dict[str, Any]:
result = {}
while args:
opt = args.pop(0)
- err = f"Invalid CLI argument '{opt}'"
+ err = f"Invalid config override '{opt}'"
if opt.startswith("--"): # new argument
orig_opt = opt
opt = opt.replace("--", "")
if "." not in opt:
- raise NoSuchOption(orig_opt)
+ if is_cli:
+ raise NoSuchOption(orig_opt)
+ else:
+ msg.fail(f"{err}: can't override top-level sections", exits=1)
if "=" in opt: # we have --opt=value
opt, value = opt.split("=", 1)
opt = opt.replace("-", "_")
@@ -98,7 +117,7 @@ def parse_config_overrides(args: List[str]) -> Dict[str, Any]:
except ValueError:
result[opt] = str(value)
else:
- msg.fail(f"{err}: override option should start with --", exits=1)
+ msg.fail(f"{err}: name should start with --", exits=1)
return result
@@ -287,7 +306,7 @@ def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False)
if dest.exists() and not force:
return None
src = str(src)
- with smart_open.open(src, mode="rb") as input_file:
+ with smart_open.open(src, mode="rb", ignore_ext=True) as input_file:
with dest.open(mode="wb") as output_file:
output_file.write(input_file.read())
@@ -327,7 +346,7 @@ def git_checkout(
)
with make_tempdir() as tmp_dir:
cmd = f"git -C {tmp_dir} clone {repo} . -b {branch}"
- ret = run_command(cmd, capture=True)
+ run_command(cmd, capture=True)
# We need Path(name) to make sure we also support subdirectories
shutil.copytree(str(tmp_dir / Path(subpath)), str(dest))
diff --git a/spacy/errors.py b/spacy/errors.py
index 81e3616be..f219496a5 100644
--- a/spacy/errors.py
+++ b/spacy/errors.py
@@ -57,7 +57,10 @@ class Warnings:
"incorrect. Modify PhraseMatcher._terminal_hash to fix.")
W024 = ("Entity '{entity}' - Alias '{alias}' combination already exists in "
"the Knowledge Base.")
- W026 = ("Unable to set all sentence boundaries from dependency parses.")
+ W026 = ("Unable to set all sentence boundaries from dependency parses. If "
+ "you are constructing a parse tree incrementally by setting "
+ "token.head values, you can probably ignore this warning. Consider "
+ "using Doc(words, ..., heads=heads, deps=deps) instead.")
W027 = ("Found a large training file of {size} bytes. Note that it may "
"be more efficient to split your training data into multiple "
"smaller JSON files instead.")
diff --git a/spacy/language.py b/spacy/language.py
index 7d463731a..4dffd9679 100644
--- a/spacy/language.py
+++ b/spacy/language.py
@@ -156,11 +156,7 @@ class Language:
raise ValueError(Errors.E918.format(vocab=vocab, vocab_type=type(Vocab)))
if vocab is True:
vectors_name = meta.get("vectors", {}).get("name")
- vocab = create_vocab(
- self.lang,
- self.Defaults,
- vectors_name=vectors_name,
- )
+ vocab = create_vocab(self.lang, self.Defaults, vectors_name=vectors_name)
else:
if (self.lang and vocab.lang) and (self.lang != vocab.lang):
raise ValueError(Errors.E150.format(nlp=self.lang, vocab=vocab.lang))
@@ -1462,7 +1458,7 @@ class Language:
# here :(
for i, (name1, proc1) in enumerate(self.pipeline):
if hasattr(proc1, "find_listeners"):
- for name2, proc2 in self.pipeline[i+1:]:
+ for name2, proc2 in self.pipeline[i + 1 :]:
if isinstance(getattr(proc2, "model", None), Model):
proc1.find_listeners(proc2.model)
diff --git a/spacy/ml/models/tok2vec.py b/spacy/ml/models/tok2vec.py
index 7ced4bd04..fec478e21 100644
--- a/spacy/ml/models/tok2vec.py
+++ b/spacy/ml/models/tok2vec.py
@@ -164,7 +164,9 @@ def MultiHashEmbed(
@registry.architectures.register("spacy.CharacterEmbed.v1")
-def CharacterEmbed(width: int, rows: int, nM: int, nC: int, also_use_static_vectors: bool):
+def CharacterEmbed(
+ width: int, rows: int, nM: int, nC: int, also_use_static_vectors: bool
+):
"""Construct an embedded representation based on character embeddings, using
a feed-forward network. A fixed number of UTF-8 byte characters are used for
each word, taken from the beginning and end of the word equally. Padding is
@@ -202,9 +204,11 @@ def CharacterEmbed(width: int, rows: int, nM: int, nC: int, also_use_static_vect
),
StaticVectors(width, dropout=0.0),
),
- with_array(Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0)),
+ with_array(
+ Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0)
+ ),
ragged2list(),
- )
+ )
else:
model = chain(
concatenate(
@@ -215,9 +219,11 @@ def CharacterEmbed(width: int, rows: int, nM: int, nC: int, also_use_static_vect
with_array(HashEmbed(nO=width, nV=rows, column=0, seed=5)),
),
),
- with_array(Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0)),
+ with_array(
+ Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0)
+ ),
ragged2list(),
- )
+ )
return model
diff --git a/spacy/schemas.py b/spacy/schemas.py
index 60655da8c..b0f26dcd7 100644
--- a/spacy/schemas.py
+++ b/spacy/schemas.py
@@ -1,4 +1,4 @@
-from typing import Dict, List, Union, Optional, Sequence, Any, Callable, Type, Tuple
+from typing import Dict, List, Union, Optional, Any, Callable, Type, Tuple
from typing import Iterable, TypeVar, TYPE_CHECKING
from enum import Enum
from pydantic import BaseModel, Field, ValidationError, validator
@@ -255,7 +255,7 @@ class ConfigSchemaPretrain(BaseModel):
batcher: Batcher = Field(..., title="Batcher for the training data")
component: str = Field(..., title="Component to find the layer to pretrain")
layer: str = Field(..., title="Layer to pretrain. Whole model if empty.")
-
+
# TODO: use a more detailed schema for this?
objective: Dict[str, Any] = Field(..., title="Pretraining objective")
# fmt: on
diff --git a/spacy/tests/doc/test_doc_api.py b/spacy/tests/doc/test_doc_api.py
index c979931b1..0579642c4 100644
--- a/spacy/tests/doc/test_doc_api.py
+++ b/spacy/tests/doc/test_doc_api.py
@@ -9,6 +9,26 @@ from spacy.attrs import ENT_TYPE, ENT_IOB, SENT_START, HEAD, DEP, MORPH
from ..util import get_doc
+def test_doc_api_init(en_vocab):
+ # set sent_start by sent_starts
+ doc = Doc(
+ en_vocab, words=["a", "b", "c", "d"], sent_starts=[True, False, True, False]
+ )
+ assert [t.is_sent_start for t in doc] == [True, False, True, False]
+
+ # set sent_start by heads
+ doc = Doc(
+ en_vocab, words=["a", "b", "c", "d"], heads=[0, 0, 2, 2], deps=["dep"] * 4
+ )
+ assert [t.is_sent_start for t in doc] == [True, False, True, False]
+
+ # heads override sent_starts
+ doc = Doc(
+ en_vocab, words=["a", "b", "c", "d"], sent_starts=[True] * 4, heads=[0, 0, 2, 2], deps=["dep"] * 4
+ )
+ assert [t.is_sent_start for t in doc] == [True, False, True, False]
+
+
@pytest.mark.parametrize("text", [["one", "two", "three"]])
def test_doc_api_compare_by_string_position(en_vocab, text):
doc = Doc(en_vocab, words=text)
diff --git a/spacy/tests/doc/test_span.py b/spacy/tests/doc/test_span.py
index ad4f49042..0c538a0eb 100644
--- a/spacy/tests/doc/test_span.py
+++ b/spacy/tests/doc/test_span.py
@@ -92,7 +92,12 @@ def test_spans_span_sent(doc, doc_not_parsed):
def test_spans_lca_matrix(en_tokenizer):
"""Test span's lca matrix generation"""
tokens = en_tokenizer("the lazy dog slept")
- doc = get_doc(tokens.vocab, words=[t.text for t in tokens], heads=[2, 1, 1, 0], deps=["dep"] * 4)
+ doc = get_doc(
+ tokens.vocab,
+ words=[t.text for t in tokens],
+ heads=[2, 1, 1, 0],
+ deps=["dep"] * 4,
+ )
lca = doc[:2].get_lca_matrix()
assert lca.shape == (2, 2)
assert lca[0, 0] == 0 # the & the -> the
diff --git a/spacy/tests/parser/test_parse_navigate.py b/spacy/tests/parser/test_parse_navigate.py
index db1e98ba0..f181a799a 100644
--- a/spacy/tests/parser/test_parse_navigate.py
+++ b/spacy/tests/parser/test_parse_navigate.py
@@ -63,7 +63,12 @@ def test_parser_parse_navigate_consistency(en_tokenizer, text, heads):
def test_parser_parse_navigate_child_consistency(en_tokenizer, text, heads):
tokens = en_tokenizer(text)
- doc = get_doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=["dep"] * len(heads))
+ doc = get_doc(
+ tokens.vocab,
+ words=[t.text for t in tokens],
+ heads=heads,
+ deps=["dep"] * len(heads),
+ )
lefts = {}
rights = {}
diff --git a/spacy/tests/pipeline/test_pipe_factories.py b/spacy/tests/pipeline/test_pipe_factories.py
index 1cf06d97f..881460704 100644
--- a/spacy/tests/pipeline/test_pipe_factories.py
+++ b/spacy/tests/pipeline/test_pipe_factories.py
@@ -345,10 +345,7 @@ def test_language_factories_invalid():
[{"a": 100, "b": 400}, {"c": 0.5, "d": 0.5}],
{"a": 0.1, "b": 0.4, "c": 0.25, "d": 0.25},
),
- (
- [{"a": 0.5, "b": 0.5}, {"b": 1.0}],
- {"a": 0.25, "b": 0.75},
- ),
+ ([{"a": 0.5, "b": 0.5}, {"b": 1.0}], {"a": 0.25, "b": 0.75},),
],
)
def test_language_factories_combine_score_weights(weights, expected):
@@ -363,16 +360,10 @@ def test_language_factories_scores():
weights1 = {"a1": 0.5, "a2": 0.5}
weights2 = {"b1": 0.2, "b2": 0.7, "b3": 0.1}
Language.factory(
- f"{name}1",
- scores=list(weights1),
- default_score_weights=weights1,
- func=func,
+ f"{name}1", scores=list(weights1), default_score_weights=weights1, func=func,
)
Language.factory(
- f"{name}2",
- scores=list(weights2),
- default_score_weights=weights2,
- func=func,
+ f"{name}2", scores=list(weights2), default_score_weights=weights2, func=func,
)
meta1 = Language.get_factory_meta(f"{name}1")
assert meta1.default_score_weights == weights1
diff --git a/spacy/tests/regression/test_issue1501-2000.py b/spacy/tests/regression/test_issue1501-2000.py
index e226c8524..71ed2ea03 100644
--- a/spacy/tests/regression/test_issue1501-2000.py
+++ b/spacy/tests/regression/test_issue1501-2000.py
@@ -212,9 +212,17 @@ def test_issue1834():
heads=[0, -1, -2, -3, -4, -5, 0, -1, -2],
deps=["dep"] * len(words),
)
- print(doc.has_annotation("DEP"), [t.head.i for t in doc], [t.is_sent_start for t in doc])
+ print(
+ doc.has_annotation("DEP"),
+ [t.head.i for t in doc],
+ [t.is_sent_start for t in doc],
+ )
new_doc = Doc(doc.vocab).from_bytes(doc.to_bytes())
- print(new_doc.has_annotation("DEP"), [t.head.i for t in new_doc], [t.is_sent_start for t in new_doc])
+ print(
+ new_doc.has_annotation("DEP"),
+ [t.head.i for t in new_doc],
+ [t.is_sent_start for t in new_doc],
+ )
assert new_doc[6].sent_start
assert new_doc.has_annotation("DEP")
assert new_doc.has_annotation("TAG")
diff --git a/spacy/tests/serialize/test_serialize_pipeline.py b/spacy/tests/serialize/test_serialize_pipeline.py
index eedad31e0..d1c4553be 100644
--- a/spacy/tests/serialize/test_serialize_pipeline.py
+++ b/spacy/tests/serialize/test_serialize_pipeline.py
@@ -136,7 +136,13 @@ def test_serialize_textcat_empty(en_vocab):
# See issue #1105
cfg = {"model": DEFAULT_TEXTCAT_MODEL}
model = registry.make_from_config(cfg, validate=True)["model"]
- textcat = TextCategorizer(en_vocab, model, labels=["ENTITY", "ACTION", "MODIFIER"], threshold=0.5, positive_label=None)
+ textcat = TextCategorizer(
+ en_vocab,
+ model,
+ labels=["ENTITY", "ACTION", "MODIFIER"],
+ threshold=0.5,
+ positive_label=None,
+ )
textcat.to_bytes(exclude=["vocab"])
diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py
index 0a2300455..a9c9d8ca5 100644
--- a/spacy/tests/test_cli.py
+++ b/spacy/tests/test_cli.py
@@ -1,16 +1,15 @@
import pytest
from click import NoSuchOption
-
from spacy.training import docs_to_json, biluo_tags_from_offsets
from spacy.training.converters import iob2docs, conll_ner2docs, conllu2docs
-from spacy.lang.en import English
from spacy.schemas import ProjectConfigSchema, RecommendationSchema, validate
from spacy.cli.init_config import init_config, RECOMMENDATIONS
from spacy.cli._util import validate_project_commands, parse_config_overrides
from spacy.cli._util import load_project_config, substitute_project_variables
-from spacy.cli._util import string_to_list
+from spacy.cli._util import string_to_list, OVERRIDES_ENV_VAR
from thinc.config import ConfigValidationError
import srsly
+import os
from .util import make_tempdir
@@ -342,6 +341,24 @@ def test_parse_config_overrides_invalid_2(args):
parse_config_overrides(args)
+def test_parse_cli_overrides():
+ os.environ[OVERRIDES_ENV_VAR] = "--x.foo bar --x.bar=12 --x.baz false --y.foo=hello"
+ result = parse_config_overrides([])
+ assert len(result) == 4
+ assert result["x.foo"] == "bar"
+ assert result["x.bar"] == 12
+ assert result["x.baz"] is False
+ assert result["y.foo"] == "hello"
+ os.environ[OVERRIDES_ENV_VAR] = "--x"
+ assert parse_config_overrides([], env_var=None) == {}
+ with pytest.raises(SystemExit):
+ parse_config_overrides([])
+ os.environ[OVERRIDES_ENV_VAR] = "hello world"
+ with pytest.raises(SystemExit):
+ parse_config_overrides([])
+ del os.environ[OVERRIDES_ENV_VAR]
+
+
@pytest.mark.parametrize("lang", ["en", "nl"])
@pytest.mark.parametrize(
"pipeline", [["tagger", "parser", "ner"], [], ["ner", "textcat", "sentencizer"]]
diff --git a/spacy/tests/test_language.py b/spacy/tests/test_language.py
index 2a24d368a..da46ad424 100644
--- a/spacy/tests/test_language.py
+++ b/spacy/tests/test_language.py
@@ -291,8 +291,7 @@ def test_spacy_blank():
@pytest.mark.parametrize(
- "value",
- [False, None, ["x", "y"], Language, Vocab],
+ "value", [False, None, ["x", "y"], Language, Vocab],
)
def test_language_init_invalid_vocab(value):
err_fragment = "invalid value"
diff --git a/spacy/tests/test_util.py b/spacy/tests/test_util.py
index 8c931d31e..1668991cd 100644
--- a/spacy/tests/test_util.py
+++ b/spacy/tests/test_util.py
@@ -95,7 +95,7 @@ def test_util_dot_section():
assert en_nlp.get_pipe("textcat").model.attrs["multi_label"] is False
# Test that default values got overwritten
assert en_config["nlp"]["pipeline"] == ["textcat"]
- assert nl_config["nlp"]["pipeline"] == [] # default value []
+ assert nl_config["nlp"]["pipeline"] == [] # default value []
# Test proper functioning of 'dot_to_object'
with pytest.raises(KeyError):
dot_to_object(en_config, "nlp.pipeline.tagger")
diff --git a/spacy/tests/training/test_readers.py b/spacy/tests/training/test_readers.py
index 898746c2a..d20a032e8 100644
--- a/spacy/tests/training/test_readers.py
+++ b/spacy/tests/training/test_readers.py
@@ -1,7 +1,6 @@
from typing import Dict, Iterable, Callable
import pytest
from thinc.api import Config
-
from spacy import Language
from spacy.util import load_model_from_config, registry, dot_to_object
from spacy.training import Example
@@ -10,19 +9,19 @@ from spacy.training import Example
def test_readers():
config_string = """
[training]
-
+
[corpora]
@readers = "myreader.v1"
[nlp]
lang = "en"
pipeline = ["tok2vec", "textcat"]
-
+
[components]
-
+
[components.tok2vec]
factory = "tok2vec"
-
+
[components.textcat]
factory = "textcat"
"""
@@ -69,19 +68,19 @@ def test_readers():
def test_cat_readers(reader, additional_config):
nlp_config_string = """
[training]
-
+
[corpora]
@readers = "PLACEHOLDER"
[nlp]
lang = "en"
pipeline = ["tok2vec", "textcat"]
-
+
[components]
-
+
[components.tok2vec]
factory = "tok2vec"
-
+
[components.textcat]
factory = "textcat"
"""
diff --git a/spacy/tests/training/test_training.py b/spacy/tests/training/test_training.py
index 1d3c72a8b..b09487965 100644
--- a/spacy/tests/training/test_training.py
+++ b/spacy/tests/training/test_training.py
@@ -34,7 +34,17 @@ def doc():
# fmt: on
nlp = English()
words = [t.text for t in nlp.make_doc(text)]
- doc = get_doc(nlp.vocab, words=words, tags=tags, pos=pos, morphs=morphs, heads=heads, deps=deps, lemmas=lemmas, ents=ents)
+ doc = get_doc(
+ nlp.vocab,
+ words=words,
+ tags=tags,
+ pos=pos,
+ morphs=morphs,
+ heads=heads,
+ deps=deps,
+ lemmas=lemmas,
+ ents=ents,
+ )
doc.cats = cats
return doc
diff --git a/spacy/tests/util.py b/spacy/tests/util.py
index 741753c89..6c67d2ee1 100644
--- a/spacy/tests/util.py
+++ b/spacy/tests/util.py
@@ -30,60 +30,21 @@ def get_doc(
morphs=None,
):
"""Create Doc object from given vocab, words and annotations."""
- if deps and not heads:
- heads = [0] * len(deps)
- headings = []
- values = []
- annotations = [pos, heads, deps, lemmas, tags, morphs]
- possible_headings = [POS, HEAD, DEP, LEMMA, TAG, MORPH]
- for a, annot in enumerate(annotations):
- if annot is not None:
- if len(annot) != len(words):
- raise ValueError(Errors.E189)
- headings.append(possible_headings[a])
- if annot is not heads:
- values.extend(annot)
- for value in values:
- vocab.strings.add(value)
-
- doc = Doc(vocab, words=words)
-
- # if there are any other annotations, set them
- if headings:
- attrs = doc.to_array(headings)
-
- j = 0
- for annot in annotations:
- if annot:
- if annot is heads:
- for i in range(len(words)):
- if attrs.ndim == 1:
- attrs[i] = heads[i]
- else:
- attrs[i, j] = heads[i]
- elif annot is morphs:
- for i in range(len(words)):
- morph_key = vocab.morphology.add(morphs[i])
- if attrs.ndim == 1:
- attrs[i] = morph_key
- else:
- attrs[i, j] = morph_key
- else:
- for i in range(len(words)):
- if attrs.ndim == 1:
- attrs[i] = doc.vocab.strings[annot[i]]
- else:
- attrs[i, j] = doc.vocab.strings[annot[i]]
- j += 1
- doc.from_array(headings, attrs)
-
- # finally, set the entities
- if ents:
- doc.ents = [
- Span(doc, start, end, label=doc.vocab.strings[label])
- for start, end, label in ents
- ]
- return doc
+ if heads is not None:
+ heads = [i + head for i, head in enumerate(heads)]
+ if ents is not None:
+ ents = [(vocab.strings[ent_type], start, end) for start, end, ent_type in ents]
+ return Doc(
+ vocab,
+ words=words,
+ pos=pos,
+ heads=heads,
+ deps=deps,
+ tags=tags,
+ ents=ents,
+ lemmas=lemmas,
+ morphs=morphs,
+ )
def get_batch(batch_size):
diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx
index 2d9de278b..c5f1f6801 100644
--- a/spacy/tokens/doc.pyx
+++ b/spacy/tokens/doc.pyx
@@ -158,17 +158,50 @@ cdef class Doc:
raise ValueError(Errors.E046.format(name=name))
return Underscore.doc_extensions.pop(name)
- def __init__(self, Vocab vocab, words=None, spaces=None, user_data=None):
+ def __init__(
+ self,
+ Vocab vocab,
+ words=None,
+ spaces=None,
+ *,
+ user_data=None,
+ tags=None,
+ pos=None,
+ morphs=None,
+ lemmas=None,
+ heads=None,
+ deps=None,
+ sent_starts=None,
+ ents=None,
+ ):
"""Create a Doc object.
vocab (Vocab): A vocabulary object, which must match any models you
want to use (e.g. tokenizer, parser, entity recognizer).
- words (list or None): A list of unicode strings to add to the document
+ words (Optional[List[str]]): A list of unicode strings to add to the document
as words. If `None`, defaults to empty list.
- spaces (list or None): A list of boolean values, of the same length as
+ spaces (Optional[List[bool]]): A list of boolean values, of the same length as
words. True means that the word is followed by a space, False means
it is not. If `None`, defaults to `[True]*len(words)`
user_data (dict or None): Optional extra data to attach to the Doc.
+ tags (Optional[List[str]]): A list of unicode strings, of the same
+ length as words, to assign as token.tag. Defaults to None.
+ pos (Optional[List[str]]): A list of unicode strings, of the same
+ length as words, to assign as token.pos. Defaults to None.
+ morphs (Optional[List[str]]): A list of unicode strings, of the same
+ length as words, to assign as token.morph. Defaults to None.
+ lemmas (Optional[List[str]]): A list of unicode strings, of the same
+ length as words, to assign as token.lemma. Defaults to None.
+ heads (Optional[List[int]]): A list of values, of the same length as
+ words, to assign as heads. Head indices are the position of the
+ head in the doc. Defaults to None.
+ deps (Optional[List[str]]): A list of unicode strings, of the same
+ length as words, to assign as token.dep. Defaults to None.
+ sent_starts (Optional[List[Union[bool, None]]]): A list of values, of
+ the same length as words, to assign as token.is_sent_start. Will be
+ overridden by heads if heads is provided. Defaults to None.
+ ents (Optional[List[Span]]): A list of spans to assign as doc.ents.
+ Defaults to None.
DOCS: https://nightly.spacy.io/api/doc#init
"""
@@ -217,6 +250,63 @@ cdef class Doc:
lexeme = self.vocab.get_by_orth(self.mem, word)
self.push_back(lexeme, has_space)
+ if heads is not None:
+ heads = [head - i for i, head in enumerate(heads)]
+ if deps and not heads:
+ heads = [0] * len(deps)
+ if sent_starts is not None:
+ for i in range(len(sent_starts)):
+ if sent_starts[i] is True:
+ sent_starts[i] = 1
+ elif sent_starts[i] is False:
+ sent_starts[i] = -1
+ elif sent_starts[i] is None or sent_starts[i] not in [-1, 0, 1]:
+ sent_starts[i] = 0
+ headings = []
+ values = []
+ annotations = [pos, heads, deps, lemmas, tags, morphs, sent_starts]
+ possible_headings = [POS, HEAD, DEP, LEMMA, TAG, MORPH, SENT_START]
+ for a, annot in enumerate(annotations):
+ if annot is not None:
+ if len(annot) != len(words):
+ raise ValueError(Errors.E189)
+ headings.append(possible_headings[a])
+ if annot is not heads and annot is not sent_starts:
+ values.extend(annot)
+ for value in values:
+ self.vocab.strings.add(value)
+
+ # if there are any other annotations, set them
+ if headings:
+ attrs = self.to_array(headings)
+
+ j = 0
+ for annot in annotations:
+ if annot:
+ if annot is heads or annot is sent_starts:
+ for i in range(len(words)):
+ if attrs.ndim == 1:
+ attrs[i] = annot[i]
+ else:
+ attrs[i, j] = annot[i]
+ elif annot is morphs:
+ for i in range(len(words)):
+ morph_key = vocab.morphology.add(morphs[i])
+ if attrs.ndim == 1:
+ attrs[i] = morph_key
+ else:
+ attrs[i, j] = morph_key
+ else:
+ for i in range(len(words)):
+ if attrs.ndim == 1:
+ attrs[i] = self.vocab.strings[annot[i]]
+ else:
+ attrs[i, j] = self.vocab.strings[annot[i]]
+ j += 1
+ self.from_array(headings, attrs)
+ if ents is not None:
+ self.ents = ents
+
@property
def _(self):
"""Custom extension attributes registered via `set_extension`."""
diff --git a/spacy/training/converters/conllu2docs.py b/spacy/training/converters/conllu2docs.py
index ebd123375..b4d8b3ac4 100644
--- a/spacy/training/converters/conllu2docs.py
+++ b/spacy/training/converters/conllu2docs.py
@@ -199,13 +199,17 @@ def doc_from_conllu_sentence(
heads.append(head)
deps.append(dep)
- doc = Doc(vocab, words=words, spaces=spaces)
+ doc = Doc(
+ vocab,
+ words=words,
+ spaces=spaces,
+ tags=tags,
+ pos=poses,
+ deps=deps,
+ lemmas=lemmas,
+ heads=heads,
+ )
for i in range(len(doc)):
- doc[i].tag_ = tags[i]
- doc[i].pos_ = poses[i]
- doc[i].dep_ = deps[i]
- doc[i].lemma_ = lemmas[i]
- doc[i].head = doc[heads[i]]
doc[i]._.merged_orth = words[i]
doc[i]._.merged_morph = morphs[i]
doc[i]._.merged_lemma = lemmas[i]
@@ -232,14 +236,17 @@ def doc_from_conllu_sentence(
heads.append(t.head.i)
deps.append(t.dep_)
- doc_x = Doc(vocab, words=words, spaces=spaces)
- for i in range(len(doc)):
- doc_x[i].tag_ = tags[i]
- doc_x[i].morph_ = morphs[i]
- doc_x[i].lemma_ = lemmas[i]
- doc_x[i].pos_ = poses[i]
- doc_x[i].dep_ = deps[i]
- doc_x[i].head = doc_x[heads[i]]
+ doc_x = Doc(
+ vocab,
+ words=words,
+ spaces=spaces,
+ tags=tags,
+ morphs=morphs,
+ lemmas=lemmas,
+ pos=poses,
+ deps=deps,
+ heads=heads,
+ )
doc_x.ents = [Span(doc_x, ent.start, ent.end, label=ent.label) for ent in doc.ents]
return doc_x
diff --git a/website/docs/api/doc.md b/website/docs/api/doc.md
index 44316ea1e..455465aa0 100644
--- a/website/docs/api/doc.md
+++ b/website/docs/api/doc.md
@@ -30,11 +30,21 @@ Construct a `Doc` object. The most common way to get a `Doc` object is via the
> doc = Doc(nlp.vocab, words=words, spaces=spaces)
> ```
-| Name | Description |
-| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `vocab` | A storage container for lexical types. ~~Vocab~~ |
-| `words` | A list of strings to add to the container. ~~Optional[List[str]]~~ |
-| `spaces` | A list of boolean values indicating whether each word has a subsequent space. Must have the same length as `words`, if specified. Defaults to a sequence of `True`. ~~Optional[List[bool]]~~ |
+| Name | Description |
+| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `vocab` | A storage container for lexical types. ~~Vocab~~ |
+| `words` | A list of strings to add to the container. ~~Optional[List[str]]~~ |
+| `spaces` | A list of boolean values indicating whether each word has a subsequent space. Must have the same length as `words`, if specified. Defaults to a sequence of `True`. ~~Optional[List[bool]]~~ |
+| _keyword-only_ | |
+| `user\_data` | Optional extra data to attach to the Doc. ~~Dict~~ |
+| tags | A list of strings, of the same length as words, to assign as `token.tag` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
+| pos | A list of strings, of the same length as words, to assign as `token.pos` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
+| morphs | A list of strings, of the same length as words, to assign as `token.morph` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
+| lemmas | A list of strings, of the same length as words, to assign as `token.lemma` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
+| heads | A list of values, of the same length as words, to assign as the head for each word. Head indices are the absolute position of the head in the doc. Defaults to `None`. ~~Optional[List[int]]~~ |
+| deps | A list of strings, of the same length as words, to assign as `token.dep` for each word. Defaults to `None`. ~~Optional[List[str]]~~ |
+| sent_starts | A list of values, of the same length as words, to assign as token.is_sent_start. Will be overridden by heads if heads is provided. Defaults to `None`. ~~Optional[List[Union[bool, None]]~~ |
+| ents | A list of spans to assign as doc.ents. Defaults to `None`. ~~Optional[List[Span]]~~ |
## Doc.\_\_getitem\_\_ {#getitem tag="method"}
diff --git a/website/docs/usage/projects.md b/website/docs/usage/projects.md
index f8d5a3761..95e20525a 100644
--- a/website/docs/usage/projects.md
+++ b/website/docs/usage/projects.md
@@ -921,6 +921,14 @@ package is installed in the same environment as spaCy, it will automatically add
[parallel training](/usage/training#parallel-training) for more details on how
it works under the hood.
+
+
+Get started with parallel training using our project template. It trains a
+simple model on a Universal Dependencies Treebank and lets you parallelize the
+training with Ray.
+
+
+
You can integrate [`spacy ray train`](/api/cli#ray-train) into your
`project.yml` just like the regular training command and pass it the config, and
optional output directory or remote storage URL and config overrides if needed.
@@ -940,10 +948,6 @@ commands:
- "training/model-best"
```
-
-
---
### Weights & Biases {#wandb}
diff --git a/website/docs/usage/training.md b/website/docs/usage/training.md
index 6e9de62c5..b63145636 100644
--- a/website/docs/usage/training.md
+++ b/website/docs/usage/training.md
@@ -214,6 +214,24 @@ overrides. Overrides are added before [variables](#config-interpolation) are
resolved, by the way – so if you need to use a value in multiple places,
reference it across your config and override it on the CLI once.
+> #### 💡 Tip: Verbose logging
+>
+> If you're using config overrides, you can set the `--verbose` flag on
+> [`spacy train`](/api/cli#train) to make spaCy log more info, including which
+> overrides were set via the CLI and environment variables.
+
+#### Adding overrides via environment variables {#config-overrides-env}
+
+Instead of defining the overrides as CLI arguments, you can also use the
+`SPACY_CONFIG_OVERRIDES` environment variable using the same argument syntax.
+This is especially useful if you're training models as part of an automated
+process. Environment variables **take precedence** over CLI overrides and values
+defined in the config file.
+
+```cli
+$ SPACY_CONFIG_OVERRIDES="--system.gpu_allocator pytorch --training.batch_size 128" ./your_script.sh
+```
+
### Defining pipeline components {#config-components}
You typically train a [pipeline](/usage/processing-pipelines) of **one or more
@@ -895,9 +913,13 @@ cluster. If it's not set, Ray will run locally.
python -m spacy ray train config.cfg --n-workers 2
```
-
+Get started with parallel training using our project template. It trains a
+simple model on a Universal Dependencies Treebank and lets you parallelize the
+training with Ray.
+
+
### How parallel training works {#parallel-training-details}
diff --git a/website/meta/sidebars.json b/website/meta/sidebars.json
index 76d5e63d6..e27817c92 100644
--- a/website/meta/sidebars.json
+++ b/website/meta/sidebars.json
@@ -75,63 +75,63 @@
{
"label": "Containers",
"items": [
- { "text": "Language", "url": "/api/language" },
{ "text": "Doc", "url": "/api/doc" },
- { "text": "Token", "url": "/api/token" },
- { "text": "Span", "url": "/api/span" },
- { "text": "Lexeme", "url": "/api/lexeme" },
+ { "text": "DocBin", "url": "/api/docbin" },
{ "text": "Example", "url": "/api/example" },
- { "text": "DocBin", "url": "/api/docbin" }
+ { "text": "Language", "url": "/api/language" },
+ { "text": "Lexeme", "url": "/api/lexeme" },
+ { "text": "Span", "url": "/api/span" },
+ { "text": "Token", "url": "/api/token" }
]
},
{
"label": "Pipeline",
"items": [
- { "text": "Tokenizer", "url": "/api/tokenizer" },
- { "text": "Tok2Vec", "url": "/api/tok2vec" },
- { "text": "Transformer", "url": "/api/transformer" },
- { "text": "Lemmatizer", "url": "/api/lemmatizer" },
- { "text": "Morphologizer", "url": "/api/morphologizer" },
- { "text": "Tagger", "url": "/api/tagger" },
{ "text": "AttributeRuler", "url": "/api/attributeruler" },
{ "text": "DependencyParser", "url": "/api/dependencyparser" },
+ { "text": "EntityLinker", "url": "/api/entitylinker" },
{ "text": "EntityRecognizer", "url": "/api/entityrecognizer" },
{ "text": "EntityRuler", "url": "/api/entityruler" },
- { "text": "EntityLinker", "url": "/api/entitylinker" },
- { "text": "TextCategorizer", "url": "/api/textcategorizer" },
- { "text": "Sentencizer", "url": "/api/sentencizer" },
+ { "text": "Lemmatizer", "url": "/api/lemmatizer" },
+ { "text": "Morphologizer", "url": "/api/morphologizer" },
+ { "text": "Pipe", "url": "/api/pipe" },
{ "text": "SentenceRecognizer", "url": "/api/sentencerecognizer" },
- { "text": "Other Functions", "url": "/api/pipeline-functions" },
- { "text": "Pipe", "url": "/api/pipe" }
+ { "text": "Sentencizer", "url": "/api/sentencizer" },
+ { "text": "Tagger", "url": "/api/tagger" },
+ { "text": "TextCategorizer", "url": "/api/textcategorizer" },
+ { "text": "Tok2Vec", "url": "/api/tok2vec" },
+ { "text": "Tokenizer", "url": "/api/tokenizer" },
+ { "text": "Transformer", "url": "/api/transformer" },
+ { "text": "Other Functions", "url": "/api/pipeline-functions" }
]
},
{
"label": "Matchers",
"items": [
+ { "text": "DependencyMatcher", "url": "/api/dependencymatcher" },
{ "text": "Matcher", "url": "/api/matcher" },
- { "text": "PhraseMatcher", "url": "/api/phrasematcher" },
- { "text": "DependencyMatcher", "url": "/api/dependencymatcher" }
+ { "text": "PhraseMatcher", "url": "/api/phrasematcher" }
]
},
{
"label": "Other",
"items": [
- { "text": "Vocab", "url": "/api/vocab" },
- { "text": "StringStore", "url": "/api/stringstore" },
- { "text": "Vectors", "url": "/api/vectors" },
+ { "text": "Corpus", "url": "/api/corpus" },
+ { "text": "KnowledgeBase", "url": "/api/kb" },
{ "text": "Lookups", "url": "/api/lookups" },
{ "text": "Morphology", "url": "/api/morphology" },
- { "text": "KnowledgeBase", "url": "/api/kb" },
{ "text": "Scorer", "url": "/api/scorer" },
- { "text": "Corpus", "url": "/api/corpus" }
+ { "text": "StringStore", "url": "/api/stringstore" },
+ { "text": "Vectors", "url": "/api/vectors" },
+ { "text": "Vocab", "url": "/api/vocab" }
]
},
{
"label": "Cython",
"items": [
{ "text": "Architecture", "url": "/api/cython" },
- { "text": "Structs", "url": "/api/cython-structs" },
- { "text": "Classes", "url": "/api/cython-classes" }
+ { "text": "Classes", "url": "/api/cython-classes" },
+ { "text": "Structs", "url": "/api/cython-structs" }
]
}
]