Tidy up env vars [ci skip]

This commit is contained in:
Ines Montani 2020-09-30 15:15:11 +02:00
parent 469f0e539c
commit 23c63eefaf
3 changed files with 13 additions and 7 deletions

View File

@ -16,6 +16,7 @@ import os
from ..schemas import ProjectConfigSchema, validate from ..schemas import ProjectConfigSchema, validate
from ..util import import_file, run_command, make_tempdir, registry, logger from ..util import import_file, run_command, make_tempdir, registry, logger
from ..util import ENV_VARS
if TYPE_CHECKING: if TYPE_CHECKING:
from pathy import Pathy # noqa: F401 from pathy import Pathy # noqa: F401
@ -39,7 +40,6 @@ commands to check and validate your config files, training and evaluation data,
and custom model implementations. and custom model implementations.
""" """
INIT_HELP = """Commands for initializing configs and pipeline packages.""" INIT_HELP = """Commands for initializing configs and pipeline packages."""
OVERRIDES_ENV_VAR = "SPACY_CONFIG_OVERRIDES"
# Wrappers for Typer's annotations. Initially created to set defaults and to # Wrappers for Typer's annotations. Initially created to set defaults and to
# keep the names short, but not needed at the moment. # keep the names short, but not needed at the moment.
@ -65,7 +65,7 @@ def setup_cli() -> None:
def parse_config_overrides( def parse_config_overrides(
args: List[str], env_var: Optional[str] = OVERRIDES_ENV_VAR args: List[str], env_var: Optional[str] = ENV_VARS.CONFIG_OVERRIDES
) -> Dict[str, Any]: ) -> Dict[str, Any]:
"""Generate a dictionary of config overrides based on the extra arguments """Generate a dictionary of config overrides based on the extra arguments
provided on the CLI, e.g. --training.batch_size to override provided on the CLI, e.g. --training.batch_size to override

View File

@ -3,10 +3,11 @@ from click import NoSuchOption
from spacy.training import docs_to_json, offsets_to_biluo_tags from spacy.training import docs_to_json, offsets_to_biluo_tags
from spacy.training.converters import iob_to_docs, conll_ner_to_docs, conllu_to_docs from spacy.training.converters import iob_to_docs, conll_ner_to_docs, conllu_to_docs
from spacy.schemas import ProjectConfigSchema, RecommendationSchema, validate from spacy.schemas import ProjectConfigSchema, RecommendationSchema, validate
from spacy.util import ENV_VARS
from spacy.cli.init_config import init_config, RECOMMENDATIONS from spacy.cli.init_config import init_config, RECOMMENDATIONS
from spacy.cli._util import validate_project_commands, parse_config_overrides from spacy.cli._util import validate_project_commands, parse_config_overrides
from spacy.cli._util import load_project_config, substitute_project_variables from spacy.cli._util import load_project_config, substitute_project_variables
from spacy.cli._util import string_to_list, OVERRIDES_ENV_VAR from spacy.cli._util import string_to_list
from thinc.api import ConfigValidationError from thinc.api import ConfigValidationError
import srsly import srsly
import os import os
@ -342,21 +343,22 @@ def test_parse_config_overrides_invalid_2(args):
def test_parse_cli_overrides(): def test_parse_cli_overrides():
os.environ[OVERRIDES_ENV_VAR] = "--x.foo bar --x.bar=12 --x.baz false --y.foo=hello" overrides = "--x.foo bar --x.bar=12 --x.baz false --y.foo=hello"
os.environ[ENV_VARS.CONFIG_OVERRIDES] = overrides
result = parse_config_overrides([]) result = parse_config_overrides([])
assert len(result) == 4 assert len(result) == 4
assert result["x.foo"] == "bar" assert result["x.foo"] == "bar"
assert result["x.bar"] == 12 assert result["x.bar"] == 12
assert result["x.baz"] is False assert result["x.baz"] is False
assert result["y.foo"] == "hello" assert result["y.foo"] == "hello"
os.environ[OVERRIDES_ENV_VAR] = "--x" os.environ[ENV_VARS.CONFIG_OVERRIDES] = "--x"
assert parse_config_overrides([], env_var=None) == {} assert parse_config_overrides([], env_var=None) == {}
with pytest.raises(SystemExit): with pytest.raises(SystemExit):
parse_config_overrides([]) parse_config_overrides([])
os.environ[OVERRIDES_ENV_VAR] = "hello world" os.environ[ENV_VARS.CONFIG_OVERRIDES] = "hello world"
with pytest.raises(SystemExit): with pytest.raises(SystemExit):
parse_config_overrides([]) parse_config_overrides([])
del os.environ[OVERRIDES_ENV_VAR] del os.environ[ENV_VARS.CONFIG_OVERRIDES]
@pytest.mark.parametrize("lang", ["en", "nl"]) @pytest.mark.parametrize("lang", ["en", "nl"])

View File

@ -71,6 +71,10 @@ logging.basicConfig(format="%(message)s")
logger = logging.getLogger("spacy") logger = logging.getLogger("spacy")
class ENV_VARS:
CONFIG_OVERRIDES = "SPACY_CONFIG_OVERRIDES"
class registry(thinc.registry): class registry(thinc.registry):
languages = catalogue.create("spacy", "languages", entry_points=True) languages = catalogue.create("spacy", "languages", entry_points=True)
architectures = catalogue.create("spacy", "architectures", entry_points=True) architectures = catalogue.create("spacy", "architectures", entry_points=True)