mirror of
https://github.com/explosion/spaCy.git
synced 2025-08-07 21:54:54 +03:00
Fix flag handling in dvc
Prior to this commit, if a flag (--verbose or --quiet) was passed to DVC, it would be added to the end of the generated dvc command line. This would result in the command being interpreted as part of the actual command to run, rather than an argument to dvc. This would result in command lines like: spacy project run preprocess --verbose That would fail with an error that there's no such directory as `--verbose`. This change puts the flags at the front of the dvc command so that they are interpreted correctly. It removes the `run_dvc_commands` function, which had been reduced to just a for loop and wasn't used elsewhere. A separate problem is that there's no way to specify the quiet behaviour to dvc from the command line, though it's unclear if that's a bug.
This commit is contained in:
parent
0b6f0643eb
commit
e13e11e1d8
|
@ -105,6 +105,14 @@ def update_dvc_config(
|
||||||
dvc_config_path.unlink()
|
dvc_config_path.unlink()
|
||||||
dvc_commands = []
|
dvc_commands = []
|
||||||
config_commands = {cmd["name"]: cmd for cmd in config.get("commands", [])}
|
config_commands = {cmd["name"]: cmd for cmd in config.get("commands", [])}
|
||||||
|
|
||||||
|
# some flags that apply to every command
|
||||||
|
flags = []
|
||||||
|
if verbose:
|
||||||
|
flags.append("--verbose")
|
||||||
|
if silent:
|
||||||
|
flags.append("--quiet")
|
||||||
|
|
||||||
for name in workflows[workflow]:
|
for name in workflows[workflow]:
|
||||||
command = config_commands[name]
|
command = config_commands[name]
|
||||||
deps = command.get("deps", [])
|
deps = command.get("deps", [])
|
||||||
|
@ -118,14 +126,16 @@ def update_dvc_config(
|
||||||
deps_cmd = [c for cl in [["-d", p] for p in deps] for c in cl]
|
deps_cmd = [c for cl in [["-d", p] for p in deps] for c in cl]
|
||||||
outputs_cmd = [c for cl in [["-o", p] for p in outputs] for c in cl]
|
outputs_cmd = [c for cl in [["-o", p] for p in outputs] for c in cl]
|
||||||
outputs_nc_cmd = [c for cl in [["-O", p] for p in outputs_no_cache] for c in cl]
|
outputs_nc_cmd = [c for cl in [["-O", p] for p in outputs_no_cache] for c in cl]
|
||||||
dvc_cmd = ["run", "-n", name, "-w", str(path), "--no-exec"]
|
|
||||||
|
dvc_cmd = ["run", *flags, "-n", name, "-w", str(path), "--no-exec"]
|
||||||
if command.get("no_skip"):
|
if command.get("no_skip"):
|
||||||
dvc_cmd.append("--always-changed")
|
dvc_cmd.append("--always-changed")
|
||||||
full_cmd = [*dvc_cmd, *deps_cmd, *outputs_cmd, *outputs_nc_cmd, *project_cmd]
|
full_cmd = [*dvc_cmd, *deps_cmd, *outputs_cmd, *outputs_nc_cmd, *project_cmd]
|
||||||
dvc_commands.append(full_cmd)
|
dvc_commands.append(full_cmd)
|
||||||
with working_dir(path):
|
with working_dir(path):
|
||||||
dvc_flags = {"--verbose": verbose, "--quiet": silent}
|
for c in dvc_commands:
|
||||||
run_dvc_commands(dvc_commands, flags=dvc_flags)
|
dvc_command = ["dvc", *c]
|
||||||
|
run_command(dvc_command)
|
||||||
with dvc_config_path.open("r+", encoding="utf8") as f:
|
with dvc_config_path.open("r+", encoding="utf8") as f:
|
||||||
content = f.read()
|
content = f.read()
|
||||||
f.seek(0, 0)
|
f.seek(0, 0)
|
||||||
|
@ -133,25 +143,6 @@ def update_dvc_config(
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def run_dvc_commands(
|
|
||||||
commands: Iterable[List[str]] = SimpleFrozenList(), flags: Dict[str, bool] = {}
|
|
||||||
) -> None:
|
|
||||||
"""Run a sequence of DVC commands in a subprocess, in order.
|
|
||||||
|
|
||||||
commands (List[List[str]]): The string commands without the leading "dvc".
|
|
||||||
flags (Dict[str, bool]): Conditional flags to be added to command. Makes it
|
|
||||||
easier to pass flags like --quiet that depend on a variable or
|
|
||||||
command-line setting while avoiding lots of nested conditionals.
|
|
||||||
"""
|
|
||||||
for c in commands:
|
|
||||||
dvc_command = ["dvc", *c]
|
|
||||||
# Add the flags if they are set to True
|
|
||||||
for flag, is_active in flags.items():
|
|
||||||
if is_active:
|
|
||||||
dvc_command.append(flag)
|
|
||||||
run_command(dvc_command)
|
|
||||||
|
|
||||||
|
|
||||||
def check_workflows(workflows: List[str], workflow: Optional[str] = None) -> None:
|
def check_workflows(workflows: List[str], workflow: Optional[str] = None) -> None:
|
||||||
"""Validate workflows provided in project.yml and check that a given
|
"""Validate workflows provided in project.yml and check that a given
|
||||||
workflow can be used to generate a DVC config.
|
workflow can be used to generate a DVC config.
|
||||||
|
|
Loading…
Reference in New Issue
Block a user