mirror of
https://github.com/explosion/spaCy.git
synced 2025-01-28 02:04:07 +03:00
e559867605
* Add utils for working with remote storage * WIP add remote_cache for project * WIP add push and pull commands * Use pathy in remote_cache * Updarte util * Update remote_cache * Update util * Update project assets * Update pull script * Update push script * Fix type annotation in util * Work on remote storage * Remove site and env hash * Fix imports * Fix type annotation * Require pathy * Require pathy * Fix import * Add a util to handle project variable substitution * Import push and pull commands * Fix pull command * Fix push command * Fix tarfile in remote_storage * Improve printing * Fiddle with status messages * Set version to v3.0.0a9 * Draft docs for spacy project remote storages * Update docs [ci skip] * Use Thinc config to simplify and unify template variables * Auto-format * Don't import Pathy globally for now Causes slow and annoying Google Cloud warning * Tidy up test * Tidy up and update tests * Update to latest Thinc * Update docs * variables -> vars * Update docs [ci skip] * Update docs [ci skip] Co-authored-by: Ines Montani <ines@ines.io>
49 lines
2.0 KiB
Python
49 lines
2.0 KiB
Python
from pathlib import Path
|
|
from wasabi import msg
|
|
from .remote_storage import RemoteStorage
|
|
from .remote_storage import get_content_hash, get_command_hash
|
|
from .._util import load_project_config
|
|
from .._util import project_cli, Arg
|
|
|
|
|
|
@project_cli.command("push")
|
|
def project_push_cli(
|
|
# fmt: off
|
|
remote: str = Arg("default", help="Name or path of remote storage"),
|
|
project_dir: Path = Arg(Path.cwd(), help="Location of project directory. Defaults to current working directory.", exists=True, file_okay=False),
|
|
# fmt: on
|
|
):
|
|
"""Persist outputs to a remote storage. You can alias remotes in your project.yml
|
|
by mapping them to storage paths. A storage can be anything that the smart-open
|
|
library can upload to, e.g. gcs, aws, ssh, local directories etc
|
|
"""
|
|
for output_path, url in project_push(project_dir, remote):
|
|
if url is None:
|
|
msg.info(f"Skipping {output_path}")
|
|
else:
|
|
msg.good(f"Pushed {output_path} to {url}")
|
|
|
|
|
|
def project_push(project_dir: Path, remote: str):
|
|
"""Persist outputs to a remote storage. You can alias remotes in your project.yml
|
|
by mapping them to storage paths. A storage can be anything that the smart-open
|
|
library can upload to, e.g. gcs, aws, ssh, local directories etc
|
|
"""
|
|
config = load_project_config(project_dir)
|
|
if remote in config.get("remotes", {}):
|
|
remote = config["remotes"][remote]
|
|
storage = RemoteStorage(project_dir, remote)
|
|
for cmd in config.get("commands", []):
|
|
cmd_hash = get_command_hash(
|
|
"", "", [project_dir / dep for dep in cmd.get("deps", [])], cmd["script"]
|
|
)
|
|
for output_path in cmd.get("outputs", []):
|
|
output_loc = project_dir / output_path
|
|
if output_loc.exists():
|
|
url = storage.push(
|
|
output_path,
|
|
command_hash=cmd_hash,
|
|
content_hash=get_content_hash(output_loc),
|
|
)
|
|
yield output_path, url
|