mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 19:39:13 +03:00
37c7c85a86
* Support nowrap setting in util.prints * Tidy up and fix whitespace * Simplify script and use read_jsonl helper * Add JSON schemas (see #2928) * Deprecate Doc.print_tree Will be replaced with Doc.to_json, which will produce a unified format * Add Doc.to_json() method (see #2928) Converts Doc objects to JSON using the same unified format as the training data. Method also supports serializing selected custom attributes in the doc._. space. * Remove outdated test * Add write_json and write_jsonl helpers * WIP: Update spacy train * Tidy up spacy train * WIP: Use wasabi for formatting * Add GoldParse helpers for JSON format * WIP: add debug-data command * Fix typo * Add missing import * Update wasabi pin * Add missing import * 💫 Refactor CLI (#2943) To be merged into #2932. ## Description - [x] refactor CLI To use [`wasabi`](https://github.com/ines/wasabi) - [x] use [`black`](https://github.com/ambv/black) for auto-formatting - [x] add `flake8` config - [x] move all messy UD-related scripts to `cli.ud` - [x] make converters function that take the opened file and return the converted data (instead of having them handle the IO) ### Types of change enhancement ## Checklist <!--- Before you submit the PR, go over this checklist and make sure you can tick off all the boxes. [] -> [x] --> - [x] I have submitted the spaCy Contributor Agreement. - [x] I ran the tests, and all new and existing tests passed. - [x] My changes don't require a change to the documentation, or if they do, I've added all required information. * Update wasabi pin * Delete old test * Update errors * Fix typo * Tidy up and format remaining code * Fix formatting * Improve formatting of messages * Auto-format remaining code * Add tok2vec stuff to spacy.train * Fix typo * Update wasabi pin * Fix path checks for when train() is called as function * Reformat and tidy up pretrain script * Update argument annotations * Raise error if model language doesn't match lang * Document new train command
95 lines
3.2 KiB
Python
95 lines
3.2 KiB
Python
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
import plac
|
|
import requests
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from wasabi import Printer
|
|
|
|
from ._messages import Messages
|
|
from .link import link
|
|
from ..util import get_package_path
|
|
from .. import about
|
|
|
|
|
|
msg = Printer()
|
|
|
|
|
|
@plac.annotations(
|
|
model=("Model to download (shortcut or name)", "positional", None, str),
|
|
direct=("Force direct download of name + version", "flag", "d", bool),
|
|
pip_args=("additional arguments to be passed to `pip install` on model install"),
|
|
)
|
|
def download(model, direct=False, *pip_args):
|
|
"""
|
|
Download compatible model from default download path using pip. Model
|
|
can be shortcut, model name or, if --direct flag is set, full model name
|
|
with version. For direct downloads, the compatibility check will be skipped.
|
|
"""
|
|
if direct:
|
|
dl = download_model("{m}/{m}.tar.gz#egg={m}".format(m=model), pip_args)
|
|
else:
|
|
shortcuts = get_json(about.__shortcuts__, "available shortcuts")
|
|
model_name = shortcuts.get(model, model)
|
|
compatibility = get_compatibility()
|
|
version = get_version(model_name, compatibility)
|
|
dl_tpl = "{m}-{v}/{m}-{v}.tar.gz#egg={m}=={v}"
|
|
dl = download_model(dl_tpl.format(m=model_name, v=version), pip_args)
|
|
if dl != 0: # if download subprocess doesn't return 0, exit
|
|
sys.exit(dl)
|
|
try:
|
|
# Get package path here because link uses
|
|
# pip.get_installed_distributions() to check if model is a
|
|
# package, which fails if model was just installed via
|
|
# subprocess
|
|
package_path = get_package_path(model_name)
|
|
link(model_name, model, force=True, model_path=package_path)
|
|
except: # noqa: E722
|
|
# Dirty, but since spacy.download and the auto-linking is
|
|
# mostly a convenience wrapper, it's best to show a success
|
|
# message and loading instructions, even if linking fails.
|
|
msg.warn(Messages.M002.format(name=model_name), Messages.M001)
|
|
|
|
|
|
def get_json(url, desc):
|
|
r = requests.get(url)
|
|
if r.status_code != 200:
|
|
msg.fail(
|
|
Messages.M003.format(code=r.status_code),
|
|
Messages.M004.format(desc=desc, version=about.__version__),
|
|
exits=1,
|
|
)
|
|
return r.json()
|
|
|
|
|
|
def get_compatibility():
|
|
version = about.__version__
|
|
version = version.rsplit(".dev", 1)[0]
|
|
comp_table = get_json(about.__compatibility__, "compatibility table")
|
|
comp = comp_table["spacy"]
|
|
if version not in comp:
|
|
msg.fail(Messages.M005, Messages.M006.format(version=version), exits=1)
|
|
return comp[version]
|
|
|
|
|
|
def get_version(model, comp):
|
|
model = model.rsplit(".dev", 1)[0]
|
|
if model not in comp:
|
|
msg.fail(
|
|
Messages.M005,
|
|
Messages.M007.format(name=model, version=about.__version__),
|
|
exits=1,
|
|
)
|
|
return comp[model][0]
|
|
|
|
|
|
def download_model(filename, user_pip_args=None):
|
|
download_url = about.__download_url__ + "/" + filename
|
|
pip_args = ["--no-cache-dir", "--no-deps"]
|
|
if user_pip_args:
|
|
pip_args.extend(user_pip_args)
|
|
cmd = [sys.executable, "-m", "pip", "install"] + pip_args + [download_url]
|
|
return subprocess.call(cmd, env=os.environ.copy())
|