spaCy/spacy/cli/download.py

93 lines
3.5 KiB
Python
Raw Normal View History

# coding: utf8
from __future__ import unicode_literals
import plac
import requests
2017-03-18 14:59:41 +03:00
import os
import subprocess
import sys
2017-05-08 00:25:29 +03:00
from .link import link
from ..util import prints, get_package_path
from .. import about
@plac.annotations(
2017-10-27 15:38:39 +03:00
model=("model to download, shortcut or name)", "positional", None, str),
direct=("force direct download. Needs model name with version and won't "
2017-10-27 15:38:39 +03:00
"perform compatibility check", "flag", "d", bool))
def download(model, direct=False):
"""
Download compatible model from default download path using pip. Model
can be shortcut, model name or, if --direct flag is set, full model name
with version.
"""
if direct:
dl = download_model('{m}/{m}.tar.gz'.format(m=model))
else:
2017-05-08 00:25:29 +03:00
shortcuts = get_json(about.__shortcuts__, "available shortcuts")
model_name = shortcuts.get(model, model)
compatibility = get_compatibility()
version = get_version(model_name, compatibility)
2017-10-27 15:38:39 +03:00
dl = download_model('{m}-{v}/{m}-{v}.tar.gz'.format(m=model_name,
v=version))
if dl != 0:
# if download subprocess doesn't return 0, exit with the respective
# exit code before doing anything else
sys.exit(dl)
try:
# Get package path here because link uses
# pip.get_installed_distributions() to check if model is a
# package, which fails if model was just installed via
# subprocess
package_path = get_package_path(model_name)
link(model_name, model, force=True,
model_path=package_path)
except:
# Dirty, but since spacy.download and the auto-linking is
# mostly a convenience wrapper, it's best to show a success
# message and loading instructions, even if linking fails.
prints(
"Creating a shortcut link for 'en' didn't work (maybe "
"you don't have admin permissions?), but you can still "
"load the model via its full package name:",
"nlp = spacy.load('%s')" % model_name,
title="Download successful but linking failed")
2017-03-17 01:23:26 +03:00
2018-01-03 22:14:50 +03:00
def get_json(url, desc):
r = requests.get(url)
if r.status_code != 200:
2017-10-27 15:38:39 +03:00
msg = ("Couldn't fetch %s. Please find a model for your spaCy "
"installation (v%s), and download it manually.")
prints(msg % (desc, about.__version__), about.__docs_models__,
title="Server error (%d)" % r.status_code, exits=1)
return r.json()
def get_compatibility():
version = about.__version__
version = version.rsplit('.dev', 1)[0]
comp_table = get_json(about.__compatibility__, "compatibility table")
comp = comp_table['spacy']
if version not in comp:
2017-05-08 00:25:29 +03:00
prints("No compatible models found for v%s of spaCy." % version,
title="Compatibility error", exits=1)
return comp[version]
def get_version(model, comp):
model = model.rsplit('.dev', 1)[0]
if model not in comp:
2017-05-08 00:25:29 +03:00
version = about.__version__
2017-10-27 15:38:39 +03:00
msg = "No compatible model found for '%s' (spaCy v%s)."
prints(msg % (model, version), title="Compatibility error", exits=1)
return comp[model][0]
def download_model(filename):
download_url = about.__download_url__ + '/' + filename
2017-10-27 15:38:39 +03:00
return subprocess.call(
[sys.executable, '-m', 'pip', 'install', '--no-cache-dir', '--no-deps',
2017-10-27 15:38:39 +03:00
download_url], env=os.environ.copy())