mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-26 01:46:28 +03:00
Merge pull request #1792 from explosion/feature-improve-model-download
💫 Improve model downloading and linking
This commit is contained in:
Matthew Honnibal
GitHub
commit
a2a06dce24
|
|
@ -31,24 +31,28 @@ def download(model, direct=False):
|
|||
version = get_version(model_name, compatibility)
|
||||
dl = download_model('{m}-{v}/{m}-{v}.tar.gz'.format(m=model_name,
|
||||
v=version))
|
||||
if dl == 0:
|
||||
try:
|
||||
# Get package path here because link uses
|
||||
# pip.get_installed_distributions() to check if model is a
|
||||
# package, which fails if model was just installed via
|
||||
# subprocess
|
||||
package_path = get_package_path(model_name)
|
||||
link(model_name, model, force=True, model_path=package_path)
|
||||
except:
|
||||
# Dirty, but since spacy.download and the auto-linking is
|
||||
# mostly a convenience wrapper, it's best to show a success
|
||||
# message and loading instructions, even if linking fails.
|
||||
prints(
|
||||
"Creating a shortcut link for 'en' didn't work (maybe "
|
||||
"you don't have admin permissions?), but you can still "
|
||||
"load the model via its full package name:",
|
||||
"nlp = spacy.load('%s')" % model_name,
|
||||
title="Download successful")
|
||||
if dl != 0:
|
||||
# if download subprocess doesn't return 0, exit with the respective
|
||||
# exit code before doing anything else
|
||||
sys.exit(dl)
|
||||
try:
|
||||
# Get package path here because link uses
|
||||
# pip.get_installed_distributions() to check if model is a
|
||||
# package, which fails if model was just installed via
|
||||
# subprocess
|
||||
package_path = get_package_path(model_name)
|
||||
link(None, model_name, model, force=True,
|
||||
model_path=package_path)
|
||||
except:
|
||||
# Dirty, but since spacy.download and the auto-linking is
|
||||
# mostly a convenience wrapper, it's best to show a success
|
||||
# message and loading instructions, even if linking fails.
|
||||
prints(
|
||||
"Creating a shortcut link for 'en' didn't work (maybe "
|
||||
"you don't have admin permissions?), but you can still "
|
||||
"load the model via its full package name:",
|
||||
"nlp = spacy.load('%s')" % model_name,
|
||||
title="Download successful but linking failed")
|
||||
|
||||
|
||||
def get_json(url, desc):
|
||||
|
|
@ -84,5 +88,5 @@ def get_version(model, comp):
|
|||
def download_model(filename):
|
||||
download_url = about.__download_url__ + '/' + filename
|
||||
return subprocess.call(
|
||||
[sys.executable, '-m', 'pip', 'install', '--no-cache-dir',
|
||||
[sys.executable, '-m', 'pip', 'install', '--no-cache-dir', '--no-deps',
|
||||
download_url], env=os.environ.copy())
|
||||
|
|
|
|||
|
|
@ -34,11 +34,18 @@ def link(origin, link_name, force=False, model_path=None):
|
|||
"located here:", path2str(spacy_loc), exits=1,
|
||||
title="Can't find the spaCy data path to create model symlink")
|
||||
link_path = util.get_data_path() / link_name
|
||||
if link_path.exists() and not force:
|
||||
if link_path.is_symlink() and not force:
|
||||
prints("To overwrite an existing link, use the --force flag.",
|
||||
title="Link %s already exists" % link_name, exits=1)
|
||||
elif link_path.exists():
|
||||
elif link_path.is_symlink(): # does a symlink exist?
|
||||
# NB: It's important to check for is_symlink here and not for exists,
|
||||
# because invalid/outdated symlinks would return False otherwise.
|
||||
link_path.unlink()
|
||||
elif link_path.exists(): # does it exist otherwise?
|
||||
# NB: Check this last because valid symlinks also "exist".
|
||||
prints("This can happen if your data directory contains a directory "
|
||||
"or file of the same name.", link_path,
|
||||
title="Can't overwrite symlink %s" % link_name, exits=1)
|
||||
try:
|
||||
symlink_to(link_path, model_path)
|
||||
except:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from __future__ import unicode_literals, print_function
|
|||
import requests
|
||||
import pkg_resources
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from ..compat import path2str, locale_escape
|
||||
from ..util import prints, get_data_path, read_json
|
||||
|
|
@ -62,6 +63,9 @@ def validate():
|
|||
"them from the data directory. Data path: {}"
|
||||
.format(path2str(get_data_path())))
|
||||
|
||||
if incompat_models or incompat_links:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def get_model_links(compat):
|
||||
links = {}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,17 @@ p
|
|||
| Direct downloads don't perform any compatibility checks and require the
|
||||
| model name to be specified with its version (e.g., #[code en_core_web_sm-1.2.0]).
|
||||
|
||||
+aside("Downloading best practices")
|
||||
| The #[code download] command is mostly intended as a convenient,
|
||||
| interactive wrapper – it performs compatibility checks and prints
|
||||
| detailed messages in case things go wrong. It's #[strong not recommended]
|
||||
| to use this command as part of an automated process. If you know which
|
||||
| model your project needs, you should consider a
|
||||
| #[+a("/usage/models#download-pip") direct download via pip], or
|
||||
| uploading the model to a local PyPi installation and fetching it straight
|
||||
| from there. This will also allow you to add it as a versioned package
|
||||
| dependency to your project.
|
||||
|
||||
+code(false, "bash", "$").
|
||||
python -m spacy download [model] [--direct]
|
||||
|
||||
|
|
@ -43,17 +54,6 @@ p
|
|||
| The installed model package in your #[code site-packages]
|
||||
| directory and a shortcut link as a symlink in #[code spacy/data].
|
||||
|
||||
+aside("Downloading best practices")
|
||||
| The #[code download] command is mostly intended as a convenient,
|
||||
| interactive wrapper – it performs compatibility checks and prints
|
||||
| detailed messages in case things go wrong. It's #[strong not recommended]
|
||||
| to use this command as part of an automated process. If you know which
|
||||
| model your project needs, you should consider a
|
||||
| #[+a("/usage/models#download-pip") direct download via pip], or
|
||||
| uploading the model to a local PyPi installation and fetching it straight
|
||||
| from there. This will also allow you to add it as a versioned package
|
||||
| dependency to your project.
|
||||
|
||||
+h(3, "link") Link
|
||||
|
||||
p
|
||||
|
|
@ -144,8 +144,14 @@ p
|
|||
| #[code pip install -U spacy] to ensure that all installed models are
|
||||
| can be used with the new version. The command is also useful to detect
|
||||
| out-of-sync model links resulting from links created in different virtual
|
||||
| environments. Prints a list of models, the installed versions, the latest
|
||||
| compatible version (if out of date) and the commands for updating.
|
||||
| environments. It will a list of models, the installed versions, the
|
||||
| latest compatible version (if out of date) and the commands for updating.
|
||||
|
||||
+aside("Automated validation")
|
||||
| You can also use the #[code validate] command as part of your build
|
||||
| process or test suite, to ensure all models are up to date before
|
||||
| proceeding. If incompatible models or shortcut links are found, it will
|
||||
| return #[code 1].
|
||||
|
||||
+code(false, "bash", "$").
|
||||
python -m spacy validate
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user