explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-06-04 05:03:06 +03:00
Code Issues Packages Projects Releases Wiki Activity

add sputnik

Browse Source
This commit is contained in:
Henning Peters 2015-11-15 15:58:21 +01:00
parent e8bd92f1e7
commit 03d2f98cd5
1 changed files with 30 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
spacy/en/download.py
View File

@ -1,65 +1,50 @@
from __future__ import print_function
import sys import sys
import os import os
import tarfile
import shutil import shutil
import plac import plac
from sputnik import Sputnik
from . import uget
try: def migrate(path):
FileExistsError data_path = os.path.join(path, 'data')
except NameError: if os.path.isdir(data_path) and not os.path.islink(data_path):
FileExistsError = Exception shutil.rmtree(data_path)
for filename in os.listdir(path):
if filename.endswith('tgz'):
os.unlink(os.path.join(path, filename))
# TODO: Read this from the same source as the setup def link(package, path):
VERSION = '0.9.9' if os.path.exists(path):
os.unlink(path)
AWS_STORE = 'https://s3-us-west-1.amazonaws.com/media.spacynlp.com' os.symlink(os.path.join(package.path, 'data'),
os.path.join(path))
ALL_DATA_DIR_URL = '%s/en_data_all-%s.tgz' % (AWS_STORE, VERSION)
DEST_DIR = os.path.dirname(os.path.abspath(__file__))
def download_file(url, download_path):
return uget.download(url, download_path, console=sys.stdout)
def install_data(url, extract_path, download_path):
try:
os.makedirs(extract_path)
except FileExistsError:
pass
tmp = download_file(url, download_path)
assert tmp == download_path
t = tarfile.open(download_path)
t.extractall(extract_path)
os.unlink(download_path)
@plac.annotations( @plac.annotations(
force=("Force overwrite", "flag", "f", bool), force=("Force overwrite", "flag", "f", bool),
) )
def main(data_size='all', force=False): def main(force=False):
filename = ALL_DATA_DIR_URL.rsplit('/', 1)[1] # TODO read version from the same source as the setup
download_path = os.path.join(DEST_DIR, filename) sputnik = Sputnik('spacy', '0.99.0', console=sys.stdout)
data_path = os.path.join(DEST_DIR, 'data')
if force and os.path.exists(download_path): path = os.path.dirname(os.path.abspath(__file__))
os.unlink(download_path)
if force and os.path.exists(data_path): command = sputnik.make_command(
shutil.rmtree(data_path) data_path=path,
repository_url=os.environ.get('REPOSITORY_URL'))
if os.path.exists(data_path): if force:
print('data already installed at %s, overwrite with --force' % DEST_DIR) command.purge()
sys.exit(1)
install_data(ALL_DATA_DIR_URL, DEST_DIR, download_path) package = command.install('en_default')
# FIXME clean up old-style packages
migrate(path)
# FIXME supply spacy with an old-style data dir
link(package, os.path.join(path, 'data'))
if __name__ == '__main__': if __name__ == '__main__':