spaCy/spacy/en/download.py

58 lines
1.4 KiB
Python
Raw Normal View History

2015-02-01 05:22:23 +03:00
from __future__ import print_function
2015-01-02 13:44:41 +03:00
from os import path
import os
import tarfile
import shutil
2015-06-08 01:48:56 +03:00
import wget
2015-01-31 05:51:56 +03:00
import plac
2015-01-02 13:44:41 +03:00
2015-02-07 19:32:33 +03:00
# TODO: Read this from the same source as the setup
2015-06-07 20:13:28 +03:00
VERSION = '0.8.5'
2015-01-02 13:44:41 +03:00
2015-02-07 19:32:33 +03:00
AWS_STORE = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com'
2015-02-07 19:32:33 +03:00
ALL_DATA_DIR_URL = '%s/en_data_all-%s.tgz' % (AWS_STORE, VERSION)
2015-02-07 20:00:36 +03:00
SM_DATA_DIR_URL = '%s/en_data_sm-%s.tgz' % (AWS_STORE, VERSION)
2015-01-25 15:01:10 +03:00
SPEECH_DATA_DIR_URL = '%s/en_data_speech-%s.tgz' % (AWS_STORE, VERSION)
2015-01-30 10:59:31 +03:00
DEST_DIR = path.join(path.dirname(__file__), 'data')
2015-01-02 13:44:41 +03:00
2015-01-25 15:01:10 +03:00
def download_file(url, out):
2015-06-08 01:48:56 +03:00
wget.download(url, out=out)
2015-01-30 10:36:24 +03:00
return url.rsplit('/', 1)[1]
2015-01-02 13:44:41 +03:00
2015-01-30 12:33:19 +03:00
def install_data(url, dest_dir):
filename = download_file(url, dest_dir)
2015-01-30 11:28:43 +03:00
t = tarfile.open(path.join(dest_dir, filename))
t.extractall(dest_dir)
def install_parser_model(url, dest_dir):
filename = download_file(url, dest_dir)
t = tarfile.open(path.join(dest_dir, filename), mode=":gz")
2015-01-30 11:28:43 +03:00
t.extractall(path.dirname(__file__))
def install_dep_vectors(url, dest_dir):
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
2015-04-19 11:31:31 +03:00
2015-01-25 15:01:10 +03:00
filename = download_file(url, dest_dir)
2015-01-31 05:51:56 +03:00
def main(data_size='all'):
if data_size == 'all':
data_url = ALL_DATA_DIR_URL
elif data_size == 'small':
data_url = SM_DATA_DIR_URL
2015-02-01 05:22:23 +03:00
if path.exists(DEST_DIR):
2015-04-12 08:04:10 +03:00
shutil.rmtree(DEST_DIR)
2015-01-31 14:48:32 +03:00
install_data(data_url, path.dirname(DEST_DIR))
2015-01-02 13:44:41 +03:00
if __name__ == '__main__':
2015-01-31 05:51:56 +03:00
plac.call(main)