explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-06-05 21:53:05 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Upd download script, having it download all data if there's no data/ directory, allowing easier compilation from source

Browse Source
This commit is contained in:
Matthew Honnibal 2015-01-30 18:04:01 +11:00
parent 0b53fd7daa
commit 998b607f65
1 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
spacy/en/download.py
View File

@ -4,19 +4,27 @@ import tarfile
import shutil import shutil
import wget import wget
PARSER_URL = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com/en_deps-0.30.tgz'
DATA_DIR_URL = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com/en_data_all-0.4.tgz'
PARSER_URL = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com/en_deps-0.30.tgz'
DEP_VECTORS_URL = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com/vec.bin' DEP_VECTORS_URL = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com/vec.bin'
DEST_DIR = path.join(path.dirname(__file__), 'data') DEST_DIR = path.join(path.dirname(__file__), 'tmp_data')
def download_file(url, out): def download_file(url, out):
wget.download(url, out=out) wget.download(url, out=out)
def install_all_data(url, dest_dir):
filename = download_file(url, dest_dir)
t = tarfile.open(path.join(dest_dir, filename), mode=":gz")
t.extractall(dest_dir)
def install_parser_model(url, dest_dir): def install_parser_model(url, dest_dir):
download_file(url, dest_dir) filename = download_file(url, dest_dir)
t = tarfile.open(path.join(DEST_DIR, 'en_deps-0.30.tgz'), mode=":gz") t = tarfile.open(path.join(dest_dir, filename), mode=":gz")
t.extractall(dest_dir) t.extractall(dest_dir)
@ -28,8 +36,11 @@ def install_dep_vectors(url, dest_dir):
def main(): def main():
install_parser_model(PARSER_URL, DEST_DIR) if not path.exists(DEST_DIR):
install_dep_vectors(DEP_VECTORS_URL, path.join(DEST_DIR, 'vocab')) install_all_data(DATA_DIR_URL, DEST_DIR)
else:
install_parser_model(PARSER_URL, DEST_DIR)
install_dep_vectors(DEP_VECTORS_URL, path.join(DEST_DIR, 'vocab'))
if __name__ == '__main__': if __name__ == '__main__':