Merge branch 'master' of github.com:honnibal/spaCy

requirements.txt

@@ -10,3 +10,4 @@ plac
 six
 ujson
 cloudpickle
+sputnik == 0.5.1

setup.py

@@ -177,9 +177,9 @@ def run_setup(exts):
         package_data=PACKAGE_DATA,
         ext_modules=exts,
         license="MIT",
-        install_requires=['numpy', 'murmurhash', 'cymem == 1.30', 'preshed == 0.44',
+        install_requires=['numpy', 'murmurhash == 0.24', 'cymem == 1.30', 'preshed == 0.44',
                           'thinc == 4.0.0', "text_unidecode", 'plac', 'six',
-                          'ujson', 'cloudpickle'],
+                          'ujson', 'cloudpickle', 'sputnik == 0.5.1'],
         setup_requires=["headers_workaround"],
         cmdclass = {'build_ext': build_ext_subclass },
     )

spacy/data/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

spacy/en/download.py

@@ -1,65 +1,49 @@
-from __future__ import print_function
 import sys
 import os
-import tarfile
 import shutil
+
 import plac
-
+from sputnik import Sputnik
-from . import uget
 
 
-try:
+def migrate(path):
-    FileExistsError
+    data_path = os.path.join(path, 'data')
-except NameError:
+    if os.path.isdir(data_path) and not os.path.islink(data_path):
-    FileExistsError = Exception
+        shutil.rmtree(data_path)
+    for filename in os.listdir(path):
+        if filename.endswith('.tgz'):
+            os.unlink(os.path.join(path, filename))
 
 
-# TODO: Read this from the same source as the setup
+def link(package, path):
-VERSION = '0.9.9'
+    if os.path.exists(path):
-
+        os.unlink(path)
-AWS_STORE = 'https://s3-us-west-1.amazonaws.com/media.spacynlp.com'
+    os.symlink(package.dir_path('data'), path)
-
-ALL_DATA_DIR_URL = '%s/en_data_all-%s.tgz' % (AWS_STORE, VERSION)
-
-DEST_DIR = os.path.dirname(os.path.abspath(__file__))
-
-
-def download_file(url, download_path):
-    return uget.download(url, download_path, console=sys.stdout)
-
-
-def install_data(url, extract_path, download_path):
-    try:
-        os.makedirs(extract_path)
-    except FileExistsError:
-        pass
-
-    tmp = download_file(url, download_path)
-    assert tmp == download_path
-    t = tarfile.open(download_path)
-    t.extractall(extract_path)
-    os.unlink(download_path)
 
 
 @plac.annotations(
     force=("Force overwrite", "flag", "f", bool),
 )
 def main(data_size='all', force=False):
-    filename = ALL_DATA_DIR_URL.rsplit('/', 1)[1]
+    # TODO read version from the same source as the setup
-    download_path = os.path.join(DEST_DIR, filename)
+    sputnik = Sputnik('spacy', '0.99.0', console=sys.stdout)
-    data_path = os.path.join(DEST_DIR, 'data')
 
-    if force and os.path.exists(download_path):
+    path = os.path.dirname(os.path.abspath(__file__))
-        os.unlink(download_path)
 
-    if force and os.path.exists(data_path):
+    command = sputnik.make_command(
-        shutil.rmtree(data_path)
+        data_path=os.path.abspath(os.path.join(path, '..', 'data')),
+        repository_url='https://index.spacy.io')
 
-    if os.path.exists(data_path):
+    if force:
-        print('data already installed at %s, overwrite with --force' % DEST_DIR)
+        command.purge()
-        sys.exit(1)
 
-    install_data(ALL_DATA_DIR_URL, DEST_DIR, download_path)
+    package = command.install('en_default')
+
+    # FIXME clean up old-style packages
+    migrate(path)
+
+    # FIXME supply spacy with an old-style data dir
+    link(package, os.path.join(path, 'data'))
 
 
 if __name__ == '__main__':

website/src/jade/home/_installation.jade

@@ -31,7 +31,7 @@ mixin Option(name, open)
 
     pre.language-bash: code
       | $ pip install spacy
-      | $ python -m spacy.en.download all
+      | $ python -m spacy.en.download --force all
 
   p
     | The download command fetches and installs about 400mb of data, for
@@ -70,7 +70,7 @@ mixin Option(name, open)
     | $ python setup.py build_ext --inplace
     | $ python -m spacy.en.download
     | $ pip install pytest
-    | $ py.test tests/
+    | $ py.test spacy/tests/
 
   p
     | Python packaging is awkward at the best of times, and it's particularly tricky