From a4662a31a98325b08188df5ea96b471da1dd1926 Mon Sep 17 00:00:00 2001
From: ines <ines@ines.io>
Date: Tue, 7 Nov 2017 12:15:35 +0100
Subject: [PATCH] Move model package templates to cli.package and update docs

---
 spacy/about.py                               |   1 -
 spacy/cli/package.py                         | 101 ++++++++++++++++---
 website/api/_top-level/_util.jade            |   2 +-
 website/api/cli.jade                         |  10 +-
 website/usage/_training/_saving-loading.jade |   4 +-
 5 files changed, 94 insertions(+), 24 deletions(-)

diff --git a/spacy/about.py b/spacy/about.py
index 4e03256c9..0c9ac70ef 100644
--- a/spacy/about.py
+++ b/spacy/about.py
@@ -15,4 +15,3 @@ __docs_models__ = 'https://spacy.io/usage/models'
 __download_url__ = 'https://github.com/explosion/spacy-models/releases/download'
 __compatibility__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json'
 __shortcuts__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/shortcuts-v2.json'
-__model_files__ = 'https://raw.githubusercontent.com/explosion/spacy-dev-resources/develop/templates/model/'
diff --git a/spacy/cli/package.py b/spacy/cli/package.py
index 5ee8a2b1e..8c590b49b 100644
--- a/spacy/cli/package.py
+++ b/spacy/cli/package.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 import plac
 import shutil
-import requests
 from pathlib import Path
 
 from ..compat import path2str, json_dumps
@@ -38,9 +37,6 @@ def package(cmd, input_dir, output_dir, meta_path=None, create_meta=False,
     if meta_path and not meta_path.exists():
         prints(meta_path, title="meta.json not found", exits=1)
 
-    template_setup = get_template('setup.py')
-    template_manifest = get_template('MANIFEST.in')
-    template_init = get_template('xx_model_name/__init__.py')
     meta_path = meta_path or input_path / 'meta.json'
     if meta_path.is_file():
         meta = util.read_json(meta_path)
@@ -58,9 +54,9 @@ def package(cmd, input_dir, output_dir, meta_path=None, create_meta=False,
     shutil.copytree(path2str(input_path),
                     path2str(package_path / model_name_v))
     create_file(main_path / 'meta.json', json_dumps(meta))
-    create_file(main_path / 'setup.py', template_setup)
-    create_file(main_path / 'MANIFEST.in', template_manifest)
-    create_file(package_path / '__init__.py', template_init)
+    create_file(main_path / 'setup.py', TEMPLATE_SETUP)
+    create_file(main_path / 'MANIFEST.in', TEMPLATE_MANIFEST)
+    create_file(package_path / '__init__.py', TEMPLATE_INIT)
     prints(main_path, "To build the package, run `python setup.py sdist` in "
            "this directory.",
            title="Successfully created package '%s'" % model_name_v)
@@ -120,9 +116,88 @@ def validate_meta(meta, keys):
     return meta
 
 
-def get_template(filepath):
-    r = requests.get(about.__model_files__ + filepath)
-    if r.status_code != 200:
-        prints("Couldn't fetch template files from GitHub.",
-               title="Server error (%d)" % r.status_code, exits=1)
-    return r.text
+TEMPLATE_SETUP = """
+#!/usr/bin/env python
+# coding: utf8
+from __future__ import unicode_literals
+
+import io
+import json
+from os import path, walk
+from shutil import copy
+from setuptools import setup
+
+
+def load_meta(fp):
+    with io.open(fp, encoding='utf8') as f:
+        return json.load(f)
+
+
+def list_files(data_dir):
+    output = []
+    for root, _, filenames in walk(data_dir):
+        for filename in filenames:
+            if not filename.startswith('.'):
+                output.append(path.join(root, filename))
+    output = [path.relpath(p, path.dirname(data_dir)) for p in output]
+    output.append('meta.json')
+    return output
+
+
+def list_requirements(meta):
+    parent_package = meta.get('parent_package', 'spacy')
+    requirements = [parent_package + meta['spacy_version']]
+    if 'setup_requires' in meta:
+        requirements += meta['setup_requires']
+    return requirements
+
+
+def setup_package():
+    root = path.abspath(path.dirname(__file__))
+    meta_path = path.join(root, 'meta.json')
+    meta = load_meta(meta_path)
+    model_name = str(meta['lang'] + '_' + meta['name'])
+    model_dir = path.join(model_name, model_name + '-' + meta['version'])
+
+    copy(meta_path, path.join(model_name))
+    copy(meta_path, model_dir)
+
+    setup(
+        name=model_name,
+        description=meta['description'],
+        author=meta['author'],
+        author_email=meta['email'],
+        url=meta['url'],
+        version=meta['version'],
+        license=meta['license'],
+        packages=[model_name],
+        package_data={model_name: list_files(model_dir)},
+        install_requires=list_requirements(meta),
+        zip_safe=False,
+    )
+
+
+if __name__ == '__main__':
+    setup_package()
+""".strip()
+
+
+TEMPLATE_MANIFEST = """
+include meta.json
+""".strip()
+
+
+TEMPLATE_INIT = """
+# coding: utf8
+from __future__ import unicode_literals
+
+from pathlib import Path
+from spacy.util import load_model_from_init_py, get_model_meta
+
+
+__version__ = get_model_meta(Path(__file__).parent)['version']
+
+
+def load(**overrides):
+    return load_model_from_init_py(__file__, **overrides)
+""".strip()
diff --git a/website/api/_top-level/_util.jade b/website/api/_top-level/_util.jade
index a08cf1276..ef12846ad 100644
--- a/website/api/_top-level/_util.jade
+++ b/website/api/_top-level/_util.jade
@@ -148,7 +148,7 @@ p
 
 p
     |  A helper function to use in the #[code load()] method of a model package's
-    |  #[+src(gh("spacy-dev-resources", "templates/model/en_model_name/__init__.py")) #[code __init__.py]].
+    |  #[+src(gh("spacy-models", "template/model/xx_model_name/__init__.py")) #[code __init__.py]].
 
 +aside-code("Example").
     from spacy.util import load_model_from_init_py
diff --git a/website/api/cli.jade b/website/api/cli.jade
index cd1cb22fb..12f5e8713 100644
--- a/website/api/cli.jade
+++ b/website/api/cli.jade
@@ -541,13 +541,9 @@ p
     |  from an existing model data directory. All data files are copied over.
     |  If the path to a #[code meta.json] is supplied, or a #[code meta.json] is
     |  found in the input directory, this file is used. Otherwise, the data can
-    |  be entered directly from the command line. The required file templates
-    |  are downloaded from
-    |  #[+src(gh("spacy-dev-resources", "templates/model")) GitHub] to make
-    |  sure you're always using the latest versions. This means you need to be
-    |  connected to the internet to use this command. After packaging, you
-    |  can run #[code python setup.py sdist] from the newly created directory
-    |  to turn your model into an installable archive file.
+    |  be entered directly from the command line. After packaging, you can run
+    |  #[code python setup.py sdist] from the newly created directory to turn
+    |  your model into an installable archive file.
 
 +code(false, "bash", "$", false, false, true).
     spacy package [input_dir] [output_dir] [--meta-path] [--create-meta] [--force]
diff --git a/website/usage/_training/_saving-loading.jade b/website/usage/_training/_saving-loading.jade
index 885b3ad24..2a6f9380b 100644
--- a/website/usage/_training/_saving-loading.jade
+++ b/website/usage/_training/_saving-loading.jade
@@ -59,8 +59,8 @@ p This command will create a model package directory that should look like this:
             └── en_example_model-1.0.0    # model data
 
 p
-    |  You can also find templates for all files in our
-    |  #[+src(gh("spacy-dev-resources", "templates/model")) spaCy dev resources].
+    |  You can also find templates for all files on
+    |  #[+src(gh("spacy-models", "template")) GitHub].
     |  If you're creating the package manually, keep in mind that the directories
     |  need to be named according to the naming conventions of
     |  #[code lang_name] and #[code lang_name-version].