2017-03-21 00:50:13 +03:00
|
|
|
# coding: utf8
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
2017-05-22 13:28:58 +03:00
|
|
|
import plac
|
2017-03-21 04:06:29 +03:00
|
|
|
import shutil
|
2017-03-21 13:17:36 +03:00
|
|
|
import requests
|
2017-03-21 00:50:13 +03:00
|
|
|
from pathlib import Path
|
|
|
|
|
2017-05-08 00:25:29 +03:00
|
|
|
from ..compat import path2str, json_dumps
|
|
|
|
from ..util import prints
|
2017-03-21 00:50:13 +03:00
|
|
|
from .. import util
|
2017-05-08 00:25:29 +03:00
|
|
|
from .. import about
|
2017-03-21 00:50:13 +03:00
|
|
|
|
|
|
|
|
2017-05-22 13:28:58 +03:00
|
|
|
@plac.annotations(
|
|
|
|
input_dir=("directory with model data", "positional", None, str),
|
|
|
|
output_dir=("output parent directory", "positional", None, str),
|
2017-08-12 22:44:15 +03:00
|
|
|
meta_path=("path to meta.json", "option", "m", str),
|
2017-10-27 15:38:39 +03:00
|
|
|
create_meta=("create meta.json, even if one exists in directory", "flag",
|
|
|
|
"c", bool),
|
|
|
|
force=("force overwriting of existing folder in output directory", "flag",
|
|
|
|
"f", bool))
|
|
|
|
def package(cmd, input_dir, output_dir, meta_path=None, create_meta=False,
|
|
|
|
force=False):
|
2017-05-27 21:01:46 +03:00
|
|
|
"""
|
|
|
|
Generate Python package for model data, including meta and required
|
2017-05-22 13:28:58 +03:00
|
|
|
installation files. A new directory will be created in the specified
|
|
|
|
output directory, and model data will be copied over.
|
|
|
|
"""
|
2017-05-08 00:25:29 +03:00
|
|
|
input_path = util.ensure_path(input_dir)
|
|
|
|
output_path = util.ensure_path(output_dir)
|
2017-08-12 22:44:15 +03:00
|
|
|
meta_path = util.ensure_path(meta_path)
|
2017-05-08 00:25:29 +03:00
|
|
|
if not input_path or not input_path.exists():
|
2017-05-22 13:28:58 +03:00
|
|
|
prints(input_path, title="Model directory not found", exits=1)
|
2017-05-08 00:25:29 +03:00
|
|
|
if not output_path or not output_path.exists():
|
2017-05-22 13:28:58 +03:00
|
|
|
prints(output_path, title="Output directory not found", exits=1)
|
2017-05-08 00:25:29 +03:00
|
|
|
if meta_path and not meta_path.exists():
|
2017-05-22 13:28:58 +03:00
|
|
|
prints(meta_path, title="meta.json not found", exits=1)
|
2017-03-21 00:50:13 +03:00
|
|
|
|
2017-03-21 13:17:36 +03:00
|
|
|
template_setup = get_template('setup.py')
|
2017-03-21 13:32:38 +03:00
|
|
|
template_manifest = get_template('MANIFEST.in')
|
2017-05-29 16:27:24 +03:00
|
|
|
template_init = get_template('xx_model_name/__init__.py')
|
2017-04-16 14:06:02 +03:00
|
|
|
meta_path = meta_path or input_path / 'meta.json'
|
2017-08-12 22:44:15 +03:00
|
|
|
if not create_meta and meta_path.is_file():
|
2017-05-08 00:25:29 +03:00
|
|
|
prints(meta_path, title="Reading meta.json from file")
|
2017-04-16 14:06:02 +03:00
|
|
|
meta = util.read_json(meta_path)
|
|
|
|
else:
|
2017-10-25 17:03:26 +03:00
|
|
|
meta = generate_meta(input_dir)
|
2017-05-27 21:02:01 +03:00
|
|
|
meta = validate_meta(meta, ['lang', 'name', 'version'])
|
2017-05-08 00:25:29 +03:00
|
|
|
|
2017-03-21 00:50:13 +03:00
|
|
|
model_name = meta['lang'] + '_' + meta['name']
|
|
|
|
model_name_v = model_name + '-' + meta['version']
|
|
|
|
main_path = output_path / model_name_v
|
|
|
|
package_path = main_path / model_name
|
|
|
|
|
2017-03-21 04:06:53 +03:00
|
|
|
create_dirs(package_path, force)
|
2017-10-27 15:38:39 +03:00
|
|
|
shutil.copytree(path2str(input_path),
|
|
|
|
path2str(package_path / model_name_v))
|
2017-04-14 00:30:47 +03:00
|
|
|
create_file(main_path / 'meta.json', json_dumps(meta))
|
2017-03-21 13:17:36 +03:00
|
|
|
create_file(main_path / 'setup.py', template_setup)
|
|
|
|
create_file(main_path / 'MANIFEST.in', template_manifest)
|
|
|
|
create_file(package_path / '__init__.py', template_init)
|
2017-10-27 15:38:39 +03:00
|
|
|
prints(main_path, "To build the package, run `python setup.py sdist` in "
|
|
|
|
"this directory.",
|
|
|
|
title="Successfully created package '%s'" % model_name_v)
|
2017-03-21 00:50:13 +03:00
|
|
|
|
|
|
|
|
2017-03-21 04:06:53 +03:00
|
|
|
def create_dirs(package_path, force):
|
|
|
|
if package_path.exists():
|
|
|
|
if force:
|
2017-05-08 00:25:29 +03:00
|
|
|
shutil.rmtree(path2str(package_path))
|
2017-03-21 04:06:53 +03:00
|
|
|
else:
|
2017-10-27 15:38:39 +03:00
|
|
|
prints(package_path, "Please delete the directory and try again, "
|
|
|
|
"or use the --force flag to overwrite existing "
|
|
|
|
"directories.", title="Package directory already exists",
|
|
|
|
exits=1)
|
2017-03-21 04:06:53 +03:00
|
|
|
Path.mkdir(package_path, parents=True)
|
|
|
|
|
|
|
|
|
2017-03-21 00:50:13 +03:00
|
|
|
def create_file(file_path, contents):
|
|
|
|
file_path.touch()
|
2017-03-29 10:11:02 +03:00
|
|
|
file_path.open('w', encoding='utf-8').write(contents)
|
2017-03-21 00:50:13 +03:00
|
|
|
|
|
|
|
|
2017-10-25 17:03:26 +03:00
|
|
|
def generate_meta(model_path):
|
|
|
|
meta = {}
|
2017-03-21 00:50:13 +03:00
|
|
|
settings = [('lang', 'Model language', 'en'),
|
|
|
|
('name', 'Model name', 'model'),
|
|
|
|
('version', 'Model version', '0.0.0'),
|
2017-10-27 15:38:39 +03:00
|
|
|
('spacy_version', 'Required spaCy version',
|
|
|
|
'>=%s,<3.0.0' % about.__version__),
|
2017-03-21 00:50:13 +03:00
|
|
|
('description', 'Model description', False),
|
|
|
|
('author', 'Author', False),
|
|
|
|
('email', 'Author email', False),
|
|
|
|
('url', 'Author website', False),
|
2017-03-21 14:24:43 +03:00
|
|
|
('license', 'License', 'CC BY-NC 3.0')]
|
2017-10-25 17:03:26 +03:00
|
|
|
nlp = util.load_model_from_path(Path(model_path))
|
|
|
|
meta['pipeline'] = nlp.pipe_names
|
|
|
|
meta['vectors'] = {'width': nlp.vocab.vectors_length,
|
|
|
|
'entries': len(nlp.vocab.vectors)}
|
|
|
|
prints("Enter the package settings for your model. The following "
|
|
|
|
"information will be read from your model data: pipeline, vectors.",
|
|
|
|
title="Generating meta.json")
|
2017-03-21 00:50:13 +03:00
|
|
|
for setting, desc, default in settings:
|
|
|
|
response = util.get_raw_input(desc, default)
|
|
|
|
meta[setting] = default if response == '' and default else response
|
2017-06-05 21:11:02 +03:00
|
|
|
if about.__title__ != 'spacy':
|
|
|
|
meta['parent_package'] = about.__title__
|
2017-03-21 00:50:13 +03:00
|
|
|
return meta
|
|
|
|
|
|
|
|
|
2017-04-16 14:13:17 +03:00
|
|
|
def validate_meta(meta, keys):
|
|
|
|
for key in keys:
|
|
|
|
if key not in meta or meta[key] == '':
|
2017-05-08 00:25:29 +03:00
|
|
|
prints("This setting is required to build your package.",
|
2017-05-22 13:28:58 +03:00
|
|
|
title='No "%s" setting found in meta.json' % key, exits=1)
|
2017-05-27 21:02:01 +03:00
|
|
|
return meta
|
2017-04-16 14:13:17 +03:00
|
|
|
|
|
|
|
|
2017-03-21 13:17:36 +03:00
|
|
|
def get_template(filepath):
|
2017-05-08 00:25:29 +03:00
|
|
|
r = requests.get(about.__model_files__ + filepath)
|
2017-03-21 13:17:36 +03:00
|
|
|
if r.status_code != 200:
|
2017-05-08 00:25:29 +03:00
|
|
|
prints("Couldn't fetch template files from GitHub.",
|
2017-05-22 13:28:58 +03:00
|
|
|
title="Server error (%d)" % r.status_code, exits=1)
|
2017-03-21 13:17:36 +03:00
|
|
|
return r.text
|