mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-25 01:16:28 +03:00
Move model package templates to cli.package and update docs
This commit is contained in:
parent
a09c096d3c
commit
a4662a31a9
|
@ -15,4 +15,3 @@ __docs_models__ = 'https://spacy.io/usage/models'
|
||||||
__download_url__ = 'https://github.com/explosion/spacy-models/releases/download'
|
__download_url__ = 'https://github.com/explosion/spacy-models/releases/download'
|
||||||
__compatibility__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json'
|
__compatibility__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json'
|
||||||
__shortcuts__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/shortcuts-v2.json'
|
__shortcuts__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/shortcuts-v2.json'
|
||||||
__model_files__ = 'https://raw.githubusercontent.com/explosion/spacy-dev-resources/develop/templates/model/'
|
|
||||||
|
|
|
@ -3,7 +3,6 @@ from __future__ import unicode_literals
|
||||||
|
|
||||||
import plac
|
import plac
|
||||||
import shutil
|
import shutil
|
||||||
import requests
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ..compat import path2str, json_dumps
|
from ..compat import path2str, json_dumps
|
||||||
|
@ -38,9 +37,6 @@ def package(cmd, input_dir, output_dir, meta_path=None, create_meta=False,
|
||||||
if meta_path and not meta_path.exists():
|
if meta_path and not meta_path.exists():
|
||||||
prints(meta_path, title="meta.json not found", exits=1)
|
prints(meta_path, title="meta.json not found", exits=1)
|
||||||
|
|
||||||
template_setup = get_template('setup.py')
|
|
||||||
template_manifest = get_template('MANIFEST.in')
|
|
||||||
template_init = get_template('xx_model_name/__init__.py')
|
|
||||||
meta_path = meta_path or input_path / 'meta.json'
|
meta_path = meta_path or input_path / 'meta.json'
|
||||||
if meta_path.is_file():
|
if meta_path.is_file():
|
||||||
meta = util.read_json(meta_path)
|
meta = util.read_json(meta_path)
|
||||||
|
@ -58,9 +54,9 @@ def package(cmd, input_dir, output_dir, meta_path=None, create_meta=False,
|
||||||
shutil.copytree(path2str(input_path),
|
shutil.copytree(path2str(input_path),
|
||||||
path2str(package_path / model_name_v))
|
path2str(package_path / model_name_v))
|
||||||
create_file(main_path / 'meta.json', json_dumps(meta))
|
create_file(main_path / 'meta.json', json_dumps(meta))
|
||||||
create_file(main_path / 'setup.py', template_setup)
|
create_file(main_path / 'setup.py', TEMPLATE_SETUP)
|
||||||
create_file(main_path / 'MANIFEST.in', template_manifest)
|
create_file(main_path / 'MANIFEST.in', TEMPLATE_MANIFEST)
|
||||||
create_file(package_path / '__init__.py', template_init)
|
create_file(package_path / '__init__.py', TEMPLATE_INIT)
|
||||||
prints(main_path, "To build the package, run `python setup.py sdist` in "
|
prints(main_path, "To build the package, run `python setup.py sdist` in "
|
||||||
"this directory.",
|
"this directory.",
|
||||||
title="Successfully created package '%s'" % model_name_v)
|
title="Successfully created package '%s'" % model_name_v)
|
||||||
|
@ -120,9 +116,88 @@ def validate_meta(meta, keys):
|
||||||
return meta
|
return meta
|
||||||
|
|
||||||
|
|
||||||
def get_template(filepath):
|
TEMPLATE_SETUP = """
|
||||||
r = requests.get(about.__model_files__ + filepath)
|
#!/usr/bin/env python
|
||||||
if r.status_code != 200:
|
# coding: utf8
|
||||||
prints("Couldn't fetch template files from GitHub.",
|
from __future__ import unicode_literals
|
||||||
title="Server error (%d)" % r.status_code, exits=1)
|
|
||||||
return r.text
|
import io
|
||||||
|
import json
|
||||||
|
from os import path, walk
|
||||||
|
from shutil import copy
|
||||||
|
from setuptools import setup
|
||||||
|
|
||||||
|
|
||||||
|
def load_meta(fp):
|
||||||
|
with io.open(fp, encoding='utf8') as f:
|
||||||
|
return json.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
def list_files(data_dir):
|
||||||
|
output = []
|
||||||
|
for root, _, filenames in walk(data_dir):
|
||||||
|
for filename in filenames:
|
||||||
|
if not filename.startswith('.'):
|
||||||
|
output.append(path.join(root, filename))
|
||||||
|
output = [path.relpath(p, path.dirname(data_dir)) for p in output]
|
||||||
|
output.append('meta.json')
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
def list_requirements(meta):
|
||||||
|
parent_package = meta.get('parent_package', 'spacy')
|
||||||
|
requirements = [parent_package + meta['spacy_version']]
|
||||||
|
if 'setup_requires' in meta:
|
||||||
|
requirements += meta['setup_requires']
|
||||||
|
return requirements
|
||||||
|
|
||||||
|
|
||||||
|
def setup_package():
|
||||||
|
root = path.abspath(path.dirname(__file__))
|
||||||
|
meta_path = path.join(root, 'meta.json')
|
||||||
|
meta = load_meta(meta_path)
|
||||||
|
model_name = str(meta['lang'] + '_' + meta['name'])
|
||||||
|
model_dir = path.join(model_name, model_name + '-' + meta['version'])
|
||||||
|
|
||||||
|
copy(meta_path, path.join(model_name))
|
||||||
|
copy(meta_path, model_dir)
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name=model_name,
|
||||||
|
description=meta['description'],
|
||||||
|
author=meta['author'],
|
||||||
|
author_email=meta['email'],
|
||||||
|
url=meta['url'],
|
||||||
|
version=meta['version'],
|
||||||
|
license=meta['license'],
|
||||||
|
packages=[model_name],
|
||||||
|
package_data={model_name: list_files(model_dir)},
|
||||||
|
install_requires=list_requirements(meta),
|
||||||
|
zip_safe=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
setup_package()
|
||||||
|
""".strip()
|
||||||
|
|
||||||
|
|
||||||
|
TEMPLATE_MANIFEST = """
|
||||||
|
include meta.json
|
||||||
|
""".strip()
|
||||||
|
|
||||||
|
|
||||||
|
TEMPLATE_INIT = """
|
||||||
|
# coding: utf8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from spacy.util import load_model_from_init_py, get_model_meta
|
||||||
|
|
||||||
|
|
||||||
|
__version__ = get_model_meta(Path(__file__).parent)['version']
|
||||||
|
|
||||||
|
|
||||||
|
def load(**overrides):
|
||||||
|
return load_model_from_init_py(__file__, **overrides)
|
||||||
|
""".strip()
|
||||||
|
|
|
@ -148,7 +148,7 @@ p
|
||||||
|
|
||||||
p
|
p
|
||||||
| A helper function to use in the #[code load()] method of a model package's
|
| A helper function to use in the #[code load()] method of a model package's
|
||||||
| #[+src(gh("spacy-dev-resources", "templates/model/en_model_name/__init__.py")) #[code __init__.py]].
|
| #[+src(gh("spacy-models", "template/model/xx_model_name/__init__.py")) #[code __init__.py]].
|
||||||
|
|
||||||
+aside-code("Example").
|
+aside-code("Example").
|
||||||
from spacy.util import load_model_from_init_py
|
from spacy.util import load_model_from_init_py
|
||||||
|
|
|
@ -541,13 +541,9 @@ p
|
||||||
| from an existing model data directory. All data files are copied over.
|
| from an existing model data directory. All data files are copied over.
|
||||||
| If the path to a #[code meta.json] is supplied, or a #[code meta.json] is
|
| If the path to a #[code meta.json] is supplied, or a #[code meta.json] is
|
||||||
| found in the input directory, this file is used. Otherwise, the data can
|
| found in the input directory, this file is used. Otherwise, the data can
|
||||||
| be entered directly from the command line. The required file templates
|
| be entered directly from the command line. After packaging, you can run
|
||||||
| are downloaded from
|
| #[code python setup.py sdist] from the newly created directory to turn
|
||||||
| #[+src(gh("spacy-dev-resources", "templates/model")) GitHub] to make
|
| your model into an installable archive file.
|
||||||
| sure you're always using the latest versions. This means you need to be
|
|
||||||
| connected to the internet to use this command. After packaging, you
|
|
||||||
| can run #[code python setup.py sdist] from the newly created directory
|
|
||||||
| to turn your model into an installable archive file.
|
|
||||||
|
|
||||||
+code(false, "bash", "$", false, false, true).
|
+code(false, "bash", "$", false, false, true).
|
||||||
spacy package [input_dir] [output_dir] [--meta-path] [--create-meta] [--force]
|
spacy package [input_dir] [output_dir] [--meta-path] [--create-meta] [--force]
|
||||||
|
|
|
@ -59,8 +59,8 @@ p This command will create a model package directory that should look like this:
|
||||||
└── en_example_model-1.0.0 # model data
|
└── en_example_model-1.0.0 # model data
|
||||||
|
|
||||||
p
|
p
|
||||||
| You can also find templates for all files in our
|
| You can also find templates for all files on
|
||||||
| #[+src(gh("spacy-dev-resources", "templates/model")) spaCy dev resources].
|
| #[+src(gh("spacy-models", "template")) GitHub].
|
||||||
| If you're creating the package manually, keep in mind that the directories
|
| If you're creating the package manually, keep in mind that the directories
|
||||||
| need to be named according to the naming conventions of
|
| need to be named according to the naming conventions of
|
||||||
| #[code lang_name] and #[code lang_name-version].
|
| #[code lang_name] and #[code lang_name-version].
|
||||||
|
|
Loading…
Reference in New Issue
Block a user