From adbcac65918137634e080e3e4689c3456cde593b Mon Sep 17 00:00:00 2001 From: ines Date: Mon, 20 Mar 2017 22:48:21 +0100 Subject: [PATCH 01/23] Fix spacing --- spacy/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spacy/util.py b/spacy/util.py index 49c51b436..d1252e41d 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -13,6 +13,7 @@ import textwrap from .attrs import TAG, HEAD, DEP, ENT_IOB, ENT_TYPE + try: basestring except NameError: From a6c036180344899471c715ad5798b78c116fdee5 Mon Sep 17 00:00:00 2001 From: ines Date: Mon, 20 Mar 2017 22:48:32 +0100 Subject: [PATCH 02/23] Handle raw_input vs input in Python 2 and 3 --- spacy/util.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spacy/util.py b/spacy/util.py index d1252e41d..f8fc76b05 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -20,6 +20,12 @@ except NameError: basestring = str +try: + raw_input +except NameError: # Python 3 + raw_input = input + + LANGUAGES = {} _data_path = pathlib.Path(__file__).parent / 'data' From 5aea327a5b5a4fe74b023d6ecad0689a41ee895f Mon Sep 17 00:00:00 2001 From: ines Date: Mon, 20 Mar 2017 22:48:56 +0100 Subject: [PATCH 03/23] Add util function to get raw user input --- spacy/util.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/spacy/util.py b/spacy/util.py index f8fc76b05..1f1cdbb6e 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -168,6 +168,17 @@ def parse_package_meta(package_path, package, require=True): return None +def get_raw_input(description, default=False): + """Get user input via raw_input / input and return input value. Takes a + description for the prompt, and an optional default value that's displayed + with the prompt.""" + + additional = ' (default: {d})'.format(d=default) if default else '' + prompt = ' {d}{a}: '.format(d=description, a=additional) + user_input = raw_input(prompt) + return user_input + + def print_table(data, **kwargs): """Print data in table format. Can either take a list of tuples or a dictionary, which will be converted to a list of tuples.""" From a54e3c2efe6d388b1cb2fa0bc9fc867165c9025d Mon Sep 17 00:00:00 2001 From: ines Date: Mon, 20 Mar 2017 22:49:36 +0100 Subject: [PATCH 04/23] Remove empty line --- spacy/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spacy/__main__.py b/spacy/__main__.py index 9addbccde..cf740c8fe 100644 --- a/spacy/__main__.py +++ b/spacy/__main__.py @@ -1,5 +1,4 @@ # coding: utf8 -# from __future__ import print_function # NB! This breaks in plac on Python 2!! #from __future__ import unicode_literals, From bf240132d70b497e6c5c57407e5ca1cfdc9b17e3 Mon Sep 17 00:00:00 2001 From: ines Date: Mon, 20 Mar 2017 22:50:13 +0100 Subject: [PATCH 05/23] Add cli.package command to build model packages --- spacy/__main__.py | 17 ++++- spacy/cli/__init__.py | 1 + spacy/cli/package.py | 149 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 spacy/cli/package.py diff --git a/spacy/__main__.py b/spacy/__main__.py index cf740c8fe..ba34c478f 100644 --- a/spacy/__main__.py +++ b/spacy/__main__.py @@ -7,12 +7,13 @@ import plac from spacy.cli import download as cli_download from spacy.cli import link as cli_link from spacy.cli import info as cli_info +from spacy.cli import package as cli_package class CLI(object): """Command-line interface for spaCy""" - commands = ('download', 'link', 'info') + commands = ('download', 'link', 'info', 'package') @plac.annotations( model=("model to download (shortcut or model name)", "positional", None, str), @@ -58,6 +59,20 @@ class CLI(object): cli_info(model, markdown) + @plac.annotations( + input_dir=("directory with model data", "positional", None, str), + output_dir=("output directory", "positional", None, str) + ) + def package(self, input_dir, output_dir): + """ + Generate Python package for model data, including meta and required + installation files. A new directory will be created in the specified + output directory. + """ + + cli_package(input_dir, output_dir) + + def __missing__(self, name): print("\n Command %r does not exist\n" % name) diff --git a/spacy/cli/__init__.py b/spacy/cli/__init__.py index 2c45b471a..2383e04b9 100644 --- a/spacy/cli/__init__.py +++ b/spacy/cli/__init__.py @@ -1,3 +1,4 @@ from .download import download from .info import info from .link import link +from .package import package diff --git a/spacy/cli/package.py b/spacy/cli/package.py new file mode 100644 index 000000000..9d1ff7183 --- /dev/null +++ b/spacy/cli/package.py @@ -0,0 +1,149 @@ +# coding: utf8 +from __future__ import unicode_literals + +import json +from shutil import copytree +from pathlib import Path + +from .. import about +from .. import util + + +def package(input_dir, output_dir): + input_path = Path(input_dir) + output_path = Path(output_dir) + check_dirs(input_path, output_path) + + meta = generate_meta() + model_name = meta['lang'] + '_' + meta['name'] + model_name_v = model_name + '-' + meta['version'] + main_path = output_path / model_name_v + package_path = main_path / model_name + + Path.mkdir(package_path, parents=True) + copytree(input_path, package_path / model_name_v) + create_file(main_path / 'meta.json', json.dumps(meta, indent=2)) + create_file(main_path / 'setup.py', TEMPLATE_SETUP.strip()) + create_file(main_path / 'MANIFEST.in', TEMPLATE_MANIFEST.strip()) + create_file(package_path / '__init__.py', TEMPLATE_INIT.strip()) + + util.print_msg( + main_path.as_posix(), + "To build the package, run python setup.py sdist in that directory.", + title="Successfully reated package {p}".format(p=model_name_v)) + + +def check_dirs(input_path, output_path): + if not input_path.exists(): + util.sys_exit(input_path.as_poisx(), title="Model directory not found") + if not output_path.exists(): + util.sys_exit(output_path.as_posix(), title="Output directory not found") + + +def create_file(file_path, contents): + file_path.touch() + file_path.write_text(contents, encoding='utf-8') + + +def generate_meta(): + settings = [('lang', 'Model language', 'en'), + ('name', 'Model name', 'model'), + ('version', 'Model version', '0.0.0'), + ('spacy_version', 'Required spaCy version', '>=2.0.0,<3.0.0'), + ('description', 'Model description', False), + ('author', 'Author', False), + ('email', 'Author email', False), + ('url', 'Author website', False), + ('license', 'License', 'MIT')] + + util.print_msg("Enter the package settings for your model.", title="Generating meta.json") + + meta = {} + for setting, desc, default in settings: + response = util.get_raw_input(desc, default) + meta[setting] = default if response == '' and default else response + return meta + + +TEMPLATE_MANIFEST = """ +include meta.json +""" + + +TEMPLATE_SETUP = """ +#!/usr/bin/env python +# coding: utf8 +from __future__ import unicode_literals + +import io +import json +from os import path, walk +from shutil import copy +from setuptools import setup + + +def load_meta(fp): + with io.open(fp, encoding='utf8') as f: + return json.load(f) + + +def list_files(data_dir): + output = [] + for root, _, filenames in walk(data_dir): + for filename in filenames: + if not filename.startswith('.'): + output.append(path.join(root, filename)) + output = [path.relpath(p, path.dirname(data_dir)) for p in output] + output.append('meta.json') + return output + + +def setup_package(): + root = path.abspath(path.dirname(__file__)) + meta_path = path.join(root, 'meta.json') + meta = load_meta(meta_path) + model_name = str(meta['lang'] + '_' + meta['name']) + model_dir = path.join(model_name, model_name + '-' + meta['version']) + + copy(meta_path, path.join(root, model_name)) + copy(meta_path, path.join(root, model_dir)) + + setup( + name=model_name, + description=meta['description'], + author=meta['author'], + author_email=meta['email'], + url=meta['url'], + version=meta['version'], + license=meta['license'], + packages=[model_name], + package_data={model_name: list_files(model_dir)}, + install_requires=['spacy' + meta['spacy_version']], + zip_safe=False, + ) + + +if __name__ == '__main__': + setup_package() +""" + + +TEMPLATE_INIT = """ +from pathlib import Path +from spacy.util import get_lang_class +import pkg_resources +import json + + +def load_meta(): + with (Path(__file__).parent / 'meta.json').open() as f: + return json.load(f) + + +def load(**kwargs): + meta = load_meta() + version = meta['version'] + data_dir = pkg_resources.resource_filename(__name__, __name__ + '-' + version) + lang = get_lang_class(meta['lang']) + return lang(path=Path(data_dir), **kwargs) +""" From b2bcdec0f608dcf64147e94f1547b267baf007b6 Mon Sep 17 00:00:00 2001 From: ines Date: Mon, 20 Mar 2017 22:50:55 +0100 Subject: [PATCH 06/23] Update docstring --- spacy/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/__main__.py b/spacy/__main__.py index ba34c478f..23d87acb3 100644 --- a/spacy/__main__.py +++ b/spacy/__main__.py @@ -67,7 +67,7 @@ class CLI(object): """ Generate Python package for model data, including meta and required installation files. A new directory will be created in the specified - output directory. + output directory, and model data will be copied over. """ cli_package(input_dir, output_dir) From 8eb9a2b35503a54ac7d3aca403e4d48516a8900b Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 02:05:14 +0100 Subject: [PATCH 07/23] Fix formatting --- spacy/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/__main__.py b/spacy/__main__.py index 23d87acb3..e539ed78d 100644 --- a/spacy/__main__.py +++ b/spacy/__main__.py @@ -32,8 +32,8 @@ class CLI(object): @plac.annotations( origin=("package name or local path to model", "positional", None, str), - link_name=("Name of shortuct link to create", "positional", None, str), - force=("Force overwriting of existing link", "flag", "f", bool) + link_name=("name of shortuct link to create", "positional", None, str), + force=("force overwriting of existing link", "flag", "f", bool) ) def link(self, origin, link_name, force=False): """ From 448a916d0d3e3af2d5fe161cd5ed5828e6517ab7 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 02:05:34 +0100 Subject: [PATCH 08/23] Add --force option to override directory --- spacy/__main__.py | 7 ++++--- spacy/cli/package.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/spacy/__main__.py b/spacy/__main__.py index e539ed78d..a5ba66fee 100644 --- a/spacy/__main__.py +++ b/spacy/__main__.py @@ -61,16 +61,17 @@ class CLI(object): @plac.annotations( input_dir=("directory with model data", "positional", None, str), - output_dir=("output directory", "positional", None, str) + output_dir=("output directory", "positional", None, str), + force=("force overwriting of existing output directory", "flag", "f", bool) ) - def package(self, input_dir, output_dir): + def package(self, input_dir, output_dir, force=False): """ Generate Python package for model data, including meta and required installation files. A new directory will be created in the specified output directory, and model data will be copied over. """ - cli_package(input_dir, output_dir) + cli_package(input_dir, output_dir, force) def __missing__(self, name): diff --git a/spacy/cli/package.py b/spacy/cli/package.py index 9d1ff7183..abd3f6e4e 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -9,7 +9,7 @@ from .. import about from .. import util -def package(input_dir, output_dir): +def package(input_dir, output_dir, force): input_path = Path(input_dir) output_path = Path(output_dir) check_dirs(input_path, output_path) From 64e38f304e1f9374e3b731e3721434fe33867bb4 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 02:06:29 +0100 Subject: [PATCH 09/23] Only import shutil --- spacy/cli/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index abd3f6e4e..d23b03821 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import json -from shutil import copytree +import shutil from pathlib import Path from .. import about @@ -21,7 +21,7 @@ def package(input_dir, output_dir, force): package_path = main_path / model_name Path.mkdir(package_path, parents=True) - copytree(input_path, package_path / model_name_v) + shutil.copytree(input_path, package_path / model_name_v) create_file(main_path / 'meta.json', json.dumps(meta, indent=2)) create_file(main_path / 'setup.py', TEMPLATE_SETUP.strip()) create_file(main_path / 'MANIFEST.in', TEMPLATE_MANIFEST.strip()) From 46bc3c36b00dfb95f90da5769181d0255f860cd8 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 02:06:37 +0100 Subject: [PATCH 10/23] Fix typo --- spacy/cli/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index d23b03821..bf424e075 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -30,7 +30,7 @@ def package(input_dir, output_dir, force): util.print_msg( main_path.as_posix(), "To build the package, run python setup.py sdist in that directory.", - title="Successfully reated package {p}".format(p=model_name_v)) + title="Successfully created package {p}".format(p=model_name_v)) def check_dirs(input_path, output_path): From 5230ed5b98e3f1e9c83b1205c87db962c5844804 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 02:06:53 +0100 Subject: [PATCH 11/23] Move directory check and overwriting/creating dirs to own function --- spacy/cli/package.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index bf424e075..59b45ab5f 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -20,7 +20,7 @@ def package(input_dir, output_dir, force): main_path = output_path / model_name_v package_path = main_path / model_name - Path.mkdir(package_path, parents=True) + create_dirs(package_path, force) shutil.copytree(input_path, package_path / model_name_v) create_file(main_path / 'meta.json', json.dumps(meta, indent=2)) create_file(main_path / 'setup.py', TEMPLATE_SETUP.strip()) @@ -40,6 +40,17 @@ def check_dirs(input_path, output_path): util.sys_exit(output_path.as_posix(), title="Output directory not found") +def create_dirs(package_path, force): + if package_path.exists(): + if force: + shutil.rmtree(package_path) + else: + util.sys_exit(package_path.as_posix(), + "Please delete the directory and try again.", + title="Package directory already exists") + Path.mkdir(package_path, parents=True) + + def create_file(file_path, contents): file_path.touch() file_path.write_text(contents, encoding='utf-8') From 3f4e3fda1d21a90b5d7b3e3fe70e650120b19c84 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 11:17:36 +0100 Subject: [PATCH 12/23] Update command and fetch file templates from GitHub While feature is still experimental, this allows files to be modified without having to ship a new version of spaCy. --- spacy/cli/package.py | 105 ++++++++----------------------------------- 1 file changed, 18 insertions(+), 87 deletions(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index 59b45ab5f..6a0f36ff9 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import json import shutil +import requests from pathlib import Path from .. import about @@ -14,7 +15,11 @@ def package(input_dir, output_dir, force): output_path = Path(output_dir) check_dirs(input_path, output_path) + template_setup = get_template('setup.py') + template_init = get_template('en_model_name/__init__.py') + template_manifest = 'include meta.json' meta = generate_meta() + model_name = meta['lang'] + '_' + meta['name'] model_name_v = model_name + '-' + meta['version'] main_path = output_path / model_name_v @@ -23,13 +28,13 @@ def package(input_dir, output_dir, force): create_dirs(package_path, force) shutil.copytree(input_path, package_path / model_name_v) create_file(main_path / 'meta.json', json.dumps(meta, indent=2)) - create_file(main_path / 'setup.py', TEMPLATE_SETUP.strip()) - create_file(main_path / 'MANIFEST.in', TEMPLATE_MANIFEST.strip()) - create_file(package_path / '__init__.py', TEMPLATE_INIT.strip()) + create_file(main_path / 'setup.py', template_setup) + create_file(main_path / 'MANIFEST.in', template_manifest) + create_file(package_path / '__init__.py', template_init) util.print_msg( main_path.as_posix(), - "To build the package, run python setup.py sdist in that directory.", + "To build the package, run `python setup.py sdist` in that directory.", title="Successfully created package {p}".format(p=model_name_v)) @@ -60,7 +65,7 @@ def generate_meta(): settings = [('lang', 'Model language', 'en'), ('name', 'Model name', 'model'), ('version', 'Model version', '0.0.0'), - ('spacy_version', 'Required spaCy version', '>=2.0.0,<3.0.0'), + ('spacy_version', 'Required spaCy version', '>=1.7.0,<2.0.0'), ('description', 'Model description', False), ('author', 'Author', False), ('email', 'Author email', False), @@ -76,85 +81,11 @@ def generate_meta(): return meta -TEMPLATE_MANIFEST = """ -include meta.json -""" - - -TEMPLATE_SETUP = """ -#!/usr/bin/env python -# coding: utf8 -from __future__ import unicode_literals - -import io -import json -from os import path, walk -from shutil import copy -from setuptools import setup - - -def load_meta(fp): - with io.open(fp, encoding='utf8') as f: - return json.load(f) - - -def list_files(data_dir): - output = [] - for root, _, filenames in walk(data_dir): - for filename in filenames: - if not filename.startswith('.'): - output.append(path.join(root, filename)) - output = [path.relpath(p, path.dirname(data_dir)) for p in output] - output.append('meta.json') - return output - - -def setup_package(): - root = path.abspath(path.dirname(__file__)) - meta_path = path.join(root, 'meta.json') - meta = load_meta(meta_path) - model_name = str(meta['lang'] + '_' + meta['name']) - model_dir = path.join(model_name, model_name + '-' + meta['version']) - - copy(meta_path, path.join(root, model_name)) - copy(meta_path, path.join(root, model_dir)) - - setup( - name=model_name, - description=meta['description'], - author=meta['author'], - author_email=meta['email'], - url=meta['url'], - version=meta['version'], - license=meta['license'], - packages=[model_name], - package_data={model_name: list_files(model_dir)}, - install_requires=['spacy' + meta['spacy_version']], - zip_safe=False, - ) - - -if __name__ == '__main__': - setup_package() -""" - - -TEMPLATE_INIT = """ -from pathlib import Path -from spacy.util import get_lang_class -import pkg_resources -import json - - -def load_meta(): - with (Path(__file__).parent / 'meta.json').open() as f: - return json.load(f) - - -def load(**kwargs): - meta = load_meta() - version = meta['version'] - data_dir = pkg_resources.resource_filename(__name__, __name__ + '-' + version) - lang = get_lang_class(meta['lang']) - return lang(path=Path(data_dir), **kwargs) -""" +def get_template(filepath): + url = 'https://raw.githubusercontent.com/explosion/spacy-dev-resources/master/templates/model/' + r = requests.get(url + filepath) + if r.status_code != 200: + util.sys_exit( + "Couldn't fetch template files from GitHub.", + title="Server error ({c})".format(c=r.status_code)) + return r.text From 09b24bc5a9dfd69d2f95c0225599c44170659351 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 11:19:21 +0100 Subject: [PATCH 13/23] Add docs for package command --- spacy/__main__.py | 2 +- website/docs/usage/cli.jade | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/spacy/__main__.py b/spacy/__main__.py index a5ba66fee..cde146cba 100644 --- a/spacy/__main__.py +++ b/spacy/__main__.py @@ -62,7 +62,7 @@ class CLI(object): @plac.annotations( input_dir=("directory with model data", "positional", None, str), output_dir=("output directory", "positional", None, str), - force=("force overwriting of existing output directory", "flag", "f", bool) + force=("force overwriting of existing folder in output directory", "flag", "f", bool) ) def package(self, input_dir, output_dir, force=False): """ diff --git a/website/docs/usage/cli.jade b/website/docs/usage/cli.jade index 990117542..4a9ba3dd1 100644 --- a/website/docs/usage/cli.jade +++ b/website/docs/usage/cli.jade @@ -103,3 +103,40 @@ p +cell #[code --help], #[code -h] +cell flag +cell Show help message and available arguments. + ++h(2, "package") Package + +tag experimental + +p + | Generate a #[+a("/docs/usage/models#own-models") model Python package] + | from an existing model data directory. All data files are copied over, + | and the meta data can be entered directly from the command line. While + | this feature is still experimental, the templates for the + | #[+src(gh("spacy-dev-resources", "templates/model/setup.py")) setup.py] and + | #[+src(gh("spacy-dev-resources", "templates/model/en_morel_name/__init__.py")) __init__.py] + | are downloaded from GitHub. This means you need to be connected to the + | internet to use this command. + ++code(false, "bash"). + python -m spacy package [input_dir] [output_dir] [--force] + ++table(["Argument", "Type", "Description"]) + +row + +cell #[code input_dir] + +cell positional + +cell Path to directory containing model data. + + +row + +cell #[code output_dir] + +cell positional + +cell Directory to create package folder in. + + +row + +cell #[code --force], #[code -f] + +cell flag + +cell Force overwriting of existing folder in output directory. + + +row + +cell #[code --help], #[code -h] + +cell flag + +cell Show help message and available arguments. From 49bbfdaac1036f052b38f6991ea65de8efc3478f Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 11:25:01 +0100 Subject: [PATCH 14/23] Add info on CLI to docs on own models --- website/docs/usage/models.jade | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/docs/usage/models.jade b/website/docs/usage/models.jade index ae1417a29..39c271df4 100644 --- a/website/docs/usage/models.jade +++ b/website/docs/usage/models.jade @@ -238,7 +238,11 @@ p | #[+a("/docs/usage/adding-languages") additional languages], you can | create a shortuct link for it by pointing #[code spacy.link] to the | model's data directory. To allow your model to be downloaded and - | installed via pip, you'll also need to generate a package for it. + | installed via pip, you'll also need to generate a package for it. You can + | do this manually, or via the new + | #[+a("/docs/usage/cli#package") #[code spacy package] command] that will + | create all required files, and walk you through generating the meta data. + +infobox("Important note") | The model packages are #[strong not suitable] for the public From cf0094187e356e9b762a1796b092734d4e30d654 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 11:32:38 +0100 Subject: [PATCH 15/23] Fetch MANIFEST.in from GitHub as well --- spacy/cli/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index 6a0f36ff9..5894ec049 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -16,8 +16,8 @@ def package(input_dir, output_dir, force): check_dirs(input_path, output_path) template_setup = get_template('setup.py') + template_manifest = get_template('MANIFEST.in') template_init = get_template('en_model_name/__init__.py') - template_manifest = 'include meta.json' meta = generate_meta() model_name = meta['lang'] + '_' + meta['name'] From fa6e3cefbb482e97d0c21c51d852bcda5f31b089 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 11:35:29 +0100 Subject: [PATCH 16/23] Simplify package command docs --- website/docs/usage/cli.jade | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/website/docs/usage/cli.jade b/website/docs/usage/cli.jade index 4a9ba3dd1..66be83923 100644 --- a/website/docs/usage/cli.jade +++ b/website/docs/usage/cli.jade @@ -111,11 +111,9 @@ p | Generate a #[+a("/docs/usage/models#own-models") model Python package] | from an existing model data directory. All data files are copied over, | and the meta data can be entered directly from the command line. While - | this feature is still experimental, the templates for the - | #[+src(gh("spacy-dev-resources", "templates/model/setup.py")) setup.py] and - | #[+src(gh("spacy-dev-resources", "templates/model/en_morel_name/__init__.py")) __init__.py] - | are downloaded from GitHub. This means you need to be connected to the - | internet to use this command. + | this feature is still experimental, the required file templates are + | downloaded from #[+src(gh("spacy-dev-resources", "templates/model")) GitHub]. + | This means you need to be connected to the internet to use this command. +code(false, "bash"). python -m spacy package [input_dir] [output_dir] [--force] From 3e134b5b2b5fdf7866ce63201be764897861a4b7 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 12:15:33 +0100 Subject: [PATCH 17/23] Make sure paths in copytree and rmtree are strings --- spacy/cli/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index 5894ec049..a5c41adec 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -26,7 +26,7 @@ def package(input_dir, output_dir, force): package_path = main_path / model_name create_dirs(package_path, force) - shutil.copytree(input_path, package_path / model_name_v) + shutil.copytree((input_path, package_path / model_name_v).as_posix()) create_file(main_path / 'meta.json', json.dumps(meta, indent=2)) create_file(main_path / 'setup.py', template_setup) create_file(main_path / 'MANIFEST.in', template_manifest) @@ -48,7 +48,7 @@ def check_dirs(input_path, output_path): def create_dirs(package_path, force): if package_path.exists(): if force: - shutil.rmtree(package_path) + shutil.rmtree(package_path.as_posix()) else: util.sys_exit(package_path.as_posix(), "Please delete the directory and try again.", From ae466475607540a0b439a808dde6a5d553c6084f Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 12:21:42 +0100 Subject: [PATCH 18/23] Fix brackets --- spacy/cli/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index a5c41adec..5072e272b 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -26,7 +26,7 @@ def package(input_dir, output_dir, force): package_path = main_path / model_name create_dirs(package_path, force) - shutil.copytree((input_path, package_path / model_name_v).as_posix()) + shutil.copytree(input_path, (package_path / model_name_v).as_posix()) create_file(main_path / 'meta.json', json.dumps(meta, indent=2)) create_file(main_path / 'setup.py', template_setup) create_file(main_path / 'MANIFEST.in', template_manifest) From 83a999ea83e7298ce9b999d91087dc8556f57956 Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 12:24:43 +0100 Subject: [PATCH 19/23] Change default license from MIT to CC --- spacy/cli/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index 5072e272b..e2c8000b3 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -70,7 +70,7 @@ def generate_meta(): ('author', 'Author', False), ('email', 'Author email', False), ('url', 'Author website', False), - ('license', 'License', 'MIT')] + ('license', 'License', 'CC BY-NC 3.0')] util.print_msg("Enter the package settings for your model.", title="Generating meta.json") From d74aa428ad6137b585ea000bdd8f2dde3c4da03d Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 12:26:00 +0100 Subject: [PATCH 20/23] Fix path --- spacy/cli/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index e2c8000b3..b4dc76e9a 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -26,7 +26,7 @@ def package(input_dir, output_dir, force): package_path = main_path / model_name create_dirs(package_path, force) - shutil.copytree(input_path, (package_path / model_name_v).as_posix()) + shutil.copytree(input_path.as_posix(), (package_path / model_name_v).as_posix()) create_file(main_path / 'meta.json', json.dumps(meta, indent=2)) create_file(main_path / 'setup.py', template_setup) create_file(main_path / 'MANIFEST.in', template_manifest) From c3a9f738960356dc395a789b786aa095bdad8e4f Mon Sep 17 00:00:00 2001 From: ines Date: Tue, 21 Mar 2017 12:35:22 +0100 Subject: [PATCH 21/23] Fix writing to file --- spacy/cli/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index b4dc76e9a..5cab2b4bc 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -58,7 +58,7 @@ def create_dirs(package_path, force): def create_file(file_path, contents): file_path.touch() - file_path.write_text(contents, encoding='utf-8') + file_path.open('w').write(contents, encoding='utf-8') def generate_meta(): From ce065e5d65bc2a880d9e1993129b6beff6468c39 Mon Sep 17 00:00:00 2001 From: ines Date: Wed, 22 Mar 2017 10:02:14 +0100 Subject: [PATCH 22/23] Fix imports --- spacy/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/spacy/__init__.py b/spacy/__init__.py index 70b3363d6..80bd1c539 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -5,7 +5,7 @@ import json from pathlib import Path from .util import set_lang_class, get_lang_class, parse_package_meta from .deprecated import resolve_model_name -from .cli.info import info +from .cli import info from . import en from . import de @@ -49,7 +49,3 @@ def load(name, **overrides): overrides['path'] = model_path return cls(**overrides) - - -def info(name, markdown): - info(name, markdown) From 4a9a1126a4aabfeb20fe555c042d333b1d6c982f Mon Sep 17 00:00:00 2001 From: ines Date: Wed, 22 Mar 2017 10:02:59 +0100 Subject: [PATCH 23/23] Update syntax highlighting color scheme --- website/assets/css/_variables.sass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/assets/css/_variables.sass b/website/assets/css/_variables.sass index bfef915be..1c38d114a 100644 --- a/website/assets/css/_variables.sass +++ b/website/assets/css/_variables.sass @@ -44,7 +44,7 @@ $color-red: #d9515d $color-green: #3ec930 $color-yellow: #f4c025 -$syntax-highlighting: ( comment: #949e9b, tag: #3ec930, number: #B084EB, selector: #FFB86C, operator: #FF2C6D, function: #09a3d5, keyword: #45A9F9, regex: #f4c025 ) +$syntax-highlighting: ( comment: #949e9b, tag: #b084eb, number: #b084eb, selector: #ffb86c, operator: #ff2c6d, function: #35b3dc, keyword: #45a9f9, regex: #f4c025 ) $pattern: $color-theme url("/assets/img/pattern_#{$theme}.jpg") center top repeat $pattern-overlay: transparent url("/assets/img/pattern_landing.jpg") center -138px no-repeat