mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 19:39:13 +03:00
37c7c85a86
* Support nowrap setting in util.prints * Tidy up and fix whitespace * Simplify script and use read_jsonl helper * Add JSON schemas (see #2928) * Deprecate Doc.print_tree Will be replaced with Doc.to_json, which will produce a unified format * Add Doc.to_json() method (see #2928) Converts Doc objects to JSON using the same unified format as the training data. Method also supports serializing selected custom attributes in the doc._. space. * Remove outdated test * Add write_json and write_jsonl helpers * WIP: Update spacy train * Tidy up spacy train * WIP: Use wasabi for formatting * Add GoldParse helpers for JSON format * WIP: add debug-data command * Fix typo * Add missing import * Update wasabi pin * Add missing import * 💫 Refactor CLI (#2943) To be merged into #2932. ## Description - [x] refactor CLI To use [`wasabi`](https://github.com/ines/wasabi) - [x] use [`black`](https://github.com/ambv/black) for auto-formatting - [x] add `flake8` config - [x] move all messy UD-related scripts to `cli.ud` - [x] make converters function that take the opened file and return the converted data (instead of having them handle the IO) ### Types of change enhancement ## Checklist <!--- Before you submit the PR, go over this checklist and make sure you can tick off all the boxes. [] -> [x] --> - [x] I have submitted the spaCy Contributor Agreement. - [x] I ran the tests, and all new and existing tests passed. - [x] My changes don't require a change to the documentation, or if they do, I've added all required information. * Update wasabi pin * Delete old test * Update errors * Fix typo * Tidy up and format remaining code * Fix formatting * Improve formatting of messages * Auto-format remaining code * Add tok2vec stuff to spacy.train * Fix typo * Update wasabi pin * Fix path checks for when train() is called as function * Reformat and tidy up pretrain script * Update argument annotations * Raise error if model language doesn't match lang * Document new train command
57 lines
2.3 KiB
Python
57 lines
2.3 KiB
Python
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
import plac
|
|
from pathlib import Path
|
|
from wasabi import Printer
|
|
|
|
from ._messages import Messages
|
|
from ..compat import symlink_to, path2str
|
|
from .. import util
|
|
|
|
|
|
@plac.annotations(
|
|
origin=("package name or local path to model", "positional", None, str),
|
|
link_name=("name of shortuct link to create", "positional", None, str),
|
|
force=("force overwriting of existing link", "flag", "f", bool),
|
|
)
|
|
def link(origin, link_name, force=False, model_path=None):
|
|
"""
|
|
Create a symlink for models within the spacy/data directory. Accepts
|
|
either the name of a pip package, or the local path to the model data
|
|
directory. Linking models allows loading them via spacy.load(link_name).
|
|
"""
|
|
msg = Printer()
|
|
if util.is_package(origin):
|
|
model_path = util.get_package_path(origin)
|
|
else:
|
|
model_path = Path(origin) if model_path is None else Path(model_path)
|
|
if not model_path.exists():
|
|
msg.fail(
|
|
Messages.M008, Messages.M009.format(path=path2str(model_path)), exits=1
|
|
)
|
|
data_path = util.get_data_path()
|
|
if not data_path or not data_path.exists():
|
|
spacy_loc = Path(__file__).parent.parent
|
|
msg.fail(Messages.M010, Messages.M011.format(path=spacy_loc), exits=1)
|
|
link_path = util.get_data_path() / link_name
|
|
if link_path.is_symlink() and not force:
|
|
msg.fail(Messages.M012.format(name=link_name), Messages.M013, exits=1)
|
|
elif link_path.is_symlink(): # does a symlink exist?
|
|
# NB: It's important to check for is_symlink here and not for exists,
|
|
# because invalid/outdated symlinks would return False otherwise.
|
|
link_path.unlink()
|
|
elif link_path.exists(): # does it exist otherwise?
|
|
# NB: Check this last because valid symlinks also "exist".
|
|
msg.fail(Messages.M014.format(name=link_name), Messages.M015, exits=1)
|
|
details = "%s --> %s" % (path2str(model_path), path2str(link_path))
|
|
try:
|
|
symlink_to(link_path, model_path)
|
|
except: # noqa: E722
|
|
# This is quite dirty, but just making sure other errors are caught.
|
|
msg.fail(Messages.M016.format(name=link_name), Messages.M017)
|
|
msg.text(details)
|
|
raise
|
|
msg.good(Messages.M018, details)
|
|
msg.text(Messages.M019.format(name=link_name))
|