mirror of
https://github.com/explosion/spaCy.git
synced 2025-01-13 18:56:36 +03:00
37c7c85a86
* Support nowrap setting in util.prints * Tidy up and fix whitespace * Simplify script and use read_jsonl helper * Add JSON schemas (see #2928) * Deprecate Doc.print_tree Will be replaced with Doc.to_json, which will produce a unified format * Add Doc.to_json() method (see #2928) Converts Doc objects to JSON using the same unified format as the training data. Method also supports serializing selected custom attributes in the doc._. space. * Remove outdated test * Add write_json and write_jsonl helpers * WIP: Update spacy train * Tidy up spacy train * WIP: Use wasabi for formatting * Add GoldParse helpers for JSON format * WIP: add debug-data command * Fix typo * Add missing import * Update wasabi pin * Add missing import * 💫 Refactor CLI (#2943) To be merged into #2932. ## Description - [x] refactor CLI To use [`wasabi`](https://github.com/ines/wasabi) - [x] use [`black`](https://github.com/ambv/black) for auto-formatting - [x] add `flake8` config - [x] move all messy UD-related scripts to `cli.ud` - [x] make converters function that take the opened file and return the converted data (instead of having them handle the IO) ### Types of change enhancement ## Checklist <!--- Before you submit the PR, go over this checklist and make sure you can tick off all the boxes. [] -> [x] --> - [x] I have submitted the spaCy Contributor Agreement. - [x] I ran the tests, and all new and existing tests passed. - [x] My changes don't require a change to the documentation, or if they do, I've added all required information. * Update wasabi pin * Delete old test * Update errors * Fix typo * Tidy up and format remaining code * Fix formatting * Improve formatting of messages * Auto-format remaining code * Add tok2vec stuff to spacy.train * Fix typo * Update wasabi pin * Fix path checks for when train() is called as function * Reformat and tidy up pretrain script * Update argument annotations * Raise error if model language doesn't match lang * Document new train command
138 lines
3.3 KiB
Python
138 lines
3.3 KiB
Python
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
import os
|
|
import sys
|
|
import ujson
|
|
import itertools
|
|
|
|
from thinc.neural.util import copy_array
|
|
|
|
try:
|
|
import cPickle as pickle
|
|
except ImportError:
|
|
import pickle
|
|
|
|
try:
|
|
import copy_reg
|
|
except ImportError:
|
|
import copyreg as copy_reg
|
|
|
|
try:
|
|
from cupy.cuda.stream import Stream as CudaStream
|
|
except ImportError:
|
|
CudaStream = None
|
|
|
|
try:
|
|
import cupy
|
|
except ImportError:
|
|
cupy = None
|
|
|
|
try:
|
|
from thinc.neural.optimizers import Optimizer # noqa: F401
|
|
except ImportError:
|
|
from thinc.neural.optimizers import Adam as Optimizer # noqa: F401
|
|
|
|
pickle = pickle
|
|
copy_reg = copy_reg
|
|
CudaStream = CudaStream
|
|
cupy = cupy
|
|
copy_array = copy_array
|
|
izip = getattr(itertools, "izip", zip)
|
|
|
|
is_windows = sys.platform.startswith("win")
|
|
is_linux = sys.platform.startswith("linux")
|
|
is_osx = sys.platform == "darwin"
|
|
|
|
# See: https://github.com/benjaminp/six/blob/master/six.py
|
|
is_python2 = sys.version_info[0] == 2
|
|
is_python3 = sys.version_info[0] == 3
|
|
is_python_pre_3_5 = is_python2 or (is_python3 and sys.version_info[1] < 5)
|
|
|
|
if is_python2:
|
|
bytes_ = str
|
|
unicode_ = unicode # noqa: F821
|
|
basestring_ = basestring # noqa: F821
|
|
input_ = raw_input # noqa: F821
|
|
json_dumps = lambda data, indent=2: ujson.dumps(
|
|
data, indent=indent, escape_forward_slashes=False
|
|
).decode("utf8")
|
|
path2str = lambda path: str(path).decode("utf8")
|
|
|
|
elif is_python3:
|
|
bytes_ = bytes
|
|
unicode_ = str
|
|
basestring_ = str
|
|
input_ = input
|
|
json_dumps = lambda data, indent=2: ujson.dumps(
|
|
data, indent=indent, escape_forward_slashes=False
|
|
)
|
|
path2str = lambda path: str(path)
|
|
|
|
|
|
def b_to_str(b_str):
|
|
if is_python2:
|
|
return b_str
|
|
# important: if no encoding is set, string becomes "b'...'"
|
|
return str(b_str, encoding="utf8")
|
|
|
|
|
|
def getattr_(obj, name, *default):
|
|
if is_python3 and isinstance(name, bytes):
|
|
name = name.decode("utf8")
|
|
return getattr(obj, name, *default)
|
|
|
|
|
|
def symlink_to(orig, dest):
|
|
if is_windows:
|
|
import subprocess
|
|
|
|
subprocess.call(["mklink", "/d", path2str(orig), path2str(dest)], shell=True)
|
|
else:
|
|
orig.symlink_to(dest)
|
|
|
|
|
|
def symlink_remove(link):
|
|
# https://stackoverflow.com/q/26554135/6400719
|
|
if os.path.isdir(path2str(link)) and is_windows:
|
|
# this should only be on Py2.7 and windows
|
|
os.rmdir(path2str(link))
|
|
else:
|
|
os.unlink(path2str(link))
|
|
|
|
|
|
def is_config(python2=None, python3=None, windows=None, linux=None, osx=None):
|
|
return (
|
|
python2 in (None, is_python2)
|
|
and python3 in (None, is_python3)
|
|
and windows in (None, is_windows)
|
|
and linux in (None, is_linux)
|
|
and osx in (None, is_osx)
|
|
)
|
|
|
|
|
|
def normalize_string_keys(old):
|
|
"""Given a dictionary, make sure keys are unicode strings, not bytes."""
|
|
new = {}
|
|
for key, value in old.items():
|
|
if isinstance(key, bytes_):
|
|
new[key.decode("utf8")] = value
|
|
else:
|
|
new[key] = value
|
|
return new
|
|
|
|
|
|
def import_file(name, loc):
|
|
loc = str(loc)
|
|
if is_python_pre_3_5:
|
|
import imp
|
|
|
|
return imp.load_source(name, loc)
|
|
else:
|
|
import importlib.util
|
|
|
|
spec = importlib.util.spec_from_file_location(name, str(loc))
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|