explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-11 09:00:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update converter to produce DocBin

Browse Source
This commit is contained in:
Matthew Honnibal 2020-06-20 03:50:13 +02:00
parent d9a8fdf4b7
commit 3a73d95dcc
1 changed files with 42 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
spacy/cli/convert.py
View File

@ -2,6 +2,7 @@ from pathlib import Path
from wasabi import Printer from wasabi import Printer
import srsly import srsly
import re import re
import sys
from .converters import conllu2json, iob2json, conll_ner2json from .converters import conllu2json, iob2json, conll_ner2json
from .converters import ner_jsonl2json from .converters import ner_jsonl2json
@ -11,15 +12,29 @@ from .converters import ner_jsonl2json
# matched by file extension and content. To add a converter, add a new # matched by file extension and content. To add a converter, add a new
# entry to this dict with the file extension mapped to the converter function # entry to this dict with the file extension mapped to the converter function
# imported from /converters. # imported from /converters.
CONVERTERS = {
"conllubio": conllu2json, DOC_CONVERTERS = {
"conllu": conllu2json, "conllubio": conllu2doc,
"conll": conllu2json, "conllu": conllu2doc,
"ner": conll_ner2json, "conll": conllu2doc,
"iob": iob2json, "ner": conll_ner2doc,
"jsonl": ner_jsonl2json, "iob": iob2doc,
"jsonl": ner_jsonl2doc,
"json": json2docs,
} }
ALL_ATTRS = [
"ORTH",
"TAG",
"HEAD",
"DEP",
"SENT_START",
"ENT_IOB",
"ENT_TYPE",
"LEMMA",
"MORPH",
]
# File types # File types
FILE_TYPES = ("json", "jsonl", "msg") FILE_TYPES = ("json", "jsonl", "msg")
FILE_TYPES_STDOUT = ("json", "jsonl") FILE_TYPES_STDOUT = ("json", "jsonl")
@ -82,7 +97,7 @@ def convert(
ner_map = srsly.read_json(ner_map_path) ner_map = srsly.read_json(ner_map_path)
# Use converter function to convert data # Use converter function to convert data
func = CONVERTERS[converter] func = CONVERTERS[converter]
data = func( docs = func(
input_data, input_data,
n_sents=n_sents, n_sents=n_sents,
seg_sents=seg_sents, seg_sents=seg_sents,
@ -93,23 +108,27 @@ def convert(
no_print=no_print, no_print=no_print,
ner_map=ner_map, ner_map=ner_map,
) )
if output_dir != "-": if write_json:
# Export data to a file data = docs2json(docs)
suffix = f".{file_type}"
output_file = Path(output_dir) / Path(input_path.parts[-1]).with_suffix(suffix)
if file_type == "json":
srsly.write_json(output_file, data)
elif file_type == "jsonl":
srsly.write_jsonl(output_file, data)
elif file_type == "msg":
srsly.write_msgpack(output_file, data)
msg.good(f"Generated output file ({len(data)} documents): {output_file}")
else: else:
# Print to stdout data = DocBin(attrs=ALL_ATTRS, docs=docs).to_bytes()
if file_type == "json":
if output_dir == "-":
if write_json:
srsly.write_json("-", data) srsly.write_json("-", data)
elif file_type == "jsonl": else:
srsly.write_jsonl("-", data) sys.stdout.write(data)
else:
# Export data to a file
if write_json:
suffix = f".{file_type}"
output_file = output_dir / input_path.parts[-1].with_suffix(suffix)
srsly.write_json(output_file, data)
else:
output_file = output_dir / input_path.parts[-1].with_suffix("spacy")
with output_file.open("wb") as file_:
file_.write(data)
msg.good(f"Generated output file ({len(data)} documents): {output_file}")
def autodetect_ner_format(input_data): def autodetect_ner_format(input_data):