explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 01:16:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add NER converter for CoNLL 2003 data

Browse Source
This commit is contained in:
Matthew Honnibal 2017-10-09 20:06:28 -05:00
parent 808d8740d6
commit 735d18654d
2 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
spacy/cli/convert.py
View File

@ -4,7 +4,7 @@ from __future__ import unicode_literals
import plac import plac
from pathlib import Path from pathlib import Path
from .converters import conllu2json, iob2json from .converters import conllu2json, iob2json, conll_ner2json
from ..util import prints from ..util import prints
# Converters are matched by file extension. To add a converter, add a new entry # Converters are matched by file extension. To add a converter, add a new entry
@ -12,9 +12,10 @@ from ..util import prints
# from /converters. # from /converters.
CONVERTERS = { CONVERTERS = {
'.conllu': conllu2json, 'conllu': conllu2json,
'.conll': conllu2json, 'conll': conllu2json,
'.iob': iob2json, 'ner': conll_ner2json,
'iob': iob2json,
} }
@ -22,9 +23,11 @@ CONVERTERS = {
input_file=("input file", "positional", None, str), input_file=("input file", "positional", None, str),
output_dir=("output directory for converted file", "positional", None, str), output_dir=("output directory for converted file", "positional", None, str),
n_sents=("Number of sentences per doc", "option", "n", int), n_sents=("Number of sentences per doc", "option", "n", int),
converter=("Name of converter (auto, iob, conllu or ner)", "option", "c", str),
morphology=("Enable appending morphology to tags", "flag", "m", bool) morphology=("Enable appending morphology to tags", "flag", "m", bool)
) )
def convert(cmd, input_file, output_dir, n_sents=1, morphology=False): def convert(cmd, input_file, output_dir, n_sents=1, morphology=False,
converter='auto'):
""" """
Convert files into JSON format for use with train command and other Convert files into JSON format for use with train command and other
experiment management functions. experiment management functions.
@ -35,9 +38,11 @@ def convert(cmd, input_file, output_dir, n_sents=1, morphology=False):
prints(input_path, title="Input file not found", exits=1) prints(input_path, title="Input file not found", exits=1)
if not output_path.exists(): if not output_path.exists():
prints(output_path, title="Output directory not found", exits=1) prints(output_path, title="Output directory not found", exits=1)
file_ext = input_path.suffix if converter == 'auto':
if not file_ext in CONVERTERS: converter = input_path.suffix[1:]
prints("Can't find converter for %s" % input_path.parts[-1], if not converter in CONVERTERS:
prints("Can't find converter for %s" % converter,
title="Unknown format", exits=1) title="Unknown format", exits=1)
CONVERTERS[file_ext](input_path, output_path, func = CONVERTERS[converter]
func(input_path, output_path,
n_sents=n_sents, use_morphology=morphology) n_sents=n_sents, use_morphology=morphology)

1
spacy/cli/converters/__init__.py
View File

@ -1,2 +1,3 @@
from .conllu2json import conllu2json from .conllu2json import conllu2json
from .iob2json import iob2json from .iob2json import iob2json
from .conll_ner2json import conll_ner2json