mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-22 11:59:14 +03:00
82159b5c19
* Updates/bugfixes for NER/IOB converters
* Converter formats `ner` and `iob` use autodetect to choose a converter if
possible
* `iob2json` is reverted to handle sentence-per-line data like
`word1|pos1|ent1 word2|pos2|ent2`
* Fix bug in `merge_sentences()` so the second sentence in each batch isn't
skipped
* `conll_ner2json` is made more general so it can handle more formats with
whitespace-separated columns
* Supports all formats where the first column is the token and the final
column is the IOB tag; if present, the second column is the POS tag
* As in CoNLL 2003 NER, blank lines separate sentences, `-DOCSTART- -X- O O`
separates documents
* Add option for segmenting sentences (new flag `-s`)
* Parser-based sentence segmentation with a provided model, otherwise with
sentencizer (new option `-b` to specify model)
* Can group sentences into documents with `n_sents` as long as sentence
segmentation is available
* Only applies automatic segmentation when there are no existing delimiters
in the data
* Provide info about settings applied during conversion with warnings and
suggestions if settings conflict or might not be not optimal.
* Add tests for common formats
* Add '(default)' back to docs for -c auto
* Add document count back to output
* Revert changes to converter output message
* Use explicit tabs in convert CLI test data
* Adjust/add messages for n_sents=1 default
* Add sample NER data to training examples
* Update README
* Add links in docs to example NER data
* Define msg within converters
70 lines
2.3 KiB
Python
70 lines
2.3 KiB
Python
# coding: utf8
|
|
from __future__ import unicode_literals
|
|
|
|
import re
|
|
from wasabi import Printer
|
|
|
|
from ...gold import iob_to_biluo
|
|
from ...util import minibatch
|
|
from .conll_ner2json import n_sents_info
|
|
|
|
|
|
def iob2json(input_data, n_sents=10, *args, **kwargs):
|
|
"""
|
|
Convert IOB files with one sentence per line and tags separated with '|'
|
|
into JSON format for use with train cli. IOB and IOB2 are accepted.
|
|
|
|
Sample formats:
|
|
|
|
I|O like|O London|I-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|O
|
|
I|O like|O London|B-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|O
|
|
I|PRP|O like|VBP|O London|NNP|I-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|O
|
|
I|PRP|O like|VBP|O London|NNP|B-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|O
|
|
"""
|
|
msg = Printer()
|
|
docs = read_iob(input_data.split("\n"))
|
|
if n_sents > 0:
|
|
n_sents_info(msg, n_sents)
|
|
docs = merge_sentences(docs, n_sents)
|
|
return docs
|
|
|
|
|
|
def read_iob(raw_sents):
|
|
sentences = []
|
|
for line in raw_sents:
|
|
if not line.strip():
|
|
continue
|
|
tokens = [t.split('|') for t in line.split()]
|
|
if len(tokens[0]) == 3:
|
|
words, pos, iob = zip(*tokens)
|
|
elif len(tokens[0]) == 2:
|
|
words, iob = zip(*tokens)
|
|
pos = ["-"] * len(words)
|
|
else:
|
|
raise ValueError(
|
|
"The sentence-per-line IOB/IOB2 file is not formatted correctly. Try checking whitespace and delimiters. See https://spacy.io/api/cli#convert"
|
|
)
|
|
biluo = iob_to_biluo(iob)
|
|
sentences.append(
|
|
[
|
|
{"orth": w, "tag": p, "ner": ent}
|
|
for (w, p, ent) in zip(words, pos, biluo)
|
|
]
|
|
)
|
|
sentences = [{"tokens": sent} for sent in sentences]
|
|
paragraphs = [{"sentences": [sent]} for sent in sentences]
|
|
docs = [{"id": i, "paragraphs": [para]} for i, para in enumerate(paragraphs)]
|
|
return docs
|
|
|
|
|
|
def merge_sentences(docs, n_sents):
|
|
merged = []
|
|
for group in minibatch(docs, size=n_sents):
|
|
group = list(group)
|
|
first = group.pop(0)
|
|
to_extend = first["paragraphs"][0]["sentences"]
|
|
for sent in group:
|
|
to_extend.extend(sent["paragraphs"][0]["sentences"])
|
|
merged.append(first)
|
|
return merged
|