2019-08-29 13:04:01 +03:00
|
|
|
from wasabi import Printer
|
2018-12-18 15:58:30 +03:00
|
|
|
|
2020-06-20 04:19:40 +03:00
|
|
|
from ...gold import iob_to_biluo, tags_to_entities
|
2019-05-11 22:12:09 +03:00
|
|
|
from ...util import minibatch
|
2020-06-20 04:19:40 +03:00
|
|
|
from .util import merge_sentences
|
2020-06-20 16:59:39 +03:00
|
|
|
from .conll_ner2docs import n_sents_info
|
2017-05-19 21:27:51 +03:00
|
|
|
|
|
|
|
|
2020-06-20 04:19:40 +03:00
|
|
|
def iob2docs(input_data, n_sents=10, no_print=False, *args, **kwargs):
|
2017-05-19 21:27:51 +03:00
|
|
|
"""
|
2019-08-29 13:04:01 +03:00
|
|
|
Convert IOB files with one sentence per line and tags separated with '|'
|
2020-06-20 04:19:40 +03:00
|
|
|
into Doc objects so they can be saved. IOB and IOB2 are accepted.
|
2019-08-29 13:04:01 +03:00
|
|
|
|
|
|
|
Sample formats:
|
|
|
|
|
|
|
|
I|O like|O London|I-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|O
|
|
|
|
I|O like|O London|B-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|O
|
|
|
|
I|PRP|O like|VBP|O London|NNP|I-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|O
|
|
|
|
I|PRP|O like|VBP|O London|NNP|B-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|O
|
2017-05-19 21:27:51 +03:00
|
|
|
"""
|
2019-10-18 19:12:59 +03:00
|
|
|
msg = Printer(no_print=no_print)
|
2019-08-29 13:04:01 +03:00
|
|
|
docs = read_iob(input_data.split("\n"))
|
|
|
|
if n_sents > 0:
|
|
|
|
n_sents_info(msg, n_sents)
|
|
|
|
docs = merge_sentences(docs, n_sents)
|
2018-11-30 22:16:14 +03:00
|
|
|
return docs
|
2017-05-19 21:27:51 +03:00
|
|
|
|
|
|
|
|
2017-10-02 17:50:26 +03:00
|
|
|
def read_iob(raw_sents):
|
2020-06-20 04:19:40 +03:00
|
|
|
docs = []
|
2017-10-02 17:50:26 +03:00
|
|
|
for line in raw_sents:
|
2017-05-19 21:27:51 +03:00
|
|
|
if not line.strip():
|
|
|
|
continue
|
2019-08-31 14:39:06 +03:00
|
|
|
tokens = [t.split("|") for t in line.split()]
|
2017-05-28 16:11:39 +03:00
|
|
|
if len(tokens[0]) == 3:
|
2020-06-20 04:19:40 +03:00
|
|
|
words, tags, iob = zip(*tokens)
|
2019-01-16 15:44:16 +03:00
|
|
|
elif len(tokens[0]) == 2:
|
2017-05-28 16:11:39 +03:00
|
|
|
words, iob = zip(*tokens)
|
2020-06-20 04:19:40 +03:00
|
|
|
tags = ["-"] * len(words)
|
2019-01-16 15:44:16 +03:00
|
|
|
else:
|
2019-02-07 22:54:07 +03:00
|
|
|
raise ValueError(
|
2019-08-29 13:04:01 +03:00
|
|
|
"The sentence-per-line IOB/IOB2 file is not formatted correctly. Try checking whitespace and delimiters. See https://spacy.io/api/cli#convert"
|
2019-02-07 22:54:07 +03:00
|
|
|
)
|
2020-06-20 04:19:40 +03:00
|
|
|
doc = Doc(vocab, words=words)
|
|
|
|
for i, tag in enumerate(pos):
|
|
|
|
doc[i].tag_ = tag
|
2017-05-26 19:32:34 +03:00
|
|
|
biluo = iob_to_biluo(iob)
|
2020-06-20 04:19:40 +03:00
|
|
|
entities = biluo_tags_to_entities(biluo)
|
|
|
|
doc.ents = [Span(doc, start=s, end=e, label=L) for (L, s, e) in entities]
|
|
|
|
docs.append(doc)
|
2017-05-19 21:27:51 +03:00
|
|
|
return docs
|