mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-25 09:26:27 +03:00
Improve sentence merging in iob2json
This commit is contained in:
parent
31681d20e0
commit
f942903429
|
@ -12,17 +12,13 @@ def iob2json(input_path, output_path, n_sents=10, *a, **k):
|
||||||
Convert IOB files into JSON format for use with train cli.
|
Convert IOB files into JSON format for use with train cli.
|
||||||
"""
|
"""
|
||||||
with input_path.open('r', encoding='utf8') as file_:
|
with input_path.open('r', encoding='utf8') as file_:
|
||||||
if n_sents:
|
sentences = read_iob(file_)
|
||||||
lines = [' '.join(para) for para in partition_all(n_sents, file_)]
|
docs = merge_sentences(sentences, n_sents)
|
||||||
else:
|
|
||||||
lines = file_
|
|
||||||
sentences = read_iob(lines)
|
|
||||||
|
|
||||||
output_filename = input_path.parts[-1].replace(".iob", ".json")
|
output_filename = input_path.parts[-1].replace(".iob", ".json")
|
||||||
output_file = output_path / output_filename
|
output_file = output_path / output_filename
|
||||||
with output_file.open('w', encoding='utf-8') as f:
|
with output_file.open('w', encoding='utf-8') as f:
|
||||||
f.write(json_dumps(sentences))
|
f.write(json_dumps(docs))
|
||||||
prints("Created %d documents" % len(sentences),
|
prints("Created %d documents" % len(docs),
|
||||||
title="Generated output file %s" % path2str(output_file))
|
title="Generated output file %s" % path2str(output_file))
|
||||||
|
|
||||||
|
|
||||||
|
@ -46,3 +42,15 @@ def read_iob(raw_sents):
|
||||||
paragraphs = [{'sentences': [sent]} for sent in sentences]
|
paragraphs = [{'sentences': [sent]} for sent in sentences]
|
||||||
docs = [{'id': 0, 'paragraphs': [para]} for para in paragraphs]
|
docs = [{'id': 0, 'paragraphs': [para]} for para in paragraphs]
|
||||||
return docs
|
return docs
|
||||||
|
|
||||||
|
def merge_sentences(docs, n_sents):
|
||||||
|
counter = 0
|
||||||
|
merged = []
|
||||||
|
for group in partition_all(n_sents, docs):
|
||||||
|
group = list(group)
|
||||||
|
first = group.pop(0)
|
||||||
|
to_extend = first['paragraphs'][0]['sentences']
|
||||||
|
for sent in group[1:]:
|
||||||
|
to_extend.extend(sent['paragraphs'][0]['sentences'])
|
||||||
|
merged.append(first)
|
||||||
|
return merged
|
||||||
|
|
Loading…
Reference in New Issue
Block a user