explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 01:16:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Improve sentence merging in iob2json

Browse Source
This commit is contained in:
Matthew Honnibal 2017-10-02 17:02:10 +02:00
parent 31681d20e0
commit f942903429
1 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
spacy/cli/converters/iob2json.py
View File

@ -12,17 +12,13 @@ def iob2json(input_path, output_path, n_sents=10, *a, **k):
Convert IOB files into JSON format for use with train cli. Convert IOB files into JSON format for use with train cli.
""" """
with input_path.open('r', encoding='utf8') as file_: with input_path.open('r', encoding='utf8') as file_:
if n_sents: sentences = read_iob(file_)
lines = [' '.join(para) for para in partition_all(n_sents, file_)] docs = merge_sentences(sentences, n_sents)
else:
lines = file_
sentences = read_iob(lines)
output_filename = input_path.parts[-1].replace(".iob", ".json") output_filename = input_path.parts[-1].replace(".iob", ".json")
output_file = output_path / output_filename output_file = output_path / output_filename
with output_file.open('w', encoding='utf-8') as f: with output_file.open('w', encoding='utf-8') as f:
f.write(json_dumps(sentences)) f.write(json_dumps(docs))
prints("Created %d documents" % len(sentences), prints("Created %d documents" % len(docs),
title="Generated output file %s" % path2str(output_file)) title="Generated output file %s" % path2str(output_file))
@ -46,3 +42,15 @@ def read_iob(raw_sents):
paragraphs = [{'sentences': [sent]} for sent in sentences] paragraphs = [{'sentences': [sent]} for sent in sentences]
docs = [{'id': 0, 'paragraphs': [para]} for para in paragraphs] docs = [{'id': 0, 'paragraphs': [para]} for para in paragraphs]
return docs return docs
def merge_sentences(docs, n_sents):
counter = 0
merged = []
for group in partition_all(n_sents, docs):
group = list(group)
first = group.pop(0)
to_extend = first['paragraphs'][0]['sentences']
for sent in group[1:]:
to_extend.extend(sent['paragraphs'][0]['sentences'])
merged.append(first)
return merged