explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-04 13:40:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Fix efficiency of JSON reading, by using ujson instead of stream

Browse Source
This commit is contained in:
Matthew Honnibal 2015-05-30 17:54:52 +02:00
parent 6bbdcc5db5
commit 9e39a206da
1 changed files with 25 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spacy/gold.pyx
View File

@ -2,6 +2,7 @@ import numpy
import codecs
import json
import ijson
import ujson
import random
import re
import os
@ -96,12 +97,14 @@ def _min_edit_path(cand_words, gold_words):
def read_json_file(loc):
print loc
if path.isdir(loc):
for filename in os.listdir(loc):
yield from read_json_file(path.join(loc, filename))
else:
with open(loc) as file_:
for doc in ijson.items(file_, 'item'):
docs = ujson.load(file_)
for doc in docs:
paragraphs = []
for paragraph in doc['paragraphs']:
sents = []
@ -122,6 +125,7 @@ def read_json_file(loc):
sents.append((
(ids, words, tags, heads, labels, ner),
sent.get('brackets', [])))
if sents:
yield (paragraph.get('raw', None), sents)