explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-06-30 18:03:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Read json files recursively from a directory, instead of requiring a single .json file

Browse Source
This commit is contained in:
Matthew Honnibal 2015-05-29 03:52:55 +02:00
parent 8f31d3b864
commit b76bbbd12c
2 changed files with 29 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/parser/train.py
View File

@ -138,8 +138,8 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
@plac.annotations( @plac.annotations(
train_loc=("Location of training json file"), train_loc=("Location of training file or directory"),
dev_loc=("Location of development json file"), dev_loc=("Location of development file or directory"),
corruption_level=("Amount of noise to add to training data", "option", "c", float), corruption_level=("Amount of noise to add to training data", "option", "c", float),
model_dir=("Location of output model directory",), model_dir=("Location of output model directory",),
out_loc=("Out location", "option", "o", str), out_loc=("Out location", "option", "o", str),

6
spacy/gold.pyx
View File

@ -4,6 +4,8 @@ import json
import ijson import ijson
import random import random
import re import re
import os
from os import path
from spacy.munge.read_ner import tags_to_entities from spacy.munge.read_ner import tags_to_entities
from libc.string cimport memset from libc.string cimport memset
@ -94,6 +96,10 @@ def _min_edit_path(cand_words, gold_words):
def read_json_file(loc): def read_json_file(loc):
if path.isdir(loc):
for filename in os.listdir(loc):
yield from read_json_file(path.join(loc, filename))
else:
with open(loc) as file_: with open(loc) as file_:
for doc in ijson.items(file_, 'item'): for doc in ijson.items(file_, 'item'):
paragraphs = [] paragraphs = []