explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-14 10:30:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Handle various data better in NewExample

Browse Source
This commit is contained in:
Matthew Honnibal 2020-06-13 15:30:12 +02:00
parent face0de74f
commit b078b05ecd
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
spacy/gold/new_example.pyx
View File

@ -148,6 +148,8 @@ def _fix_legacy_dict_data(predicted, example_dict):
for key, value in example_dict.items(): for key, value in example_dict.items():
if key in ("token_annotation", "doc_annotation"): if key in ("token_annotation", "doc_annotation"):
pass pass
elif key == "ids":
pass
elif key in ("cats", "links") and value: elif key in ("cats", "links") and value:
doc_dict[key] = value doc_dict[key] = value
elif key in ("ner", "entities") and value: elif key in ("ner", "entities") and value:
@ -168,10 +170,15 @@ def _fix_legacy_dict_data(predicted, example_dict):
old_token_dict = token_dict old_token_dict = token_dict
token_dict = {} token_dict = {}
for key, value in old_token_dict.items(): for key, value in old_token_dict.items():
if key in remapping: if key in ("text", "ids", "entities", "ner", "brackets"):
pass
elif key in remapping:
token_dict[remapping[key]] = value token_dict[remapping[key]] = value
else: else:
raise ValueError(f"Unknown attr: {key}") raise ValueError(f"Unknown attr: {key}")
if "HEAD" in token_dict and "SENT_START" in token_dict:
# If heads are set, we don't also redundantly specify SENT_START.
token_dict.pop("SENT_START")
return { return {
"token_annotation": token_dict, "token_annotation": token_dict,
"doc_annotation": doc_dict "doc_annotation": doc_dict