explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-25 17:36:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix multiple entries per custom extension in doc json (#11551)

Browse Source 
* Fix multiple extensions and character offset

* Rename token_start/end to start/end

* Refactor Doc.from_json based on review

* Iterate over user_data items

* Only add non-empty underscore entries

Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
This commit is contained in:
Edward 2022-10-19 15:52:47 +02:00 committed by GitHub
parent a1eacaa8db
commit d66ccb8eb0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 42 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy/schemas.py
View File

@ -519,9 +519,9 @@ class DocJSONSchema(BaseModel):
title="Any custom data stored in the document's _ attribute", title="Any custom data stored in the document's _ attribute",
alias="_", alias="_",
) )
underscore_token: Optional[Dict[StrictStr, Dict[StrictStr, Any]]] = Field( underscore_token: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field(
None, title="Any custom data stored in the token's _ attribute" None, title="Any custom data stored in the token's _ attribute"
) )
underscore_span: Optional[Dict[StrictStr, Dict[StrictStr, Any]]] = Field( underscore_span: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field(
None, title="Any custom data stored in the span's _ attribute" None, title="Any custom data stored in the span's _ attribute"
) )

25
spacy/tests/doc/test_json_doc_conversion.py
View File

@ -128,7 +128,9 @@ def test_doc_to_json_with_token_span_attributes(doc):
doc._.json_test1 = "hello world" doc._.json_test1 = "hello world"
doc._.json_test2 = [1, 2, 3] doc._.json_test2 = [1, 2, 3]
doc[0:1]._.span_test = "span_attribute" doc[0:1]._.span_test = "span_attribute"
doc[0:2]._.span_test = "span_attribute_2"
doc[0]._.token_test = 117 doc[0]._.token_test = 117
doc[1]._.token_test = 118
doc.spans["span_group"] = [doc[0:1]] doc.spans["span_group"] = [doc[0:1]]
json_doc = doc.to_json( json_doc = doc.to_json(
underscore=["json_test1", "json_test2", "token_test", "span_test"] underscore=["json_test1", "json_test2", "token_test", "span_test"]
@ -139,8 +141,10 @@ def test_doc_to_json_with_token_span_attributes(doc):
assert json_doc["_"]["json_test2"] == [1, 2, 3] assert json_doc["_"]["json_test2"] == [1, 2, 3]
assert "underscore_token" in json_doc assert "underscore_token" in json_doc
assert "underscore_span" in json_doc assert "underscore_span" in json_doc
assert json_doc["underscore_token"]["token_test"]["value"] == 117 assert json_doc["underscore_token"]["token_test"][0]["value"] == 117
assert json_doc["underscore_span"]["span_test"]["value"] == "span_attribute" assert json_doc["underscore_token"]["token_test"][1]["value"] == 118
assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute"
assert json_doc["underscore_span"]["span_test"][1]["value"] == "span_attribute_2"
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
@ -161,8 +165,8 @@ def test_doc_to_json_with_custom_user_data(doc):
assert json_doc["_"]["json_test"] == "hello world" assert json_doc["_"]["json_test"] == "hello world"
assert "underscore_token" in json_doc assert "underscore_token" in json_doc
assert "underscore_span" in json_doc assert "underscore_span" in json_doc
assert json_doc["underscore_token"]["token_test"]["value"] == 117 assert json_doc["underscore_token"]["token_test"][0]["value"] == 117
assert json_doc["underscore_span"]["span_test"]["value"] == "span_attribute" assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute"
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
@ -181,8 +185,8 @@ def test_doc_to_json_with_token_span_same_identifier(doc):
assert json_doc["_"]["my_ext"] == "hello world" assert json_doc["_"]["my_ext"] == "hello world"
assert "underscore_token" in json_doc assert "underscore_token" in json_doc
assert "underscore_span" in json_doc assert "underscore_span" in json_doc
assert json_doc["underscore_token"]["my_ext"]["value"] == 117 assert json_doc["underscore_token"]["my_ext"][0]["value"] == 117
assert json_doc["underscore_span"]["my_ext"]["value"] == "span_attribute" assert json_doc["underscore_span"]["my_ext"][0]["value"] == "span_attribute"
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc
@ -195,10 +199,9 @@ def test_doc_to_json_with_token_attributes_missing(doc):
doc[0]._.token_test = 117 doc[0]._.token_test = 117
json_doc = doc.to_json(underscore=["span_test"]) json_doc = doc.to_json(underscore=["span_test"])
assert "underscore_token" in json_doc
assert "underscore_span" in json_doc assert "underscore_span" in json_doc
assert json_doc["underscore_span"]["span_test"]["value"] == "span_attribute" assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute"
assert "token_test" not in json_doc["underscore_token"] assert "underscore_token" not in json_doc
assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0
@ -283,7 +286,9 @@ def test_json_to_doc_with_token_span_attributes(doc):
doc._.json_test1 = "hello world" doc._.json_test1 = "hello world"
doc._.json_test2 = [1, 2, 3] doc._.json_test2 = [1, 2, 3]
doc[0:1]._.span_test = "span_attribute" doc[0:1]._.span_test = "span_attribute"
doc[0:2]._.span_test = "span_attribute_2"
doc[0]._.token_test = 117 doc[0]._.token_test = 117
doc[1]._.token_test = 118
json_doc = doc.to_json( json_doc = doc.to_json(
underscore=["json_test1", "json_test2", "token_test", "span_test"] underscore=["json_test1", "json_test2", "token_test", "span_test"]
@ -295,7 +300,9 @@ def test_json_to_doc_with_token_span_attributes(doc):
assert new_doc._.json_test1 == "hello world" assert new_doc._.json_test1 == "hello world"
assert new_doc._.json_test2 == [1, 2, 3] assert new_doc._.json_test2 == [1, 2, 3]
assert new_doc[0]._.token_test == 117 assert new_doc[0]._.token_test == 117
assert new_doc[1]._.token_test == 118
assert new_doc[0:1]._.span_test == "span_attribute" assert new_doc[0:1]._.span_test == "span_attribute"
assert new_doc[0:2]._.span_test == "span_attribute_2"
assert new_doc.user_data == doc.user_data assert new_doc.user_data == doc.user_data
assert new_doc.to_bytes(exclude=["user_data"]) == doc.to_bytes( assert new_doc.to_bytes(exclude=["user_data"]) == doc.to_bytes(
exclude=["user_data"] exclude=["user_data"]

48
spacy/tokens/doc.pyx
View File

@ -1608,24 +1608,20 @@ cdef class Doc:
Doc.set_extension(attr) Doc.set_extension(attr)
self._.set(attr, doc_json["_"][attr]) self._.set(attr, doc_json["_"][attr])
if doc_json.get("underscore_token", {}): for token_attr in doc_json.get("underscore_token", {}):
for token_attr in doc_json["underscore_token"]: if not Token.has_extension(token_attr):
token_start = doc_json["underscore_token"][token_attr]["token_start"] Token.set_extension(token_attr)
value = doc_json["underscore_token"][token_attr]["value"] for token_data in doc_json["underscore_token"][token_attr]:
start = token_by_char(self.c, self.length, token_data["start"])
if not Token.has_extension(token_attr): value = token_data["value"]
Token.set_extension(token_attr) self[start]._.set(token_attr, value)
self[token_start]._.set(token_attr, value)
if doc_json.get("underscore_span", {}): for span_attr in doc_json.get("underscore_span", {}):
for span_attr in doc_json["underscore_span"]: if not Span.has_extension(span_attr):
token_start = doc_json["underscore_span"][span_attr]["token_start"] Span.set_extension(span_attr)
token_end = doc_json["underscore_span"][span_attr]["token_end"] for span_data in doc_json["underscore_span"][span_attr]:
value = doc_json["underscore_span"][span_attr]["value"] value = span_data["value"]
self.char_span(span_data["start"], span_data["end"])._.set(span_attr, value)
if not Span.has_extension(span_attr):
Span.set_extension(span_attr)
self[token_start:token_end]._.set(span_attr, value)
return self return self
def to_json(self, underscore=None): def to_json(self, underscore=None):
@ -1673,30 +1669,34 @@ cdef class Doc:
if underscore: if underscore:
user_keys = set() user_keys = set()
if self.user_data: if self.user_data:
data["_"] = {} for data_key, value in self.user_data.copy().items():
data["underscore_token"] = {}
data["underscore_span"] = {}
for data_key in self.user_data:
if type(data_key) == tuple and len(data_key) >= 4 and data_key[0] == "._.": if type(data_key) == tuple and len(data_key) >= 4 and data_key[0] == "._.":
attr = data_key[1] attr = data_key[1]
start = data_key[2] start = data_key[2]
end = data_key[3] end = data_key[3]
if attr in underscore: if attr in underscore:
user_keys.add(attr) user_keys.add(attr)
value = self.user_data[data_key]
if not srsly.is_json_serializable(value): if not srsly.is_json_serializable(value):
raise ValueError(Errors.E107.format(attr=attr, value=repr(value))) raise ValueError(Errors.E107.format(attr=attr, value=repr(value)))
# Check if doc attribute # Check if doc attribute
if start is None: if start is None:
if "_" not in data:
data["_"] = {}
data["_"][attr] = value data["_"][attr] = value
# Check if token attribute # Check if token attribute
elif end is None: elif end is None:
if "underscore_token" not in data:
data["underscore_token"] = {}
if attr not in data["underscore_token"]: if attr not in data["underscore_token"]:
data["underscore_token"][attr] = {"token_start": start, "value": value} data["underscore_token"][attr] = []
data["underscore_token"][attr].append({"start": start, "value": value})
# Else span attribute # Else span attribute
else: else:
if "underscore_span" not in data:
data["underscore_span"] = {}
if attr not in data["underscore_span"]: if attr not in data["underscore_span"]:
data["underscore_span"][attr] = {"token_start": start, "token_end": end, "value": value} data["underscore_span"][attr] = []
data["underscore_span"][attr].append({"start": start, "end": end, "value": value})
for attr in underscore: for attr in underscore:
if attr not in user_keys: if attr not in user_keys: