explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-07-17 11:42:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

partial annotation support

Browse Source
This commit is contained in:
Peter Baumgartner 2023-01-12 09:10:39 -05:00
parent 28c31048c9
commit a944e55291
1 changed files with 29 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
spacy/cli/debug_data.py
View File

@ -708,11 +708,23 @@ def debug_data(
n = gold_train_data["no_lemma_annotations"] n = gold_train_data["no_lemma_annotations"]
msg.warn(f"{n} docs with no lemma annotations.") msg.warn(f"{n} docs with no lemma annotations.")
else: else:
msg.good("All training docs have complete lemma annotations.") msg.good("All training docs have lemma annotations.")
if gold_dev_data["no_lemma_annotations"] > 0: if gold_dev_data["no_lemma_annotations"] > 0:
n = gold_dev_data["no_lemma_annotations"] n = gold_dev_data["no_lemma_annotations"]
msg.warn(f"{n} docs with no lemma annotations.") msg.warn(f"{n} docs with no lemma annotations.")
else:
msg.good("All dev docs have lemma annotations.")
if gold_train_data["partial_lemma_annotations"] > 0:
n = gold_train_data["partial_lemma_annotations"]
msg.info(f"{n} docs with partial lemma annotations.")
else:
msg.good("All training docs have complete lemma annotations.")
if gold_dev_data["partial_lemma_annotations"] > 0:
n = gold_dev_data["partial_lemma_annotations"]
msg.info(f"{n} docs with partial lemma annotations.")
else: else:
msg.good("All dev docs have complete lemma annotations.") msg.good("All dev docs have complete lemma annotations.")
@ -779,6 +791,7 @@ def _compile_gold(
"texts": set(), "texts": set(),
"lemmatizer_trees": set(), "lemmatizer_trees": set(),
"no_lemma_annotations": 0, "no_lemma_annotations": 0,
"partial_lemma_annotations": 0,
"n_low_cardinality_lemmas": 0, "n_low_cardinality_lemmas": 0,
} }
if "trainable_lemmatizer" in factory_names: if "trainable_lemmatizer" in factory_names:
@ -916,7 +929,9 @@ def _compile_gold(
# from EditTreeLemmatizer._labels_from_data # from EditTreeLemmatizer._labels_from_data
if all(token.lemma == 0 for token in gold): if all(token.lemma == 0 for token in gold):
data["no_lemma_annotations"] += 1 data["no_lemma_annotations"] += 1
else: continue
if any(token.lemma == 0 for token in gold):
data["partial_lemma_annotations"] += 1
lemma_set = set() lemma_set = set()
for token in gold: for token in gold:
if token.lemma != 0: if token.lemma != 0: