From 1fb8fc6ea9e9f290af12c1ea1b9755757e64c610 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sat, 6 Jun 2020 14:24:35 +0200 Subject: [PATCH] Add Example class --- spacy/_gold/example.py | 199 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 spacy/_gold/example.py diff --git a/spacy/_gold/example.py b/spacy/_gold/example.py new file mode 100644 index 000000000..db9e10093 --- /dev/null +++ b/spacy/_gold/example.py @@ -0,0 +1,199 @@ +from .annotation import TokenAnnotation, DocAnnotation +from .gold_parse import GoldParse + + +class Example: + def __init__(self, doc_annotation=None, token_annotation=None, doc=None, + goldparse=None): + """ Doc can either be text, or an actual Doc """ + self.doc = doc + self.doc_annotation = doc_annotation if doc_annotation else DocAnnotation() + self.token_annotation = token_annotation if token_annotation else TokenAnnotation() + self.goldparse = goldparse + + @classmethod + def from_gold(cls, goldparse, doc=None): + doc_annotation = DocAnnotation(cats=goldparse.cats, links=goldparse.links) + token_annotation = goldparse.get_token_annotation() + return cls(doc_annotation, token_annotation, doc) + + @classmethod + def from_dict(cls, example_dict, doc=None): + token_dict = example_dict.get("token_annotation", {}) + token_annotation = TokenAnnotation.from_dict(token_dict) + doc_dict = example_dict.get("doc_annotation", {}) + doc_annotation = DocAnnotation.from_dict(doc_dict) + return cls(doc_annotation, token_annotation, doc) + + def to_dict(self): + """ Note that this method does NOT export the doc, only the annotations ! """ + token_dict = self.token_annotation.to_dict() + doc_dict = self.doc_annotation.to_dict() + return {"token_annotation": token_dict, "doc_annotation": doc_dict} + + @property + def text(self): + if self.doc is None: + return None + if isinstance(self.doc, Doc): + return self.doc.text + return self.doc + + @property + def gold(self): + if self.goldparse is None: + doc, gold = self.get_gold_parses()[0] + self.goldparse = gold + return self.goldparse + + def set_token_annotation(self, ids=None, words=None, tags=None, pos=None, + morphs=None, lemmas=None, heads=None, deps=None, + entities=None, sent_starts=None, brackets=None): + self.token_annotation = TokenAnnotation(ids=ids, words=words, tags=tags, + pos=pos, morphs=morphs, lemmas=lemmas, heads=heads, + deps=deps, entities=entities, + sent_starts=sent_starts, brackets=brackets) + + def set_doc_annotation(self, cats=None, links=None): + if cats: + self.doc_annotation.cats = cats + if links: + self.doc_annotation.links = links + + def split_sents(self): + """ Split the token annotations into multiple Examples based on + sent_starts and return a list of the new Examples""" + if not self.token_annotation.words: + return [self] + s_example = Example(doc=None, doc_annotation=self.doc_annotation) + s_ids, s_words, s_tags, s_pos, s_morphs = [], [], [], [], [] + s_lemmas, s_heads, s_deps, s_ents, s_sent_starts = [], [], [], [], [] + s_brackets = [] + sent_start_i = 0 + t = self.token_annotation + split_examples = [] + for i in range(len(t.words)): + if i > 0 and t.sent_starts[i] == 1: + s_example.set_token_annotation(ids=s_ids, + words=s_words, tags=s_tags, pos=s_pos, morphs=s_morphs, + lemmas=s_lemmas, heads=s_heads, deps=s_deps, + entities=s_ents, sent_starts=s_sent_starts, + brackets=s_brackets) + split_examples.append(s_example) + s_example = Example(doc=None, doc_annotation=self.doc_annotation) + s_ids, s_words, s_tags, s_pos, s_heads = [], [], [], [], [] + s_deps, s_ents, s_morphs, s_lemmas = [], [], [], [] + s_sent_starts, s_brackets = [], [] + sent_start_i = i + s_ids.append(t.get_id(i)) + s_words.append(t.get_word(i)) + s_tags.append(t.get_tag(i)) + s_pos.append(t.get_pos(i)) + s_morphs.append(t.get_morph(i)) + s_lemmas.append(t.get_lemma(i)) + s_heads.append(t.get_head(i) - sent_start_i) + s_deps.append(t.get_dep(i)) + s_ents.append(t.get_entity(i)) + s_sent_starts.append(t.get_sent_start(i)) + for b_end, b_label in t.brackets_by_start.get(i, []): + s_brackets.append( + (i - sent_start_i, b_end - sent_start_i, b_label) + ) + i += 1 + s_example.set_token_annotation(ids=s_ids, words=s_words, tags=s_tags, + pos=s_pos, morphs=s_morphs, lemmas=s_lemmas, heads=s_heads, + deps=s_deps, entities=s_ents, sent_starts=s_sent_starts, + brackets=s_brackets) + split_examples.append(s_example) + return split_examples + + + def get_gold_parses(self, merge=True, vocab=None, make_projective=False, + ignore_misaligned=False): + """Return a list of (doc, GoldParse) objects. + If merge is set to True, keep all Token annotations as one big list.""" + d = self.doc_annotation + # merge == do not modify Example + if merge: + t = self.token_annotation + doc = self.doc + if doc is None or not isinstance(doc, Doc): + if not vocab: + raise ValueError(Errors.E998) + doc = Doc(vocab, words=t.words) + try: + gp = GoldParse.from_annotation(doc, d, t, + make_projective=make_projective) + except AlignmentError: + if ignore_misaligned: + gp = None + else: + raise + return [(doc, gp)] + # not merging: one GoldParse per sentence, defining docs with the words + # from each sentence + else: + parses = [] + split_examples = self.split_sents() + for split_example in split_examples: + if not vocab: + raise ValueError(Errors.E998) + split_doc = Doc(vocab, words=split_example.token_annotation.words) + try: + gp = GoldParse.from_annotation(split_doc, d, + split_example.token_annotation, + make_projective=make_projective) + except AlignmentError: + if ignore_misaligned: + gp = None + else: + raise + if gp is not None: + parses.append((split_doc, gp)) + return parses + + @classmethod + def to_example_objects(cls, examples, make_doc=None, keep_raw_text=False): + """ + Return a list of Example objects, from a variety of input formats. + make_doc needs to be provided when the examples contain text strings and keep_raw_text=False + """ + if isinstance(examples, Example): + return [examples] + if isinstance(examples, tuple): + examples = [examples] + converted_examples = [] + for ex in examples: + if isinstance(ex, Example): + converted_examples.append(ex) + # convert string to Doc to Example + elif isinstance(ex, str): + if keep_raw_text: + converted_examples.append(Example(doc=ex)) + else: + doc = make_doc(ex) + converted_examples.append(Example(doc=doc)) + # convert Doc to Example + elif isinstance(ex, Doc): + converted_examples.append(Example(doc=ex)) + # convert tuples to Example + elif isinstance(ex, tuple) and len(ex) == 2: + doc, gold = ex + gold_dict = {} + # convert string to Doc + if isinstance(doc, str) and not keep_raw_text: + doc = make_doc(doc) + # convert dict to GoldParse + if isinstance(gold, dict): + gold_dict = gold + if doc is not None or gold.get("words", None) is not None: + gold = GoldParse(doc, **gold) + else: + gold = None + if gold is not None: + converted_examples.append(Example.from_gold(goldparse=gold, doc=doc)) + else: + raise ValueError(Errors.E999.format(gold_dict=gold_dict)) + else: + converted_examples.append(ex) + return converted_examples