From 2d929ffc5db25e89f3dc5c0113dc93386d3c20ae Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Sun, 1 Apr 2018 17:42:33 +0200 Subject: [PATCH] Handle list-valued GoldParse values --- spacy/pipeline.pyx | 2 ++ spacy/syntax/arc_eager.pyx | 4 ++++ spacy/syntax/ner.pyx | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/spacy/pipeline.pyx b/spacy/pipeline.pyx index 01a2b16e4..b1e4313a8 100644 --- a/spacy/pipeline.pyx +++ b/spacy/pipeline.pyx @@ -500,6 +500,8 @@ class Tagger(Pipe): for tag in gold.tags: if tag is None: correct[idx] = guesses[idx] + elif isinstance(tag, list): # Leave these latent for now. + correct[idx] = guesses[idx] else: correct[idx] = tag_index[tag] idx += 1 diff --git a/spacy/syntax/arc_eager.pyx b/spacy/syntax/arc_eager.pyx index e51b61bdb..d04328d44 100644 --- a/spacy/syntax/arc_eager.pyx +++ b/spacy/syntax/arc_eager.pyx @@ -479,6 +479,10 @@ cdef class ArcEager(TransitionSystem): if head is None or dep is None: gold.c.heads[i] = i gold.c.has_dep[i] = False + elif isinstance(head, list): + # TODO: This is where the fused token stuff will happen + gold.c.heads[i] = i + gold.c.has_dep[i] = False else: if head > i: action = LEFT diff --git a/spacy/syntax/ner.pyx b/spacy/syntax/ner.pyx index d56008ca0..224fe33e6 100644 --- a/spacy/syntax/ner.pyx +++ b/spacy/syntax/ner.pyx @@ -111,7 +111,10 @@ cdef class BiluoPushDown(TransitionSystem): if not self.has_gold(gold): return None for i in range(gold.length): - gold.c.ner[i] = self.lookup_transition(gold.ner[i]) + if isinstance(gold.ner[i], list): + gold.c.ner[i] = self.lookup_transition(None) + else: + gold.c.ner[i] = self.lookup_transition(gold.ner[i]) return gold def get_beam_annot(self, Beam beam):