explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-04-27 20:33:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

Exclude states with no matching gold annotations from parsing

Browse Source
This commit is contained in:
Matthew Honnibal 2017-05-22 10:30:12 -05:00
parent 83ffd16474
commit e2136232f9
5 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
spacy/syntax/arc_eager.pyx
View File

@ -350,7 +350,9 @@ cdef class ArcEager(TransitionSystem):
def __get__(self): def __get__(self):
return (SHIFT, REDUCE, LEFT, RIGHT, BREAK) return (SHIFT, REDUCE, LEFT, RIGHT, BREAK)
cdef int preprocess_gold(self, GoldParse gold) except -1: def preprocess_gold(self, GoldParse gold):
if all([h is None for h in gold.heads]):
return None
for i in range(gold.length): for i in range(gold.length):
if gold.heads[i] is None: # Missing values if gold.heads[i] is None: # Missing values
gold.c.heads[i] = i gold.c.heads[i] = i
@ -361,6 +363,7 @@ cdef class ArcEager(TransitionSystem):
label = 'ROOT' label = 'ROOT'
gold.c.heads[i] = gold.heads[i] gold.c.heads[i] = gold.heads[i]
gold.c.labels[i] = self.strings[label] gold.c.labels[i] = self.strings[label]
return gold
cdef Transition lookup_transition(self, object name) except *: cdef Transition lookup_transition(self, object name) except *:
if '-' in name: if '-' in name:

5
spacy/syntax/ner.pyx
View File

@ -95,9 +95,12 @@ cdef class BiluoPushDown(TransitionSystem):
else: else:
return MOVE_NAMES[move] + '-' + self.strings[label] return MOVE_NAMES[move] + '-' + self.strings[label]
cdef int preprocess_gold(self, GoldParse gold) except -1: def preprocess_gold(self, GoldParse gold):
if all([tag == '-' for tag in gold.ner]):
return None
for i in range(gold.length): for i in range(gold.length):
gold.c.ner[i] = self.lookup_transition(gold.ner[i]) gold.c.ner[i] = self.lookup_transition(gold.ner[i])
return gold
cdef Transition lookup_transition(self, object name) except *: cdef Transition lookup_transition(self, object name) except *:
if name == '-' or name == None: if name == '-' or name == None:

5
spacy/syntax/nn_parser.pyx
View File

@ -318,15 +318,14 @@ cdef class Parser:
golds = [golds] golds = [golds]
cuda_stream = get_cuda_stream() cuda_stream = get_cuda_stream()
for gold in golds: golds = [self.moves.preprocess_gold(g) for g in golds]
self.moves.preprocess_gold(gold)
states = self.moves.init_batch(docs) states = self.moves.init_batch(docs)
state2vec, vec2scores = self.get_batch_model(len(states), tokvecs, cuda_stream, state2vec, vec2scores = self.get_batch_model(len(states), tokvecs, cuda_stream,
drop) drop)
todo = [(s, g) for (s, g) in zip(states, golds) todo = [(s, g) for (s, g) in zip(states, golds)
if not s.is_final()] if not s.is_final() and g is not None]
backprops = [] backprops = []
cdef float loss = 0. cdef float loss = 0.

2
spacy/syntax/transition_system.pxd
View File

@ -43,8 +43,6 @@ cdef class TransitionSystem:
cdef int initialize_state(self, StateC* state) nogil cdef int initialize_state(self, StateC* state) nogil
cdef int finalize_state(self, StateC* state) nogil cdef int finalize_state(self, StateC* state) nogil
cdef int preprocess_gold(self, GoldParse gold) except -1
cdef Transition lookup_transition(self, object name) except * cdef Transition lookup_transition(self, object name) except *
cdef Transition init_transition(self, int clas, int move, int label) except * cdef Transition init_transition(self, int clas, int move, int label) except *

2
spacy/syntax/transition_system.pyx
View File

@ -70,7 +70,7 @@ cdef class TransitionSystem:
def finalize_doc(self, doc): def finalize_doc(self, doc):
pass pass
cdef int preprocess_gold(self, GoldParse gold) except -1: def preprocess_gold(self, GoldParse gold):
raise NotImplementedError raise NotImplementedError
cdef Transition lookup_transition(self, object name) except *: cdef Transition lookup_transition(self, object name) except *: