mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-11 04:08:09 +03:00
322 lines
11 KiB
Cython
322 lines
11 KiB
Cython
from __future__ import unicode_literals
|
|
|
|
from ._state cimport State
|
|
from ._state cimport has_head, get_idx, get_s0, get_n0
|
|
from ._state cimport is_final, at_eol, pop_stack, push_stack, add_dep
|
|
from ._state cimport head_in_buffer, children_in_buffer
|
|
from ._state cimport head_in_stack, children_in_stack
|
|
|
|
from ..structs cimport TokenC
|
|
|
|
|
|
DEF NON_MONOTONIC = True
|
|
DEF USE_BREAK = True
|
|
|
|
|
|
cdef enum:
|
|
SHIFT
|
|
REDUCE
|
|
LEFT
|
|
RIGHT
|
|
BREAK_SHIFT
|
|
BREAK_RIGHT
|
|
N_MOVES
|
|
|
|
|
|
cdef inline bint _can_shift(const State* s) nogil:
|
|
return not at_eol(s)
|
|
|
|
|
|
cdef inline bint _can_right(const State* s) nogil:
|
|
return s.stack_len >= 1 and not at_eol(s)
|
|
|
|
|
|
cdef inline bint _can_left(const State* s) nogil:
|
|
if NON_MONOTONIC:
|
|
return s.stack_len >= 1
|
|
else:
|
|
return s.stack_len >= 1 and not has_head(get_s0(s))
|
|
|
|
|
|
cdef inline bint _can_reduce(const State* s) nogil:
|
|
if NON_MONOTONIC:
|
|
return s.stack_len >= 2
|
|
else:
|
|
return s.stack_len >= 2 and has_head(get_s0(s))
|
|
|
|
|
|
cdef inline bint _can_break_shift(const State* s) nogil:
|
|
cdef int i
|
|
if not USE_BREAK:
|
|
return False
|
|
elif not _can_shift(s):
|
|
return False
|
|
else:
|
|
# P. 757
|
|
# In UPP, if Shift(F) or RightArc(F) fail to result in a single parsing
|
|
# tree, they cannot be performed as well.
|
|
seen_headless = False
|
|
for i in range(s.stack_len):
|
|
if s.sent[s.stack[i]].head == 0:
|
|
return False
|
|
return True
|
|
|
|
|
|
cdef inline bint _can_break_right(const State* s) nogil:
|
|
cdef int i
|
|
if not USE_BREAK:
|
|
return False
|
|
elif not _can_right(s):
|
|
return False
|
|
else:
|
|
# P. 757
|
|
# In UPP, if Shift(F) or RightArc(F) fail to result in a single parsing
|
|
# tree, they cannot be performed as well.
|
|
seen_headless = False
|
|
for i in range(s.stack_len):
|
|
if s.sent[s.stack[i]].head == 0:
|
|
if seen_headless:
|
|
return False
|
|
else:
|
|
seen_headless = True
|
|
return True
|
|
|
|
|
|
cdef int _shift_cost(const State* s, const int* gold) except -1:
|
|
assert not at_eol(s)
|
|
cost = 0
|
|
cost += head_in_stack(s, s.i, gold)
|
|
cost += children_in_stack(s, s.i, gold)
|
|
if NON_MONOTONIC:
|
|
cost += gold[s.stack[0]] == s.i
|
|
return cost
|
|
|
|
|
|
cdef int _right_cost(const State* s, const int* gold) except -1:
|
|
assert s.stack_len >= 1
|
|
cost = 0
|
|
if gold[s.i] == s.stack[0]:
|
|
return cost
|
|
cost += head_in_buffer(s, s.i, gold)
|
|
cost += children_in_stack(s, s.i, gold)
|
|
cost += head_in_stack(s, s.i, gold)
|
|
if NON_MONOTONIC:
|
|
cost += gold[s.stack[0]] == s.i
|
|
return cost
|
|
|
|
|
|
cdef int _left_cost(const State* s, const int* gold) except -1:
|
|
assert s.stack_len >= 1
|
|
cost = 0
|
|
if gold[s.stack[0]] == s.i:
|
|
return cost
|
|
|
|
cost += head_in_buffer(s, s.stack[0], gold)
|
|
cost += children_in_buffer(s, s.stack[0], gold)
|
|
if NON_MONOTONIC and s.stack_len >= 2:
|
|
cost += gold[s.stack[0]] == s.stack[-1]
|
|
return cost
|
|
|
|
|
|
cdef int _reduce_cost(const State* s, const int* gold) except -1:
|
|
cdef int cost = 0
|
|
cost += children_in_buffer(s, s.stack[0], gold)
|
|
if NON_MONOTONIC:
|
|
cost += head_in_buffer(s, s.stack[0], gold)
|
|
return cost
|
|
|
|
|
|
cdef int _break_shift_cost(const State* s, const int* gold) except -1:
|
|
cdef int cost = _shift_cost(s, gold)
|
|
# When we break, we Reduce all of the words on the stack. So, the Break
|
|
# cost is the sum of the Reduce costs
|
|
for i in range(s.stack_len):
|
|
cost += children_in_buffer(s, s.stack[i], gold)
|
|
if NON_MONOTONIC:
|
|
cost += head_in_buffer(s, s.stack[i], gold)
|
|
return cost
|
|
|
|
|
|
cdef int _break_right_cost(const State* s, const int* gold) except -1:
|
|
cdef int cost = _right_cost(s, gold)
|
|
# When we break, we Reduce all of the words on the stack. So, the Break
|
|
# cost is the sum of the Reduce costs
|
|
for i in range(s.stack_len):
|
|
cost += children_in_buffer(s, s.stack[i], gold)
|
|
if NON_MONOTONIC:
|
|
cost += head_in_buffer(s, s.stack[i], gold)
|
|
return cost
|
|
|
|
|
|
cdef class TransitionSystem:
|
|
def __init__(self, list left_labels, list right_labels):
|
|
self.mem = Pool()
|
|
left_labels.sort()
|
|
right_labels.sort()
|
|
if 'ROOT' in right_labels:
|
|
right_labels.pop(right_labels.index('ROOT'))
|
|
if 'ROOT' in left_labels:
|
|
left_labels.pop(left_labels.index('ROOT'))
|
|
self.n_moves = 3 + len(left_labels) + len(right_labels) + len(right_labels)
|
|
moves = <Transition*>self.mem.alloc(self.n_moves, sizeof(Transition))
|
|
cdef int i = 0
|
|
moves[i].move = SHIFT
|
|
moves[i].label = 0
|
|
moves[i].clas = i
|
|
i += 1
|
|
moves[i].move = REDUCE
|
|
moves[i].label = 0
|
|
moves[i].clas = i
|
|
i += 1
|
|
self.label_ids = {'ROOT': 0}
|
|
cdef int label_id
|
|
for label_str in left_labels:
|
|
label_str = unicode(label_str)
|
|
label_id = self.label_ids.setdefault(label_str, len(self.label_ids))
|
|
moves[i].move = LEFT
|
|
moves[i].label = label_id
|
|
moves[i].clas = i
|
|
i += 1
|
|
for label_str in right_labels:
|
|
label_str = unicode(label_str)
|
|
label_id = self.label_ids.setdefault(label_str, len(self.label_ids))
|
|
moves[i].move = RIGHT
|
|
moves[i].label = label_id
|
|
moves[i].clas = i
|
|
i += 1
|
|
moves[i].move = BREAK_SHIFT
|
|
moves[i].label = 0
|
|
moves[i].clas = i
|
|
i += 1
|
|
for label_str in right_labels:
|
|
label_str = unicode(label_str)
|
|
label_id = self.label_ids.setdefault(label_str, len(self.label_ids))
|
|
moves[i].move = BREAK_RIGHT
|
|
moves[i].label = label_id
|
|
moves[i].clas = i
|
|
i += 1
|
|
self._moves = moves
|
|
|
|
cdef int transition(self, State *s, const Transition* t) except -1:
|
|
if t.move == SHIFT:
|
|
# Set the dep label, in case we need it after we reduce
|
|
if NON_MONOTONIC:
|
|
get_s0(s).dep = t.label
|
|
push_stack(s)
|
|
elif t.move == LEFT:
|
|
add_dep(s, s.i, s.stack[0], t.label)
|
|
pop_stack(s)
|
|
elif t.move == RIGHT:
|
|
add_dep(s, s.stack[0], s.i, t.label)
|
|
push_stack(s)
|
|
elif t.move == REDUCE:
|
|
add_dep(s, s.stack[-1], s.stack[0], get_s0(s).dep)
|
|
pop_stack(s)
|
|
elif t.move == BREAK_RIGHT:
|
|
add_dep(s, s.stack[0], s.i, t.label)
|
|
push_stack(s)
|
|
while s.stack_len != 0:
|
|
if not has_head(get_s0(s)):
|
|
get_s0(s).dep = 0
|
|
s.stack -= 1
|
|
s.stack_len -= 1
|
|
if not at_eol(s):
|
|
push_stack(s)
|
|
elif t.move == BREAK_SHIFT:
|
|
push_stack(s)
|
|
get_s0(s).dep = 0
|
|
s.stack -= s.stack_len
|
|
s.stack_len = 0
|
|
if not at_eol(s):
|
|
push_stack(s)
|
|
else:
|
|
raise Exception(t.move)
|
|
|
|
cdef Transition best_valid(self, const weight_t* scores, const State* s) except *:
|
|
cdef bint[N_MOVES] valid
|
|
valid[SHIFT] = _can_shift(s)
|
|
valid[LEFT] = _can_left(s)
|
|
valid[RIGHT] = _can_right(s)
|
|
valid[REDUCE] = _can_reduce(s)
|
|
valid[BREAK_SHIFT] = _can_break_shift(s)
|
|
valid[BREAK_RIGHT] = _can_break_right(s)
|
|
|
|
cdef int best = -1
|
|
cdef weight_t score = 0
|
|
cdef weight_t best_r_score = -9000
|
|
cdef int best_r_label = -1
|
|
cdef int i
|
|
for i in range(self.n_moves):
|
|
if valid[self._moves[i].move] and (best == -1 or scores[i] > score):
|
|
best = i
|
|
score = scores[i]
|
|
if self._moves[i].move == RIGHT and scores[i] > best_r_score:
|
|
best_r_label = self._moves[i].label
|
|
assert best >= 0
|
|
cdef Transition t = self._moves[best]
|
|
t.score = score
|
|
if t.move == SHIFT:
|
|
t.label = best_r_label
|
|
return t
|
|
|
|
cdef Transition best_gold(self, Transition* guess, const weight_t* scores,
|
|
const State* s,
|
|
const int* gold_heads, const int* gold_labels) except *:
|
|
# If we can create a gold dependency, only one action can be correct
|
|
cdef int[N_MOVES] unl_costs
|
|
unl_costs[SHIFT] = _shift_cost(s, gold_heads) if _can_shift(s) else -1
|
|
unl_costs[LEFT] = _left_cost(s, gold_heads) if _can_left(s) else -1
|
|
unl_costs[RIGHT] = _right_cost(s, gold_heads) if _can_right(s) else -1
|
|
unl_costs[REDUCE] = _reduce_cost(s, gold_heads) if _can_reduce(s) else -1
|
|
unl_costs[BREAK_SHIFT] = _break_shift_cost(s, gold_heads) if _can_break_shift(s) else -1
|
|
unl_costs[BREAK_RIGHT] = _break_right_cost(s, gold_heads) if _can_break_right(s) else -1
|
|
|
|
guess.cost = unl_costs[guess.move]
|
|
cdef Transition t
|
|
cdef int target_label
|
|
cdef int i
|
|
if gold_heads[s.stack[0]] == s.i:
|
|
target_label = gold_labels[s.stack[0]]
|
|
if guess.move == LEFT:
|
|
guess.cost += guess.label != target_label
|
|
for i in range(self.n_moves):
|
|
t = self._moves[i]
|
|
if t.move == LEFT and t.label == target_label:
|
|
return t
|
|
elif gold_heads[s.i] == s.stack[0]:
|
|
target_label = gold_labels[s.i]
|
|
if guess.move == RIGHT or guess.move == BREAK_RIGHT:
|
|
guess.cost += guess.label != target_label
|
|
for i in range(self.n_moves):
|
|
t = self._moves[i]
|
|
if (t.move == RIGHT or t.move == BREAK_RIGHT) and t.label == target_label:
|
|
return t
|
|
|
|
cdef int best = -1
|
|
cdef weight_t score = -9000
|
|
for i in range(self.n_moves):
|
|
t = self._moves[i]
|
|
if unl_costs[t.move] == 0 and (best == -1 or scores[i] > score):
|
|
best = i
|
|
score = scores[i]
|
|
t = self._moves[best]
|
|
t.score = score
|
|
if best < 0:
|
|
msg = ("No gold move found for configuration.\n"
|
|
"Is the gold-standard parse a projective tree?\n"
|
|
"S unl cost: %d\n"
|
|
"D unl cost: %d\n"
|
|
"L unl cost: %d\n"
|
|
"R unl cost: %d\n"
|
|
"S0, S0 gold: %d, %d\n"
|
|
"N0, N0 gold: %d, %d\n"
|
|
|
|
)
|
|
fields = [unl_costs[SHIFT], unl_costs[REDUCE], unl_costs[LEFT],
|
|
unl_costs[RIGHT],
|
|
s.stack[0], gold_heads[s.stack[0]],
|
|
s.i, gold_heads[s.i]]
|
|
|
|
raise Exception(msg % tuple(fields))
|
|
return t
|