mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-26 01:46:28 +03:00
* Work on beam parser
This commit is contained in:
Matthew Honnibal
parent
de7c6c48d8
commit
e2a9a68b66
338
spacy/syntax/beam_parser.pyx
Normal file
338
spacy/syntax/beam_parser.pyx
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
# cython: profile=True
|
||||
# cython: experimental_cpp_class_def=True
|
||||
"""
|
||||
MALT-style dependency parser
|
||||
"""
|
||||
from __future__ import unicode_literals
|
||||
cimport cython
|
||||
|
||||
from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF
|
||||
|
||||
from libc.stdint cimport uint32_t, uint64_t
|
||||
from libc.string cimport memset, memcpy
|
||||
from libc.stdlib cimport rand
|
||||
from libc.math cimport log, exp
|
||||
import random
|
||||
import os.path
|
||||
from os import path
|
||||
import shutil
|
||||
import json
|
||||
|
||||
from cymem.cymem cimport Pool, Address
|
||||
from murmurhash.mrmr cimport hash64
|
||||
from thinc.typedefs cimport weight_t, class_t, feat_t, atom_t, hash_t
|
||||
|
||||
|
||||
from util import Config
|
||||
|
||||
from thinc.linear.features cimport ConjunctionExtracter
|
||||
from thinc.structs cimport FeatureC
|
||||
|
||||
from thinc.extra.search cimport Beam
|
||||
from thinc.extra.search cimport MaxViolation
|
||||
|
||||
from ..structs cimport TokenC
|
||||
|
||||
from ..tokens.doc cimport Doc
|
||||
from ..strings cimport StringStore
|
||||
|
||||
from .transition_system cimport TransitionSystem, Transition
|
||||
|
||||
from ..gold cimport GoldParse
|
||||
|
||||
from . import _parse_features
|
||||
from ._parse_features cimport CONTEXT_SIZE
|
||||
from ._parse_features cimport fill_context
|
||||
from .stateclass cimport StateClass
|
||||
from .parser cimport Parser
|
||||
from .parser cimport ParserPerceptron
|
||||
|
||||
DEBUG = False
|
||||
def set_debug(val):
|
||||
global DEBUG
|
||||
DEBUG = val
|
||||
|
||||
|
||||
def get_templates(name):
|
||||
pf = _parse_features
|
||||
if name == 'ner':
|
||||
return pf.ner
|
||||
elif name == 'debug':
|
||||
return pf.unigrams
|
||||
else:
|
||||
return (pf.unigrams + pf.s0_n0 + pf.s1_n0 + pf.s1_s0 + pf.s0_n1 + pf.n0_n1 + \
|
||||
pf.tree_shape + pf.trigrams)
|
||||
|
||||
|
||||
cdef int BEAM_WIDTH = 8
|
||||
|
||||
|
||||
cdef class BeamParser(Parser):
|
||||
cdef public int beam_width
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.beam_width = kwargs.get('beam_width', BEAM_WIDTH)
|
||||
Parser.__init__(self, *args, **kwargs)
|
||||
|
||||
cdef int parseC(self, TokenC* tokens, int length, int nr_feat, int nr_class) with gil:
|
||||
self._parseC(tokens, length, nr_feat, nr_class)
|
||||
|
||||
cdef int _parseC(self, TokenC* tokens, int length, int nr_feat, int nr_class) except -1:
|
||||
|
||||
cdef Beam beam = Beam(self.moves.n_moves, self.beam_width)
|
||||
beam.initialize(_init_state, length, tokens)
|
||||
beam.check_done(_check_final_state, NULL)
|
||||
while not beam.is_done:
|
||||
self._advance_beam(beam, None, False)
|
||||
state = <StateClass>beam.at(0)
|
||||
self.moves.finalize_state(state.c)
|
||||
for i in range(length):
|
||||
tokens[i] = state.c._sent[i]
|
||||
_cleanup(beam)
|
||||
|
||||
def train(self, Doc tokens, GoldParse gold_parse):
|
||||
self.moves.preprocess_gold(gold_parse)
|
||||
cdef Beam pred = Beam(self.moves.n_moves, self.beam_width)
|
||||
pred.initialize(_init_state, tokens.length, tokens.c)
|
||||
pred.check_done(_check_final_state, NULL)
|
||||
|
||||
cdef Beam gold = Beam(self.moves.n_moves, self.beam_width)
|
||||
gold.initialize(_init_state, tokens.length, tokens.c)
|
||||
gold.check_done(_check_final_state, NULL)
|
||||
violn = MaxViolation()
|
||||
|
||||
while not pred.is_done and not gold.is_done:
|
||||
self._advance_beam(pred, gold_parse, False)
|
||||
self._advance_beam(gold, gold_parse, True)
|
||||
violn.check(pred, gold)
|
||||
self.model.time += 1
|
||||
if pred.is_done and pred.loss == 0:
|
||||
pass
|
||||
elif pred.is_done and pred.loss > 0:
|
||||
self._update(tokens, pred.histories[0], -1.0)
|
||||
self._update(tokens, gold.histories[0], 1.0)
|
||||
elif violn.cost > 0:
|
||||
self._update(tokens, violn.p_hist, -1.0)
|
||||
self._update(tokens, violn.g_hist, 1.0)
|
||||
_cleanup(pred)
|
||||
_cleanup(gold)
|
||||
return pred.loss
|
||||
|
||||
def _advance_beam(self, Beam beam, GoldParse gold, bint follow_gold):
|
||||
cdef atom_t[CONTEXT_SIZE] context
|
||||
cdef Pool mem = Pool()
|
||||
features = <FeatureC*>mem.alloc(self.model.nr_feat, sizeof(FeatureC))
|
||||
cdef ParserPerceptron model = self.model
|
||||
for i in range(beam.size):
|
||||
stcls = <StateClass>beam.at(i)
|
||||
if not stcls.c.is_final():
|
||||
fill_context(context, stcls.c)
|
||||
nr_feat = model.extracter.set_features(features, context)
|
||||
self.model.set_scoresC(beam.scores[i], features, nr_feat, 1)
|
||||
self.moves.set_valid(beam.is_valid[i], stcls.c)
|
||||
if gold is not None:
|
||||
for i in range(beam.size):
|
||||
stcls = <StateClass>beam.at(i)
|
||||
if not stcls.c.is_final():
|
||||
self.moves.set_costs(beam.is_valid[i], beam.costs[i], stcls, gold)
|
||||
if follow_gold:
|
||||
for j in range(self.moves.n_moves):
|
||||
beam.is_valid[i][j] *= beam.costs[i][j] == 0
|
||||
beam.advance(_transition_state, _hash_state, <void*>self.moves.c)
|
||||
beam.check_done(_check_final_state, NULL)
|
||||
|
||||
def _maxent_update_dense(self, doc, pred_scores, pred_hist, gold_scores,
|
||||
gold_hist, step_size=0.001):
|
||||
for i, history in enumerate(pred_hist):
|
||||
stcls = StateClass.init(doc.c, doc.length)
|
||||
self.moves.initialize_state(stcls.c)
|
||||
for j, clas in enumerate(history):
|
||||
fill_context(context, stcls.c)
|
||||
nr_feat = model.extracter.set_features(features, context)
|
||||
|
||||
self.moves.set_valid(is_valid, stcls)
|
||||
# Move weight away from this outcome
|
||||
for i in range(nr_class):
|
||||
costs[i] = 0.0
|
||||
costs[clas] = 1.0
|
||||
self.update(features, nr_feat, True, costs, is_valid, False)
|
||||
|
||||
self.moves.c[clas].do(stcls.c, self.moves.c[clas].label)
|
||||
for i, history in enumerate(gold_hist):
|
||||
stcls = StateClass.init(doc.c, doc.length)
|
||||
self.moves.initialize_state(stcls.c)
|
||||
for j, clas in enumerate(history):
|
||||
fill_context(context, stcls.c)
|
||||
nr_feat = model.extracter.set_features(features, context)
|
||||
|
||||
self.moves.set_valid(is_valid, stcls)
|
||||
# Move weight towards this outcome
|
||||
for i in range(nr_class):
|
||||
costs[i] = 1.0
|
||||
costs[clas] = 0.0
|
||||
self.update(features, nr_feat, True, costs, is_valid, False)
|
||||
|
||||
self.moves.c[clas].do(stcls.c, self.moves.c[clas].label)
|
||||
|
||||
def _maxent_update(self, doc, pred_scores, pred_hist, gold_scores, gold_hist,
|
||||
step_size=0.001):
|
||||
cdef weight_t Z, gZ, value
|
||||
cdef feat_t feat
|
||||
cdef class_t clas
|
||||
gZ, g_counts = self._maxent_counts(doc, gold_scores, gold_hist)
|
||||
Z, counts = self._maxent_counts(doc, pred_scores, pred_hist)
|
||||
update = {}
|
||||
if gZ > 0:
|
||||
for (clas, feat), value in g_counts.items():
|
||||
update[(clas, feat)] = value / gZ
|
||||
Z += gZ
|
||||
for (clas, feat), value in counts.items():
|
||||
update.setdefault((clas, feat), 0.0)
|
||||
update[(clas, feat)] -= value / Z
|
||||
for (clas, feat), value in update.items():
|
||||
if value < 1000:
|
||||
self.model.update_weight(feat, clas, step_size * value)
|
||||
|
||||
def _maxent_counts(self, Doc doc, scores, history):
|
||||
cdef Pool mem = Pool()
|
||||
cdef atom_t[CONTEXT_SIZE] context
|
||||
features = <FeatureC*>mem.alloc(self.model.nr_feat, sizeof(FeatureC))
|
||||
|
||||
cdef StateClass stcls
|
||||
|
||||
cdef class_t clas
|
||||
cdef ParserPerceptron model = self.model
|
||||
|
||||
cdef weight_t Z = 0.0
|
||||
cdef weight_t score
|
||||
counts = {}
|
||||
for i, (score, history) in enumerate(zip(scores, history)):
|
||||
prob = exp(score)
|
||||
if prob < 1e-6:
|
||||
continue
|
||||
stcls = StateClass.init(doc.c, doc.length)
|
||||
self.moves.initialize_state(stcls.c)
|
||||
for clas in history:
|
||||
fill_context(context, stcls.c)
|
||||
nr_feat = model.extracter.set_features(features, context)
|
||||
for feat in features[:nr_feat]:
|
||||
key = (clas, feat.key)
|
||||
counts[key] = counts.get(key, 0.0) + feat.value
|
||||
self.moves.c[clas].do(stcls.c, self.moves.c[clas].label)
|
||||
for key in counts:
|
||||
counts[key] *= prob
|
||||
Z += prob
|
||||
return Z, counts
|
||||
|
||||
def _update(self, Doc tokens, list hist, weight_t inc):
|
||||
cdef Pool mem = Pool()
|
||||
cdef atom_t[CONTEXT_SIZE] context
|
||||
features = <FeatureC*>mem.alloc(self.model.nr_feat, sizeof(FeatureC))
|
||||
|
||||
cdef StateClass stcls = StateClass.init(tokens.c, tokens.length)
|
||||
self.moves.initialize_state(stcls.c)
|
||||
|
||||
cdef class_t clas
|
||||
cdef ParserPerceptron model = self.model
|
||||
for clas in hist:
|
||||
fill_context(context, stcls.c)
|
||||
nr_feat = model.extracter.set_features(features, context)
|
||||
for feat in features[:nr_feat]:
|
||||
model.update_weight(feat.key, clas, feat.value * inc)
|
||||
self.moves.c[clas].do(stcls.c, self.moves.c[clas].label)
|
||||
|
||||
|
||||
# These are passed as callbacks to thinc.search.Beam
|
||||
cdef int _transition_state(void* _dest, void* _src, class_t clas, void* _moves) except -1:
|
||||
dest = <StateClass>_dest
|
||||
src = <StateClass>_src
|
||||
moves = <const Transition*>_moves
|
||||
dest.clone(src)
|
||||
moves[clas].do(dest.c, moves[clas].label)
|
||||
|
||||
|
||||
cdef void* _init_state(Pool mem, int length, void* tokens) except NULL:
|
||||
cdef StateClass st = StateClass.init(<const TokenC*>tokens, length)
|
||||
# Ensure sent_start is set to 0 throughout
|
||||
for i in range(st.c.length):
|
||||
st.c._sent[i].sent_start = False
|
||||
st.c._sent[i].l_edge = i
|
||||
st.c._sent[i].r_edge = i
|
||||
st.fast_forward()
|
||||
Py_INCREF(st)
|
||||
return <void*>st
|
||||
|
||||
|
||||
cdef int _check_final_state(void* _state, void* extra_args) except -1:
|
||||
return (<StateClass>_state).is_final()
|
||||
|
||||
|
||||
def _cleanup(Beam beam):
|
||||
for i in range(beam.width):
|
||||
Py_XDECREF(<PyObject*>beam._states[i].content)
|
||||
Py_XDECREF(<PyObject*>beam._parents[i].content)
|
||||
|
||||
|
||||
cdef hash_t _hash_state(void* _state, void* _) except 0:
|
||||
state = <StateClass>_state
|
||||
#return <uint64_t>state.c
|
||||
return state.c.hash()
|
||||
|
||||
|
||||
#
|
||||
# def _advance_beam(self, Beam beam, GoldParse gold, bint follow_gold, words):
|
||||
# cdef atom_t[CONTEXT_SIZE] context
|
||||
# cdef int i, j, cost
|
||||
# cdef bint is_valid
|
||||
# cdef const Transition* move
|
||||
#
|
||||
# for i in range(beam.size):
|
||||
# state = <StateClass>beam.at(i)
|
||||
# if not state.is_final():
|
||||
# # What the model is predicting here:
|
||||
# # We know, separately, the probability of the current state
|
||||
# # We can think of a state as a sequence of (action, score) pairs
|
||||
# # We obtain a state by doing reduce(state, [act for act, score in scores])
|
||||
# # We obtain its probability by doing sum(score for act, score in scores)
|
||||
# #
|
||||
# # So after running the forward pass, we have this output layer...
|
||||
# # The output layer has N nodes in its output, for our N moves
|
||||
# # The model asserts that:
|
||||
# #
|
||||
# # P(actions[i](state)) == score + output[i]
|
||||
# #
|
||||
# # i.e. each node holds a score that means "This is the difference
|
||||
# # in goodness that will occur if you apply this action to this state.
|
||||
# # If you apply this action, this is how I would judge the state."
|
||||
# self.model.set_scoresC(beam.scores[i], eg)
|
||||
# self.moves.set_validC(beam.is_valid[i], state)
|
||||
# if gold is not None:
|
||||
# for i in range(beam.size):
|
||||
# state = <StateClass>beam.at(i)
|
||||
# if not stcls.is_final():
|
||||
# self.moves.set_costsC(beam.costs[i], beam.is_valid[i],
|
||||
# state, gold)
|
||||
# if follow_gold:
|
||||
# for j in range(self.moves.n_moves):
|
||||
# beam.is_valid[i][j] *= beam.costs[i][j] == 0
|
||||
# beam.advance(_transition_state, _hash_state, <void*>self.moves.c)
|
||||
# beam.check_done(_check_final_state, NULL)
|
||||
#
|
||||
# def _update(self, Doc doc, g_hist, p_hist, loss):
|
||||
# pred = StateClass(doc)
|
||||
# gold = StateClass(doc)
|
||||
# for g_move, p_move in zip(g_hist, p_hist):
|
||||
# self.model(pred_eg)
|
||||
# self.model(gold_eg)
|
||||
#
|
||||
# margin = pred_eg.scores[p_move] - gold_eg.scores[g_move] + 1
|
||||
# if margin > 0:
|
||||
# gold_eg.losses[g_move] = margin
|
||||
# self.model.update(gold_eg)
|
||||
# pred_eg.losses[p_move] = -margin
|
||||
# self.model.update(pred_eg.guess)
|
||||
# self.c.moves[g_move].do(gold)
|
||||
# self.c.moves[p_move].do(pred)
|
||||
#
|
||||
#
|
||||
#
|
||||
Loading…
Reference in New Issue
Block a user