spaCy/spacy/syntax/parser.pyx

# cython: infer_types=True
# cython: cdivision=True
# cython: profile=True
"""
MALT-style dependency parser
"""
from __future__ import unicode_literals
cimport cython
cimport cython.parallel

from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF
from cpython.exc cimport PyErr_CheckSignals

from libc.stdint cimport uint32_t, uint64_t
from libc.string cimport memset, memcpy
from libc.stdlib cimport malloc, calloc, free

import os.path
from collections import Counter
from os import path
import shutil
import json
import sys
from .nonproj import PseudoProjectivity
import numpy
import random
cimport numpy as np
np.import_array()

from cymem.cymem cimport Pool, Address
from murmurhash.mrmr cimport hash64, hash32
from thinc.typedefs cimport weight_t, class_t, feat_t, atom_t, hash_t
from thinc.linear.avgtron cimport AveragedPerceptron
from thinc.linalg cimport VecVec
from thinc.structs cimport SparseArrayC
from preshed.maps cimport MapStruct
from preshed.maps cimport map_get
from thinc.neural.ops import NumpyOps
from thinc.neural.optimizers import Adam
from thinc.neural.optimizers import SGD

from thinc.structs cimport FeatureC
from thinc.structs cimport ExampleC
from thinc.extra.eg cimport Example

from util import Config

from ..structs cimport TokenC

from ..tokens.doc cimport Doc
from ..strings cimport StringStore

from .transition_system import OracleError
from .transition_system cimport TransitionSystem, Transition

from ..gold cimport GoldParse

from . import _parse_features
from ._parse_features cimport CONTEXT_SIZE
from ._parse_features cimport fill_context
from .stateclass cimport StateClass
from ._state cimport StateC
from .._ml cimport LinearModel


DEBUG = False
def set_debug(val):
    global DEBUG
    DEBUG = val


def get_templates(name):
    pf = _parse_features
    if name == 'ner':
        return pf.ner
    elif name == 'debug':
        return pf.unigrams
    elif name.startswith('embed'):
        return (pf.words, pf.tags, pf.labels)
    else:
        return (pf.unigrams + pf.s0_n0 + pf.s1_n0 + pf.s1_s0 + pf.s0_n1 + pf.n0_n1 + \
                pf.tree_shape + pf.trigrams)


#cdef class ParserModel(AveragedPerceptron):
#    cdef int set_featuresC(self, atom_t* context, FeatureC* features,
#            const StateC* state) nogil:
#        fill_context(context, state)
#        nr_feat = self.extracter.set_features(features, context)
#        return nr_feat
#
#    def update(self, Example eg, itn=0):
#        '''Does regression on negative cost. Sort of cute?'''
#        self.time += 1
#        best = arg_max_if_gold(eg.c.scores, eg.c.costs, eg.c.nr_class)
#        guess = eg.guess
#        cdef weight_t loss = 0.0
#        if guess == best:
#            return loss
#        for clas in [guess, best]:
#            loss += (-eg.c.costs[clas] - eg.c.scores[clas]) ** 2
#            d_loss = eg.c.scores[clas] - -eg.c.costs[clas]
#            for feat in eg.c.features[:eg.c.nr_feat]:
#                self.update_weight_ftrl(feat.key, clas, feat.value * d_loss)
#        return loss
#
#    def update_from_histories(self, TransitionSystem moves, Doc doc, histories, weight_t min_grad=0.0):
#        cdef Pool mem = Pool()
#        features = <FeatureC*>mem.alloc(self.nr_feat, sizeof(FeatureC))
#
#        cdef StateClass stcls
#
#        cdef class_t clas
#        self.time += 1
#        cdef atom_t[CONTEXT_SIZE] atoms
#        histories = [(grad, hist) for grad, hist in histories if abs(grad) >= min_grad and hist]
#        if not histories:
#            return None
#        gradient = [Counter() for _ in range(max([max(h)+1 for _, h in histories]))]
#        for d_loss, history in histories:
#            stcls = StateClass.init(doc.c, doc.length)
#            moves.initialize_state(stcls.c)
#            for clas in history:
#                nr_feat = self.set_featuresC(atoms, features, stcls.c)
#                clas_grad = gradient[clas]
#                for feat in features[:nr_feat]:
#                    clas_grad[feat.key] += d_loss * feat.value
#                moves.c[clas].do(stcls.c, moves.c[clas].label)
#        cdef feat_t key
#        cdef weight_t d_feat
#        for clas, clas_grad in enumerate(gradient):
#            for key, d_feat in clas_grad.items():
#                if d_feat != 0:
#                    self.update_weight_ftrl(key, clas, d_feat)
#

cdef class ParserModel(LinearModel):
    cdef int set_featuresC(self, atom_t* context, FeatureC* features,
            const StateC* state) nogil:
        fill_context(context, state)
        nr_feat = self.extracter.set_features(features, context)
        return nr_feat


cdef class Parser:
    """Base class of the DependencyParser and EntityRecognizer."""
    @classmethod
    def load(cls, path, Vocab vocab, TransitionSystem=None, require=False, **cfg):
        """Load the statistical model from the supplied path.

        Arguments:
            path (Path):
                The path to load from.
            vocab (Vocab):
                The vocabulary. Must be shared by the documents to be processed.
            require (bool):
                Whether to raise an error if the files are not found.
        Returns (Parser):
            The newly constructed object.
        """
        with (path / 'config.json').open() as file_:
            cfg = json.load(file_)
        # TODO: remove this shim when we don't have to support older data
        if 'labels' in cfg and 'actions' not in cfg:
            cfg['actions'] = cfg.pop('labels')
        self = cls(vocab, TransitionSystem=TransitionSystem, model=None, **cfg)
        if (path / 'model').exists():
            self.model.load(str(path / 'model'))
        elif require:
            raise IOError(
                "Required file %s/model not found when loading" % str(path))
        return self

    def __init__(self, Vocab vocab, TransitionSystem=None, ParserModel model=None, **cfg):
        """Create a Parser.

        Arguments:
            vocab (Vocab):
                The vocabulary object. Must be shared with documents to be processed.
            model (thinc.linear.AveragedPerceptron):
                The statistical model.
        Returns (Parser):
            The newly constructed object.
        """
        if TransitionSystem is None:
            TransitionSystem = self.TransitionSystem
        self.vocab = vocab
        actions = TransitionSystem.get_actions(**cfg)
        self.moves = TransitionSystem(vocab.strings, actions)
        # TODO: Remove this when we no longer need to support old-style models
        if isinstance(cfg.get('features'), basestring):
            cfg['features'] = get_templates(cfg['features'])
        elif 'features' not in cfg:
            cfg['features'] = self.feature_templates
        self.model = ParserModel(self.moves.n_moves, cfg['features'],
                                 size=2**18,
                                 learn_rate=cfg.get('learn_rate', 0.001))
        #self.model.l1_penalty = cfg.get('L1', 1e-8)
        #self.model.learn_rate = cfg.get('learn_rate', 0.001)

        self.optimizer = SGD(NumpyOps(), cfg.get('learn_rate', 0.001),
                             momentum=0.9)

        self.cfg = cfg

    def __reduce__(self):
        return (Parser, (self.vocab, self.moves, self.model), None, None)

    def __call__(self, Doc tokens):
        """Apply the entity recognizer, setting the annotations onto the Doc object.

        Arguments:
            doc (Doc): The document to be processed.
        Returns:
            None
        """
        cdef int nr_feat = self.model.nr_feat
        with nogil:
            status = self.parseC(tokens.c, tokens.length, nr_feat, self.moves.n_moves)
        # Check for KeyboardInterrupt etc. Untested
        PyErr_CheckSignals()
        if status != 0:
            raise ParserStateError(tokens)
        self.moves.finalize_doc(tokens)

    def pipe(self, stream, int batch_size=1000, int n_threads=2):
        """Process a stream of documents.

        Arguments:
            stream: The sequence of documents to process.
            batch_size (int):
                The number of documents to accumulate into a working set.
            n_threads (int):
                The number of threads with which to work on the buffer in parallel.
        Yields (Doc): Documents, in order.
        """
        cdef Pool mem = Pool()
        cdef TokenC** doc_ptr = <TokenC**>mem.alloc(batch_size, sizeof(TokenC*))
        cdef int* lengths = <int*>mem.alloc(batch_size, sizeof(int))
        cdef Doc doc
        cdef int i
        cdef int nr_feat = self.model.nr_feat
        cdef int status
        queue = []
        for doc in stream:
            doc_ptr[len(queue)] = doc.c
            lengths[len(queue)] = doc.length
            queue.append(doc)
            if len(queue) == batch_size:
                with nogil:
                    for i in cython.parallel.prange(batch_size, num_threads=n_threads):
                        status = self.parseC(doc_ptr[i], lengths[i], nr_feat, self.moves.n_moves)
                        if status != 0:
                            with gil:
                                raise ParserStateError(queue[i])
                PyErr_CheckSignals()
                for doc in queue:
                    self.moves.finalize_doc(doc)
                    yield doc
                queue = []
        batch_size = len(queue)
        with nogil:
            for i in cython.parallel.prange(batch_size, num_threads=n_threads):
                status = self.parseC(doc_ptr[i], lengths[i], nr_feat, self.moves.n_moves)
                if status != 0:
                    with gil:
                        raise ParserStateError(queue[i])
        PyErr_CheckSignals()
        for doc in queue:
            self.moves.finalize_doc(doc)
            yield doc

    cdef int parseC(self, TokenC* tokens, int length, int nr_feat, int nr_class) with gil:
        state = new StateC(tokens, length)
        # NB: This can change self.moves.n_moves!
        self.moves.initialize_state(state)

        cdef ExampleC eg
        eg.nr_feat = nr_feat
        eg.nr_atom = CONTEXT_SIZE
        eg.nr_class = nr_class
        eg.features = <FeatureC*>calloc(sizeof(FeatureC), nr_feat)
        eg.atoms = <atom_t*>calloc(sizeof(atom_t), CONTEXT_SIZE)
        eg.scores = <weight_t*>calloc(sizeof(weight_t), nr_class)
        eg.is_valid = <int*>calloc(sizeof(int), nr_class)
        cdef int i
        while not state.is_final():
            eg.nr_feat = self.model.set_featuresC(eg.atoms, eg.features, state)
            self.moves.set_valid(eg.is_valid, state)
            self.model.set_scoresC(eg.scores, eg.features, eg.nr_feat)

            guess = VecVec.arg_max_if_true(eg.scores, eg.is_valid, eg.nr_class)

            action = self.moves.c[guess]
            if not eg.is_valid[guess]:
                return 1

            action.do(state, action.label)
            memset(eg.scores, 0, sizeof(eg.scores[0]) * eg.nr_class)
            for i in range(eg.nr_class):
                eg.is_valid[i] = 1
        self.moves.finalize_state(state)
        for i in range(length):
            tokens[i] = state._sent[i]
        del state
        free(eg.features)
        free(eg.atoms)
        free(eg.scores)
        free(eg.is_valid)
        return 0

    def update(self, Doc tokens, GoldParse gold, itn=0):
        """Update the statistical model.

        Arguments:
            doc (Doc):
                The example document for the update.
            gold (GoldParse):
                The gold-standard annotations, to calculate the loss.
        Returns (float):
            The loss on this example.
        """
        self.moves.preprocess_gold(gold)
        cdef StateClass stcls = StateClass.init(tokens.c, tokens.length)
        self.moves.initialize_state(stcls.c)

        cdef int nr_class = self.model.nr_class
        cdef Pool mem = Pool()
        d_scores = <weight_t*>mem.alloc(nr_class, sizeof(weight_t))
        scores = <weight_t*>mem.alloc(nr_class, sizeof(weight_t))
        costs = <weight_t*>mem.alloc(nr_class, sizeof(weight_t))
        features = <FeatureC*>mem.alloc(self.model.nr_feat, sizeof(FeatureC))
        is_valid = <int*>mem.alloc(self.moves.n_moves, sizeof(int))
        cdef atom_t[CONTEXT_SIZE] context

        cdef weight_t loss = 0
        cdef Transition action
        words = [w.text for w in tokens]
 
        while not stcls.is_final():

            nr_feat = self.model.set_featuresC(context, features, stcls.c)
            self.moves.set_costs(is_valid, costs, stcls, gold)
            self.model.set_scoresC(scores, features, nr_feat)

            guess = VecVec.arg_max_if_true(scores, is_valid, nr_class)
            best = arg_max_if_gold(scores, costs, nr_class)

            self.model.regression_lossC(d_scores, scores, costs)
            self.model.set_gradientC(d_scores, features, nr_feat) 

            action = self.moves.c[guess]
            action.do(stcls.c, action.label)
            #print(scores[guess], scores[best], d_scores[guess], costs[guess],
            #    self.moves.move_name(action.move, action.label), stcls.print_state(words))

            loss += scores[guess]
            memset(context, 0, sizeof(context))
            memset(features, 0, sizeof(features[0]) * nr_feat)
            memset(scores, 0, sizeof(scores[0]) * nr_class)
            memset(d_scores, 0, sizeof(d_scores[0]) * nr_class)
            memset(costs, 0, sizeof(costs[0]) * nr_class)
            for i in range(nr_class):
                is_valid[i] = 1
        #if itn % 100 == 0:
        #    self.optimizer(self.model.model[0].ravel(),
        #        self.model.model[1].ravel(), key=1)
        return loss

    def step_through(self, Doc doc):
        """Set up a stepwise state, to introspect and control the transition sequence.

        Arguments:
            doc (Doc): The document to step through.
        Returns (StepwiseState):
            A state object, to step through the annotation process.
        """
        return StepwiseState(self, doc)

    def from_transition_sequence(self, Doc doc, sequence):
        """Control the annotations on a document by specifying a transition sequence
        to follow.

        Arguments:
            doc (Doc): The document to annotate.
            sequence: A sequence of action names, as unicode strings.
        Returns: None
        """
        with self.step_through(doc) as stepwise:
            for transition in sequence:
                stepwise.transition(transition)

    def add_label(self, label):
        # Doesn't set label into serializer -- subclasses override it to do that.
        for action in self.moves.action_types:
            self.moves.add_action(action, label)


cdef class StepwiseState:
    cdef readonly StateClass stcls
    cdef readonly Example eg
    cdef readonly Doc doc
    cdef readonly Parser parser

    def __init__(self, Parser parser, Doc doc):
        self.parser = parser
        self.doc = doc
        self.stcls = StateClass.init(doc.c, doc.length)
        self.parser.moves.initialize_state(self.stcls.c)
        self.eg = Example(
            nr_class=self.parser.moves.n_moves,
            nr_atom=CONTEXT_SIZE,
            nr_feat=self.parser.model.nr_feat)

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.finish()

    @property
    def is_final(self):
        return self.stcls.is_final()

    @property
    def stack(self):
        return self.stcls.stack

    @property
    def queue(self):
        return self.stcls.queue

    @property
    def heads(self):
        return [self.stcls.H(i) for i in range(self.stcls.c.length)]

    @property
    def deps(self):
        return [self.doc.vocab.strings[self.stcls.c._sent[i].dep]
                for i in range(self.stcls.c.length)]

    def predict(self):
        self.eg.reset()
        self.eg.c.nr_feat = self.parser.model.set_featuresC(self.eg.c.atoms, self.eg.c.features,
                                                            self.stcls.c)
        self.parser.moves.set_valid(self.eg.c.is_valid, self.stcls.c)
        self.parser.model.set_scoresC(self.eg.c.scores,
            self.eg.c.features, self.eg.c.nr_feat)

        cdef Transition action = self.parser.moves.c[self.eg.guess]
        return self.parser.moves.move_name(action.move, action.label)

    def transition(self, action_name=None):
        if action_name is None:
            action_name = self.predict()
        moves = {'S': 0, 'D': 1, 'L': 2, 'R': 3}
        if action_name == '_':
            action_name = self.predict()
            action = self.parser.moves.lookup_transition(action_name)
        elif action_name == 'L' or action_name == 'R':
            self.predict()
            move = moves[action_name]
            clas = _arg_max_clas(self.eg.c.scores, move, self.parser.moves.c,
                                 self.eg.c.nr_class)
            action = self.parser.moves.c[clas]
        else:
            action = self.parser.moves.lookup_transition(action_name)
        action.do(self.stcls.c, action.label)

    def finish(self):
        if self.stcls.is_final():
            self.parser.moves.finalize_state(self.stcls.c)
        self.doc.set_parse(self.stcls.c._sent)
        self.parser.moves.finalize_doc(self.doc)


class ParserStateError(ValueError):
    def __init__(self, doc):
        ValueError.__init__(self,
            "Error analysing doc -- no valid actions available. This should "
            "never happen, so please report the error on the issue tracker. "
            "Here's the thread to do so --- reopen it if it's closed:\n"
            "https://github.com/spacy-io/spaCy/issues/429\n"
            "Please include the text that the parser failed on, which is:\n"
            "%s" % repr(doc.text))

cdef int arg_max_if_gold(const weight_t* scores, const weight_t* costs, int n) nogil:
    cdef int best = -1
    for i in range(n):
        if costs[i] <= 0:
            if best == -1 or scores[i] > scores[best]:
                best = i
    return best


cdef int _arg_max_clas(const weight_t* scores, int move, const Transition* actions,
                       int nr_class) except -1:
    cdef weight_t score = 0
    cdef int mode = -1
    cdef int i
    for i in range(nr_class):
        if actions[i].move == move and (mode == -1 or scores[i] >= score):
            mode = i
            score = scores[i]
    return mode