spaCy/spacy/tagger.pyx

# cython: profile=True
from __future__ import unicode_literals
from __future__ import division

from os import path
import os
import shutil
import random
import json
import cython

from thinc.features cimport Feature, count_feats


def setup_model_dir(tag_names, tag_map, templates, model_dir):
    if path.exists(model_dir):
        shutil.rmtree(model_dir)
    os.mkdir(model_dir)
    config = {
        'templates': templates,
        'tag_names': tag_names,
        'tag_map': tag_map
    }
    with open(path.join(model_dir, 'config.json'), 'w') as file_:
        json.dump(config, file_)


cdef class Tagger:
    """Predict some type of tag, using greedy decoding.  The tagger reads its
    model and configuration from disk.
    """
    def __init__(self, model_dir):
        self.mem = Pool()
        cfg = json.load(open(path.join(model_dir, 'config.json')))
        templates = cfg['templates']
        univ_counts = {}
        cdef unicode tag
        cdef unicode univ_tag
        tag_names = cfg['tag_names']
        self.extractor = Extractor(templates)
        self.model = LinearModel(len(tag_names) + 1, self.extractor.n_templ+2) # TODO
        if path.exists(path.join(model_dir, 'model')):
            self.model.load(path.join(model_dir, 'model'))

    cdef class_t predict(self, atom_t* context, object golds=None) except *:
        """Predict the tag of tokens[i].

        >>> tokens = EN.tokenize(u'An example sentence.')
        >>> tag = EN.pos_tagger.predict(0, tokens)
        >>> assert tag == EN.pos_tagger.tag_id('DT') == 5
        """
        cdef int n_feats
        cdef const Feature* feats = self.extractor.get_feats(context, &n_feats)
        cdef const weight_t* scores = self.model.get_scores(feats, n_feats)
        guess = _arg_max(scores, self.model.nr_class)
        if golds is not None and guess not in golds:
            best = _arg_max_among(scores, golds)
            counts = {guess: {}, best: {}}
            count_feats(counts[guess], feats, n_feats, -1)
            count_feats(counts[best], feats, n_feats, 1)
            self.model.update(counts)
        return guess


cdef int _arg_max(const weight_t* scores, int n_classes) except -1:
    cdef int best = 0
    cdef weight_t score = scores[best]
    cdef int i
    for i in range(1, n_classes):
        if scores[i] >= score:
            score = scores[i]
            best = i
    return best


cdef int _arg_max_among(const weight_t* scores, list classes) except -1:
    cdef int best = classes[0]
    cdef weight_t score = scores[best]
    cdef class_t clas
    for clas in classes:
        if scores[clas] > score:
            score = scores[clas]
            best = clas
    return best