explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-26 17:24:41 +03:00
Code Issues Packages Projects Releases Wiki Activity

WIP on refactor, with hidde pre-computing

Browse Source
This commit is contained in:
Matthew Honnibal 2017-05-07 02:02:43 +02:00
parent b439e04f8d
commit bdf2dba9fb
4 changed files with 216 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
bin/parser/train_ud.py
View File

@ -82,6 +82,7 @@ def organize_data(vocab, train_sents):
def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None):
LangClass = spacy.util.get_lang_class(lang_name)
train_sents = list(read_conllx(train_loc))
dev_sents = list(read_conllx(dev_loc))
train_sents = PseudoProjectivity.preprocess_training_data(train_sents)
actions = ArcEager.get_actions(gold_parses=train_sents)
@ -136,6 +137,7 @@ def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None):
parser = DependencyParser(vocab, actions=actions, features=features, L1=0.0)
Xs, ys = organize_data(vocab, train_sents)
dev_Xs, dev_ys = organize_data(vocab, dev_sents)
Xs = Xs[:100]
ys = ys[:100]
with encoder.model.begin_training(Xs[:100], ys[:100]) as (trainer, optimizer):
@ -145,13 +147,13 @@ def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None):
parser.begin_training(docs, ys)
nn_loss = [0.]
def track_progress():
scorer = score_model(vocab, encoder, tagger, parser, Xs, ys)
scorer = score_model(vocab, encoder, tagger, parser, dev_Xs, dev_ys)
itn = len(nn_loss)
print('%d:\t%.3f\t%.3f\t%.3f' % (itn, nn_loss[-1], scorer.uas, scorer.tags_acc))
nn_loss.append(0.)
trainer.each_epoch.append(track_progress)
trainer.batch_size = 6
trainer.nb_epoch = 10000
trainer.batch_size = 12
trainer.nb_epoch = 2
for docs, golds in trainer.iterate(Xs, ys, progress_bar=False):
docs = [Doc(vocab, words=[w.text for w in doc]) for doc in docs]
tokvecs, upd_tokvecs = encoder.begin_update(docs)
@ -163,10 +165,20 @@ def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None):
upd_tokvecs(d_tokvecs, sgd=optimizer)
nn_loss[-1] += loss
nlp = LangClass(vocab=vocab, tagger=tagger, parser=parser)
nlp.end_training(model_dir)
scorer = score_model(vocab, tagger, parser, read_conllx(dev_loc))
print('%d:\t%.3f\t%.3f\t%.3f' % (itn, scorer.uas, scorer.las, scorer.tags_acc))
#nlp.end_training(model_dir)
#scorer = score_model(vocab, tagger, parser, read_conllx(dev_loc))
#print('%d:\t%.3f\t%.3f\t%.3f' % (itn, scorer.uas, scorer.las, scorer.tags_acc))
if __name__ == '__main__':
import cProfile
import pstats
if 0:
plac.call(main)
else:
cProfile.runctx("plac.call(main)", globals(), locals(), "Profile.prof")
s = pstats.Stats("Profile.prof")
s.strip_dirs().sort_stats("time").print_stats()
plac.call(main)

236
spacy/_ml.py
View File

@ -21,181 +21,23 @@ def get_col(idx):
return layerize(forward)
def build_model(state2vec, width, depth, nr_class):
with Model.define_operators({'>>': chain, '**': clone}):
model = (
state2vec
>> Maxout(width, 1344)
>> Maxout(width, width)
>> Affine(nr_class, width)
)
return model
def build_debug_model(state2vec, width, depth, nr_class):
with Model.define_operators({'>>': chain, '**': clone}):
model = (
state2vec
>> Maxout(width)
>> Affine(nr_class)
)
return model
def build_debug_state2vec(width, nr_vector=1000, nF=1, nB=0, nS=1, nL=2, nR=2):
ops = Model.ops
def forward(tokens_attrs_vectors, drop=0.):
tokens, attr_vals, tokvecs = tokens_attrs_vectors
orig_tokvecs_shape = tokvecs.shape
tokvecs = tokvecs.reshape((tokvecs.shape[0], tokvecs.shape[1] *
tokvecs.shape[2]))
vector = tokvecs
def backward(d_vector, sgd=None):
d_tokvecs = vector.reshape(orig_tokvecs_shape)
return (tokens, d_tokvecs)
return vector, backward
model = layerize(forward)
return model
def build_state2vec(nr_context_tokens, width, nr_vector=1000):
ops = Model.ops
with Model.define_operators({'|': concatenate, '+': add, '>>': chain}):
hiddens = [get_col(i) >> Affine(width) for i in range(nr_context_tokens)]
model = (
get_token_vectors
>> add(*hiddens)
>> Maxout(width)
)
return model
def print_shape(prefix):
def forward(X, drop=0.):
return X, lambda dX, **kwargs: dX
return layerize(forward)
@layerize
def get_token_vectors(tokens_attrs_vectors, drop=0.):
ops = Model.ops
tokens, attrs, vectors = tokens_attrs_vectors
def backward(d_output, sgd=None):
return (tokens, d_output)
return vectors, backward
def build_parser_state2vec(width, nr_vector=1000, nF=1, nB=0, nS=1, nL=2, nR=2):
embed_tags = _reshape(chain(get_col(0), HashEmbed(16, nr_vector)))
embed_deps = _reshape(chain(get_col(1), HashEmbed(16, nr_vector)))
ops = embed_tags.ops
def forward(tokens_attrs_vectors, drop=0.):
tokens, attr_vals, tokvecs = tokens_attrs_vectors
tagvecs, bp_tagvecs = embed_deps.begin_update(attr_vals, drop=drop)
depvecs, bp_depvecs = embed_tags.begin_update(attr_vals, drop=drop)
orig_tokvecs_shape = tokvecs.shape
tokvecs = tokvecs.reshape((tokvecs.shape[0], tokvecs.shape[1] *
tokvecs.shape[2]))
shapes = (tagvecs.shape, depvecs.shape, tokvecs.shape)
assert tagvecs.shape[0] == depvecs.shape[0] == tokvecs.shape[0], shapes
vector = ops.xp.hstack((tagvecs, depvecs, tokvecs))
def backward(d_vector, sgd=None):
d_tagvecs, d_depvecs, d_tokvecs = backprop_concatenate(d_vector, shapes)
assert d_tagvecs.shape == shapes[0], (d_tagvecs.shape, shapes)
assert d_depvecs.shape == shapes[1], (d_depvecs.shape, shapes)
assert d_tokvecs.shape == shapes[2], (d_tokvecs.shape, shapes)
bp_tagvecs(d_tagvecs)
bp_depvecs(d_depvecs)
d_tokvecs = d_tokvecs.reshape(orig_tokvecs_shape)
return (tokens, d_tokvecs)
return vector, backward
model = layerize(forward)
model._layers = [embed_tags, embed_deps]
return model
def backprop_concatenate(gradient, shapes):
grads = []
start = 0
for shape in shapes:
end = start + shape[1]
grads.append(gradient[:, start : end])
start = end
return grads
def _reshape(layer):
'''Transforms input with shape
(states, tokens, features)
into input with shape:
(states * tokens, features)
So that it can be used with a token-wise feature extraction layer, e.g.
an embedding layer. The embedding layer outputs:
(states * tokens, ndim)
But we want to concatenate the vectors for the tokens, so we produce:
(states, tokens * ndim)
We then need to reverse the transforms to do the backward pass. Recall
the simple rule here: each layer is a map:
inputs -> (outputs, (d_outputs->d_inputs))
So the shapes must match like this:
shape of forward input == shape of backward output
shape of backward input == shape of forward output
'''
def forward(X__bfm, drop=0.):
b, f, m = X__bfm.shape
B = b*f
M = f*m
X__Bm = X__bfm.reshape((B, m))
y__Bn, bp_yBn = layer.begin_update(X__Bm, drop=drop)
n = y__Bn.shape[1]
N = f * n
y__bN = y__Bn.reshape((b, N))
def backward(dy__bN, sgd=None):
dy__Bn = dy__bN.reshape((B, n))
dX__Bm = bp_yBn(dy__Bn, sgd)
if dX__Bm is None:
return None
else:
return dX__Bm.reshape((b, f, m))
return y__bN, backward
model = layerize(forward)
model._layers.append(layer)
return model
@layerize
def flatten(seqs, drop=0.):
ops = Model.ops
def finish_update(d_X, sgd=None):
return d_X
X = ops.xp.concatenate([ops.asarray(seq) for seq in seqs])
return X, finish_update
def build_tok2vec(lang, width, depth=2, embed_size=1000):
cols = [ID, LOWER, PREFIX, SUFFIX, SHAPE, TAG]
with Model.define_operators({'>>': chain, '|': concatenate, '**': clone}):
#static = get_col(cols.index(ID)) >> StaticVectors(lang, width)
lower = get_col(cols.index(LOWER)) >> HashEmbed(width, embed_size)
prefix = get_col(cols.index(PREFIX)) >> HashEmbed(width, embed_size)
suffix = get_col(cols.index(SUFFIX)) >> HashEmbed(width, embed_size)
shape = get_col(cols.index(SHAPE)) >> HashEmbed(width, embed_size)
tag = get_col(cols.index(TAG)) >> HashEmbed(width, embed_size)
prefix = get_col(cols.index(PREFIX)) >> HashEmbed(width//4, embed_size)
suffix = get_col(cols.index(SUFFIX)) >> HashEmbed(width//4, embed_size)
shape = get_col(cols.index(SHAPE)) >> HashEmbed(width//4, embed_size)
tag = get_col(cols.index(TAG)) >> HashEmbed(width//2, embed_size)
tok2vec = (
doc2feats(cols)
>> with_flatten(
#(static | prefix | suffix | shape)
(lower | prefix | suffix | shape | tag)
>> Maxout(width, width*5)
#>> (ExtractWindow(nW=1) >> Maxout(width, width*3))
#>> (ExtractWindow(nW=1) >> Maxout(width, width*3))
>> Maxout(width)
>> (ExtractWindow(nW=1) >> Maxout(width, width*3))
>> (ExtractWindow(nW=1) >> Maxout(width, width*3))
)
)
return tok2vec
@ -208,3 +50,67 @@ def doc2feats(cols):
return feats, None
model = layerize(forward)
return model
def build_feature_precomputer(model, feat_maps):
'''Allow a model to be "primed" by pre-computing input features in bulk.
This is used for the parser, where we want to take a batch of documents,
and compute vectors for each (token, position) pair. These vectors can then
be reused, especially for beam-search.
Let's say we're using 12 features for each state, e.g. word at start of
buffer, three words on stack, their children, etc. In the normal arc-eager
system, a document of length N is processed in 2*N states. This means we'll
create 2*N*12 feature vectors --- but if we pre-compute, we only need
N*12 vector computations. The saving for beam-search is much better:
if we have a beam of k, we'll normally make 2*N*12*K computations --
so we can save the factor k. This also gives a nice CPU/GPU division:
we can do all our hard maths up front, packed into large multiplications,
and do the hard-to-program parsing on the CPU.
'''
def precompute(input_vectors):
cached, backprops = zip(*[lyr.begin_update(input_vectors)
for lyr in feat_maps)
def forward(batch_token_ids, drop=0.):
output = ops.allocate((batch_size, output_width))
# i: batch index
# j: position index (i.e. N0, S0, etc
# tok_i: Index of the token within its document
for i, token_ids in enumerate(batch_token_ids):
for j, tok_i in enumerate(token_ids):
output[i] += cached[j][tok_i]
def backward(d_vector, sgd=None):
d_inputs = ops.allocate((batch_size, n_feat, vec_width))
for i, token_ids in enumerate(batch_token_ids):
for j in range(len(token_ids)):
d_inputs[i][j] = backprops[j](d_vector, sgd)
# Return the IDs, so caller can associate to correct token
return (batch_token_ids, d_inputs)
return vector, backward
return chain(layerize(forward), model)
return precompute
def print_shape(prefix):
def forward(X, drop=0.):
return X, lambda dX, **kwargs: dX
return layerize(forward)
@layerize
def get_token_vectors(tokens_attrs_vectors, drop=0.):
ops = Model.ops
tokens, attrs, vectors = tokens_attrs_vectors
def backward(d_output, sgd=None):
return (tokens, d_output)
return vectors, backward
@layerize
def flatten(seqs, drop=0.):
ops = Model.ops
def finish_update(d_X, sgd=None):
return d_X
X = ops.xp.concatenate([ops.asarray(seq) for seq in seqs])
return X, finish_update

253
spacy/syntax/parser.pyx
View File

@ -44,9 +44,7 @@ from ..strings cimport StringStore
from ..gold cimport GoldParse
from ..attrs cimport TAG, DEP
from .._ml import build_parser_state2vec, build_model
from .._ml import build_state2vec, build_model
from .._ml import build_debug_state2vec, build_debug_model
from .._ml import build_state2vec, build_model, precompute_hiddens
USE_FTRL = True
@ -114,12 +112,12 @@ cdef class Parser:
def __reduce__(self):
return (Parser, (self.vocab, self.moves, self.model), None, None)
def build_model(self, width=64, nr_vector=1000, nF=1, nB=1, nS=1, nL=1, nR=1, **_):
def build_model(self, width=32, nr_vector=1000, nF=1, nB=1, nS=1, nL=1, nR=1, **_):
nr_context_tokens = StateClass.nr_context_tokens(nF, nB, nS, nL, nR)
state2vec = build_state2vec(nr_context_tokens, width, nr_vector)
#state2vec = build_debug_state2vec(width, nr_vector)
model = build_debug_model(state2vec, width*2, 2, self.moves.n_moves)
return model
return build_model_precomputer(
build_model(state2vec, width*2, 2, self.moves.n_moves)
build_feature_maps(nr_context_tokens, width, nr_vector))
def __call__(self, Doc tokens):
"""
@ -132,7 +130,7 @@ cdef class Parser:
"""
self.parse_batch([tokens])
self.moves.finalize_doc(tokens)
def pipe(self, stream, int batch_size=1000, int n_threads=2):
"""
Process a stream of documents.
@ -167,158 +165,50 @@ cdef class Parser:
yield doc
def parse_batch(self, docs):
states = self._init_states(docs)
nr_class = self.moves.n_moves
cdef Doc doc
cdef StateClass state
cdef int guess
tokvecs = [d.tensor for d in docs]
all_states = list(states)
todo = zip(states, tokvecs)
model, states = self.init_batch(docs)
todo = list(states)
while todo:
states, tokvecs = zip(*todo)
scores, _ = self._begin_update(states, tokvecs)
for state, guess in zip(states, scores.argmax(axis=1)):
action = self.moves.c[guess]
action.do(state.c, action.label)
todo = filter(lambda sp: not sp[0].py_is_final(), todo)
for state, doc in zip(all_states, docs):
todo = model(todo)
for state, doc in zip(states, docs):
self.moves.finalize_state(state.c)
for i in range(doc.length):
doc.c[i] = state.c._sent[i]
def begin_training(self, docs, golds):
for gold in golds:
self.moves.preprocess_gold(gold)
states = self._init_states(docs)
tokvecs = [d.tensor for d in docs]
d_tokens = [self.model.ops.allocate(d.tensor.shape) for d in docs]
nr_class = self.moves.n_moves
costs = self.model.ops.allocate((len(docs), nr_class), dtype='f')
gradients = self.model.ops.allocate((len(docs), nr_class), dtype='f')
is_valid = self.model.ops.allocate((len(docs), nr_class), dtype='i')
attr_names = self.model.ops.allocate((2,), dtype='i')
attr_names[0] = TAG
attr_names[1] = DEP
features = self._get_features(states, tokvecs, attr_names)
self.model.begin_training(features)
def update(self, docs, golds, drop=0., sgd=None):
if isinstance(docs, Doc) and isinstance(golds, GoldParse):
return self.update([docs], [golds], drop=drop)
for gold in golds:
self.moves.preprocess_gold(gold)
states = self._init_states(docs)
tokvecs = [d.tensor for d in docs]
model, states = self.init_batch(docs)
d_tokens = [self.model.ops.allocate(d.tensor.shape) for d in docs]
nr_class = self.moves.n_moves
output = list(d_tokens)
todo = zip(states, tokvecs, golds, d_tokens)
assert len(states) == len(todo)
losses = []
todo = zip(states, golds, d_tokens)
while todo:
states, tokvecs, golds, d_tokens = zip(*todo)
scores, finish_update = self._begin_update(states, tokvecs)
token_ids, batch_token_grads = finish_update(golds, sgd=sgd, losses=losses,
force_gold=False)
states, golds, d_tokens = zip(*todo)
states, finish_update = model.begin_update(states)
d_state_features = finish_update(golds, sgd=sgd)
for i, tok_ids in enumerate(token_ids):
for j, tok_i in enumerate(tok_ids):
if tok_i >= 0:
d_tokens[i][tok_i] += batch_token_grads[i, j]
self._transition_batch(states, scores)
d_tokens[i][tok_i] += d_state_features[i, j]
# Get unfinished states (and their matching gold and token gradients)
todo = filter(lambda sp: not sp[0].py_is_final(), todo)
return output, sum(losses)
def _begin_update(self, states, tokvecs, drop=0.):
nr_class = self.moves.n_moves
attr_names = self.model.ops.allocate((2,), dtype='i')
attr_names[0] = TAG
attr_names[1] = DEP
def begin_training(self, docs, golds):
for gold in golds:
self.moves.preprocess_gold(gold)
states = self._init_states(docs)
tokvecs = [d.tensor for d in docs]
features = self._get_features(states, tokvecs, attr_names)
scores, finish_update = self.model.begin_update(features, drop=drop)
assert scores.shape[0] == len(states), (len(states), scores.shape)
assert len(scores.shape) == 2
is_valid = self.model.ops.allocate((len(states), nr_class), dtype='i')
self._validate_batch(is_valid, states)
softmaxed = self.model.ops.softmax(scores)
softmaxed *= is_valid
softmaxed /= softmaxed.sum(axis=1).reshape((softmaxed.shape[0], 1))
def backward(golds, sgd=None, losses=[], force_gold=False):
nonlocal softmaxed
costs = self.model.ops.allocate((len(states), nr_class), dtype='f')
d_scores = self.model.ops.allocate((len(states), nr_class), dtype='f')
features = self._get_features(states, tokvecs)
self.model.begin_training(features)
self._cost_batch(costs, is_valid, states, golds)
self._set_gradient(d_scores, scores, is_valid, costs)
losses.append(numpy.abs(d_scores).sum())
if force_gold:
softmaxed *= costs <= 0
return finish_update(d_scores, sgd=sgd)
return softmaxed, backward
def _init_states(self, docs):
states = []
cdef Doc doc
cdef StateClass state
for i, doc in enumerate(docs):
state = StateClass.init(doc.c, doc.length)
self.moves.initialize_state(state.c)
states.append(state)
return states
def _get_features(self, states, all_tokvecs, attr_names,
nF=1, nB=0, nS=2, nL=2, nR=2):
n_tokens = states[0].nr_context_tokens(nF, nB, nS, nL, nR)
vector_length = all_tokvecs[0].shape[1]
tokens = self.model.ops.allocate((len(states), n_tokens), dtype='int32')
features = self.model.ops.allocate((len(states), n_tokens, attr_names.shape[0]), dtype='uint64')
tokvecs = self.model.ops.allocate((len(states), n_tokens, vector_length), dtype='f')
for i, state in enumerate(states):
state.set_context_tokens(tokens[i], nF, nB, nS, nL, nR)
state.set_attributes(features[i], tokens[i], attr_names)
state.set_token_vectors(tokvecs[i], all_tokvecs[i], tokens[i])
return (tokens, features, tokvecs)
def _validate_batch(self, int[:, ::1] is_valid, states):
cdef StateClass state
cdef int i
for i, state in enumerate(states):
self.moves.set_valid(&is_valid[i, 0], state.c)
def _cost_batch(self, weight_t[:, ::1] costs, int[:, ::1] is_valid,
states, golds):
cdef int i
cdef StateClass state
cdef GoldParse gold
for i, (state, gold) in enumerate(zip(states, golds)):
self.moves.set_costs(&is_valid[i, 0], &costs[i, 0], state, gold)
def _transition_batch(self, states, scores):
cdef StateClass state
cdef int guess
for state, guess in zip(states, scores.argmax(axis=1)):
action = self.moves.c[guess]
action.do(state.c, action.label)
def _set_gradient(self, gradients, scores, is_valid, costs):
"""Do multi-label log loss"""
cdef double Z, gZ, max_, g_max
n = gradients.shape[0]
scores = scores * is_valid
g_scores = scores * is_valid * (costs <= 0.)
exps = numpy.exp(scores - scores.max(axis=1).reshape((n, 1)))
exps *= is_valid
g_exps = numpy.exp(g_scores - g_scores.max(axis=1).reshape((n, 1)))
g_exps *= costs <= 0.
g_exps *= is_valid
gradients[:] = exps / exps.sum(axis=1).reshape((n, 1))
gradients -= g_exps / g_exps.sum(axis=1).reshape((n, 1))
def step_through(self, Doc doc, GoldParse gold=None):
"""
@ -355,6 +245,97 @@ cdef class Parser:
self.cfg.setdefault('extra_labels', []).append(label)
def _transition_batch(self, states, scores):
cdef StateClass state
cdef int guess
for state, guess in zip(states, scores.argmax(axis=1)):
action = self.moves.c[guess]
action.do(state.c, action.label)
def _set_gradient(self, gradients, scores, is_valid, costs):
"""Do multi-label log loss"""
cdef double Z, gZ, max_, g_max
n = gradients.shape[0]
scores = scores * is_valid
g_scores = scores * is_valid * (costs <= 0.)
exps = numpy.exp(scores - scores.max(axis=1).reshape((n, 1)))
exps *= is_valid
g_exps = numpy.exp(g_scores - g_scores.max(axis=1).reshape((n, 1)))
g_exps *= costs <= 0.
g_exps *= is_valid
gradients[:] = exps / exps.sum(axis=1).reshape((n, 1))
gradients -= g_exps / g_exps.sum(axis=1).reshape((n, 1))
def _begin_update(self, model, states, tokvecs, drop=0.):
nr_class = self.moves.n_moves
attr_names = self.model.ops.allocate((2,), dtype='i')
attr_names[0] = TAG
attr_names[1] = DEP
features = self._get_features(states, tokvecs, attr_names)
scores, finish_update = self.model.begin_update(features, drop=drop)
assert scores.shape[0] == len(states), (len(states), scores.shape)
assert len(scores.shape) == 2
is_valid = self.model.ops.allocate((len(states), nr_class), dtype='i')
self._validate_batch(is_valid, states)
softmaxed = self.model.ops.softmax(scores)
softmaxed *= is_valid
softmaxed /= softmaxed.sum(axis=1).reshape((softmaxed.shape[0], 1))
def backward(golds, sgd=None, losses=[], force_gold=False):
nonlocal softmaxed
costs = self.model.ops.allocate((len(states), nr_class), dtype='f')
d_scores = self.model.ops.allocate((len(states), nr_class), dtype='f')
self._cost_batch(costs, is_valid, states, golds)
self._set_gradient(d_scores, scores, is_valid, costs)
losses.append(numpy.abs(d_scores).sum())
if force_gold:
softmaxed *= costs <= 0
return finish_update(d_scores, sgd=sgd)
return softmaxed, backward
def _init_states(self, docs):
states = []
cdef Doc doc
cdef StateClass state
for i, doc in enumerate(docs):
state = StateClass.init(doc.c, doc.length)
self.moves.initialize_state(state.c)
states.append(state)
return states
def _validate_batch(self, int[:, ::1] is_valid, states):
cdef StateClass state
cdef int i
for i, state in enumerate(states):
self.moves.set_valid(&is_valid[i, 0], state.c)
def _cost_batch(self, weight_t[:, ::1] costs, int[:, ::1] is_valid,
states, golds):
cdef int i
cdef StateClass state
cdef GoldParse gold
for i, (state, gold) in enumerate(zip(states, golds)):
self.moves.set_costs(&is_valid[i, 0], &costs[i, 0], state, gold)
def _get_features(self, states, all_tokvecs, attr_names,
nF=1, nB=0, nS=2, nL=2, nR=2):
n_tokens = states[0].nr_context_tokens(nF, nB, nS, nL, nR)
vector_length = all_tokvecs[0].shape[1]
tokens = self.model.ops.allocate((len(states), n_tokens), dtype='int32')
features = self.model.ops.allocate((len(states), n_tokens, attr_names.shape[0]), dtype='uint64')
tokvecs = self.model.ops.allocate((len(states), n_tokens, vector_length), dtype='f')
for i, state in enumerate(states):
state.set_context_tokens(tokens[i], nF, nB, nS, nL, nR)
state.set_attributes(features[i], tokens[i], attr_names)
state.set_token_vectors(tokvecs[i], all_tokvecs[i], tokens[i])
return (tokens, features, tokvecs)
cdef int dropout(FeatureC* feats, int nr_feat, float prob) except -1:
if prob <= 0 or prob >= 1.:
return 0

19
spacy/syntax/stateclass.pyx
View File

@ -48,7 +48,7 @@ cdef class StateClass:
@classmethod
def nr_context_tokens(cls, int nF, int nB, int nS, int nL, int nR):
return 4
return 5
def set_context_tokens(self, int[:] output, nF=1, nB=0, nS=2,
nL=2, nR=2):
@ -56,14 +56,15 @@ cdef class StateClass:
output[1] = self.B(1)
output[2] = self.S(0)
output[3] = self.S(1)
#output[4] = self.L(self.S(0), 1)
#output[5] = self.L(self.S(0), 2)
#output[6] = self.R(self.S(0), 1)
#output[7] = self.R(self.S(0), 2)
#output[7] = self.L(self.S(1), 1)
#output[8] = self.L(self.S(1), 2)
#output[9] = self.R(self.S(1), 1)
#output[10] = self.R(self.S(1), 2)
output[4] = self.S(2)
#output[5] = self.L(self.S(0), 1)
#output[6] = self.L(self.S(0), 2)
#output[7] = self.R(self.S(0), 1)
#output[8] = self.R(self.S(0), 2)
#output[10] = self.L(self.S(1), 1)
#output[11] = self.L(self.S(1), 2)
#output[12] = self.R(self.S(1), 1)
#output[13] = self.R(self.S(1), 2)
def set_attributes(self, uint64_t[:, :] vals, int[:] tokens, int[:] names):
cdef int i, j, tok_i