explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-26 17:24:41 +03:00
Code Issues Packages Projects Releases Wiki Activity

Learns things

Browse Source
This commit is contained in:
Matthew Honnibal 2017-05-06 17:37:36 +02:00
parent 8e48b58cd6
commit bcf4cd0a5f
4 changed files with 35 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bin/parser/train_ud.py
View File

@ -137,8 +137,8 @@ def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None):
Xs, ys = organize_data(vocab, train_sents)
Xs = Xs[:1]
ys = ys[:1]
Xs = Xs[:10]
ys = ys[:10]
with encoder.model.begin_training(Xs[:100], ys[:100]) as (trainer, optimizer):
docs = list(Xs)
for doc in docs:
@ -151,8 +151,8 @@ def main(lang_name, train_loc, dev_loc, model_dir, clusters_loc=None):
print('%d:\t%.3f\t%.3f\t%.3f' % (itn, nn_loss[-1], scorer.uas, scorer.tags_acc))
nn_loss.append(0.)
trainer.each_epoch.append(track_progress)
trainer.batch_size = 1
trainer.nb_epoch = 100
trainer.batch_size = 2
trainer.nb_epoch = 10000
for docs, golds in trainer.iterate(Xs, ys, progress_bar=False):
docs = [Doc(vocab, words=[w.text for w in doc]) for doc in docs]
tokvecs, upd_tokvecs = encoder.begin_update(docs)

11
spacy/_ml.py
View File

@ -5,7 +5,7 @@ from thinc.neural._classes.hash_embed import HashEmbed
from thinc.neural._classes.convolution import ExtractWindow
from thinc.neural._classes.static_vectors import StaticVectors
from .attrs import ID, PREFIX, SUFFIX, SHAPE, TAG, DEP
from .attrs import ID, LOWER, PREFIX, SUFFIX, SHAPE, TAG, DEP
def get_col(idx):
@ -147,19 +147,20 @@ def flatten(seqs, drop=0.):
def build_tok2vec(lang, width, depth=2, embed_size=1000):
cols = [ID, PREFIX, SUFFIX, SHAPE]
cols = [ID, LOWER, PREFIX, SUFFIX, SHAPE, TAG]
with Model.define_operators({'>>': chain, '|': concatenate, '**': clone}):
#static = get_col(cols.index(ID)) >> StaticVectors(lang, width)
lower = get_col(cols.index(ID)) >> HashEmbed(width, embed_size)
lower = get_col(cols.index(LOWER)) >> HashEmbed(width, embed_size)
prefix = get_col(cols.index(PREFIX)) >> HashEmbed(width, embed_size)
suffix = get_col(cols.index(SUFFIX)) >> HashEmbed(width, embed_size)
shape = get_col(cols.index(SHAPE)) >> HashEmbed(width, embed_size)
tag = get_col(cols.index(TAG)) >> HashEmbed(width, embed_size)
tok2vec = (
doc2feats(cols)
>> with_flatten(
#(static | prefix | suffix | shape)
(lower | prefix | suffix | shape)
>> Maxout(width, width*4)
(lower | prefix | suffix | shape | tag)
>> Maxout(width, width*5)
>> (ExtractWindow(nW=1) >> Maxout(width, width*3))
>> (ExtractWindow(nW=1) >> Maxout(width, width*3))
)

31
spacy/syntax/parser.pyx
View File

@ -113,7 +113,7 @@ cdef class Parser:
def __reduce__(self):
return (Parser, (self.vocab, self.moves, self.model), None, None)
def build_model(self, width=8, nr_vector=1000, nF=1, nB=1, nS=1, nL=1, nR=1, **_):
def build_model(self, width=32, nr_vector=1000, nF=1, nB=1, nS=1, nL=1, nR=1, **_):
state2vec = build_debug_state2vec(width, nr_vector, nF, nB, nL, nR)
model = build_debug_model(state2vec, width, 2, self.moves.n_moves)
return model
@ -197,7 +197,7 @@ cdef class Parser:
attr_names = self.model.ops.allocate((2,), dtype='i')
attr_names[0] = TAG
attr_names[1] = DEP
features = self._get_features(states, tokvecs, attr_names)
self.model.begin_training(features)
@ -214,11 +214,12 @@ cdef class Parser:
output = list(d_tokens)
todo = zip(states, tokvecs, golds, d_tokens)
assert len(states) == len(todo)
loss = 0.
losses = []
while todo:
states, tokvecs, golds, d_tokens = zip(*todo)
scores, finish_update = self._begin_update(states, tokvecs)
token_ids, batch_token_grads = finish_update(golds, sgd=sgd)
token_ids, batch_token_grads = finish_update(golds, sgd=sgd, losses=losses,
force_gold=False)
for i, tok_i in enumerate(token_ids):
d_tokens[i][tok_i] += batch_token_grads[i]
@ -226,7 +227,7 @@ cdef class Parser:
# Get unfinished states (and their matching gold and token gradients)
todo = filter(lambda sp: not sp[0].py_is_final(), todo)
return output, loss
return output, sum(losses)
def _begin_update(self, states, tokvecs, drop=0.):
nr_class = self.moves.n_moves
@ -240,14 +241,17 @@ cdef class Parser:
self._validate_batch(is_valid, states)
softmaxed = self.model.ops.softmax(scores)
softmaxed *= is_valid
softmaxed /= softmaxed.sum(axis=1)
print('Scores', softmaxed[0])
def backward(golds, sgd=None):
softmaxed /= softmaxed.sum(axis=1).reshape((softmaxed.shape[0], 1))
def backward(golds, sgd=None, losses=[], force_gold=False):
nonlocal softmaxed
costs = self.model.ops.allocate((len(states), nr_class), dtype='f')
d_scores = self.model.ops.allocate((len(states), nr_class), dtype='f')
self._cost_batch(costs, is_valid, states, golds)
self._set_gradient(d_scores, scores, is_valid, costs)
losses.append(numpy.abs(d_scores).sum())
if force_gold:
softmaxed *= costs <= 0
return finish_update(d_scores, sgd=sgd)
return softmaxed, backward
@ -298,17 +302,16 @@ cdef class Parser:
def _set_gradient(self, gradients, scores, is_valid, costs):
"""Do multi-label log loss"""
cdef double Z, gZ, max_, g_max
n = gradients.shape[0]
scores = scores * is_valid
g_scores = scores * is_valid * (costs <= 0.)
exps = numpy.exp(scores - scores.max(axis=1))
exps = numpy.exp(scores - scores.max(axis=1).reshape((n, 1)))
exps *= is_valid
g_exps = numpy.exp(g_scores - g_scores.max(axis=1))
g_exps = numpy.exp(g_scores - g_scores.max(axis=1).reshape((n, 1)))
g_exps *= costs <= 0.
g_exps *= is_valid
gradients[:] = exps / exps.sum(axis=1)
gradients -= g_exps / g_exps.sum(axis=1)
print('Gradient', gradients[0])
print('Costs', costs[0])
gradients[:] = exps / exps.sum(axis=1).reshape((n, 1))
gradients -= g_exps / g_exps.sum(axis=1).reshape((n, 1))
def step_through(self, Doc doc, GoldParse gold=None):
"""

16
spacy/syntax/stateclass.pyx
View File

@ -47,18 +47,18 @@ cdef class StateClass:
return ' '.join((third, second, top, '|', n0, n1))
def nr_context_tokens(self, int nF, int nB, int nS, int nL, int nR):
return 3
#return 1+nF+nB+nS + nL + (nS * nL) + (nS * nR)
return 8
def set_context_tokens(self, int[:] output, nF=1, nB=0, nS=2,
nL=2, nR=2):
output[0] = self.B(0)
output[1] = self.S(0)
output[2] = self.S(1)
#output[3] = self.L(self.S(0), 1)
#output[4] = self.L(self.S(0), 2)
#output[5] = self.R(self.S(0), 1)
#output[6] = self.R(self.S(0), 2)
output[1] = self.B(1)
output[2] = self.S(0)
output[3] = self.S(1)
output[4] = self.L(self.S(0), 1)
output[5] = self.L(self.S(0), 2)
output[6] = self.R(self.S(0), 1)
output[7] = self.R(self.S(0), 2)
#output[7] = self.L(self.S(1), 1)
#output[8] = self.L(self.S(1), 2)
#output[9] = self.R(self.S(1), 1)