explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-10-30 23:47:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

Try different drop_layer structure in Tok2Vec

Browse Source
This commit is contained in:
Matthew Honnibal 2017-08-12 08:56:57 -05:00
parent cd5ecedf6a
commit 4ab0c8c8e9
1 changed files with 12 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
spacy/_ml.py
View File

@ -210,14 +210,14 @@ class PrecomputableMaxouts(Model):
return Yfp, backward return Yfp, backward
def drop_layer(layer, factor=1.0): def drop_layer(layer, factor=2.):
def drop_layer_fwd(X, drop=0.): def drop_layer_fwd(X, drop=0.):
drop *= factor drop *= factor
mask = layer.ops.get_dropout_mask((1,), drop) mask = layer.ops.get_dropout_mask((1,), drop)
if mask is not None and mask[0] == 0.: if mask is None or mask > 0:
return X, lambda dX, sgd=None: dX
else:
return layer.begin_update(X, drop=drop) return layer.begin_update(X, drop=drop)
else:
return X, lambda dX, sgd=None: dX
return wrap(drop_layer_fwd, layer) return wrap(drop_layer_fwd, layer)
@ -229,17 +229,17 @@ def Tok2Vec(width, embed_size, preprocess=None):
suffix = get_col(cols.index(SUFFIX)) >> HashEmbed(width, embed_size//2, name='embed_suffix') suffix = get_col(cols.index(SUFFIX)) >> HashEmbed(width, embed_size//2, name='embed_suffix')
shape = get_col(cols.index(SHAPE)) >> HashEmbed(width, embed_size//2, name='embed_shape') shape = get_col(cols.index(SHAPE)) >> HashEmbed(width, embed_size//2, name='embed_shape')
embed = (norm | prefix | suffix | shape ) embed = (norm | prefix | suffix | shape ) >> Maxout(width, width*4, pieces=3)
tok2vec = ( tok2vec = (
with_flatten( with_flatten(
asarray(Model.ops, dtype='uint64') asarray(Model.ops, dtype='uint64')
>> uniqued(embed >> Maxout(width, width*4, pieces=3), column=5) >> uniqued(embed, column=5)
>> Residual( >> drop_layer(
Residual(
(ExtractWindow(nW=1) >> ReLu(width, width*3)) (ExtractWindow(nW=1) >> ReLu(width, width*3))
>> (ExtractWindow(nW=1) >> ReLu(width, width*3)) )
>> (ExtractWindow(nW=1) >> ReLu(width, width*3)) ) ** 4, pad=4
>> (ExtractWindow(nW=1) >> ReLu(width, width*3)) )
), pad=4)
) )
if preprocess not in (False, None): if preprocess not in (False, None):
tok2vec = preprocess >> tok2vec tok2vec = preprocess >> tok2vec