explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-14 21:57:15 +03:00
Code Issues Packages Projects Releases Wiki Activity

Restore LayerNorm for mish

Browse Source
This commit is contained in:
Matthw Honnibal 2019-10-24 17:35:10 +02:00
parent 134056684f
commit 7d81d17ce5
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spacy/_ml.py
View File

@ -382,13 +382,15 @@ def CNN(width, depth, pieces, nW=1):
if pieces == 1: if pieces == 1:
layer = chain( layer = chain(
ExtractWindow(nW=nW), ExtractWindow(nW=nW),
LN(Mish(width, width * (nW*2+1))) Mish(width, width*(nW*2+1)),
LN(nO=width)
) )
return clone(Residual(layer), depth)
else: else:
layer = chain( layer = chain(
ExtractWindow(nW=nW), ExtractWindow(nW=nW),
LN(Maxout(width, width * (nW*2+1), pieces=pieces))) LN(Maxout(width, width * (nW*2+1), pieces=pieces)))
return clone(Residual(layer), depth) return clone(Residual(layer), depth)
def SelfAttention(width, depth, pieces): def SelfAttention(width, depth, pieces):