From 7d81d17ce541f0dfb4417ecaba028dba8fdc42f6 Mon Sep 17 00:00:00 2001 From: Matthw Honnibal Date: Thu, 24 Oct 2019 17:35:10 +0200 Subject: [PATCH] Restore LayerNorm for mish --- spacy/_ml.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spacy/_ml.py b/spacy/_ml.py index 95341e747..b5925c2b1 100644 --- a/spacy/_ml.py +++ b/spacy/_ml.py @@ -382,13 +382,15 @@ def CNN(width, depth, pieces, nW=1): if pieces == 1: layer = chain( ExtractWindow(nW=nW), - LN(Mish(width, width * (nW*2+1))) + Mish(width, width*(nW*2+1)), + LN(nO=width) ) + return clone(Residual(layer), depth) else: layer = chain( ExtractWindow(nW=nW), LN(Maxout(width, width * (nW*2+1), pieces=pieces))) - return clone(Residual(layer), depth) + return clone(Residual(layer), depth) def SelfAttention(width, depth, pieces):