explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-14 13:47:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rescale gradients for mlm

Browse Source
This commit is contained in:
Matthw Honnibal 2019-10-24 17:35:37 +02:00
parent 7d81d17ce5
commit 73b1f651d4
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/_ml.py
View File

@ -966,6 +966,8 @@ def masked_language_model(vocab, model, mask_prob=0.15):
def mlm_backward(d_output, sgd=None):
d_output *= 1 - mask
# Rescale gradient for number of instances.
d_output *= mask.size - mask.sum()
return backprop(d_output, sgd=sgd)
return output, mlm_backward