diff --git a/spacy/_ml.py b/spacy/_ml.py
index 9147fa450..3cfc6c2ca 100644
--- a/spacy/_ml.py
+++ b/spacy/_ml.py
@@ -43,8 +43,8 @@ def cosine(vec1, vec2):
 def create_default_optimizer(ops, **cfg):
     learn_rate = util.env_opt('learn_rate', 0.001)
     beta1 = util.env_opt('optimizer_B1', 0.9)
-    beta2 = util.env_opt('optimizer_B2', 0.999)
-    eps = util.env_opt('optimizer_eps', 1e-08)
+    beta2 = util.env_opt('optimizer_B2', 0.9)
+    eps = util.env_opt('optimizer_eps', 1e-12)
     L2 = util.env_opt('L2_penalty', 1e-6)
     max_grad_norm = util.env_opt('grad_norm_clip', 1.)
     optimizer = Adam(ops, learn_rate, L2=L2, beta1=beta1,