From b2e93be867be16acee8ccc6f95e4fb1ebf7d86cf Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Fri, 3 Apr 2020 13:02:46 +0200 Subject: [PATCH] Optimizer defaults (#5244) * set optimizer defaults to mimic thinc 7 + bump to dev6 * larger error range for senter overfitting test --- spacy/about.py | 2 +- spacy/tests/pipeline/test_senter.py | 2 +- spacy/util.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/spacy/about.py b/spacy/about.py index 0c0a2d002..6fa1f4c0b 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -1,6 +1,6 @@ # fmt: off __title__ = "spacy" -__version__ = "3.0.0.dev4" +__version__ = "3.0.0.dev6" __release__ = True __download_url__ = "https://github.com/explosion/spacy-models/releases/download" __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json" diff --git a/spacy/tests/pipeline/test_senter.py b/spacy/tests/pipeline/test_senter.py index 411768e5f..197fdca6e 100644 --- a/spacy/tests/pipeline/test_senter.py +++ b/spacy/tests/pipeline/test_senter.py @@ -33,7 +33,7 @@ def test_overfitting_IO(): for i in range(200): losses = {} nlp.update(TRAIN_DATA, sgd=optimizer, losses=losses) - assert losses["senter"] < 0.0001 + assert losses["senter"] < 0.001 # test the trained model test_text = "I like purple eggs. They eat ham. You like yellow eggs." diff --git a/spacy/util.py b/spacy/util.py index 37649c5e6..ef9082140 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -790,7 +790,8 @@ def create_default_optimizer(): beta2 = env_opt("optimizer_B2", 0.999) eps = env_opt("optimizer_eps", 1e-8) L2 = env_opt("L2_penalty", 1e-6) - grad_clip = env_opt("grad_norm_clip", 1.0) + grad_clip = env_opt("grad_norm_clip", 10.0) + L2_is_weight_decay = env_opt("L2_is_weight_decay", False) optimizer = Adam( learn_rate, L2=L2, @@ -799,5 +800,6 @@ def create_default_optimizer(): eps=eps, ops=ops, grad_clip=grad_clip, + L2_is_weight_decay=L2_is_weight_decay, ) return optimizer