Optimizer defaults (#5244)

* set optimizer defaults to mimic thinc 7 + bump to dev6

* larger error range for senter overfitting test

spacy/about.py

@@ -1,6 +1,6 @@
 # fmt: off
 __title__ = "spacy"
-__version__ = "3.0.0.dev4"
+__version__ = "3.0.0.dev6"
 __release__ = True
 __download_url__ = "https://github.com/explosion/spacy-models/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"

spacy/tests/pipeline/test_senter.py

@@ -33,7 +33,7 @@ def test_overfitting_IO():
     for i in range(200):
         losses = {}
         nlp.update(TRAIN_DATA, sgd=optimizer, losses=losses)
-    assert losses["senter"] < 0.0001
+    assert losses["senter"] < 0.001
 
     # test the trained model
     test_text = "I like purple eggs. They eat ham. You like yellow eggs."

spacy/util.py

@@ -790,7 +790,8 @@ def create_default_optimizer():
     beta2 = env_opt("optimizer_B2", 0.999)
     eps = env_opt("optimizer_eps", 1e-8)
     L2 = env_opt("L2_penalty", 1e-6)
-    grad_clip = env_opt("grad_norm_clip", 1.0)
+    grad_clip = env_opt("grad_norm_clip", 10.0)
+    L2_is_weight_decay = env_opt("L2_is_weight_decay", False)
     optimizer = Adam(
         learn_rate,
         L2=L2,
@@ -799,5 +800,6 @@ def create_default_optimizer():
         eps=eps,
         ops=ops,
         grad_clip=grad_clip,
+        L2_is_weight_decay=L2_is_weight_decay,
     )
     return optimizer