go with Gemma2 softclamp values

lucidrains · Jul 6, 2024 · 80be134 · 80be134
1 parent 832464f
commit 80be134
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.31.9',
+  version = '1.31.10',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

diff --git a/x_transformers/attend.py b/x_transformers/attend.py
@@ -82,7 +82,7 @@ def __init__(
         qk_norm = False,
         flash = False,
         softclamp_logits = False,
-        logit_softclamp_value = 30.,
+        logit_softclamp_value = 50.,
         add_zero_kv = False,
         cope = None,
         onnxable = False,

diff --git a/x_transformers/x_transformers.py b/x_transformers/x_transformers.py
@@ -1272,7 +1272,7 @@ def __init__(
         shift_tokens = 0,
         sandwich_norm = False,
         softclamp_output = False,
-        softclamp_output_value = 50.,
+        softclamp_output_value = 30.,
         resi_dual = False,
         resi_dual_scale = 1.,
         zero_init_branch_output = False,