handle key padding mask directly passed into Attend

lucidrains · Jul 6, 2024 · 832464f · 832464f
1 parent 65e9d36
commit 832464f
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.31.8',
+  version = '1.31.9',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

diff --git a/x_transformers/attend.py b/x_transformers/attend.py
@@ -268,6 +268,11 @@ def forward(
 
         causal = self.causal
 
+        # handle key padding mask
+
+        if exists(mask) and mask.ndim == 2:
+            mask = rearrange(mask, 'b j -> b 1 1 j')
+
         # handle kv cached decoding
 
         if n == 1 and causal: