gracefully handle length feature used for group by (axolotl-ai-cloud#565

)
mkeoliya · Sep 13, 2023 · 374a914 · 374a914
1 parent e6a5ed2
commit 374a914
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/axolotl/utils/dataloader.py b/src/axolotl/utils/dataloader.py
@@ -223,6 +223,8 @@ def __iter__(self):
                 concatenated = {}
                 batched_data = [self.dataset[batch_idx] for batch_idx in batch]
                 for feature in features:
+                    if feature == "length":
+                        continue
                     if feature == "attention_mask":
                         arrays = [
                             (attn_mask_cum_idx + idx + 1) * np.array(item[feature])