Revert "update the default block size (#1137)" (#1162)

This reverts commit 25d61ad.
huggingface · Jul 6, 2023 · d5b4636 · d5b4636
1 parent 2eab7ab
commit d5b4636
Showing 1 changed file with 6 additions and 4 deletions.
diff --git a/examples/onnxruntime/training/language-modeling/run_clm.py b/examples/onnxruntime/training/language-modeling/run_clm.py
@@ -435,10 +435,12 @@ def tokenize_function(examples):
 
     if data_args.block_size is None:
         block_size = tokenizer.model_max_length
-        logger.warning(
-            f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). "
-            "You can change that default value by passing --block_size xxx."
-        )
+        if block_size > 1024:
+            logger.warning(
+                f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). "
+                "Picking 1024 instead. You can change that default value by passing --block_size xxx."
+            )
+            block_size = 1024
     else:
         if data_args.block_size > tokenizer.model_max_length:
             logger.warning(