support to disable exllama for gptq (#604)

* support to disable exllama for gptq * update property instead of item * fix config key
axolotl-ai-cloud · Sep 19, 2023 · faecff9 · faecff9
1 parent aa656e0
commit faecff9
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/examples/llama-2/gptq-lora.yml b/examples/llama-2/gptq-lora.yml
@@ -2,7 +2,7 @@ base_model: TheBloke/Llama-2-7B-GPTQ
 base_model_config: TheBloke/Llama-2-7B-GPTQ
 is_llama_derived_model: false
 gptq: true
-gptq_bits: 4
+gptq_disable_exllama: true
 model_type: AutoModelForCausalLM
 tokenizer_type: LlamaTokenizer
 tokenizer_use_fast: true
@@ -62,8 +62,6 @@ xformers_attention:
 flash_attention:
 sdp_attention:
 flash_optimum:
-gptq_groupsize:
-gptq_model_v1:
 warmup_steps: 100
 eval_steps:
 save_steps:

diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py
@@ -196,6 +196,10 @@ def load_model(
         if not hasattr(model_config, "quantization_config"):
             LOG.warning("model config does not contain quantization_config information")
         else:
+            if cfg.gptq_disable_exllama is not None:
+                model_config.quantization_config[
+                    "disable_exllama"
+                ] = cfg.gptq_disable_exllama
             model_kwargs["quantization_config"] = GPTQConfig(
                 **model_config.quantization_config
             )