Module path importing

instructlab · Jun 20, 2024 · c27e20b · c27e20b
1 parent 8141247
commit c27e20b
Showing 1 changed file with 9 additions and 2 deletions.
diff --git a/src/instructlab/training/tokenizer_utils.py b/src/instructlab/training/tokenizer_utils.py
@@ -19,8 +19,15 @@ class SpecialTokens:
 
 
 #TODO: Replace with specified template path
-from instructlab.training.chat_templates.ibm_generic_tmpl import SPECIAL_TOKENS, CHAT_TEMPLATE
-
+#from instructlab.training.chat_templates.ibm_generic_tmpl import SPECIAL_TOKENS, CHAT_TEMPLATE
+import importlib.util
+import sys
+spec = importlib.util.spec_from_file_location("ibm_generic_tmpl", "chat_templates/ibm_generic_tmpl.py")
+module = importlib.util.module_from_spec(spec)
+sys.modules["ibm_generic_tmpl"] = module
+spec.loader.exec_module(module)
+SPECIAL_TOKENS = module.SPECIAL_TOKENS
+CHAT_TEMPLATE = module.CHAT_TEMPLATE
 
 def setup_tokenizer(
     model_name_or_path, SPECIAL_TOKENS=SPECIAL_TOKENS, CHAT_TEMPLATE=CHAT_TEMPLATE