Raiff1982 commited on
Commit
5604872
·
1 Parent(s): 5d33959
Files changed (1) hide show
  1. AICoreAGIX_with_TB.py +5 -0
AICoreAGIX_with_TB.py CHANGED
@@ -109,6 +109,11 @@ class AICoreAGIX:
109
  except Exception as e2:
110
  logger.warning(f"[Tokenizer Load]: Full fallback to gpt2. Reason: {e2}")
111
  return AutoTokenizer.from_pretrained("gpt2")
 
 
 
 
 
112
 
113
  def _safe_load_model(self, model_name):
114
  try:
 
109
  except Exception as e2:
110
  logger.warning(f"[Tokenizer Load]: Full fallback to gpt2. Reason: {e2}")
111
  return AutoTokenizer.from_pretrained("gpt2")
112
+ try:
113
+ return AutoTokenizer.from_pretrained("gpt2", local_files_only=True)
114
+ except Exception as e3:
115
+ logger.error(f"[Tokenizer Load]: Even fallback to GPT2 failed: {e3}")
116
+ raise RuntimeError("Tokenizer load completely failed.")
117
 
118
  def _safe_load_model(self, model_name):
119
  try: