Update app.py

Fixed the torch_dtype bug. Needed it initialize the dtype when loading the model rather than later.

app.py

@@ -95,7 +95,7 @@ class ModelWrapper:
 
         print(f"Loading model: {model_name}...")
         if "8b" in model_name.lower():
-            config = AutoConfig.from_pretrained(model_name)
+            config = AutoConfig.from_pretrained(model_name, torch_dtype=torch.bfloat16)
             with init_empty_weights():
                 model_empty = AutoModelForCausalLM.from_config(config)
 
@@ -104,7 +104,6 @@ class ModelWrapper:
                 model_name,
                 device_map="auto",
                 offload_folder="offload",
-                torch_dtype=torch.bfloat16
             ).eval()
         else:
             self.model = AutoModelForCausalLM.from_pretrained(