Fix

app.py

@@ -6,7 +6,6 @@ from transformers import (
     StoppingCriteria,
     StoppingCriteriaList,
     TextIteratorStreamer,
-    BitsAndBytesConfig,
 )
 import gradio as gr
 import torch
@@ -19,18 +18,10 @@ max_new_tokens = 1024
 
 print(f"Starting to load the model {model_name}")
 
-bnb_config = BitsAndBytesConfig(
-    llm_int8_enable_fp32_cpu_offload=True,
-    load_in_8bit=True,
-)
-
 m = AutoModelForCausalLM.from_pretrained(
     model_name,
-    device_map = {
-        "transformer.h": 0,
-        "": "cpu",
-    },
-    quantization_config=bnb_config,
+    device_map=0,
+    #load_in_8bit=True,
 )
 
 m.config.pad_token_id = m.config.eos_token_id