Create handler.py

handler.py

@@ -0,0 +1,59 @@
+from typing import Dict, List, Any
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+
+class EndpointHandler():
+    def __init__(self, path=""):
+        # Load model and tokenizer
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.model = AutoModelForCausalLM.from_pretrained(
+            path,
+            device_map="auto",
+            torch_dtype=torch.bfloat16,
+        )
+        
+    def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        data args:
+            inputs (:str): a string to be generated from
+            parameters (:dict): generation parameters
+        Return:
+            A :obj:`list` | `dict`: will be serialized and returned
+        """
+        
+        # Get the input text
+        inputs = data.pop("inputs", data)
+        parameters = data.pop("parameters", {})
+        
+        # Set default parameters
+        max_new_tokens = parameters.get("max_new_tokens", 400)
+        temperature = parameters.get("temperature", 0.7)
+        do_sample = parameters.get("do_sample", True)
+        top_p = parameters.get("top_p", 0.9)
+        return_full_text = parameters.get("return_full_text", True)
+        
+        # Tokenize the input
+        input_ids = self.tokenizer(inputs, return_tensors="pt").to(self.model.device)
+        
+        # Generate text
+        with torch.no_grad():
+            generated_ids = self.model.generate(
+                **input_ids,
+                max_new_tokens=max_new_tokens,
+                temperature=temperature,
+                do_sample=do_sample,
+                top_p=top_p,
+                pad_token_id=self.tokenizer.eos_token_id,
+                eos_token_id=self.tokenizer.eos_token_id,
+            )
+        
+        # Decode the generated text
+        if return_full_text:
+            generated_text = self.tokenizer.decode(generated_ids[0], skip_special_tokens=True)
+        else:
+            # Only return the newly generated part
+            new_tokens = generated_ids[0][input_ids["input_ids"].shape[1]:]
+            generated_text = self.tokenizer.decode(new_tokens, skip_special_tokens=True)
+        
+        return [{"generated_text": generated_text}]