Spaces:
Running
Running
File size: 2,249 Bytes
fb856e1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
from pydantic import BaseModel
from openai import OpenAI
import os
from dotenv import load_dotenv
MODEL = "gemini-2.0-flash"
class Evaluation(BaseModel):
is_acceptable: bool
feedback: str
class Evaluator:
def __init__(self, name="", model=MODEL):
load_dotenv(override=True)
google_api_key = os.getenv('GOOGLE_API_KEY')
self.name=name
self.model=model
self._gemini = OpenAI(api_key=google_api_key, base_url="https://generativelanguage.googleapis.com/v1beta/openai/")
def _evaluator_system_prompt(self):
return f"You are an evaluator that decides whether a response to a question is acceptable. \
You are provided with a conversation between a User and an Agent. Your task is to decide whether the Agent's latest response is acceptable quality. \
The Agent is playing the role of {self.name} and is representing {self.name} on their website. \
The Agent has been instructed to be professional and engaging, as if talking to a potential client or future employer who came across the website. \
The Agent has been provided with context on {self.name} in the form of their summary, experience and CV. \
With this context, please evaluate the latest response, replying with whether the response is acceptable and your feedback."
def _evaluator_user_prompt(self, reply, message, history):
user_prompt = f"Here's the conversation between the User and the Agent: \n\n{history}\n\n"
user_prompt += f"Here's the latest message from the User: \n\n{message}\n\n"
user_prompt += f"Here's the latest response from the Agent: \n\n{reply}\n\n"
user_prompt += "Please evaluate the response, replying with whether it is acceptable and your feedback."
return user_prompt
def evaluate(self, reply, message, history) -> Evaluation:
messages = [{"role": "system", "content": self._evaluator_system_prompt()}] + [{"role": "user", "content": self._evaluator_user_prompt(reply, message, history)}]
response = self._gemini.beta.chat.completions.parse(model=self.model, messages=messages, response_format=Evaluation)
return response.choices[0].message.parsed
|