File size: 6,401 Bytes
10e9b7d
 
eccf8e4
3c4371f
10e9b7d
7cbe9e1
 
 
a5a7826
 
 
547bea9
 
4e6c049
 
547bea9
4e6c049
 
 
547bea9
4e6c049
 
 
547bea9
4e6c049
547bea9
 
 
 
 
 
 
a5a7826
4e6c049
e80aab9
3db6293
e80aab9
31243f4
 
aacb082
7cbe9e1
 
 
31243f4
7cbe9e1
 
 
 
 
 
 
547bea9
 
 
 
 
7cbe9e1
 
a5a7826
4021bf3
547bea9
9348b59
 
 
3c4371f
9348b59
7e4a06b
31243f4
 
e80aab9
31243f4
 
 
 
19dc6ec
9348b59
3c4371f
31243f4
eccf8e4
31243f4
7d65c66
31243f4
 
19dc6ec
31243f4
e80aab9
31243f4
e80aab9
7d65c66
 
3c4371f
31243f4
 
 
 
 
 
7d65c66
 
 
31243f4
19dc6ec
31243f4
 
 
 
19dc6ec
 
 
 
 
e80aab9
31243f4
e80aab9
7d65c66
e80aab9
 
31243f4
9348b59
e80aab9
3c4371f
 
 
e80aab9
31243f4
 
7d65c66
9348b59
e80aab9
9348b59
e80aab9
31243f4
0ee0419
e514fd7
 
9348b59
 
e514fd7
 
e80aab9
 
9348b59
e80aab9
31243f4
9088b99
7d65c66
e80aab9
31243f4
 
9348b59
31243f4
e80aab9
 
 
3c4371f
 
19dc6ec
7d65c66
3c4371f
 
7d65c66
3c4371f
19dc6ec
7d65c66
19dc6ec
7d65c66
 
 
19dc6ec
7d65c66
3c4371f
31243f4
19dc6ec
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
import os
import gradio as gr
import requests
import pandas as pd

from agent import build_graph
from langchain_core.messages import HumanMessage

import re

def extract_answer(text: str) -> str:
    """
    Clean and extract the final answer from agent output.
    Removes prefixes like 'FINAL ANSWER:', trims punctuation,
    and normalizes separators.
    """
    # 提取 final answer 后内容
    match = re.search(r"(final\s*answer|answer\s*is)[::]?\s*(.+)", text, re.IGNORECASE)
    answer = match.group(2) if match else text

    # 清理格式
    answer = answer.strip().lstrip(":").strip()     # ✅ 去掉前导冒号和空格
    answer = answer.rstrip('.').strip()

    # 多项格式化
    if ',' in answer:
        answer = ",".join(part.strip() for part in answer.split(','))
    if ';' in answer:
        answer = "; ".join(part.strip() for part in answer.split(';'))

    return answer



# --- Constants ---
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"

# --- Basic Agent Definition ---
class BasicAgent:
    def __init__(self, provider: str = "openai"):
        print(f"Initializing LangGraph Agent with provider: {provider}")
        self.graph = build_graph(provider=provider)

    def __call__(self, question: str) -> str:
        print(f"Running LangGraph Agent on question: {question[:50]}...")
        try:
            messages = [HumanMessage(content=question)]
            result = self.graph.invoke({"messages": messages})
            outputs = result["messages"]
            for m in reversed(outputs):
                if m.type == "ai":
                    raw_answer = m.content
                    clean = extract_answer(raw_answer)
                    print(f"Extracted clean answer: {clean}")
                    return clean
            return ""
        except Exception as e:
            print(f"LangGraph Agent error: {e}")
            return f"Error: {str(e)}"


def run_and_submit_all(username: str):
    if not username:
        return "❌ Please enter your Hugging Face username.", None

    space_id = os.getenv("SPACE_ID")
    api_url = DEFAULT_API_URL
    questions_url = f"{api_url}/questions"
    submit_url = f"{api_url}/submit"

    try:
        agent = BasicAgent()
    except Exception as e:
        return f"Error initializing agent: {e}", None

    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "N/A"

    print(f"Fetching questions from: {questions_url}")
    try:
        response = requests.get(questions_url, timeout=15)
        response.raise_for_status()
        questions_data = response.json()
        if not questions_data:
            return "Fetched questions list is empty or invalid format.", None
        print(f"Fetched {len(questions_data)} questions.")
    except requests.exceptions.RequestException as e:
        return f"Error fetching questions: {e}", None

    results_log = []
    answers_payload = []
    print(f"Running agent on {len(questions_data)} questions...")
    for item in questions_data:
        task_id = item.get("task_id")
        question_text = item.get("question")
        if not task_id or question_text is None:
            continue
        try:
            submitted_answer = agent(question_text)
            answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
        except Exception as e:
            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})

    if not answers_payload:
        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)

    submission_data = {
        "username": username.strip(),
        "agent_code": agent_code,
        "answers": answers_payload
    }

    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
    try:
        response = requests.post(submit_url, json=submission_data, timeout=60)
        response.raise_for_status()
        result_data = response.json()
        final_status = (
            f"✅ Submission Successful!\n"
            f"User: {result_data.get('username')}\n"
            f"Overall Score: {result_data.get('score', 'N/A')}% "
            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
            f"Message: {result_data.get('message', 'No message received.')}"
        )
        results_df = pd.DataFrame(results_log)
        return final_status, results_df
    except Exception as e:
        return f"❌ Submission Failed: {e}", pd.DataFrame(results_log)

# --- Gradio Interface ---
with gr.Blocks() as demo:
    gr.Markdown("# Basic Agent Evaluation Runner")
    gr.Markdown(
        """
        **Instructions:**
        1. Please enter your Hugging Face username below manually.
        2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see your score.
        ---
        """
    )

    username_box = gr.Textbox(label="Your Hugging Face Username (for submission)", placeholder="e.g. johndoe")

    run_button = gr.Button("Run Evaluation & Submit All Answers")
    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)

    run_button.click(
        fn=run_and_submit_all,
        inputs=[username_box],
        outputs=[status_output, results_table]
    )

if __name__ == "__main__":
    print("\n" + "-"*30 + " App Starting " + "-"*30)
    space_host_startup = os.getenv("SPACE_HOST")
    space_id_startup = os.getenv("SPACE_ID")

    if space_host_startup:
        print(f"✅ SPACE_HOST found: {space_host_startup}")
        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
    else:
        print("ℹ️  SPACE_HOST not found (running locally?).")

    if space_id_startup:
        print(f"✅ SPACE_ID found: {space_id_startup}")
        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
    else:
        print("ℹ️  SPACE_ID not found. Repo URL cannot be determined.")

    print("-"*(60 + len(" App Starting ")) + "\n")
    print("Launching Gradio Interface for Basic Agent Evaluation...")
    demo.launch(debug=True, share=False)