Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -1,845 +1,875 @@
|
|
1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2 |
import json
|
3 |
-
import
|
4 |
-
import
|
5 |
-
import operator
|
6 |
-
from typing import Dict, List, Any, Optional
|
7 |
import time
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24 |
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
|
|
|
|
|
|
47 |
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
|
72 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
73 |
}
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
|
82 |
-
if
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
'file_name': question_data.get('file_name')
|
90 |
-
}
|
91 |
-
})
|
92 |
-
|
93 |
-
# Priority 2: Web search for factual information
|
94 |
-
if analysis['needs_web_search'] or analysis['is_factual_question']:
|
95 |
-
plan.append({
|
96 |
-
'action': 'web_search',
|
97 |
-
'tool': 'web_search',
|
98 |
-
'priority': 2,
|
99 |
-
'params': {
|
100 |
-
'query': self._extract_search_query(analysis['question_text'])
|
101 |
-
}
|
102 |
-
})
|
103 |
-
|
104 |
-
# Priority 3: Calculations
|
105 |
-
if analysis['needs_calculation']:
|
106 |
-
plan.append({
|
107 |
-
'action': 'calculate',
|
108 |
-
'tool': 'calculator',
|
109 |
-
'priority': 3,
|
110 |
-
'params': {}
|
111 |
-
})
|
112 |
-
|
113 |
-
# Priority 4: Text analysis
|
114 |
-
plan.append({
|
115 |
-
'action': 'analyze_text',
|
116 |
-
'tool': 'text_analyzer',
|
117 |
-
'priority': 4,
|
118 |
-
'params': {
|
119 |
-
'text': analysis['question_text']
|
120 |
-
}
|
121 |
-
})
|
122 |
-
|
123 |
-
return sorted(plan, key=lambda x: x['priority'])
|
124 |
-
|
125 |
-
def _execute_plan(self, plan: List[Dict], question_data: Dict) -> Dict:
|
126 |
-
"""Execute the planned steps"""
|
127 |
-
results = {}
|
128 |
-
|
129 |
-
for step in plan:
|
130 |
-
tool_name = step['tool']
|
131 |
-
action = step['action']
|
132 |
|
133 |
-
|
134 |
-
|
|
|
|
|
135 |
|
136 |
-
|
137 |
-
|
138 |
-
|
139 |
-
|
140 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
141 |
|
142 |
-
|
143 |
-
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
# Extract numbers and operations from question and file data
|
149 |
-
calculation_input = self._prepare_calculation_input(
|
150 |
-
question_data, results
|
151 |
-
)
|
152 |
-
if calculation_input:
|
153 |
-
results['calculation'] = self.tools[tool_name].calculate(
|
154 |
-
calculation_input
|
155 |
-
)
|
156 |
-
|
157 |
-
elif action == 'analyze_text':
|
158 |
-
results['text_analysis'] = self.tools[tool_name].analyze(
|
159 |
-
step['params']['text'],
|
160 |
-
context=results
|
161 |
-
)
|
162 |
-
|
163 |
-
except Exception as e:
|
164 |
-
print(f"Error in {action}: {e}")
|
165 |
-
results[f'{action}_error'] = str(e)
|
166 |
-
|
167 |
-
return results
|
168 |
-
|
169 |
-
def _extract_search_query(self, question: str) -> str:
|
170 |
-
"""Extract relevant search query from question"""
|
171 |
-
# Remove question words and extract key terms
|
172 |
-
question_words = ['what', 'who', 'when', 'where', 'how', 'why', 'is', 'are', 'was', 'were']
|
173 |
-
words = question.lower().split()
|
174 |
-
|
175 |
-
# Keep important words, remove common question words
|
176 |
-
filtered_words = [word for word in words if word not in question_words and len(word) > 2]
|
177 |
-
|
178 |
-
return ' '.join(filtered_words[:6]) # Limit to 6 words
|
179 |
-
|
180 |
-
def _prepare_calculation_input(self, question_data: Dict, results: Dict) -> Optional[str]:
|
181 |
-
"""Prepare input for calculator based on question and available data"""
|
182 |
-
question = question_data.get('question', '')
|
183 |
-
|
184 |
-
# Extract numbers from question
|
185 |
-
numbers = re.findall(r'\d+\.?\d*', question)
|
186 |
-
|
187 |
-
# Look for mathematical operations
|
188 |
-
if 'sum' in question.lower() or 'total' in question.lower():
|
189 |
-
if numbers:
|
190 |
-
return '+'.join(numbers)
|
191 |
-
elif 'multiply' in question.lower() or 'product' in question.lower():
|
192 |
-
if numbers:
|
193 |
-
return '*'.join(numbers)
|
194 |
-
elif 'average' in question.lower():
|
195 |
-
if numbers:
|
196 |
-
return f"({'+'.join(numbers)})/{len(numbers)}"
|
197 |
-
|
198 |
-
# Check if file data contains numbers for calculation
|
199 |
-
if 'file_data' in results and isinstance(results['file_data'], dict):
|
200 |
-
file_numbers = results['file_data'].get('numbers', [])
|
201 |
-
if file_numbers and ('sum' in question.lower() or 'total' in question.lower()):
|
202 |
-
return '+'.join(map(str, file_numbers))
|
203 |
-
|
204 |
-
return None
|
205 |
-
|
206 |
-
def _generate_final_answer(self, results: Dict, question_data: Dict) -> str:
|
207 |
-
"""Generate final answer based on execution results"""
|
208 |
-
question = question_data.get('question', '').lower()
|
209 |
-
|
210 |
-
# Priority order for answer selection
|
211 |
-
if 'calculation' in results and results['calculation'] is not None:
|
212 |
-
return str(results['calculation'])
|
213 |
-
|
214 |
-
if 'file_data' in results and isinstance(results['file_data'], dict):
|
215 |
-
# Look for specific answer in file data
|
216 |
-
if 'answer' in results['file_data']:
|
217 |
-
return str(results['file_data']['answer'])
|
218 |
-
elif 'summary' in results['file_data']:
|
219 |
-
return str(results['file_data']['summary'])
|
220 |
-
|
221 |
-
if 'search_data' in results and results['search_data']:
|
222 |
-
# Extract answer from search results
|
223 |
-
for result in results['search_data']:
|
224 |
-
if isinstance(result, dict) and 'summary' in result:
|
225 |
-
return result['summary']
|
226 |
-
|
227 |
-
if 'text_analysis' in results:
|
228 |
-
return str(results['text_analysis'])
|
229 |
-
|
230 |
-
return "Unable to determine answer"
|
231 |
|
232 |
-
def
|
233 |
-
"""
|
234 |
-
if
|
235 |
-
return "
|
236 |
|
237 |
-
# Convert to string and strip whitespace
|
238 |
-
answer = str(answer).strip()
|
239 |
-
|
240 |
-
# Remove common prefixes that might cause exact match failures
|
241 |
-
prefixes_to_remove = [
|
242 |
-
'the answer is: ',
|
243 |
-
'answer: ',
|
244 |
-
'final answer: ',
|
245 |
-
'result: ',
|
246 |
-
'solution: '
|
247 |
-
]
|
248 |
-
|
249 |
-
answer_lower = answer.lower()
|
250 |
-
for prefix in prefixes_to_remove:
|
251 |
-
if answer_lower.startswith(prefix):
|
252 |
-
answer = answer[len(prefix):].strip()
|
253 |
-
break
|
254 |
-
|
255 |
-
# Handle numeric answers
|
256 |
-
if self._is_numeric_answer(answer):
|
257 |
-
return self._format_numeric_answer(answer)
|
258 |
-
|
259 |
-
# Handle yes/no answers
|
260 |
-
if answer.lower() in ['yes', 'no', 'true', 'false']:
|
261 |
-
return answer.lower()
|
262 |
-
|
263 |
-
# Return cleaned text answer
|
264 |
-
return answer
|
265 |
-
|
266 |
-
def _is_numeric_answer(self, answer: str) -> bool:
|
267 |
-
"""Check if answer is numeric"""
|
268 |
try:
|
269 |
-
|
270 |
-
|
271 |
-
|
272 |
-
|
273 |
-
|
274 |
-
|
275 |
-
|
276 |
-
|
277 |
-
|
278 |
-
|
279 |
-
|
280 |
-
|
281 |
-
|
282 |
-
|
283 |
-
|
284 |
-
return
|
285 |
-
|
286 |
-
|
287 |
-
|
288 |
-
"""Simple web search tool (implement with your preferred search API)"""
|
289 |
-
|
290 |
-
def search(self, query: str, max_results: int = 3) -> List[Dict]:
|
291 |
-
"""Perform web search - implement with your preferred search service"""
|
292 |
-
print(f"Web search: {query}")
|
293 |
-
|
294 |
-
# Placeholder implementation
|
295 |
-
# Replace with actual search API (DuckDuckGo, Google Custom Search, etc.)
|
296 |
-
return [
|
297 |
-
{
|
298 |
-
'title': f'Search result for: {query}',
|
299 |
-
'summary': f'Information about {query}',
|
300 |
-
'url': 'https://example.com'
|
301 |
-
}
|
302 |
-
]
|
303 |
|
|
|
|
|
|
|
304 |
|
305 |
-
class
|
306 |
-
"""
|
307 |
|
308 |
-
|
309 |
-
|
|
|
310 |
try:
|
311 |
-
|
312 |
-
expression = expression.replace(' ', '')
|
313 |
|
314 |
-
|
315 |
-
|
316 |
-
if not all(c in allowed_chars for c in expression):
|
317 |
-
raise ValueError("Invalid characters in expression")
|
318 |
|
319 |
-
|
320 |
-
|
321 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
322 |
|
323 |
-
|
|
|
324 |
|
325 |
except Exception as e:
|
326 |
-
|
327 |
-
return
|
328 |
|
329 |
-
|
330 |
-
|
331 |
-
|
332 |
-
|
333 |
-
|
334 |
-
|
335 |
-
|
336 |
-
|
337 |
-
|
338 |
-
|
339 |
-
|
340 |
-
|
341 |
-
|
342 |
-
|
343 |
-
|
344 |
-
|
345 |
-
|
346 |
-
|
347 |
-
|
348 |
-
|
349 |
-
|
350 |
-
|
351 |
-
|
352 |
-
|
353 |
-
|
354 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
355 |
|
356 |
-
|
357 |
|
|
|
|
|
|
|
358 |
|
359 |
-
class
|
360 |
-
"""
|
361 |
-
|
362 |
-
def __init__(self, api_base_url: str):
|
363 |
-
self.api_base_url = api_base_url
|
364 |
|
365 |
-
def
|
366 |
-
|
367 |
-
|
368 |
-
|
369 |
-
|
370 |
-
|
371 |
-
|
372 |
-
|
373 |
-
return self._process_csv(file_content)
|
374 |
-
elif file_name.endswith('.txt'):
|
375 |
-
return self._process_text(file_content)
|
376 |
-
elif file_name.endswith('.json'):
|
377 |
-
return self._process_json(file_content)
|
378 |
-
else:
|
379 |
-
return self._process_generic(file_content)
|
380 |
-
|
381 |
-
except Exception as e:
|
382 |
-
print(f"File processing error: {e}")
|
383 |
-
return {'error': str(e)}
|
384 |
-
|
385 |
-
def _download_file(self, task_id: str) -> bytes:
|
386 |
-
"""Download file from API"""
|
387 |
-
response = requests.get(f"{self.api_base_url}/files/{task_id}")
|
388 |
-
response.raise_for_status()
|
389 |
-
return response.content
|
390 |
-
|
391 |
-
def _process_csv(self, content: bytes) -> Dict:
|
392 |
-
"""Process CSV file"""
|
393 |
try:
|
394 |
-
|
395 |
-
|
396 |
-
|
397 |
-
# Convert bytes to string
|
398 |
-
text_content = content.decode('utf-8')
|
399 |
-
|
400 |
-
# Parse CSV
|
401 |
-
reader = csv.reader(io.StringIO(text_content))
|
402 |
-
rows = list(reader)
|
403 |
-
|
404 |
-
if not rows:
|
405 |
-
return {'error': 'Empty CSV file'}
|
406 |
-
|
407 |
-
headers = rows[0] if rows else []
|
408 |
-
data_rows = rows[1:] if len(rows) > 1 else []
|
409 |
-
|
410 |
-
# Extract numbers for potential calculations
|
411 |
-
numbers = []
|
412 |
-
for row in data_rows:
|
413 |
-
for cell in row:
|
414 |
-
try:
|
415 |
-
numbers.append(float(cell))
|
416 |
-
except ValueError:
|
417 |
-
continue
|
418 |
-
|
419 |
-
return {
|
420 |
-
'type': 'csv',
|
421 |
-
'headers': headers,
|
422 |
-
'rows': data_rows,
|
423 |
-
'row_count': len(data_rows),
|
424 |
-
'numbers': numbers,
|
425 |
-
'summary': f'CSV with {len(headers)} columns and {len(data_rows)} rows'
|
426 |
-
}
|
427 |
|
428 |
-
|
429 |
-
|
430 |
-
|
431 |
-
def _process_text(self, content: bytes) -> Dict:
|
432 |
-
"""Process text file"""
|
433 |
-
try:
|
434 |
-
text = content.decode('utf-8')
|
435 |
-
|
436 |
-
# Extract numbers from text
|
437 |
-
numbers = [float(match) for match in re.findall(r'\d+\.?\d*', text)]
|
438 |
-
|
439 |
-
# Basic text analysis
|
440 |
-
lines = text.split('\n')
|
441 |
-
words = text.split()
|
442 |
-
|
443 |
-
return {
|
444 |
-
'type': 'text',
|
445 |
-
'content': text,
|
446 |
-
'line_count': len(lines),
|
447 |
-
'word_count': len(words),
|
448 |
-
'numbers': numbers,
|
449 |
-
'summary': f'Text file with {len(lines)} lines and {len(words)} words'
|
450 |
-
}
|
451 |
|
452 |
-
|
453 |
-
return {'error': f'Text processing failed: {e}'}
|
454 |
-
|
455 |
-
def _process_json(self, content: bytes) -> Dict:
|
456 |
-
"""Process JSON file"""
|
457 |
-
try:
|
458 |
-
data = json.loads(content.decode('utf-8'))
|
459 |
|
460 |
-
#
|
461 |
-
|
462 |
|
463 |
-
return
|
464 |
-
'type': 'json',
|
465 |
-
'data': data,
|
466 |
-
'numbers': numbers,
|
467 |
-
'summary': f'JSON file with {len(data) if isinstance(data, (list, dict)) else 1} items'
|
468 |
-
}
|
469 |
|
470 |
except Exception as e:
|
471 |
-
return
|
472 |
|
473 |
-
def
|
474 |
-
"""Process
|
475 |
-
|
476 |
-
|
477 |
-
try:
|
478 |
-
text = content.decode('utf-8')
|
479 |
-
return self._process_text(content)
|
480 |
-
except UnicodeDecodeError:
|
481 |
-
# Binary file
|
482 |
-
return {
|
483 |
-
'type': 'binary',
|
484 |
-
'size': len(content),
|
485 |
-
'summary': f'Binary file of {len(content)} bytes'
|
486 |
-
}
|
487 |
-
|
488 |
-
except Exception as e:
|
489 |
-
return {'error': f'Generic processing failed: {e}'}
|
490 |
-
|
491 |
-
def _extract_numbers_from_json(self, data, numbers=None):
|
492 |
-
"""Recursively extract numbers from JSON structure"""
|
493 |
-
if numbers is None:
|
494 |
-
numbers = []
|
495 |
|
496 |
-
|
497 |
-
numbers.append(float(data))
|
498 |
-
elif isinstance(data, dict):
|
499 |
-
for value in data.values():
|
500 |
-
self._extract_numbers_from_json(value, numbers)
|
501 |
-
elif isinstance(data, list):
|
502 |
-
for item in data:
|
503 |
-
self._extract_numbers_from_json(item, numbers)
|
504 |
|
505 |
-
return numbers
|
506 |
-
|
507 |
-
|
508 |
-
class TextAnalyzerTool:
|
509 |
-
"""Tool for analyzing and extracting information from text"""
|
510 |
-
|
511 |
-
def analyze(self, text: str, context: Dict = None) -> str:
|
512 |
-
"""Analyze text and extract relevant information"""
|
513 |
try:
|
514 |
-
|
515 |
-
|
|
|
|
|
|
|
|
|
516 |
|
517 |
-
#
|
518 |
-
|
519 |
-
return self._analyze_question_pattern(text, context)
|
520 |
|
521 |
-
|
522 |
-
if any(word in text.lower() for word in ['calculate', 'sum', 'total', 'average']):
|
523 |
-
return self._analyze_calculation_pattern(text, context)
|
524 |
|
525 |
-
#
|
526 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
527 |
|
528 |
except Exception as e:
|
529 |
-
|
530 |
-
|
531 |
-
|
532 |
-
"""Extract important keywords from text"""
|
533 |
-
# Simple keyword extraction
|
534 |
-
words = re.findall(r'\b[A-Za-z]{3,}\b', text.lower())
|
535 |
-
|
536 |
-
# Remove common stop words
|
537 |
-
stop_words = {'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had', 'her', 'was', 'one', 'our', 'out', 'day', 'get', 'has', 'him', 'his', 'how', 'man', 'new', 'now', 'old', 'see', 'two', 'way', 'who', 'boy', 'did', 'its', 'let', 'put', 'say', 'she', 'too', 'use'}
|
538 |
-
|
539 |
-
keywords = [word for word in words if word not in stop_words]
|
540 |
-
|
541 |
-
# Return most frequent keywords
|
542 |
-
from collections import Counter
|
543 |
-
return [word for word, count in Counter(keywords).most_common(10)]
|
544 |
-
|
545 |
-
def _analyze_question_pattern(self, text: str, context: Dict) -> str:
|
546 |
-
"""Analyze question patterns to extract answers"""
|
547 |
-
# This is where you'd implement more sophisticated NLP
|
548 |
-
# For now, return a simple analysis
|
549 |
-
|
550 |
-
if context and 'search_data' in context:
|
551 |
-
search_results = context['search_data']
|
552 |
-
if search_results and isinstance(search_results, list) and len(search_results) > 0:
|
553 |
-
return search_results[0].get('summary', 'No summary available')
|
554 |
-
|
555 |
-
return "Unable to extract specific answer from question pattern"
|
556 |
-
|
557 |
-
def _analyze_calculation_pattern(self, text: str, context: Dict) -> str:
|
558 |
-
"""Analyze calculation patterns"""
|
559 |
-
if context and 'calculation' in context:
|
560 |
-
return str(context['calculation'])
|
561 |
-
|
562 |
-
# Extract numbers for potential calculation
|
563 |
-
numbers = re.findall(r'\d+\.?\d*', text)
|
564 |
-
if numbers:
|
565 |
-
return f"Found numbers: {', '.join(numbers)}"
|
566 |
-
|
567 |
-
return "No calculation pattern found"
|
568 |
-
|
569 |
-
|
570 |
-
# Main execution functions
|
571 |
-
def test_agent_on_random_question(api_base_url: str):
|
572 |
-
"""Test the agent on a random question"""
|
573 |
-
agent = GAIAAgent(api_base_url)
|
574 |
|
575 |
-
|
576 |
-
|
577 |
-
|
578 |
-
|
579 |
|
580 |
-
|
581 |
-
|
582 |
-
print("=" * 50)
|
583 |
-
print(f"Task ID: {question.get('task_id')}")
|
584 |
-
print(f"Question: {question.get('question')}")
|
585 |
-
print(f"File: {question.get('file_name', 'None')}")
|
586 |
-
print("-" * 50)
|
587 |
|
588 |
-
|
589 |
-
start_time = time.time()
|
590 |
-
answer = agent.solve_question(question)
|
591 |
-
end_time = time.time()
|
592 |
-
|
593 |
-
print(f"Agent Answer: {answer}")
|
594 |
-
print(f"Processing Time: {end_time - start_time:.2f} seconds")
|
595 |
-
print("=" * 50)
|
596 |
-
|
597 |
-
return {
|
598 |
-
'task_id': question.get('task_id'),
|
599 |
-
'question': question.get('question'),
|
600 |
-
'agent_answer': answer,
|
601 |
-
'processing_time': end_time - start_time
|
602 |
-
}
|
603 |
-
|
604 |
-
except Exception as e:
|
605 |
-
print(f"Error testing random question: {e}")
|
606 |
-
return None
|
607 |
-
|
608 |
-
|
609 |
-
def run_full_evaluation(api_base_url: str, username: str, agent_code_url: str):
|
610 |
-
"""Run the complete evaluation on all 20 questions"""
|
611 |
-
agent = GAIAAgent(api_base_url)
|
612 |
-
|
613 |
-
try:
|
614 |
-
# Get all questions
|
615 |
-
response = requests.get(f"{api_base_url}/questions")
|
616 |
-
questions = response.json()
|
617 |
-
|
618 |
-
print(f"Starting evaluation on {len(questions)} questions...")
|
619 |
-
|
620 |
-
answers = []
|
621 |
-
successful_answers = 0
|
622 |
|
623 |
for i, question in enumerate(questions):
|
624 |
-
print(f"\n{'='*60}")
|
625 |
-
print(f"PROCESSING QUESTION {i+1}/{len(questions)}")
|
626 |
-
print(f"{'='*60}")
|
627 |
-
print(f"Task ID: {question.get('task_id')}")
|
628 |
-
print(f"Question: {question.get('question')[:100]}...")
|
629 |
-
|
630 |
try:
|
|
|
|
|
|
|
631 |
start_time = time.time()
|
632 |
-
answer = agent.solve_question(question)
|
633 |
-
end_time = time.time()
|
634 |
|
635 |
-
|
636 |
-
|
637 |
-
|
638 |
-
|
|
|
|
|
639 |
|
640 |
-
|
641 |
-
print(f"Time: {end_time - start_time:.2f}s")
|
642 |
|
643 |
-
|
644 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
645 |
|
646 |
except Exception as e:
|
647 |
-
|
648 |
-
|
649 |
-
|
650 |
-
|
651 |
-
|
652 |
-
|
653 |
-
|
654 |
-
|
655 |
-
|
656 |
-
|
657 |
-
|
658 |
-
|
659 |
-
|
660 |
-
|
661 |
-
|
662 |
-
|
663 |
-
|
664 |
-
|
665 |
-
|
666 |
-
|
667 |
-
|
668 |
-
|
669 |
-
|
670 |
-
|
671 |
-
|
672 |
-
|
673 |
-
|
674 |
-
|
675 |
-
|
676 |
-
|
677 |
-
|
678 |
-
|
679 |
-
'username': username,
|
680 |
-
'agent_code': agent_code_url,
|
681 |
-
'answers': answers
|
682 |
-
}
|
683 |
-
|
684 |
-
response = requests.post(f"{api_base_url}/submit", json=submission_data)
|
685 |
-
|
686 |
-
if response.status_code == 200:
|
687 |
-
result = response.json()
|
688 |
-
print(f"✅ Submission successful!")
|
689 |
-
print(f"Score: {result.get('score', 'N/A')}%")
|
690 |
-
print(f"Rank: {result.get('rank', 'N/A')}")
|
691 |
-
return result
|
692 |
-
else:
|
693 |
-
print(f"❌ Submission failed: {response.status_code}")
|
694 |
-
print(f"Response: {response.text}")
|
695 |
-
return None
|
696 |
-
|
697 |
-
except Exception as e:
|
698 |
-
print(f"Error submitting results: {e}")
|
699 |
-
return None
|
700 |
|
|
|
|
|
|
|
|
|
|
|
701 |
|
702 |
-
|
703 |
-
|
704 |
-
|
705 |
-
|
706 |
-
|
707 |
-
|
708 |
-
|
709 |
-
|
710 |
-
|
711 |
-
|
712 |
-
# Test on a few random questions first
|
713 |
-
print("1. Testing on random questions...")
|
714 |
-
for i in range(3):
|
715 |
-
print(f"\n--- Random Test {i+1} ---")
|
716 |
-
test_result = test_agent_on_random_question(API_BASE_URL)
|
717 |
-
if test_result:
|
718 |
-
print(f"✅ Test {i+1} completed")
|
719 |
-
else:
|
720 |
-
print(f"❌ Test {i+1} failed")
|
721 |
-
|
722 |
-
# Ask user if they want to run full evaluation
|
723 |
-
user_input = input("\nRun full evaluation on all 20 questions? (y/n): ")
|
724 |
|
725 |
-
|
726 |
-
|
727 |
-
|
728 |
-
print("=" * 60)
|
729 |
|
730 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
731 |
|
732 |
-
|
733 |
-
|
734 |
-
|
735 |
-
|
736 |
-
|
737 |
-
|
738 |
-
|
739 |
-
|
740 |
-
print(f"📈 Keep improving! You need 30% to earn the certificate.")
|
741 |
-
else:
|
742 |
-
print(f"❌ Evaluation failed. Please check your implementation.")
|
743 |
-
|
744 |
-
else:
|
745 |
-
print("Evaluation cancelled. Use the test functions to debug your agent first.")
|
746 |
|
|
|
|
|
|
|
747 |
|
748 |
-
|
|
|
|
|
749 |
|
750 |
-
def
|
751 |
-
"""
|
752 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
753 |
|
754 |
-
|
755 |
-
# Get specific question (you'd need to implement this endpoint or find the question in the list)
|
756 |
-
response = requests.get(f"{api_base_url}/questions")
|
757 |
-
questions = response.json()
|
758 |
-
question = next((q for q in questions if q.get('task_id') == task_id), None)
|
759 |
-
else:
|
760 |
-
# Get random question
|
761 |
-
response = requests.get(f"{api_base_url}/random-question")
|
762 |
-
question = response.json()
|
763 |
|
764 |
-
|
765 |
-
|
766 |
-
|
|
|
|
|
|
|
767 |
|
768 |
-
|
769 |
-
|
770 |
-
|
771 |
-
print(f"Question: {question.get('question')}")
|
772 |
-
print(f"File: {question.get('file_name', 'None')}")
|
773 |
-
print("-" * 40)
|
774 |
|
775 |
-
|
776 |
-
analysis = agent._analyze_question(question)
|
777 |
-
print("Analysis Results:")
|
778 |
-
for key, value in analysis.items():
|
779 |
-
print(f" {key}: {value}")
|
780 |
|
781 |
-
#
|
782 |
-
|
783 |
-
print(f"\nExecution Plan:")
|
784 |
-
for i, step in enumerate(plan):
|
785 |
-
print(f" {i+1}. {step['action']} (priority: {step['priority']})")
|
786 |
|
787 |
-
return
|
788 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
789 |
|
790 |
-
|
791 |
-
|
792 |
-
|
793 |
-
|
794 |
-
|
795 |
-
|
796 |
-
|
797 |
-
|
798 |
-
|
799 |
-
|
800 |
-
|
801 |
-
|
802 |
-
|
803 |
-
|
804 |
-
|
805 |
-
|
806 |
-
|
807 |
-
|
808 |
-
|
809 |
-
|
810 |
-
|
811 |
-
|
812 |
-
|
813 |
-
|
814 |
-
|
815 |
-
|
816 |
-
|
817 |
-
|
818 |
-
|
819 |
-
|
820 |
-
|
821 |
-
|
822 |
-
|
823 |
-
|
824 |
-
|
825 |
-
|
826 |
-
|
827 |
-
|
828 |
-
|
829 |
-
|
830 |
-
|
831 |
-
|
832 |
-
|
833 |
-
|
834 |
-
|
835 |
-
|
836 |
-
|
837 |
-
|
838 |
-
|
839 |
-
|
840 |
-
|
841 |
-
|
842 |
-
|
843 |
-
|
844 |
-
|
845 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
GAIA Benchmark AI Agent - Hugging Face Space
|
4 |
+
============================================
|
5 |
+
|
6 |
+
A Gradio-based web interface for running GAIA benchmark evaluations
|
7 |
+
on Hugging Face Spaces with GPU acceleration.
|
8 |
+
"""
|
9 |
+
|
10 |
+
import gradio as gr
|
11 |
+
import torch
|
12 |
import json
|
13 |
+
import os
|
14 |
+
import logging
|
|
|
|
|
15 |
import time
|
16 |
+
import re
|
17 |
+
from datetime import datetime
|
18 |
+
from typing import Dict, List, Optional, Tuple, Any
|
19 |
+
from dataclasses import dataclass
|
20 |
+
import pandas as pd
|
21 |
+
from pathlib import Path
|
22 |
|
23 |
+
# Core ML libraries
|
24 |
+
from transformers import (
|
25 |
+
AutoTokenizer,
|
26 |
+
AutoModelForCausalLM,
|
27 |
+
BitsAndBytesConfig,
|
28 |
+
pipeline
|
29 |
+
)
|
30 |
+
from datasets import load_dataset
|
31 |
+
from huggingface_hub import HfApi, hf_hub_download
|
32 |
+
|
33 |
+
# Setup logging
|
34 |
+
logging.basicConfig(level=logging.INFO)
|
35 |
+
logger = logging.getLogger(__name__)
|
36 |
+
|
37 |
+
# ================================
|
38 |
+
# CORE DATA STRUCTURES
|
39 |
+
# ================================
|
40 |
+
|
41 |
+
@dataclass
|
42 |
+
class GAIAQuestion:
|
43 |
+
"""Structure for GAIA benchmark questions"""
|
44 |
+
task_id: str
|
45 |
+
question: str
|
46 |
+
level: int
|
47 |
+
final_answer: Optional[str] = None
|
48 |
+
file_name: Optional[str] = None
|
49 |
+
annotator_metadata: Optional[Dict] = None
|
50 |
+
|
51 |
+
@classmethod
|
52 |
+
def from_dict(cls, data: dict):
|
53 |
+
return cls(**{k: v for k, v in data.items() if k in cls.__annotations__})
|
54 |
+
|
55 |
+
@dataclass
|
56 |
+
class GAIAResponse:
|
57 |
+
"""Structure for GAIA responses"""
|
58 |
+
task_id: str
|
59 |
+
model_answer: str
|
60 |
+
reasoning_trace: str
|
61 |
+
final_answer: str
|
62 |
+
processing_time: float = 0.0
|
63 |
+
confidence_score: float = 0.0
|
64 |
+
|
65 |
+
# ================================
|
66 |
+
# GAIA PROMPT MANAGEMENT
|
67 |
+
# ================================
|
68 |
+
|
69 |
+
class GAIAPromptManager:
|
70 |
+
"""Manages GAIA-specific prompting and formatting"""
|
71 |
|
72 |
+
GAIA_SYSTEM_PROMPT = """You are a general AI assistant. I will ask you a question. Report your thoughts, and finish your answer with the following template:
|
73 |
+
|
74 |
+
FINAL ANSWER: [YOUR FINAL ANSWER]
|
75 |
+
|
76 |
+
YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string."""
|
77 |
+
|
78 |
+
@staticmethod
|
79 |
+
def create_gaia_prompt(question: str) -> str:
|
80 |
+
"""Create properly formatted GAIA prompt"""
|
81 |
+
return f"{GAIAPromptManager.GAIA_SYSTEM_PROMPT}\n\nQuestion: {question}\n\nLet me think step by step:"
|
82 |
+
|
83 |
+
@staticmethod
|
84 |
+
def extract_final_answer(response: str) -> Tuple[str, str]:
|
85 |
+
"""Extract final answer and reasoning from model response"""
|
86 |
+
final_answer_pattern = r"FINAL ANSWER:\s*(.+?)(?:\n|$)"
|
87 |
+
match = re.search(final_answer_pattern, response, re.IGNORECASE | re.DOTALL)
|
88 |
+
|
89 |
+
if match:
|
90 |
+
final_answer = match.group(1).strip()
|
91 |
+
reasoning_end = match.start()
|
92 |
+
reasoning = response[:reasoning_end].strip()
|
93 |
+
else:
|
94 |
+
lines = response.strip().split('\n')
|
95 |
+
final_answer = lines[-1].strip() if lines else ""
|
96 |
+
reasoning = '\n'.join(lines[:-1]) if len(lines) > 1 else response
|
97 |
|
98 |
+
return final_answer, reasoning
|
99 |
+
|
100 |
+
# ================================
|
101 |
+
# HF SPACES OPTIMIZED MODEL MANAGER
|
102 |
+
# ================================
|
103 |
+
|
104 |
+
class HFSpaceModelManager:
|
105 |
+
"""Hugging Face Spaces optimized model manager"""
|
106 |
+
|
107 |
+
# Space-friendly models with different capabilities
|
108 |
+
SPACE_MODELS = {
|
109 |
+
"Fast & Light": {
|
110 |
+
"name": "microsoft/DialoGPT-medium",
|
111 |
+
"size": "~345MB",
|
112 |
+
"speed": "Fast",
|
113 |
+
"quality": "Good",
|
114 |
+
"gpu_required": False
|
115 |
+
},
|
116 |
+
"Balanced": {
|
117 |
+
"name": "stabilityai/stablelm-zephyr-3b",
|
118 |
+
"size": "~3GB",
|
119 |
+
"speed": "Medium",
|
120 |
+
"quality": "Better",
|
121 |
+
"gpu_required": True
|
122 |
+
},
|
123 |
+
"High Quality": {
|
124 |
+
"name": "HuggingFaceH4/zephyr-7b-beta",
|
125 |
+
"size": "~7GB",
|
126 |
+
"speed": "Slower",
|
127 |
+
"quality": "Best",
|
128 |
+
"gpu_required": True
|
129 |
+
},
|
130 |
+
"Instruction Following": {
|
131 |
+
"name": "mistralai/Mistral-7B-Instruct-v0.1",
|
132 |
+
"size": "~7GB",
|
133 |
+
"speed": "Medium",
|
134 |
+
"quality": "Excellent",
|
135 |
+
"gpu_required": True
|
136 |
}
|
137 |
+
}
|
138 |
+
|
139 |
+
def __init__(self, model_choice: str = "Fast & Light"):
|
140 |
+
self.model_config = self.SPACE_MODELS[model_choice]
|
141 |
+
self.model_name = self.model_config["name"]
|
142 |
+
self.tokenizer = None
|
143 |
+
self.model = None
|
144 |
+
self.pipeline = None
|
145 |
+
self.device = "cuda" if torch.cuda.is_available() else "cpu"
|
146 |
+
|
147 |
+
def load_model(self, progress_callback=None) -> str:
|
148 |
+
"""Load model with progress updates"""
|
149 |
+
try:
|
150 |
+
if progress_callback:
|
151 |
+
progress_callback(0.1, "Loading tokenizer...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
152 |
|
153 |
+
# Load tokenizer
|
154 |
+
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
|
155 |
+
if self.tokenizer.pad_token is None:
|
156 |
+
self.tokenizer.pad_token = self.tokenizer.eos_token
|
157 |
|
158 |
+
if progress_callback:
|
159 |
+
progress_callback(0.3, "Configuring model...")
|
160 |
+
|
161 |
+
# Configure quantization for GPU spaces
|
162 |
+
quantization_config = None
|
163 |
+
if self.device == "cuda" and "7b" in self.model_name.lower():
|
164 |
+
quantization_config = BitsAndBytesConfig(
|
165 |
+
load_in_4bit=True,
|
166 |
+
bnb_4bit_compute_dtype=torch.float16,
|
167 |
+
bnb_4bit_use_double_quant=True,
|
168 |
+
bnb_4bit_quant_type="nf4"
|
169 |
+
)
|
170 |
+
|
171 |
+
if progress_callback:
|
172 |
+
progress_callback(0.6, "Loading model weights...")
|
173 |
+
|
174 |
+
# Load model
|
175 |
+
self.model = AutoModelForCausalLM.from_pretrained(
|
176 |
+
self.model_name,
|
177 |
+
quantization_config=quantization_config,
|
178 |
+
device_map="auto" if self.device == "cuda" else None,
|
179 |
+
torch_dtype=torch.float16 if self.device == "cuda" else torch.float32,
|
180 |
+
trust_remote_code=True
|
181 |
+
)
|
182 |
+
|
183 |
+
if progress_callback:
|
184 |
+
progress_callback(0.9, "Creating pipeline...")
|
185 |
+
|
186 |
+
# Create pipeline
|
187 |
+
self.pipeline = pipeline(
|
188 |
+
"text-generation",
|
189 |
+
model=self.model,
|
190 |
+
tokenizer=self.tokenizer,
|
191 |
+
max_new_tokens=384,
|
192 |
+
temperature=0.7,
|
193 |
+
do_sample=True,
|
194 |
+
pad_token_id=self.tokenizer.eos_token_id,
|
195 |
+
device=0 if self.device == "cuda" else -1
|
196 |
+
)
|
197 |
+
|
198 |
+
if progress_callback:
|
199 |
+
progress_callback(1.0, "Model loaded successfully!")
|
200 |
|
201 |
+
return f"✅ Model '{self.model_name}' loaded successfully on {self.device.upper()}"
|
202 |
+
|
203 |
+
except Exception as e:
|
204 |
+
error_msg = f"❌ Error loading model: {str(e)}"
|
205 |
+
logger.error(error_msg)
|
206 |
+
return error_msg
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
207 |
|
208 |
+
def generate_response(self, prompt: str, max_tokens: int = 384) -> str:
|
209 |
+
"""Generate response with error handling"""
|
210 |
+
if self.pipeline is None:
|
211 |
+
return "❌ Model not loaded. Please load a model first."
|
212 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
213 |
try:
|
214 |
+
# Truncate prompt if too long
|
215 |
+
max_input_length = 1000
|
216 |
+
if len(prompt) > max_input_length:
|
217 |
+
prompt = prompt[:max_input_length] + "..."
|
218 |
+
|
219 |
+
outputs = self.pipeline(
|
220 |
+
prompt,
|
221 |
+
max_new_tokens=max_tokens,
|
222 |
+
temperature=0.7,
|
223 |
+
do_sample=True,
|
224 |
+
return_full_text=False,
|
225 |
+
pad_token_id=self.tokenizer.eos_token_id
|
226 |
+
)
|
227 |
+
|
228 |
+
response = outputs[0]['generated_text'].strip()
|
229 |
+
return response
|
230 |
+
|
231 |
+
except Exception as e:
|
232 |
+
return f"❌ Error generating response: {str(e)}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
233 |
|
234 |
+
# ================================
|
235 |
+
# DATASET MANAGEMENT
|
236 |
+
# ================================
|
237 |
|
238 |
+
class GAIADatasetManager:
|
239 |
+
"""Manages GAIA dataset loading and sample generation"""
|
240 |
|
241 |
+
@staticmethod
|
242 |
+
def load_gaia_dataset(split: str = "test", max_questions: int = None) -> Tuple[List[GAIAQuestion], str]:
|
243 |
+
"""Load GAIA dataset from Hugging Face Hub"""
|
244 |
try:
|
245 |
+
dataset = load_dataset("gaia-benchmark/GAIA", split=split, trust_remote_code=True)
|
|
|
246 |
|
247 |
+
questions = []
|
248 |
+
items = dataset[:max_questions] if max_questions else dataset
|
|
|
|
|
249 |
|
250 |
+
for i, item in enumerate(items):
|
251 |
+
question = GAIAQuestion(
|
252 |
+
task_id=item.get('task_id', f'gaia_{split}_{i:03d}'),
|
253 |
+
question=item['Question'],
|
254 |
+
level=item['Level'],
|
255 |
+
final_answer=item.get('Final answer', None),
|
256 |
+
file_name=item.get('file_name', None),
|
257 |
+
annotator_metadata=item.get('Annotator Metadata', None)
|
258 |
+
)
|
259 |
+
questions.append(question)
|
260 |
|
261 |
+
status = f"✅ Loaded {len(questions)} questions from GAIA {split} split"
|
262 |
+
return questions, status
|
263 |
|
264 |
except Exception as e:
|
265 |
+
error_msg = f"❌ Error loading GAIA dataset: {str(e)}"
|
266 |
+
return GAIADatasetManager.get_sample_questions(), error_msg
|
267 |
|
268 |
+
@staticmethod
|
269 |
+
def get_sample_questions() -> List[GAIAQuestion]:
|
270 |
+
"""Get sample questions for testing"""
|
271 |
+
sample_data = [
|
272 |
+
{
|
273 |
+
"task_id": "sample_001",
|
274 |
+
"question": "What is the capital of France?",
|
275 |
+
"level": 1,
|
276 |
+
"final_answer": "Paris"
|
277 |
+
},
|
278 |
+
{
|
279 |
+
"task_id": "sample_002",
|
280 |
+
"question": "Calculate 144 divided by 12.",
|
281 |
+
"level": 1,
|
282 |
+
"final_answer": "12"
|
283 |
+
},
|
284 |
+
{
|
285 |
+
"task_id": "sample_003",
|
286 |
+
"question": "What is the largest planet in our solar system?",
|
287 |
+
"level": 1,
|
288 |
+
"final_answer": "Jupiter"
|
289 |
+
},
|
290 |
+
{
|
291 |
+
"task_id": "sample_004",
|
292 |
+
"question": "Convert 100 degrees Celsius to Fahrenheit.",
|
293 |
+
"level": 2,
|
294 |
+
"final_answer": "212"
|
295 |
+
},
|
296 |
+
{
|
297 |
+
"task_id": "sample_005",
|
298 |
+
"question": "List the first three even numbers greater than zero.",
|
299 |
+
"level": 1,
|
300 |
+
"final_answer": "2, 4, 6"
|
301 |
+
},
|
302 |
+
{
|
303 |
+
"task_id": "sample_006",
|
304 |
+
"question": "What year did the Berlin Wall fall?",
|
305 |
+
"level": 1,
|
306 |
+
"final_answer": "1989"
|
307 |
+
},
|
308 |
+
{
|
309 |
+
"task_id": "sample_007",
|
310 |
+
"question": "What is the chemical symbol for water?",
|
311 |
+
"level": 1,
|
312 |
+
"final_answer": "H2O"
|
313 |
+
},
|
314 |
+
{
|
315 |
+
"task_id": "sample_008",
|
316 |
+
"question": "How many continents are there?",
|
317 |
+
"level": 1,
|
318 |
+
"final_answer": "7"
|
319 |
+
}
|
320 |
+
]
|
321 |
|
322 |
+
return [GAIAQuestion.from_dict(data) for data in sample_data]
|
323 |
|
324 |
+
# ================================
|
325 |
+
# MAIN GAIA AGENT FOR HF SPACES
|
326 |
+
# ================================
|
327 |
|
328 |
+
class GAIASpaceAgent:
|
329 |
+
"""Main GAIA agent optimized for Hugging Face Spaces"""
|
|
|
|
|
|
|
330 |
|
331 |
+
def __init__(self):
|
332 |
+
self.model_manager = None
|
333 |
+
self.prompt_manager = GAIAPromptManager()
|
334 |
+
self.current_model = None
|
335 |
+
self.evaluation_results: List[GAIAResponse] = []
|
336 |
+
|
337 |
+
def initialize_model(self, model_choice: str, progress=gr.Progress()) -> str:
|
338 |
+
"""Initialize model with progress tracking"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
339 |
try:
|
340 |
+
progress(0, desc="Initializing model manager...")
|
341 |
+
self.model_manager = HFSpaceModelManager(model_choice)
|
342 |
+
self.current_model = model_choice
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
343 |
|
344 |
+
# Load model with progress updates
|
345 |
+
def progress_callback(value, desc):
|
346 |
+
progress(value, desc=desc)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
347 |
|
348 |
+
result = self.model_manager.load_model(progress_callback)
|
|
|
|
|
|
|
|
|
|
|
|
|
349 |
|
350 |
+
# Clear any previous results when changing models
|
351 |
+
self.evaluation_results = []
|
352 |
|
353 |
+
return result
|
|
|
|
|
|
|
|
|
|
|
354 |
|
355 |
except Exception as e:
|
356 |
+
return f"❌ Failed to initialize model: {str(e)}"
|
357 |
|
358 |
+
def process_single_question(self, question_text: str, progress=gr.Progress()) -> Tuple[str, str, str, float]:
|
359 |
+
"""Process a single question with detailed output"""
|
360 |
+
if self.model_manager is None or self.model_manager.pipeline is None:
|
361 |
+
return "❌ No model loaded", "", "", 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
362 |
|
363 |
+
start_time = time.time()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
364 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
365 |
try:
|
366 |
+
progress(0.2, desc="Creating GAIA prompt...")
|
367 |
+
|
368 |
+
# Create GAIA prompt
|
369 |
+
prompt = self.prompt_manager.create_gaia_prompt(question_text)
|
370 |
+
|
371 |
+
progress(0.4, desc="Generating response...")
|
372 |
|
373 |
+
# Generate response
|
374 |
+
raw_response = self.model_manager.generate_response(prompt)
|
|
|
375 |
|
376 |
+
progress(0.8, desc="Extracting final answer...")
|
|
|
|
|
377 |
|
378 |
+
# Extract final answer and reasoning
|
379 |
+
final_answer, reasoning = self.prompt_manager.extract_final_answer(raw_response)
|
380 |
+
|
381 |
+
processing_time = time.time() - start_time
|
382 |
+
|
383 |
+
progress(1.0, desc="Complete!")
|
384 |
+
|
385 |
+
return final_answer, raw_response, reasoning, processing_time
|
386 |
|
387 |
except Exception as e:
|
388 |
+
processing_time = time.time() - start_time
|
389 |
+
error_msg = f"❌ Error processing question: {str(e)}"
|
390 |
+
return error_msg, "", "", processing_time
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
391 |
|
392 |
+
def batch_evaluate(self, questions: List[GAIAQuestion], progress=gr.Progress()) -> Tuple[str, str, str]:
|
393 |
+
"""Evaluate multiple questions with progress tracking"""
|
394 |
+
if self.model_manager is None:
|
395 |
+
return "❌ No model loaded", "", ""
|
396 |
|
397 |
+
results = []
|
398 |
+
total_questions = len(questions)
|
|
|
|
|
|
|
|
|
|
|
399 |
|
400 |
+
progress(0, desc=f"Starting evaluation of {total_questions} questions...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
401 |
|
402 |
for i, question in enumerate(questions):
|
|
|
|
|
|
|
|
|
|
|
|
|
403 |
try:
|
404 |
+
progress((i + 1) / total_questions,
|
405 |
+
desc=f"Processing question {i + 1}/{total_questions}: {question.task_id}")
|
406 |
+
|
407 |
start_time = time.time()
|
|
|
|
|
408 |
|
409 |
+
# Create prompt and generate response
|
410 |
+
prompt = self.prompt_manager.create_gaia_prompt(question.question)
|
411 |
+
raw_response = self.model_manager.generate_response(prompt)
|
412 |
+
|
413 |
+
# Extract final answer
|
414 |
+
final_answer, reasoning = self.prompt_manager.extract_final_answer(raw_response)
|
415 |
|
416 |
+
processing_time = time.time() - start_time
|
|
|
417 |
|
418 |
+
# Create response object
|
419 |
+
response = GAIAResponse(
|
420 |
+
task_id=question.task_id,
|
421 |
+
model_answer=raw_response,
|
422 |
+
reasoning_trace=reasoning,
|
423 |
+
final_answer=final_answer,
|
424 |
+
processing_time=processing_time
|
425 |
+
)
|
426 |
+
|
427 |
+
results.append(response)
|
428 |
+
self.evaluation_results.append(response)
|
429 |
|
430 |
except Exception as e:
|
431 |
+
logger.error(f"Error processing {question.task_id}: {e}")
|
432 |
+
error_response = GAIAResponse(
|
433 |
+
task_id=question.task_id,
|
434 |
+
model_answer=f"Error: {str(e)}",
|
435 |
+
reasoning_trace="Processing failed",
|
436 |
+
final_answer="ERROR",
|
437 |
+
processing_time=0.0
|
438 |
+
)
|
439 |
+
results.append(error_response)
|
440 |
+
self.evaluation_results.append(error_response)
|
441 |
+
|
442 |
+
# Generate summary
|
443 |
+
summary = self._generate_summary(results)
|
444 |
+
|
445 |
+
# Generate detailed results
|
446 |
+
detailed_results = self._generate_detailed_results(results, questions)
|
447 |
+
|
448 |
+
# Generate downloadable JSONL
|
449 |
+
jsonl_content = self._generate_jsonl(results)
|
450 |
+
|
451 |
+
return summary, detailed_results, jsonl_content
|
452 |
+
|
453 |
+
def _generate_summary(self, results: List[GAIAResponse]) -> str:
|
454 |
+
"""Generate evaluation summary"""
|
455 |
+
total = len(results)
|
456 |
+
errors = sum(1 for r in results if r.final_answer == "ERROR")
|
457 |
+
successful = total - errors
|
458 |
+
avg_time = sum(r.processing_time for r in results) / total if total > 0 else 0
|
459 |
+
total_time = sum(r.processing_time for r in results)
|
460 |
+
|
461 |
+
summary = f"""
|
462 |
+
# 📊 GAIA Evaluation Summary
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
463 |
|
464 |
+
## Overall Statistics
|
465 |
+
- **Total Questions**: {total}
|
466 |
+
- **Successful**: {successful}
|
467 |
+
- **Errors**: {errors}
|
468 |
+
- **Success Rate**: {(successful/total*100):.1f}%
|
469 |
|
470 |
+
## Performance Metrics
|
471 |
+
- **Average Processing Time**: {avg_time:.2f}s
|
472 |
+
- **Total Processing Time**: {total_time:.2f}s
|
473 |
+
- **Questions per Minute**: {(total/(total_time/60)):.1f}
|
474 |
+
|
475 |
+
## Model Information
|
476 |
+
- **Model**: {self.current_model}
|
477 |
+
- **Device**: {self.model_manager.device.upper() if self.model_manager else 'Unknown'}
|
478 |
+
"""
|
479 |
+
return summary
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
480 |
|
481 |
+
def _generate_detailed_results(self, results: List[GAIAResponse], questions: List[GAIAQuestion]) -> str:
|
482 |
+
"""Generate detailed results breakdown"""
|
483 |
+
detailed = "# 📋 Detailed Results\n\n"
|
|
|
484 |
|
485 |
+
for i, (result, question) in enumerate(zip(results, questions), 1):
|
486 |
+
status = "✅" if result.final_answer != "ERROR" else "❌"
|
487 |
+
|
488 |
+
detailed += f"""
|
489 |
+
## Question {i}: {question.task_id} {status}
|
490 |
+
|
491 |
+
**Question**: {question.question}
|
492 |
+
|
493 |
+
**Model Answer**: {result.final_answer}
|
494 |
+
|
495 |
+
**Expected Answer**: {question.final_answer if question.final_answer else 'N/A'}
|
496 |
+
|
497 |
+
**Processing Time**: {result.processing_time:.2f}s
|
498 |
+
|
499 |
+
**Level**: {question.level}
|
500 |
+
|
501 |
+
---
|
502 |
+
"""
|
503 |
+
|
504 |
+
return detailed
|
505 |
+
|
506 |
+
def _generate_jsonl(self, results: List[GAIAResponse]) -> str:
|
507 |
+
"""Generate JSONL format for download"""
|
508 |
+
jsonl_lines = []
|
509 |
+
for result in results:
|
510 |
+
line = {
|
511 |
+
"task_id": result.task_id,
|
512 |
+
"model_answer": result.model_answer,
|
513 |
+
"reasoning_trace": result.reasoning_trace
|
514 |
+
}
|
515 |
+
jsonl_lines.append(json.dumps(line))
|
516 |
|
517 |
+
return '\n'.join(jsonl_lines)
|
518 |
+
|
519 |
+
# ================================
|
520 |
+
# GLOBAL AGENT INSTANCE
|
521 |
+
# ================================
|
522 |
+
|
523 |
+
# Initialize global agent
|
524 |
+
gaia_agent = GAIASpaceAgent()
|
|
|
|
|
|
|
|
|
|
|
|
|
525 |
|
526 |
+
# ================================
|
527 |
+
# GRADIO INTERFACE FUNCTIONS
|
528 |
+
# ================================
|
529 |
|
530 |
+
def load_model_interface(model_choice: str, progress=gr.Progress()):
|
531 |
+
"""Interface function for model loading"""
|
532 |
+
return gaia_agent.initialize_model(model_choice, progress)
|
533 |
|
534 |
+
def single_question_interface(question: str, progress=gr.Progress()):
|
535 |
+
"""Interface function for single question processing"""
|
536 |
+
if not question.strip():
|
537 |
+
return "Please enter a question", "", "", "0.00s"
|
538 |
+
|
539 |
+
final_answer, full_response, reasoning, proc_time = gaia_agent.process_single_question(question, progress)
|
540 |
+
|
541 |
+
return (
|
542 |
+
final_answer,
|
543 |
+
full_response,
|
544 |
+
reasoning,
|
545 |
+
f"{proc_time:.2f}s"
|
546 |
+
)
|
547 |
+
|
548 |
+
def batch_evaluate_interface(dataset_choice: str, max_questions: int, progress=gr.Progress()):
|
549 |
+
"""Interface function for batch evaluation"""
|
550 |
+
if gaia_agent.model_manager is None:
|
551 |
+
return "❌ Please load a model first", "", ""
|
552 |
|
553 |
+
progress(0.1, desc="Loading dataset...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
554 |
|
555 |
+
# Load questions based on choice
|
556 |
+
if dataset_choice == "Sample Questions":
|
557 |
+
questions = GAIADatasetManager.get_sample_questions()
|
558 |
+
status_msg = f"✅ Loaded {len(questions)} sample questions"
|
559 |
+
else:
|
560 |
+
questions, status_msg = GAIADatasetManager.load_gaia_dataset("test", max_questions)
|
561 |
|
562 |
+
# Limit questions
|
563 |
+
if max_questions and len(questions) > max_questions:
|
564 |
+
questions = questions[:max_questions]
|
|
|
|
|
|
|
565 |
|
566 |
+
progress(0.2, desc=f"{status_msg}. Starting evaluation...")
|
|
|
|
|
|
|
|
|
567 |
|
568 |
+
# Run evaluation
|
569 |
+
summary, detailed, jsonl = gaia_agent.batch_evaluate(questions, progress)
|
|
|
|
|
|
|
570 |
|
571 |
+
return summary, detailed, jsonl
|
572 |
|
573 |
+
def get_model_info(model_choice: str):
|
574 |
+
"""Get information about selected model"""
|
575 |
+
if model_choice in HFSpaceModelManager.SPACE_MODELS:
|
576 |
+
config = HFSpaceModelManager.SPACE_MODELS[model_choice]
|
577 |
+
return f"""
|
578 |
+
**Model**: {config['name']}
|
579 |
+
**Size**: {config['size']}
|
580 |
+
**Speed**: {config['speed']}
|
581 |
+
**Quality**: {config['quality']}
|
582 |
+
**GPU Required**: {'Yes' if config['gpu_required'] else 'No'}
|
583 |
+
"""
|
584 |
+
return "Model information not available"
|
585 |
|
586 |
+
# ================================
|
587 |
+
# GRADIO APP CREATION
|
588 |
+
# ================================
|
589 |
+
|
590 |
+
def create_gaia_app():
|
591 |
+
"""Create the main Gradio application"""
|
592 |
+
|
593 |
+
with gr.Blocks(
|
594 |
+
title="GAIA Benchmark AI Agent",
|
595 |
+
theme=gr.themes.Soft(),
|
596 |
+
css="""
|
597 |
+
.gradio-container {
|
598 |
+
font-family: 'Arial', sans-serif;
|
599 |
+
}
|
600 |
+
.main-header {
|
601 |
+
text-align: center;
|
602 |
+
background: linear-gradient(45deg, #2196F3, #21CBF3);
|
603 |
+
-webkit-background-clip: text;
|
604 |
+
-webkit-text-fill-color: transparent;
|
605 |
+
font-size: 2.5em;
|
606 |
+
font-weight: bold;
|
607 |
+
margin-bottom: 20px;
|
608 |
+
}
|
609 |
+
"""
|
610 |
+
) as app:
|
611 |
+
|
612 |
+
# Header
|
613 |
+
gr.HTML("""
|
614 |
+
<div class="main-header">
|
615 |
+
🧠 GAIA Benchmark AI Agent
|
616 |
+
</div>
|
617 |
+
<p style="text-align: center; font-size: 1.2em; color: #666;">
|
618 |
+
Evaluate AI models on the GAIA benchmark with step-by-step reasoning
|
619 |
+
</p>
|
620 |
+
""")
|
621 |
+
|
622 |
+
with gr.Tabs():
|
623 |
+
|
624 |
+
# ===============================
|
625 |
+
# TAB 1: MODEL SETUP
|
626 |
+
# ===============================
|
627 |
+
with gr.Tab("🔧 Model Setup"):
|
628 |
+
gr.Markdown("## Choose and Load Your Model")
|
629 |
+
|
630 |
+
with gr.Row():
|
631 |
+
with gr.Column(scale=2):
|
632 |
+
model_dropdown = gr.Dropdown(
|
633 |
+
choices=list(HFSpaceModelManager.SPACE_MODELS.keys()),
|
634 |
+
value="Fast & Light",
|
635 |
+
label="Select Model",
|
636 |
+
info="Choose based on your quality vs speed preference"
|
637 |
+
)
|
638 |
+
|
639 |
+
model_info = gr.Markdown(
|
640 |
+
value=get_model_info("Fast & Light"),
|
641 |
+
label="Model Information"
|
642 |
+
)
|
643 |
+
|
644 |
+
load_btn = gr.Button("🚀 Load Model", variant="primary", size="lg")
|
645 |
+
|
646 |
+
with gr.Column(scale=1):
|
647 |
+
gpu_info = gr.Markdown(f"""
|
648 |
+
### 🖥️ System Info
|
649 |
+
**CUDA Available**: {torch.cuda.is_available()}
|
650 |
+
{f"**GPU**: {torch.cuda.get_device_name(0)}" if torch.cuda.is_available() else "**Device**: CPU"}
|
651 |
+
""")
|
652 |
+
|
653 |
+
model_status = gr.Textbox(
|
654 |
+
label="Model Status",
|
655 |
+
value="No model loaded",
|
656 |
+
interactive=False
|
657 |
+
)
|
658 |
+
|
659 |
+
# Update model info when selection changes
|
660 |
+
model_dropdown.change(
|
661 |
+
fn=get_model_info,
|
662 |
+
inputs=[model_dropdown],
|
663 |
+
outputs=[model_info]
|
664 |
+
)
|
665 |
+
|
666 |
+
# Load model when button clicked
|
667 |
+
load_btn.click(
|
668 |
+
fn=load_model_interface,
|
669 |
+
inputs=[model_dropdown],
|
670 |
+
outputs=[model_status]
|
671 |
+
)
|
672 |
+
|
673 |
+
# ===============================
|
674 |
+
# TAB 2: SINGLE QUESTION
|
675 |
+
# ===============================
|
676 |
+
with gr.Tab("❓ Single Question"):
|
677 |
+
gr.Markdown("## Test Individual Questions")
|
678 |
+
|
679 |
+
with gr.Row():
|
680 |
+
with gr.Column():
|
681 |
+
question_input = gr.Textbox(
|
682 |
+
label="Enter your question",
|
683 |
+
placeholder="e.g., What is the capital of France?",
|
684 |
+
lines=3
|
685 |
+
)
|
686 |
+
|
687 |
+
process_btn = gr.Button("🤔 Process Question", variant="primary")
|
688 |
+
|
689 |
+
# Example questions
|
690 |
+
gr.Markdown("### 💡 Example Questions:")
|
691 |
+
example_questions = [
|
692 |
+
"What is the capital of France?",
|
693 |
+
"Calculate 144 divided by 12",
|
694 |
+
"What is the largest planet in our solar system?",
|
695 |
+
"Convert 100 degrees Celsius to Fahrenheit"
|
696 |
+
]
|
697 |
+
|
698 |
+
for i, example in enumerate(example_questions):
|
699 |
+
gr.Button(
|
700 |
+
f"📝 {example}",
|
701 |
+
size="sm"
|
702 |
+
).click(
|
703 |
+
lambda x=example: x,
|
704 |
+
outputs=[question_input]
|
705 |
+
)
|
706 |
+
|
707 |
+
with gr.Column():
|
708 |
+
final_answer_output = gr.Textbox(
|
709 |
+
label="🎯 Final Answer",
|
710 |
+
interactive=False
|
711 |
+
)
|
712 |
+
|
713 |
+
processing_time = gr.Textbox(
|
714 |
+
label="⏱️ Processing Time",
|
715 |
+
interactive=False
|
716 |
+
)
|
717 |
+
|
718 |
+
with gr.Accordion("🧠 Full Response", open=False):
|
719 |
+
full_response = gr.Textbox(
|
720 |
+
label="Complete Model Response",
|
721 |
+
lines=8,
|
722 |
+
interactive=False
|
723 |
+
)
|
724 |
+
|
725 |
+
with gr.Accordion("🔍 Reasoning Trace", open=False):
|
726 |
+
reasoning_trace = gr.Textbox(
|
727 |
+
label="Step-by-step Reasoning",
|
728 |
+
lines=6,
|
729 |
+
interactive=False
|
730 |
+
)
|
731 |
+
|
732 |
+
# Process single question
|
733 |
+
process_btn.click(
|
734 |
+
fn=single_question_interface,
|
735 |
+
inputs=[question_input],
|
736 |
+
outputs=[final_answer_output, full_response, reasoning_trace, processing_time]
|
737 |
+
)
|
738 |
+
|
739 |
+
# ===============================
|
740 |
+
# TAB 3: BATCH EVALUATION
|
741 |
+
# ===============================
|
742 |
+
with gr.Tab("📊 Batch Evaluation"):
|
743 |
+
gr.Markdown("## Evaluate Multiple Questions")
|
744 |
+
|
745 |
+
with gr.Row():
|
746 |
+
dataset_choice = gr.Radio(
|
747 |
+
choices=["Sample Questions", "GAIA Test Set"],
|
748 |
+
value="Sample Questions",
|
749 |
+
label="Dataset Choice",
|
750 |
+
info="Start with sample questions to test your setup"
|
751 |
+
)
|
752 |
+
|
753 |
+
max_questions = gr.Slider(
|
754 |
+
minimum=1,
|
755 |
+
maximum=50,
|
756 |
+
value=5,
|
757 |
+
step=1,
|
758 |
+
label="Max Questions",
|
759 |
+
info="Number of questions to evaluate"
|
760 |
+
)
|
761 |
+
|
762 |
+
evaluate_btn = gr.Button("🚀 Start Batch Evaluation", variant="primary", size="lg")
|
763 |
+
|
764 |
+
with gr.Row():
|
765 |
+
with gr.Column():
|
766 |
+
summary_output = gr.Markdown(
|
767 |
+
label="📊 Evaluation Summary",
|
768 |
+
value="No evaluation completed yet"
|
769 |
+
)
|
770 |
+
|
771 |
+
with gr.Column():
|
772 |
+
download_output = gr.File(
|
773 |
+
label="💾 Download Results (JSONL)",
|
774 |
+
visible=False
|
775 |
+
)
|
776 |
+
|
777 |
+
with gr.Accordion("📋 Detailed Results", open=False):
|
778 |
+
detailed_output = gr.Markdown(
|
779 |
+
value="Run an evaluation to see detailed results"
|
780 |
+
)
|
781 |
+
|
782 |
+
# Batch evaluation
|
783 |
+
def batch_eval_with_download(*args):
|
784 |
+
summary, detailed, jsonl_content = batch_evaluate_interface(*args)
|
785 |
+
|
786 |
+
# Save JSONL for download
|
787 |
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
788 |
+
filename = f"gaia_results_{timestamp}.jsonl"
|
789 |
+
|
790 |
+
with open(filename, 'w') as f:
|
791 |
+
f.write(jsonl_content)
|
792 |
+
|
793 |
+
return summary, detailed, filename
|
794 |
+
|
795 |
+
evaluate_btn.click(
|
796 |
+
fn=batch_eval_with_download,
|
797 |
+
inputs=[dataset_choice, max_questions],
|
798 |
+
outputs=[summary_output, detailed_output, download_output]
|
799 |
+
).then(
|
800 |
+
lambda: gr.update(visible=True),
|
801 |
+
outputs=[download_output]
|
802 |
+
)
|
803 |
+
|
804 |
+
# ===============================
|
805 |
+
# TAB 4: INFORMATION
|
806 |
+
# ===============================
|
807 |
+
with gr.Tab("ℹ️ Information"):
|
808 |
+
gr.Markdown("""
|
809 |
+
# 🧠 GAIA Benchmark AI Agent
|
810 |
+
|
811 |
+
## What is GAIA?
|
812 |
+
GAIA (General AI Assistant) is a benchmark designed to test AI assistants on real-world questions that require:
|
813 |
+
- **Reasoning**: Multi-step logical thinking
|
814 |
+
- **Multi-modality**: Handling text, images, and other file types
|
815 |
+
- **Web browsing**: Finding and using external information
|
816 |
+
- **Tool use**: Calculator, code execution, etc.
|
817 |
+
|
818 |
+
## 🎯 How to Use This Space
|
819 |
+
|
820 |
+
### 1. Model Setup
|
821 |
+
- Choose a model based on your needs (speed vs quality)
|
822 |
+
- Load the model (this may take a few minutes)
|
823 |
+
- Wait for "Model loaded successfully" message
|
824 |
+
|
825 |
+
### 2. Test Single Questions
|
826 |
+
- Start with the "Single Question" tab
|
827 |
+
- Try example questions to verify everything works
|
828 |
+
- Enter your own questions to test model capabilities
|
829 |
+
|
830 |
+
### 3. Batch Evaluation
|
831 |
+
- Use "Sample Questions" first to test your setup
|
832 |
+
- Then try "GAIA Test Set" for real benchmark evaluation
|
833 |
+
- Download results in JSONL format for submission
|
834 |
+
|
835 |
+
## 📊 Model Recommendations
|
836 |
+
|
837 |
+
| Model | Best For | Memory | Speed | Quality |
|
838 |
+
|-------|----------|---------|-------|---------|
|
839 |
+
| Fast & Light | Quick testing | Low | Fast | Good |
|
840 |
+
| Balanced | General use | Medium | Medium | Better |
|
841 |
+
| High Quality | Best results | High | Slow | Best |
|
842 |
+
| Instruction Following | Complex reasoning | High | Medium | Excellent |
|
843 |
+
|
844 |
+
## 🔗 Resources
|
845 |
+
- [GAIA Paper](https://arxiv.org/abs/2311.12983)
|
846 |
+
- [GAIA Leaderboard](https://huggingface.co/spaces/gaia-benchmark/leaderboard)
|
847 |
+
- [Hugging Face Spaces Documentation](https://huggingface.co/docs/hub/spaces)
|
848 |
+
|
849 |
+
## 🚀 Output Format
|
850 |
+
Results are saved in GAIA leaderboard format:
|
851 |
+
```json
|
852 |
+
{"task_id": "gaia_001", "model_answer": "[FULL RESPONSE]", "reasoning_trace": "[REASONING]"}
|
853 |
+
```
|
854 |
+
|
855 |
+
## ⚡ Tips for Best Results
|
856 |
+
1. **Start Small**: Test with sample questions first
|
857 |
+
2. **Choose Right Model**: Balance speed vs quality for your needs
|
858 |
+
3. **Monitor GPU**: Larger models need GPU acceleration
|
859 |
+
4. **Download Results**: Save JSONL files for leaderboard submission
|
860 |
+
""")
|
861 |
+
|
862 |
+
return app
|
863 |
+
|
864 |
+
# ================================
|
865 |
+
# MAIN APPLICATION
|
866 |
+
# ================================
|
867 |
+
|
868 |
+
if __name__ == "__main__":
|
869 |
+
# Create and launch the Gradio app
|
870 |
+
app = create_gaia_app()
|
871 |
+
app.launch(
|
872 |
+
server_name="0.0.0.0",
|
873 |
+
server_port=7860,
|
874 |
+
share=False
|
875 |
+
)
|