Spaces:
Running
Running
""" | |
Simple BigCodeArena - A simplified AI coding battle arena | |
Focuses on core functionality: two models, automatic code extraction, and execution | |
""" | |
import gradio as gr | |
from gradio_sandboxcomponent import SandboxComponent | |
import pandas as pd | |
import datetime | |
import os | |
import asyncio | |
import concurrent.futures | |
import random | |
import time | |
import numpy as np | |
from collections import defaultdict | |
from datasets import Dataset, load_dataset | |
# Import Elo calculation utilities | |
from elo_calculation import ( | |
calculate_elo_with_confidence_intervals, | |
create_ranking_dataframe, | |
) | |
# Import ranking functionality | |
from ranking import ( | |
load_ranking_data, | |
update_ranking_display, | |
force_update_ranking_display, | |
create_ranking_tab, | |
setup_ranking_handlers, | |
) | |
# Import voting functionality | |
from voting import ( | |
handle_vote, | |
save_vote_to_hf, | |
serialize_interactions, | |
create_vote_ui, | |
should_show_vote_buttons, | |
get_vote_ui_updates, | |
setup_vote_handlers, | |
) | |
# Import completion utilities | |
from completion import make_config, registered_api_completion | |
from sandbox.prompts import GENERAL_SANDBOX_INSTRUCTION | |
# Import code extraction utilities | |
from sandbox.code_analyzer import ( | |
SandboxEnvironment, | |
extract_code_from_markdown, | |
) | |
# Import sandbox execution functions | |
from sandbox.code_runner import ( | |
run_html_sandbox, | |
run_react_sandbox, | |
run_vue_sandbox, | |
run_pygame_sandbox, | |
run_gradio_sandbox, | |
run_streamlit_sandbox, | |
run_code_interpreter, | |
run_c_code, | |
run_cpp_code, | |
run_java_code, | |
run_golang_code, | |
run_rust_code, | |
mermaid_to_html, | |
javascript_to_html | |
) | |
# Import sandbox telemetry | |
from sandbox.sandbox_telemetry import log_sandbox_telemetry_gradio_fn | |
# Create a proper sandbox state structure | |
def create_sandbox_state() -> dict: | |
"""Create a new sandbox state for a model""" | |
return { | |
'enable_sandbox': True, | |
'enabled_round': 0, | |
'sandbox_run_round': 0, | |
'edit_round': 0, | |
'sandbox_environment': SandboxEnvironment.AUTO, | |
'auto_selected_sandbox_environment': None, | |
'sandbox_instruction': "Run the extracted code in the appropriate sandbox environment", | |
'code_to_execute': "", | |
'code_dependencies': ([], []), | |
'btn_list_length': 5, | |
'sandbox_id': None, | |
'chat_session_id': None, | |
'conv_id': None, | |
"sandbox_output": None, | |
"sandbox_error": None, | |
} | |
def reset_sandbox_state(state: dict) -> dict: | |
"""Reset the sandbox state""" | |
state['enabled_round'] = 0 | |
state['sandbox_run_round'] = 0 | |
state['edit_round'] = 0 | |
state['auto_selected_sandbox_environment'] = None | |
state['code_to_execute'] = "" | |
state['code_dependencies'] = ([], []) | |
state['sandbox_error'] = None | |
state['sandbox_output'] = None | |
state['sandbox_id'] = None | |
state['conv_id'] = None | |
state['chat_session_id'] = None | |
return state | |
# Load API configuration | |
def load_api_config(): | |
"""Load API configuration from yaml file""" | |
try: | |
config = make_config("api_config.yaml") | |
return config | |
except Exception as e: | |
return {} | |
# Global variables | |
api_config = load_api_config() | |
available_models = list(api_config.keys()) if api_config else [] | |
# HuggingFace dataset configuration | |
HF_DATASET_NAME = os.getenv("HF_DATASET_NAME") | |
HF_TOKEN = os.getenv("HF_TOKEN") | |
def get_random_models(): | |
"""Get two random models from available models using weighted sampling""" | |
if len(available_models) < 2: | |
return available_models[0] if available_models else None, available_models[0] if available_models else None | |
# Use get_battle_pair for weighted sampling | |
return get_battle_pair(available_models, {}, [], {}, []) | |
# Configuration for battle sampling | |
ANON_MODELS = [] # Models that should not battle against each other in anonymous mode | |
BATTLE_STRICT_TARGETS = {} # Strict battle targets for specific models | |
def get_sample_weight(model, outage_models, sampling_weights, sampling_boost_models=None): | |
"""Get the sampling weight for a model""" | |
# Check if model is in outage | |
if model in outage_models: | |
return 0 | |
# Get base weight from API config | |
model_config = api_config.get(model, {}) | |
base_weight = model_config.get('weight', 1.0) # Default weight is 1.0 | |
# Apply custom sampling weights if provided | |
if model in sampling_weights: | |
base_weight *= sampling_weights[model] | |
# Apply boost if model is in boost list | |
if sampling_boost_models and model in sampling_boost_models: | |
base_weight *= 2.0 # Example boost factor | |
return base_weight | |
def is_model_match_pattern(model, pattern): | |
"""Check if model matches a pattern (for battle strict targets)""" | |
# Simple pattern matching - can be extended for more complex patterns | |
if isinstance(pattern, str): | |
return pattern in model | |
elif isinstance(pattern, list): | |
return any(p in model for p in pattern) | |
return False | |
def get_battle_pair( | |
models, battle_targets, outage_models, sampling_weights, sampling_boost_models | |
): | |
""" | |
Sample a pair of models for battle using weighted sampling. | |
Args: | |
models: List of available model names | |
battle_targets: Dict mapping models to their preferred battle targets | |
outage_models: List of models currently in outage | |
sampling_weights: Dict of custom sampling weights per model | |
sampling_boost_models: List of models to boost in sampling | |
Returns: | |
Tuple of (model_a, model_b) for battle | |
""" | |
if len(models) == 1: | |
return models[0], models[0] | |
# Calculate weights for all models | |
model_weights = [] | |
for model in models: | |
weight = get_sample_weight( | |
model, outage_models, sampling_weights, sampling_boost_models | |
) | |
model_weights.append(weight) | |
total_weight = np.sum(model_weights) | |
if total_weight == 0: | |
# Fallback to uniform sampling if all weights are 0 | |
return random.sample(models, 2) | |
model_weights = np.array(model_weights) / total_weight | |
# Sample first model | |
chosen_idx = np.random.choice(len(models), p=model_weights) | |
chosen_model = models[chosen_idx] | |
# Find eligible rival models | |
rival_models = [] | |
rival_weights = [] | |
for model in models: | |
if model == chosen_model: | |
continue | |
if model in ANON_MODELS and chosen_model in ANON_MODELS: | |
continue | |
if chosen_model in BATTLE_STRICT_TARGETS: | |
if not is_model_match_pattern(model, BATTLE_STRICT_TARGETS[chosen_model]): | |
continue | |
if model in BATTLE_STRICT_TARGETS: | |
if not is_model_match_pattern(chosen_model, BATTLE_STRICT_TARGETS[model]): | |
continue | |
weight = get_sample_weight(model, outage_models, sampling_weights) | |
if ( | |
weight != 0 | |
and chosen_model in battle_targets | |
and model in battle_targets[chosen_model] | |
): | |
# boost to higher chance for targeted battles | |
weight = 0.5 * total_weight / len(battle_targets[chosen_model]) | |
rival_models.append(model) | |
rival_weights.append(weight) | |
if not rival_models: | |
# Fallback: if no eligible rivals, pick any other model | |
rival_models = [m for m in models if m != chosen_model] | |
if rival_models: | |
rival_model = random.choice(rival_models) | |
else: | |
rival_model = chosen_model | |
else: | |
rival_weights = np.array(rival_weights) / np.sum(rival_weights) | |
rival_idx = np.random.choice(len(rival_models), p=rival_weights) | |
rival_model = rival_models[rival_idx] | |
# Randomly swap order | |
swap = np.random.randint(2) | |
if swap == 0: | |
return chosen_model, rival_model | |
else: | |
return rival_model, chosen_model | |
def create_chat_state(model_name: str) -> dict: | |
"""Create a new chat state for a model""" | |
return { | |
"model_name": model_name, | |
"messages": [], | |
"sandbox_state": create_sandbox_state(), | |
"has_output": False, | |
"generating": False, # Track if model is currently generating | |
"interactions": [], # Store user interactions | |
} | |
def generate_response_with_completion(state, temperature, max_tokens): | |
"""Generate response using the completion API system with full conversation history""" | |
if state is None: | |
return state, "" | |
# Get the last user message | |
user_message = None | |
for msg in reversed(state["messages"]): | |
if msg["role"] == "user": | |
user_message = msg["content"] | |
break | |
if not user_message: | |
return state, "" | |
# Prepare messages for API call - include full conversation history | |
messages = [{"role": "system", "content": GENERAL_SANDBOX_INSTRUCTION}] | |
for msg in state["messages"]: | |
if msg["role"] in ["user", "assistant"] and msg["content"] is not None: | |
messages.append({"role": msg["role"], "content": msg["content"]}) | |
# Get model config | |
model_name = state["model_name"] | |
if model_name not in api_config: | |
return state, f"Error: Model {model_name} not configured" | |
model_config = api_config[model_name] | |
api_type = model_config.get("api_type", "openai") | |
# retrieve the api completion function from register | |
api_completion_func = registered_api_completion[api_type] | |
# build arguments for api completions | |
# Use the actual model identifier from config, not the display name | |
actual_model = model_config.get("model", model_name) | |
kwargs = { | |
"model": actual_model, | |
"temperature": temperature, | |
"max_tokens": max_tokens, | |
"api_dict": model_config.get("endpoints", [{}])[0] if model_config.get("endpoints") else None, | |
"messages": messages, | |
} | |
output = api_completion_func(**kwargs) | |
# Extract the answer from the response | |
if isinstance(output, dict) and "answer" in output: | |
response_text = output["answer"] | |
# Return response as dict with content and interaction keys | |
response_dict = { | |
"content": response_text, | |
"interaction": state.get("interactions", []) | |
} | |
return state, response_dict | |
else: | |
error_msg = f"Error: Invalid response format from {api_type}" | |
# Return error as dict with content and interaction keys | |
error_dict = { | |
"content": error_msg, | |
"interaction": state.get("interactions", []) | |
} | |
return state, error_dict | |
def generate_response_async(state, temperature, max_tokens): | |
"""Async wrapper for generate_response_with_completion""" | |
loop = asyncio.new_event_loop() | |
asyncio.set_event_loop(loop) | |
try: | |
# Run the synchronous function in a thread pool | |
with concurrent.futures.ThreadPoolExecutor() as executor: | |
future = executor.submit(generate_response_with_completion, state, temperature, max_tokens) | |
return future.result() | |
finally: | |
loop.close() | |
async def generate_responses_parallel(state0, state1, temperature, max_tokens): | |
"""Generate responses for both models in parallel with error handling""" | |
loop = asyncio.get_event_loop() | |
# Run both model generations in parallel | |
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: | |
future0 = loop.run_in_executor(executor, generate_response_with_completion, state0, temperature, max_tokens) | |
future1 = loop.run_in_executor(executor, generate_response_with_completion, state1, temperature, max_tokens) | |
# Wait for both to complete with error handling | |
try: | |
result0, result1 = await asyncio.gather(future0, future1, return_exceptions=True) | |
# Handle exceptions | |
if isinstance(result0, Exception): | |
result0 = (state0, {"content": f"Error: {str(result0)}", "interaction": []}) | |
if isinstance(result1, Exception): | |
result1 = (state1, {"content": f"Error: {str(result1)}", "interaction": []}) | |
except Exception as e: | |
# Fallback to sequential processing | |
result0 = generate_response_with_completion(state0, temperature, max_tokens) | |
result1 = generate_response_with_completion(state1, temperature, max_tokens) | |
return result0, result1 | |
def extract_and_execute_code(message, sandbox_state): | |
"""Extract code from message and prepare for execution""" | |
if not message: | |
return sandbox_state, "", "" | |
# Extract code using the same logic as code_runner.py | |
extract_result = extract_code_from_markdown( | |
message=message, | |
enable_auto_env=True | |
) | |
if extract_result is None: | |
return sandbox_state, "", "" | |
code, code_language, env_selection, install_command = extract_result | |
# Update sandbox state (now a dictionary) | |
sandbox_state['code_to_execute'] = code | |
sandbox_state['install_command'] = install_command | |
sandbox_state['auto_selected_sandbox_environment'] = env_selection | |
return sandbox_state, code, str(env_selection) | |
def add_text_and_generate(state0, state1, text, temperature, max_tokens, model_a, model_b): | |
"""Add text and generate responses for both models""" | |
if not text.strip(): | |
return state0, state1, "", "", "", "", "", "", "", "", "", "", "", "" | |
# Initialize states if needed | |
if state0 is None or state1 is None: | |
if state0 is None: | |
state0 = create_chat_state(model_a) | |
if state1 is None: | |
state1 = create_chat_state(model_b) | |
# Add user message to both states | |
state0["messages"].append({"role": "user", "content": text}) | |
state1["messages"].append({"role": "user", "content": text}) | |
# Mark that generation is starting - this will be used to hide vote buttons | |
state0["generating"] = True | |
state1["generating"] = True | |
# Generate responses in parallel | |
start_time = time.time() | |
loop = asyncio.new_event_loop() | |
asyncio.set_event_loop(loop) | |
try: | |
result0, result1 = loop.run_until_complete( | |
generate_responses_parallel(state0, state1, temperature, max_tokens) | |
) | |
state0, response0 = result0 | |
state1, response1 = result1 | |
generation_time = time.time() - start_time | |
except Exception as e: | |
# Fallback to sequential processing | |
state0, response0 = generate_response_with_completion(state0, temperature, max_tokens) | |
state1, response1 = generate_response_with_completion(state1, temperature, max_tokens) | |
generation_time = time.time() - start_time | |
finally: | |
loop.close() | |
# Add the assistant responses to the message history | |
state0["messages"].append({"role": "assistant", "content": response0["content"]}) | |
state1["messages"].append({"role": "assistant", "content": response1["content"]}) | |
# Format chat history for display | |
chat0 = format_chat_history(state0["messages"]) | |
chat1 = format_chat_history(state1["messages"]) | |
# Extract code from responses for sandbox | |
sandbox_state0 = ( | |
state0.get("sandbox_state", create_sandbox_state()) | |
if state0 | |
else create_sandbox_state() | |
) | |
sandbox_state1 = ( | |
state1.get("sandbox_state", create_sandbox_state()) | |
if state1 | |
else create_sandbox_state() | |
) | |
sandbox_state0, code0, env0 = extract_and_execute_code(response0["content"], sandbox_state0) | |
sandbox_state1, code1, env1 = extract_and_execute_code(response1["content"], sandbox_state1) | |
# Update sandbox states in the main states | |
if state0 is not None: | |
state0["sandbox_state"] = sandbox_state0 | |
state0["has_output"] = True | |
state0["generating"] = False # Mark generation as complete | |
if state1 is not None: | |
state1["sandbox_state"] = sandbox_state1 | |
state1["has_output"] = True | |
state1["generating"] = False # Mark generation as complete | |
# Clear previous sandbox outputs when new message is sent | |
sandbox_output0 = "" | |
sandbox_output1 = "" | |
# Force clear sandbox components to ensure refresh | |
sandbox_component_update0 = gr.update(value=("", False, []), visible=False) | |
sandbox_component_update1 = gr.update(value=("", False, []), visible=False) | |
# Also clear the sandbox view components to show fresh results | |
sandbox_view_a = "" | |
sandbox_view_b = "" | |
# Run sandbox executions in parallel if both models have code | |
if code0.strip() or code1.strip(): | |
sandbox_start_time = time.time() | |
loop = asyncio.new_event_loop() | |
asyncio.set_event_loop(loop) | |
try: | |
# Prepare sandbox execution parameters | |
install_command0 = sandbox_state0.get('install_command', "") if code0.strip() else "" | |
install_command1 = sandbox_state1.get('install_command', "") if code1.strip() else "" | |
# Run both sandbox executions in parallel | |
result0, result1 = loop.run_until_complete( | |
run_sandboxes_parallel( | |
sandbox_state0, code0, install_command0, | |
sandbox_state1, code1, install_command1 | |
) | |
) | |
sandbox_time = time.time() - sandbox_start_time | |
# Process results for model A | |
if code0.strip(): | |
sandbox_url0, sandbox_output0, sandbox_error0 = result0 | |
# Check if this is a web-based environment that should use SandboxComponent | |
env_type = sandbox_state0.get('auto_selected_sandbox_environment') or sandbox_state0.get('sandbox_environment') | |
# Use the URL directly from the function return | |
if sandbox_url0: | |
# Force refresh by using a unique key and clearing first | |
sandbox_component_update0 = gr.update( | |
value=(sandbox_url0, True, []), | |
visible=True, | |
key=f"sandbox_a_{int(time.time() * 1000)}" # Unique key to force refresh | |
) | |
# Update sandbox view with output and errors | |
if sandbox_output0: | |
sandbox_view_a += sandbox_output0 | |
if sandbox_error0: | |
sandbox_view_a = f"<details closed><summary><strong>🚨 Errors/Warnings</strong></summary>\n\n```\n{sandbox_error0}\n```\n\n</details>\n\n" + sandbox_view_a | |
# Process results for model B | |
if code1.strip(): | |
sandbox_url1, sandbox_output1, sandbox_error1 = result1 | |
# Check if this is a web-based environment that should use SandboxComponent | |
env_type = sandbox_state1.get('auto_selected_sandbox_environment') or sandbox_state1.get('sandbox_environment') | |
# Use the URL directly from the function return | |
if sandbox_url1: | |
# Force refresh by using a unique key and clearing first | |
sandbox_component_update1 = gr.update( | |
value=(sandbox_url1, True, []), | |
visible=True, | |
key=f"sandbox_b_{int(time.time() * 1000)}" # Unique key to force refresh | |
) | |
if sandbox_output1: | |
sandbox_view_b += sandbox_output1 | |
if sandbox_error1: | |
sandbox_view_b = f"<details closed><summary><strong>🚨 Errors/Warnings</strong></summary>\n\n```\n{sandbox_error1}\n```\n\n</details>\n\n" + sandbox_view_b | |
except Exception as e: | |
# Fallback to sequential processing | |
if code0.strip(): | |
install_command0 = sandbox_state0.get('install_command', "") | |
sandbox_url0, sandbox_output0, sandbox_error0 = run_sandbox_code(sandbox_state0, code0, install_command0) | |
if sandbox_url0: | |
sandbox_component_update0 = gr.update( | |
value=(sandbox_url0, True, []), | |
visible=True, | |
key=f"sandbox_a_fallback_{int(time.time() * 1000)}" | |
) | |
if sandbox_output0: | |
sandbox_view_a += sandbox_output0 | |
if sandbox_error0: | |
sandbox_view_a = f"<details closed><summary><strong>🚨 Errors/Warnings</strong></summary>\n\n```\n{sandbox_error0}\n```\n\n</details>\n\n" + sandbox_view_a | |
if code1.strip(): | |
install_command1 = sandbox_state1.get('install_command', "") | |
sandbox_url1, sandbox_output1, sandbox_error1 = run_sandbox_code(sandbox_state1, code1, install_command1) | |
if sandbox_url1: | |
sandbox_component_update1 = gr.update( | |
value=(sandbox_url1, True, []), | |
visible=True, | |
key=f"sandbox_b_fallback_{int(time.time() * 1000)}" | |
) | |
if sandbox_output1: | |
sandbox_view_b += f"## Output\n{sandbox_output1}" | |
if sandbox_error1: | |
sandbox_view_b = f"<details closed><summary><strong>🚨 Errors/Warnings</strong></summary>\n\n```\n{sandbox_error1}\n```\n\n</details>\n\n" + sandbox_view_b | |
sandbox_time = time.time() - sandbox_start_time | |
finally: | |
loop.close() | |
else: | |
# No code to execute, but still ensure sandbox components are cleared | |
sandbox_component_update0 = gr.update(value=("", False, []), visible=False) | |
sandbox_component_update1 = gr.update(value=("", False, []), visible=False) | |
# Calculate conversation statistics | |
turn_count_a = ( | |
len( | |
[ | |
msg | |
for msg in state0["messages"] | |
if msg["role"] == "assistant" and msg["content"] | |
] | |
) | |
if state0 | |
else 0 | |
) | |
turn_count_b = ( | |
len( | |
[ | |
msg | |
for msg in state1["messages"] | |
if msg["role"] == "assistant" and msg["content"] | |
] | |
) | |
if state1 | |
else 0 | |
) | |
# Format conversation statistics | |
chat_stats_a = f"**Conversation:** {turn_count_a} turns | **Total Messages:** {len(state0['messages']) if state0 else 0}" | |
chat_stats_b = f"**Conversation:** {turn_count_b} turns | **Total Messages:** {len(state1['messages']) if state1 else 0}" | |
# Get install commands from sandbox states | |
install_command0 = sandbox_state0.get('install_command', '') if sandbox_state0 else '' | |
install_command1 = sandbox_state1.get('install_command', '') if sandbox_state1 else '' | |
return state0, state1, chat0, chat1, response0, response1, code0, code1, env0, env1, sandbox_state0, sandbox_state1, sandbox_output0, sandbox_output1, sandbox_component_update0, sandbox_component_update1, chat_stats_a, chat_stats_b, sandbox_view_a, sandbox_view_b, install_command0, install_command1 | |
def format_chat_history(messages): | |
"""Format messages for chat display with turn numbers""" | |
formatted = [] | |
for msg in messages: | |
if msg["role"] == "user" and msg["content"]: | |
# Add turn number to user messages | |
formatted.append({ | |
"role": "user", | |
"content": msg['content'] | |
}) | |
elif msg["role"] == "assistant" and msg["content"]: | |
# Add turn number to assistant messages | |
formatted.append({ | |
"role": "assistant", | |
"content": msg['content'] | |
}) | |
return formatted | |
def clear_chat(state0, state1): | |
"""Clear chat history""" | |
if state0 and "sandbox_state" in state0: | |
reset_sandbox_state(state0["sandbox_state"]) | |
state0["interactions"] = [] # Clear interactions | |
state0["generating"] = False # Reset generating flag | |
if state1 and "sandbox_state" in state1: | |
reset_sandbox_state(state1["sandbox_state"]) | |
state1["interactions"] = [] # Clear interactions | |
state1["generating"] = False # Reset generating flag | |
# Get current model names for display | |
model_a, model_b = get_random_models() | |
print(f"Model A: {model_a}, Model B: {model_b}") | |
return ( | |
None, # state0 | |
None, # state1 | |
"", # chatbot_a | |
"", # chatbot_b | |
"", # response_a | |
"", # response_b | |
"", # code_a | |
"", # code_b | |
None, # sandbox_state0 | |
None, # sandbox_state1 | |
"", # sandbox_view_a | |
"", # sandbox_view_b | |
gr.update(value=("", False, []), visible=False), # sandbox_component_a | |
gr.update(value=("", False, []), visible=False), # sandbox_component_b | |
"**Conversation:** 0 turns | **Total Messages:** 0", # chat_stats_a | |
"**Conversation:** 0 turns | **Total Messages:** 0", # chat_stats_b | |
"", # sandbox_view_a (duplicate) | |
"", # sandbox_view_b (duplicate) | |
"", # install_command_a | |
"", # install_command_b | |
f"**Model A:** {model_a}", # model_display_a | |
f"**Model B:** {model_b}", # model_display_b | |
"", # text_input | |
gr.update(visible=False), # vote_section | |
gr.update(visible=False), # vote_buttons_row | |
"", # vote_status | |
gr.update(interactive=False), # vote_left_btn | |
gr.update(interactive=False), # vote_right_btn | |
gr.update(interactive=False), # vote_tie_btn | |
gr.update(interactive=False), # vote_both_bad_btn | |
) | |
def retry_last_message(state0, state1, model_a, model_b): | |
"""Retry the last user message""" | |
if not state0 or not state1: | |
return state0, state1, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" | |
# Get the last user message | |
last_user_message = "" | |
for msg in reversed(state0["messages"]): | |
if msg["role"] == "user": | |
last_user_message = msg["content"] | |
break | |
if not last_user_message: | |
return state0, state1, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" | |
# Remove the last user message and assistant responses from both states | |
if state0["messages"] and state0["messages"][-1]["role"] == "assistant": | |
state0["messages"].pop() # Remove last assistant response | |
if state0["messages"] and state0["messages"][-1]["role"] == "user": | |
state0["messages"].pop() # Remove last user message | |
if state1["messages"] and state1["messages"][-1]["role"] == "assistant": | |
state1["messages"].pop() # Remove last assistant response | |
if state1["messages"] and state1["messages"][-1]["role"] == "user": | |
state1["messages"].pop() # Remove last user message | |
# Generate new responses with the same message | |
result = add_text_and_generate(state0, state1, last_user_message, 0.4, 8192, model_a, model_b) | |
# Extract the state from the result | |
new_state0, new_state1 = result[0], result[1] | |
# Check if both models have output and are not generating to show vote buttons | |
show_vote_buttons = should_show_vote_buttons(new_state0, new_state1) | |
# Return all the original outputs plus the updated state for run buttons | |
return ( | |
new_state0, # state0 | |
new_state1, # state1 | |
result[2], # chatbot_a (chat0) | |
result[3], # chatbot_b (chat1) | |
( | |
result[4]["content"] if isinstance(result[4], dict) else result[4] | |
), # response_a (response0) | |
( | |
result[5]["content"] if isinstance(result[5], dict) else result[5] | |
), # response_b (response1) | |
result[6], # code_a (code0) | |
result[7], # code_b (code1) | |
result[10] if len(result) > 10 else "", # sandbox_state0 | |
result[11] if len(result) > 11 else "", # sandbox_state1 | |
result[12] if len(result) > 12 else "", # sandbox_output0 | |
result[13] if len(result) > 13 else "", # sandbox_output1 | |
( | |
result[14] if len(result) > 14 else gr.update(visible=False) | |
), # sandbox_component_update0 | |
( | |
result[15] if len(result) > 15 else gr.update(visible=False) | |
), # sandbox_component_update1 | |
( | |
result[16] if len(result) > 16 else "**Conversation:** 0 turns" | |
), # chat_stats_a | |
( | |
result[17] if len(result) > 17 else "**Conversation:** 0 turns" | |
), # chat_stats_b | |
result[18] if len(result) > 18 else "", # sandbox_view_a | |
result[19] if len(result) > 19 else "", # sandbox_view_b | |
new_state0, # state0_var | |
new_state1, # state1_var | |
last_user_message, # Keep original text input | |
f"**Model A:** {model_a}", # Update model display A | |
f"**Model B:** {model_b}", # Update model display B | |
gr.update(visible=show_vote_buttons), # vote_section | |
gr.update(visible=show_vote_buttons), # vote_buttons_row | |
gr.update(visible=False), # vote_status | |
gr.update(interactive=show_vote_buttons), # vote_left_btn | |
gr.update(interactive=show_vote_buttons), # vote_right_btn | |
gr.update(interactive=show_vote_buttons), # vote_tie_btn | |
gr.update(interactive=show_vote_buttons), # vote_both_bad_btn | |
) | |
def send_to_left_only(state0, state1, text, temperature, max_tokens, model_a, model_b): | |
"""Send message to left model (Model A) only""" | |
if not text.strip(): | |
return state0, state1, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" | |
# Initialize states if needed | |
if state0 is None: | |
state0 = create_chat_state(model_a) | |
if state1 is None: | |
state1 = create_chat_state(model_b) | |
# Add user message to left state only | |
state0["messages"].append({"role": "user", "content": text}) | |
state0["generating"] = True | |
# Generate response for left model only | |
state0, response0 = generate_response_with_completion(state0, temperature, max_tokens) | |
state0["messages"].append({"role": "assistant", "content": response0["content"]}) | |
state0["has_output"] = True | |
state0["generating"] = False | |
# Format chat history for display | |
chat0 = format_chat_history(state0["messages"]) | |
chat1 = format_chat_history(state1["messages"]) if state1 else [] | |
# Extract code from response for sandbox | |
sandbox_state0 = state0.get("sandbox_state", create_sandbox_state()) | |
sandbox_state0, code0, env0 = extract_and_execute_code(response0["content"], sandbox_state0) | |
state0["sandbox_state"] = sandbox_state0 | |
# Clear previous sandbox outputs | |
sandbox_output0 = "" | |
sandbox_component_update0 = gr.update(value=("", False, []), visible=False) | |
sandbox_view_a = "" | |
# Run sandbox execution if there's code | |
if code0.strip(): | |
install_command0 = sandbox_state0.get('install_command', "") | |
sandbox_url0, sandbox_output0, sandbox_error0 = run_sandbox_code(sandbox_state0, code0, install_command0) | |
if sandbox_url0: | |
sandbox_component_update0 = gr.update( | |
value=(sandbox_url0, True, []), | |
visible=True, | |
key=f"sandbox_a_{int(time.time() * 1000)}" | |
) | |
if sandbox_output0: | |
sandbox_view_a += f"# Output\n{sandbox_output0}" | |
if sandbox_error0: | |
sandbox_view_a = f"<details closed><summary><strong>🚨 Errors/Warnings</strong></summary>\n\n```\n{sandbox_error0.strip()}\n```\n\n</details>\n\n" + sandbox_view_a | |
# Calculate conversation statistics | |
turn_count_a = len([msg for msg in state0["messages"] if msg["role"] == "assistant" and msg["content"]]) | |
turn_count_b = len([msg for msg in state1["messages"] if msg["role"] == "assistant" and msg["content"]]) if state1 else 0 | |
chat_stats_a = f"**Conversation:** {turn_count_a} turns | **Total Messages:** {len(state0['messages'])}" | |
chat_stats_b = f"**Conversation:** {turn_count_b} turns | **Total Messages:** {len(state1['messages']) if state1 else 0}" | |
# Don't show vote buttons since only one model responded | |
show_vote_buttons = False | |
return ( | |
state0, # state0 | |
state1, # state1 | |
chat0, # chatbot_a | |
chat1, # chatbot_b | |
( | |
response0["content"] if isinstance(response0, dict) else response0 | |
), # response_a | |
"", # response_b (empty) | |
code0, # code_a | |
"", # code_b (empty) | |
sandbox_state0, # sandbox_state0 | |
( | |
state1.get("sandbox_state", create_sandbox_state()) | |
if state1 | |
else create_sandbox_state() | |
), # sandbox_state1 | |
sandbox_output0, # sandbox_output0 | |
"", # sandbox_output1 (empty) | |
sandbox_component_update0, # sandbox_component_update0 | |
gr.update(value=("", False, []), visible=False), # sandbox_component_update1 | |
chat_stats_a, # chat_stats_a | |
chat_stats_b, # chat_stats_b | |
sandbox_view_a, # sandbox_view_a | |
"", # sandbox_view_b (empty) | |
state0, # state0_var | |
state1, # state1_var | |
state0.get('install_command', ''), # state0_install_command | |
state1.get('install_command', ''), # state1_install_command | |
text, # Keep original text input | |
f"**Model A:** {model_a}", # Update model display A | |
f"**Model B:** {model_b}", # Update model display B | |
gr.update(visible=show_vote_buttons), # vote_section | |
gr.update(visible=show_vote_buttons), # vote_buttons_row | |
gr.update(visible=False), # vote_status | |
gr.update(interactive=show_vote_buttons), # vote_left_btn | |
gr.update(interactive=show_vote_buttons), # vote_right_btn | |
gr.update(interactive=show_vote_buttons), # vote_tie_btn | |
gr.update(interactive=show_vote_buttons), # vote_both_bad_btn | |
) | |
def send_to_right_only(state0, state1, text, temperature, max_tokens, model_a, model_b): | |
"""Send message to right model (Model B) only""" | |
if not text.strip(): | |
return state0, state1, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" | |
# Initialize states if needed | |
if state0 is None: | |
state0 = create_chat_state(model_a) | |
if state1 is None: | |
state1 = create_chat_state(model_b) | |
# Add user message to right state only | |
state1["messages"].append({"role": "user", "content": text}) | |
state1["generating"] = True | |
# Generate response for right model only | |
state1, response1 = generate_response_with_completion(state1, temperature, max_tokens) | |
state1["messages"].append({"role": "assistant", "content": response1["content"]}) | |
state1["has_output"] = True | |
state1["generating"] = False | |
# Format chat history for display | |
chat0 = format_chat_history(state0["messages"]) if state0 else [] | |
chat1 = format_chat_history(state1["messages"]) | |
# Extract code from response for sandbox | |
sandbox_state1 = state1.get("sandbox_state", create_sandbox_state()) | |
sandbox_state1, code1, env1 = extract_and_execute_code(response1["content"], sandbox_state1) | |
state1["sandbox_state"] = sandbox_state1 | |
# Clear previous sandbox outputs | |
sandbox_output1 = "" | |
sandbox_component_update1 = gr.update(value=("", False, []), visible=False) | |
sandbox_view_b = "" | |
# Run sandbox execution if there's code | |
if code1.strip(): | |
install_command1 = sandbox_state1.get('install_command', "") | |
sandbox_url1, sandbox_output1, sandbox_error1 = run_sandbox_code(sandbox_state1, code1, install_command1) | |
if sandbox_url1: | |
sandbox_component_update1 = gr.update( | |
value=(sandbox_url1, True, []), | |
visible=True, | |
key=f"sandbox_b_{int(time.time() * 1000)}" | |
) | |
if sandbox_output1: | |
sandbox_view_b += f"# Output\n{sandbox_output1}" | |
if sandbox_error1: | |
sandbox_view_b = f"<details closed><summary><strong>🚨 Errors/Warnings</strong></summary>\n\n```\n{sandbox_error1.strip()}\n```\n\n</details>\n\n" + sandbox_view_b | |
# Calculate conversation statistics | |
turn_count_a = len([msg for msg in state0["messages"] if msg["role"] == "assistant" and msg["content"]]) if state0 else 0 | |
turn_count_b = len([msg for msg in state1["messages"] if msg["role"] == "assistant" and msg["content"]]) | |
chat_stats_a = f"**Conversation:** {turn_count_a} turns | **Total Messages:** {len(state0['messages']) if state0 else 0}" | |
chat_stats_b = f"**Conversation:** {turn_count_b} turns | **Total Messages:** {len(state1['messages'])}" | |
# Don't show vote buttons since only one model responded | |
show_vote_buttons = False | |
return ( | |
state0, # state0 | |
state1, # state1 | |
chat0, # chatbot_a | |
chat1, # chatbot_b | |
"", # response_a (empty) | |
( | |
response1["content"] if isinstance(response1, dict) else response1 | |
), # response_b | |
"", # code_a (empty) | |
code1, # code_b | |
( | |
state0.get("sandbox_state", create_sandbox_state()) | |
if state0 | |
else create_sandbox_state() | |
), # sandbox_state0 | |
sandbox_state1, # sandbox_state1 | |
"", # sandbox_output0 (empty) | |
sandbox_output1, # sandbox_output1 | |
gr.update(value=("", False, []), visible=False), # sandbox_component_update0 | |
sandbox_component_update1, # sandbox_component_update1 | |
chat_stats_a, # chat_stats_a | |
chat_stats_b, # chat_stats_b | |
"", # sandbox_view_a (empty) | |
sandbox_view_b, # sandbox_view_b | |
state0, # state0_var | |
state1, # state1_var | |
state0.get('install_command', ''), # state0_install_command | |
state1.get('install_command', ''), # state1_install_command | |
text, # Keep original text input | |
f"**Model A:** {model_a}", # Update model display A | |
f"**Model B:** {model_b}", # Update model display B | |
gr.update(visible=show_vote_buttons), # vote_section | |
gr.update(visible=show_vote_buttons), # vote_buttons_row | |
gr.update(visible=False), # vote_status | |
gr.update(interactive=show_vote_buttons), # vote_left_btn | |
gr.update(interactive=show_vote_buttons), # vote_right_btn | |
gr.update(interactive=show_vote_buttons), # vote_tie_btn | |
gr.update(interactive=show_vote_buttons), # vote_both_bad_btn | |
) | |
def rerun_code_execution(state, current_code: str, current_install_command: str, model_name: str) -> tuple[dict, str, str, str]: | |
"""Re-run code execution for a specific model using the current code and install command from the UI components""" | |
if not state or not state.get("sandbox_state"): | |
return state, "", "", "" | |
sandbox_state = state["sandbox_state"] | |
if not current_code.strip(): | |
return state, "", "", "No code to re-run" | |
# Update the sandbox state with the current code and install command | |
sandbox_state['code_to_execute'] = current_code | |
sandbox_state['install_command'] = current_install_command | |
# Re-run the code execution with the updated code and install command | |
sandbox_url, sandbox_output, sandbox_error = run_sandbox_code(sandbox_state, current_code, current_install_command) | |
# Update sandbox view with new output | |
sandbox_view = "" | |
if sandbox_output: | |
sandbox_view += sandbox_output | |
if sandbox_error: | |
sandbox_view = f"<details closed><summary><strong>🚨 Errors/Warnings</strong></summary>\n\n```\n{sandbox_error}\n```\n\n</details>\n\n" + sandbox_view | |
# Update sandbox component if we have a URL | |
sandbox_component_update = None | |
if sandbox_url: | |
sandbox_component_update = gr.update( | |
value=(sandbox_url, True, []), | |
visible=True, | |
key=f"sandbox_{model_name.lower()}_{int(time.time() * 1000)}" # Unique key to force refresh | |
) | |
return state, sandbox_view, sandbox_component_update, sandbox_error | |
def run_sandbox_code(sandbox_state: dict, code: str, install_command: str) -> tuple[str, str, str]: | |
"""Run code in the appropriate sandbox environment""" | |
if not code.strip(): | |
return "", "", "No code to run" | |
# Update sandbox state | |
sandbox_state['code_to_execute'] = code | |
sandbox_state['install_command'] = install_command | |
# Determine environment | |
env = sandbox_state.get('auto_selected_sandbox_environment') or sandbox_state.get('sandbox_environment') | |
try: | |
if env == SandboxEnvironment.HTML: | |
sandbox_url, sandbox_id, stderr = run_html_sandbox(code, install_command, sandbox_state.get('sandbox_id')) | |
sandbox_state['sandbox_id'] = sandbox_id | |
return sandbox_url, "", stderr | |
elif env == SandboxEnvironment.REACT: | |
result = run_react_sandbox(code, install_command, sandbox_state.get('sandbox_id')) | |
sandbox_state['sandbox_id'] = result['sandbox_id'] | |
return result['sandbox_url'], "", result['stderr'] | |
elif env == SandboxEnvironment.VUE: | |
result = run_vue_sandbox(code, install_command, sandbox_state.get('sandbox_id')) | |
sandbox_state['sandbox_id'] = result['sandbox_id'] | |
return result['sandbox_url'], "", result['stderr'] | |
elif env == SandboxEnvironment.PYGAME: | |
result = run_pygame_sandbox(code, install_command, sandbox_state.get('sandbox_id')) | |
sandbox_state['sandbox_id'] = result['sandbox_id'] | |
return result['sandbox_url'], "", result['stderr'] | |
elif env == SandboxEnvironment.GRADIO: | |
sandbox_url, sandbox_id, stderr = run_gradio_sandbox(code, install_command, sandbox_state.get('sandbox_id')) | |
sandbox_state['sandbox_id'] = sandbox_id | |
return sandbox_url, "", stderr | |
elif env == SandboxEnvironment.STREAMLIT: | |
sandbox_url, sandbox_id, stderr = run_streamlit_sandbox(code, install_command, sandbox_state.get('sandbox_id')) | |
sandbox_state['sandbox_id'] = sandbox_id | |
return sandbox_url, "", stderr | |
elif env == SandboxEnvironment.MERMAID: | |
# Convert Mermaid to HTML and run in HTML sandbox | |
html_code = mermaid_to_html(code, theme='light') | |
sandbox_url, sandbox_id, stderr = run_html_sandbox(html_code, install_command, sandbox_state.get('sandbox_id')) | |
sandbox_state['sandbox_id'] = sandbox_id | |
return sandbox_url, "", stderr | |
elif env == SandboxEnvironment.PYTHON_RUNNER: | |
output, stderr = run_code_interpreter(code, 'python', install_command) | |
return "", output, stderr | |
elif env == SandboxEnvironment.JAVASCRIPT_RUNNER: | |
html_code = javascript_to_html(code) | |
sandbox_url, sandbox_id, stderr = run_html_sandbox(html_code, install_command, sandbox_state.get('sandbox_id')) | |
sandbox_state['sandbox_id'] = sandbox_id | |
return sandbox_url, "", stderr | |
elif env == SandboxEnvironment.C_RUNNER: | |
output, stderr = run_c_code(code, sandbox_state.get('sandbox_id')) | |
return "", output, stderr | |
elif env == SandboxEnvironment.CPP_RUNNER: | |
output, stderr = run_cpp_code(code, sandbox_state.get('sandbox_id')) | |
return "", output, stderr | |
elif env == SandboxEnvironment.JAVA_RUNNER: | |
output, stderr = run_java_code(code, sandbox_state.get('sandbox_id')) | |
return "", output, stderr | |
elif env == SandboxEnvironment.GOLANG_RUNNER: | |
output, stderr = run_golang_code(code, sandbox_state.get('sandbox_id')) | |
return "", output, stderr | |
elif env == SandboxEnvironment.RUST_RUNNER: | |
output, stderr = run_rust_code(code, sandbox_state.get('sandbox_id')) | |
return "", output, stderr | |
else: | |
# Fallback to Python runner | |
output, stderr = run_code_interpreter(code, 'python', install_command) | |
return "", output, stderr | |
except Exception as e: | |
return "", "", str(e) | |
async def run_sandbox_code_async(sandbox_state: dict, code: str, install_command: str) -> tuple[str, str, str]: | |
"""Async wrapper for run_sandbox_code""" | |
loop = asyncio.get_event_loop() | |
# Run sandbox execution in a thread pool to avoid blocking | |
with concurrent.futures.ThreadPoolExecutor() as executor: | |
future = loop.run_in_executor(executor, run_sandbox_code, sandbox_state, code, install_command) | |
return await future | |
async def run_sandboxes_parallel(sandbox_state0, code0, install_command0, sandbox_state1, code1, install_command1): | |
"""Run both sandbox executions in parallel with error handling""" | |
loop = asyncio.get_event_loop() | |
# Create tasks for both sandbox executions | |
task0 = loop.run_in_executor(None, run_sandbox_code, sandbox_state0, code0, install_command0) | |
task1 = loop.run_in_executor(None, run_sandbox_code, sandbox_state1, code1, install_command1) | |
# Wait for both to complete with error handling | |
try: | |
result0, result1 = await asyncio.gather(task0, task1, return_exceptions=True) | |
# Handle exceptions | |
if isinstance(result0, Exception): | |
result0 = ("", "", f"Sandbox execution error: {str(result0)}") | |
if isinstance(result1, Exception): | |
result1 = ("", "", f"Sandbox execution error: {str(result1)}") | |
except Exception as e: | |
# Fallback to sequential processing | |
result0 = run_sandbox_code(sandbox_state0, code0, install_command0) | |
result1 = run_sandbox_code(sandbox_state1, code1, install_command1) | |
return result0, result1 | |
def instantiate_send_button(): | |
"""Create a send button with icon""" | |
return gr.Button( | |
"🚀", | |
size="lg", | |
scale=0, | |
min_width=60, | |
variant="primary", | |
elem_id="send-btn" | |
) | |
def instantiate_retry_button(): | |
"""Create a retry button with icon""" | |
return gr.Button( | |
"🔄", | |
size="lg", | |
scale=0, | |
min_width=60, | |
variant="secondary", | |
elem_id="retry-btn" | |
) | |
def instantiate_send_left_button(): | |
"""Create a send left button with icon""" | |
return gr.Button( | |
"⬅️", | |
size="lg", | |
scale=0, | |
min_width=60, | |
variant="secondary", | |
elem_id="send-left-btn" | |
) | |
def instantiate_send_right_button(): | |
"""Create a send right button with icon""" | |
return gr.Button( | |
"➡️", | |
size="lg", | |
scale=0, | |
min_width=60, | |
variant="secondary", | |
elem_id="send-right-btn" | |
) | |
def instantiate_clear_button(): | |
"""Create a clear button with icon""" | |
return gr.Button( | |
"🗑️", | |
size="sm", | |
scale=0, | |
min_width=40, | |
variant="secondary", | |
elem_id="clear-btn" | |
) | |
def build_ui(): | |
"""Build a UI for the coding arena with integrated sandbox""" | |
with gr.Blocks(title="BigCodeArena", theme=gr.themes.Soft()) as demo: | |
# Add custom CSS for centering and button styling | |
demo.css = """ | |
.center-text { | |
text-align: center !important; | |
} | |
.input-row { | |
display: flex; | |
align-items: center; | |
gap: 12px; | |
} | |
.input-row .gr-textbox { | |
flex: 1; | |
} | |
.input-row .gr-button { | |
flex-shrink: 0; | |
height: 40px; | |
font-size: 16px; | |
} | |
.button-grid { | |
display: flex; | |
flex-direction: column; | |
gap: 8px; | |
} | |
.button-grid .gr-row { | |
display: flex; | |
gap: 8px; | |
} | |
.button-grid .gr-button { | |
flex: 1; | |
min-width: 60px; | |
} | |
""" | |
gr.Markdown("# 🌸 BigCodeArena - Start Your Vibe Coding!", elem_classes="center-text") | |
gr.HTML(""" | |
<p align="center" style="font-size: 1.2em;"> | |
<a href="https://github.com/bigcode-project/bigcodearena">GitHub</a> | | |
<a href="https://huggingface.co/collections/bigcode/bigcodearena-68cd3a196e5147cc45f8ea3d">HF Collection</a> | | |
<a href="https://arxiv.org/abs/2510.08697">Arxiv</a> | |
</p> | |
""") | |
# Main tabs | |
with gr.Tabs(): | |
# Arena Tab | |
with gr.Tab("🥊 Arena", id="arena"): | |
# Model display (non-interactive) | |
with gr.Row(): | |
with gr.Column(): | |
model_display_a = gr.Markdown( | |
"**Model A:** Loading...", visible=False | |
) | |
with gr.Column(): | |
model_display_b = gr.Markdown( | |
"**Model B:** Loading...", visible=False | |
) | |
# Sandbox section with tabs for each model - Collapsible and open by default | |
with gr.Accordion("🏗️ Code Execution & Sandbox", open=True): | |
with gr.Row(): | |
# Model A Sandbox | |
with gr.Column(): | |
gr.Markdown("### Model A Sandbox") | |
with gr.Tabs() as tabs_a: | |
with gr.Tab("View", id=0): | |
sandbox_view_a = gr.Markdown( | |
"**Sandbox output will appear here automatically**" | |
) | |
sandbox_component_a = SandboxComponent( | |
value=("", False, []), | |
label="Model A Sandbox", | |
visible=False, | |
) | |
with gr.Tab("Code", id=1): | |
code_a = gr.Code( | |
label="Code Editor", | |
language="python", | |
lines=8, | |
interactive=True, | |
) | |
install_command_a = gr.Textbox( | |
label="Install Command", | |
placeholder="bash command to install dependencies", | |
interactive=True, | |
lines=1, | |
) | |
rerun_code_a_btn = gr.Button( | |
"🔄 Re-run Code", | |
) | |
# Model B Sandbox | |
with gr.Column(): | |
gr.Markdown("### Model B Sandbox") | |
with gr.Tabs() as tabs_b: | |
with gr.Tab("View", id=2): | |
sandbox_view_b = gr.Markdown( | |
"**Sandbox output will appear here automatically**" | |
) | |
sandbox_component_b = SandboxComponent( | |
value=("", False, []), | |
label="Model B Sandbox", | |
visible=False, | |
) | |
with gr.Tab("Code", id=3): | |
code_b = gr.Code( | |
label="Code Editor", | |
language="python", | |
lines=8, | |
interactive=True, | |
) | |
install_command_b = gr.Textbox( | |
label="Install Command", | |
placeholder="bash command to install dependencies", | |
interactive=True, | |
lines=1, | |
) | |
rerun_code_b_btn = gr.Button( | |
"🔄 Re-run Code", | |
) | |
# Vote UI components | |
vote_components = create_vote_ui() | |
vote_section = vote_components["vote_section"] | |
vote_buttons_row = vote_components["vote_buttons_row"] | |
vote_left_btn = vote_components["vote_left_btn"] | |
vote_right_btn = vote_components["vote_right_btn"] | |
vote_tie_btn = vote_components["vote_tie_btn"] | |
vote_both_bad_btn = vote_components["vote_both_bad_btn"] | |
vote_status = vote_components["vote_status"] | |
# Main chat interface - Collapsible and hidden by default | |
with gr.Accordion("💬 Chat Interface", open=False): | |
with gr.Row(): | |
with gr.Column(): | |
gr.Markdown("## Model A") | |
chatbot_a = gr.Chatbot( | |
label="Model A", | |
height=300, | |
show_copy_button=True, | |
type="messages", | |
) | |
chat_stats_a = gr.Markdown("**Conversation:** 0 turns") | |
with gr.Column(): | |
gr.Markdown("## Model B") | |
chatbot_b = gr.Chatbot( | |
label="Model B", | |
height=300, | |
show_copy_button=True, | |
type="messages", | |
) | |
chat_stats_b = gr.Markdown("**Conversation:** 0 turns") | |
# Input section with 2x2 button grid | |
with gr.Row(elem_classes="input-row"): | |
text_input = gr.Textbox( | |
label="Enter your coding prompt", | |
placeholder="e.g., 'Write a Python function to calculate fibonacci numbers'", | |
lines=1, | |
scale=1 | |
) | |
with gr.Column(scale=0, min_width=140, elem_classes="button-grid"): | |
with gr.Row(): | |
send_btn = instantiate_send_button() | |
retry_btn = instantiate_retry_button() | |
with gr.Row(): | |
send_left_btn = instantiate_send_left_button() | |
send_right_btn = instantiate_send_right_button() | |
# Additional control buttons | |
with gr.Row(): | |
clear_btn = gr.Button("🗑️ Clear Chat", variant="secondary") | |
refresh_models_btn = gr.Button( | |
"🔄 New Random Models", variant="secondary" | |
) | |
# Advanced Settings (Collapsible) | |
with gr.Accordion("⚙️ Advanced Settings", open=False): | |
with gr.Row(): | |
with gr.Column(scale=1): | |
temperature = gr.Slider( | |
minimum=0.0, | |
maximum=1.0, | |
value=0.4, | |
step=0.1, | |
label="Temperature", | |
) | |
with gr.Column(scale=1): | |
max_tokens = gr.Slider( | |
minimum=1024, | |
maximum=32768, | |
value=8192, | |
label="Max Tokens", | |
) | |
# Examples | |
gr.Examples( | |
examples=[ | |
[ | |
"使用SVG绘制春节主题的动态图案,包括:1)一个红色的灯笼,带有金色的流苏 2)一个金色的福字,使用书法字体 3)背景添加一些烟花效果 4)在灯笼和福字周围添加一些祥云图案。确保图案布局美观,颜色搭配符合春节传统风格。" | |
], | |
[ | |
"SVGを使用して日本の伝統的な和柄パターンを描画してください。1)波紋(さざなみ)模様 2)市松模様 3)麻の葉模様 4)雷文(らいもん)模様を含めてください。色は伝統的な日本の色(藍色、朱色、金色など)を使用し、レイアウトはバランスよく配置してください。" | |
], | |
[ | |
"Write HTML with P5.js that simulates 25 particles in a vacuum space of a cylindrical container, bouncing within its boundaries. Use different colors for each ball and ensure they leave a trail showing their movement. Add a slow rotation of the container to give better view of what's going on in the scene. Make sure to create proper collision detection and physic rules to ensure particles remain in the container. Add an external spherical container. Add a slow zoom in and zoom out effect to the whole scene." | |
], | |
[ | |
"Write a Python script to scrape NVIDIA's stock price for the past month using the yfinance library. Clean the data and create an interactive visualization using Matplotlib. Include: 1) A candlestick chart showing daily price movements 2) A line chart with 7-day and 30-day moving averages. Add hover tooltips showing exact values and date. Make the layout professional with proper titles and axis labels." | |
], | |
[ | |
"Write a Python script that uses the Gradio library to create a chatbot that can have conversations with users. The chatbot should maintain conversation history, display messages in a chat interface, and respond to user inputs. Include a text input field for users to type messages and a chatbot component to display the conversation. The bot should give helpful responses and remember the context of the conversation." | |
], | |
[ | |
"Write a Todo list app using React.js. The app should allow users to add, delete, and mark tasks as completed. Include features like filtering tasks by status (completed, active), sorting tasks by priority, and displaying the total number of tasks." | |
], | |
[ | |
"Write a Python script using the Streamlit library to create a web application for uploading and displaying files. The app should allow users to upload files of type .csv or .txt. If a .csv file is uploaded, display its contents as a table using Streamlit's st.dataframe() method. If a .txt file is uploaded, display its content as plain text." | |
], | |
[ | |
"Write a Python function to solve the Trapping Rain Water problem. The function should take a list of non-negative integers representing the height of bars in a histogram and return the total amount of water trapped between the bars after raining. Use an efficient algorithm with a time complexity of O(n)." | |
], | |
[ | |
"Create a simple Pygame script for a game where the player controls a bouncing ball that changes direction when it collides with the edges of the window. Add functionality for the player to control a paddle using arrow keys, aiming to keep the ball from touching the bottom of the screen. Include basic collision detection and a scoring system that increases as the ball bounces off the paddle. You need to add clickable buttons to start the game, and reset the game." | |
], | |
[ | |
"Create a financial management Dashboard using Vue.js, focusing on local data handling without APIs. Include features like a clean dashboard for tracking income and expenses, dynamic charts for visualizing finances, and a budget planner. Implement functionalities for adding, editing, and deleting transactions, as well as filtering by date or category. Ensure responsive design and smooth user interaction for an intuitive experience." | |
], | |
[ | |
"Create a Mermaid diagram to visualize a flowchart of a user login process. Include the following steps: User enters login credentials; Credentials are validated; If valid, the user is directed to the dashboard; If invalid, an error message is shown, and the user can retry or reset the password." | |
], | |
[ | |
"Write a Python function to calculate the Fibonacci sequence up to n numbers. Then write test cases to verify the function works correctly for edge cases like negative numbers, zero, and large inputs." | |
], | |
[ | |
"Build an HTML page for a Kanban board with three columns with Vue.js: To Do, In Progress, and Done. Each column should allow adding, moving, and deleting tasks. Implement drag-and-drop functionality using Vue Draggable and persist the state using Vuex." | |
], | |
[ | |
"Develop a Streamlit app that takes a CSV file as input and provides: 1) Basic statistics about the data 2) Interactive visualizations using Plotly 3) A data cleaning interface with options to handle missing values 4) An option to download the cleaned data." | |
], | |
[ | |
"Write an HTML page with embedded JavaScript that creates an interactive periodic table. Each element should display its properties on hover and allow filtering by category (metals, non-metals, etc.). Include a search bar to find elements by name or symbol." | |
], | |
[ | |
"Here's a Python function that sorts a list of dictionaries by a specified key:\n\n```python\ndef sort_dicts(data, key):\n return sorted(data, key=lambda x: x[key])\n```\n\nWrite test cases to verify the function works correctly for edge cases like empty lists, missing keys, and different data types. If you use unittest, please use `unittest.main(argv=['first-arg-is-ignored'], exit=False)` to run the tests." | |
], | |
[ | |
"Create a React component for a fitness tracker that shows: 1) Daily step count 2) Calories burned 3) Distance walked 4) A progress bar for daily goals." | |
], | |
[ | |
"Build a Vue.js dashboard for monitoring server health. Include: 1) Real-time CPU and memory usage graphs 2) Disk space visualization 3) Network activity monitor 4) Alerts for critical thresholds." | |
], | |
[ | |
"Write a C program that calculates and prints the first 100 prime numbers in a formatted table with 10 numbers per row. Include a function to check if a number is prime and use it in your solution." | |
], | |
[ | |
"Write a C++ program that implements a simple calculator using object-oriented programming. Create a Calculator class with methods for addition, subtraction, multiplication, and division. Include error handling for division by zero." | |
], | |
[ | |
"Write a Rust program that generates and prints a Pascal's Triangle with 10 rows. Format the output to center-align the numbers in each row." | |
], | |
[ | |
"Write a Java program that simulates a simple bank account system. Create a BankAccount class with methods for deposit, withdrawal, and balance inquiry. Include error handling for insufficient funds and demonstrate its usage with a few transactions." | |
], | |
[ | |
"Write a Go program that calculates and prints the Fibonacci sequence up to the 50th number. Format the output in a table with 5 numbers per row and include the index of each Fibonacci number." | |
], | |
[ | |
"Write a C program that calculates and prints a histogram of letter frequencies from a predefined string. Use ASCII art to display the histogram vertically." | |
], | |
[ | |
"Write a C++ program that implements a simple stack data structure with push, pop, and peek operations. Demonstrate its usage by reversing a predefined string using the stack." | |
], | |
[ | |
"Write a Rust program that calculates and prints the first 20 happy numbers. Include a function to check if a number is happy and use it in your solution." | |
], | |
[ | |
"Write a Java program that implements a simple binary search algorithm. Create a sorted array of integers and demonstrate searching for different values, including cases where the value is found and not found." | |
], | |
[ | |
"Write a Go program that generates and prints a multiplication table from 1 to 12. Format the output in a neat grid with proper alignment." | |
], | |
[ | |
"Write a Python script with Gradio that applies a halftone effect to uploaded images. The app should allow users to upload an image file and convert it to a simple halftone pattern using basic image processing. Include a slider to control the dot size and display both the original and processed images. Use only built-in Python libraries and basic image manipulation techniques." | |
], | |
], | |
example_labels=[ | |
"🏮 春节主题图案", | |
"🎎 日本の伝統的な和柄パターン", | |
"🌐 Particles in a Spherical Container", | |
"💹 NVIDIA Stock Analysis with Matplotlib", | |
"💬 Chatbot with Gradio", | |
"📝 Todo List App with React.js", | |
"📂 File Upload Web App with Streamlit", | |
"💦 Solve Trapping Rain Water Problem", | |
"🎮 Pygame Bouncing Ball Game", | |
"💳 Financial Dashboard with Vue.js", | |
"🔑 User Login Process Flowchart", | |
"🔢 Fibonacci Sequence with Tests", | |
"📌 Vue Kanban Board", | |
"🧹 Streamlit Data Cleaning App", | |
"⚗️ Interactive Periodic Table with React", | |
"📚 Dictionary Sorting Tests in Python", | |
"🏋️♂️ Fitness Tracker with React", | |
"🖥️ Vue Server Monitoring", | |
"🔢 Prime Numbers in C", | |
"🧮 OOP Calculator in C++", | |
"🔷 Pascal's Triangle in Rust", | |
"🏛️ Bank Account Simulation in Java", | |
"🐰 Fibonacci Sequence in Go", | |
"📊 Letter Frequency Histogram in C", | |
"📦 Stack Implementation in C++", | |
"😄 Happy Numbers in Rust", | |
"🔎 Binary Search in Java", | |
"✖️ Multiplication Table in Go", | |
"🖨️ Halftone Image Effect with Gradio", | |
], | |
examples_per_page=100, | |
label="Example Prompts", | |
inputs=[text_input], | |
) | |
# Ranking Tab | |
ranking_table, ranking_last_update, ranking_timer = create_ranking_tab() | |
# Note Tab | |
with gr.Tab("📝 Note", id="note"): | |
gr.Markdown("## 📋 Important Information") | |
# Privacy Notice Section | |
gr.Markdown("### Privacy Notice") | |
gr.Markdown( | |
""" | |
Your conversations and certain other personal information will be disclosed to the relevant AI providers | |
and may otherwise be disclosed publicly to help support our community and advance AI research. | |
**Do not submit to our Services any personal information or other sensitive information that you would not | |
want to be shared publicly.** By continuing to use our Services, you acknowledge and direct us to engage | |
in such sharing. | |
""" | |
) | |
# Citation Section | |
gr.Markdown("### Citation") | |
gr.Markdown( | |
""" | |
If you use BigCodeArena in your research, please cite our work: | |
```bibtex | |
@article{zhuo2025bigcodearena, | |
title={BigCodeArena: Unveiling More Reliable Human Preferences in Code Generation via Execution}, | |
author={Zhuo, Terry Yue and Jin, Xiaolong and Liu, Hange and Jiang, Juyong and Liu, Tianyang and Gong, Chen and Bishnoi, Bhupesh and Mishra, Vaisakhi and Suppa, Marek and Ziems, Noah and Utpala, Saiteja and Xu, Ming and Song, Guangyu and Li, Kaixin and Cao, Yuhan and Liu, Bo and Liu, Zheng and Abdurakhmanova, Sabina and Yu, Wenhao and Jia, Mengzhao and Yao, Jihan and Hamilton, Kenneth and Shridhar, Kumar and Vu, Minh Chien and Wang, Dingmin and Liu, Jiawei and Wang, Zijian and Liu, Qian and Hui, Binyuan and Risdal, Meg and Khaliq, Ahsen and Sood, Atin and Xing, Zhenchang and Ahmad, Wasi Uddin and Grundy, John and Lo, David and Zhu, Banghua and Du, Xiaoning and Scholak, Torsten and von Werra, Leandro}, | |
year={2025} | |
} | |
``` | |
""" | |
) | |
# Event handlers | |
# Create state variables for the run buttons | |
state0_var = gr.State() | |
state1_var = gr.State() | |
# Create state variables for model names - initialized per user session | |
model_a_state = gr.State() | |
model_b_state = gr.State() | |
# Add telemetry logging for user interactions after state variables are created | |
# We need to create a wrapper function to extract the sandbox state from the main state | |
def log_telemetry_a(state0, sandbox_ui): | |
if state0 and "sandbox_state" in state0: | |
# Print user interactions for debugging | |
if sandbox_ui and len(sandbox_ui) > 2: | |
interactions = sandbox_ui[2] # Third element is user_interaction_records | |
if interactions: | |
# Store interactions in the state | |
if "interactions" not in state0: | |
state0["interactions"] = [] | |
state0["interactions"].extend(interactions) | |
return log_sandbox_telemetry_gradio_fn(state0["sandbox_state"], sandbox_ui) | |
return None | |
def log_telemetry_b(state1, sandbox_ui): | |
if state1 and "sandbox_state" in state1: | |
# Print user interactions for debugging | |
if sandbox_ui and len(sandbox_ui) > 2: | |
interactions = sandbox_ui[2] # Third element is user_interaction_records | |
if interactions: | |
# Store interactions in the state | |
if "interactions" not in state1: | |
state1["interactions"] = [] | |
state1["interactions"].extend(interactions) | |
return log_sandbox_telemetry_gradio_fn(state1["sandbox_state"], sandbox_ui) | |
return None | |
sandbox_component_a.change( | |
fn=log_telemetry_a, | |
inputs=[state0_var, sandbox_component_a], | |
) | |
sandbox_component_b.change( | |
fn=log_telemetry_b, | |
inputs=[state1_var, sandbox_component_b], | |
) | |
# Create response components (hidden but needed for outputs) | |
response_a = gr.Markdown("", visible=False) | |
response_b = gr.Markdown("", visible=False) | |
# Create a wrapper function that handles both the main execution and state update | |
def send_and_update_state(state0, state1, text, temp, max_tok, model_a, model_b): | |
# Hide vote buttons immediately when generation starts | |
initial_vote_visibility = False | |
# Call the main function | |
result = add_text_and_generate(state0, state1, text, temp, max_tok, model_a, model_b) | |
# Extract the state from the result | |
new_state0, new_state1 = result[0], result[1] | |
# Check if both models have output and are not generating to show vote buttons | |
show_vote_buttons = should_show_vote_buttons(new_state0, new_state1) | |
# Return all the original outputs plus the updated state for run buttons | |
# Make sure all outputs are properly formatted for their expected types | |
return ( | |
new_state0, # state0 | |
new_state1, # state1 | |
result[2], # chatbot_a (chat0) | |
result[3], # chatbot_b (chat1) | |
( | |
result[4]["content"] if isinstance(result[4], dict) else result[4] | |
), # response_a (response0) | |
( | |
result[5]["content"] if isinstance(result[5], dict) else result[5] | |
), # response_b (response1) | |
result[6], # code_a (code0) | |
result[7], # code_b (code1) | |
result[10] if len(result) > 10 else "", # sandbox_state0 | |
result[11] if len(result) > 11 else "", # sandbox_state1 | |
result[12] if len(result) > 12 else "", # sandbox_output0 | |
result[13] if len(result) > 13 else "", # sandbox_output1 | |
( | |
result[14] if len(result) > 14 else gr.update(visible=False) | |
), # sandbox_component_update0 | |
( | |
result[15] if len(result) > 15 else gr.update(visible=False) | |
), # sandbox_component_update1 | |
( | |
result[16] if len(result) > 16 else "**Conversation:** 0 turns" | |
), # chat_stats_a | |
( | |
result[17] if len(result) > 17 else "**Conversation:** 0 turns" | |
), # chat_stats_b | |
result[18] if len(result) > 18 else "", # sandbox_view_a | |
result[19] if len(result) > 19 else "", # sandbox_view_b | |
new_state0, # state0_var | |
new_state1, # state1_var | |
new_state0.get('install_command', ''), # state0_install_command | |
new_state1.get('install_command', ''), # state1_install_command | |
text, # Keep original text input | |
f"**Model A:** {model_a}", # Update model display A | |
f"**Model B:** {model_b}", # Update model display B | |
gr.update(visible=show_vote_buttons), # vote_section | |
gr.update(visible=show_vote_buttons), # vote_buttons_row | |
gr.update(visible=False), # vote_status | |
gr.update(interactive=show_vote_buttons), # vote_left_btn | |
gr.update(interactive=show_vote_buttons), # vote_right_btn | |
gr.update(interactive=show_vote_buttons), # vote_tie_btn | |
gr.update(interactive=show_vote_buttons), # vote_both_bad_btn | |
) | |
send_btn.click( | |
fn=send_and_update_state, | |
inputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
text_input, | |
temperature, | |
max_tokens, | |
model_a_state, # Use per-user model A | |
model_b_state, # Use per-user model B | |
], | |
outputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
chatbot_a, | |
chatbot_b, | |
response_a, | |
response_b, | |
code_a, | |
code_b, | |
gr.State(), # sandbox_state0 | |
gr.State(), # sandbox_state1 | |
sandbox_view_a, # sandbox output for model A | |
sandbox_view_b, # sandbox output for model B | |
sandbox_component_a, # sandbox component for model A | |
sandbox_component_b, # sandbox component for model B | |
chat_stats_a, # Conversation statistics for model A | |
chat_stats_b, # Conversation statistics for model B | |
sandbox_view_a, # Sandbox view for model A | |
sandbox_view_b, # Sandbox view for model B | |
state0_var, # Updated state for run button A | |
state1_var, # Updated state for run button B | |
install_command_a, # Install command for model A | |
install_command_b, # Install command for model B | |
text_input, # Clear the text input after sending | |
model_display_a, # Update model display A | |
model_display_b, # Update model display B | |
vote_section, # Show/hide vote section | |
vote_buttons_row, # Show/hide vote buttons | |
vote_status, # Vote status message | |
vote_left_btn, # vote_left_btn | |
vote_right_btn, # vote_right_btn | |
vote_tie_btn, # vote_tie_btn | |
vote_both_bad_btn, # vote_both_bad_btn | |
], | |
) | |
# Add Enter key submission support to textbox | |
text_input.submit( | |
fn=send_and_update_state, | |
inputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
text_input, | |
temperature, | |
max_tokens, | |
model_a_state, # Use per-user model A | |
model_b_state, # Use per-user model B | |
], | |
outputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
chatbot_a, | |
chatbot_b, | |
response_a, | |
response_b, | |
code_a, | |
code_b, | |
gr.State(), # sandbox_state0 | |
gr.State(), # sandbox_state1 | |
sandbox_view_a, # sandbox output for model A | |
sandbox_view_b, # sandbox output for model B | |
sandbox_component_a, # sandbox component for model A | |
sandbox_component_b, # sandbox component for model B | |
chat_stats_a, # Conversation statistics for model A | |
chat_stats_b, # Conversation statistics for model B | |
sandbox_view_a, # Sandbox view for model A | |
sandbox_view_b, # Sandbox view for model B | |
state0_var, # Updated state for run button A | |
state1_var, # Updated state for run button B | |
install_command_a, # Install command for model A | |
install_command_b, # Install command for model B | |
text_input, # Clear the text input after sending | |
model_display_a, # Update model display A | |
model_display_b, # Update model display B | |
vote_section, # Show/hide vote section | |
vote_buttons_row, # Show/hide vote buttons | |
vote_status, # Vote status message | |
vote_left_btn, # vote_left_btn | |
vote_right_btn, # vote_right_btn | |
vote_tie_btn, # vote_tie_btn | |
vote_both_bad_btn, # vote_both_bad_btn | |
], | |
) | |
# Retry button handler | |
retry_btn.click( | |
fn=retry_last_message, | |
inputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
model_a_state, # Use per-user model A | |
model_b_state, # Use per-user model B | |
], | |
outputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
chatbot_a, | |
chatbot_b, | |
response_a, | |
response_b, | |
code_a, | |
code_b, | |
gr.State(), # sandbox_state0 | |
gr.State(), # sandbox_state1 | |
sandbox_view_a, # sandbox output for model A | |
sandbox_view_b, # sandbox output for model B | |
sandbox_component_a, # sandbox component for model A | |
sandbox_component_b, # sandbox component for model B | |
chat_stats_a, # Conversation statistics for model A | |
chat_stats_b, # Conversation statistics for model B | |
sandbox_view_a, # Sandbox view for model A | |
sandbox_view_b, # Sandbox view for model B | |
state0_var, # Updated state for run button A | |
state1_var, # Updated state for run button B | |
install_command_a, # Install command for model A | |
install_command_b, # Install command for model B | |
text_input, # Clear the text input after sending | |
model_display_a, # Update model display A | |
model_display_b, # Update model display B | |
vote_section, # Show/hide vote section | |
vote_buttons_row, # Show/hide vote buttons | |
vote_status, # Vote status message | |
vote_left_btn, # vote_left_btn | |
vote_right_btn, # vote_right_btn | |
vote_tie_btn, # vote_tie_btn | |
vote_both_bad_btn, # vote_both_bad_btn | |
], | |
) | |
# Send left button handler | |
send_left_btn.click( | |
fn=send_to_left_only, | |
inputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
text_input, | |
temperature, | |
max_tokens, | |
model_a_state, # Use per-user model A | |
model_b_state, # Use per-user model B | |
], | |
outputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
chatbot_a, | |
chatbot_b, | |
response_a, | |
response_b, | |
code_a, | |
code_b, | |
gr.State(), # sandbox_state0 | |
gr.State(), # sandbox_state1 | |
sandbox_view_a, # sandbox output for model A | |
sandbox_view_b, # sandbox output for model B | |
sandbox_component_a, # sandbox component for model A | |
sandbox_component_b, # sandbox component for model B | |
chat_stats_a, # Conversation statistics for model A | |
chat_stats_b, # Conversation statistics for model B | |
sandbox_view_a, # Sandbox view for model A | |
sandbox_view_b, # Sandbox view for model B | |
state0_var, # Updated state for run button A | |
state1_var, # Updated state for run button B | |
install_command_a, # Install command for model A | |
install_command_b, # Install command for model B | |
text_input, # Clear the text input after sending | |
model_display_a, # Update model display A | |
model_display_b, # Update model display B | |
vote_section, # Show/hide vote section | |
vote_buttons_row, # Show/hide vote buttons | |
vote_status, # Vote status message | |
vote_left_btn, # vote_left_btn | |
vote_right_btn, # vote_right_btn | |
vote_tie_btn, # vote_tie_btn | |
vote_both_bad_btn, # vote_both_bad_btn | |
], | |
) | |
# Send right button handler | |
send_right_btn.click( | |
fn=send_to_right_only, | |
inputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
text_input, | |
temperature, | |
max_tokens, | |
model_a_state, # Use per-user model A | |
model_b_state, # Use per-user model B | |
], | |
outputs=[ | |
state0_var, # state0 | |
state1_var, # state1 | |
chatbot_a, | |
chatbot_b, | |
response_a, | |
response_b, | |
code_a, | |
code_b, | |
gr.State(), # sandbox_state0 | |
gr.State(), # sandbox_state1 | |
sandbox_view_a, # sandbox output for model A | |
sandbox_view_b, # sandbox output for model B | |
sandbox_component_a, # sandbox component for model A | |
sandbox_component_b, # sandbox component for model B | |
chat_stats_a, # Conversation statistics for model A | |
chat_stats_b, # Conversation statistics for model B | |
sandbox_view_a, # Sandbox view for model A | |
sandbox_view_b, # Sandbox view for model B | |
state0_var, # Updated state for run button A | |
state1_var, # Updated state for run button B | |
install_command_a, # Install command for model A | |
install_command_b, # Install command for model B | |
text_input, # Clear the text input after sending | |
model_display_a, # Update model display A | |
model_display_b, # Update model display B | |
vote_section, # Show/hide vote section | |
vote_buttons_row, # Show/hide vote buttons | |
vote_status, # Vote status message | |
vote_left_btn, # vote_left_btn | |
vote_right_btn, # vote_right_btn | |
vote_tie_btn, # vote_tie_btn | |
vote_both_bad_btn, # vote_both_bad_btn | |
], | |
) | |
clear_btn.click( | |
fn=clear_chat, | |
inputs=[state0_var, state1_var], | |
outputs=[ | |
state0_var, # Reset state0 | |
state1_var, # Reset state1 | |
chatbot_a, # Clear chatbot_a | |
chatbot_b, # Clear chatbot_b | |
response_a, # Clear response_a | |
response_b, # Clear response_b | |
code_a, # Clear code_a | |
code_b, # Clear code_b | |
gr.State(None), # Reset sandbox_state0 | |
gr.State(None), # Reset sandbox_state1 | |
sandbox_view_a, # Clear sandbox_view_a | |
sandbox_view_b, # Clear sandbox_view_b | |
sandbox_component_a, # Hide sandbox_component_a | |
sandbox_component_b, # Hide sandbox_component_b | |
chat_stats_a, # Reset conversation statistics for model A | |
chat_stats_b, # Reset conversation statistics for model B | |
sandbox_view_a, # Reset sandbox view for model A | |
sandbox_view_b, # Reset sandbox view for model B | |
state0_var, # Updated state for run button A | |
state1_var, # Updated state for run button B | |
install_command_a, # Clear install command for model A | |
install_command_b, # Clear install command for model B | |
model_display_a, # Reset model display A | |
model_display_b, # Reset model display B | |
text_input, # Clear text input | |
vote_section, # Hide vote section | |
vote_buttons_row, # Hide vote buttons | |
vote_status, # Clear vote status | |
vote_left_btn, # Disable vote buttons | |
vote_right_btn, # Disable vote buttons | |
vote_tie_btn, # Disable vote buttons | |
vote_both_bad_btn, # Disable vote buttons | |
] | |
) | |
# Refresh models button handler | |
def refresh_models(): | |
new_model_a, new_model_b = get_random_models() | |
print(f"Refresh models - Model A: {new_model_a}, Model B: {new_model_b}") | |
return ( | |
None, # Reset state0 | |
None, # Reset state1 | |
"", # Clear chat A | |
"", # Clear chat B | |
"", # Clear response A | |
"", # Clear response B | |
"", # Clear code A | |
"", # Clear code B | |
gr.State(None), # Reset sandbox state A | |
gr.State(None), # Reset sandbox state B | |
"", # Clear sandbox view A | |
"", # Clear sandbox view B | |
gr.update(visible=False), # Hide sandbox component A | |
gr.update(visible=False), # Hide sandbox component B | |
"**Conversation:** 0 turns | **Total Messages:** 0", # Reset stats A | |
"**Conversation:** 0 turns | **Total Messages:** 0", # Reset stats B | |
"", # Clear sandbox view A | |
"", # Clear sandbox view B | |
None, # Reset state0_var | |
None, # Reset state1_var | |
f"**Model A:** {new_model_a}", # Update model display A | |
f"**Model B:** {new_model_b}", # Update model display B | |
gr.update(visible=False), # Hide vote section | |
gr.update(visible=False), # Hide vote buttons | |
gr.update(visible=False), # Clear vote status | |
new_model_a, # Update model_a_state | |
new_model_b, # Update model_b_state | |
) | |
refresh_models_btn.click( | |
fn=refresh_models, | |
inputs=[], | |
outputs=[ | |
state0_var, | |
state1_var, | |
chatbot_a, | |
chatbot_b, | |
response_a, | |
response_b, | |
code_a, | |
code_b, | |
gr.State(None), | |
gr.State(None), | |
sandbox_view_a, | |
sandbox_view_b, | |
sandbox_component_a, | |
sandbox_component_b, | |
chat_stats_a, | |
chat_stats_b, | |
sandbox_view_a, | |
sandbox_view_b, | |
state0_var, | |
state1_var, | |
model_display_a, # Update model display A | |
model_display_b, # Update model display B | |
vote_section, # Hide vote section | |
vote_buttons_row, # Hide vote buttons | |
vote_status, # Clear vote status | |
model_a_state, # Update model_a_state | |
model_b_state, # Update model_b_state | |
], | |
) | |
# Setup vote handlers | |
def process_vote(state0, state1, vote_type, current_text): | |
# Save the vote and get updates | |
message, ranking_update, last_update = handle_vote( | |
state0, state1, vote_type | |
) | |
# Get the model names from the current session | |
model_a = state0["model_name"] if state0 else "Unknown" | |
model_b = state1["model_name"] if state1 else "Unknown" | |
# Always show thank you message and clear everything immediately | |
gr.Info("Thank you for your vote! 🎉 Your feedback has been recorded and new models have been selected.", duration=5) | |
gr.Info(f"Model B: {model_b}", duration=20) | |
gr.Info(f"Model A: {model_a}", duration=20) | |
# revval the model names in the info message | |
gr.Info(f"Now you can see model names! 👀", duration=20) | |
# Get new random models for the next session | |
new_model_a, new_model_b = get_random_models() | |
print(f"Post-vote new models - Model A: {new_model_a}, Model B: {new_model_b}") | |
# Clear everything and start fresh immediately, but preserve examples | |
return ( | |
message, # vote status message | |
gr.update(), # Keep state0 unchanged | |
gr.update(), # Keep state1 unchanged | |
gr.update(), # Keep chatbot_a unchanged | |
gr.update(), # Keep chatbot_b unchanged | |
gr.update(), # Keep response_a unchanged | |
gr.update(), # Keep response_b unchanged | |
gr.update(), # Keep code_a unchanged | |
gr.update(), # Keep code_b unchanged | |
gr.update(), # Keep sandbox_view_a unchanged | |
gr.update(), # Keep sandbox_view_b unchanged | |
gr.update(), # Keep sandbox_component_a unchanged | |
gr.update(), # Keep sandbox_component_b unchanged | |
gr.update(), # Keep chat_stats_a unchanged | |
gr.update(), # Keep chat_stats_b unchanged | |
gr.update(), # Keep model_display_a unchanged | |
gr.update(), # Keep model_display_b unchanged | |
gr.update(visible=False), # Hide vote_section | |
gr.update(visible=False), # Hide vote_buttons_row | |
None, # Reset state0_var | |
None, # Reset state1_var | |
gr.update(), # Keep existing ranking_table (no refresh needed) | |
gr.update(), # Keep existing ranking_last_update (no refresh needed) | |
gr.update(interactive=False), # Disable vote_left_btn | |
gr.update(interactive=False), # Disable vote_right_btn | |
gr.update(interactive=False), # Disable vote_tie_btn | |
gr.update(interactive=False), # Disable vote_both_bad_btn | |
"", # Clear text_input to preserve examples | |
new_model_a, # Update model_a_state with new model | |
new_model_b, # Update model_b_state with new model | |
) | |
# Re-run code button handlers | |
def rerun_code_a(state0, current_code_a, current_install_command_a): | |
"""Re-run code execution for Model A""" | |
updated_state, sandbox_view, sandbox_component_update, error = rerun_code_execution(state0, current_code_a, current_install_command_a, "A") | |
return ( | |
updated_state, # state0_var | |
sandbox_view, # sandbox_view_a | |
sandbox_component_update if sandbox_component_update else gr.skip(), # sandbox_component_a | |
) | |
def rerun_code_b(state1, current_code_b, current_install_command_b): | |
"""Re-run code execution for Model B""" | |
updated_state, sandbox_view, sandbox_component_update, error = rerun_code_execution(state1, current_code_b, current_install_command_b, "B") | |
return ( | |
updated_state, # state1_var | |
sandbox_view, # sandbox_view_b | |
sandbox_component_update if sandbox_component_update else gr.skip(), # sandbox_component_b | |
) | |
def change_to_view_a(): | |
return gr.Tabs(selected=0) | |
def change_to_view_b(): | |
return gr.Tabs(selected=2) | |
rerun_code_a_btn.click( | |
fn=change_to_view_a, | |
inputs=[], | |
outputs=[tabs_a] | |
).then( | |
fn=rerun_code_a, | |
inputs=[state0_var, code_a, install_command_a], | |
outputs=[ | |
state0_var, # state0_var | |
sandbox_view_a, # sandbox_view_a | |
sandbox_component_a, # sandbox_component_a | |
] | |
) | |
rerun_code_b_btn.click( | |
fn=change_to_view_b, | |
inputs=[], | |
outputs=[tabs_b] | |
).then( | |
fn=rerun_code_b, | |
inputs=[state1_var, code_b, install_command_b], | |
outputs=[ | |
state1_var, # state1_var | |
sandbox_view_b, # sandbox_view_b | |
sandbox_component_b, # sandbox_component_b | |
] | |
) | |
# Vote button click handlers | |
for vote_btn, vote_type in [ | |
(vote_left_btn, "left"), | |
(vote_right_btn, "right"), | |
(vote_tie_btn, "tie"), | |
(vote_both_bad_btn, "both_bad"), | |
]: | |
vote_btn.click( | |
fn=process_vote, | |
inputs=[state0_var, state1_var, gr.State(vote_type), text_input], | |
outputs=[ | |
vote_status, # vote status message | |
state0_var, # state0 | |
state1_var, # state1 | |
chatbot_a, # chatbot_a | |
chatbot_b, # chatbot_b | |
response_a, # response_a | |
response_b, # response_b | |
code_a, # code_a | |
code_b, # code_b | |
sandbox_view_a, # sandbox_view_a | |
sandbox_view_b, # sandbox_view_b | |
sandbox_component_a, # sandbox_component_a | |
sandbox_component_b, # sandbox_component_b | |
chat_stats_a, # chat_stats_a | |
chat_stats_b, # chat_stats_b | |
model_display_a, # model_display_a | |
model_display_b, # model_display_b | |
vote_section, # vote_section | |
vote_buttons_row, # vote_buttons_row | |
state0_var, # state0_var (duplicate for state management) | |
state1_var, # state1_var (duplicate for state management) | |
ranking_table, # ranking_table | |
ranking_last_update, # ranking_last_update | |
vote_left_btn, # vote_left_btn | |
vote_right_btn, # vote_right_btn | |
vote_tie_btn, # vote_tie_btn | |
vote_both_bad_btn, # vote_both_bad_btn | |
text_input, # text_input (to preserve examples) | |
model_a_state, # Update model_a_state with new model | |
model_b_state, # Update model_b_state with new model | |
], | |
) | |
# Setup ranking handlers | |
setup_ranking_handlers(demo, ranking_table, ranking_last_update, ranking_timer) | |
# Initialize random models for each user session when they load the page | |
def init_models(): | |
"""Initialize random models for this user session""" | |
model_a, model_b = get_random_models() | |
print(f"New user session - Model A: {model_a}, Model B: {model_b}") | |
return ( | |
model_a, # model_a_state | |
model_b, # model_b_state | |
f"**Model A:** {model_a}", # model_display_a | |
f"**Model B:** {model_b}", # model_display_b | |
) | |
demo.load( | |
fn=init_models, | |
inputs=[], | |
outputs=[ | |
model_a_state, | |
model_b_state, | |
model_display_a, | |
model_display_b, | |
] | |
) | |
return demo | |
def main(): | |
"""Main function to run the Simple BigCodeArena app""" | |
# Build the UI - models are now selected per user session | |
demo = build_ui() | |
# Launch the app | |
demo.launch( | |
server_name="0.0.0.0", | |
server_port=7860, | |
share=False, | |
debug=True | |
) | |
if __name__ == "__main__": | |
main() | |