|
|
import gradio as gr |
|
|
import json, os |
|
|
from huggingface_hub import hf_hub_download |
|
|
from compute_perp import prep_evaluator, numberic_compare, check_equal |
|
|
from compute_sc import sc_evaluator |
|
|
from compute_rpc import wpc_evaluator |
|
|
import numpy as np |
|
|
|
|
|
def greet(name): |
|
|
return "Hello " + name + "!!" |
|
|
|
|
|
json_file = {"predict": [], "answer": [], "completion": [], "mean_logprob": [], "prompt": []} |
|
|
|
|
|
demo = gr.Blocks() |
|
|
with demo: |
|
|
paper_title = gr.HTML("""<div align='center'><h1>[NeurIPS 2025] A Theoretical Study on Bridging Internal Probability and Self-Consistency for LLM Reasoning</h1></div>""") |
|
|
paper_info = gr.HTML("""<div align="center"><h3><a href="https://arxiv.org/pdf/2502.00511">📄 [Paper]</a> • <a href="https://wnjxyk.github.io/RPC">🌐 [Project]</a> • <a href="#" onclick="document.getElementById('bibtex-popup').style.display='block';">📚 [BibTeX]</a><h3> |
|
|
<div id="bibtex-popup" style="display:none; position:fixed; top:50%; left:50%; transform:translate(-50%, -50%); background:white; padding:20px; border:1px solid #ccc; box-shadow:0 0 10px rgba(0,0,0,0.2); z-index:1000; max-width:80%; overflow:auto;"> |
|
|
<pre style="white-space:pre-wrap; font-size:12px; text-align:left;">@inproceedings{zhou24theoretical, |
|
|
author = {Zhou, Zhi and Tan, Yuhao and Li, Zenan and Yao, Yuan and Guo, Lan-Zhe and Li, Yu-Feng and Ma, Xiaoxing}, |
|
|
title = {A Theorecial Study on Bridging Internal Probability and Self-Consistency for LLM Reasoning}, |
|
|
booktitle = {Advances in Neural Information Processing Systems}, |
|
|
year = {2025}, |
|
|
}</pre> |
|
|
<button onclick="document.getElementById('bibtex-popup').style.display='none';" style="margin-top:10px; padding:5px 10px;">Close</button> |
|
|
</div></div>""") |
|
|
|
|
|
with gr.Column(): |
|
|
gr.Markdown("## 1. Experimental Settings") |
|
|
with gr.Row(): |
|
|
dataset = gr.Dropdown( |
|
|
choices=["MATH", "MathOdyssey", "AIME", "OlympiadBench"], |
|
|
value="MathOdyssey", |
|
|
label="Dataset", |
|
|
interactive=True |
|
|
) |
|
|
model = gr.Dropdown( |
|
|
choices=["Deepseek-Math-RL-7B", "InternLM2-Math-Plus-1.8B", "InternLM2-Math-Plus-7B"], |
|
|
value="InternLM2-Math-Plus-7B", |
|
|
label="Model", |
|
|
interactive=True |
|
|
) |
|
|
k_value = gr.Dropdown( |
|
|
choices=[8, 16, 32, 64, 128], |
|
|
value=128, |
|
|
label="K (Number of Sampled Reasoning Paths)", |
|
|
interactive=True |
|
|
) |
|
|
seed = gr.Number( |
|
|
label="Random Seed", |
|
|
value=998244353, |
|
|
step=1, |
|
|
interactive=True |
|
|
) |
|
|
def update_k_value(dataset_choice): |
|
|
if dataset_choice == "MATH": |
|
|
return gr.update(choices=[8, 16, 32, 64], value=min(64, k_value.value)) |
|
|
else: |
|
|
return gr.update(choices=[8, 16, 32, 64, 128], value=k_value.value) |
|
|
dataset.change(fn=update_k_value, inputs=dataset, outputs=k_value) |
|
|
load_btn = gr.Button("Load All Problems") |
|
|
|
|
|
with gr.Column(visible=False) as content_column: |
|
|
gr.Markdown("## 2. Problem Selection") |
|
|
with gr.Group(): |
|
|
data_info = gr.Textbox(label="Experiment Info", value="") |
|
|
problem_id = gr.Dropdown( |
|
|
choices=[1], |
|
|
value=1, |
|
|
label="Problem ID (We removed (1) problems that were unlikely to be answered correctly using any of the methods; (2) easy problems)", |
|
|
interactive=True |
|
|
) |
|
|
with gr.Row(): |
|
|
problem_prompt = gr.Textbox(label="Problem Prompt", value="", scale=3) |
|
|
problem_answer = gr.Textbox(label="Problem Answer", value="", scale=1) |
|
|
def update_problem_info(problem_id): |
|
|
return gr.update(value=json_file['prompt'][problem_id-1], label=f"Problem#{problem_id} Prompt"), gr.update(value=json_file['answer'][problem_id-1], label=f"Problem#{problem_id} Answer") |
|
|
problem_id.change(fn=update_problem_info, inputs=problem_id, outputs=[problem_prompt, problem_answer]) |
|
|
run_btn = gr.Button("Run Evaluation") |
|
|
|
|
|
with gr.Column(visible=False) as result_column: |
|
|
gr.Markdown("## 3. Experiment Result") |
|
|
with gr.Row(): |
|
|
with gr.Column(): |
|
|
gr.Markdown("### PPL (Internal Probability)") |
|
|
ppl_result = gr.Markdown() |
|
|
with gr.Column(): |
|
|
gr.Markdown("### SC (Self-Consistency)") |
|
|
sc_result = gr.Markdown(value="") |
|
|
with gr.Column(): |
|
|
gr.Markdown("### RPC (Ours)") |
|
|
rpc_result = gr.Markdown(value="") |
|
|
|
|
|
def get_available_problems(): |
|
|
global json_file |
|
|
answer = np.array(json_file["accuracy"]).mean(axis=0) |
|
|
|
|
|
|
|
|
available_indices = np.where((answer > 0.3) & (answer < 0.5))[0] |
|
|
available_indices = available_indices + 1 |
|
|
|
|
|
return available_indices.tolist() |
|
|
|
|
|
|
|
|
def load(dataset, model, k_value, seed): |
|
|
try: |
|
|
repo_id = { |
|
|
"MATH": "WNJXYK/MATH-Reasoning-Paths", |
|
|
"MathOdyssey": "WNJXYK/MathOdyssey-Reasoning-Paths", |
|
|
"AIME": "WNJXYK/AIME_1983_2024-Reasoning-Paths", |
|
|
"OlympiadBench": "WNJXYK/OlympiadBench-Reasoning-Paths" |
|
|
}[dataset] |
|
|
filename = f"{model}.json" |
|
|
|
|
|
yield f"Downloading sampled reasoning paths from Hugging Face {repo_id}...", gr.update(visible=False), gr.update(), gr.update(), gr.update(visible=False) |
|
|
file_path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type="dataset") |
|
|
|
|
|
global json_file |
|
|
with open(file_path, 'r', encoding='utf-8') as f: |
|
|
json_file = json.load(f) |
|
|
clist = get_available_problems() |
|
|
os.remove(file_path) |
|
|
|
|
|
yield "Loading complete! You can now select a problem ID.", gr.update(visible=True), gr.update(value=f"Dataset: {dataset}\tModel: {model}\tK: {k_value}\tSeed: {seed}"), gr.update(choices=clist, value=clist[0]), gr.update(visible=False) |
|
|
|
|
|
except Exception as e: |
|
|
yield f"Error: {str(e)}" |
|
|
|
|
|
def res_to_str(correct, answers, topk=10): |
|
|
answers = sorted(answers, key=lambda x: x[1], reverse=True) |
|
|
response = "| # | Answer | Probability | Correct |\n|---|--------|------------|--------|\n" |
|
|
for i in range(min(len(answers), topk)): |
|
|
correct_mark = "✅" if answers[i][2] else "❌" |
|
|
wrapped_answer = answers[i][0] if len(answers[i][0]) <= 10 else answers[i][0][:10] + "..." |
|
|
response += f"| Top-{i+1} | {wrapped_answer} | {answers[i][1]:.2f} | {correct_mark} |\n" |
|
|
return response |
|
|
|
|
|
def evaluate(problem_id): |
|
|
ppl_correct, ppl_answers = prep_evaluator( |
|
|
json_file["predict"][problem_id-1], |
|
|
json_file["completion"][problem_id-1], |
|
|
json_file["mean_logprob"][problem_id-1], |
|
|
json_file["answer"][problem_id-1], |
|
|
numberic_compare, |
|
|
check_equal |
|
|
) |
|
|
|
|
|
sc_correct, sc_answers = sc_evaluator( |
|
|
json_file["predict"][problem_id-1], |
|
|
json_file["completion"][problem_id-1], |
|
|
json_file["mean_logprob"][problem_id-1], |
|
|
json_file["answer"][problem_id-1], |
|
|
numberic_compare, |
|
|
check_equal |
|
|
) |
|
|
|
|
|
rpc_correct, rpc_answers = wpc_evaluator( |
|
|
json_file["predict"][problem_id-1], |
|
|
json_file["completion"][problem_id-1], |
|
|
json_file["mean_logprob"][problem_id-1], |
|
|
json_file["answer"][problem_id-1], |
|
|
numberic_compare, |
|
|
check_equal |
|
|
) |
|
|
|
|
|
return gr.update(visible=True), gr.update(value=res_to_str(ppl_correct, ppl_answers)), gr.update(value=res_to_str(sc_correct, sc_answers)), gr.update(value=res_to_str(rpc_correct, rpc_answers)) |
|
|
|
|
|
load_btn.click(fn=load, inputs=[dataset, model, k_value, seed],outputs=[load_btn, content_column, data_info, problem_id, result_column], show_progress="inside") |
|
|
run_btn.click(fn=evaluate, inputs=problem_id, outputs=[result_column, ppl_result, sc_result, rpc_result]) |
|
|
|
|
|
if __name__ == "__main__": |
|
|
demo.launch() |
