llm_eval_script/aws_claude.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
https://docs.aws.amazon.com/bedrock/latest/userguide/api-inference-examples-claude-messages-code-examples.html
https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html

https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-text-completion.html

https://docs.aws.amazon.com/bedrock/latest/userguide/inference-invoke.html

https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-examples.html

"""
import argparse
from datetime import datetime
import json
import os
from pathlib import Path
import sys
import time
from zoneinfo import ZoneInfo  # Python 3.9+ 自带，无需安装

pwd = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(pwd, "../"))

import boto3

from project_settings import environment, project_path


def get_args():
    """
python3 aws_claude.py --model_name anthropic.claude-instant-v1 \
--eval_dataset_name agent-lingoace-zh-400-choice.jsonl \
--client "us_west(47.88.76.239)" \
--create_time_str 20250723-interval-10 \
--interval 10

python3 aws_claude.py --model_name anthropic.claude-v2 \
--eval_dataset_name agent-lingoace-zh-400-choice.jsonl \
--client "us_west(47.88.76.239)" \
--create_time_str 20250723-interval-10 \
--interval 10

    """
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--model_name",
        # default="anthropic.claude-v2",
        default="anthropic.claude-instant-v1",

        # default="anthropic.claude-v2:1",
        # default="anthropic.claude-instant-v1:2",
        # default="anthropic.claude-v2:0",
        type=str
    )
    parser.add_argument(
        "--eval_dataset_name",
        default="agent-lingoace-zh-400-choice.jsonl",
        # default="arc-easy-1000-choice.jsonl",
        type=str
    )
    parser.add_argument(
        "--eval_dataset_dir",
        default=(project_path / "data/dataset").as_posix(),
        type=str
    )
    parser.add_argument(
        "--eval_data_dir",
        default=(project_path / "data/eval_data").as_posix(),
        type=str
    )
    parser.add_argument(
        "--client",
        default="shenzhen_sase",
        type=str
    )
    parser.add_argument(
        "--service",
        default="aws_us_east",
        type=str
    )
    parser.add_argument(
        "--create_time_str",
        default="null",
        type=str
    )
    parser.add_argument(
        "--interval",
        default=10,
        type=int
    )
    args = parser.parse_args()
    return args


def main():
    args = get_args()

    service = environment.get(key=args.service, dtype=json.loads)
    aws_access_key_id = service["AWS_ACCESS_KEY_ID"]
    aws_secret_access_key = service["AWS_SECRET_ACCESS_KEY"]
    aws_default_region = service["AWS_DEFAULT_REGION"]

    os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id
    os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key
    os.environ["AWS_DEFAULT_REGION"] = aws_default_region

    client = boto3.client(
        service_name="bedrock-runtime",
        region_name=aws_default_region
    )

    eval_dataset_dir = Path(args.eval_dataset_dir)
    eval_dataset_dir.mkdir(parents=True, exist_ok=True)
    eval_data_dir = Path(args.eval_data_dir)
    eval_data_dir.mkdir(parents=True, exist_ok=True)

    if args.create_time_str == "null":
        tz = ZoneInfo("Asia/Shanghai")
        now = datetime.now(tz)
        create_time_str = now.strftime("%Y%m%d_%H%M%S")
        # create_time_str = "20250722_173400"
    else:
        create_time_str = args.create_time_str

    eval_dataset = eval_dataset_dir / args.eval_dataset_name

    output_file = eval_data_dir / f"aws_claude/anthropic/{args.model_name}/{args.client}/{args.service}/{create_time_str}/{args.eval_dataset_name}"
    output_file.parent.mkdir(parents=True, exist_ok=True)

    total = 0
    total_correct = 0

    # finished
    finished_idx_set = set()
    if os.path.exists(output_file.as_posix()):
        with open(output_file.as_posix(), "r", encoding="utf-8") as f:
            for row in f:
                row = json.loads(row)
                idx = row["idx"]
                total = row["total"]
                total_correct = row["total_correct"]
                finished_idx_set.add(idx)
    print(f"finished count: {len(finished_idx_set)}")

    with open(eval_dataset.as_posix(), "r", encoding="utf-8") as fin, open(output_file.as_posix(), "a+", encoding="utf-8") as fout:
        for row in fin:
            row = json.loads(row)
            idx = row["idx"]
            prompt = row["prompt"]
            response = row["response"]

            if idx in finished_idx_set:
                continue
            finished_idx_set.add(idx)

            body = {
                "anthropic_version": "bedrock-2023-05-31",
                "messages": [
                    {
                        "role": "user",
                        "content": [{"type": "text", "text": prompt}]
                    }
                ],
                "max_tokens": 1000,
                "temperature": 0.5,
                "top_p": 0.95,
                # "thinking": {
                #     "type": "enabled",
                #     "budget_tokens": 1
                # },
            }

            try:
                # client.converse()
                time.sleep(args.interval)
                print(f"sleep: {args.interval}")
                time_begin = time.time()
                llm_response = client.invoke_model(
                    modelId=args.model_name,
                    body=json.dumps(body),
                    contentType="application/json"
                )

                llm_response = json.loads(llm_response["body"].read())
                # print(result['content'][0]['text'])
                time_cost = time.time() - time_begin
                print(f"time_cost: {time_cost}")

            except Exception as e:
                print(f"request failed, error type: {type(e)}, error text: {str(e)}")
                continue

            prediction = llm_response["content"][0]["text"]

            correct = 1 if prediction == response else 0

            total += 1
            total_correct += correct
            score = total_correct / total

            row_ = {
                "idx": idx,
                "prompt": prompt,
                "response": response,
                "prediction": prediction,
                "correct": correct,
                "total": total,
                "total_correct": total_correct,
                "score": score,
                "time_cost": time_cost,
            }
            row_ = json.dumps(row_, ensure_ascii=False)
            fout.write(f"{row_}\n")

    return


if __name__ == "__main__":
    main()