美团 LongCat 开源 General 365：推理评测的及格线，大多数模型都没摸到

# general365_eval.py — 最小可运行评测脚本
# 前置依赖：pip install datasets openai pyyaml

from datasets import load_dataset
from openai import OpenAI
import json, yaml, os

# 1. 加载 General 365 评测集（假设发布在 HuggingFace）
# 实际 repo 名以 LongCat 官方公告为准，此处为示意
ds = load_dataset("longcat/general365", split="test")

# 2. 配置你的模型端点
client = OpenAI(
    base_url=os.getenv("MODEL_BASE_URL", "https://api.openai.com/v1"),
    api_key=os.getenv("MODEL_API_KEY", "sk-xxx"),
)
MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o")

# 3. 构造推理 prompt模板
PROMPT_TEMPLATE = """你需要逐步推理并回答以下问题。请按以下格式输出：

## 推理过程
[请写出你的每一步推理，不要跳步]

## 最终答案
[用一个简短的答案作结]

题目：{question}

选项（如有）：{options}
"""

def call_model(prompt: str) -> str:
    resp = client.chat.completions.create(
        model=MODEL_NAME,
        messages=[{"role": "user", "content": prompt}],
        temperature=0.1,  # 推理评测尽量低温度，减少随机性
        max_tokens=1024,
    )
    return resp.choices[0].message.content

# 4. 批量评测
results = []
for item in ds:
    prompt = PROMPT_TEMPLATE.format(
        question=item["question"],
        options=item.get("options", "无"),
    )
    output = call_model(prompt)
    results.append({
        "id": item["id"],
        "category": item.get("category", "unknown"),
        "ground_truth": item["answer"],
        "model_output": output,
    })
    print(f"[{item['id']}] done — category: {item.get('category')}")

# 5. 保存原始结果，后续可对接 General 365 的官方评分脚本
with open("general365_results.json", "w", encoding="utf-8") as f:
    json.dump(results, f, ensure_ascii=False, indent=2)

print(f"评测完成，共 {len(results)} 题，结果已保存。")

# 示例：评测本地 vLLM 部署的模型
export MODEL_BASE_URL="http://localhost:8000/v1"
export MODEL_API_KEY="token-abc123"
export MODEL_NAME="Qwen2.5-72B-Instruct"

python general365_eval.py

# quick_score.py — 快速答案准确率统计
import json

with open("general365_results.json", "r", encoding="utf-8") as f:
    results = json.load(f)

correct = 0
for r in results:
    # 简单匹配：模型输出中是否包含标准答案
    if r["ground_truth"].strip() in r["model_output"]:
        correct += 1

acc = correct / len(results) * 100
print(f"答案准确率：{acc:.1f}% （{correct}/{len(results)}）")
print(f"对照：Gemini 3 Pro 在 General 365 上为 62.8%")

# error_analysis.py — 按推理类别统计错误分布
from collections import Counter

error_by_cat = Counter()
for r in results:
    if r["ground_truth"].strip() not in r["model_output"]:
        error_by_cat[r["category"]] += 1

print("错误分布（类别: 错误数）：")
for cat, cnt in error_by_cat.most_common():
    total_in_cat = sum(1 for r in results if r["category"] == cat)
    print(f"  {cat}: {cnt}/{total_in_cat} 错误率 {cnt/total_in_cat*100:.0f}%")

# prompt_strategies.yaml — 三种推理 prompt 策略配置
strategies:
  baseline:
    template: "请逐步推理并回答：{question}"
    temperature: 0.1
  chain_of_thought:
    template: "让我们一步步思考。首先分析问题条件，然后逐步推导，最后给出答案。\n问题：{question}"
    temperature: 0.1
  self_verify:
    template: "请推理并回答问题，完成后再检查一遍你的推理过程是否有逻辑错误。\n问题：{question}"
    temperature: 0.2