通用推理不是学科推理的翻版：美团 LongCat General 365 带来了什么

"""
最小化 General 365 评测脚本
假设数据格式为：
{
  "id": "q001",
  "category": "causal_reasoning",
  "difficulty": 2,
  "question": "...",
  "options": ["A...", "B...", "C...", "D..."],
  "answer": "B"
}

依赖：pip install openai datasets tqdm
"""

import json
import os
from pathlib import Path
from openai import OpenAI
from tqdm import tqdm

# ---- 配置 ----
DATA_PATH = Path("general365/data/general365.json")  # 从仓库下载后放这里
MODEL_NAME = "deepseek-chat"  # 替换为你想评测的模型
API_BASE = "https://api.deepseek.com"  # 替换为对应 API 地址
API_KEY = os.environ.get("LLM_API_KEY", "your-key-here")

client = OpenAI(base_url=API_BASE, api_key=API_KEY)

# ---- 加载题目 ----
with open(DATA_PATH, "r", encoding="utf-8") as f:
    questions = json.load(f)

print(f"共加载 {len(questions)} 题")

# ---- 构造 prompt ----
def build_prompt(item: dict) -> str:
    options_text = "\n".join(
        f"{chr(65+i)}. {opt}" for i, opt in enumerate(item["options"])
    )
    return (
        f"请仅根据逻辑推理回答以下问题，不要引入题面以外的知识。\n\n"
        f"题目：{item['question']}\n\n"
        f"选项：\n{options_text}\n\n"
        f"请只输出选项字母（A/B/C/D），不要输出其他内容。"
    )

# ---- 评测 ----
results = []
for item in tqdm(questions, desc="评测中"):
    prompt = build_prompt(item)
    try:
        resp = client.chat.completions.create(
            model=MODEL_NAME,
            messages=[{"role": "user", "content": prompt}],
            temperature=0.0,  # 推理评测用零温度，减少随机性
            max_tokens=10,
        )
        prediction = resp.choices[0].message.content.strip().upper()
        # 只取第一个字母
        prediction = prediction[0] if prediction and prediction[0] in "ABCD" else "X"
    except Exception as e:
        prediction = "X"
        print(f"题目 {item['id']} 请求失败: {e}")

    results.append({
        "id": item["id"],
        "category": item["category"],
        "difficulty": item["difficulty"],
        "prediction": prediction,
        "answer": item["answer"],
        "correct": prediction == item["answer"],
    })

# ---- 统计 ----
total = len(results)
correct = sum(r["correct"] for r in results)
print(f"\n总准确率: {correct}/{total} = {correct/total:.2%}")

# 按推理类型统计
from collections import defaultdict
by_category = defaultdict(list)
by_difficulty = defaultdict(list)
for r in results:
    by_category[r["category"]].append(r)
    by_difficulty[r["difficulty"]].append(r)

print("\n--- 按推理类型 ---")
for cat, items in sorted(by_category.items()):
    acc = sum(i["correct"] for i in items) / len(items)
    print(f"  {cat}: {acc:.2%} ({len(items)} 题)")

print("\n--- 按难度等级 ---")
for diff, items in sorted(by_difficulty.items()):
    acc = sum(i["correct"] for i in items) / len(items)
    print(f"  难度 {diff}: {acc:.2%} ({len(items)} 题)")

# ---- 保存详细结果 ----
with open("general365_eval_results.json", "w", encoding="utf-8") as f:
    json.dump(results, f, ensure_ascii=False, indent=2)
print("\n详细结果已保存到 general365_eval_results.json")

# 1. 克隆数据仓库（假设 LongCat 将数据发布在 GitHub）
git clone https://github.com/LongCat-AI/General365.git
# 数据文件一般在 general365/data/ 目录下

# 2. 安装依赖
pip install openai tqdm

# 3. 设置 API Key
export LLM_API_KEY="sk-xxx"

# 4. 运行评测
python eval_general365.py

# 不推荐：让模型一次完成多步推理
prompt = "根据用户行为数据，推断流失原因，并给出挽回策略。"

# 推荐：拆步验证
step1 = "根据以下行为数据，列出最可能的2个流失原因。只输出原因，不输出策略。"
step2 = f"基于原因：{step1_result}，针对每个原因给出一条挽回动作。只输出动作。"