Claude Code 质量滑坡复盘：三个"小改动"叠加的六周灾难

"""
quality_drift_detector.py
检测 LLM 产品层配置变化是否导致质量漂移
依赖：anthropic Python SDK (pip install anthropic)
运行前设置环境变量：export ANTHROPIC_API_KEY=your_key
"""

import os
import json
from anthropic import Anthropic

client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))

# 测试题：需要多步推理的编程任务，对推理深度敏感
TEST_PROMPTS = [
    {
        "id": "multi_step_debug",
        "prompt": "给定这段 Python 代码，找出所有 bug 并逐一解释原因和修复方案：\n"
                  "def merge_dicts(a, b):\n    return {k: a[k] for k in a if k not in b} | b",
        "expected_depth": 3,  # 期望至少识别 3 个独立问题
    },
    {
        "id": "refactor_plan",
        "prompt": "把一个 500 行的单文件 Flask 应用重构为模块化结构，"
                  "给出目录布局、文件职责划分和迁移步骤。",
        "expected_depth": 4,  # 期望至少 4 个步骤
    },
]

# 两种系统提示词：完整版 vs 截断版（模拟 verbosity limit）
SYSTEM_FULL = (
    "你是一位资深软件工程师。回答时必须："
    "1. 先分析问题结构再给出方案；"
    "2. 对每个建议说明理由和潜在风险；"
    "3. 如果有多种方案，比较优劣；"
    "4. 给出可直接运行的代码示例。"
)

SYSTEM_TRUNCATED = "你是一位资深软件工程师。回答时要分析问题再给方案。"  # 模拟截断后只剩一句话

def count_reasoning_steps(response_text: str) -> int:
    """粗略统计回答中独立推理步骤的数量"""
    # 按编号列表、分号分隔的句子等启发式计数
    steps = 0
    for line in response_text.split("\n"):
        line = line.strip()
        if line and (line[0].isdigit() or line.startswith("-") or line.startswith("*")):
            steps += 1
    # 也按"首先/其次/然后/最后"等连接词计数
    connector_words = ["首先", "其次", "然后", "接着", "最后", "另外", "同时"]
    for word in connector_words:
        steps += response_text.count(word)
    return max(steps, 1)

def run_test(prompt: str, system: str, thinking_budget: int = 2000) -> str:
    """调用 Claude API，控制 thinking budget 和系统提示词"""
    response = client.messages.create(
        model="claude-sonnet-4-20250514",
        max_tokens=4096,
        system=system,
        messages=[{"role": "user", "content": prompt}],
        # 使用 extended thinking 控制推理深度
        thinking={
            "type": "enabled",
            "budget_tokens": thinking_budget,
        },
    )
    # 提取文本内容
    text_blocks = [b.text for b in response.content if b.type == "text"]
    return "".join(text_blocks)

def detect_drift():
    """对比不同配置下的质量差异"""
    results = {}

    configs = {
        "baseline": {"system": SYSTEM_FULL, "thinking_budget": 2000},
        "low_effort": {"system": SYSTEM_FULL, "thinking_budget": 500},    # 模拟推理降档
        "truncated_sys": {"system": SYSTEM_TRUNCATED, "thinking_budget": 2000},  # 模拟提示词截断
        "both_bad": {"system": SYSTEM_TRUNCATED, "thinking_budget": 500},  # 模拟叠加
    }

    for test in TEST_PROMPTS:
        print(f"\n=== 测试题: {test['id']} ===")
        for config_name, config in configs.items():
            text = run_test(
                test["prompt"],
                config["system"],
                config["thinking_budget"],
            )
            steps = count_reasoning_steps(text)
            drift = test["expected_depth"] - steps
            label = "✓" if drift <= 0 else f"⚠ 缺 {drift} 步"
            print(f"  {config_name:15s} | 推理步骤: {steps:3d} | 期望: {test['expected_depth']} | {label}")
            results.setdefault(test["id"], {})[config_name] = {
                "steps": steps,
                "expected": test["expected_depth"],
                "drift": drift,
            }

    # 输出汇总
    print("\n=== 漂移汇总 ===")
    for test_id, configs_result in results.items():
        baseline_steps = configs_result["baseline"]["steps"]
        for name, r in configs_result.items():
            delta = r["steps"] - baseline_steps
            print(f"  {test_id} | {name:15s} | 相对基线变化: {delta:+d} 步")

    return results

if __name__ == "__main__":
    detect_drift()

pip install anthropic
export ANTHROPIC_API_KEY=your_key_here
python quality_drift_detector.py

# 如果你用 Claude Code 的缓存功能，加一条监控：
# 每轮对话后检查 thinking tokens 数量是否异常衰减

# 伪代码逻辑（嵌入你的对话管线）：
# round_1_thinking = 2000
# round_2_thinking = extract_thinking_length(response_2)
# if round_2_thinking < round_1_thinking * 0.5:
#     log_warning("thinking tokens 衰减超过 50%，可能缓存截断")
#     disable_cache_for_next_round()