分布式故障诊断终于有了统一考场：阿里云开源 RCA Benchmark

git clone https://github.com/alibaba/rca-benchmark.git
cd rca-benchmark

# 安装依赖（项目基于 Python 3.10+）
pip install -r requirements.txt

# 验证数据集完整性
python scripts/verify_dataset.py --check-all

# eval_config.yaml — 选择你要评估的场景和维度
benchmark:
  scenario_type: microservice        # 微服务 / container / network / storage
  difficulty: hard                   # easy / medium / hard
  noise_level: 0.1                   # 0~1，模拟观测数据噪声比例
  missing_data_ratio: 0.05           # 模拟部分数据缺失

evaluation:
  metrics:
    - accuracy                       # 根因定位准确率
    - reasoning_chain_coverage       # 推理链完整度
    - step_efficiency                # 诊断效率
    - robustness                     # 鲁棒性（需搭配 noise_level > 0）
  acceptable_answer_range: 2         # 根因节点允许 ±2 跳的偏差

agent:
  type: llm_based
  model: your-model-name
  api_endpoint: https://your-api-endpoint/v1/chat/completions
  max_interaction_rounds: 10         # Agent 最多交互轮次

# agent_adapter.py — 把你的 Agent 包装成 Benchmark 能调用的接口
from rca_benchmark.adapter import BaseAgentAdapter
from rca_benchmark.schema import Observation, DiagnosisResult

class MyRCAAgentAdapter(BaseAgentAdapter):
    """适配你的 LLM-based RCA Agent 到 Benchmark 评估协议"""

    def __init__(self, model_endpoint: str, max_rounds: int = 10):
        self.endpoint = model_endpoint
        self.max_rounds = max_rounds

    def diagnose(self, observation: Observation) -> DiagnosisResult:
        """
        接收标准化的观测数据，返回诊断结果。
        observation 包含：metrics, traces, logs, alert_events
        """
        # 构造给 LLM 的提示词——这里是你自己的设计
        prompt = self._build_prompt(observation)

        # 多轮交互：Agent 可以主动请求更多数据
        reasoning_chain = []
        for round_idx in range(self.max_rounds):
            response = self._call_llm(prompt, reasoning_chain)
            step = self._parse_step(response)

            reasoning_chain.append(step)

            if step.is_conclusive:
                # Agent 给出了最终根因判断
                return DiagnosisResult(
                    root_cause_node=step.root_cause,
                    reasoning_chain=reasoning_chain,
                    confidence=step.confidence,
                    rounds_used=round_idx + 1,
                )

            # Agent 请求补充观测数据
            if step.requested_data:
                extra = observation.get_supplement(step.requested_data)
                prompt = self._update_prompt(prompt, extra)

        # 超过最大轮次仍未定位
        return DiagnosisResult(
            root_cause_node=None,
            reasoning_chain=reasoning_chain,
            confidence=0.0,
            rounds_used=self.max_rounds,
            timed_out=True,
        )

    def _build_prompt(self, obs: Observation) -> str:
        # 将 metrics/traces/logs 格式化为 LLM 可理解的文本
        # 这部分是你 Agent 的核心设计，Benchmark 不限定格式
        parts = []
        if obs.metrics:
            parts.append(f"## Metrics\n{obs.metrics.to_dataframe().to_string()}")
        if obs.traces:
            parts.append(f"## Traces\n{obs.traces.to_summary_string()}")
        if obs.logs:
            parts.append(f"## Logs (recent)\n{obs.logs.tail(50).to_string()}")
        if obs.alerts:
            parts.append(f"## Alerts\n{obs.alerts.to_string()}")

        return (
            "你是一名分布式系统故障诊断专家。"
            "根据以下观测数据，逐步推理故障根因。\n\n"
            + "\n\n".join(parts)
        )

    # _call_llm, _parse_step, _update_prompt 按你的实际实现补充

# 单场景评估
python run_eval.py \
  --config eval_config.yaml \
  --adapter agent_adapter.py \
  --output results/microservice_hard/

# 批量评估（遍历所有微服务场景）
python run_eval.py \
  --config eval_config.yaml \
  --adapter agent_adapter.py \
  --scenario-filter microservice \
  --output results/microservice_all/

# 生成对比报告
python scripts/generate_report.py \
  --input results/microservice_all/ \
  --format markdown \
  --output report.md

# .github/workflows/rca-eval.yml 示例片段
name: RCA Agent Eval
on:
  push:
    paths: ["agent/**", "prompts/**"]

jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install dependencies
        run: pip install -r requirements.txt
      - name: Run RCA Benchmark
        run: |
          python run_eval.py \
            --config eval_config.yaml \
            --adapter agent_adapter.py \
            --output results/${{ github.sha }}/
      - name: Check accuracy threshold
        run: |
          python scripts/check_threshold.py \
            --input results/${{ github.sha }}/ \
            --min-accuracy 0.7 \
            --min-chain-coverage 0.6