Agent 跑起来容易，跑得好难——Litefuse 用 EDD 闭环让效果可量化

# 安装依赖
# pip install litefuse openai

import os
from openai import OpenAI
from litefuse import LitefuseClient, EvaluationConfig

# 1. 初始化 Litefuse 客户端
lf = LitefuseClient(
    public_key=os.environ["LITEFUSE_PUBLIC_KEY"],
    secret_key=os.environ["LITEFUSE_SECRET_KEY"],
    # 可选：自托管地址，进一步降低成本
    # host="http://your-self-hosted-litefuse:3000"
)

# 2. 创建一个 trace，代表一次完整的 Agent 执行
trace = lf.trace(
    name="rag-agent-query",
    metadata={"user_id": "u-1234", "session_id": "s-5678"},
    tags=["production", "v1.2"]
)

# 3. 在 trace 内记录 LLM 调用（span）
client = OpenAI()

with trace.span(name="llm-generate", kind="llm") as llm_span:
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": "你是一个知识助手，根据检索结果回答问题。"},
            {"role": "user", "content": "什么是 EDD？"}
        ],
        temperature=0.3
    )
    answer = response.choices[0].message.content

    # 把输入输出和 token 用量记录到 span
    llm_span.update(
        input={"messages": [
            {"role": "system", "content": "你是一个知识助手..."},
            {"role": "user", "content": "什么是 EDD？"}
        ]},
        output={"content": answer},
        usage={
            "prompt_tokens": response.usage.prompt_tokens,
            "completion_tokens": response.usage.completion_tokens,
            "total_tokens": response.usage.total_tokens
        }
    )

# 4. 记录工具调用 span（模拟检索）
with trace.span(name="tool-search", kind="tool") as tool_span:
    # 实际项目中这里调用你的向量数据库
    search_results = ["EDD 是评估驱动开发方法论..."]
    tool_span.update(
        input={"query": "什么是 EDD？", "top_k": 3},
        output={"results": search_results},
        status="success"
    )

# 5. 结束 trace
trace.update(output={"answer": answer})
trace.end()

# 6. 配置并运行评估
eval_config = EvaluationConfig(
    name="rag-quality-v1",
    # LLM-as-judge：用模型评判回答质量
    evaluations=[
        {
            "type": "llm_judge",
            "model": "gpt-4o-mini",
            "prompt": (
                "请评估以下回答的质量，按 1-5 分打分。\n"
                "维度：准确性、完整性、简洁性。\n"
                "问题：什么是 EDD？\n"
                "回答：{answer}\n"
                "请输出 JSON：{\"accuracy\": N, \"completeness\": N, \"conciseness\": N}"
            ),
            "input_mapping": {"answer": "$.trace.output.answer"}
        },
        # 规则评估：检查回答长度是否合理
        {
            "type": "rule",
            "rule": "len(answer) >= 50 and len(answer) <= 500",
            "input_mapping": {"answer": "$.trace.output.answer"}
        }
    ]
)

result = lf.evaluate(trace_id=trace.id, config=eval_config)
print(f"评估结果: {result.scores}")
# 示例输出: {"accuracy": 4, "completeness": 3, "conciseness": 4, "rule_length": True}

# 拉取并启动 Litefuse 自托管服务
docker run -d \
  --name litefuse \
  -p 3000:3000 \
  -e DATABASE_URL=postgresql://postgres:password@pg-host:5432/litefuse \
  -e REDIS_URL=redis://redis-host:6379 \
  -v litefuse-data:/data \
  litefuse/litefuse:latest

# 验证服务可用
curl http://localhost:3000/api/health