GPT‑6.1 Sol 主打低价高能：上线前先用真实任务验证它

单次有效任务成本 = 模型调用成本 + 重试成本 + 验证成本 + 人工返工成本

export LLM_BASE_URL="https://api.example.com/v1"
export LLM_API_KEY="replace-with-your-key"
export SOL_MODEL="gpt-6.1-sol"
export BASELINE_MODEL="your-current-model"
python3 benchmark_models.py

import json
import os
import time
import urllib.request

BASE_URL = os.environ["LLM_BASE_URL"].rstrip("/")
API_KEY = os.environ["LLM_API_KEY"]
MODELS = [
    os.environ.get("SOL_MODEL", "gpt-6.1-sol"),
    os.environ.get("BASELINE_MODEL", "baseline-model"),
]

TASKS = [
    {
        "id": "office-summary",
        "prompt": (
            "把下面的项目记录整理为 JSON，字段为 risks、owners、deadlines。"
            "不要补充原文中不存在的信息。\n\n"
            "记录：支付接口延迟优化由林工负责，周五前提交方案；"
            "数据迁移尚未指定负责人，目标日期为 6 月 30 日。"
        ),
    },
    {
        "id": "coding-fix",
        "prompt": (
            "修复下面的 Python 函数，使空列表返回 0，并给出完整代码和两个断言。\n\n"
            "def average(values):\n"
            "    return sum(values) / len(values)"
        ),
    },
    {
        "id": "safe-action-plan",
        "prompt": (
            "你需要清理服务器上的旧日志。只输出操作计划，不执行命令。"
            "必须包含预览、备份、路径限制和人工确认步骤。"
        ),
    },
]


def call_model(model, prompt):
    payload = {
        "model": model,
        "messages": [
            {
                "role": "system",
                "content": "准确完成任务；信息不足时明确说明，不要猜测。",
            },
            {"role": "user", "content": prompt},
        ],
        "temperature": 0,
    }
    request = urllib.request.Request(
        f"{BASE_URL}/chat/completions",
        data=json.dumps(payload).encode("utf-8"),
        headers={
            "Authorization": f"Bearer {API_KEY}",
            "Content-Type": "application/json",
        },
        method="POST",
    )

    started = time.perf_counter()
    with urllib.request.urlopen(request, timeout=120) as response:
        result = json.load(response)
    elapsed_ms = round((time.perf_counter() - started) * 1000)

    return {
        "latency_ms": elapsed_ms,
        "output": result["choices"][0]["message"]["content"],
        "usage": result.get("usage", {}),
    }


def main():
    results = []
    for task in TASKS:
        for model in MODELS:
            try:
                result = call_model(model, task["prompt"])
                results.append({"task": task["id"], "model": model, **result})
                print(f"completed: {task['id']} / {model}")
            except Exception as exc:
                results.append(
                    {"task": task["id"], "model": model, "error": str(exc)}
                )
                print(f"failed: {task['id']} / {model}: {exc}")

    with open("benchmark-results.json", "w", encoding="utf-8") as file:
        json.dump(results, file, ensure_ascii=False, indent=2)


if __name__ == "__main__":
    main()

agent_policy:
  default_mode: propose_only

  filesystem:
    allow_read:
      - "/workspace/**"
    allow_write:
      - "/workspace/tmp/**"
    deny:
      - "**/.env"
      - "**/id_rsa"
      - "/etc/**"

  shell:
    allow:
      - "pytest"
      - "npm test"
      - "git diff"
    require_approval:
      - "git push*"
      - "kubectl apply*"
      - "terraform apply*"
    deny:
      - "rm -rf*"
      - "curl * | sh"

  browser:
    require_approval_for:
      - "send_message"
      - "submit_payment"
      - "change_permissions"
      - "delete_resource"

  audit:
    log_prompts: true
    log_tool_calls: true
    redact_secrets: true