把基础模型升级从数月压缩到数小时：评测器、智能体循环与提示词爬山

cat > samples.jsonl <<'EOF'
{"input":"Translate for dubbing: The train leaves in five minutes.","reference":"火车五分钟后开出。","max_chars":12}
{"input":"Translate for dubbing: Please keep the door open.","reference":"请让门开着。","max_chars":8}
EOF

export MODEL_URL="https://model.example.com/generate"
export RATER_URL="https://rater.example.com/score"
export API_TOKEN="replace-me"

# migrate.py
import json
import os
import statistics
import urllib.request

MODEL_URL = os.environ["MODEL_URL"]
RATER_URL = os.environ["RATER_URL"]
TOKEN = os.environ["API_TOKEN"]

CANDIDATE_PROMPTS = [
    "Translate into concise Chinese suitable for video dubbing. Preserve meaning.",
    "Translate into natural, concise Chinese for dubbing. Preserve all facts and prefer shorter phrasing.",
]


def post_json(url, payload):
    request = urllib.request.Request(
        url,
        data=json.dumps(payload).encode("utf-8"),
        headers={
            "Authorization": f"Bearer {TOKEN}",
            "Content-Type": "application/json",
        },
        method="POST",
    )
    with urllib.request.urlopen(request, timeout=60) as response:
        return json.load(response)


def load_samples(path="samples.jsonl"):
    with open(path, encoding="utf-8") as file:
        return [json.loads(line) for line in file if line.strip()]


def evaluate_prompt(prompt, samples):
    rows = []
    for sample in samples:
        generated = post_json(MODEL_URL, {
            "system": prompt,
            "input": sample["input"],
            "temperature": 0,
        })["text"]

        rating = post_json(RATER_URL, {
            "rubric": {
                "meaning": "Preserve the meaning and factual details.",
                "fluency": "Use natural Chinese suitable for speech.",
            },
            "input": sample["input"],
            "reference": sample["reference"],
            "candidate": generated,
        })

        length_ok = len(generated) <= sample["max_chars"]
        semantic_score = float(rating["score"])
        final_score = semantic_score if length_ok else semantic_score - 0.25
        rows.append({
            "input": sample["input"],
            "output": generated,
            "semantic_score": semantic_score,
            "length_ok": length_ok,
            "score": final_score,
        })

    return statistics.mean(row["score"] for row in rows), rows


def main():
    samples = load_samples()
    reports = []

    for prompt in CANDIDATE_PROMPTS:
        score, rows = evaluate_prompt(prompt, samples)
        reports.append({"prompt": prompt, "score": score, "rows": rows})

    reports.sort(key=lambda item: item["score"], reverse=True)
    print(json.dumps({"winner": reports[0], "all_candidates": reports}, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()

python migrate.py > migration-report.json