从排行榜数字到可重放证据：UK AISI 与 EvalEval 的基准复现思路

import argparse
import hashlib
import json
import platform
import random
import sys
from datetime import datetime, timezone
from pathlib import Path

CASES = [
    {'id': 'geo-1', 'prompt': 'Capital of France?', 'expected': 'paris'},
    {'id': 'math-1', 'prompt': 'What is 2 + 2?', 'expected': '4'},
    {'id': 'color-1', 'prompt': 'Color of a clear daytime sky?', 'expected': 'blue'},
]


def canonical_bytes(value):
    return json.dumps(
        value,
        ensure_ascii=False,
        sort_keys=True,
        separators=(',', ':'),
    ).encode('utf-8')


def sha256_bytes(value):
    return hashlib.sha256(value).hexdigest()


def invoke_mock(prompt):
    text = prompt.lower()
    if 'france' in text:
        return 'Paris'
    if '2 + 2' in text:
        return '4'
    if 'daytime sky' in text:
        return 'Blue'
    return 'unknown'


def exact_match(response, expected):
    return int(response.strip().casefold() == expected.strip().casefold())


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--model', default='mock-model-v1')
    parser.add_argument('--seed', type=int, default=7)
    parser.add_argument('--output', default='run.json')
    args = parser.parse_args()

    random.seed(args.seed)
    runner_path = Path(__file__)

    config = {
        'model': args.model,
        'seed': args.seed,
        'temperature': 0,
        'evaluator': 'casefold-exact-match-v1',
    }

    records = []
    for case in CASES:
        response = invoke_mock(case['prompt'])
        records.append({
            'case_id': case['id'],
            'prompt': case['prompt'],
            'prompt_sha256': sha256_bytes(case['prompt'].encode('utf-8')),
            'expected': case['expected'],
            'response': response,
            'response_sha256': sha256_bytes(response.encode('utf-8')),
            'score': exact_match(response, case['expected']),
            'status': 'ok',
        })

    artifact = {
        'schema_version': '1.0',
        'created_at': datetime.now(timezone.utc).isoformat(),
        'dataset_sha256': sha256_bytes(canonical_bytes(CASES)),
        'runner_sha256': sha256_bytes(runner_path.read_bytes()),
        'config': config,
        'environment': {
            'python': sys.version,
            'platform': platform.platform(),
        },
        'summary': {
            'correct': sum(item['score'] for item in records),
            'total': len(records),
            'accuracy': sum(item['score'] for item in records) / len(records),
        },
        'records': records,
    }

    Path(args.output).write_text(
        json.dumps(artifact, ensure_ascii=False, indent=2),
        encoding='utf-8',
    )
    print(json.dumps(artifact['summary'], indent=2))
    print(f'Wrote reproducibility artifact to {args.output}')


if __name__ == '__main__':
    main()

python benchmark.py --model mock-model-v1 --seed 7 --output run-a.json
python benchmark.py --model mock-model-v1 --seed 7 --output run-b.json

python - <<'PY'
import json

a = json.load(open('run-a.json', encoding='utf-8'))
b = json.load(open('run-b.json', encoding='utf-8'))

for key in ('dataset_sha256', 'runner_sha256', 'config', 'records', 'summary'):
    print(f'{key}:', 'MATCH' if a[key] == b[key] else 'DIFFERENT')
PY

schema_version: '1.0'
run:
  benchmark: qa-mini
  dataset_digest: sha256:REPLACE_ME
  model: provider/model-snapshot
  seed: 7
  decoding:
    temperature: 0
    top_p: 1
    max_output_tokens: 128
scoring:
  evaluator: exact-match
  evaluator_version: '1.0.0'
artifacts:
  raw_responses: artifacts/responses.jsonl
  scores: artifacts/scores.jsonl
  summary: artifacts/summary.json