用 Azure AI Foundry 管理模型、成本与质量：从选模型到规模化运营的全流程实践

# pip install azure-ai-projects azure-identity

from azure.ai.projects import AIProjectClient
from azure.identity import DefaultAzureCredential

# 从环境变量读取：AZURE_AI_PROJECT_ENDPOINT
project_client = AIProjectClient(
    endpoint="https://your-project.cognitiveservices.azure.com",
    credential=DefaultAzureCredential(),
)

# 列出当前项目可用的模型部署
deployments = project_client.deployments.list()

for d in deployments:
    print(f"模型: {d.model_name}  |  类型: {d.model_type}  |  SKU: {d.sku_name}  |  速率限制: {d.rate_limits}")

import os
from openai import AzureOpenAI

# 简单任务用小模型，复杂任务用大模型
CHEAP_MODEL = "gpt-4o-mini"   # 成本约 $0.15/1M input tokens
EXPENSIVE_MODEL = "gpt-4o"    # 成本约 $2.5/1M input tokens

client = AzureOpenAI(
    api_key=os.getenv("AZURE_OPENAI_API_KEY"),
    api_version="2024-08-01-preview",
    azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
)

def smart_complete(prompt: str, max_tokens: int = 500, token_budget: int = 2000) -> str:
    """根据 prompt 长度和 token 预算自动选模型"""

    # 粗估 input token 数（英文约 4 字符/token，中文约 2 字符/token）
    estimated_input_tokens = len(prompt) // 3

    if estimated_input_tokens + max_tokens > token_budget:
        model = CHEAP_MODEL
        print(f"⚠️ 预估超预算，降级到 {model}")
    else:
        model = EXPENSIVE_MODEL
        print(f"✅ 预算充足，使用 {model}")

    response = client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": prompt}],
        max_tokens=max_tokens,
    )

    # 打印实际消耗（生产环境应写入监控系统）
    usage = response.usage
    print(f"实际消耗: input={usage.prompt_tokens}, output={usage.completion_tokens}, model={model}")

    return response.choices[0].message.content

# 测试：短 prompt 用大模型，长 prompt 自动降级
short_result = smart_complete("用一句话解释什么是微服务")
long_prompt = "请详细分析以下 5000 字的产品需求文档..." + "冗余填充内容" * 200
long_result = smart_complete(long_prompt, max_tokens=800, token_budget=1500)

from azure.ai.projects import AIProjectClient
from azure.identity import DefaultAzureCredential
from azure.ai.evaluation import QAEvaluator, evaluate

project_client = AIProjectClient(
    endpoint="https://your-project.cognitiveservices.azure.com",
    credential=DefaultAzureCredential(),
)

# 1. 准备评估数据集（实际应从文件加载）
eval_data = [
    {
        "question": "Azure Functions 的冷启动延迟典型值是多少？",
        "answer": "通常在 1-5 秒之间，取决于运行时语言和包大小。",  # 标准答案
    },
    {
        "question": "什么是 Cosmos DB 的 RU？",
        "answer": "Request Unit，是 Cosmos DB 中衡量操作成本的抽象单位，1 RU ≈ 读取 1KB 文档的成本。",
    },
]

# 2. 用 GPT-4o-mini 生成候选输出（此处简化，实际应批量调用模型）
model_outputs = []
for item in eval_data:
    # 假设你已经拿到了模型的回答
    model_outputs.append({
        "question": item["question"],
        "answer": item["answer"],
        "model_response": "Azure Functions 冷启动大约需要几秒钟时间。",  # 模型实际输出
    })

# 3. 运行评估
qa_eval = QAEvaluator(model_config=project_client.models.get("gpt-4o-mini"))

result = evaluate(
    evaluation_name="model-quality-check",
    data=model_outputs,
    evaluators={"qa": qa_eval},
)

# 打印每条结果的分数
for row in result.rows:
    print(f"问题: {row['question']}")
    print(f"  QA 准确性分数: {row.get('qa.accuracy', 'N/A')}")
    print(f"  QA 相关性分数: {row.get('qa.relevance', 'N/A')}")

# 模型部署配置示例（用于 Azure AI Foundry 项目）
apiVersion: 2024-08-01
kind: Deployment
metadata:
  name: gpt-4o-mini-prod
spec:
  model:
    name: gpt-4o-mini
    version: "2024-07-18"       # 锁定版本，避免静默升级
  sku:
    name: GlobalStandard
    capacity: 50                # 速率上限：50 requests/min
  contentSafety:
    enabled: true               # 开启内容安全过滤
    severityThreshold: medium   # 中等及以上风险内容拦截
  tags:
    env: production
    team: platform-core
    costCenter: ai-infra-001    # 成本归属标签