AI 硬件上市两个月后，真正值得复盘的是什么

# export_and_quantize.py
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer

model_name = "your-org/intent-classifier-small"
model = AutoModelForSequenceClassification.from_pretrained(model_name)
tokenizer = AutoTokenizer.from_pretrained(model_name)

# 导出为 ONNX
dummy_input = tokenizer("帮我打开客厅的灯", return_tensors="pt")
torch.onnx.export(
    model,
    (dummy_input["input_ids"], dummy_input["attention_mask"]),
    "intent_model.onnx",
    input_names=["input_ids", "attention_mask"],
    output_names=["logits"],
    dynamic_axes={
        "input_ids": {0: "batch", 1: "seq_len"},
        "attention_mask": {0: "batch", 1: "seq_len"},
    },
    opset_version=14,
)

# 用 onnxruntime 量化（需要先安装 onnxruntime-tools）
# 命令行方式更简洁：
# python -m onnxruntime.quantization.preprocess --input intent_model.onnx --output intent_model_preprocessed.onnx

# 安装依赖
pip install onnxruntime onnx

# 预处理模型（量化前必须步骤）
python -m onnxruntime.quantization.preprocess \
  --input intent_model.onnx \
  --output intent_model_preprocessed.onnx

# 执行 INT8 动态量化
python -c "
from onnxruntime.quantization import quantize_dynamic, QuantType
quantize_dynamic(
    'intent_model_preprocessed.onnx',
    'intent_model_int8.onnx',
    weight_type=QuantType.QInt8,
)
print('量化完成，模型已保存为 intent_model_int8.onnx')
"

# edge_inference_benchmark.py
"""在边缘设备上运行 ONNX 推理并测量延迟"""
import time
import numpy as np
import onnxruntime as ort

# 加载量化模型
session = ort.InferenceSession(
    "intent_model_int8.onnx",
    providers=["CPUExecutionProvider"],  # 边缘设备通常只有 CPU
)

# 模拟用户输入（实际项目中替换为真实 tokenizer 输出）
input_ids = np.array([[101, 3352, 1059, 2140, 749, 1383, 639, 102]], dtype=np.int64)
attention_mask = np.array([[1, 1, 1, 1, 1, 1, 1, 1]], dtype=np.int64)

# 单次推理
result = session.run(
    ["logits"],
    {"input_ids": input_ids, "attention_mask": attention_mask},
)
print(f"推理结果 logits: {result[0]}")

# 批量延迟测试（100 次取均值和 P95）
latencies = []
for _ in range(100):
    start = time.perf_counter()
    session.run(
        ["logits"],
        {"input_ids": input_ids, "attention_mask": attention_mask},
    )
    latencies.append((time.perf_counter() - start) * 1000)  # 转为 ms

avg = np.mean(latencies)
p95 = np.percentile(latencies, 95)
print(f"平均延迟: {avg:.1f}ms | P95 延迟: {p95:.1f}ms")
print(f"如果 P95 > 150ms，在硬件上用户会感知到明显等待，需要进一步优化")

# 每次模型变更后跑一次 benchmark，结果追加到 CSV
# 方便对比不同量化策略、不同芯片的实际表现

python edge_inference_benchmark.py | tee /tmp/bench_result.txt

# 提取关键指标写入记录
echo "$(date +%Y-%m-%d),intent_model_int8,$(grep '平均延迟' /tmp/bench_result.txt | grep -oP '[\d.]+ms' | head -1),$(grep 'P95' /tmp/bench_result.txt | grep -oP '[\d.]+ms' | head -1)" >> bench_history.csv

# 查看历史对比
column -t -s',' bench_history.csv