AI 工具评估实战：从业务任务到可复现生产选型框架

产品 + 模型版本 + 系统指令 + 工具
+ 检索/索引版本 + 采样参数 + 区域 + 策略 + 日期

workload: support-ticket-resolution
population:
  languages: [zh, en]
  length_buckets: [short, medium, long]
  risk_slices: [refund, account-access, self-harm]
required_outputs:
  schema: ticket-decision-v3
  evidence: cited-policy-sections
hard_constraints:
  regions: [approved-region]
  retention: no-training
  p95_latency_ms: 2500
  critical_safety_failures: 0
quality_gates:
  policy_accuracy_min: 0.94
  schema_validity_min: 0.995
  abstention_recall_min: 0.90
operations:
  cancellation: required
  usage_export: required
  rollback: required

基线通过 -> 候选通过：保持
基线失败 -> 候选通过：修复
基线通过 -> 候选失败：回归
基线失败 -> 候选失败：未解决

总成本 =
  模型输入 + 模型输出 + Cache/Reasoning 用量
  + 检索 + 工具/API 调用 + 审核
  + 存储 + 出网 + 重试 + 人工复核
  + 预留基础设施和运维

每个成功结果成本 = 总成本 / 合格业务结果数

package main

import (
	"fmt"
	"sort"
)

type Result struct {
	Passed          bool
	CriticalFailure bool
	LatencyMS       int
	CostUSD         float64
}

type Candidate struct {
	Name    string
	Results []Result
}

type Summary struct {
	Name           string
	Eligible       bool
	Passed         int
	Total          int
	P95LatencyMS   int
	TotalCostUSD   float64
	CostPerSuccess float64
}

func summarize(candidate Candidate, minPassRate float64, maxP95MS int) (Summary, error) {
	if candidate.Name == "" || len(candidate.Results) == 0 {
		return Summary{}, fmt.Errorf("candidate needs a name and results")
	}

	latencies := make([]int, 0, len(candidate.Results))
	passed := 0
	criticalFailures := 0
	totalCost := 0.0

	for i, result := range candidate.Results {
		if result.LatencyMS < 0 || result.CostUSD < 0 {
			return Summary{}, fmt.Errorf("result %d has a negative measurement", i)
		}
		if result.Passed {
			passed++
		}
		if result.CriticalFailure {
			criticalFailures++
		}
		latencies = append(latencies, result.LatencyMS)
		totalCost += result.CostUSD
	}

	sort.Ints(latencies)
	p95Index := (95*len(latencies) + 99) / 100
	p95 := latencies[p95Index-1]
	passRate := float64(passed) / float64(len(candidate.Results))

	costPerSuccess := 0.0
	if passed > 0 {
		costPerSuccess = totalCost / float64(passed)
	}

	return Summary{
		Name:           candidate.Name,
		Eligible:       criticalFailures == 0 && passRate >= minPassRate && p95 <= maxP95MS,
		Passed:         passed,
		Total:          len(candidate.Results),
		P95LatencyMS:   p95,
		TotalCostUSD:   totalCost,
		CostPerSuccess: costPerSuccess,
	}, nil
}

func main() {
	candidates := []Candidate{
		{Name: "baseline", Results: []Result{
			{Passed: true, LatencyMS: 900, CostUSD: 0.018},
			{Passed: true, LatencyMS: 1100, CostUSD: 0.020},
			{Passed: false, LatencyMS: 800, CostUSD: 0.017},
		}},
		{Name: "candidate", Results: []Result{
			{Passed: true, LatencyMS: 700, CostUSD: 0.010},
			{Passed: true, LatencyMS: 760, CostUSD: 0.011},
			{Passed: true, LatencyMS: 820, CostUSD: 0.012},
		}},
	}

	for _, candidate := range candidates {
		summary, err := summarize(candidate, 0.80, 1200)
		if err != nil {
			panic(err)
		}
		fmt.Printf(
			"%s eligible=%t passed=%d/%d p95_ms=%d cost_per_success=$%.4f\n",
			summary.Name,
			summary.Eligible,
			summary.Passed,
			summary.Total,
			summary.P95LatencyMS,
			summary.CostPerSuccess,
		)
	}
}

baseline eligible=false passed=2/3 p95_ms=1100 cost_per_success=$0.0275
candidate eligible=true passed=3/3 p95_ms=820 cost_per_success=$0.0110

{
  "workload": "support-ticket-resolution",
  "candidate": "provider/model@revision",
  "fixture_set": "support-eval-v12",
  "prompt_revision": "sha256:...",
  "evaluator_revision": "rubric-v5",
  "hard_gates": {
    "critical_safety_failures": 0,
    "policy_accuracy_min": 0.94,
    "p95_latency_ms_max": 2500
  },
  "result": "shadow_only",
  "missing_coverage": ["voice", "new-market-locale"]
}