Cursor、Claude Code 与 Copilot：如何做可复现评测

decision: choose a default tool for bounded maintenance pull requests
candidates:
  - candidate-a-local-agent
  - candidate-b-cloud-agent
  - current-human-workflow
repositories:
  - typescript-service
  - python-library
task_slices:
  - bug-fix
  - test-addition
  - dependency-safe-update
  - narrow-refactor
excluded:
  - production-operations
  - schema-migrations
  - security-critical-auth-changes
primary_outcome: accepted_change_without_material_rework
hard_gates:
  - no_policy_violation
  - no_unapproved_network
  - no_secret_access
  - required_tests_pass

人类分钟数 = 主动操作 + 评审 + 返工
墙钟分钟数 = 从委派到终态的经过时间

总成本
  = 工具和计算直接成本
  + 人力费率 * 人类分钟数 / 60
  + 事故和回滚成本

每个已验收变更成本
  = 总成本 / 已验收变更数

{
  "experiment_id": "coding-tools-maintenance-v1",
  "labor_cost_per_hour": 120,
  "gates": {
    "min_acceptance_rate": 0.6,
    "max_policy_violations": 0,
    "max_regressions": 0,
    "max_cost_per_accepted_change": 900
  },
  "trials": [
    {
      "trial_id": "candidate-a-task-001",
      "tool": "candidate-a",
      "task_id": "task-001",
      "task_type": "bug-fix",
      "tool_version": "pinned-or-vendor-managed",
      "model": "pinned-or-vendor-managed",
      "environment_hash": "sha256:...",
      "prompt_hash": "sha256:...",
      "status": "accepted",
      "active_minutes": 12,
      "review_minutes": 18,
      "rework_minutes": 4,
      "wall_minutes": 41,
      "direct_cost": 7.4,
      "regressions": 0,
      "policy_violations": 0
    }
  ]
}

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"sort"
)

var validStatuses = map[string]bool{
	"accepted":                 true,
	"rejected_correctness":     true,
	"rejected_scope":           true,
	"rejected_security":        true,
	"rejected_maintainability": true,
	"clarification_required":   true,
	"budget_exhausted":         true,
	"environment_failure":      true,
	"invalid_trial":            true,
}

type Gates struct {
	MinAcceptanceRate        *float64 `json:"min_acceptance_rate"`
	MaxPolicyViolations      *float64 `json:"max_policy_violations"`
	MaxRegressions           *float64 `json:"max_regressions"`
	MaxCostPerAcceptedChange *float64 `json:"max_cost_per_accepted_change"`
}

type Trial struct {
	TrialID          string   `json:"trial_id"`
	Tool             string   `json:"tool"`
	TaskID           string   `json:"task_id"`
	TaskType         string   `json:"task_type"`
	ToolVersion      string   `json:"tool_version"`
	Model            string   `json:"model"`
	EnvironmentHash  string   `json:"environment_hash"`
	PromptHash       string   `json:"prompt_hash"`
	Status           string   `json:"status"`
	ActiveMinutes    *float64 `json:"active_minutes"`
	ReviewMinutes    *float64 `json:"review_minutes"`
	ReworkMinutes    *float64 `json:"rework_minutes"`
	WallMinutes      *float64 `json:"wall_minutes"`
	DirectCost       *float64 `json:"direct_cost"`
	Regressions      *float64 `json:"regressions"`
	PolicyViolations *float64 `json:"policy_violations"`
}

type Experiment struct {
	ExperimentID     string   `json:"experiment_id"`
	LaborCostPerHour *float64 `json:"labor_cost_per_hour"`
	Gates            Gates    `json:"gates"`
	Trials           []Trial  `json:"trials"`
}

type Summary struct {
	ValidTrials                 int        `json:"valid_trials"`
	OperationalFailures         int        `json:"operational_failures"`
	Accepted                    int        `json:"accepted"`
	AcceptanceRate              float64    `json:"acceptance_rate"`
	AcceptanceRateWilson95      [2]float64 `json:"acceptance_rate_wilson_95"`
	MedianHumanMinutes          float64    `json:"median_human_minutes"`
	MedianWallMinutes           float64    `json:"median_wall_minutes"`
	AcceptedGoodputPerHumanHour *float64   `json:"accepted_goodput_per_human_hour"`
	TotalCost                   float64    `json:"total_cost"`
	CostPerAcceptedChange       *float64   `json:"cost_per_accepted_change"`
	Regressions                 float64    `json:"regressions"`
	PolicyViolations            float64    `json:"policy_violations"`
}

func number(value *float64, field string) (float64, error) {
	if value == nil || math.IsNaN(*value) || math.IsInf(*value, 0) || *value < 0 {
		return 0, fmt.Errorf("%s must be finite and non-negative", field)
	}
	return *value, nil
}

func isDecisionStatus(status string) bool {
	return status != "environment_failure" && status != "invalid_trial"
}

func validate(experiment Experiment) error {
	if experiment.ExperimentID == "" {
		return errors.New("experiment_id is required")
	}
	labor, err := number(experiment.LaborCostPerHour, "labor_cost_per_hour")
	if err != nil || labor < 0 {
		return errors.New("labor_cost_per_hour must be finite and non-negative")
	}
	if len(experiment.Trials) == 0 {
		return errors.New("trials must not be empty")
	}

	gates := map[string]*float64{
		"min_acceptance_rate":          experiment.Gates.MinAcceptanceRate,
		"max_policy_violations":        experiment.Gates.MaxPolicyViolations,
		"max_regressions":              experiment.Gates.MaxRegressions,
		"max_cost_per_accepted_change": experiment.Gates.MaxCostPerAcceptedChange,
	}
	for name, value := range gates {
		if _, err := number(value, name); err != nil {
			return err
		}
	}
	if *experiment.Gates.MinAcceptanceRate > 1 {
		return errors.New("min_acceptance_rate must not exceed 1")
	}

	seenTrials := map[string]bool{}
	tools := map[string]bool{}
	decisionCounts := map[string]map[string]int{}
	for index, trial := range experiment.Trials {
		if trial.TrialID == "" || trial.Tool == "" || trial.TaskID == "" ||
			trial.TaskType == "" || trial.ToolVersion == "" || trial.Model == "" ||
			trial.EnvironmentHash == "" || trial.PromptHash == "" {
			return fmt.Errorf("trials[%d] has missing identity", index)
		}
		tools[trial.Tool] = true
		if seenTrials[trial.TrialID] {
			return fmt.Errorf("duplicate trial_id %q", trial.TrialID)
		}
		seenTrials[trial.TrialID] = true
		if !validStatuses[trial.Status] {
			return fmt.Errorf("trials[%d] has invalid status %q", index, trial.Status)
		}
		for name, value := range map[string]*float64{
			"active_minutes":    trial.ActiveMinutes,
			"review_minutes":    trial.ReviewMinutes,
			"rework_minutes":    trial.ReworkMinutes,
			"wall_minutes":      trial.WallMinutes,
			"direct_cost":       trial.DirectCost,
			"regressions":       trial.Regressions,
			"policy_violations": trial.PolicyViolations,
		} {
			if _, err := number(value, fmt.Sprintf("trials[%d].%s", index, name)); err != nil {
				return err
			}
		}
		if isDecisionStatus(trial.Status) {
			if decisionCounts[trial.Tool] == nil {
				decisionCounts[trial.Tool] = map[string]int{}
			}
			decisionCounts[trial.Tool][trial.TaskID]++
			if decisionCounts[trial.Tool][trial.TaskID] > 1 {
				return fmt.Errorf("multiple decision results for %s/%s", trial.Tool, trial.TaskID)
			}
		}
	}

	if len(decisionCounts) < 2 {
		return errors.New("at least two tools need decision-bearing trials")
	}
	if len(decisionCounts) != len(tools) {
		return errors.New("every tool needs at least one decision-bearing trial")
	}
	var baseline []string
	for tool, taskCounts := range decisionCounts {
		tasks := make([]string, 0, len(taskCounts))
		for task := range taskCounts {
			tasks = append(tasks, task)
		}
		sort.Strings(tasks)
		if baseline == nil {
			baseline = tasks
			continue
		}
		if fmt.Sprint(tasks) != fmt.Sprint(baseline) {
			return fmt.Errorf("tool %s does not cover the paired task set", tool)
		}
	}
	return nil
}

func wilson(successes, total int) [2]float64 {
	if total == 0 {
		return [2]float64{}
	}
	const z = 1.96
	p := float64(successes) / float64(total)
	denominator := 1 + z*z/float64(total)
	center := (p + z*z/(2*float64(total))) / denominator
	spread := z * math.Sqrt(
		(p*(1-p)+z*z/(4*float64(total)))/float64(total),
	) / denominator
	return [2]float64{math.Max(0, center-spread), math.Min(1, center+spread)}
}

func median(values []float64) float64 {
	sort.Float64s(values)
	middle := len(values) / 2
	if len(values)%2 == 1 {
		return values[middle]
	}
	return (values[middle-1] + values[middle]) / 2
}

func summarize(experiment Experiment) map[string]Summary {
	grouped := map[string][]Trial{}
	for _, trial := range experiment.Trials {
		grouped[trial.Tool] = append(grouped[trial.Tool], trial)
	}

	output := map[string]Summary{}
	for tool, trials := range grouped {
		var validTrials, operationalFailures, accepted int
		var humanMinutes, wallMinutes []float64
		var totalHuman, totalDirect, regressions, violations float64
		for _, trial := range trials {
			human := *trial.ActiveMinutes + *trial.ReviewMinutes + *trial.ReworkMinutes
			humanMinutes = append(humanMinutes, human)
			wallMinutes = append(wallMinutes, *trial.WallMinutes)
			totalHuman += human
			totalDirect += *trial.DirectCost
			regressions += *trial.Regressions
			violations += *trial.PolicyViolations
			if isDecisionStatus(trial.Status) {
				validTrials++
				if trial.Status == "accepted" {
					accepted++
				}
			} else {
				operationalFailures++
			}
		}
		totalCost := totalDirect + *experiment.LaborCostPerHour*totalHuman/60
		var goodput, costPerAccepted *float64
		if totalHuman > 0 {
			value := float64(accepted) * 60 / totalHuman
			goodput = &value
		}
		if accepted > 0 {
			value := totalCost / float64(accepted)
			costPerAccepted = &value
		}
		output[tool] = Summary{
			ValidTrials:                 validTrials,
			OperationalFailures:         operationalFailures,
			Accepted:                    accepted,
			AcceptanceRate:              float64(accepted) / float64(validTrials),
			AcceptanceRateWilson95:      wilson(accepted, validTrials),
			MedianHumanMinutes:          median(humanMinutes),
			MedianWallMinutes:           median(wallMinutes),
			AcceptedGoodputPerHumanHour: goodput,
			TotalCost:                   totalCost,
			CostPerAcceptedChange:       costPerAccepted,
			Regressions:                 regressions,
			PolicyViolations:            violations,
		}
	}
	return output
}

func gateFailures(summaries map[string]Summary, gates Gates) []string {
	var failures []string
	for tool, result := range summaries {
		if result.AcceptanceRate < *gates.MinAcceptanceRate {
			failures = append(failures, tool+" failed min_acceptance_rate")
		}
		if result.PolicyViolations > *gates.MaxPolicyViolations {
			failures = append(failures, tool+" failed max_policy_violations")
		}
		if result.Regressions > *gates.MaxRegressions {
			failures = append(failures, tool+" failed max_regressions")
		}
		if result.CostPerAcceptedChange == nil ||
			*result.CostPerAcceptedChange > *gates.MaxCostPerAcceptedChange {
			failures = append(failures, tool+" failed max_cost_per_accepted_change")
		}
	}
	sort.Strings(failures)
	return failures
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, "usage: go run coding_tool_gate.go EXPERIMENT.json")
		os.Exit(2)
	}
	file, err := os.Open(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	defer file.Close()

	var experiment Experiment
	decoder := json.NewDecoder(file)
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(&experiment); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	var extra any
	if err := decoder.Decode(&extra); err != io.EOF {
		fmt.Fprintln(os.Stderr, "experiment must contain exactly one JSON value")
		os.Exit(1)
	}
	if err := validate(experiment); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	summaries := summarize(experiment)
	encoded, _ := json.MarshalIndent(summaries, "", "  ")
	fmt.Println(string(encoded))
	if failures := gateFailures(summaries, experiment.Gates); len(failures) > 0 {
		for _, failure := range failures {
			fmt.Fprintln(os.Stderr, failure)
		}
		os.Exit(1)
	}
}

go run coding_tool_gate.go experiment.json