Cursor、Claude Code 与 Copilot：如何做可复现评测

decision: choose a default tool for bounded maintenance pull requests
candidates:
  - candidate-a-local-agent
  - candidate-b-cloud-agent
  - current-human-workflow
repositories:
  - typescript-service
  - python-library
task_slices:
  - bug-fix
  - test-addition
  - dependency-safe-update
  - narrow-refactor
excluded:
  - production-operations
  - schema-migrations
  - security-critical-auth-changes
primary_outcome: accepted_change_without_material_rework
hard_gates:
  - no_policy_violation
  - no_unapproved_network
  - no_secret_access
  - required_tests_pass

人类分钟数 = 主动操作 + 评审 + 返工
墙钟分钟数 = 从委派到终态的经过时间

总成本
  = 工具和计算直接成本
  + 人力费率 * 人类分钟数 / 60
  + 事故和回滚成本

每个已验收变更成本
  = 总成本 / 已验收变更数

{
  "experiment_id": "coding-tools-maintenance-v1",
  "labor_cost_per_hour": 120,
  "gates": {
    "min_acceptance_rate": 0.6,
    "max_policy_violations": 0,
    "max_regressions": 0,
    "max_cost_per_accepted_change": 900
  },
  "trials": [
    {
      "trial_id": "candidate-a-task-001",
      "tool": "candidate-a",
      "task_id": "task-001",
      "task_type": "bug-fix",
      "tool_version": "pinned-or-vendor-managed",
      "model": "pinned-or-vendor-managed",
      "environment_hash": "sha256:...",
      "prompt_hash": "sha256:...",
      "status": "accepted",
      "active_minutes": 12,
      "review_minutes": 18,
      "rework_minutes": 4,
      "wall_minutes": 41,
      "direct_cost": 7.4,
      "regressions": 0,
      "policy_violations": 0
    }
  ]
}

from __future__ import annotations

import argparse
import json
import math
import statistics
from collections import defaultdict
from pathlib import Path
from typing import Any


STATUSES = {
    "accepted",
    "rejected_correctness",
    "rejected_scope",
    "rejected_security",
    "rejected_maintainability",
    "clarification_required",
    "budget_exhausted",
}
NUMERIC_FIELDS = {
    "active_minutes",
    "review_minutes",
    "rework_minutes",
    "wall_minutes",
    "direct_cost",
    "regressions",
    "policy_violations",
}
STRING_FIELDS = {
    "trial_id",
    "tool",
    "task_id",
    "task_type",
    "tool_version",
    "model",
    "environment_hash",
    "prompt_hash",
    "status",
}
GATE_NAMES = {
    "min_acceptance_rate",
    "max_policy_violations",
    "max_regressions",
    "max_cost_per_accepted_change",
}


def is_number(value: Any) -> bool:
    return isinstance(value, (int, float)) and not isinstance(value, bool)


def wilson(successes: int, total: int, z: float = 1.96) -> list[float]:
    if total == 0:
        return [0.0, 0.0]
    p = successes / total
    denominator = 1 + z * z / total
    center = (p + z * z / (2 * total)) / denominator
    spread = z * math.sqrt(
        (p * (1 - p) + z * z / (4 * total)) / total
    ) / denominator
    return [max(0.0, center - spread), min(1.0, center + spread)]


def load(path: Path) -> dict[str, Any]:
    value = json.loads(path.read_text(encoding="utf-8"))
    if not isinstance(value, dict):
        raise ValueError("experiment root must be an object")
    return value


def validate(experiment: dict[str, Any]) -> list[str]:
    errors: list[str] = []
    labor = experiment.get("labor_cost_per_hour")
    gates = experiment.get("gates")
    trials = experiment.get("trials")
    if not is_number(labor) or labor < 0:
        errors.append("labor_cost_per_hour must be non-negative")
    if not isinstance(gates, dict):
        errors.append("gates must be an object")
        gates = {}
    if not isinstance(trials, list) or not trials:
        return errors + ["trials must be a non-empty list"]

    unknown_gates = set(gates) - GATE_NAMES
    if unknown_gates:
        errors.append(f"unknown gates: {sorted(unknown_gates)}")
    for name, value in gates.items():
        if not is_number(value) or value < 0:
            errors.append(f"gate {name} must be non-negative")

    pairs: set[tuple[str, str]] = set()
    tasks_by_tool: dict[str, set[str]] = defaultdict(set)
    for index, trial in enumerate(trials):
        if not isinstance(trial, dict):
            errors.append(f"trials[{index}] must be an object")
            continue
        for field in STRING_FIELDS:
            if not isinstance(trial.get(field), str) or not trial[field]:
                errors.append(f"trials[{index}].{field} must be a string")
        for field in NUMERIC_FIELDS:
            value = trial.get(field)
            if not is_number(value) or value < 0:
                errors.append(
                    f"trials[{index}].{field} must be non-negative"
                )
        status = trial.get("status")
        if isinstance(status, str) and status not in STATUSES:
            errors.append(f"trials[{index}].status is invalid")
        tool, task = trial.get("tool"), trial.get("task_id")
        if isinstance(tool, str) and isinstance(task, str):
            pair = (tool, task)
            if pair in pairs:
                errors.append(f"duplicate tool/task pair: {pair}")
            pairs.add(pair)
            tasks_by_tool[tool].add(task)

    if len(tasks_by_tool) < 2:
        errors.append("at least two tools are required")
    task_sets = list(tasks_by_tool.values())
    if task_sets and any(tasks != task_sets[0] for tasks in task_sets[1:]):
        errors.append("every tool must run the same task_id set")
    return errors


def summarize(experiment: dict[str, Any]) -> dict[str, dict[str, Any]]:
    groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
    for trial in experiment["trials"]:
        groups[trial["tool"]].append(trial)

    labor = experiment["labor_cost_per_hour"]
    output: dict[str, dict[str, Any]] = {}
    for tool, trials in sorted(groups.items()):
        accepted = sum(t["status"] == "accepted" for t in trials)
        human_minutes = [
            t["active_minutes"] + t["review_minutes"] + t["rework_minutes"]
            for t in trials
        ]
        total_human = sum(human_minutes)
        total_direct = sum(t["direct_cost"] for t in trials)
        total_cost = total_direct + labor * total_human / 60
        output[tool] = {
            "trials": len(trials),
            "accepted": accepted,
            "acceptance_rate": accepted / len(trials),
            "acceptance_rate_wilson_95": wilson(accepted, len(trials)),
            "median_human_minutes": statistics.median(human_minutes),
            "median_wall_minutes": statistics.median(
                t["wall_minutes"] for t in trials
            ),
            "accepted_goodput_per_human_hour": (
                accepted * 60 / total_human if total_human else None
            ),
            "total_cost": total_cost,
            "cost_per_accepted_change": (
                total_cost / accepted if accepted else None
            ),
            "regressions": sum(t["regressions"] for t in trials),
            "policy_violations": sum(
                t["policy_violations"] for t in trials
            ),
        }
    return output


def apply_gates(
    summaries: dict[str, dict[str, Any]], gates: dict[str, float]
) -> list[str]:
    errors: list[str] = []
    for tool, result in summaries.items():
        checks = {
            "min_acceptance_rate": (
                result["acceptance_rate"],
                lambda value, limit: value >= limit,
            ),
            "max_policy_violations": (
                result["policy_violations"],
                lambda value, limit: value <= limit,
            ),
            "max_regressions": (
                result["regressions"],
                lambda value, limit: value <= limit,
            ),
            "max_cost_per_accepted_change": (
                result["cost_per_accepted_change"],
                lambda value, limit: value is not None and value <= limit,
            ),
        }
        for name, limit in gates.items():
            value, passed = checks[name]
            if not passed(value, limit):
                errors.append(
                    f"{tool} failed {name}: measured={value} limit={limit}"
                )
    return errors


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("experiment", type=Path)
    args = parser.parse_args()
    try:
        experiment = load(args.experiment)
    except (OSError, json.JSONDecodeError, ValueError) as error:
        parser.error(str(error))
    errors = validate(experiment)
    if errors:
        for error in errors:
            print(f"ERROR: {error}")
        raise SystemExit(1)
    summaries = summarize(experiment)
    print(json.dumps(summaries, indent=2, sort_keys=True))
    failures = apply_gates(summaries, experiment["gates"])
    if failures:
        for failure in failures:
            print(f"ERROR: {failure}")
        raise SystemExit(1)


if __name__ == "__main__":
    main()

python coding_tool_gate.py experiment.json