Prompt CI/CD：版本、评测、发布门禁与回滚

{
  "releaseId": "support-rag/sha256:4e89c6...",
  "sourceCommit": "9f5cb7...",
  "prompt": {
    "artifact": "prompts/support-answer.json",
    "sha256": "c8f4a2..."
  },
  "model": {
    "provider": "provider-a",
    "revision": "immutable-model-revision",
    "parameters": {
      "temperature": 0.2,
      "maxOutputTokens": 900
    }
  },
  "tools": {
    "schemaSha256": "2a70b1...",
    "policySha256": "79945d..."
  },
  "retrieval": {
    "corpusSnapshot": "support-docs-184",
    "indexBuild": "index-2026-08-09-03"
  },
  "output": {
    "schemaSha256": "b5cb3a...",
    "parserRevision": "parser/7d34fe..."
  },
  "evaluation": {
    "datasetRevision": "support-eval/31",
    "runnerRevision": "eval-runner/8c18a0...",
    "judgeRevision": "judge-rubric/12"
  }
}

task: refund_explanation
locale: zh-CN
risk: financial
tenant_policy: standard
input_source: production_sample
expected_contract: cited_answer_without_mutation

package main

import (
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"os"
	"sort"
)

var requiredManifestPaths = [][]string{
	{"prompt", "sha256"},
	{"model", "revision"},
	{"tools", "schemaSha256"},
	{"tools", "policySha256"},
	{"retrieval", "indexBuild"},
	{"output", "schemaSha256"},
	{"evaluation", "datasetRevision"},
	{"evaluation", "runnerRevision"},
}

type EvalResult struct {
	CaseID  string             `json:"caseId"`
	Repeat  int                `json:"repeat"`
	Slice   string             `json:"slice"`
	Metrics map[string]float64 `json:"metrics"`
}

type Rule struct {
	Metric       string   `json:"metric"`
	Slice        string   `json:"slice"`
	Direction    string   `json:"direction"`
	MinimumDelta *float64 `json:"minimumDelta,omitempty"`
	MaximumDelta *float64 `json:"maximumDelta,omitempty"`
}

type Policy struct {
	Confidence       float64 `json:"confidence"`
	BootstrapSamples int     `json:"bootstrapSamples"`
	Seed             int64   `json:"seed"`
	Rules            []Rule  `json:"rules"`
}

type Check struct {
	Metric    string     `json:"metric"`
	Slice     string     `json:"slice"`
	Pairs     int        `json:"pairs"`
	MeanDelta float64    `json:"meanDelta"`
	Interval  [2]float64 `json:"interval"`
	Passed    bool       `json:"passed"`
}

type Report struct {
	ReleaseDigest string  `json:"releaseDigest"`
	Passed        bool    `json:"passed"`
	Checks        []Check `json:"checks"`
}

type sampleKey struct {
	CaseID string
	Repeat int
}

func loadJSON(path string, target any) error {
	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(target); err != nil {
		return err
	}
	if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
		return errors.New("expected one JSON value")
	}
	return nil
}

func nestedValue(document map[string]any, path []string) (any, error) {
	var value any = document
	for _, key := range path {
		object, ok := value.(map[string]any)
		if !ok {
			return nil, fmt.Errorf("missing manifest field: %v", path)
		}
		value, ok = object[key]
		if !ok {
			return nil, fmt.Errorf("missing manifest field: %v", path)
		}
	}
	if text, ok := value.(string); ok && text == "" {
		return nil, fmt.Errorf("empty manifest field: %v", path)
	}
	return value, nil
}

func releaseDigest(manifest map[string]any) (string, error) {
	for _, path := range requiredManifestPaths {
		if _, err := nestedValue(manifest, path); err != nil {
			return "", err
		}
	}
	payload, err := json.Marshal(manifest)
	if err != nil {
		return "", err
	}
	digest := sha256.Sum256(payload)
	return fmt.Sprintf("sha256:%x", digest), nil
}

func selectResults(
	rows []EvalResult,
	metric string,
	slice string,
) (map[sampleKey]float64, error) {
	selected := make(map[sampleKey]float64)
	for _, row := range rows {
		if row.Slice != slice {
			continue
		}
		if row.CaseID == "" || row.Repeat < 0 {
			return nil, errors.New("caseId must be non-empty and repeat nonnegative")
		}
		value, ok := row.Metrics[metric]
		if !ok {
			return nil, fmt.Errorf("missing metric %q for case %q", metric, row.CaseID)
		}
		key := sampleKey{CaseID: row.CaseID, Repeat: row.Repeat}
		if _, exists := selected[key]; exists {
			return nil, fmt.Errorf("duplicate result for %q repeat %d", row.CaseID, row.Repeat)
		}
		selected[key] = value
	}
	return selected, nil
}

func pairedDifferences(
	baseline []EvalResult,
	candidate []EvalResult,
	metric string,
	slice string,
) ([]float64, error) {
	left, err := selectResults(baseline, metric, slice)
	if err != nil {
		return nil, err
	}
	right, err := selectResults(candidate, metric, slice)
	if err != nil {
		return nil, err
	}
	if len(left) == 0 || len(left) != len(right) {
		return nil, fmt.Errorf("unpaired or empty results for %s/%s", metric, slice)
	}
	keys := make([]sampleKey, 0, len(left))
	for key := range left {
		if _, exists := right[key]; !exists {
			return nil, fmt.Errorf("candidate is missing %+v", key)
		}
		keys = append(keys, key)
	}
	sort.Slice(keys, func(i, j int) bool {
		if keys[i].CaseID == keys[j].CaseID {
			return keys[i].Repeat < keys[j].Repeat
		}
		return keys[i].CaseID < keys[j].CaseID
	})
	differences := make([]float64, 0, len(keys))
	for _, key := range keys {
		differences = append(differences, right[key]-left[key])
	}
	return differences, nil
}

func mean(values []float64) float64 {
	total := 0.0
	for _, value := range values {
		total += value
	}
	return total / float64(len(values))
}

func bootstrapInterval(
	values []float64,
	confidence float64,
	samples int,
	seed int64,
) [2]float64 {
	random := rand.New(rand.NewSource(seed))
	estimates := make([]float64, samples)
	for sample := range estimates {
		draw := make([]float64, len(values))
		for index := range draw {
			draw[index] = values[random.Intn(len(values))]
		}
		estimates[sample] = mean(draw)
	}
	sort.Float64s(estimates)
	tail := (1 - confidence) / 2
	low := int(tail * float64(samples-1))
	high := int((1 - tail) * float64(samples-1))
	return [2]float64{estimates[low], estimates[high]}
}

func evaluateGate(
	baseline []EvalResult,
	candidate []EvalResult,
	policy Policy,
) ([]Check, bool, error) {
	if policy.Confidence <= 0 || policy.Confidence >= 1 {
		return nil, false, errors.New("confidence must be between zero and one")
	}
	if policy.BootstrapSamples < 100 || len(policy.Rules) == 0 {
		return nil, false, errors.New("bootstrapSamples must be at least 100 and rules cannot be empty")
	}
	checks := make([]Check, 0, len(policy.Rules))
	for index, rule := range policy.Rules {
		differences, err := pairedDifferences(
			baseline,
			candidate,
			rule.Metric,
			rule.Slice,
		)
		if err != nil {
			return nil, false, err
		}
		interval := bootstrapInterval(
			differences,
			policy.Confidence,
			policy.BootstrapSamples,
			policy.Seed+int64(index),
		)
		passed := false
		switch rule.Direction {
		case "higher":
			if rule.MinimumDelta == nil {
				return nil, false, errors.New("higher rule requires minimumDelta")
			}
			passed = interval[0] >= *rule.MinimumDelta
		case "lower":
			if rule.MaximumDelta == nil {
				return nil, false, errors.New("lower rule requires maximumDelta")
			}
			passed = interval[1] <= *rule.MaximumDelta
		default:
			return nil, false, fmt.Errorf("unknown direction %q", rule.Direction)
		}
		checks = append(checks, Check{
			Metric:    rule.Metric,
			Slice:     rule.Slice,
			Pairs:     len(differences),
			MeanDelta: mean(differences),
			Interval:  interval,
			Passed:    passed,
		})
	}
	for _, check := range checks {
		if !check.Passed {
			return checks, false, nil
		}
	}
	return checks, true, nil
}

func main() {
	var manifest map[string]any
	var baseline, candidate []EvalResult
	var policy Policy
	for path, target := range map[string]any{
		"release-manifest.json":  &manifest,
		"baseline-results.json":  &baseline,
		"candidate-results.json": &candidate,
		"release-policy.json":    &policy,
	} {
		if err := loadJSON(path, target); err != nil {
			fmt.Fprintln(os.Stderr, path+":", err)
			os.Exit(1)
		}
	}
	digest, err := releaseDigest(manifest)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	checks, passed, err := evaluateGate(baseline, candidate, policy)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	payload, err := json.MarshalIndent(
		Report{ReleaseDigest: digest, Passed: passed, Checks: checks},
		"",
		"  ",
	)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if err := os.WriteFile("gate-report.json", append(payload, '\n'), 0o644); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if !passed {
		fmt.Fprintln(os.Stderr, "release gate failed; inspect gate-report.json")
		os.Exit(1)
	}
}

{
  "confidence": 0.95,
  "bootstrapSamples": 10000,
  "seed": 20260809,
  "rules": [
    {
      "metric": "taskScore",
      "slice": "all",
      "direction": "higher",
      "minimumDelta": -0.01
    },
    {
      "metric": "unsafeActionRate",
      "slice": "financial",
      "direction": "lower",
      "maximumDelta": 0.0
    }
  ]
}

name: prompt-contracts
on:
  pull_request:
    paths:
      - "prompts/**"
      - "evals/**"
      - "release-policy.json"

permissions:
  contents: read

jobs:
  deterministic-contracts:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Check trusted repository workspace
        run: |
          test -f scripts/validate-release.go
          go run scripts/validate-release.go --no-network