feat(eval): moat claim surface hard-rejects ineligible scorecards (age-sb0)

Add ao eval scenario-moat aggregation with fail-closed rejection of
moat_eligible=false plumbing scorecards before any moat verdict is rendered.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Boden Fuller
2026-06-17 08:38:20 -04:00
parent 37f97bce19
commit 00fc9858eb
13 changed files with 485 additions and 6 deletions
+73
View File
@@ -0,0 +1,73 @@
// practices: [llm-eval-harness]
package main
import (
"errors"
"fmt"
"github.com/spf13/cobra"
aoeval "github.com/boshu2/agentops/cli/internal/eval"
)
var (
evalScenarioMoatScorecards []string
evalScenarioMoatOutput string
)
var evalScenarioMoatCmd = &cobra.Command{
Use: "scenario-moat",
Short: "Aggregate moat-eligible scenario A/B scorecards into a publication verdict",
Long: `Render a moat positive/null/inconclusive verdict over one or more
ScenarioDeltaScorecard JSON artifacts from ao eval scenario-ab.
The claim surface fail-closes on any scorecard with moat_eligible=false — a
fact-recall/plumbing scorecard can pass its own gate but must NEVER be aggregated
into a moat verdict (age-6ys/age-sb0). See docs/evals/applied-ood-claim-rule.md.`,
Args: cobra.NoArgs,
SilenceUsage: true,
RunE: func(cmd *cobra.Command, _ []string) error {
if len(evalScenarioMoatScorecards) == 0 {
return fmt.Errorf("at least one --scorecard path is required")
}
cards := make([]aoeval.ScenarioDeltaScorecard, 0, len(evalScenarioMoatScorecards))
for _, path := range evalScenarioMoatScorecards {
card, err := aoeval.LoadScenarioDeltaScorecard(path)
if err != nil {
return err
}
cards = append(cards, card)
}
result, err := aoeval.AggregateMoatClaim(cards)
if err != nil {
var ineligible aoeval.ErrMoatIneligibleScorecard
if errors.As(err, &ineligible) {
fmt.Fprintf(cmd.ErrOrStderr(), "REJECTED: %s\n", err)
}
return err
}
if err := aoeval.WriteMoatClaimResult(evalScenarioMoatOutput, result); err != nil {
return err
}
if GetOutput() == "json" {
return writeEvalJSON(cmd, result)
}
fmt.Fprintf(cmd.OutOrStdout(),
"scenario-moat: verdict=%s scenarios=%d mean_delta=%.4f\n %s\n",
result.Verdict, result.ScenarioCount, result.MeanAggregateDelta, result.Reason,
)
if evalScenarioMoatOutput != "" {
fmt.Fprintf(cmd.OutOrStdout(), "Moat claim result: %s\n", evalScenarioMoatOutput)
}
if result.Verdict == aoeval.MoatVerdictInconclusive {
return fmt.Errorf("moat claim inconclusive — cannot publish positive or honest null")
}
return nil
},
}
func init() {
evalScenarioMoatCmd.Flags().StringArrayVar(&evalScenarioMoatScorecards, "scorecard", nil, "Path to a ScenarioDeltaScorecard JSON (repeatable)")
evalScenarioMoatCmd.Flags().StringVar(&evalScenarioMoatOutput, "output", "", "Write the MoatClaimResult JSON to this path")
evalCmd.AddCommand(evalScenarioMoatCmd)
}
+62
View File
@@ -0,0 +1,62 @@
package main
import (
"bytes"
"path/filepath"
"runtime"
"strings"
"testing"
)
func withMoatCmdReset(t *testing.T) {
t.Helper()
t.Cleanup(func() {
evalScenarioMoatScorecards = nil
evalScenarioMoatOutput = ""
})
}
func runScenarioMoatCmd(t *testing.T, scorecards []string, outPath string) (string, error) {
t.Helper()
cmd := evalScenarioMoatCmd
var buf bytes.Buffer
cmd.SetOut(&buf)
cmd.SetErr(&buf)
evalScenarioMoatScorecards = scorecards
evalScenarioMoatOutput = outPath
err := cmd.RunE(cmd, nil)
return buf.String(), err
}
func TestEvalScenarioMoatRejectsPlumbingScorecard(t *testing.T) {
withMoatCmdReset(t)
root := repoRootForEvalCmd(t)
plumbing := filepath.Join(root, "evals/scenarios/fixtures/scenario-ab-fact-recall-plumbing.scorecard.json")
_, err := runScenarioMoatCmd(t, []string{plumbing}, "")
if err == nil {
t.Fatal("expected error when aggregating moat_eligible=false scorecard")
}
if !strings.Contains(err.Error(), "moat_eligible=false") {
t.Fatalf("error = %q, want moat_eligible=false rejection", err.Error())
}
}
func TestEvalScenarioMoatPositiveFromFixture(t *testing.T) {
withMoatCmdReset(t)
root := repoRootForEvalCmd(t)
valid := filepath.Join(root, "evals/scenarios/fixtures/scenario-ab-valid-redacted.scorecard.json")
out := filepath.Join(t.TempDir(), "moat-claim.json")
_, err := runScenarioMoatCmd(t, []string{valid}, out)
if err != nil {
t.Fatalf("runScenarioMoatCmd: %v", err)
}
}
func repoRootForEvalCmd(t *testing.T) string {
t.Helper()
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
return filepath.Join(filepath.Dir(file), "..", "..", "..")
}
+16
View File
@@ -1697,6 +1697,22 @@ ao eval scenario-ab [flags]
--token-budget int Fail the gate if summed arm token cost exceeds this (0 = default 200000)
```
#### `ao eval scenario-moat`
Render a moat positive/null/inconclusive verdict over one or more
```
ao eval scenario-moat [flags]
```
**Flags:**
```
-h, --help help for scenario-moat
--output string Write the MoatClaimResult JSON to this path
--scorecard stringArray Path to a ScenarioDeltaScorecard JSON (repeatable)
```
#### `ao eval scorecard`
Build an eval scorecard from run records
+167
View File
@@ -0,0 +1,167 @@
package eval
import (
"encoding/json"
"fmt"
"os"
"strings"
"time"
)
// MoatClaimVerdict is the publication-tier verdict for the gold/corpus axis when
// aggregating moat-eligible ScenarioDeltaScorecards. See
// docs/evals/applied-ood-claim-rule.md.
type MoatClaimVerdict string
const (
// MoatVerdictPositive means eligible scorecards collectively support a moat
// positive claim under the locked publication rule.
MoatVerdictPositive MoatClaimVerdict = "moat_positive"
// MoatVerdictHonestNull means eligible scorecards had headroom but the corpus
// did not improve work — a valid null result.
MoatVerdictHonestNull MoatClaimVerdict = "honest_null"
// MoatVerdictInconclusive means the inputs cannot support positive or null.
MoatVerdictInconclusive MoatClaimVerdict = "inconclusive"
)
// MoatClaimResult is the persisted output of the moat claim aggregation surface
// (age-sb0). It renders a moat positive/null/inconclusive verdict over one or
// more scenario A/B scorecards.
type MoatClaimResult struct {
SchemaVersion int `json:"schema_version"`
GeneratedAt time.Time `json:"generated_at"`
Verdict MoatClaimVerdict `json:"verdict"`
Reason string `json:"reason"`
ScenarioCount int `json:"scenario_count"`
MeanAggregateDelta float64 `json:"mean_aggregate_delta"`
ScorecardScenarioIDs []string `json:"scorecard_scenario_ids"`
ExcludedCeiling []string `json:"excluded_ceiling_violation,omitempty"`
ExcludedGateFail []string `json:"excluded_gate_fail,omitempty"`
}
// ErrMoatIneligibleScorecard is returned when aggregation is asked to include a
// scorecard with moat_eligible=false. The claim surface fail-closes rather than
// silently mixing plumbing into a moat verdict (age-6ys/age-sb0).
type ErrMoatIneligibleScorecard struct {
ScenarioID string
VerdictClass string
}
func (e ErrMoatIneligibleScorecard) Error() string {
class := e.VerdictClass
if class == "" {
class = "unknown"
}
return fmt.Sprintf(
"moat claim aggregation refused: scorecard %q has moat_eligible=false (verdict_class=%q, NOT-moat-evidence/plumbing); cannot aggregate into a moat verdict (age-6ys/age-sb0)",
e.ScenarioID, class,
)
}
// LoadScenarioDeltaScorecard reads a ScenarioDeltaScorecard JSON artifact.
func LoadScenarioDeltaScorecard(path string) (ScenarioDeltaScorecard, error) {
data, err := os.ReadFile(path)
if err != nil {
return ScenarioDeltaScorecard{}, fmt.Errorf("read scenario delta scorecard %s: %w", path, err)
}
var card ScenarioDeltaScorecard
if err := json.Unmarshal(data, &card); err != nil {
return ScenarioDeltaScorecard{}, fmt.Errorf("decode scenario delta scorecard %s: %w", path, err)
}
if strings.TrimSpace(card.ScenarioID) == "" {
return ScenarioDeltaScorecard{}, fmt.Errorf("scenario delta scorecard %s has no scenario_id", path)
}
return card, nil
}
// AggregateMoatClaim renders a moat positive/null/inconclusive verdict over the
// provided scorecards. It fail-closes on any moat_eligible=false input — plumbing
// scorecards must never be aggregated into a moat claim (age-sb0).
func AggregateMoatClaim(cards []ScenarioDeltaScorecard) (MoatClaimResult, error) {
if len(cards) == 0 {
return MoatClaimResult{}, fmt.Errorf("moat claim aggregation requires at least one scorecard")
}
for _, card := range cards {
if !card.MoatEligible {
return MoatClaimResult{}, ErrMoatIneligibleScorecard{
ScenarioID: card.ScenarioID,
VerdictClass: card.VerdictClass,
}
}
}
now := time.Now().UTC()
result := MoatClaimResult{
SchemaVersion: 1,
GeneratedAt: now,
ScenarioCount: len(cards),
}
for _, card := range cards {
result.ScorecardScenarioIDs = append(result.ScorecardScenarioIDs, card.ScenarioID)
}
var admissible []ScenarioDeltaScorecard
var positiveEligible []ScenarioDeltaScorecard
for _, card := range cards {
if card.CeilingViolation {
result.ExcludedCeiling = append(result.ExcludedCeiling, card.ScenarioID)
continue
}
admissible = append(admissible, card)
if card.Gate.Pass && card.AggregateDelta > 0 {
positiveEligible = append(positiveEligible, card)
} else if !card.Gate.Pass {
result.ExcludedGateFail = append(result.ExcludedGateFail, card.ScenarioID)
}
}
if len(admissible) == 0 {
result.Verdict = MoatVerdictInconclusive
result.Reason = "no admissible scorecards after excluding ceiling violations"
return result, nil
}
var deltaSum float64
allHadHeadroom := true
for _, card := range admissible {
deltaSum += card.AggregateDelta
if card.Without.Score >= card.SatisfactionThreshold {
allHadHeadroom = false
}
}
result.MeanAggregateDelta = roundDelta(deltaSum / float64(len(admissible)))
switch {
case len(positiveEligible) == len(admissible):
result.Verdict = MoatVerdictPositive
result.Reason = fmt.Sprintf(
"all %d admissible scorecards have positive delta and passed gate; mean aggregate_delta=%.4f",
len(admissible), result.MeanAggregateDelta,
)
case allHadHeadroom && len(positiveEligible) == 0:
result.Verdict = MoatVerdictHonestNull
result.Reason = fmt.Sprintf(
"admissible scorecards had headroom but corpus did not beat control; mean aggregate_delta=%.4f",
result.MeanAggregateDelta,
)
default:
result.Verdict = MoatVerdictInconclusive
result.Reason = "mixed headroom or delta signals across admissible scorecards — cannot claim positive or honest null"
}
return result, nil
}
// WriteMoatClaimResult persists a MoatClaimResult JSON artifact.
func WriteMoatClaimResult(path string, result MoatClaimResult) error {
if strings.TrimSpace(path) == "" {
return fmt.Errorf("output path is required")
}
data, err := json.MarshalIndent(result, "", " ")
if err != nil {
return fmt.Errorf("marshal moat claim result: %w", err)
}
if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil {
return fmt.Errorf("write moat claim result: %w", err)
}
return nil
}
+123
View File
@@ -0,0 +1,123 @@
package eval
import (
"errors"
"path/filepath"
"runtime"
"testing"
)
func TestAggregateMoatClaimRejectsIneligibleScorecard(t *testing.T) {
plumbing := ScenarioDeltaScorecard{
ScenarioID: "s-plumbing-001",
VerdictClass: VerdictClassFactRecall,
MoatEligible: false,
AggregateDelta: 1.0,
SatisfactionThreshold: 0.8,
Without: ScenarioArmResult{Score: 0.1},
With: ScenarioArmResult{Score: 1.0},
Gate: ScenarioGate{Pass: true},
}
_, err := AggregateMoatClaim([]ScenarioDeltaScorecard{plumbing})
if err == nil {
t.Fatal("expected error for moat_eligible=false scorecard")
}
var ineligible ErrMoatIneligibleScorecard
if !errors.As(err, &ineligible) {
t.Fatalf("expected ErrMoatIneligibleScorecard, got %T: %v", err, err)
}
if ineligible.ScenarioID != "s-plumbing-001" {
t.Errorf("ScenarioID = %q, want s-plumbing-001", ineligible.ScenarioID)
}
}
func TestAggregateMoatClaimPositive(t *testing.T) {
card := ScenarioDeltaScorecard{
ScenarioID: "s-applied-001",
VerdictClass: VerdictClassAppliedOOD,
MoatEligible: true,
AggregateDelta: 0.57,
SatisfactionThreshold: 0.8,
Without: ScenarioArmResult{Score: 0.35},
With: ScenarioArmResult{Score: 0.92},
Gate: ScenarioGate{Pass: true},
}
result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card})
if err != nil {
t.Fatalf("AggregateMoatClaim: %v", err)
}
if result.Verdict != MoatVerdictPositive {
t.Errorf("Verdict = %q, want %q", result.Verdict, MoatVerdictPositive)
}
}
func TestAggregateMoatClaimHonestNull(t *testing.T) {
card := ScenarioDeltaScorecard{
ScenarioID: "s-applied-002",
VerdictClass: VerdictClassAppliedOOD,
MoatEligible: true,
AggregateDelta: -0.07,
SatisfactionThreshold: 0.8,
Without: ScenarioArmResult{Score: 0.35},
With: ScenarioArmResult{Score: 0.28},
Gate: ScenarioGate{Pass: false, Reasons: []string{"aggregate_delta <= 0", "with-gold score < threshold"}},
}
result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card})
if err != nil {
t.Fatalf("AggregateMoatClaim: %v", err)
}
if result.Verdict != MoatVerdictHonestNull {
t.Errorf("Verdict = %q, want %q", result.Verdict, MoatVerdictHonestNull)
}
}
func TestAggregateMoatClaimExcludesCeilingViolation(t *testing.T) {
card := ScenarioDeltaScorecard{
ScenarioID: "s-ceiling-001",
VerdictClass: VerdictClassAppliedOOD,
MoatEligible: true,
CeilingViolation: true,
AggregateDelta: 0,
SatisfactionThreshold: 0.8,
Without: ScenarioArmResult{Score: 0.95},
With: ScenarioArmResult{Score: 0.95},
Gate: ScenarioGate{Pass: false},
}
result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card})
if err != nil {
t.Fatalf("AggregateMoatClaim: %v", err)
}
if result.Verdict != MoatVerdictInconclusive {
t.Errorf("Verdict = %q, want %q", result.Verdict, MoatVerdictInconclusive)
}
if len(result.ExcludedCeiling) != 1 {
t.Errorf("ExcludedCeiling = %v, want [s-ceiling-001]", result.ExcludedCeiling)
}
}
func TestLoadScenarioDeltaScorecardFromFixture(t *testing.T) {
fixture := filepath.Join(repoRoot(t), "evals/scenarios/fixtures/scenario-ab-valid-redacted.scorecard.json")
card, err := LoadScenarioDeltaScorecard(fixture)
if err != nil {
t.Fatalf("LoadScenarioDeltaScorecard: %v", err)
}
if !card.MoatEligible {
t.Error("fixture scorecard should be moat_eligible")
}
result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card})
if err != nil {
t.Fatalf("AggregateMoatClaim: %v", err)
}
if result.Verdict != MoatVerdictPositive {
t.Errorf("fixture Verdict = %q, want %q", result.Verdict, MoatVerdictPositive)
}
}
func repoRoot(t *testing.T) string {
t.Helper()
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
return filepath.Join(filepath.Dir(file), "..", "..", "..")
}
+7
View File
@@ -561,6 +561,13 @@
"kind": "leaf",
"reason": "Covered by release smoke tests, direct command tests, or command handler tests."
},
{
"category": "public-tested",
"command": "eval scenario-moat",
"coverage_status": "covered",
"kind": "leaf",
"reason": "Covered by release smoke tests, direct command tests, or command handler tests."
},
{
"category": "public-tested",
"command": "eval scorecard",
+1
View File
@@ -84,6 +84,7 @@
| `ao eval outcomes ingest` | `public-stateful-fixture-needed` | `allowlisted` | Maps an Outcomes score to the council verdict record; core logic unit-tested (eval_outcomes_ingest_test.go ingestOutcomesScore); CLI smoke needs a score.json fixture (follow-up). |
| `ao eval run` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. |
| `ao eval scenario-ab` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. |
| `ao eval scenario-moat` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. |
| `ao eval scorecard` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. |
| `ao eval suite n-required` | `public-stateful-fixture-needed` | `allowlisted` | Computes eval-suite thresholds and needs a representative suite fixture. |
| `ao eval suite verdict` | `public-stateful-fixture-needed` | `allowlisted` | Projects eval-suite verdicts and needs a representative suite fixture. |
+3
View File
@@ -118,3 +118,6 @@ partly done:
as moat evidence; defined the applied-OOD class + validity checklist + locked
publication rule. Mechanical enforcement landed as `verdict_class` /
`moat_eligible` on `ScenarioDeltaScorecard`.
- 2026-06-18 (age-sb0): moat claim aggregation surface (`ao eval scenario-moat`)
fail-closes on `moat_eligible=false` inputs; renders moat_positive/honest_null/
inconclusive over eligible scorecards only.
@@ -41,7 +41,7 @@
},
"expectations": [
{"type": "exit_code", "value": 0},
{"type": "stdout_contains", "value": "cli-command-headings: top=88 sub=219 all=307"},
{"type": "stdout_contains", "value": "cli-command-headings: top=88 sub=220 all=308"},
{"type": "stdout_contains", "value": "cli-help-matrix-ok"}
],
"dimensions": ["correctness", "runtime_compatibility", "artifact_quality"],
@@ -17,7 +17,7 @@ top_count="$(rg -c '^### `ao ' "$DOCS_PATH")"
sub_count="$(rg -c '^#### `ao ' "$DOCS_PATH")"
all_count="$(rg -c '^#{3,4} `ao ' "$DOCS_PATH")"
if [[ "$top_count" != "88" || "$sub_count" != "219" || "$all_count" != "307" ]]; then
if [[ "$top_count" != "88" || "$sub_count" != "220" || "$all_count" != "308" ]]; then
printf 'unexpected command heading counts: top=%s sub=%s all=%s\n' "$top_count" "$sub_count" "$all_count" >&2
exit 1
fi
@@ -25,7 +25,7 @@ fi
# shellcheck disable=SC2016 # literal backticks delimit generated Markdown command headings.
mapfile -t commands < <(rg '^#{3,4} `ao ' "$DOCS_PATH" | sed -E 's/^.*`([^`]+)`.*/\1/')
if [[ "${#commands[@]}" -ne 307 ]]; then
if [[ "${#commands[@]}" -ne 308 ]]; then
printf 'unexpected command matrix size: %s\n' "${#commands[@]}" >&2
exit 1
fi
@@ -0,0 +1,27 @@
{
"schema_version": 1,
"scenario_id": "s-fixture-plumbing-001",
"scenario_path": "evals/scenarios/fixtures/sentinel-recall-plumbing.json",
"generated_at": "2026-06-17T12:00:00Z",
"without_gold": {
"arm": "without_gold",
"score": 0.0,
"token_cost": 500
},
"with_gold": {
"arm": "with_gold",
"score": 1.0,
"token_cost": 600
},
"aggregate_delta": 1.0,
"satisfaction_threshold": 0.8,
"token_budget": 200000,
"ceiling_violation": false,
"verdict_class": "fact-recall",
"moat_eligible": false,
"gate": {
"pass": true,
"reasons": []
},
"_note": "Plumbing fixture for age-sb0: must be rejected by moat claim aggregation."
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"schema_version": 2,
"generated_at": "2026-06-17T11:58:35Z",
"generated_at": "2026-06-17T12:37:19Z",
"summary": {
"skills": 71,
"hooks": 0,
+2 -2
View File
@@ -1366,8 +1366,8 @@
{
"name": "plan",
"source_skill": "skills/plan",
"source_hash": "129c2fec737a15f8b0abf10dbedc706ec52d94c74bc1bbbbaac28f90ca018275",
"generated_hash": "343d33512062b6232f30820a2f29d35a8a4e8b6e02e2194efb7032d5a90a5648"
"source_hash": "29465f3bacb12c1bc18a694b71c14987e217c3c0ee32f0123a617a27817fb260",
"generated_hash": "eb02068c559fdb80a0fada49a6438aad7ef6fd7f81a6e33024d7ecabcb77d00b"
},
{
"name": "post-mortem",