mirror of
https://github.com/boshu2/agentops.git
synced 2026-09-14 15:08:13 +08:00
feat(eval): moat claim surface hard-rejects ineligible scorecards (age-sb0)
Add ao eval scenario-moat aggregation with fail-closed rejection of moat_eligible=false plumbing scorecards before any moat verdict is rendered. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
// practices: [llm-eval-harness]
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
aoeval "github.com/boshu2/agentops/cli/internal/eval"
|
||||
)
|
||||
|
||||
var (
|
||||
evalScenarioMoatScorecards []string
|
||||
evalScenarioMoatOutput string
|
||||
)
|
||||
|
||||
var evalScenarioMoatCmd = &cobra.Command{
|
||||
Use: "scenario-moat",
|
||||
Short: "Aggregate moat-eligible scenario A/B scorecards into a publication verdict",
|
||||
Long: `Render a moat positive/null/inconclusive verdict over one or more
|
||||
ScenarioDeltaScorecard JSON artifacts from ao eval scenario-ab.
|
||||
|
||||
The claim surface fail-closes on any scorecard with moat_eligible=false — a
|
||||
fact-recall/plumbing scorecard can pass its own gate but must NEVER be aggregated
|
||||
into a moat verdict (age-6ys/age-sb0). See docs/evals/applied-ood-claim-rule.md.`,
|
||||
Args: cobra.NoArgs,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if len(evalScenarioMoatScorecards) == 0 {
|
||||
return fmt.Errorf("at least one --scorecard path is required")
|
||||
}
|
||||
cards := make([]aoeval.ScenarioDeltaScorecard, 0, len(evalScenarioMoatScorecards))
|
||||
for _, path := range evalScenarioMoatScorecards {
|
||||
card, err := aoeval.LoadScenarioDeltaScorecard(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cards = append(cards, card)
|
||||
}
|
||||
result, err := aoeval.AggregateMoatClaim(cards)
|
||||
if err != nil {
|
||||
var ineligible aoeval.ErrMoatIneligibleScorecard
|
||||
if errors.As(err, &ineligible) {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "REJECTED: %s\n", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := aoeval.WriteMoatClaimResult(evalScenarioMoatOutput, result); err != nil {
|
||||
return err
|
||||
}
|
||||
if GetOutput() == "json" {
|
||||
return writeEvalJSON(cmd, result)
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(),
|
||||
"scenario-moat: verdict=%s scenarios=%d mean_delta=%.4f\n %s\n",
|
||||
result.Verdict, result.ScenarioCount, result.MeanAggregateDelta, result.Reason,
|
||||
)
|
||||
if evalScenarioMoatOutput != "" {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Moat claim result: %s\n", evalScenarioMoatOutput)
|
||||
}
|
||||
if result.Verdict == aoeval.MoatVerdictInconclusive {
|
||||
return fmt.Errorf("moat claim inconclusive — cannot publish positive or honest null")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
evalScenarioMoatCmd.Flags().StringArrayVar(&evalScenarioMoatScorecards, "scorecard", nil, "Path to a ScenarioDeltaScorecard JSON (repeatable)")
|
||||
evalScenarioMoatCmd.Flags().StringVar(&evalScenarioMoatOutput, "output", "", "Write the MoatClaimResult JSON to this path")
|
||||
evalCmd.AddCommand(evalScenarioMoatCmd)
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func withMoatCmdReset(t *testing.T) {
|
||||
t.Helper()
|
||||
t.Cleanup(func() {
|
||||
evalScenarioMoatScorecards = nil
|
||||
evalScenarioMoatOutput = ""
|
||||
})
|
||||
}
|
||||
|
||||
func runScenarioMoatCmd(t *testing.T, scorecards []string, outPath string) (string, error) {
|
||||
t.Helper()
|
||||
cmd := evalScenarioMoatCmd
|
||||
var buf bytes.Buffer
|
||||
cmd.SetOut(&buf)
|
||||
cmd.SetErr(&buf)
|
||||
evalScenarioMoatScorecards = scorecards
|
||||
evalScenarioMoatOutput = outPath
|
||||
err := cmd.RunE(cmd, nil)
|
||||
return buf.String(), err
|
||||
}
|
||||
|
||||
func TestEvalScenarioMoatRejectsPlumbingScorecard(t *testing.T) {
|
||||
withMoatCmdReset(t)
|
||||
root := repoRootForEvalCmd(t)
|
||||
plumbing := filepath.Join(root, "evals/scenarios/fixtures/scenario-ab-fact-recall-plumbing.scorecard.json")
|
||||
_, err := runScenarioMoatCmd(t, []string{plumbing}, "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error when aggregating moat_eligible=false scorecard")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "moat_eligible=false") {
|
||||
t.Fatalf("error = %q, want moat_eligible=false rejection", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvalScenarioMoatPositiveFromFixture(t *testing.T) {
|
||||
withMoatCmdReset(t)
|
||||
root := repoRootForEvalCmd(t)
|
||||
valid := filepath.Join(root, "evals/scenarios/fixtures/scenario-ab-valid-redacted.scorecard.json")
|
||||
out := filepath.Join(t.TempDir(), "moat-claim.json")
|
||||
_, err := runScenarioMoatCmd(t, []string{valid}, out)
|
||||
if err != nil {
|
||||
t.Fatalf("runScenarioMoatCmd: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func repoRootForEvalCmd(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("runtime.Caller failed")
|
||||
}
|
||||
return filepath.Join(filepath.Dir(file), "..", "..", "..")
|
||||
}
|
||||
@@ -1697,6 +1697,22 @@ ao eval scenario-ab [flags]
|
||||
--token-budget int Fail the gate if summed arm token cost exceeds this (0 = default 200000)
|
||||
```
|
||||
|
||||
#### `ao eval scenario-moat`
|
||||
|
||||
Render a moat positive/null/inconclusive verdict over one or more
|
||||
|
||||
```
|
||||
ao eval scenario-moat [flags]
|
||||
```
|
||||
|
||||
**Flags:**
|
||||
|
||||
```
|
||||
-h, --help help for scenario-moat
|
||||
--output string Write the MoatClaimResult JSON to this path
|
||||
--scorecard stringArray Path to a ScenarioDeltaScorecard JSON (repeatable)
|
||||
```
|
||||
|
||||
#### `ao eval scorecard`
|
||||
|
||||
Build an eval scorecard from run records
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MoatClaimVerdict is the publication-tier verdict for the gold/corpus axis when
|
||||
// aggregating moat-eligible ScenarioDeltaScorecards. See
|
||||
// docs/evals/applied-ood-claim-rule.md.
|
||||
type MoatClaimVerdict string
|
||||
|
||||
const (
|
||||
// MoatVerdictPositive means eligible scorecards collectively support a moat
|
||||
// positive claim under the locked publication rule.
|
||||
MoatVerdictPositive MoatClaimVerdict = "moat_positive"
|
||||
// MoatVerdictHonestNull means eligible scorecards had headroom but the corpus
|
||||
// did not improve work — a valid null result.
|
||||
MoatVerdictHonestNull MoatClaimVerdict = "honest_null"
|
||||
// MoatVerdictInconclusive means the inputs cannot support positive or null.
|
||||
MoatVerdictInconclusive MoatClaimVerdict = "inconclusive"
|
||||
)
|
||||
|
||||
// MoatClaimResult is the persisted output of the moat claim aggregation surface
|
||||
// (age-sb0). It renders a moat positive/null/inconclusive verdict over one or
|
||||
// more scenario A/B scorecards.
|
||||
type MoatClaimResult struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Verdict MoatClaimVerdict `json:"verdict"`
|
||||
Reason string `json:"reason"`
|
||||
ScenarioCount int `json:"scenario_count"`
|
||||
MeanAggregateDelta float64 `json:"mean_aggregate_delta"`
|
||||
ScorecardScenarioIDs []string `json:"scorecard_scenario_ids"`
|
||||
ExcludedCeiling []string `json:"excluded_ceiling_violation,omitempty"`
|
||||
ExcludedGateFail []string `json:"excluded_gate_fail,omitempty"`
|
||||
}
|
||||
|
||||
// ErrMoatIneligibleScorecard is returned when aggregation is asked to include a
|
||||
// scorecard with moat_eligible=false. The claim surface fail-closes rather than
|
||||
// silently mixing plumbing into a moat verdict (age-6ys/age-sb0).
|
||||
type ErrMoatIneligibleScorecard struct {
|
||||
ScenarioID string
|
||||
VerdictClass string
|
||||
}
|
||||
|
||||
func (e ErrMoatIneligibleScorecard) Error() string {
|
||||
class := e.VerdictClass
|
||||
if class == "" {
|
||||
class = "unknown"
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"moat claim aggregation refused: scorecard %q has moat_eligible=false (verdict_class=%q, NOT-moat-evidence/plumbing); cannot aggregate into a moat verdict (age-6ys/age-sb0)",
|
||||
e.ScenarioID, class,
|
||||
)
|
||||
}
|
||||
|
||||
// LoadScenarioDeltaScorecard reads a ScenarioDeltaScorecard JSON artifact.
|
||||
func LoadScenarioDeltaScorecard(path string) (ScenarioDeltaScorecard, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return ScenarioDeltaScorecard{}, fmt.Errorf("read scenario delta scorecard %s: %w", path, err)
|
||||
}
|
||||
var card ScenarioDeltaScorecard
|
||||
if err := json.Unmarshal(data, &card); err != nil {
|
||||
return ScenarioDeltaScorecard{}, fmt.Errorf("decode scenario delta scorecard %s: %w", path, err)
|
||||
}
|
||||
if strings.TrimSpace(card.ScenarioID) == "" {
|
||||
return ScenarioDeltaScorecard{}, fmt.Errorf("scenario delta scorecard %s has no scenario_id", path)
|
||||
}
|
||||
return card, nil
|
||||
}
|
||||
|
||||
// AggregateMoatClaim renders a moat positive/null/inconclusive verdict over the
|
||||
// provided scorecards. It fail-closes on any moat_eligible=false input — plumbing
|
||||
// scorecards must never be aggregated into a moat claim (age-sb0).
|
||||
func AggregateMoatClaim(cards []ScenarioDeltaScorecard) (MoatClaimResult, error) {
|
||||
if len(cards) == 0 {
|
||||
return MoatClaimResult{}, fmt.Errorf("moat claim aggregation requires at least one scorecard")
|
||||
}
|
||||
for _, card := range cards {
|
||||
if !card.MoatEligible {
|
||||
return MoatClaimResult{}, ErrMoatIneligibleScorecard{
|
||||
ScenarioID: card.ScenarioID,
|
||||
VerdictClass: card.VerdictClass,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
result := MoatClaimResult{
|
||||
SchemaVersion: 1,
|
||||
GeneratedAt: now,
|
||||
ScenarioCount: len(cards),
|
||||
}
|
||||
for _, card := range cards {
|
||||
result.ScorecardScenarioIDs = append(result.ScorecardScenarioIDs, card.ScenarioID)
|
||||
}
|
||||
|
||||
var admissible []ScenarioDeltaScorecard
|
||||
var positiveEligible []ScenarioDeltaScorecard
|
||||
for _, card := range cards {
|
||||
if card.CeilingViolation {
|
||||
result.ExcludedCeiling = append(result.ExcludedCeiling, card.ScenarioID)
|
||||
continue
|
||||
}
|
||||
admissible = append(admissible, card)
|
||||
if card.Gate.Pass && card.AggregateDelta > 0 {
|
||||
positiveEligible = append(positiveEligible, card)
|
||||
} else if !card.Gate.Pass {
|
||||
result.ExcludedGateFail = append(result.ExcludedGateFail, card.ScenarioID)
|
||||
}
|
||||
}
|
||||
|
||||
if len(admissible) == 0 {
|
||||
result.Verdict = MoatVerdictInconclusive
|
||||
result.Reason = "no admissible scorecards after excluding ceiling violations"
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var deltaSum float64
|
||||
allHadHeadroom := true
|
||||
for _, card := range admissible {
|
||||
deltaSum += card.AggregateDelta
|
||||
if card.Without.Score >= card.SatisfactionThreshold {
|
||||
allHadHeadroom = false
|
||||
}
|
||||
}
|
||||
result.MeanAggregateDelta = roundDelta(deltaSum / float64(len(admissible)))
|
||||
|
||||
switch {
|
||||
case len(positiveEligible) == len(admissible):
|
||||
result.Verdict = MoatVerdictPositive
|
||||
result.Reason = fmt.Sprintf(
|
||||
"all %d admissible scorecards have positive delta and passed gate; mean aggregate_delta=%.4f",
|
||||
len(admissible), result.MeanAggregateDelta,
|
||||
)
|
||||
case allHadHeadroom && len(positiveEligible) == 0:
|
||||
result.Verdict = MoatVerdictHonestNull
|
||||
result.Reason = fmt.Sprintf(
|
||||
"admissible scorecards had headroom but corpus did not beat control; mean aggregate_delta=%.4f",
|
||||
result.MeanAggregateDelta,
|
||||
)
|
||||
default:
|
||||
result.Verdict = MoatVerdictInconclusive
|
||||
result.Reason = "mixed headroom or delta signals across admissible scorecards — cannot claim positive or honest null"
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// WriteMoatClaimResult persists a MoatClaimResult JSON artifact.
|
||||
func WriteMoatClaimResult(path string, result MoatClaimResult) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("output path is required")
|
||||
}
|
||||
data, err := json.MarshalIndent(result, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal moat claim result: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil {
|
||||
return fmt.Errorf("write moat claim result: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAggregateMoatClaimRejectsIneligibleScorecard(t *testing.T) {
|
||||
plumbing := ScenarioDeltaScorecard{
|
||||
ScenarioID: "s-plumbing-001",
|
||||
VerdictClass: VerdictClassFactRecall,
|
||||
MoatEligible: false,
|
||||
AggregateDelta: 1.0,
|
||||
SatisfactionThreshold: 0.8,
|
||||
Without: ScenarioArmResult{Score: 0.1},
|
||||
With: ScenarioArmResult{Score: 1.0},
|
||||
Gate: ScenarioGate{Pass: true},
|
||||
}
|
||||
_, err := AggregateMoatClaim([]ScenarioDeltaScorecard{plumbing})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for moat_eligible=false scorecard")
|
||||
}
|
||||
var ineligible ErrMoatIneligibleScorecard
|
||||
if !errors.As(err, &ineligible) {
|
||||
t.Fatalf("expected ErrMoatIneligibleScorecard, got %T: %v", err, err)
|
||||
}
|
||||
if ineligible.ScenarioID != "s-plumbing-001" {
|
||||
t.Errorf("ScenarioID = %q, want s-plumbing-001", ineligible.ScenarioID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateMoatClaimPositive(t *testing.T) {
|
||||
card := ScenarioDeltaScorecard{
|
||||
ScenarioID: "s-applied-001",
|
||||
VerdictClass: VerdictClassAppliedOOD,
|
||||
MoatEligible: true,
|
||||
AggregateDelta: 0.57,
|
||||
SatisfactionThreshold: 0.8,
|
||||
Without: ScenarioArmResult{Score: 0.35},
|
||||
With: ScenarioArmResult{Score: 0.92},
|
||||
Gate: ScenarioGate{Pass: true},
|
||||
}
|
||||
result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card})
|
||||
if err != nil {
|
||||
t.Fatalf("AggregateMoatClaim: %v", err)
|
||||
}
|
||||
if result.Verdict != MoatVerdictPositive {
|
||||
t.Errorf("Verdict = %q, want %q", result.Verdict, MoatVerdictPositive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateMoatClaimHonestNull(t *testing.T) {
|
||||
card := ScenarioDeltaScorecard{
|
||||
ScenarioID: "s-applied-002",
|
||||
VerdictClass: VerdictClassAppliedOOD,
|
||||
MoatEligible: true,
|
||||
AggregateDelta: -0.07,
|
||||
SatisfactionThreshold: 0.8,
|
||||
Without: ScenarioArmResult{Score: 0.35},
|
||||
With: ScenarioArmResult{Score: 0.28},
|
||||
Gate: ScenarioGate{Pass: false, Reasons: []string{"aggregate_delta <= 0", "with-gold score < threshold"}},
|
||||
}
|
||||
result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card})
|
||||
if err != nil {
|
||||
t.Fatalf("AggregateMoatClaim: %v", err)
|
||||
}
|
||||
if result.Verdict != MoatVerdictHonestNull {
|
||||
t.Errorf("Verdict = %q, want %q", result.Verdict, MoatVerdictHonestNull)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateMoatClaimExcludesCeilingViolation(t *testing.T) {
|
||||
card := ScenarioDeltaScorecard{
|
||||
ScenarioID: "s-ceiling-001",
|
||||
VerdictClass: VerdictClassAppliedOOD,
|
||||
MoatEligible: true,
|
||||
CeilingViolation: true,
|
||||
AggregateDelta: 0,
|
||||
SatisfactionThreshold: 0.8,
|
||||
Without: ScenarioArmResult{Score: 0.95},
|
||||
With: ScenarioArmResult{Score: 0.95},
|
||||
Gate: ScenarioGate{Pass: false},
|
||||
}
|
||||
result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card})
|
||||
if err != nil {
|
||||
t.Fatalf("AggregateMoatClaim: %v", err)
|
||||
}
|
||||
if result.Verdict != MoatVerdictInconclusive {
|
||||
t.Errorf("Verdict = %q, want %q", result.Verdict, MoatVerdictInconclusive)
|
||||
}
|
||||
if len(result.ExcludedCeiling) != 1 {
|
||||
t.Errorf("ExcludedCeiling = %v, want [s-ceiling-001]", result.ExcludedCeiling)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadScenarioDeltaScorecardFromFixture(t *testing.T) {
|
||||
fixture := filepath.Join(repoRoot(t), "evals/scenarios/fixtures/scenario-ab-valid-redacted.scorecard.json")
|
||||
card, err := LoadScenarioDeltaScorecard(fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadScenarioDeltaScorecard: %v", err)
|
||||
}
|
||||
if !card.MoatEligible {
|
||||
t.Error("fixture scorecard should be moat_eligible")
|
||||
}
|
||||
result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card})
|
||||
if err != nil {
|
||||
t.Fatalf("AggregateMoatClaim: %v", err)
|
||||
}
|
||||
if result.Verdict != MoatVerdictPositive {
|
||||
t.Errorf("fixture Verdict = %q, want %q", result.Verdict, MoatVerdictPositive)
|
||||
}
|
||||
}
|
||||
|
||||
func repoRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("runtime.Caller failed")
|
||||
}
|
||||
return filepath.Join(filepath.Dir(file), "..", "..", "..")
|
||||
}
|
||||
@@ -561,6 +561,13 @@
|
||||
"kind": "leaf",
|
||||
"reason": "Covered by release smoke tests, direct command tests, or command handler tests."
|
||||
},
|
||||
{
|
||||
"category": "public-tested",
|
||||
"command": "eval scenario-moat",
|
||||
"coverage_status": "covered",
|
||||
"kind": "leaf",
|
||||
"reason": "Covered by release smoke tests, direct command tests, or command handler tests."
|
||||
},
|
||||
{
|
||||
"category": "public-tested",
|
||||
"command": "eval scorecard",
|
||||
|
||||
@@ -84,6 +84,7 @@
|
||||
| `ao eval outcomes ingest` | `public-stateful-fixture-needed` | `allowlisted` | Maps an Outcomes score to the council verdict record; core logic unit-tested (eval_outcomes_ingest_test.go ingestOutcomesScore); CLI smoke needs a score.json fixture (follow-up). |
|
||||
| `ao eval run` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. |
|
||||
| `ao eval scenario-ab` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. |
|
||||
| `ao eval scenario-moat` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. |
|
||||
| `ao eval scorecard` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. |
|
||||
| `ao eval suite n-required` | `public-stateful-fixture-needed` | `allowlisted` | Computes eval-suite thresholds and needs a representative suite fixture. |
|
||||
| `ao eval suite verdict` | `public-stateful-fixture-needed` | `allowlisted` | Projects eval-suite verdicts and needs a representative suite fixture. |
|
||||
|
||||
@@ -118,3 +118,6 @@ partly done:
|
||||
as moat evidence; defined the applied-OOD class + validity checklist + locked
|
||||
publication rule. Mechanical enforcement landed as `verdict_class` /
|
||||
`moat_eligible` on `ScenarioDeltaScorecard`.
|
||||
- 2026-06-18 (age-sb0): moat claim aggregation surface (`ao eval scenario-moat`)
|
||||
fail-closes on `moat_eligible=false` inputs; renders moat_positive/honest_null/
|
||||
inconclusive over eligible scorecards only.
|
||||
|
||||
@@ -41,7 +41,7 @@
|
||||
},
|
||||
"expectations": [
|
||||
{"type": "exit_code", "value": 0},
|
||||
{"type": "stdout_contains", "value": "cli-command-headings: top=88 sub=219 all=307"},
|
||||
{"type": "stdout_contains", "value": "cli-command-headings: top=88 sub=220 all=308"},
|
||||
{"type": "stdout_contains", "value": "cli-help-matrix-ok"}
|
||||
],
|
||||
"dimensions": ["correctness", "runtime_compatibility", "artifact_quality"],
|
||||
|
||||
@@ -17,7 +17,7 @@ top_count="$(rg -c '^### `ao ' "$DOCS_PATH")"
|
||||
sub_count="$(rg -c '^#### `ao ' "$DOCS_PATH")"
|
||||
all_count="$(rg -c '^#{3,4} `ao ' "$DOCS_PATH")"
|
||||
|
||||
if [[ "$top_count" != "88" || "$sub_count" != "219" || "$all_count" != "307" ]]; then
|
||||
if [[ "$top_count" != "88" || "$sub_count" != "220" || "$all_count" != "308" ]]; then
|
||||
printf 'unexpected command heading counts: top=%s sub=%s all=%s\n' "$top_count" "$sub_count" "$all_count" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -25,7 +25,7 @@ fi
|
||||
# shellcheck disable=SC2016 # literal backticks delimit generated Markdown command headings.
|
||||
mapfile -t commands < <(rg '^#{3,4} `ao ' "$DOCS_PATH" | sed -E 's/^.*`([^`]+)`.*/\1/')
|
||||
|
||||
if [[ "${#commands[@]}" -ne 307 ]]; then
|
||||
if [[ "${#commands[@]}" -ne 308 ]]; then
|
||||
printf 'unexpected command matrix size: %s\n' "${#commands[@]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"scenario_id": "s-fixture-plumbing-001",
|
||||
"scenario_path": "evals/scenarios/fixtures/sentinel-recall-plumbing.json",
|
||||
"generated_at": "2026-06-17T12:00:00Z",
|
||||
"without_gold": {
|
||||
"arm": "without_gold",
|
||||
"score": 0.0,
|
||||
"token_cost": 500
|
||||
},
|
||||
"with_gold": {
|
||||
"arm": "with_gold",
|
||||
"score": 1.0,
|
||||
"token_cost": 600
|
||||
},
|
||||
"aggregate_delta": 1.0,
|
||||
"satisfaction_threshold": 0.8,
|
||||
"token_budget": 200000,
|
||||
"ceiling_violation": false,
|
||||
"verdict_class": "fact-recall",
|
||||
"moat_eligible": false,
|
||||
"gate": {
|
||||
"pass": true,
|
||||
"reasons": []
|
||||
},
|
||||
"_note": "Plumbing fixture for age-sb0: must be rejected by moat claim aggregation."
|
||||
}
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"generated_at": "2026-06-17T11:58:35Z",
|
||||
"generated_at": "2026-06-17T12:37:19Z",
|
||||
"summary": {
|
||||
"skills": 71,
|
||||
"hooks": 0,
|
||||
|
||||
@@ -1366,8 +1366,8 @@
|
||||
{
|
||||
"name": "plan",
|
||||
"source_skill": "skills/plan",
|
||||
"source_hash": "129c2fec737a15f8b0abf10dbedc706ec52d94c74bc1bbbbaac28f90ca018275",
|
||||
"generated_hash": "343d33512062b6232f30820a2f29d35a8a4e8b6e02e2194efb7032d5a90a5648"
|
||||
"source_hash": "29465f3bacb12c1bc18a694b71c14987e217c3c0ee32f0123a617a27817fb260",
|
||||
"generated_hash": "eb02068c559fdb80a0fada49a6438aad7ef6fd7f81a6e33024d7ecabcb77d00b"
|
||||
},
|
||||
{
|
||||
"name": "post-mortem",
|
||||
|
||||
Reference in New Issue
Block a user